From ceb5add20d5f3a8b091f76318d99063e91be4551 Mon Sep 17 00:00:00 2001 From: Andrew Kent Date: Tue, 1 Sep 2026 23:21:37 -0600 Subject: [PATCH 1/2] VCR --- .../__files/v1_experiment-08983257c32b.json | 1 + .../__files/v1_experiment-1aa85d713cc8.json | 1 + .../__files/v1_experiment-409b337a72c9.json | 1 + .../__files/v1_experiment-4c3bb7c53c36.json | 1 + .../__files/v1_experiment-640904b89031.json | 1 + .../__files/v1_experiment-81c93d1c26be.json | 1 + .../__files/v1_experiment-a1296773fd64.json | 1 + .../__files/v1_experiment-a6fe1b460300.json | 1 + .../__files/v1_experiment-b5bb3690fa91.json | 1 + .../__files/v1_experiment-d25fd7674fdc.json | 1 + .../__files/v1_experiment-f4bcbfaef3af.json | 1 + .../__files/v1_project-2f16e640e582.json | 1 + .../__files/v1_project-30867e7088bd.json | 1 + .../__files/v1_project-410ff9f133da.json | 1 + .../__files/v1_project-4891298f2969.json | 1 + .../__files/v1_project-543a07178006.json | 1 + .../__files/v1_project-8d973730b6e1.json | 1 + .../__files/v1_project-b8c4d7b1d75a.json | 1 + .../__files/v1_project-c0742bb3c63f.json | 1 + .../__files/v1_project-dd7665d7a48a.json | 1 + .../__files/v1_project-e548ebcb1218.json | 1 + .../__files/v1_project-e74886687a34.json | 1 + .../__files/v1_project-e7e35e493e43.json | 1 + .../api_apikey_login-08d26d2faef8.json | 10 ++--- .../api_apikey_login-228248286fba.json | 10 ++--- .../api_apikey_login-22a5aef01e2c.json | 10 ++--- .../api_apikey_login-4139f91c7e72.json | 10 ++--- .../api_apikey_login-54c5187c4ebe.json | 11 +++-- .../api_apikey_login-684178b0ab9d.json | 10 ++--- .../api_apikey_login-9611c99e6aac.json | 10 ++--- .../api_apikey_login-a008f9904c48.json | 10 ++--- .../api_apikey_login-a7fce53bd211.json | 11 ++--- .../api_apikey_login-cdbc56f8e830.json | 10 ++--- .../api_apikey_login-fd38313bebba.json | 10 ++--- .../api_apikey_login-fda483633056.json | 10 ++--- .../mappings/v1_experiment-08983257c32b.json | 41 +++++++++++++++++ .../mappings/v1_experiment-1aa85d713cc8.json | 41 +++++++++++++++++ .../mappings/v1_experiment-409b337a72c9.json | 41 +++++++++++++++++ .../mappings/v1_experiment-4c3bb7c53c36.json | 41 +++++++++++++++++ .../mappings/v1_experiment-640904b89031.json | 44 +++++++++++++++++++ .../mappings/v1_experiment-81c93d1c26be.json | 41 +++++++++++++++++ .../mappings/v1_experiment-a1296773fd64.json | 41 +++++++++++++++++ .../mappings/v1_experiment-a6fe1b460300.json | 41 +++++++++++++++++ .../mappings/v1_experiment-b5bb3690fa91.json | 41 +++++++++++++++++ .../mappings/v1_experiment-d25fd7674fdc.json | 41 +++++++++++++++++ .../mappings/v1_experiment-f4bcbfaef3af.json | 43 ++++++++++++++++++ .../mappings/v1_project-2f16e640e582.json | 41 +++++++++++++++++ .../mappings/v1_project-30867e7088bd.json | 41 +++++++++++++++++ .../mappings/v1_project-410ff9f133da.json | 41 +++++++++++++++++ .../mappings/v1_project-4891298f2969.json | 41 +++++++++++++++++ .../mappings/v1_project-543a07178006.json | 41 +++++++++++++++++ .../mappings/v1_project-8d973730b6e1.json | 40 +++++++++++++++++ .../mappings/v1_project-b8c4d7b1d75a.json | 41 +++++++++++++++++ .../mappings/v1_project-c0742bb3c63f.json | 41 +++++++++++++++++ .../mappings/v1_project-dd7665d7a48a.json | 41 +++++++++++++++++ .../mappings/v1_project-e548ebcb1218.json | 41 +++++++++++++++++ .../mappings/v1_project-e74886687a34.json | 41 +++++++++++++++++ .../mappings/v1_project-e7e35e493e43.json | 41 +++++++++++++++++ 58 files changed, 1031 insertions(+), 61 deletions(-) create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-08983257c32b.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-1aa85d713cc8.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-409b337a72c9.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-4c3bb7c53c36.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-640904b89031.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-81c93d1c26be.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-a1296773fd64.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-a6fe1b460300.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-b5bb3690fa91.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-d25fd7674fdc.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-f4bcbfaef3af.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-2f16e640e582.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-30867e7088bd.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-410ff9f133da.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-4891298f2969.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-543a07178006.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-8d973730b6e1.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-b8c4d7b1d75a.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-c0742bb3c63f.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-dd7665d7a48a.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-e548ebcb1218.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-e74886687a34.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-e7e35e493e43.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-08983257c32b.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-1aa85d713cc8.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-409b337a72c9.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-4c3bb7c53c36.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-640904b89031.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-81c93d1c26be.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-a1296773fd64.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-a6fe1b460300.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-b5bb3690fa91.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-d25fd7674fdc.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-f4bcbfaef3af.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-2f16e640e582.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-30867e7088bd.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-410ff9f133da.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-4891298f2969.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-543a07178006.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-8d973730b6e1.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-b8c4d7b1d75a.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-c0742bb3c63f.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-dd7665d7a48a.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-e548ebcb1218.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-e74886687a34.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-e7e35e493e43.json diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-08983257c32b.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-08983257c32b.json new file mode 100644 index 00000000..e1ed3149 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-08983257c32b.json @@ -0,0 +1 @@ +{"id":"a2899b42-b4c6-4189-94ec-ae309cb28bd9","project_id":"f1e858a4-58e3-408f-983f-016760d7fa25","name":"serial-test","description":null,"created":"2026-08-26T02:02:00.029Z","repo_info":null,"commit":null,"base_exp_id":null,"deleted_at":null,"dataset_id":null,"dataset_version":null,"internal_metadata":null,"parameters_id":null,"parameters_version":null,"public":false,"user_id":"a5ca7f9c-bf20-40c4-a82b-5c992f6a38f5","metadata":null,"tags":null} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-1aa85d713cc8.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-1aa85d713cc8.json new file mode 100644 index 00000000..312274f0 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-1aa85d713cc8.json @@ -0,0 +1 @@ +{"id":"f22ad541-31ff-4889-bd1f-952b4b63859c","project_id":"f1e858a4-58e3-408f-983f-016760d7fa25","name":"config-default-concurrency-test","description":null,"created":"2026-09-01T20:39:05.777Z","repo_info":null,"commit":null,"base_exp_id":null,"deleted_at":null,"dataset_id":null,"dataset_version":null,"internal_metadata":null,"parameters_id":null,"parameters_version":null,"public":false,"user_id":"a5ca7f9c-bf20-40c4-a82b-5c992f6a38f5","metadata":null,"tags":null} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-409b337a72c9.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-409b337a72c9.json new file mode 100644 index 00000000..862e9646 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-409b337a72c9.json @@ -0,0 +1 @@ +{"id":"e48d6ebf-42f0-45fa-b91c-8ce73f04f9f4","project_id":"f1e858a4-58e3-408f-983f-016760d7fa25","name":"failing-task-test","description":null,"created":"2026-08-26T02:02:02.086Z","repo_info":null,"commit":null,"base_exp_id":null,"deleted_at":null,"dataset_id":null,"dataset_version":null,"internal_metadata":null,"parameters_id":null,"parameters_version":null,"public":false,"user_id":"a5ca7f9c-bf20-40c4-a82b-5c992f6a38f5","metadata":null,"tags":null} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-4c3bb7c53c36.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-4c3bb7c53c36.json new file mode 100644 index 00000000..f4367e54 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-4c3bb7c53c36.json @@ -0,0 +1 @@ +{"id":"7f8cb336-e1bc-4d4e-9541-03c4dee2e1b3","project_id":"f1e858a4-58e3-408f-983f-016760d7fa25","name":"byo-executor-test","description":null,"created":"2026-08-26T02:01:58.096Z","repo_info":null,"commit":null,"base_exp_id":null,"deleted_at":null,"dataset_id":null,"dataset_version":null,"internal_metadata":null,"parameters_id":null,"parameters_version":null,"public":false,"user_id":"a5ca7f9c-bf20-40c4-a82b-5c992f6a38f5","metadata":null,"tags":null} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-640904b89031.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-640904b89031.json new file mode 100644 index 00000000..3428c91e --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-640904b89031.json @@ -0,0 +1 @@ +{"id":"73e8737d-a8eb-4ab1-b049-a3338c63eab3","project_id":"f1e858a4-58e3-408f-983f-016760d7fa25","name":"bad-score-test","description":null,"created":"2026-08-26T02:01:57.012Z","repo_info":null,"commit":null,"base_exp_id":null,"deleted_at":null,"dataset_id":null,"dataset_version":null,"internal_metadata":null,"parameters_id":null,"parameters_version":null,"public":false,"user_id":"a5ca7f9c-bf20-40c4-a82b-5c992f6a38f5","metadata":null,"tags":null} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-81c93d1c26be.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-81c93d1c26be.json new file mode 100644 index 00000000..b8eb1de5 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-81c93d1c26be.json @@ -0,0 +1 @@ +{"id":"e7c0ab1f-9757-45cc-84f0-cb271d9bc398","project_id":"f1e858a4-58e3-408f-983f-016760d7fa25","name":"concurrency-test","description":null,"created":"2026-08-26T02:02:01.048Z","repo_info":null,"commit":null,"base_exp_id":null,"deleted_at":null,"dataset_id":null,"dataset_version":null,"internal_metadata":null,"parameters_id":null,"parameters_version":null,"public":false,"user_id":"a5ca7f9c-bf20-40c4-a82b-5c992f6a38f5","metadata":null,"tags":null} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-a1296773fd64.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-a1296773fd64.json new file mode 100644 index 00000000..ffd293b2 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-a1296773fd64.json @@ -0,0 +1 @@ +{"id":"ddfe67fc-9a9d-4913-8a12-c051e05e25c0","project_id":"f1e858a4-58e3-408f-983f-016760d7fa25","name":"supplied-executor-concurrency-test","description":null,"created":"2026-09-01T22:59:28.691Z","repo_info":null,"commit":null,"base_exp_id":null,"deleted_at":null,"dataset_id":null,"dataset_version":null,"internal_metadata":null,"parameters_id":null,"parameters_version":null,"public":false,"user_id":"a5ca7f9c-bf20-40c4-a82b-5c992f6a38f5","metadata":null,"tags":null} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-a6fe1b460300.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-a6fe1b460300.json new file mode 100644 index 00000000..6acca0d4 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-a6fe1b460300.json @@ -0,0 +1 @@ +{"id":"3061a4e5-bfa5-4e84-a9c4-14fbe08bd653","project_id":"f1e858a4-58e3-408f-983f-016760d7fa25","name":"start-test","description":null,"created":"2026-08-26T02:02:03.020Z","repo_info":null,"commit":null,"base_exp_id":null,"deleted_at":null,"dataset_id":null,"dataset_version":null,"internal_metadata":null,"parameters_id":null,"parameters_version":null,"public":false,"user_id":"a5ca7f9c-bf20-40c4-a82b-5c992f6a38f5","metadata":null,"tags":null} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-b5bb3690fa91.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-b5bb3690fa91.json new file mode 100644 index 00000000..3ea4e6e6 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-b5bb3690fa91.json @@ -0,0 +1 @@ +{"id":"870f752d-d4fe-4718-8a4f-0ee73d90eb20","project_id":"f1e858a4-58e3-408f-983f-016760d7fa25","name":"backpressure-test","description":null,"created":"2026-09-01T22:59:26.961Z","repo_info":null,"commit":null,"base_exp_id":null,"deleted_at":null,"dataset_id":null,"dataset_version":null,"internal_metadata":null,"parameters_id":null,"parameters_version":null,"public":false,"user_id":"a5ca7f9c-bf20-40c4-a82b-5c992f6a38f5","metadata":null,"tags":null} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-d25fd7674fdc.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-d25fd7674fdc.json new file mode 100644 index 00000000..3d253393 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-d25fd7674fdc.json @@ -0,0 +1 @@ +{"id":"37adfc69-2ad9-4c38-ad1e-fe8d3294add5","project_id":"f1e858a4-58e3-408f-983f-016760d7fa25","name":"max-concurrency-test","description":null,"created":"2026-08-26T02:01:59.022Z","repo_info":null,"commit":null,"base_exp_id":null,"deleted_at":null,"dataset_id":null,"dataset_version":null,"internal_metadata":null,"parameters_id":null,"parameters_version":null,"public":false,"user_id":"a5ca7f9c-bf20-40c4-a82b-5c992f6a38f5","metadata":null,"tags":null} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-f4bcbfaef3af.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-f4bcbfaef3af.json new file mode 100644 index 00000000..3428c91e --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-f4bcbfaef3af.json @@ -0,0 +1 @@ +{"id":"73e8737d-a8eb-4ab1-b049-a3338c63eab3","project_id":"f1e858a4-58e3-408f-983f-016760d7fa25","name":"bad-score-test","description":null,"created":"2026-08-26T02:01:57.012Z","repo_info":null,"commit":null,"base_exp_id":null,"deleted_at":null,"dataset_id":null,"dataset_version":null,"internal_metadata":null,"parameters_id":null,"parameters_version":null,"public":false,"user_id":"a5ca7f9c-bf20-40c4-a82b-5c992f6a38f5","metadata":null,"tags":null} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-2f16e640e582.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-2f16e640e582.json new file mode 100644 index 00000000..89452950 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-2f16e640e582.json @@ -0,0 +1 @@ +{"objects":[{"id":"f1e858a4-58e3-408f-983f-016760d7fa25","org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","name":"java-unit-test","description":null,"created":"2026-05-07T16:55:48.127Z","deleted_at":null,"user_id":"a5ca7f9c-bf20-40c4-a82b-5c992f6a38f5","settings":{"remote_eval_sources":[{"url":"http://localhost:8301","name":"localjava","description":null}]}}]} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-30867e7088bd.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-30867e7088bd.json new file mode 100644 index 00000000..89452950 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-30867e7088bd.json @@ -0,0 +1 @@ +{"objects":[{"id":"f1e858a4-58e3-408f-983f-016760d7fa25","org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","name":"java-unit-test","description":null,"created":"2026-05-07T16:55:48.127Z","deleted_at":null,"user_id":"a5ca7f9c-bf20-40c4-a82b-5c992f6a38f5","settings":{"remote_eval_sources":[{"url":"http://localhost:8301","name":"localjava","description":null}]}}]} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-410ff9f133da.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-410ff9f133da.json new file mode 100644 index 00000000..89452950 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-410ff9f133da.json @@ -0,0 +1 @@ +{"objects":[{"id":"f1e858a4-58e3-408f-983f-016760d7fa25","org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","name":"java-unit-test","description":null,"created":"2026-05-07T16:55:48.127Z","deleted_at":null,"user_id":"a5ca7f9c-bf20-40c4-a82b-5c992f6a38f5","settings":{"remote_eval_sources":[{"url":"http://localhost:8301","name":"localjava","description":null}]}}]} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-4891298f2969.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-4891298f2969.json new file mode 100644 index 00000000..89452950 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-4891298f2969.json @@ -0,0 +1 @@ +{"objects":[{"id":"f1e858a4-58e3-408f-983f-016760d7fa25","org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","name":"java-unit-test","description":null,"created":"2026-05-07T16:55:48.127Z","deleted_at":null,"user_id":"a5ca7f9c-bf20-40c4-a82b-5c992f6a38f5","settings":{"remote_eval_sources":[{"url":"http://localhost:8301","name":"localjava","description":null}]}}]} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-543a07178006.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-543a07178006.json new file mode 100644 index 00000000..89452950 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-543a07178006.json @@ -0,0 +1 @@ +{"objects":[{"id":"f1e858a4-58e3-408f-983f-016760d7fa25","org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","name":"java-unit-test","description":null,"created":"2026-05-07T16:55:48.127Z","deleted_at":null,"user_id":"a5ca7f9c-bf20-40c4-a82b-5c992f6a38f5","settings":{"remote_eval_sources":[{"url":"http://localhost:8301","name":"localjava","description":null}]}}]} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-8d973730b6e1.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-8d973730b6e1.json new file mode 100644 index 00000000..89452950 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-8d973730b6e1.json @@ -0,0 +1 @@ +{"objects":[{"id":"f1e858a4-58e3-408f-983f-016760d7fa25","org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","name":"java-unit-test","description":null,"created":"2026-05-07T16:55:48.127Z","deleted_at":null,"user_id":"a5ca7f9c-bf20-40c4-a82b-5c992f6a38f5","settings":{"remote_eval_sources":[{"url":"http://localhost:8301","name":"localjava","description":null}]}}]} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-b8c4d7b1d75a.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-b8c4d7b1d75a.json new file mode 100644 index 00000000..89452950 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-b8c4d7b1d75a.json @@ -0,0 +1 @@ +{"objects":[{"id":"f1e858a4-58e3-408f-983f-016760d7fa25","org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","name":"java-unit-test","description":null,"created":"2026-05-07T16:55:48.127Z","deleted_at":null,"user_id":"a5ca7f9c-bf20-40c4-a82b-5c992f6a38f5","settings":{"remote_eval_sources":[{"url":"http://localhost:8301","name":"localjava","description":null}]}}]} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-c0742bb3c63f.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-c0742bb3c63f.json new file mode 100644 index 00000000..89452950 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-c0742bb3c63f.json @@ -0,0 +1 @@ +{"objects":[{"id":"f1e858a4-58e3-408f-983f-016760d7fa25","org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","name":"java-unit-test","description":null,"created":"2026-05-07T16:55:48.127Z","deleted_at":null,"user_id":"a5ca7f9c-bf20-40c4-a82b-5c992f6a38f5","settings":{"remote_eval_sources":[{"url":"http://localhost:8301","name":"localjava","description":null}]}}]} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-dd7665d7a48a.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-dd7665d7a48a.json new file mode 100644 index 00000000..89452950 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-dd7665d7a48a.json @@ -0,0 +1 @@ +{"objects":[{"id":"f1e858a4-58e3-408f-983f-016760d7fa25","org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","name":"java-unit-test","description":null,"created":"2026-05-07T16:55:48.127Z","deleted_at":null,"user_id":"a5ca7f9c-bf20-40c4-a82b-5c992f6a38f5","settings":{"remote_eval_sources":[{"url":"http://localhost:8301","name":"localjava","description":null}]}}]} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-e548ebcb1218.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-e548ebcb1218.json new file mode 100644 index 00000000..89452950 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-e548ebcb1218.json @@ -0,0 +1 @@ +{"objects":[{"id":"f1e858a4-58e3-408f-983f-016760d7fa25","org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","name":"java-unit-test","description":null,"created":"2026-05-07T16:55:48.127Z","deleted_at":null,"user_id":"a5ca7f9c-bf20-40c4-a82b-5c992f6a38f5","settings":{"remote_eval_sources":[{"url":"http://localhost:8301","name":"localjava","description":null}]}}]} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-e74886687a34.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-e74886687a34.json new file mode 100644 index 00000000..89452950 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-e74886687a34.json @@ -0,0 +1 @@ +{"objects":[{"id":"f1e858a4-58e3-408f-983f-016760d7fa25","org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","name":"java-unit-test","description":null,"created":"2026-05-07T16:55:48.127Z","deleted_at":null,"user_id":"a5ca7f9c-bf20-40c4-a82b-5c992f6a38f5","settings":{"remote_eval_sources":[{"url":"http://localhost:8301","name":"localjava","description":null}]}}]} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-e7e35e493e43.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-e7e35e493e43.json new file mode 100644 index 00000000..89452950 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-e7e35e493e43.json @@ -0,0 +1 @@ +{"objects":[{"id":"f1e858a4-58e3-408f-983f-016760d7fa25","org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","name":"java-unit-test","description":null,"created":"2026-05-07T16:55:48.127Z","deleted_at":null,"user_id":"a5ca7f9c-bf20-40c4-a82b-5c992f6a38f5","settings":{"remote_eval_sources":[{"url":"http://localhost:8301","name":"localjava","description":null}]}}]} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/api_apikey_login-08d26d2faef8.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/api_apikey_login-08d26d2faef8.json index deca6570..907583a0 100644 --- a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/api_apikey_login-08d26d2faef8.json +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/api_apikey_login-08d26d2faef8.json @@ -10,15 +10,15 @@ "bodyFileName" : "api_apikey_login-08d26d2faef8.json", "headers" : { "X-Cache" : "Miss from cloudfront", - "X-Amz-Cf-Pop" : "SEA900-P9", - "Date" : "Mon, 31 Aug 2026 17:36:47 GMT", - "Via" : "1.1 63560dd3f856b0f7bfe68a0cad46924a.cloudfront.net (CloudFront)", + "X-Amz-Cf-Pop" : "SEA900-P10", + "Date" : "Wed, 02 Sep 2026 05:16:17 GMT", + "Via" : "1.1 a40ac7dad0e348fc93799233c9af5960.cloudfront.net (CloudFront)", "Strict-Transport-Security" : "max-age=31536000; includeSubDomains", "Access-Control-Expose-Headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url", "Cache-Control" : "no-store, no-cache, must-revalidate, proxy-revalidate", "ETag" : "W/\"178-kKbrsfThc+xXlggXc9RQYV+Gr9I\"", "Access-Control-Allow-Credentials" : "true", - "X-Amz-Cf-Id" : "bPeEz-Kr3NcW34gPD74I3NxAPaN6JlxbKrsVJ1nphiAXmKxfBYpHAA==", + "X-Amz-Cf-Id" : "cIC5pK9ztQGRtyPY4TG-kOt0A5T6RyyB-B98pIKigKYNfmz1xMvJxw==", "Vary" : "Origin, Accept-Encoding", "Expires" : "0", "Surrogate-Control" : "no-store", @@ -30,5 +30,5 @@ "scenarioName" : "scenario-1-api-apikey-login", "requiredScenarioState" : "Started", "newScenarioState" : "scenario-1-api-apikey-login-2", - "insertionIndex" : 188 + "insertionIndex" : 222 } \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/api_apikey_login-228248286fba.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/api_apikey_login-228248286fba.json index 5c93a823..339cd8ba 100644 --- a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/api_apikey_login-228248286fba.json +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/api_apikey_login-228248286fba.json @@ -10,15 +10,15 @@ "bodyFileName" : "api_apikey_login-228248286fba.json", "headers" : { "X-Cache" : "Miss from cloudfront", - "X-Amz-Cf-Pop" : "SEA900-P9", - "Date" : "Mon, 31 Aug 2026 17:34:24 GMT", - "Via" : "1.1 bef90eae512e70457d6a8a77b097a124.cloudfront.net (CloudFront)", + "X-Amz-Cf-Pop" : "SEA900-P10", + "Date" : "Wed, 02 Sep 2026 05:16:19 GMT", + "Via" : "1.1 f8731007efc5ab360d90cee573a1e916.cloudfront.net (CloudFront)", "Strict-Transport-Security" : "max-age=31536000; includeSubDomains", "Access-Control-Expose-Headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url", "Cache-Control" : "no-store, no-cache, must-revalidate, proxy-revalidate", "ETag" : "W/\"178-kKbrsfThc+xXlggXc9RQYV+Gr9I\"", "Access-Control-Allow-Credentials" : "true", - "X-Amz-Cf-Id" : "Qu6yfa4c-niBlO2R0ZiMQe5KVggvZYydqHci_JvYmMrZuw66mhXmvA==", + "X-Amz-Cf-Id" : "fpjGMXVOttkN6DHWVPW4X73OaQL7iB8df5W4J2y4u9tNe1MUXqlhbA==", "Vary" : "Origin, Accept-Encoding", "Expires" : "0", "Surrogate-Control" : "no-store", @@ -30,5 +30,5 @@ "scenarioName" : "scenario-1-api-apikey-login", "requiredScenarioState" : "scenario-1-api-apikey-login-3", "newScenarioState" : "scenario-1-api-apikey-login-4", - "insertionIndex" : 129 + "insertionIndex" : 217 } \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/api_apikey_login-22a5aef01e2c.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/api_apikey_login-22a5aef01e2c.json index a64685e8..c9e60845 100644 --- a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/api_apikey_login-22a5aef01e2c.json +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/api_apikey_login-22a5aef01e2c.json @@ -10,15 +10,15 @@ "bodyFileName" : "api_apikey_login-22a5aef01e2c.json", "headers" : { "X-Cache" : "Miss from cloudfront", - "X-Amz-Cf-Pop" : "SEA900-P9", - "Date" : "Mon, 31 Aug 2026 17:35:04 GMT", - "Via" : "1.1 b54238be18861f9bb1272bf1cb10e040.cloudfront.net (CloudFront)", + "X-Amz-Cf-Pop" : "SEA900-P10", + "Date" : "Wed, 02 Sep 2026 05:16:24 GMT", + "Via" : "1.1 b3ccaedda78c63d5967b57382ceb4cbe.cloudfront.net (CloudFront)", "Strict-Transport-Security" : "max-age=31536000; includeSubDomains", "Access-Control-Expose-Headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url", "Cache-Control" : "no-store, no-cache, must-revalidate, proxy-revalidate", "ETag" : "W/\"178-kKbrsfThc+xXlggXc9RQYV+Gr9I\"", "Access-Control-Allow-Credentials" : "true", - "X-Amz-Cf-Id" : "j1E_lRf_p-_WlQePU8GnXdpmKB5vVgJ0iQ_GrziMDuk92yBTnpuL4A==", + "X-Amz-Cf-Id" : "rEfDTp59m1WYkZxB3YY1R7GbM8tH_aFKSJJdDHju2McabUc_8eVDlw==", "Vary" : "Origin, Accept-Encoding", "Expires" : "0", "Surrogate-Control" : "no-store", @@ -30,5 +30,5 @@ "scenarioName" : "scenario-1-api-apikey-login", "requiredScenarioState" : "scenario-1-api-apikey-login-10", "newScenarioState" : "scenario-1-api-apikey-login-11", - "insertionIndex" : 110 + "insertionIndex" : 196 } \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/api_apikey_login-4139f91c7e72.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/api_apikey_login-4139f91c7e72.json index 3afa8d2d..c3269a5c 100644 --- a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/api_apikey_login-4139f91c7e72.json +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/api_apikey_login-4139f91c7e72.json @@ -10,15 +10,15 @@ "bodyFileName" : "api_apikey_login-4139f91c7e72.json", "headers" : { "X-Cache" : "Miss from cloudfront", - "X-Amz-Cf-Pop" : "SEA900-P9", - "Date" : "Mon, 31 Aug 2026 17:34:26 GMT", - "Via" : "1.1 b54238be18861f9bb1272bf1cb10e040.cloudfront.net (CloudFront)", + "X-Amz-Cf-Pop" : "SEA900-P10", + "Date" : "Wed, 02 Sep 2026 05:16:19 GMT", + "Via" : "1.1 0eb43913f9caf453beb959a8a836a688.cloudfront.net (CloudFront)", "Strict-Transport-Security" : "max-age=31536000; includeSubDomains", "Access-Control-Expose-Headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url", "Cache-Control" : "no-store, no-cache, must-revalidate, proxy-revalidate", "ETag" : "W/\"178-kKbrsfThc+xXlggXc9RQYV+Gr9I\"", "Access-Control-Allow-Credentials" : "true", - "X-Amz-Cf-Id" : "BNPuPLxQJJzNiu1Z68Tc3t-zU4V9noEifBUQzWnkc_Bay7kib2gwxQ==", + "X-Amz-Cf-Id" : "U5fDC81oHt2oTDwio9zwag_JaGCjguyGdlttBhR8l9Dol8OQ5h8exA==", "Vary" : "Origin, Accept-Encoding", "Expires" : "0", "Surrogate-Control" : "no-store", @@ -30,5 +30,5 @@ "scenarioName" : "scenario-1-api-apikey-login", "requiredScenarioState" : "scenario-1-api-apikey-login-4", "newScenarioState" : "scenario-1-api-apikey-login-5", - "insertionIndex" : 127 + "insertionIndex" : 214 } \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/api_apikey_login-54c5187c4ebe.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/api_apikey_login-54c5187c4ebe.json index 9c059bcc..ef90283f 100644 --- a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/api_apikey_login-54c5187c4ebe.json +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/api_apikey_login-54c5187c4ebe.json @@ -10,15 +10,15 @@ "bodyFileName" : "api_apikey_login-54c5187c4ebe.json", "headers" : { "X-Cache" : "Miss from cloudfront", - "X-Amz-Cf-Pop" : "SEA900-P9", - "Date" : "Mon, 31 Aug 2026 17:35:08 GMT", - "Via" : "1.1 a3134c0c893f03d1e9a9c657d09af7cc.cloudfront.net (CloudFront)", + "X-Amz-Cf-Pop" : "SEA900-P10", + "Date" : "Wed, 02 Sep 2026 05:16:26 GMT", + "Via" : "1.1 77f3c89ffd619275648d49ad13868570.cloudfront.net (CloudFront)", "Strict-Transport-Security" : "max-age=31536000; includeSubDomains", "Access-Control-Expose-Headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url", "Cache-Control" : "no-store, no-cache, must-revalidate, proxy-revalidate", "ETag" : "W/\"178-kKbrsfThc+xXlggXc9RQYV+Gr9I\"", "Access-Control-Allow-Credentials" : "true", - "X-Amz-Cf-Id" : "n2h0oFc39MiB4qZGROWv85GXhvUcrloF1Vk1m5SIKC-2tmI0bkE1YQ==", + "X-Amz-Cf-Id" : "fpcUWQAq1iRZ3lspUJKSZamoeX-UJRYtnRBP-bRxt5jQ9UHYqCRWxg==", "Vary" : "Origin, Accept-Encoding", "Expires" : "0", "Surrogate-Control" : "no-store", @@ -29,6 +29,5 @@ "persistent" : true, "scenarioName" : "scenario-1-api-apikey-login", "requiredScenarioState" : "scenario-1-api-apikey-login-12", - "newScenarioState" : "scenario-1-api-apikey-login-13", - "insertionIndex" : 106 + "insertionIndex" : 190 } \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/api_apikey_login-684178b0ab9d.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/api_apikey_login-684178b0ab9d.json index a1475d62..05785d47 100644 --- a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/api_apikey_login-684178b0ab9d.json +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/api_apikey_login-684178b0ab9d.json @@ -10,15 +10,15 @@ "bodyFileName" : "api_apikey_login-684178b0ab9d.json", "headers" : { "X-Cache" : "Miss from cloudfront", - "X-Amz-Cf-Pop" : "SEA900-P9", - "Date" : "Mon, 31 Aug 2026 17:34:58 GMT", - "Via" : "1.1 d49bde7225e80ca0dc457ff2b8b4343e.cloudfront.net (CloudFront)", + "X-Amz-Cf-Pop" : "SEA900-P10", + "Date" : "Wed, 02 Sep 2026 05:16:23 GMT", + "Via" : "1.1 74e8c76139b8c7f9b11d5e4441c2a7a2.cloudfront.net (CloudFront)", "Strict-Transport-Security" : "max-age=31536000; includeSubDomains", "Access-Control-Expose-Headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url", "Cache-Control" : "no-store, no-cache, must-revalidate, proxy-revalidate", "ETag" : "W/\"178-kKbrsfThc+xXlggXc9RQYV+Gr9I\"", "Access-Control-Allow-Credentials" : "true", - "X-Amz-Cf-Id" : "zJPHh5Advrp225DDEex7adDrv8ZeDx_cUylyDkH35R6dLza9OSN58g==", + "X-Amz-Cf-Id" : "vB5k6LWSyxpP_HVksApsquEs5oy-e_M8xoIbLLF5ApIhOG9T1pf62w==", "Vary" : "Origin, Accept-Encoding", "Expires" : "0", "Surrogate-Control" : "no-store", @@ -30,5 +30,5 @@ "scenarioName" : "scenario-1-api-apikey-login", "requiredScenarioState" : "scenario-1-api-apikey-login-8", "newScenarioState" : "scenario-1-api-apikey-login-9", - "insertionIndex" : 115 + "insertionIndex" : 202 } \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/api_apikey_login-9611c99e6aac.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/api_apikey_login-9611c99e6aac.json index fe943b10..af1160b6 100644 --- a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/api_apikey_login-9611c99e6aac.json +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/api_apikey_login-9611c99e6aac.json @@ -10,15 +10,15 @@ "bodyFileName" : "api_apikey_login-9611c99e6aac.json", "headers" : { "X-Cache" : "Miss from cloudfront", - "X-Amz-Cf-Pop" : "SEA900-P9", - "Date" : "Mon, 31 Aug 2026 17:34:31 GMT", - "Via" : "1.1 b3a8bdee20374465a3f2aa64f19ec30e.cloudfront.net (CloudFront)", + "X-Amz-Cf-Pop" : "SEA900-P10", + "Date" : "Wed, 02 Sep 2026 05:16:20 GMT", + "Via" : "1.1 b3ccaedda78c63d5967b57382ceb4cbe.cloudfront.net (CloudFront)", "Strict-Transport-Security" : "max-age=31536000; includeSubDomains", "Access-Control-Expose-Headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url", "Cache-Control" : "no-store, no-cache, must-revalidate, proxy-revalidate", "ETag" : "W/\"178-kKbrsfThc+xXlggXc9RQYV+Gr9I\"", "Access-Control-Allow-Credentials" : "true", - "X-Amz-Cf-Id" : "ROXvzBKxUn_zgMNs1-BbW5J_FTq7LCKl2nrnLR8sTpoAYqtAFEPRWQ==", + "X-Amz-Cf-Id" : "5Z_ghrPhfu3bThjf6xSTwFUp3iF-hmEHKgQUTZMyFmHxjepRPtNK3A==", "Vary" : "Origin, Accept-Encoding", "Expires" : "0", "Surrogate-Control" : "no-store", @@ -30,5 +30,5 @@ "scenarioName" : "scenario-1-api-apikey-login", "requiredScenarioState" : "scenario-1-api-apikey-login-5", "newScenarioState" : "scenario-1-api-apikey-login-6", - "insertionIndex" : 123 + "insertionIndex" : 211 } \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/api_apikey_login-a008f9904c48.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/api_apikey_login-a008f9904c48.json index c62cad34..a07719e2 100644 --- a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/api_apikey_login-a008f9904c48.json +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/api_apikey_login-a008f9904c48.json @@ -10,15 +10,15 @@ "bodyFileName" : "api_apikey_login-a008f9904c48.json", "headers" : { "X-Cache" : "Miss from cloudfront", - "X-Amz-Cf-Pop" : "SEA900-P9", - "Date" : "Mon, 31 Aug 2026 17:35:02 GMT", - "Via" : "1.1 e088ff8bff69861ed7fd37fbb518f0c2.cloudfront.net (CloudFront)", + "X-Amz-Cf-Pop" : "SEA900-P10", + "Date" : "Wed, 02 Sep 2026 05:16:24 GMT", + "Via" : "1.1 cb2aa1abf9fb243a4dc4cb073a92424e.cloudfront.net (CloudFront)", "Strict-Transport-Security" : "max-age=31536000; includeSubDomains", "Access-Control-Expose-Headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url", "Cache-Control" : "no-store, no-cache, must-revalidate, proxy-revalidate", "ETag" : "W/\"178-kKbrsfThc+xXlggXc9RQYV+Gr9I\"", "Access-Control-Allow-Credentials" : "true", - "X-Amz-Cf-Id" : "EBcJg6PORi1Wyl2YxuYSxCjW3PDP6XPCCV5fmeUER0rgjza1lFsnOw==", + "X-Amz-Cf-Id" : "NCvlrpCrAI4EvvDRFxRYCAbJWZOpk-vaGpaRRvZ3dFJ1OFFMnft5Mg==", "Vary" : "Origin, Accept-Encoding", "Expires" : "0", "Surrogate-Control" : "no-store", @@ -30,5 +30,5 @@ "scenarioName" : "scenario-1-api-apikey-login", "requiredScenarioState" : "scenario-1-api-apikey-login-9", "newScenarioState" : "scenario-1-api-apikey-login-10", - "insertionIndex" : 112 + "insertionIndex" : 199 } \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/api_apikey_login-a7fce53bd211.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/api_apikey_login-a7fce53bd211.json index dd6b5267..bc08df82 100644 --- a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/api_apikey_login-a7fce53bd211.json +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/api_apikey_login-a7fce53bd211.json @@ -10,15 +10,15 @@ "bodyFileName" : "api_apikey_login-a7fce53bd211.json", "headers" : { "X-Cache" : "Miss from cloudfront", - "X-Amz-Cf-Pop" : "SEA900-P9", - "Date" : "Mon, 31 Aug 2026 17:36:50 GMT", - "Via" : "1.1 b3a8bdee20374465a3f2aa64f19ec30e.cloudfront.net (CloudFront)", + "X-Amz-Cf-Pop" : "SEA900-P10", + "Date" : "Wed, 02 Sep 2026 05:16:18 GMT", + "Via" : "1.1 0df7f27a01014ab815259ca2d88193c6.cloudfront.net (CloudFront)", "Strict-Transport-Security" : "max-age=31536000; includeSubDomains", "Access-Control-Expose-Headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url", "Cache-Control" : "no-store, no-cache, must-revalidate, proxy-revalidate", "ETag" : "W/\"178-kKbrsfThc+xXlggXc9RQYV+Gr9I\"", "Access-Control-Allow-Credentials" : "true", - "X-Amz-Cf-Id" : "yGdyjanZFfSiKQ0w3WAMR-CuMALKqeXicdItPp0zW_YwFqueWBNCFg==", + "X-Amz-Cf-Id" : "0O2fPbX6KGx7UDTJ6fjEtt17a3RDgZgukWeXP_ROqA46LmjhrH4Sjw==", "Vary" : "Origin, Accept-Encoding", "Expires" : "0", "Surrogate-Control" : "no-store", @@ -29,5 +29,6 @@ "persistent" : true, "scenarioName" : "scenario-1-api-apikey-login", "requiredScenarioState" : "scenario-1-api-apikey-login-2", - "insertionIndex" : 187 + "newScenarioState" : "scenario-1-api-apikey-login-3", + "insertionIndex" : 220 } \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/api_apikey_login-cdbc56f8e830.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/api_apikey_login-cdbc56f8e830.json index 61cb80c3..145d914a 100644 --- a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/api_apikey_login-cdbc56f8e830.json +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/api_apikey_login-cdbc56f8e830.json @@ -10,15 +10,15 @@ "bodyFileName" : "api_apikey_login-cdbc56f8e830.json", "headers" : { "X-Cache" : "Miss from cloudfront", - "X-Amz-Cf-Pop" : "SEA900-P9", - "Date" : "Mon, 31 Aug 2026 17:35:07 GMT", - "Via" : "1.1 e8b22a8fa8511f8f972b4965a9af80b4.cloudfront.net (CloudFront)", + "X-Amz-Cf-Pop" : "SEA900-P10", + "Date" : "Wed, 02 Sep 2026 05:16:25 GMT", + "Via" : "1.1 fbb003dfc0617e3e058e3dac791dfd5a.cloudfront.net (CloudFront)", "Strict-Transport-Security" : "max-age=31536000; includeSubDomains", "Access-Control-Expose-Headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url", "Cache-Control" : "no-store, no-cache, must-revalidate, proxy-revalidate", "ETag" : "W/\"178-kKbrsfThc+xXlggXc9RQYV+Gr9I\"", "Access-Control-Allow-Credentials" : "true", - "X-Amz-Cf-Id" : "-EILwmdJbOiH4bhUusMroi1wiOrkMzX8nFxDqEWa-70cYzzl5TFpgQ==", + "X-Amz-Cf-Id" : "LRzUk7A5vNLXU8sc5k7lcmRQpBlRe-Tvsxr99DSW4HbeG6haibCr5w==", "Vary" : "Origin, Accept-Encoding", "Expires" : "0", "Surrogate-Control" : "no-store", @@ -30,5 +30,5 @@ "scenarioName" : "scenario-1-api-apikey-login", "requiredScenarioState" : "scenario-1-api-apikey-login-11", "newScenarioState" : "scenario-1-api-apikey-login-12", - "insertionIndex" : 108 + "insertionIndex" : 193 } \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/api_apikey_login-fd38313bebba.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/api_apikey_login-fd38313bebba.json index f0341d64..f404625e 100644 --- a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/api_apikey_login-fd38313bebba.json +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/api_apikey_login-fd38313bebba.json @@ -10,15 +10,15 @@ "bodyFileName" : "api_apikey_login-fd38313bebba.json", "headers" : { "X-Cache" : "Miss from cloudfront", - "X-Amz-Cf-Pop" : "SEA900-P9", - "Date" : "Mon, 31 Aug 2026 17:34:57 GMT", - "Via" : "1.1 0ddbf3138c96d4b7c9f8047edb515414.cloudfront.net (CloudFront)", + "X-Amz-Cf-Pop" : "SEA900-P10", + "Date" : "Wed, 02 Sep 2026 05:16:22 GMT", + "Via" : "1.1 ffe9646b2ea911744e2d51fc0715cedc.cloudfront.net (CloudFront)", "Strict-Transport-Security" : "max-age=31536000; includeSubDomains", "Access-Control-Expose-Headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url", "Cache-Control" : "no-store, no-cache, must-revalidate, proxy-revalidate", "ETag" : "W/\"178-kKbrsfThc+xXlggXc9RQYV+Gr9I\"", "Access-Control-Allow-Credentials" : "true", - "X-Amz-Cf-Id" : "69k-ecAVj2Lnm-UzhkZja5fnS32iYIU3loV2Z5iW2gO3a2YF5MIGhg==", + "X-Amz-Cf-Id" : "6W3iaaNhnJW4QXhbwXX-Ny3xVKHUC_7otrnY7xDZnMRrxTMfNAanDQ==", "Vary" : "Origin, Accept-Encoding", "Expires" : "0", "Surrogate-Control" : "no-store", @@ -30,5 +30,5 @@ "scenarioName" : "scenario-1-api-apikey-login", "requiredScenarioState" : "scenario-1-api-apikey-login-7", "newScenarioState" : "scenario-1-api-apikey-login-8", - "insertionIndex" : 117 + "insertionIndex" : 205 } \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/api_apikey_login-fda483633056.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/api_apikey_login-fda483633056.json index 3fd19586..3f2b4612 100644 --- a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/api_apikey_login-fda483633056.json +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/api_apikey_login-fda483633056.json @@ -10,15 +10,15 @@ "bodyFileName" : "api_apikey_login-fda483633056.json", "headers" : { "X-Cache" : "Miss from cloudfront", - "X-Amz-Cf-Pop" : "SEA900-P9", - "Date" : "Mon, 31 Aug 2026 17:34:55 GMT", - "Via" : "1.1 88f286e23c15fc2f62a741db8207a67a.cloudfront.net (CloudFront)", + "X-Amz-Cf-Pop" : "SEA900-P10", + "Date" : "Wed, 02 Sep 2026 05:16:21 GMT", + "Via" : "1.1 82fa7f20ab5a12301da8e01f9493e222.cloudfront.net (CloudFront)", "Strict-Transport-Security" : "max-age=31536000; includeSubDomains", "Access-Control-Expose-Headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url", "Cache-Control" : "no-store, no-cache, must-revalidate, proxy-revalidate", "ETag" : "W/\"178-kKbrsfThc+xXlggXc9RQYV+Gr9I\"", "Access-Control-Allow-Credentials" : "true", - "X-Amz-Cf-Id" : "jGMIS0vFeMAUNqIOfm8F4ygcn0y-1PRmZFCcMH3u_boO7_v__VJTrg==", + "X-Amz-Cf-Id" : "e8tFNa9JscmsTVjGSnVGDaO-wzk69sBYdtoUlmnM9DCwmDpfiwHE6A==", "Vary" : "Origin, Accept-Encoding", "Expires" : "0", "Surrogate-Control" : "no-store", @@ -30,5 +30,5 @@ "scenarioName" : "scenario-1-api-apikey-login", "requiredScenarioState" : "scenario-1-api-apikey-login-6", "newScenarioState" : "scenario-1-api-apikey-login-7", - "insertionIndex" : 119 + "insertionIndex" : 208 } \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-08983257c32b.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-08983257c32b.json new file mode 100644 index 00000000..8c2ec01d --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-08983257c32b.json @@ -0,0 +1,41 @@ +{ + "id" : "5b64a1d5-cf8f-3a99-a25d-94e2daff200e", + "name" : "v1_experiment", + "request" : { + "url" : "/v1/experiment", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"project_id\":\"f1e858a4-58e3-408f-983f-016760d7fa25\",\"name\":\"serial-test\"}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_experiment-08983257c32b.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "X-Amz-Cf-Pop" : "SEA900-P10", + "Date" : "Wed, 02 Sep 2026 05:16:22 GMT", + "Via" : "1.1 83d24992402f7b214901ab76fbdc11e2.cloudfront.net (CloudFront)", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains", + "Access-Control-Expose-Headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url", + "Cache-Control" : "no-store, no-cache, must-revalidate, proxy-revalidate", + "ETag" : "W/\"1bf-97m5BoF5VHRxK0jj4iLHHlNNYB4\"", + "Access-Control-Allow-Credentials" : "true", + "X-Amz-Cf-Id" : "c0grgEz0gtxARuuLz0GCW7fbc8jVAysoW4WwE-VE7d_tR1Axt3_6eg==", + "Vary" : "Origin, Accept-Encoding", + "Expires" : "0", + "Surrogate-Control" : "no-store", + "Content-Type" : "application/json; charset=utf-8" + } + }, + "uuid" : "5b64a1d5-cf8f-3a99-a25d-94e2daff200e", + "persistent" : true, + "insertionIndex" : 204 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-1aa85d713cc8.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-1aa85d713cc8.json new file mode 100644 index 00000000..799dc478 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-1aa85d713cc8.json @@ -0,0 +1,41 @@ +{ + "id" : "bef86057-c3e8-3b36-b0ea-be2e85330a0b", + "name" : "v1_experiment", + "request" : { + "url" : "/v1/experiment", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"project_id\":\"f1e858a4-58e3-408f-983f-016760d7fa25\",\"name\":\"config-default-concurrency-test\"}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_experiment-1aa85d713cc8.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "X-Amz-Cf-Pop" : "SEA900-P10", + "Date" : "Wed, 02 Sep 2026 05:16:18 GMT", + "Via" : "1.1 ffe9646b2ea911744e2d51fc0715cedc.cloudfront.net (CloudFront)", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains", + "Access-Control-Expose-Headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url", + "Cache-Control" : "no-store, no-cache, must-revalidate, proxy-revalidate", + "ETag" : "W/\"1d3-mDmDvj0ti3MLK5KlyF9Qj+KScEg\"", + "Access-Control-Allow-Credentials" : "true", + "X-Amz-Cf-Id" : "qZYcm9PgK1sdc-fjRu_PdYwHLsmXjwkf1HAF9FqXE2OueG2SH4Pv3w==", + "Vary" : "Origin, Accept-Encoding", + "Expires" : "0", + "Surrogate-Control" : "no-store", + "Content-Type" : "application/json; charset=utf-8" + } + }, + "uuid" : "bef86057-c3e8-3b36-b0ea-be2e85330a0b", + "persistent" : true, + "insertionIndex" : 219 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-409b337a72c9.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-409b337a72c9.json new file mode 100644 index 00000000..8bbe7c6f --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-409b337a72c9.json @@ -0,0 +1,41 @@ +{ + "id" : "72603ff3-d67e-3e6c-acaa-8f9e27a4992d", + "name" : "v1_experiment", + "request" : { + "url" : "/v1/experiment", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"project_id\":\"f1e858a4-58e3-408f-983f-016760d7fa25\",\"name\":\"failing-task-test\"}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_experiment-409b337a72c9.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "X-Amz-Cf-Pop" : "SEA900-P10", + "Date" : "Wed, 02 Sep 2026 05:16:25 GMT", + "Via" : "1.1 4ac8d091dce10e726cfc5404bfed72b8.cloudfront.net (CloudFront)", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains", + "Access-Control-Expose-Headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url", + "Cache-Control" : "no-store, no-cache, must-revalidate, proxy-revalidate", + "ETag" : "W/\"1c5-0xoT2bc4TJrBN/DjLJgLlbr9jio\"", + "Access-Control-Allow-Credentials" : "true", + "X-Amz-Cf-Id" : "9KB3UHD3YQnWzlRzNzE2DTbybpILaUaIAj9xifFANrxpzxd_Y8-mzA==", + "Vary" : "Origin, Accept-Encoding", + "Expires" : "0", + "Surrogate-Control" : "no-store", + "Content-Type" : "application/json; charset=utf-8" + } + }, + "uuid" : "72603ff3-d67e-3e6c-acaa-8f9e27a4992d", + "persistent" : true, + "insertionIndex" : 192 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-4c3bb7c53c36.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-4c3bb7c53c36.json new file mode 100644 index 00000000..df1063ae --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-4c3bb7c53c36.json @@ -0,0 +1,41 @@ +{ + "id" : "8964d68a-597d-3eb5-b4e9-e3aa97abb792", + "name" : "v1_experiment", + "request" : { + "url" : "/v1/experiment", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"project_id\":\"f1e858a4-58e3-408f-983f-016760d7fa25\",\"name\":\"byo-executor-test\"}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_experiment-4c3bb7c53c36.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "X-Amz-Cf-Pop" : "SEA900-P10", + "Date" : "Wed, 02 Sep 2026 05:16:20 GMT", + "Via" : "1.1 4ac8d091dce10e726cfc5404bfed72b8.cloudfront.net (CloudFront)", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains", + "Access-Control-Expose-Headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url", + "Cache-Control" : "no-store, no-cache, must-revalidate, proxy-revalidate", + "ETag" : "W/\"1c5-XHOo3Sc+fQrk1dpnwRSa1Vcfu6M\"", + "Access-Control-Allow-Credentials" : "true", + "X-Amz-Cf-Id" : "la7wSkitfmU0PVE4wGTMx6YcXKW9RCXicWLh5cFqQwojOmaiVsGnmA==", + "Vary" : "Origin, Accept-Encoding", + "Expires" : "0", + "Surrogate-Control" : "no-store", + "Content-Type" : "application/json; charset=utf-8" + } + }, + "uuid" : "8964d68a-597d-3eb5-b4e9-e3aa97abb792", + "persistent" : true, + "insertionIndex" : 213 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-640904b89031.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-640904b89031.json new file mode 100644 index 00000000..b3df5b56 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-640904b89031.json @@ -0,0 +1,44 @@ +{ + "id" : "4df85c81-84c8-319f-a4b1-dd6eb4a70708", + "name" : "v1_experiment", + "request" : { + "url" : "/v1/experiment", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"project_id\":\"f1e858a4-58e3-408f-983f-016760d7fa25\",\"name\":\"bad-score-test\"}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_experiment-640904b89031.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "X-Amz-Cf-Pop" : "SEA900-P10", + "Date" : "Wed, 02 Sep 2026 05:16:19 GMT", + "Via" : "1.1 0df7f27a01014ab815259ca2d88193c6.cloudfront.net (CloudFront)", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains", + "Access-Control-Expose-Headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url", + "Cache-Control" : "no-store, no-cache, must-revalidate, proxy-revalidate", + "ETag" : "W/\"1c2-LL5Gg+/1eJtV3ksHwkyQRHY7WIc\"", + "Access-Control-Allow-Credentials" : "true", + "X-Amz-Cf-Id" : "TUmtBd073JSpp2RBwdfZq2iIKID_yf0Wh2dm8zVQBJJ8eWHL0NbF7A==", + "Vary" : "Origin, Accept-Encoding", + "Expires" : "0", + "Surrogate-Control" : "no-store", + "Content-Type" : "application/json; charset=utf-8" + } + }, + "uuid" : "4df85c81-84c8-319f-a4b1-dd6eb4a70708", + "persistent" : true, + "scenarioName" : "scenario-3-v1-experiment", + "requiredScenarioState" : "Started", + "newScenarioState" : "scenario-3-v1-experiment-2", + "insertionIndex" : 216 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-81c93d1c26be.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-81c93d1c26be.json new file mode 100644 index 00000000..8b53b2a9 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-81c93d1c26be.json @@ -0,0 +1,41 @@ +{ + "id" : "2d9ed8b9-81c9-35f1-b790-3fff0d2237b0", + "name" : "v1_experiment", + "request" : { + "url" : "/v1/experiment", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"project_id\":\"f1e858a4-58e3-408f-983f-016760d7fa25\",\"name\":\"concurrency-test\"}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_experiment-81c93d1c26be.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "X-Amz-Cf-Pop" : "SEA900-P10", + "Date" : "Wed, 02 Sep 2026 05:16:24 GMT", + "Via" : "1.1 b669d9add7767f73665f1f8b7e8cd802.cloudfront.net (CloudFront)", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains", + "Access-Control-Expose-Headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url", + "Cache-Control" : "no-store, no-cache, must-revalidate, proxy-revalidate", + "ETag" : "W/\"1c4-grPA721Zvp9tyi/CCXRTJj+1nVk\"", + "Access-Control-Allow-Credentials" : "true", + "X-Amz-Cf-Id" : "oScHSb0cpl-vBmVoDYRGeDGJzhafnwFlLw-UCxru1C-Q96Hm9bN7sg==", + "Vary" : "Origin, Accept-Encoding", + "Expires" : "0", + "Surrogate-Control" : "no-store", + "Content-Type" : "application/json; charset=utf-8" + } + }, + "uuid" : "2d9ed8b9-81c9-35f1-b790-3fff0d2237b0", + "persistent" : true, + "insertionIndex" : 198 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-a1296773fd64.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-a1296773fd64.json new file mode 100644 index 00000000..65d6ed3e --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-a1296773fd64.json @@ -0,0 +1,41 @@ +{ + "id" : "c24486a2-2de0-3d14-b3cf-aa8755b0ab63", + "name" : "v1_experiment", + "request" : { + "url" : "/v1/experiment", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"project_id\":\"f1e858a4-58e3-408f-983f-016760d7fa25\",\"name\":\"supplied-executor-concurrency-test\"}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_experiment-a1296773fd64.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "X-Amz-Cf-Pop" : "SEA900-P10", + "Date" : "Wed, 02 Sep 2026 05:16:25 GMT", + "Via" : "1.1 ee5f8da78d4211a93c9dba8864a4067e.cloudfront.net (CloudFront)", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains", + "Access-Control-Expose-Headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url", + "Cache-Control" : "no-store, no-cache, must-revalidate, proxy-revalidate", + "ETag" : "W/\"1d6-o5VGeBLncWhhFTItbGvxP0e28oQ\"", + "Access-Control-Allow-Credentials" : "true", + "X-Amz-Cf-Id" : "iKLgvT512ETWbYhfJ3ifSNQUgCa0rBkdjvjqCz22mPaxravGoWftFA==", + "Vary" : "Origin, Accept-Encoding", + "Expires" : "0", + "Surrogate-Control" : "no-store", + "Content-Type" : "application/json; charset=utf-8" + } + }, + "uuid" : "c24486a2-2de0-3d14-b3cf-aa8755b0ab63", + "persistent" : true, + "insertionIndex" : 195 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-a6fe1b460300.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-a6fe1b460300.json new file mode 100644 index 00000000..1513b507 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-a6fe1b460300.json @@ -0,0 +1,41 @@ +{ + "id" : "b79ecd18-2d9e-3c6a-a81c-855252f53364", + "name" : "v1_experiment", + "request" : { + "url" : "/v1/experiment", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"project_id\":\"f1e858a4-58e3-408f-983f-016760d7fa25\",\"name\":\"start-test\"}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_experiment-a6fe1b460300.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "X-Amz-Cf-Pop" : "SEA900-P10", + "Date" : "Wed, 02 Sep 2026 05:16:26 GMT", + "Via" : "1.1 dbfd9bcc806d4c322e72b461b2458112.cloudfront.net (CloudFront)", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains", + "Access-Control-Expose-Headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url", + "Cache-Control" : "no-store, no-cache, must-revalidate, proxy-revalidate", + "ETag" : "W/\"1be-+xWx4aXHBjzJNFK6Xn4fSlVfCe4\"", + "Access-Control-Allow-Credentials" : "true", + "X-Amz-Cf-Id" : "tAlHb1bV938e8jdhV3f2GTcqNAA-woyJzZ_ke5b1VcRhPu7yitaPaQ==", + "Vary" : "Origin, Accept-Encoding", + "Expires" : "0", + "Surrogate-Control" : "no-store", + "Content-Type" : "application/json; charset=utf-8" + } + }, + "uuid" : "b79ecd18-2d9e-3c6a-a81c-855252f53364", + "persistent" : true, + "insertionIndex" : 189 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-b5bb3690fa91.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-b5bb3690fa91.json new file mode 100644 index 00000000..c68810bf --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-b5bb3690fa91.json @@ -0,0 +1,41 @@ +{ + "id" : "86fc2d09-371e-35e9-aa8a-d24bd70fbc5f", + "name" : "v1_experiment", + "request" : { + "url" : "/v1/experiment", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"project_id\":\"f1e858a4-58e3-408f-983f-016760d7fa25\",\"name\":\"backpressure-test\"}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_experiment-b5bb3690fa91.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "X-Amz-Cf-Pop" : "SEA900-P10", + "Date" : "Wed, 02 Sep 2026 05:16:23 GMT", + "Via" : "1.1 4ac8d091dce10e726cfc5404bfed72b8.cloudfront.net (CloudFront)", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains", + "Access-Control-Expose-Headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url", + "Cache-Control" : "no-store, no-cache, must-revalidate, proxy-revalidate", + "ETag" : "W/\"1c5-bDIQA3jOnL5SKrWX+mgZW9fBA4c\"", + "Access-Control-Allow-Credentials" : "true", + "X-Amz-Cf-Id" : "a7sCoArRYNY98wWfh1TDlhzYnXzCfk9l46M_9edHIu4c3MFcLz8tmg==", + "Vary" : "Origin, Accept-Encoding", + "Expires" : "0", + "Surrogate-Control" : "no-store", + "Content-Type" : "application/json; charset=utf-8" + } + }, + "uuid" : "86fc2d09-371e-35e9-aa8a-d24bd70fbc5f", + "persistent" : true, + "insertionIndex" : 201 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-d25fd7674fdc.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-d25fd7674fdc.json new file mode 100644 index 00000000..f30aa64c --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-d25fd7674fdc.json @@ -0,0 +1,41 @@ +{ + "id" : "8414d439-b521-35f4-8bee-0a54e09dee2b", + "name" : "v1_experiment", + "request" : { + "url" : "/v1/experiment", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"project_id\":\"f1e858a4-58e3-408f-983f-016760d7fa25\",\"name\":\"max-concurrency-test\"}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_experiment-d25fd7674fdc.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "X-Amz-Cf-Pop" : "SEA900-P10", + "Date" : "Wed, 02 Sep 2026 05:16:21 GMT", + "Via" : "1.1 cb2aa1abf9fb243a4dc4cb073a92424e.cloudfront.net (CloudFront)", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains", + "Access-Control-Expose-Headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url", + "Cache-Control" : "no-store, no-cache, must-revalidate, proxy-revalidate", + "ETag" : "W/\"1c8-pRyrCQTW84pbwgU7+aHf5A2snnM\"", + "Access-Control-Allow-Credentials" : "true", + "X-Amz-Cf-Id" : "Gqcx-UeeTTIatKzz2LvKZAZpPOwEUPtbeApqojMLMklnnfiZxhZ2vA==", + "Vary" : "Origin, Accept-Encoding", + "Expires" : "0", + "Surrogate-Control" : "no-store", + "Content-Type" : "application/json; charset=utf-8" + } + }, + "uuid" : "8414d439-b521-35f4-8bee-0a54e09dee2b", + "persistent" : true, + "insertionIndex" : 207 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-f4bcbfaef3af.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-f4bcbfaef3af.json new file mode 100644 index 00000000..e77cd0ca --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-f4bcbfaef3af.json @@ -0,0 +1,43 @@ +{ + "id" : "52ce3247-31fb-354d-bb03-8abc792fb339", + "name" : "v1_experiment", + "request" : { + "url" : "/v1/experiment", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"project_id\":\"f1e858a4-58e3-408f-983f-016760d7fa25\",\"name\":\"bad-score-test\"}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_experiment-f4bcbfaef3af.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "X-Amz-Cf-Pop" : "SEA900-P10", + "Date" : "Wed, 02 Sep 2026 05:16:20 GMT", + "Via" : "1.1 b3ccaedda78c63d5967b57382ceb4cbe.cloudfront.net (CloudFront)", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains", + "Access-Control-Expose-Headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url", + "Cache-Control" : "no-store, no-cache, must-revalidate, proxy-revalidate", + "ETag" : "W/\"1c2-LL5Gg+/1eJtV3ksHwkyQRHY7WIc\"", + "Access-Control-Allow-Credentials" : "true", + "X-Amz-Cf-Id" : "1uuKpdw8Q4mJ4cQE0EkJyJPeEI1bJjBT4L4iSiyV4mIwAhcssWorng==", + "Vary" : "Origin, Accept-Encoding", + "Expires" : "0", + "Surrogate-Control" : "no-store", + "Content-Type" : "application/json; charset=utf-8" + } + }, + "uuid" : "52ce3247-31fb-354d-bb03-8abc792fb339", + "persistent" : true, + "scenarioName" : "scenario-3-v1-experiment", + "requiredScenarioState" : "scenario-3-v1-experiment-2", + "insertionIndex" : 210 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-2f16e640e582.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-2f16e640e582.json new file mode 100644 index 00000000..7ff5eb32 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-2f16e640e582.json @@ -0,0 +1,41 @@ +{ + "id" : "c3af4f3c-ca1e-31b5-93e9-65ff1add676f", + "name" : "v1_project", + "request" : { + "urlPath" : "/v1/project", + "method" : "GET", + "queryParameters" : { + "project_name" : { + "hasExactly" : [ { + "equalTo" : "java-unit-test" + } ] + } + } + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_project-2f16e640e582.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "X-Amz-Cf-Pop" : "SEA900-P10", + "Date" : "Wed, 02 Sep 2026 05:16:23 GMT", + "Via" : "1.1 cb2aa1abf9fb243a4dc4cb073a92424e.cloudfront.net (CloudFront)", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains", + "Access-Control-Expose-Headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url", + "Cache-Control" : "no-store, no-cache, must-revalidate, proxy-revalidate", + "ETag" : "W/\"169-XiwCuJsCqAZuAH8JspCgkYonnKw\"", + "Access-Control-Allow-Credentials" : "true", + "X-Amz-Cf-Id" : "d_jfQha_H3KhK9-in4sTyHL86vSs1cGgIEiRQPDMLjDMnzxdnTWbpg==", + "Vary" : "Origin, Accept-Encoding", + "Expires" : "0", + "Surrogate-Control" : "no-store", + "Content-Type" : "application/json; charset=utf-8" + } + }, + "uuid" : "c3af4f3c-ca1e-31b5-93e9-65ff1add676f", + "persistent" : true, + "scenarioName" : "scenario-2-v1-project", + "requiredScenarioState" : "scenario-2-v1-project-9", + "newScenarioState" : "scenario-2-v1-project-10", + "insertionIndex" : 200 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-30867e7088bd.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-30867e7088bd.json new file mode 100644 index 00000000..cd26e3bd --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-30867e7088bd.json @@ -0,0 +1,41 @@ +{ + "id" : "6606a3c7-099c-3495-b32d-59502385df8a", + "name" : "v1_project", + "request" : { + "urlPath" : "/v1/project", + "method" : "GET", + "queryParameters" : { + "project_name" : { + "hasExactly" : [ { + "equalTo" : "java-unit-test" + } ] + } + } + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_project-30867e7088bd.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "X-Amz-Cf-Pop" : "SEA900-P10", + "Date" : "Wed, 02 Sep 2026 05:16:22 GMT", + "Via" : "1.1 e6b2537b87653726af8a79e6da505188.cloudfront.net (CloudFront)", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains", + "Access-Control-Expose-Headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url", + "Cache-Control" : "no-store, no-cache, must-revalidate, proxy-revalidate", + "ETag" : "W/\"169-XiwCuJsCqAZuAH8JspCgkYonnKw\"", + "Access-Control-Allow-Credentials" : "true", + "X-Amz-Cf-Id" : "BisVV__hX_pxMdgschWS5I2tGXLBV6xsIwpilfN0CTbEWavc677D_Q==", + "Vary" : "Origin, Accept-Encoding", + "Expires" : "0", + "Surrogate-Control" : "no-store", + "Content-Type" : "application/json; charset=utf-8" + } + }, + "uuid" : "6606a3c7-099c-3495-b32d-59502385df8a", + "persistent" : true, + "scenarioName" : "scenario-2-v1-project", + "requiredScenarioState" : "scenario-2-v1-project-8", + "newScenarioState" : "scenario-2-v1-project-9", + "insertionIndex" : 203 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-410ff9f133da.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-410ff9f133da.json new file mode 100644 index 00000000..51214295 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-410ff9f133da.json @@ -0,0 +1,41 @@ +{ + "id" : "2c0e6f0a-b1c8-3b66-8435-51de30a1a7dd", + "name" : "v1_project", + "request" : { + "urlPath" : "/v1/project", + "method" : "GET", + "queryParameters" : { + "project_name" : { + "hasExactly" : [ { + "equalTo" : "java-unit-test" + } ] + } + } + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_project-410ff9f133da.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "X-Amz-Cf-Pop" : "SEA900-P10", + "Date" : "Wed, 02 Sep 2026 05:16:20 GMT", + "Via" : "1.1 fbb003dfc0617e3e058e3dac791dfd5a.cloudfront.net (CloudFront)", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains", + "Access-Control-Expose-Headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url", + "Cache-Control" : "no-store, no-cache, must-revalidate, proxy-revalidate", + "ETag" : "W/\"169-XiwCuJsCqAZuAH8JspCgkYonnKw\"", + "Access-Control-Allow-Credentials" : "true", + "X-Amz-Cf-Id" : "jrq7fA7lr7jNe4RiZm_Ggs9s_AzfdVS2dnfX60EZVboDld21wsU88w==", + "Vary" : "Origin, Accept-Encoding", + "Expires" : "0", + "Surrogate-Control" : "no-store", + "Content-Type" : "application/json; charset=utf-8" + } + }, + "uuid" : "2c0e6f0a-b1c8-3b66-8435-51de30a1a7dd", + "persistent" : true, + "scenarioName" : "scenario-2-v1-project", + "requiredScenarioState" : "scenario-2-v1-project-5", + "newScenarioState" : "scenario-2-v1-project-6", + "insertionIndex" : 212 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-4891298f2969.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-4891298f2969.json new file mode 100644 index 00000000..74163e3c --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-4891298f2969.json @@ -0,0 +1,41 @@ +{ + "id" : "f0270406-fe88-3aec-a5a7-ee6d2435ae22", + "name" : "v1_project", + "request" : { + "urlPath" : "/v1/project", + "method" : "GET", + "queryParameters" : { + "project_name" : { + "hasExactly" : [ { + "equalTo" : "java-unit-test" + } ] + } + } + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_project-4891298f2969.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "X-Amz-Cf-Pop" : "SEA900-P10", + "Date" : "Wed, 02 Sep 2026 05:16:17 GMT", + "Via" : "1.1 7605973575a3551426b82751020317de.cloudfront.net (CloudFront)", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains", + "Access-Control-Expose-Headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url", + "Cache-Control" : "no-store, no-cache, must-revalidate, proxy-revalidate", + "ETag" : "W/\"169-XiwCuJsCqAZuAH8JspCgkYonnKw\"", + "Access-Control-Allow-Credentials" : "true", + "X-Amz-Cf-Id" : "vQ9qigXABPAuprZ5qBrkKvtPEhYP3tu4X6D4797lLLEdib7JarNSmw==", + "Vary" : "Origin, Accept-Encoding", + "Expires" : "0", + "Surrogate-Control" : "no-store", + "Content-Type" : "application/json; charset=utf-8" + } + }, + "uuid" : "f0270406-fe88-3aec-a5a7-ee6d2435ae22", + "persistent" : true, + "scenarioName" : "scenario-2-v1-project", + "requiredScenarioState" : "Started", + "newScenarioState" : "scenario-2-v1-project-2", + "insertionIndex" : 223 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-543a07178006.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-543a07178006.json new file mode 100644 index 00000000..6a71e471 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-543a07178006.json @@ -0,0 +1,41 @@ +{ + "id" : "1c224e28-c695-3150-80a1-b131fa0ac88b", + "name" : "v1_project", + "request" : { + "urlPath" : "/v1/project", + "method" : "GET", + "queryParameters" : { + "project_name" : { + "hasExactly" : [ { + "equalTo" : "java-unit-test" + } ] + } + } + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_project-543a07178006.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "X-Amz-Cf-Pop" : "SEA900-P10", + "Date" : "Wed, 02 Sep 2026 05:16:22 GMT", + "Via" : "1.1 170efbc424be9181bda5d0fcd6e41f30.cloudfront.net (CloudFront)", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains", + "Access-Control-Expose-Headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url", + "Cache-Control" : "no-store, no-cache, must-revalidate, proxy-revalidate", + "ETag" : "W/\"169-XiwCuJsCqAZuAH8JspCgkYonnKw\"", + "Access-Control-Allow-Credentials" : "true", + "X-Amz-Cf-Id" : "Mja_ACcJFzolgy8QfAV79t4xmTYCKo8iY8Adc4ThasenKK3NzWoGFw==", + "Vary" : "Origin, Accept-Encoding", + "Expires" : "0", + "Surrogate-Control" : "no-store", + "Content-Type" : "application/json; charset=utf-8" + } + }, + "uuid" : "1c224e28-c695-3150-80a1-b131fa0ac88b", + "persistent" : true, + "scenarioName" : "scenario-2-v1-project", + "requiredScenarioState" : "scenario-2-v1-project-7", + "newScenarioState" : "scenario-2-v1-project-8", + "insertionIndex" : 206 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-8d973730b6e1.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-8d973730b6e1.json new file mode 100644 index 00000000..a932990d --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-8d973730b6e1.json @@ -0,0 +1,40 @@ +{ + "id" : "26cdf151-c5ec-3cc9-b1e7-35507f5e4bac", + "name" : "v1_project", + "request" : { + "urlPath" : "/v1/project", + "method" : "GET", + "queryParameters" : { + "project_name" : { + "hasExactly" : [ { + "equalTo" : "java-unit-test" + } ] + } + } + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_project-8d973730b6e1.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "X-Amz-Cf-Pop" : "SEA900-P10", + "Date" : "Wed, 02 Sep 2026 05:16:26 GMT", + "Via" : "1.1 b669d9add7767f73665f1f8b7e8cd802.cloudfront.net (CloudFront)", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains", + "Access-Control-Expose-Headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url", + "Cache-Control" : "no-store, no-cache, must-revalidate, proxy-revalidate", + "ETag" : "W/\"169-XiwCuJsCqAZuAH8JspCgkYonnKw\"", + "Access-Control-Allow-Credentials" : "true", + "X-Amz-Cf-Id" : "wFeDJrI08z8pguaFFhHFzFqbee6viP6JLr_qNiMbdhW61agPqSd4VQ==", + "Vary" : "Origin, Accept-Encoding", + "Expires" : "0", + "Surrogate-Control" : "no-store", + "Content-Type" : "application/json; charset=utf-8" + } + }, + "uuid" : "26cdf151-c5ec-3cc9-b1e7-35507f5e4bac", + "persistent" : true, + "scenarioName" : "scenario-2-v1-project", + "requiredScenarioState" : "scenario-2-v1-project-12", + "insertionIndex" : 191 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-b8c4d7b1d75a.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-b8c4d7b1d75a.json new file mode 100644 index 00000000..102c84e8 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-b8c4d7b1d75a.json @@ -0,0 +1,41 @@ +{ + "id" : "1847086f-3d9b-36e6-aa25-20d61660d6bd", + "name" : "v1_project", + "request" : { + "urlPath" : "/v1/project", + "method" : "GET", + "queryParameters" : { + "project_name" : { + "hasExactly" : [ { + "equalTo" : "java-unit-test" + } ] + } + } + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_project-b8c4d7b1d75a.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "X-Amz-Cf-Pop" : "SEA900-P10", + "Date" : "Wed, 02 Sep 2026 05:16:25 GMT", + "Via" : "1.1 b669d9add7767f73665f1f8b7e8cd802.cloudfront.net (CloudFront)", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains", + "Access-Control-Expose-Headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url", + "Cache-Control" : "no-store, no-cache, must-revalidate, proxy-revalidate", + "ETag" : "W/\"169-XiwCuJsCqAZuAH8JspCgkYonnKw\"", + "Access-Control-Allow-Credentials" : "true", + "X-Amz-Cf-Id" : "y-_v-02_V6--WMKTIBI63KfAMgObOvhMeSNKr2igVikNYzAjDBdQXQ==", + "Vary" : "Origin, Accept-Encoding", + "Expires" : "0", + "Surrogate-Control" : "no-store", + "Content-Type" : "application/json; charset=utf-8" + } + }, + "uuid" : "1847086f-3d9b-36e6-aa25-20d61660d6bd", + "persistent" : true, + "scenarioName" : "scenario-2-v1-project", + "requiredScenarioState" : "scenario-2-v1-project-11", + "newScenarioState" : "scenario-2-v1-project-12", + "insertionIndex" : 194 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-c0742bb3c63f.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-c0742bb3c63f.json new file mode 100644 index 00000000..ea1d7463 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-c0742bb3c63f.json @@ -0,0 +1,41 @@ +{ + "id" : "02e68d25-de8f-38c9-b80e-6c8c4390e791", + "name" : "v1_project", + "request" : { + "urlPath" : "/v1/project", + "method" : "GET", + "queryParameters" : { + "project_name" : { + "hasExactly" : [ { + "equalTo" : "java-unit-test" + } ] + } + } + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_project-c0742bb3c63f.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "X-Amz-Cf-Pop" : "SEA900-P10", + "Date" : "Wed, 02 Sep 2026 05:16:21 GMT", + "Via" : "1.1 170efbc424be9181bda5d0fcd6e41f30.cloudfront.net (CloudFront)", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains", + "Access-Control-Expose-Headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url", + "Cache-Control" : "no-store, no-cache, must-revalidate, proxy-revalidate", + "ETag" : "W/\"169-XiwCuJsCqAZuAH8JspCgkYonnKw\"", + "Access-Control-Allow-Credentials" : "true", + "X-Amz-Cf-Id" : "BdBDKFNLys-fhpQgUW1dDYUHVwZ-5LgB6jyFd3BK7Im7fRrI-GJMsg==", + "Vary" : "Origin, Accept-Encoding", + "Expires" : "0", + "Surrogate-Control" : "no-store", + "Content-Type" : "application/json; charset=utf-8" + } + }, + "uuid" : "02e68d25-de8f-38c9-b80e-6c8c4390e791", + "persistent" : true, + "scenarioName" : "scenario-2-v1-project", + "requiredScenarioState" : "scenario-2-v1-project-6", + "newScenarioState" : "scenario-2-v1-project-7", + "insertionIndex" : 209 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-dd7665d7a48a.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-dd7665d7a48a.json new file mode 100644 index 00000000..b1042b87 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-dd7665d7a48a.json @@ -0,0 +1,41 @@ +{ + "id" : "0254f542-fbbd-37af-903b-d925ff1ca4af", + "name" : "v1_project", + "request" : { + "urlPath" : "/v1/project", + "method" : "GET", + "queryParameters" : { + "project_name" : { + "hasExactly" : [ { + "equalTo" : "java-unit-test" + } ] + } + } + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_project-dd7665d7a48a.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "X-Amz-Cf-Pop" : "SEA900-P10", + "Date" : "Wed, 02 Sep 2026 05:16:19 GMT", + "Via" : "1.1 98b0bd91dae8cd13230da037f5a4d5e0.cloudfront.net (CloudFront)", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains", + "Access-Control-Expose-Headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url", + "Cache-Control" : "no-store, no-cache, must-revalidate, proxy-revalidate", + "ETag" : "W/\"169-XiwCuJsCqAZuAH8JspCgkYonnKw\"", + "Access-Control-Allow-Credentials" : "true", + "X-Amz-Cf-Id" : "467kCtUocQJkAM7_BY-LX3MPHUYQPbB1-tkQAKqBqE2aBFCwkkgc3g==", + "Vary" : "Origin, Accept-Encoding", + "Expires" : "0", + "Surrogate-Control" : "no-store", + "Content-Type" : "application/json; charset=utf-8" + } + }, + "uuid" : "0254f542-fbbd-37af-903b-d925ff1ca4af", + "persistent" : true, + "scenarioName" : "scenario-2-v1-project", + "requiredScenarioState" : "scenario-2-v1-project-4", + "newScenarioState" : "scenario-2-v1-project-5", + "insertionIndex" : 215 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-e548ebcb1218.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-e548ebcb1218.json new file mode 100644 index 00000000..753b4b21 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-e548ebcb1218.json @@ -0,0 +1,41 @@ +{ + "id" : "8aebafad-efbc-3e6a-af96-87882f574aa1", + "name" : "v1_project", + "request" : { + "urlPath" : "/v1/project", + "method" : "GET", + "queryParameters" : { + "project_name" : { + "hasExactly" : [ { + "equalTo" : "java-unit-test" + } ] + } + } + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_project-e548ebcb1218.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "X-Amz-Cf-Pop" : "SEA900-P10", + "Date" : "Wed, 02 Sep 2026 05:16:24 GMT", + "Via" : "1.1 a53bab1af200813b8f27e3c0a28b4964.cloudfront.net (CloudFront)", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains", + "Access-Control-Expose-Headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url", + "Cache-Control" : "no-store, no-cache, must-revalidate, proxy-revalidate", + "ETag" : "W/\"169-XiwCuJsCqAZuAH8JspCgkYonnKw\"", + "Access-Control-Allow-Credentials" : "true", + "X-Amz-Cf-Id" : "2FbRht71bcgaB5hB5gXNag-RSNfqh_6BryO4gdTNwW4KCavpuHtVEg==", + "Vary" : "Origin, Accept-Encoding", + "Expires" : "0", + "Surrogate-Control" : "no-store", + "Content-Type" : "application/json; charset=utf-8" + } + }, + "uuid" : "8aebafad-efbc-3e6a-af96-87882f574aa1", + "persistent" : true, + "scenarioName" : "scenario-2-v1-project", + "requiredScenarioState" : "scenario-2-v1-project-10", + "newScenarioState" : "scenario-2-v1-project-11", + "insertionIndex" : 197 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-e74886687a34.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-e74886687a34.json new file mode 100644 index 00000000..c6444299 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-e74886687a34.json @@ -0,0 +1,41 @@ +{ + "id" : "b0b988b5-0416-376e-8651-152095d5b692", + "name" : "v1_project", + "request" : { + "urlPath" : "/v1/project", + "method" : "GET", + "queryParameters" : { + "project_name" : { + "hasExactly" : [ { + "equalTo" : "java-unit-test" + } ] + } + } + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_project-e74886687a34.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "X-Amz-Cf-Pop" : "SEA900-P10", + "Date" : "Wed, 02 Sep 2026 05:16:18 GMT", + "Via" : "1.1 dbfd9bcc806d4c322e72b461b2458112.cloudfront.net (CloudFront)", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains", + "Access-Control-Expose-Headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url", + "Cache-Control" : "no-store, no-cache, must-revalidate, proxy-revalidate", + "ETag" : "W/\"169-XiwCuJsCqAZuAH8JspCgkYonnKw\"", + "Access-Control-Allow-Credentials" : "true", + "X-Amz-Cf-Id" : "qa2QY9XA3Dp2q5xCFMV6FYXw5f93p0p9xYLWX8l8FE1O9DmqE4JWLw==", + "Vary" : "Origin, Accept-Encoding", + "Expires" : "0", + "Surrogate-Control" : "no-store", + "Content-Type" : "application/json; charset=utf-8" + } + }, + "uuid" : "b0b988b5-0416-376e-8651-152095d5b692", + "persistent" : true, + "scenarioName" : "scenario-2-v1-project", + "requiredScenarioState" : "scenario-2-v1-project-3", + "newScenarioState" : "scenario-2-v1-project-4", + "insertionIndex" : 218 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-e7e35e493e43.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-e7e35e493e43.json new file mode 100644 index 00000000..ac609e8b --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-e7e35e493e43.json @@ -0,0 +1,41 @@ +{ + "id" : "d2f428e5-f001-3e3a-9b24-5e1d7065518c", + "name" : "v1_project", + "request" : { + "urlPath" : "/v1/project", + "method" : "GET", + "queryParameters" : { + "project_name" : { + "hasExactly" : [ { + "equalTo" : "java-unit-test" + } ] + } + } + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_project-e7e35e493e43.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "X-Amz-Cf-Pop" : "SEA900-P10", + "Date" : "Wed, 02 Sep 2026 05:16:17 GMT", + "Via" : "1.1 73b0c4a85645a8031ba157e0b3e28ffc.cloudfront.net (CloudFront)", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains", + "Access-Control-Expose-Headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url", + "Cache-Control" : "no-store, no-cache, must-revalidate, proxy-revalidate", + "ETag" : "W/\"169-XiwCuJsCqAZuAH8JspCgkYonnKw\"", + "Access-Control-Allow-Credentials" : "true", + "X-Amz-Cf-Id" : "n86u2jAoxv1yeryLmxU0IAAJIuSYCp-bRbPbssT4_6Gny6aBD50eXQ==", + "Vary" : "Origin, Accept-Encoding", + "Expires" : "0", + "Surrogate-Control" : "no-store", + "Content-Type" : "application/json; charset=utf-8" + } + }, + "uuid" : "d2f428e5-f001-3e3a-9b24-5e1d7065518c", + "persistent" : true, + "scenarioName" : "scenario-2-v1-project", + "requiredScenarioState" : "scenario-2-v1-project-2", + "newScenarioState" : "scenario-2-v1-project-3", + "insertionIndex" : 221 +} \ No newline at end of file From 3aabe47f940e8d6fb6ee6177bcce48f443a50e1d Mon Sep 17 00:00:00 2001 From: Andrew Kent Date: Tue, 25 Aug 2026 20:19:42 -0600 Subject: [PATCH 2/2] execute evals concurrently --- .../java/dev/braintrust/BraintrustUtils.java | 221 ++++++++ .../dev/braintrust/config/BaseConfig.java | 8 + .../braintrust/config/BraintrustConfig.java | 65 +++ .../dev/braintrust/devserver/Devserver.java | 516 +++++++++++++----- .../java/dev/braintrust/eval/Classifier.java | 4 + .../main/java/dev/braintrust/eval/Eval.java | 293 +++++++++- .../java/dev/braintrust/eval/EvalResult.java | 253 ++++++++- .../main/java/dev/braintrust/eval/Scorer.java | 4 + .../main/java/dev/braintrust/eval/Task.java | 5 + .../dev/braintrust/eval/TracedClassifier.java | 4 + .../dev/braintrust/eval/TracedScorer.java | 4 + .../braintrust/trace/BraintrustTracing.java | 20 +- .../dev/braintrust/BraintrustUtilsTest.java | 71 +++ .../config/BraintrustConfigTest.java | 14 + .../devserver/DevserverConcurrencyTest.java | 422 ++++++++++++++ .../braintrust/devserver/DevserverTest.java | 35 +- .../braintrust/eval/EvalConcurrencyTest.java | 501 +++++++++++++++++ .../java/dev/braintrust/eval/EvalTest.java | 5 + jvm-compat-tests/java21/build.gradle | 51 ++ .../braintrust/jvmcompat/Java21SmokeTest.java | 97 ++++ settings.gradle | 2 + 21 files changed, 2414 insertions(+), 181 deletions(-) create mode 100644 braintrust-sdk/src/test/java/dev/braintrust/devserver/DevserverConcurrencyTest.java create mode 100644 braintrust-sdk/src/test/java/dev/braintrust/eval/EvalConcurrencyTest.java create mode 100644 jvm-compat-tests/java21/build.gradle create mode 100644 jvm-compat-tests/java21/src/test/java/dev/braintrust/jvmcompat/Java21SmokeTest.java diff --git a/braintrust-sdk/src/main/java/dev/braintrust/BraintrustUtils.java b/braintrust-sdk/src/main/java/dev/braintrust/BraintrustUtils.java index 2e39122a..26b8ab86 100644 --- a/braintrust-sdk/src/main/java/dev/braintrust/BraintrustUtils.java +++ b/braintrust-sdk/src/main/java/dev/braintrust/BraintrustUtils.java @@ -3,11 +3,21 @@ import dev.braintrust.api.BraintrustApiClient; import dev.braintrust.openapi.model.Organization; import dev.braintrust.openapi.model.Project; +import java.lang.invoke.MethodHandles; import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.concurrent.AbstractExecutorService; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.Semaphore; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import javax.annotation.Nonnull; public class BraintrustUtils { @@ -71,4 +81,215 @@ public static List append(List list, T value) { result.add(value); return result; } + + /** + * An executor that runs at most {@code maxSize} tasks concurrently and backpressures whoever + * submits to it once that limit is reached. + * + *

NOTE: On Java 21+ each task gets its own virtual thread; on older JVMs this falls back to + * a bounded pool of platform daemon threads. + */ + public static ExecutorService newExecutor(int maxSize, String threadNamePrefix) { + if (maxSize < 1) { + throw new IllegalArgumentException("max size must be 1 or more: " + maxSize); + } + var virtual = VirtualThreads.newExecutor(threadNamePrefix); + if (virtual != null) { + return new BoundedExecutorService(virtual, maxSize, threadNamePrefix); + } + return newPlatformThreadExecutor(maxSize, threadNamePrefix); + } + + /** A pool of at most {@code maxSize} daemon platform threads. Used on Java < 21. */ + private static ExecutorService newPlatformThreadExecutor(int maxSize, String threadNamePrefix) { + var counter = new AtomicInteger(); + return new ThreadPoolExecutor( + 0, + maxSize, + 60, + TimeUnit.SECONDS, + new SynchronousQueue<>(), + r -> { + var thread = new Thread(r, threadNamePrefix + counter.incrementAndGet()); + thread.setDaemon(true); + return thread; + }, + (task, pool) -> { + try { + while (!pool.isShutdown()) { + if (pool.getQueue().offer(task, 100, TimeUnit.MILLISECONDS)) { + return; + } + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RejectedExecutionException( + "interrupted while waiting to submit to " + threadNamePrefix, e); + } + throw new RejectedExecutionException( + "executor is shut down: " + threadNamePrefix); + }); + } + + /** + * Reflective access to the Java 21 virtual thread APIs. + * + *

The SDK compiles against Java 17 so {@code Thread.ofVirtual()} and {@code + * Executors.newThreadPerTaskExecutor} aren't on the compile classpath. We look them up once at + * class-init time and fall back to platform threads if anything is missing. + */ + private static final class VirtualThreads { + /** + * {@code Executors.newThreadPerTaskExecutor(ThreadFactory)}, or null if unavailable. Java + * 21 is the first release where virtual threads are a final (non-preview) feature. + */ + private static final java.lang.invoke.MethodHandle NEW_THREAD_PER_TASK_EXECUTOR; + + /** {@code Thread.ofVirtual()}, or null if unavailable. */ + private static final java.lang.invoke.MethodHandle OF_VIRTUAL; + + /** {@code Thread.Builder.OfVirtual.name(String prefix, long start)}. */ + private static final java.lang.invoke.MethodHandle BUILDER_NAME; + + /** {@code Thread.Builder.factory()}. */ + private static final java.lang.invoke.MethodHandle BUILDER_FACTORY; + + static { + java.lang.invoke.MethodHandle newExecutor = null; + java.lang.invoke.MethodHandle ofVirtual = null; + java.lang.invoke.MethodHandle name = null; + java.lang.invoke.MethodHandle factory = null; + if (Runtime.version().feature() >= 21) { + try { + var lookup = MethodHandles.publicLookup(); + var builder = Class.forName("java.lang.Thread$Builder"); + var ofVirtualType = Class.forName("java.lang.Thread$Builder$OfVirtual"); + + newExecutor = + lookup.findStatic( + java.util.concurrent.Executors.class, + "newThreadPerTaskExecutor", + java.lang.invoke.MethodType.methodType( + ExecutorService.class, ThreadFactory.class)); + ofVirtual = + lookup.findStatic( + Thread.class, + "ofVirtual", + java.lang.invoke.MethodType.methodType(ofVirtualType)); + name = + lookup.findVirtual( + ofVirtualType, + "name", + java.lang.invoke.MethodType.methodType( + ofVirtualType, String.class, long.class)); + factory = + lookup.findVirtual( + builder, + "factory", + java.lang.invoke.MethodType.methodType(ThreadFactory.class)); + } catch (ReflectiveOperationException | RuntimeException e) { + // Not a JVM we can use virtual threads on; fall back to platform threads. + newExecutor = null; + ofVirtual = null; + name = null; + factory = null; + } + } + NEW_THREAD_PER_TASK_EXECUTOR = newExecutor; + OF_VIRTUAL = ofVirtual; + BUILDER_NAME = name; + BUILDER_FACTORY = factory; + } + + /** + * A thread-per-task executor backed by virtual threads named {@code 0}, {@code + * 1}, ..., or null when virtual threads aren't available on this JVM. + */ + static ExecutorService newExecutor(String threadNamePrefix) { + if (NEW_THREAD_PER_TASK_EXECUTOR == null) { + return null; + } + try { + var builder = OF_VIRTUAL.invoke(); + builder = BUILDER_NAME.invoke(builder, threadNamePrefix, 0L); + var factory = (ThreadFactory) BUILDER_FACTORY.invoke(builder); + return (ExecutorService) NEW_THREAD_PER_TASK_EXECUTOR.invoke(factory); + } catch (Throwable t) { + return null; + } + } + + private VirtualThreads() {} + } + + /** + * Caps a thread-per-task executor at {@code maxSize} concurrent tasks, blocking the submitter + * while the limit is saturated. + * + *

Deliberately a semaphore over a thread-per-task executor rather than a pool: virtual + * threads are meant to be created per task, not pooled and reused. + */ + private static final class BoundedExecutorService extends AbstractExecutorService { + private final ExecutorService delegate; + private final Semaphore permits; + private final String threadNamePrefix; + + BoundedExecutorService(ExecutorService delegate, int maxSize, String threadNamePrefix) { + this.delegate = delegate; + this.permits = new Semaphore(maxSize); + this.threadNamePrefix = threadNamePrefix; + } + + @Override + public void execute(Runnable command) { + if (delegate.isShutdown()) { + throw new RejectedExecutionException("executor is shut down: " + threadNamePrefix); + } + try { + permits.acquire(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RejectedExecutionException( + "interrupted while waiting to submit to " + threadNamePrefix, e); + } + try { + delegate.execute( + () -> { + try { + command.run(); + } finally { + permits.release(); + } + }); + } catch (RuntimeException | Error e) { + permits.release(); + throw e; + } + } + + @Override + public void shutdown() { + delegate.shutdown(); + } + + @Override + public List shutdownNow() { + return delegate.shutdownNow(); + } + + @Override + public boolean isShutdown() { + return delegate.isShutdown(); + } + + @Override + public boolean isTerminated() { + return delegate.isTerminated(); + } + + @Override + public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException { + return delegate.awaitTermination(timeout, unit); + } + } } diff --git a/braintrust-sdk/src/main/java/dev/braintrust/config/BaseConfig.java b/braintrust-sdk/src/main/java/dev/braintrust/config/BaseConfig.java index 6b00d152..e3dd9349 100644 --- a/braintrust-sdk/src/main/java/dev/braintrust/config/BaseConfig.java +++ b/braintrust-sdk/src/main/java/dev/braintrust/config/BaseConfig.java @@ -75,4 +75,12 @@ protected T cast(@Nonnull String value, @Nonnull Class clazz) { } return NULL_OVERRIDE.equals(settingValue) ? null : settingValue; } + + /** throws if number is not positive */ + protected final int assertPositive(int number) { + if (number <= 0) { + throw new IllegalArgumentException("positive number required. got: " + number); + } + return number; + } } diff --git a/braintrust-sdk/src/main/java/dev/braintrust/config/BraintrustConfig.java b/braintrust-sdk/src/main/java/dev/braintrust/config/BraintrustConfig.java index 5a34ff31..5b05962d 100644 --- a/braintrust-sdk/src/main/java/dev/braintrust/config/BraintrustConfig.java +++ b/braintrust-sdk/src/main/java/dev/braintrust/config/BraintrustConfig.java @@ -64,6 +64,41 @@ public final class BraintrustConfig extends BaseConfig { /** Custom X509 trust manager for OTLP exporter. Builder-only field, not backed by envars. */ private final X509TrustManager x509TrustManager; + /** + * Sizes the thread pools the SDK creates for batch work (e.g. eval execution), and so how much + * of that work runs at once. Used only when the caller has not supplied an executor of their + * own. + */ + private final int defaultMaxConcurrency = + assertPositive(getConfig("BRAINTRUST_DEFAULT_MAX_CONCURRENCY", 3)); + + /** + * How many spans/log records may sit in the OTel batch processor's queue waiting to be + * exported. Anything that ends while the queue is full is silently dropped, so a run + * that produces spans faster than they can be exported — a high-concurrency eval, say — loses + * data. Raise this to buy headroom for bursty runs. + * + *

The queue is a fixed-size array allocated up front, so this trades memory for headroom; + * keep it to a sane number rather than something like {@link Integer#MAX_VALUE}. + */ + private final int otelMaxQueueSize = + assertPositive(getConfig("BRAINTRUST_OTEL_MAX_QUEUE_SIZE", 4096)); + + /** + * How many spans/log records the OTel batch processor sends per export request. The processor + * exports as soon as this many are queued, so raising it means fewer, larger requests — which + * is usually what drains a backed-up queue faster. + */ + private final int otelMaxExportBatchSize = + assertPositive(getConfig("BRAINTRUST_OTEL_MAX_EXPORT_BATCH_SIZE", 1024)); + + /** + * How long the OTel batch processor waits before exporting a partial batch, in milliseconds. It + * does not wait this long when a full batch is already queued. + */ + private final int otelExportIntervalMillis = + assertPositive(getConfig("BRAINTRUST_OTEL_EXPORT_INTERVAL_MILLIS", 5_000)); + /** CORS origins to allow when running remote eval devserver */ private final String devserverCorsOriginWhitelistCsv = getConfig( @@ -101,6 +136,14 @@ private BraintrustConfig( // should never happen throw new RuntimeException("A project name or ID is required."); } + if (otelMaxExportBatchSize > otelMaxQueueSize) { + throw new IllegalArgumentException( + "BRAINTRUST_OTEL_MAX_EXPORT_BATCH_SIZE must not exceed" + + " BRAINTRUST_OTEL_MAX_QUEUE_SIZE: " + + otelMaxExportBatchSize + + " > " + + otelMaxQueueSize); + } this.sslContext = sslContext != null ? sslContext : SSLContext.getDefault(); if (x509TrustManager != null) { @@ -260,6 +303,28 @@ public Builder devserverCorsOriginWhitelistCsv(String csv) { return this; } + public Builder defaultMaxConcurrency(int maxConcurrency) { + envOverrides.put("BRAINTRUST_DEFAULT_MAX_CONCURRENCY", String.valueOf(maxConcurrency)); + return this; + } + + public Builder otelMaxQueueSize(int maxQueueSize) { + envOverrides.put("BRAINTRUST_OTEL_MAX_QUEUE_SIZE", String.valueOf(maxQueueSize)); + return this; + } + + public Builder otelMaxExportBatchSize(int maxExportBatchSize) { + envOverrides.put( + "BRAINTRUST_OTEL_MAX_EXPORT_BATCH_SIZE", String.valueOf(maxExportBatchSize)); + return this; + } + + public Builder otelExportIntervalMillis(int exportIntervalMillis) { + envOverrides.put( + "BRAINTRUST_OTEL_EXPORT_INTERVAL_MILLIS", String.valueOf(exportIntervalMillis)); + return this; + } + public BraintrustConfig build() { return new BraintrustConfig(envOverrides, sslContext, x509TrustManager); } diff --git a/braintrust-sdk/src/main/java/dev/braintrust/devserver/Devserver.java b/braintrust-sdk/src/main/java/dev/braintrust/devserver/Devserver.java index 3618fd70..d9b3569f 100644 --- a/braintrust-sdk/src/main/java/dev/braintrust/devserver/Devserver.java +++ b/braintrust-sdk/src/main/java/dev/braintrust/devserver/Devserver.java @@ -26,9 +26,14 @@ import java.io.OutputStream; import java.net.InetSocketAddress; import java.nio.charset.StandardCharsets; +import java.time.Duration; import java.util.*; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.function.Function; import java.util.regex.Pattern; @@ -44,6 +49,9 @@ public class Devserver { private static final Pattern PREVIEW_DOMAIN_PATTERN = Pattern.compile("^https://[^/]+\\.preview\\.braintrust\\.dev$"); + /** How long {@link #stop()} waits for in-flight experiment snapshots to finish. */ + private static final Duration SNAPSHOT_SHUTDOWN_TIMEOUT = Duration.ofSeconds(60); + // Allowed headers for CORS private static final String ALLOWED_HEADERS = String.join( @@ -87,6 +95,31 @@ public class Devserver { private final @Nullable String orgName; private final Map> evals; private @Nullable HttpServer server; + + /** Threads for HTTP request handling. Holds no threads until the first request arrives. */ + private final ExecutorService executor = Executors.newCachedThreadPool(); + + /** + * Threads that eval cases run on, kept separate from the HTTP pool. Submitting to this pool + * blocks once it is saturated, and playground runs submit cases while occupying an HTTP thread + * — so sharing one pool with request handling would deadlock. Experiment snapshots keep using + * it after their response has been sent, so it outlives individual requests. + * + *

Like the HTTP pool it holds no threads while idle, so there is nothing to gain from + * deferring its creation to {@link #start()}. + */ + private final ExecutorService evalExecutor; + + private final int maxConcurrency; + + /** + * Experiment snapshots that are still running. A snapshot outlives the request that triggered + * it (the caller gets the experiment link immediately and the cases keep running), so the + * server has to remember them: {@link #stop()} waits on them rather than truncating an + * experiment someone is already looking at. Finished runs are swept out as new ones arrive. + */ + private final Set runningSnapshots = ConcurrentHashMap.newKeySet(); + private final @Nullable Consumer traceBuilderHook; private final @Nullable Consumer configBuilderHook; @@ -101,6 +134,12 @@ private Devserver(Builder builder) { this.orgName = builder.orgName; this.traceBuilderHook = builder.traceBuilderHook; this.configBuilderHook = builder.configBuilderHook; + this.maxConcurrency = + builder.maxConcurrency != null + ? builder.maxConcurrency + : config.defaultMaxConcurrency(); + this.evalExecutor = + BraintrustUtils.newExecutor(maxConcurrency, "braintrust-devserver-eval-"); Map> evalMap = new HashMap<>(); for (RemoteEval eval : builder.evals) { if (evalMap.containsKey(eval.getName())) { @@ -128,9 +167,13 @@ public synchronized void start() throws IOException { if (server != null) { throw new IllegalStateException("Server is already running"); } + if (executor.isShutdown()) { + // The pools are final and shut down by stop(), so a stopped devserver is done for good. + throw new IllegalStateException("Server has been stopped; build a new one to restart"); + } server = HttpServer.create(new InetSocketAddress(host, port), 0); - server.setExecutor(Executors.newCachedThreadPool()); + server.setExecutor(executor); server.createContext("/", withCors(this::handleHealthCheck)); server.createContext("/list", withCors(this::handleList)); @@ -146,10 +189,68 @@ public synchronized void stop() { if (server != null) { server.stop(0); server = null; + executor.shutdown(); + // Snapshot experiments run in the background and their link has already been handed + // to the user, so give them a bounded chance to finish. This has to happen before the + // pool is shut down: a snapshot still draining its dataset submits its remaining cases + // as it goes, and those submits would be rejected. + awaitRunningSnapshots(SNAPSHOT_SHUTDOWN_TIMEOUT); + // Lets evals still running in the background finish, but rejects new work. + evalExecutor.shutdown(); log.info("Braintrust dev server stopped"); } } + /** + * Remembers a snapshot run so {@link #stop()} can wait on it, sweeping out the ones that have + * since finished. Snapshots are the only evals that outlive their request, and there are few of + * them, so a sweep per snapshot is cheaper than anything that watches them continuously. + */ + private void trackSnapshot(EvalResult result) { + runningSnapshots.removeIf(EvalResult::isDone); + // This may race with stop(): shutdown's best-effort sweep can finish before this result is + // added. That is acceptable once server shutdown is underway, when in-flight requests and + // their background evals are no longer guaranteed to complete. + runningSnapshots.add(result); + } + + /** + * Blocks until every snapshot still running has finished, or {@code timeout} elapses across all + * of them. + * + * @return true if they all finished + */ + private boolean awaitRunningSnapshots(Duration timeout) { + runningSnapshots.removeIf(EvalResult::isDone); + if (runningSnapshots.isEmpty()) { + return true; + } + log.info( + "Waiting up to {}s for {} snapshot experiment(s) to finish", + timeout.toSeconds(), + runningSnapshots.size()); + var deadline = System.nanoTime() + timeout.toNanos(); + for (var result : runningSnapshots) { + var remaining = Duration.ofNanos(Math.max(0, deadline - System.nanoTime())); + try { + if (!result.awaitCompletion(remaining)) { + // Out of budget. The cases run on daemon threads, so whatever this snapshot + // has not flushed by the time the JVM exits is lost. + log.warn( + "Stopped waiting: snapshot experiment(s) still running and may be" + + " incomplete: {}", + result.getExperimentUrl()); + return false; + } + } catch (Exception e) { + // The run aborted; awaitCompletion rethrows its error. Nothing to do about it at + // shutdown and the other snapshots still deserve their wait. + } + } + runningSnapshots.clear(); + return true; + } + private void handleHealthCheck(HttpExchange exchange) throws IOException { if (!"GET".equals(exchange.getRequestMethod())) { sendResponse(exchange, 405, "text/plain", "Method Not Allowed"); @@ -417,134 +518,95 @@ private void handleStreamingEval( final var braintrustParent = parentInfo.braintrustParent(); final var braintrustGeneration = parentInfo.generation(); - // NOTE: this code is serial but written in a thread-safe manner to support - // concurrent dataset fetching and eval execution - extractDataset( - request, - apiClient, - eval.getInputConverter(), - eval.getOutputConverter()) - .forEach( - datasetCase -> { - var evalSpan = - tracer.spanBuilder("eval") - .setNoParent() - .setSpanKind(SpanKind.CLIENT) - .setAttribute( - PARENT, - braintrustParent.toParentValue()) - .startSpan(); - Context evalContext = Context.current().with(evalSpan); - evalContext = - BraintrustContext.setParentInBaggage( - evalContext, - braintrustParent.type(), - braintrustParent.id()); - // Make the eval context (with span and baggage) current - try (var rootScope = evalContext.makeCurrent()) { - final TaskResult taskResult; - { // run task - var taskSpan = tracer.spanBuilder("task").startSpan(); - try (var unused = - Context.current() - .with(taskSpan) - .makeCurrent()) { - var task = eval.getTask(); - try { - taskResult = - task.apply( - datasetCase, mergedParameters); - } catch (Exception e) { - taskSpan.setStatus( - StatusCode.ERROR, e.getMessage()); - taskSpan.recordException(e); - taskSpan.end(); - evalSpan.setStatus( - StatusCode.ERROR, e.getMessage()); - log.debug( - "Task threw exception for input: " - + datasetCase.input(), - e); - // Set eval span attributes so Braintrust can - // resolve the trace - setEvalSpanAttributesForError( - evalSpan, - braintrustParent, - braintrustGeneration, - datasetCase); - // Send progress event even on error so the - // Playground can link to the trace - sendProgressEvent( - os, - evalSpan.getSpanContext().getSpanId(), - datasetCase.origin(), - eval.getName(), - null); - // run scoreForTaskException on each scorer - List> allScorersForError = - new ArrayList<>(eval.getScorers()); - allScorersForError.addAll(remoteScorers); - for (var scorer : allScorersForError) { - runScoreForTaskException( - tracer, - evalSpan, - braintrustParent, - braintrustGeneration, - scorer, - e, - datasetCase, - scoresByName); - } - return; - } - // Send progress event for task completion - sendProgressEvent( - os, - evalSpan.getSpanContext().getSpanId(), - datasetCase.origin(), - eval.getName(), - taskResult.result()); - setTaskSpanAttributes( - taskSpan, - braintrustParent, - braintrustGeneration, - datasetCase, - taskResult); - } finally { - taskSpan.end(); - } - // setting eval span attributes here because we need the - // task output - setEvalSpanAttributes( - evalSpan, - braintrustParent, - braintrustGeneration, - datasetCase, - taskResult); - } - // run scorers - one score span per scorer - // Combine local scorers from RemoteEval with remote scorers - // from request - List> allScorers = - new ArrayList<>(eval.getScorers()); - allScorers.addAll(remoteScorers); - for (var scorer : allScorers) { - runScorer( + // Cases are evaluated concurrently on the eval pool. The cursor is drained by + // this (request) thread, which submits cases as fast as the pool accepts them and + // then waits for them; it never runs a case itself, so it cannot deadlock against + // the pool. How many cases run at once is the pool's business, not ours. + // + // streamFailure is set by the first case whose progress event can't be written, + // i.e. the client disconnected. Nobody is listening any more, so the run stops + // there rather than evaluating (and billing for) the remaining cases. runFailure + // similarly stops the run when a case hits an unrecoverable error, but remains + // reportable to the connected client as an SSE error. + var streamFailure = new AtomicReference(); + var runFailure = new AtomicReference(); + var caseContext = Context.current(); + // The request thread starts as one pending party so completion cannot win while + // cases are still being registered. The counter and single future avoid retaining + // one Future for every case in a potentially large remote dataset. + var pendingCases = new AtomicInteger(1); + var casesCompleted = new CompletableFuture(); + int submittedCount = 0; + Throwable drainError = null; + try (var cursor = + extractDataset( + request, + apiClient, + eval.getInputConverter(), + eval.getOutputConverter()) + .openCursor()) { + for (var next = cursor.next(); next.isPresent(); next = cursor.next()) { + if (streamFailure.get() != null || runFailure.get() != null) { + break; + } + var datasetCase = next.get(); + pendingCases.incrementAndGet(); + try { + evalExecutor.execute( + () -> { + try { + evalPlaygroundCase( + os, + eval, tracer, - evalSpan, + caseContext, + mergedParameters, braintrustParent, braintrustGeneration, - scorer, - taskResult, - scoresByName); + remoteScorers, + scoresByName, + streamFailure, + runFailure, + datasetCase); + } finally { + caseCompleted(pendingCases, casesCompleted); } - } catch (IOException e) { - throw new RuntimeException( - "Failed to send progress event", e); - } finally { - evalSpan.end(); - } - }); + }); + submittedCount++; + } catch (Throwable t) { + // execute() rejected the case, so undo the registration before + // surfacing the drain failure below. + caseCompleted(pendingCases, casesCompleted); + throw t; + } + } + } catch (Throwable t) { + // e.g. a failure fetching the next page of a dataset, or a submit rejected + // because the server stopped. Cases already submitted still finish below. + drainError = t; + } + caseCompleted(pendingCases, casesCompleted); + casesCompleted.join(); + var brokenStream = streamFailure.get(); + if (brokenStream != null) { + // There is nowhere to send a summary, a done or even an error event, so just + // log and let the finally below close the stream. + log.warn( + "Playground run aborted: client stream closed after {} case(s)", + submittedCount, + brokenStream); + return; + } + var caseFailure = runFailure.get(); + if (caseFailure instanceof Exception caseException) { + throw caseException; + } + if (caseFailure != null) { + throw new RuntimeException("Eval case failed", caseFailure); + } + if (drainError != null) { + throw new RuntimeException("Failed to evaluate dataset", drainError); + } // Aggregate scores Map scoreSummaries = new LinkedHashMap<>(); @@ -590,14 +652,160 @@ private void handleStreamingEval( } } + private static void caseCompleted( + AtomicInteger pendingCases, CompletableFuture casesCompleted) { + if (pendingCases.decrementAndGet() == 0) { + casesCompleted.complete(null); + } + } + + /** + * Evaluates one case of a playground run: emits the {@code eval}/{@code task}/{@code score} + * spans under the playground parent, streams a {@code progress} event, and accumulates scores + * into {@code scoresByName}. + * + *

Runs on the eval pool, one case per thread, so the OpenTelemetry scopes it opens stay + * thread-confined. Task and initial scorer errors are handled by their scorer fallbacks; an + * error that escapes those fallbacks is recorded in {@code runFailure} and aborts the run. + * Failing to write to the SSE stream records {@code streamFailure} instead because the client + * is gone and cannot receive an error event. + */ + private void evalPlaygroundCase( + OutputStream os, + RemoteEval eval, + Tracer tracer, + Context caseContext, + Parameters mergedParameters, + BraintrustUtils.Parent braintrustParent, + @Nullable String braintrustGeneration, + List> remoteScorers, + Map> scoresByName, + AtomicReference streamFailure, + AtomicReference runFailure, + DatasetCase datasetCase) { + if (streamFailure.get() != null || runFailure.get() != null) { + // The run has already failed; don't start work whose results will be discarded. + return; + } + try (var caseScope = caseContext.makeCurrent()) { + var evalSpan = + tracer.spanBuilder("eval") + .setNoParent() + .setSpanKind(SpanKind.CLIENT) + .setAttribute(PARENT, braintrustParent.toParentValue()) + .startSpan(); + Context evalContext = Context.current().with(evalSpan); + evalContext = + BraintrustContext.setParentInBaggage( + evalContext, braintrustParent.type(), braintrustParent.id()); + // Make the eval context (with span and baggage) current + try (var rootScope = evalContext.makeCurrent()) { + final TaskResult taskResult; + { // run task + var taskSpan = tracer.spanBuilder("task").startSpan(); + try (var unused = Context.current().with(taskSpan).makeCurrent()) { + var task = eval.getTask(); + try { + taskResult = task.apply(datasetCase, mergedParameters); + } catch (Exception e) { + taskSpan.setStatus(StatusCode.ERROR, e.getMessage()); + taskSpan.recordException(e); + taskSpan.end(); + evalSpan.setStatus(StatusCode.ERROR, e.getMessage()); + log.debug("Task threw exception for input: " + datasetCase.input(), e); + // Set eval span attributes so Braintrust can resolve the trace + setEvalSpanAttributesForError( + evalSpan, braintrustParent, braintrustGeneration, datasetCase); + // Send progress event even on error so the Playground can link + // to the trace + sendProgressEvent( + os, + evalSpan.getSpanContext().getSpanId(), + datasetCase.origin(), + eval.getName(), + null); + // run scoreForTaskException on each scorer + List> allScorersForError = + new ArrayList<>(eval.getScorers()); + allScorersForError.addAll(remoteScorers); + for (var scorer : allScorersForError) { + runScoreForTaskException( + tracer, + evalSpan, + braintrustParent, + braintrustGeneration, + scorer, + e, + datasetCase, + scoresByName); + } + return; + } + // Send progress event for task completion + sendProgressEvent( + os, + evalSpan.getSpanContext().getSpanId(), + datasetCase.origin(), + eval.getName(), + taskResult.result()); + setTaskSpanAttributes( + taskSpan, + braintrustParent, + braintrustGeneration, + datasetCase, + taskResult); + } finally { + taskSpan.end(); + } + // setting eval span attributes here because we need the task output + setEvalSpanAttributes( + evalSpan, + braintrustParent, + braintrustGeneration, + datasetCase, + taskResult); + } + // run scorers - one score span per scorer. Combine local scorers from + // RemoteEval with remote scorers from request + List> allScorers = new ArrayList<>(eval.getScorers()); + allScorers.addAll(remoteScorers); + for (var scorer : allScorers) { + runScorer( + tracer, + evalSpan, + braintrustParent, + braintrustGeneration, + scorer, + taskResult, + scoresByName); + } + } finally { + evalSpan.end(); + } + } catch (IOException e) { + // The SSE stream is broken — the client disconnected. Abort the run: the cases still + // to come have nowhere to report to. + streamFailure.compareAndSet(null, e); + } catch (Throwable t) { + if (runFailure.compareAndSet(null, t)) { + log.warn("Aborting playground run after case failed: {}", datasetCase.input(), t); + } + } + } + /** * Handles an experiment "snapshot" run: a remote eval triggered as an Experiment from the UI * (no playground parent). Rather than re-implementing experiment creation and span emission, it - * builds a first-class {@link Eval} and runs it synchronously — so snapshots get the exact same - * behavior as a normal {@code Eval.run()} (experiment creation with {@code ensure_new}, dataset - * id/version linkage for Braintrust-backed datasets, standard span shape). When the run - * completes it streams a single {@code summary} (with the created experiment's id/name/url) and - * a {@code done} event. + * builds a first-class {@link Eval} — so snapshots get the exact same behavior as a normal + * {@code Eval.run()} (experiment creation with {@code ensure_new}, dataset id/version linkage + * for Braintrust-backed datasets, standard span shape). + * + *

The eval is started with {@link Eval#start()} rather than run to completion: as soon as + * the experiment exists this streams a single {@code summary} (with the created experiment's + * id/name/url) and a {@code done} event, then returns while the cases keep evaluating on the + * server's executor. Errors creating the experiment surface to the caller; failures once the + * run is underway are logged by the eval. The started run is registered with {@link + * #trackSnapshot} so that stopping the server waits for it rather than truncating it. * *

Unlike playground runs, snapshots do not stream per-case {@code progress} events: the user * is handed the experiment link and views results in the experiment UI. @@ -619,10 +827,7 @@ private void handleExperimentSnapshot( List> allScorers = new ArrayList<>(eval.getScorers()); allScorers.addAll(remoteScorers); - // TODO: when async evals are supported, simply begin the eval and hand back the link to the - // user and finish eval in the background - - var evalResult = + var evalBuilder = Eval.builder() .name(experimentName) .config(braintrust.config()) @@ -643,9 +848,15 @@ private void handleExperimentSnapshot( : request.getParameters()) // Each snapshot run should produce a distinct experiment even if a prior // run used the same name (the backend dedupes the name on conflict). - .ensureNew(true) - .build() - .run(); + .ensureNew(true); + + // Run cases on the eval pool so they outlive this request, and hand the experiment link + // back as soon as the experiment exists rather than waiting for the run to finish. That + // pool decides how many cases run at once, so maxConcurrency is deliberately not set here. + evalBuilder.executor(evalExecutor); + var evalResult = evalBuilder.build().start(); + // The run outlives this request; register it so stop() can wait on it. + trackSnapshot(evalResult); // Snapshots don't stream per-scorer progress. The scores are recorded on the experiment // and visible via the experiment link. @@ -844,7 +1055,12 @@ private void recordScores( } Map scorerScores = new LinkedHashMap<>(); for (Score score : scores) { - scoresByName.computeIfAbsent(score.name(), k -> new ArrayList<>()).add(score.value()); + // Cases score concurrently. computeIfAbsent is atomic but the add() is not, so the + // list itself has to be synchronized. + scoresByName + .computeIfAbsent( + score.name(), k -> Collections.synchronizedList(new ArrayList<>())) + .add(score.value()); scorerScores.put(score.name(), score.value()); } setScoreSpanAttributes( @@ -853,7 +1069,10 @@ private void recordScores( private void sendSSEEvent(OutputStream os, String eventType, String data) throws IOException { String event = "event: " + eventType + "\n" + "data: " + data + "\n\n"; - synchronized (this) { + // Lock the stream, not the server: concurrent cases of one run must not interleave a + // partial event, but concurrent *requests* write to different streams and shouldn't + // serialize against each other (or against the synchronized start()/stop()). + synchronized (os) { os.write(event.getBytes(StandardCharsets.UTF_8)); } } @@ -1378,6 +1597,7 @@ public static class Builder { private @Nullable Consumer traceBuilderHook = null; private @Nullable Consumer configBuilderHook = null; + private @Nullable Integer maxConcurrency = null; public Devserver build() { if (evals.isEmpty()) { @@ -1394,6 +1614,24 @@ public Builder config(BraintrustConfig config) { return this; } + /** + * Sizes the server's eval thread pool, and so how many cases it evaluates at once across + * all in-flight playground runs and experiment snapshots. When unset, falls back to {@link + * BraintrustConfig#defaultMaxConcurrency()}. + * + *

Cases run concurrently, so a {@link RemoteEval}'s task and scorers must be thread-safe + * — which {@code RemoteEval} already requires. Pass {@code 1} to evaluate cases one at a + * time. + */ + public Builder maxConcurrency(int maxConcurrency) { + if (maxConcurrency < 1) { + throw new IllegalArgumentException( + "maxConcurrency must be at least 1, got " + maxConcurrency); + } + this.maxConcurrency = maxConcurrency; + return this; + } + public Builder registerEval(RemoteEval eval) { this.evals.add(eval); return this; diff --git a/braintrust-sdk/src/main/java/dev/braintrust/eval/Classifier.java b/braintrust-sdk/src/main/java/dev/braintrust/eval/Classifier.java index e37b3e3e..8d7ac301 100644 --- a/braintrust-sdk/src/main/java/dev/braintrust/eval/Classifier.java +++ b/braintrust-sdk/src/main/java/dev/braintrust/eval/Classifier.java @@ -2,6 +2,7 @@ import java.util.List; import java.util.function.Function; +import javax.annotation.concurrent.ThreadSafe; /** * A classifier categorizes and labels eval outputs, producing zero or more structured {@link @@ -11,9 +12,12 @@ * the span name and as the default grouping key for classifications whose own {@code name} is * blank). * + *

Implementations must be thread safe. + * * @param type of the input data * @param type of the output data */ +@ThreadSafe public interface Classifier { String INVALID_CLASSIFICATION_MESSAGE = "When returning structured classifier results, each classification must be a non-empty" diff --git a/braintrust-sdk/src/main/java/dev/braintrust/eval/Eval.java b/braintrust-sdk/src/main/java/dev/braintrust/eval/Eval.java index ed3613a2..2392188f 100644 --- a/braintrust-sdk/src/main/java/dev/braintrust/eval/Eval.java +++ b/braintrust-sdk/src/main/java/dev/braintrust/eval/Eval.java @@ -17,7 +17,12 @@ import io.opentelemetry.api.trace.SpanKind; import io.opentelemetry.api.trace.StatusCode; import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.context.Context; import java.util.*; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -47,6 +52,14 @@ public final class Eval { private final @Nonnull Map metadata; private final @Nonnull Parameters parameters; private final boolean ensureNew; + private final int maxConcurrency; + private final @Nullable Executor executor; + + /** + * True when no executor was supplied, so each run creates its own pool and is responsible for + * shutting it down when it finishes. + */ + private final boolean ownsExecutor; private Eval(Builder builder) { this.experimentName = builder.experimentName; @@ -65,11 +78,38 @@ private Eval(Builder builder) { this.metadata = Map.copyOf(builder.metadata); this.parameters = builder.buildParameters(); this.ensureNew = builder.ensureNew; + this.maxConcurrency = + builder.maxConcurrency != null + ? builder.maxConcurrency + : config.defaultMaxConcurrency(); + this.executor = builder.executor; + this.ownsExecutor = builder.executor == null; } - /** Runs the evaluation and returns results. */ + /** Runs the evaluation to completion and returns the results. */ public EvalResult run() { - try (var cursor = dataset.openCursor()) { + var result = start(); + result.awaitCompletion(); + return result; + } + + /** + * Creates the experiment and begins evaluating cases in the background, returning as soon as + * the experiment exists on Braintrust. + * + *

The returned {@link EvalResult} carries the experiment id, name and url immediately — so + * callers can surface the link right away — while its cases are still being evaluated. Use + * {@link EvalResult#isDone()} and {@link EvalResult#awaitCompletion()} to observe the run. + * + *

Errors raised while creating the experiment are thrown from this method. Errors raised + * once the run is underway are reported through {@link EvalResult#awaitCompletion()}. + */ + public EvalResult start() { + var state = new EvalResult.RunState(); + var cursor = dataset.openCursor(); + final EvalResult result; + final Executor caseExecutor; + try { Optional datasetVersion = Optional.empty(); Optional datasetId = Optional.empty(); if (dataset instanceof DatasetBrainstoreImpl) { @@ -94,8 +134,6 @@ public EvalResult run() { var experiment = new ExperimentsApi(client).postExperiment(createExperiment); - cursor.forEach(datasetCase -> evalOne(experiment.getId().toString(), datasetCase)); - // Use the experiment's actual name from the response: with ensure_new the backend may // dedupe a conflicting name (e.g. "foo" -> "foo-2f8ca776"), and the URL must point at // the real, created experiment. @@ -109,11 +147,186 @@ public EvalResult run() { project.getName()) .toASCIIString(), resolvedName); - return new EvalResult(experiment.getId().toString(), resolvedName, experimentUrl); + result = + new EvalResult( + experiment.getId().toString(), resolvedName, experimentUrl, state); + + caseExecutor = + ownsExecutor + ? BraintrustUtils.newExecutor(maxConcurrency, "braintrust-eval-worker-") + : Objects.requireNonNull(executor); + } catch (Throwable t) { + cursor.close(); + throw t; + } + + // Each case re-establishes the context that was current when the run was started. The eval + // span itself is created with setNoParent(), so this carries baggage rather than parentage. + var callerContext = Context.current(); + var experimentId = Objects.requireNonNull(result.getExperimentId()); + var coordinator = + new Thread( + () -> + evalAllCases( + cursor, caseExecutor, experimentId, callerContext, state), + "braintrust-eval-coordinator"); + coordinator.setDaemon(true); + coordinator.start(); + return result; + } + + /** + * Drains the dataset cursor on this (coordinator) thread, submitting each case to {@code + * caseExecutor} as fast as it will take them, then waits for every submitted case to finish. + * + *

The eval does not limit how many cases are in flight — the executor does. The default + * executor blocks the submit below once every worker is busy, so the loop advances at the pace + * the pool sets and the whole dataset is never materialized in memory. A caller-supplied + * executor with an unbounded queue accepts every case immediately and provides no such + * backpressure. + * + *

The drain is deliberately single-threaded: {@link Dataset.Cursor} is + * {@code @NotThreadSafe} and its {@code next()} may make network calls, so one thread pulls + * cases and fans them out. + * + *

Runs on the coordinator thread, never on a worker. Both submitting and waiting can block + * until a case finishes, so doing either from inside the pool that runs the cases would + * deadlock once the pool is saturated. + */ + private void evalAllCases( + Dataset.Cursor> cursor, + Executor caseExecutor, + String experimentId, + Context callerContext, + EvalResult.RunState state) { + // The coordinator starts as one pending party so completion cannot win while cases are + // still being registered. This counter and single future use constant memory for the run. + var pendingCases = new AtomicInteger(1); + var casesCompleted = new CompletableFuture(); + Throwable fatal = null; + try (cursor) { + for (var next = cursor.next(); next.isPresent(); next = cursor.next()) { + if (state.isAborting()) { + // A case hit a fatal error (see EvalAbortedException). Stop feeding the pool; + // cases already in flight finish or skip themselves. + break; + } + var datasetCase = next.get(); + pendingCases.incrementAndGet(); + try { + caseExecutor.execute( + () -> { + try { + evalOneTracked(experimentId, callerContext, datasetCase, state); + } finally { + caseCompleted(pendingCases, casesCompleted); + } + }); + } catch (Throwable t) { + // execute() rejected the case, so undo the registration before surfacing the + // drain failure below. + caseCompleted(pendingCases, casesCompleted); + throw t; + } + } + } catch (Throwable t) { + // e.g. a failure fetching the next page of a dataset, or a rejected submit. Cases + // already submitted still get to finish below. + fatal = t; + } + caseCompleted(pendingCases, casesCompleted); + casesCompleted.join(); + if (ownsExecutor && caseExecutor instanceof ExecutorService owned) { + owned.shutdown(); + } + if (fatal == null) { + // A case may have aborted the run (a scorer fallback that threw, an out-of-range + // score); that error is fatal to the run, not just to its case. + fatal = state.getFailure(); + } + if (fatal != null) { + // Callers that never await (e.g. a run started with start() and left to finish in the + // background) would otherwise never see this. + log.error( + "Eval aborted for experiment {} after {} case(s)", + experimentId, + state.getCasesSucceeded() + state.getCasesFailed(), + fatal); + } else { + log.debug( + "Eval complete for experiment {}: {} succeeded, {} failed", + experimentId, + state.getCasesSucceeded(), + state.getCasesFailed()); + } + state.complete(fatal); + } + + private static void caseCompleted( + AtomicInteger pendingCases, CompletableFuture casesCompleted) { + if (pendingCases.decrementAndGet() == 0) { + casesCompleted.complete(null); + } + } + + /** + * Evaluates one case and records the outcome on {@code state}. An ordinary throw is contained + * to this case and logged: one bad case must not abort the rest of the run. An {@link + * EvalAbortedException} is different — it signals a problem with the eval itself (see the + * places that throw it), so it aborts the whole run. + */ + private void evalOneTracked( + String experimentId, + Context callerContext, + DatasetCase datasetCase, + EvalResult.RunState state) { + if (state.isAborting()) { + // Another case already aborted the run; don't start work we're about to throw away. + return; + } + try (var unused = callerContext.makeCurrent()) { + if (evalOne(experimentId, datasetCase)) { + state.caseSucceeded(); + } else { + state.caseFailed(); + } + } catch (EvalAbortedException e) { + state.caseFailed(); + log.error("Aborting eval for experiment {}", experimentId, e.getCause()); + // Reported to the caller through EvalResult#awaitCompletion(), which rethrows the + // original error rather than this marker. + state.abort(e.getCause()); + } catch (Exception e) { + state.caseFailed(); + log.warn("Eval case failed for input: {}", datasetCase.input(), e); + } catch (Error error) { + // JVM-level failures are not isolated case failures. Continuing could make resource + // exhaustion or a broken runtime worse, so stop the run and surface the original Error. + state.caseFailed(); + log.error("Aborting eval after JVM error for input: {}", datasetCase.input(), error); + state.abort(error); + } + } + + /** + * Marks an error that must abort the entire run rather than just failing its case: the eval is + * misconfigured or its scorers are broken, so every remaining case would hit the same problem. + * Never escapes {@link #evalOneTracked}, which unwraps it onto the run's state. + */ + private static final class EvalAbortedException extends RuntimeException { + EvalAbortedException(Throwable cause) { + super(cause); } } - private void evalOne(String experimentId, DatasetCase datasetCase) { + /** + * Evaluates a single case. Runs entirely on one thread so that the OpenTelemetry scopes it + * opens stay thread-confined. + * + * @return {@code true} if the task completed or at least one scorer fallback recovered a task + * exception by producing scores; {@code false} otherwise + */ + private boolean evalOne(String experimentId, DatasetCase datasetCase) { var rootSpan = tracer.spanBuilder("eval") // TODO: allow names for eval cases .setNoParent() // each eval case is its own trace @@ -160,11 +373,14 @@ private void evalOne(String experimentId, DatasetCase datasetCase taskSpan.end(); rootSpan.setStatus(StatusCode.ERROR, e.getMessage()); log.debug("Task threw exception for input: " + datasetCase.input(), e); - // run scoreForTaskException on each scorer + // A task exception only fails the case when no scorer fallback recovers it. + boolean recovered = false; for (var scorer : scorers) { - runScoreForTaskException(experimentId, rootSpan, scorer, e, datasetCase); + recovered |= + runScoreForTaskException( + experimentId, rootSpan, scorer, e, datasetCase); } - return; + return recovered; } taskSpan.end(); } @@ -221,6 +437,7 @@ private void evalOne(String experimentId, DatasetCase datasetCase } finally { rootSpan.end(); } + return true; } /** @@ -251,7 +468,11 @@ private void runScorer( scoreSpan.recordException(e); log.debug("Scorer '{}' threw exception", scorer.getName(), e); // fall back to scoreForScorerException — if this throws, eval aborts - scores = scorer.scoreForScorerException(e, taskResult); + try { + scores = scorer.scoreForScorerException(e, taskResult); + } catch (Throwable fatal) { + throw new EvalAbortedException(fatal); + } } recordScores(scoreSpan, rootSpan, scorer, scores); } finally { @@ -262,8 +483,10 @@ private void runScorer( /** * Runs {@link Scorer#scoreForTaskException} when the task threw. If the fallback throws, the * eval aborts. + * + * @return whether the fallback produced scores and therefore recovered the case */ - private void runScoreForTaskException( + private boolean runScoreForTaskException( String experimentId, Span rootSpan, Scorer scorer, @@ -274,9 +497,15 @@ private void runScoreForTaskException( .setAttribute(PARENT, "experiment_id:" + experimentId) .startSpan(); try (var unused = BraintrustContext.ofExperiment(experimentId, scoreSpan).makeCurrent()) { - // if this throws, it propagates and the eval aborts - var scores = scorer.scoreForTaskException(taskException, datasetCase); + final List scores; + try { + scores = scorer.scoreForTaskException(taskException, datasetCase); + } catch (Throwable fatal) { + // if this throws, the eval aborts + throw new EvalAbortedException(fatal); + } recordScores(scoreSpan, rootSpan, scorer, scores); + return scores != null && !scores.isEmpty(); } finally { scoreSpan.end(); } @@ -369,9 +598,12 @@ private void recordScores( final Map scorerScores = new LinkedHashMap<>(); for (var score : scores) { if (score.value() < 0.0 || score.value() > 1.0) { - throw new RuntimeException( - "score must be between 0 and 1: %s : %s" - .formatted(scorer.getName(), score)); + // A scorer that returns an out-of-range score is broken, not unlucky: abort the + // run rather than let every remaining case hit the same bug. + throw new EvalAbortedException( + new RuntimeException( + "score must be between 0 and 1: %s : %s" + .formatted(scorer.getName(), score))); } scorerScores.put(score.name(), score.value()); } @@ -406,6 +638,8 @@ public static final class Builder { private @Nonnull List tags = List.of(); private @Nonnull Map metadata = Map.of(); private boolean ensureNew = false; + private @Nullable Integer maxConcurrency = null; + private @Nullable Executor executor; public Eval build() { if (config == null) { @@ -453,6 +687,33 @@ public Builder apiClient(BraintrustApiClient apiClient) { return apiClient(apiClient.openApiClient()); } + /** Sets the maximum amount of cases which can be evaluated simultaneously. */ + public Builder maxConcurrency(int maxConcurrency) { + if (maxConcurrency < 1) { + throw new IllegalArgumentException( + "maxConcurrency must be at least 1, got " + maxConcurrency); + } + this.maxConcurrency = maxConcurrency; + return this; + } + + /** + * Sets the executor that eval cases run on, replacing the pool the eval would otherwise + * create for itself (see {@link #maxConcurrency(int)}, which is then ignored). + * + *

This executor decides how many cases run at once, and it is also the eval's only + * backpressure: the eval submits cases as fast as the executor accepts them. An executor + * with an unbounded queue therefore accepts the entire dataset up front and holds every + * pending case in memory. Prefer one that blocks or otherwise bounds its queue. + * + *

An executor supplied here is never shut down by the SDK — the caller owns its + * lifecycle. + */ + public Builder executor(@Nonnull Executor executor) { + this.executor = Objects.requireNonNull(executor); + return this; + } + public Builder tracer(Tracer tracer) { this.tracer = tracer; return this; diff --git a/braintrust-sdk/src/main/java/dev/braintrust/eval/EvalResult.java b/braintrust-sdk/src/main/java/dev/braintrust/eval/EvalResult.java index 7b2b44b4..90a341fc 100644 --- a/braintrust-sdk/src/main/java/dev/braintrust/eval/EvalResult.java +++ b/braintrust-sdk/src/main/java/dev/braintrust/eval/EvalResult.java @@ -1,24 +1,267 @@ package dev.braintrust.eval; +import java.time.Duration; +import java.time.Instant; +import java.util.Optional; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import javax.annotation.Nonnull; import javax.annotation.Nullable; import lombok.Getter; -import lombok.SneakyThrows; -/** Results of all eval cases of an experiment. */ +/** + * Results of all eval cases of an experiment. + * + *

The experiment identifiers are available as soon as this object exists, but the run itself may + * still be in progress: {@link Eval#start()} hands back a result whose cases are still being + * evaluated, so that callers can surface the experiment link immediately. Use {@link #isDone()} and + * {@link #awaitCompletion()} to observe the run. A result returned by {@link Eval#run()} is always + * already complete. + */ public class EvalResult { @Getter private final @Nullable String experimentId; @Getter private final @Nullable String experimentName; @Getter private final String experimentUrl; + private final @Nonnull RunState state; - @SneakyThrows EvalResult( - @Nullable String experimentId, @Nullable String experimentName, String experimentUrl) { + @Nullable String experimentId, + @Nullable String experimentName, + String experimentUrl, + @Nonnull RunState state) { this.experimentId = experimentId; this.experimentName = experimentName; this.experimentUrl = experimentUrl; + this.state = state; + } + + /** true if the eval has completed execution */ + public boolean isDone() { + return state.isDone(); + } + + /** + * wait until the eval finishes running, or return right away if already done. + * + *

If the run aborted with an error, that error is rethrown here. + */ + public void awaitCompletion() { + state.await(); + } + + /** + * wait until the eval finishes running, or return right away if already done, giving up after + * {@code timeout}. + * + * @return true if the eval completed, false if the timeout elapsed while it was still running + */ + public boolean awaitCompletion(@Nonnull Duration timeout) { + return state.await(timeout); + } + + /** when the eval started running */ + public Instant getStartedAt() { + return state.getStartedAt(); + } + + /** + * how long the eval took to run, or empty while it is still running — the duration is not known + * until every case has finished + */ + public Optional getDuration() { + return state.getDuration(); + } + + /** number of cases that have finished evaluating so far, whether they succeeded or failed */ + public int getCasesExecuted() { + return state.getCasesExecuted(); + } + + /** num of eval case runs that finished wihout throwing */ + int getCasesSucceeded() { + return state.getCasesSucceeded(); + } + + /** num of throwables that escaped an eval case run */ + int getCasesFailed() { + return state.getCasesFailed(); } public String createReportString() { - return "Experiment complete. View results in braintrust: %s".formatted(experimentUrl); + if (!isDone()) { + return "Experiment is running (%d case(s) executed so far). View live results in braintrust: %s" + .formatted(getCasesExecuted(), experimentUrl); + } + var elapsed = getDuration().map(EvalResult::formatDuration).orElse("an unknown duration"); + var failed = getCasesFailed(); + if (failed > 0) { + return "Experiment complete in %s: %d case(s) executed, %d failed. View results in braintrust: %s" + .formatted(elapsed, getCasesExecuted(), failed, experimentUrl); + } + return "Experiment complete in %s: %d case(s) executed. View results in braintrust: %s" + .formatted(elapsed, getCasesExecuted(), experimentUrl); + } + + /** Renders a duration as "X minutes, Y seconds". Hours and beyond roll up into the minutes. */ + private static String formatDuration(Duration duration) { + var minutes = duration.toMinutes(); + var seconds = duration.toSecondsPart(); + return "%d %s, %d %s" + .formatted( + minutes, + minutes == 1 ? "minute" : "minutes", + seconds, + seconds == 1 ? "second" : "seconds"); + } + + /** + * Mutable execution state of a single eval run, shared between the {@link Eval} executing the + * run and the enclosing {@link EvalResult}, which exposes a read-only view of it. + * + *

An {@link Eval} may hand back its {@link EvalResult} before the run has finished (see + * {@link Eval#start()}), so this state is written by the eval's coordinator and worker threads + * while the caller reads it. All members are thread-safe. + */ + static final class RunState { + private final CountDownLatch completed = new CountDownLatch(1); + private final AtomicInteger casesSucceeded = new AtomicInteger(); + private final AtomicInteger casesFailed = new AtomicInteger(); + private final AtomicReference failure = new AtomicReference<>(); + + /** When the run began. Wall-clock, for reporting only. */ + private final Instant startedAt = Instant.now(); + + /** + * Monotonic counterpart to {@link #startedAt}, used to measure {@link #duration} — a wall + * clock can step backwards mid-run and yield a negative elapsed time. + */ + private final long startedAtNanos = System.nanoTime(); + + /** How long the run took, set once by {@link #complete}. Null while it is still running. */ + private final AtomicReference duration = new AtomicReference<>(); + + /** Returns a state that is already complete, for callers that never ran anything. */ + static RunState alreadyComplete() { + var state = new RunState(); + state.complete(null); + return state; + } + + /** + * Records an error fatal enough to abort the whole run (as opposed to failing a single + * case). The first error wins; the coordinator stops submitting cases, cases that have not + * started skip themselves, and the run completes with this error. + */ + void abort(Throwable error) { + failure.compareAndSet(null, error); + } + + /** true once {@link #abort} has been called and the run is winding down. */ + boolean isAborting() { + return failure.get() != null; + } + + boolean isDone() { + return completed.getCount() == 0; + } + + /** + * Blocks until the run completes. If the run failed with an error that aborted it, that + * error is rethrown here. + */ + void await() { + try { + completed.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException("interrupted while awaiting eval completion", e); + } + throwIfFailed(); + } + + /** + * Blocks until the run completes or the timeout elapses. + * + * @return true if the run completed, false if the timeout elapsed first + */ + boolean await(Duration timeout) { + final boolean done; + try { + done = completed.await(timeout.toMillis(), TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException("interrupted while awaiting eval completion", e); + } + if (done) { + throwIfFailed(); + } + return done; + } + + private void throwIfFailed() { + var t = failure.get(); + if (t == null) { + return; + } + if (t instanceof RuntimeException re) { + throw re; + } + if (t instanceof Error err) { + throw err; + } + throw new RuntimeException("eval failed", t); + } + + /** + * Marks the run complete. A non-null {@code error} means the run aborted before every case + * was evaluated (as opposed to individual cases failing, which is tracked by {@link + * #caseFailed()} and does not abort the run). + */ + void complete(@Nullable Throwable error) { + failure.compareAndSet(null, error); + // Set before counting down so that isDone() never observes a run without a duration. + duration.compareAndSet(null, Duration.ofNanos(System.nanoTime() - startedAtNanos)); + completed.countDown(); + } + + void caseSucceeded() { + casesSucceeded.incrementAndGet(); + } + + void caseFailed() { + casesFailed.incrementAndGet(); + } + + Instant getStartedAt() { + return startedAt; + } + + Optional getDuration() { + return Optional.ofNullable(duration.get()); + } + + /** + * Total cases that finished evaluating, successfully or not. Derived from the two counters + * rather than tracked separately so it can never drift from them; the sum is not read + * atomically, so a mid-run read may miss a case that is between the two increments. + */ + int getCasesExecuted() { + return casesSucceeded.get() + casesFailed.get(); + } + + int getCasesSucceeded() { + return casesSucceeded.get(); + } + + int getCasesFailed() { + return casesFailed.get(); + } + + @Nullable + Throwable getFailure() { + return failure.get(); + } } } diff --git a/braintrust-sdk/src/main/java/dev/braintrust/eval/Scorer.java b/braintrust-sdk/src/main/java/dev/braintrust/eval/Scorer.java index 300a5607..021c2748 100644 --- a/braintrust-sdk/src/main/java/dev/braintrust/eval/Scorer.java +++ b/braintrust-sdk/src/main/java/dev/braintrust/eval/Scorer.java @@ -7,14 +7,18 @@ import java.util.function.BiFunction; import java.util.function.Function; import javax.annotation.Nullable; +import javax.annotation.concurrent.ThreadSafe; /** * A scorer evaluates the result of a task against a dataset case, producing a score between 0 * (inclusive) and 1 (inclusive). * + *

Implementations must be thread safe. + * * @param type of the input data * @param type of the output data */ +@ThreadSafe public interface Scorer { String getName(); diff --git a/braintrust-sdk/src/main/java/dev/braintrust/eval/Task.java b/braintrust-sdk/src/main/java/dev/braintrust/eval/Task.java index 9072a9b1..4ceacf86 100644 --- a/braintrust-sdk/src/main/java/dev/braintrust/eval/Task.java +++ b/braintrust-sdk/src/main/java/dev/braintrust/eval/Task.java @@ -1,5 +1,7 @@ package dev.braintrust.eval; +import javax.annotation.concurrent.ThreadSafe; + /** * A task function that runs against a single dataset case and produces an output. * @@ -11,9 +13,12 @@ * single-arg {@link #apply(DatasetCase)} method instead — the two-arg version delegates to it by * default. * + *

Implementations must be thread safe. + * * @param type of the input data * @param type of the output data */ +@ThreadSafe public interface Task { /** * Executes this task against a single dataset case, with access to merged eval parameters. diff --git a/braintrust-sdk/src/main/java/dev/braintrust/eval/TracedClassifier.java b/braintrust-sdk/src/main/java/dev/braintrust/eval/TracedClassifier.java index db67d78b..b3a7aa2b 100644 --- a/braintrust-sdk/src/main/java/dev/braintrust/eval/TracedClassifier.java +++ b/braintrust-sdk/src/main/java/dev/braintrust/eval/TracedClassifier.java @@ -2,6 +2,7 @@ import dev.braintrust.trace.BrainstoreTrace; import java.util.List; +import javax.annotation.concurrent.ThreadSafe; /** * A classifier that receives access to the full distributed trace of the task that was evaluated. @@ -10,9 +11,12 @@ * invocations, or other spans produced during task execution — not just the final {@link * TaskResult}. * + *

Implementations must be thread safe. + * * @param type of the input data * @param type of the output data */ +@ThreadSafe public interface TracedClassifier extends Classifier { /** diff --git a/braintrust-sdk/src/main/java/dev/braintrust/eval/TracedScorer.java b/braintrust-sdk/src/main/java/dev/braintrust/eval/TracedScorer.java index 352a7130..9267a150 100644 --- a/braintrust-sdk/src/main/java/dev/braintrust/eval/TracedScorer.java +++ b/braintrust-sdk/src/main/java/dev/braintrust/eval/TracedScorer.java @@ -2,6 +2,7 @@ import dev.braintrust.trace.BrainstoreTrace; import java.util.List; +import javax.annotation.concurrent.ThreadSafe; /** * A scorer that receives access to the full distributed trace of the task that was evaluated. @@ -10,9 +11,12 @@ * invocations, or other spans produced during task execution — not just the final {@code * TaskResult}. * + *

Implementations must be thread safe. + * * @param type of the input data * @param type of the output data */ +@ThreadSafe public interface TracedScorer extends Scorer { /** diff --git a/braintrust-sdk/src/main/java/dev/braintrust/trace/BraintrustTracing.java b/braintrust-sdk/src/main/java/dev/braintrust/trace/BraintrustTracing.java index c3574038..99964e5b 100644 --- a/braintrust-sdk/src/main/java/dev/braintrust/trace/BraintrustTracing.java +++ b/braintrust-sdk/src/main/java/dev/braintrust/trace/BraintrustTracing.java @@ -124,9 +124,9 @@ static void enable( @Nonnull List additionalDelegates, @Nonnull SdkLoggerProviderBuilder loggerProviderBuilder, @Nonnull SdkMeterProviderBuilder meterProviderBuilder) { - final Duration exportInterval = Duration.ofSeconds(5); - final int maxQueueSize = 2048; - final int maxExportBatchSize = 512; + final Duration exportInterval = Duration.ofMillis(config.otelExportIntervalMillis()); + final int maxQueueSize = config.otelMaxQueueSize(); + final int maxExportBatchSize = config.otelMaxExportBatchSize(); log.info(sdkInfoLogMessage()); // Create resource first so BraintrustSpanProcessor can access service.name @@ -174,14 +174,16 @@ static void enable( log.debug("Shutting down. Force-Flushing all otel data."); var result = CompletableResultCode.ofAll( - // run all flushes in parallel. Should (rarely) - // block for approx 10 seconds max Stream.of(spanProcessor.shutdown(), logProcessor.shutdown()) - .map(operation -> operation.join(10, TimeUnit.SECONDS)) + .map(operation -> operation.join(60, TimeUnit.SECONDS)) .toList()); - log.debug( - "otel shutdown complete. Flush done: %s, Flush successful: %s" - .formatted(result.isDone(), result.isSuccess())); + if (!result.isDone() || !result.isSuccess()) { + log.warn( + "failed to successfully shutdown otel. Flush done: %s, Flush successful: %s" + .formatted(result.isDone(), result.isSuccess())); + } else { + log.debug("otel shutdown successful"); + } }); } diff --git a/braintrust-sdk/src/test/java/dev/braintrust/BraintrustUtilsTest.java b/braintrust-sdk/src/test/java/dev/braintrust/BraintrustUtilsTest.java index 99359c51..b0463b71 100644 --- a/braintrust-sdk/src/test/java/dev/braintrust/BraintrustUtilsTest.java +++ b/braintrust-sdk/src/test/java/dev/braintrust/BraintrustUtilsTest.java @@ -2,8 +2,12 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.net.URI; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import org.junit.jupiter.api.Test; public class BraintrustUtilsTest { @@ -38,4 +42,71 @@ void testParseParent() { () -> BraintrustUtils.parseParent(""), "Should throw on empty string"); } + + /** + * The platform-thread fallback used on Java < 21. The virtual-thread path is covered by + * jvm-compat-tests/java21, since this suite runs on Java 17. + */ + @Test + void testNewExecutorUsesDaemonThreads() throws Exception { + var executor = BraintrustUtils.newExecutor(2, "braintrust-utils-test-"); + try { + var thread = executor.submit(Thread::currentThread).get(10, TimeUnit.SECONDS); + assertTrue(thread.isDaemon(), "executor threads must be daemons"); + assertTrue( + thread.getName().startsWith("braintrust-utils-test-"), + "unexpected thread name: " + thread.getName()); + } finally { + executor.shutdown(); + } + } + + @Test + void testNewExecutorCapsConcurrency() throws Exception { + int maxSize = 3; + var executor = BraintrustUtils.newExecutor(maxSize, "braintrust-utils-bounded-"); + var inFlight = new AtomicInteger(); + var peak = new AtomicInteger(); + var release = new CountDownLatch(1); + var done = new CountDownLatch(maxSize); + try { + for (int i = 0; i < maxSize; i++) { + executor.execute( + () -> { + peak.accumulateAndGet(inFlight.incrementAndGet(), Math::max); + try { + release.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } finally { + inFlight.decrementAndGet(); + done.countDown(); + } + }); + } + + var submitted = new CountDownLatch(1); + var submitter = + new Thread( + () -> { + executor.execute(() -> {}); + submitted.countDown(); + }); + submitter.setDaemon(true); + submitter.start(); + + assertTrue( + !submitted.await(500, TimeUnit.MILLISECONDS), + "submitting past maxSize should have backpressured the caller"); + + release.countDown(); + assertTrue(done.await(10, TimeUnit.SECONDS), "tasks did not finish"); + assertTrue(submitted.await(10, TimeUnit.SECONDS), "blocked submit never went through"); + submitter.join(); + + assertEquals(maxSize, peak.get(), "concurrency exceeded maxSize"); + } finally { + executor.shutdown(); + } + } } diff --git a/braintrust-sdk/src/test/java/dev/braintrust/config/BraintrustConfigTest.java b/braintrust-sdk/src/test/java/dev/braintrust/config/BraintrustConfigTest.java index d395ef5c..bf7a9b8a 100644 --- a/braintrust-sdk/src/test/java/dev/braintrust/config/BraintrustConfigTest.java +++ b/braintrust-sdk/src/test/java/dev/braintrust/config/BraintrustConfigTest.java @@ -74,6 +74,20 @@ public void testBuilderHasMethodForEveryField() { } } + @Test + void rejectsOtelExportBatchSizeLargerThanQueue() { + var thrown = + assertThrows( + IllegalArgumentException.class, + () -> + BraintrustConfig.builder() + .otelMaxQueueSize(512) + .otelMaxExportBatchSize(1024) + .build()); + + assertTrue(thrown.getMessage().contains("must not exceed")); + } + @Test void testDefaultSslContextWhenNotProvided() throws Exception { // Create config without custom SSL context diff --git a/braintrust-sdk/src/test/java/dev/braintrust/devserver/DevserverConcurrencyTest.java b/braintrust-sdk/src/test/java/dev/braintrust/devserver/DevserverConcurrencyTest.java new file mode 100644 index 00000000..7d46cab2 --- /dev/null +++ b/braintrust-sdk/src/test/java/dev/braintrust/devserver/DevserverConcurrencyTest.java @@ -0,0 +1,422 @@ +package dev.braintrust.devserver; + +import static org.junit.jupiter.api.Assertions.*; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import dev.braintrust.BraintrustUtils; +import dev.braintrust.TestHarness; +import dev.braintrust.eval.DatasetCase; +import dev.braintrust.eval.Score; +import dev.braintrust.eval.Scorer; +import dev.braintrust.eval.TaskResult; +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.net.HttpURLConnection; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.*; + +/** Covers concurrent case execution on the devserver's playground path. */ +@Slf4j +class DevserverConcurrencyTest { + private static final int TEST_PORT = 8302; + private static final String TEST_URL = "http://localhost:" + TEST_PORT; + private static final ObjectMapper JSON_MAPPER = new ObjectMapper(); + private static final int MAX_CONCURRENCY = 4; + + private static final String BLOCKING_EVAL = "concurrency-blocking-eval"; + private static final String SCORES_EVAL = "concurrency-scores-eval"; + private static final String SLOW_EVAL = "concurrency-slow-eval"; + private static final String BROKEN_SCORER_EVAL = "concurrency-broken-scorer-eval"; + private static final String BROKEN_TASK_FALLBACK_EVAL = "concurrency-broken-task-fallback-eval"; + + private static final BraintrustUtils.Parent PLAYGROUND_PARENT = + new BraintrustUtils.Parent("playground_id", "ceea7422-3507-4d1c-a5f7-7acf41d9fac2"); + + private static Devserver server; + private static Thread serverThread; + private static TestHarness testHarness; + + // Observed concurrency of the blocking eval's task, reset per test. + private static final AtomicInteger inFlight = new AtomicInteger(); + private static final AtomicInteger peak = new AtomicInteger(); + // When set, every task counts down and then waits for all of them to arrive. + private static volatile CountDownLatch gate; + // Cases of SLOW_EVAL that started, reset per test. + private static final AtomicInteger slowTasksRun = new AtomicInteger(); + + @BeforeAll + static void setUp() throws Exception { + testHarness = TestHarness.setup(); + + var blockingEval = + RemoteEval.builder() + .name(BLOCKING_EVAL) + .taskFunction( + input -> { + peak.accumulateAndGet(inFlight.incrementAndGet(), Math::max); + try { + var g = gate; + if (g != null) { + g.countDown(); + assertTrue( + g.await(10, TimeUnit.SECONDS), + "tasks did not run concurrently"); + } else { + Thread.sleep(25); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } finally { + inFlight.decrementAndGet(); + } + return "ok"; + }) + .scorer(Scorer.of("static_scorer", (expected, result) -> 1.0)) + .build(); + + // Each case's input *is* its score, so a lost score in the shared aggregate changes the + // reported average. With identical scores per case the race would be invisible. + var scoresEval = + RemoteEval.builder() + .name(SCORES_EVAL) + .taskFunction(input -> input) + .scorer( + Scorer.of( + "varying_scorer", + (expected, result) -> Double.parseDouble(result))) + .build(); + + var brokenScorerEval = + RemoteEval.builder() + .name(BROKEN_SCORER_EVAL) + .taskFunction(input -> input) + .scorer( + new Scorer<>() { + @Override + public String getName() { + return "broken_scorer"; + } + + @Override + public List score(TaskResult result) { + throw new IllegalStateException("scorer failed"); + } + + @Override + public List scoreForScorerException( + Exception scorerException, + TaskResult result) { + throw new IllegalStateException("fallback failed"); + } + }) + .build(); + + var brokenTaskFallbackEval = + RemoteEval.builder() + .name(BROKEN_TASK_FALLBACK_EVAL) + .taskFunction( + input -> { + throw new IllegalStateException("task failed"); + }) + .scorer( + new Scorer<>() { + @Override + public String getName() { + return "broken_task_fallback"; + } + + @Override + public List score(TaskResult result) { + return List.of(); + } + + @Override + public List scoreForTaskException( + Exception taskException, + DatasetCase datasetCase) { + throw new IllegalStateException("task fallback failed"); + } + }) + .build(); + + // Slow enough that a client can disconnect while cases are still queued up behind it. + var slowEval = + RemoteEval.builder() + .name(SLOW_EVAL) + .taskFunction( + input -> { + slowTasksRun.incrementAndGet(); + try { + Thread.sleep(100); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + return "ok"; + }) + .scorer(Scorer.of("static_scorer", (expected, result) -> 1.0)) + .build(); + + server = + Devserver.builder() + .config(testHarness.braintrust().config()) + .registerEval(blockingEval) + .registerEval(scoresEval) + .registerEval(slowEval) + .registerEval(brokenScorerEval) + .registerEval(brokenTaskFallbackEval) + .maxConcurrency(MAX_CONCURRENCY) + .host("localhost") + .port(TEST_PORT) + .build(); + + serverThread = + new Thread( + () -> { + try { + server.start(); + } catch (Exception e) { + log.error("unable to start dev server", e); + } + }); + serverThread.start(); + Thread.sleep(1000); + } + + @AfterAll + @SneakyThrows + static void tearDown() { + if (server != null) { + server.stop(); + } + if (serverThread != null) { + serverThread.join(30_000); + if (serverThread.isAlive()) { + serverThread.interrupt(); + } + } + } + + @BeforeEach + void resetCounters() { + inFlight.set(0); + peak.set(0); + slowTasksRun.set(0); + gate = null; + } + + @Test + void casesRunConcurrently() throws Exception { + // Every task blocks until all MAX_CONCURRENCY of them have arrived, so the run can only + // complete if they genuinely execute at the same time. + gate = new CountDownLatch(MAX_CONCURRENCY); + + var events = runPlaygroundEval(BLOCKING_EVAL, inputs(MAX_CONCURRENCY)); + + assertEquals( + MAX_CONCURRENCY, + events.stream().filter(e -> "progress".equals(e.get("event"))).count(), + "every case should report progress"); + assertEquals(MAX_CONCURRENCY, peak.get(), "expected all cases in flight at once"); + } + + @Test + void maxConcurrencyIsRespected() throws Exception { + int caseCount = MAX_CONCURRENCY * 3; + var events = runPlaygroundEval(BLOCKING_EVAL, inputs(caseCount)); + + assertEquals( + caseCount, events.stream().filter(e -> "progress".equals(e.get("event"))).count()); + assertTrue(peak.get() > 1, "expected concurrent execution, peak was " + peak.get()); + assertTrue( + peak.get() <= MAX_CONCURRENCY, "exceeded maxConcurrency, peak was " + peak.get()); + } + + @Test + void scoreAggregationIsCorrectUnderConcurrency() throws Exception { + // Every case scores differently, so the summary average is only right if each concurrent + // case's score made it into the shared aggregate. The case count is high to make the + // narrow computeIfAbsent-then-add window land reliably. + int caseCount = 300; + var caseInputs = new ArrayList(); + double expectedSum = 0.0; + for (int i = 1; i <= caseCount; i++) { + double value = i / (double) caseCount; + expectedSum += value; + caseInputs.add(String.valueOf(value)); + } + double expectedAverage = expectedSum / caseCount; + + var events = runPlaygroundEval(SCORES_EVAL, caseInputs); + + var summary = + events.stream() + .filter(e -> "summary".equals(e.get("event"))) + .findFirst() + .orElseThrow(() -> new AssertionError("no summary event")); + JsonNode scores = JSON_MAPPER.readTree(summary.get("data")).get("scores"); + assertNotNull(scores.get("varying_scorer"), "summary should carry the scorer's average"); + assertEquals( + expectedAverage, + scores.get("varying_scorer").get("score").asDouble(), + 1e-9, + "every case's score must be counted exactly once"); + } + + @Test + void scorerFallbackFailureSendsErrorInsteadOfSuccessfulCompletion() throws Exception { + var events = runPlaygroundEval(BROKEN_SCORER_EVAL, inputs(4)); + + assertTrue( + events.stream().anyMatch(e -> "error".equals(e.get("event"))), + "a fatal scorer failure should be reported to the client"); + assertFalse( + events.stream() + .anyMatch( + e -> + "summary".equals(e.get("event")) + || "done".equals(e.get("event"))), + "a failed run must not report successful completion"); + } + + @Test + void taskFallbackFailureSendsErrorInsteadOfSuccessfulCompletion() throws Exception { + var events = runPlaygroundEval(BROKEN_TASK_FALLBACK_EVAL, inputs(4)); + + assertTrue( + events.stream().anyMatch(e -> "error".equals(e.get("event"))), + "a fatal task fallback failure should be reported to the client"); + assertFalse( + events.stream() + .anyMatch( + e -> + "summary".equals(e.get("event")) + || "done".equals(e.get("event"))), + "a failed run must not report successful completion"); + } + + @Test + void clientDisconnectAbortsTheRun() throws Exception { + // The playground streams per-case progress over SSE. Once the client is gone there is + // nobody to stream to, so the remaining cases must not be evaluated. + int caseCount = 200; + var conn = postPlaygroundEval(SLOW_EVAL, inputs(caseCount)); + assertEquals(200, conn.getResponseCode()); + + // Read far enough to know the run is under way, then drop the connection. + try (var reader = + new BufferedReader( + new InputStreamReader(conn.getInputStream(), StandardCharsets.UTF_8))) { + String line; + while ((line = reader.readLine()) != null) { + if (line.startsWith("event: progress")) { + break; + } + } + assertNotNull(line, "expected at least one progress event before disconnecting"); + } + conn.disconnect(); + + // Wait for the server to stop starting cases: the count holding still means the run ended. + int settled = slowTasksRun.get(); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(30); + while (System.nanoTime() < deadline) { + Thread.sleep(500); + int current = slowTasksRun.get(); + if (current == settled) { + break; + } + settled = current; + } + assertTrue( + settled < caseCount, + "every case ran despite the client disconnecting: " + settled + "/" + caseCount); + } + + private static List inputs(int count) { + var list = new ArrayList(); + for (int i = 0; i < count; i++) { + list.add("case-" + i); + } + return list; + } + + /** POSTs a streaming playground eval with inline cases and returns the parsed SSE events. */ + private List> runPlaygroundEval(String evalName, List caseInputs) + throws Exception { + var conn = postPlaygroundEval(evalName, caseInputs); + assertEquals(200, conn.getResponseCode()); + return readSSEEvents(conn); + } + + /** POSTs a streaming playground eval with inline cases, leaving the response unread. */ + private HttpURLConnection postPlaygroundEval(String evalName, List caseInputs) + throws Exception { + var evalRequest = new EvalRequest(); + evalRequest.setName(evalName); + evalRequest.setStream(true); + evalRequest.setParent( + Map.of( + "object_type", PLAYGROUND_PARENT.type(), + "object_id", PLAYGROUND_PARENT.id())); + + var dataSpec = new EvalRequest.DataSpec(); + var cases = new ArrayList(); + for (String input : caseInputs) { + var c = new EvalRequest.EvalCaseData(); + c.setInput(input); + c.setExpected(input); + cases.add(c); + } + dataSpec.setData(cases); + evalRequest.setData(dataSpec); + + String requestBody = JSON_MAPPER.writeValueAsString(evalRequest); + + HttpURLConnection conn = + (HttpURLConnection) new URI(TEST_URL + "/eval").toURL().openConnection(); + conn.setRequestMethod("POST"); + conn.setRequestProperty("Content-Type", "application/json"); + conn.setRequestProperty("x-bt-auth-token", testHarness.braintrustApiKey()); + conn.setRequestProperty("x-bt-project-id", TestHarness.defaultProjectId()); + conn.setRequestProperty("x-bt-org-name", TestHarness.defaultOrgName()); + conn.setDoOutput(true); + conn.getOutputStream().write(requestBody.getBytes(StandardCharsets.UTF_8)); + conn.getOutputStream().flush(); + return conn; + } + + private List> readSSEEvents(HttpURLConnection conn) throws Exception { + var events = new ArrayList>(); + try (BufferedReader reader = + new BufferedReader( + new InputStreamReader(conn.getInputStream(), StandardCharsets.UTF_8))) { + String line; + String currentEvent = null; + var currentData = new StringBuilder(); + while ((line = reader.readLine()) != null) { + if (line.startsWith("event: ")) { + currentEvent = line.substring(7); + } else if (line.startsWith("data: ")) { + currentData.append(line.substring(6)); + } else if (line.isEmpty() && currentEvent != null) { + events.add(Map.of("event", currentEvent, "data", currentData.toString())); + currentEvent = null; + currentData = new StringBuilder(); + } + } + } + return events; + } +} diff --git a/braintrust-sdk/src/test/java/dev/braintrust/devserver/DevserverTest.java b/braintrust-sdk/src/test/java/dev/braintrust/devserver/DevserverTest.java index f3ad627c..6d8159b6 100644 --- a/braintrust-sdk/src/test/java/dev/braintrust/devserver/DevserverTest.java +++ b/braintrust-sdk/src/test/java/dev/braintrust/devserver/DevserverTest.java @@ -23,6 +23,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.concurrent.TimeUnit; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.*; @@ -759,19 +760,29 @@ void testExperimentEval() throws Exception { String experimentId = summaryData.get("experimentId").asText(); // Spans should be parented to experiment_id: using the standard Eval span shape. - List allSpans = testHarness.awaitExportedSpans(); + // The snapshot response is sent as soon as the experiment exists — cases keep evaluating + // on the server's executor after that — so poll until this experiment's spans arrive + // rather than assuming the run finished when the response did. String expectedParent = "experiment_id:" + experimentId; - var evalSpans = - allSpans.stream() - .filter(s -> s.getName().equals("eval")) - .filter( - s -> - expectedParent.equals( - s.getAttributes() - .get( - AttributeKey.stringKey( - "braintrust.parent")))) - .toList(); + List evalSpans = List.of(); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(30); + while (true) { + evalSpans = + testHarness.awaitExportedSpans().stream() + .filter(s -> s.getName().equals("eval")) + .filter( + s -> + expectedParent.equals( + s.getAttributes() + .get( + AttributeKey.stringKey( + "braintrust.parent")))) + .toList(); + if (evalSpans.size() >= 2 || System.nanoTime() > deadline) { + break; + } + Thread.sleep(100); + } assertEquals(2, evalSpans.size(), "Should have 2 eval spans parented to the experiment"); for (SpanData evalSpan : evalSpans) { diff --git a/braintrust-sdk/src/test/java/dev/braintrust/eval/EvalConcurrencyTest.java b/braintrust-sdk/src/test/java/dev/braintrust/eval/EvalConcurrencyTest.java new file mode 100644 index 00000000..782896bc --- /dev/null +++ b/braintrust-sdk/src/test/java/dev/braintrust/eval/EvalConcurrencyTest.java @@ -0,0 +1,501 @@ +package dev.braintrust.eval; + +import static org.junit.jupiter.api.Assertions.*; + +import dev.braintrust.TestHarness; +import java.time.Duration; +import java.time.Instant; +import java.util.Optional; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import lombok.SneakyThrows; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** Covers concurrent case execution and the run-state exposed on {@link EvalResult}. */ +public class EvalConcurrencyTest { + private TestHarness testHarness; + + @BeforeEach + void beforeEach() { + testHarness = TestHarness.setup(); + } + + private DatasetCase[] cases(int n) { + @SuppressWarnings("unchecked") + DatasetCase[] cases = new DatasetCase[n]; + for (int i = 0; i < n; i++) { + cases[i] = DatasetCase.of("input-" + i, "fruit"); + } + return cases; + } + + @Test + @SneakyThrows + public void casesRunConcurrently() { + int caseCount = 10; + // Every task blocks until all of them have arrived. This can only complete if the tasks + // genuinely run at the same time. + var allArrived = new CountDownLatch(caseCount); + var peak = new AtomicInteger(); + var inFlight = new AtomicInteger(); + + var eval = + testHarness + .braintrust() + .evalBuilder() + .name("concurrency-test") + .cases(cases(caseCount)) + .maxConcurrency(caseCount) + .taskFunction( + input -> { + peak.accumulateAndGet(inFlight.incrementAndGet(), Math::max); + allArrived.countDown(); + try { + assertTrue( + allArrived.await(10, TimeUnit.SECONDS), + "tasks did not run concurrently"); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } finally { + inFlight.decrementAndGet(); + } + return "fruit"; + }) + .scorers(Scorer.of("s", r -> "fruit".equals(r) ? 1.0 : 0.0)) + .build(); + + var result = eval.run(); + assertEquals(caseCount, peak.get(), "expected all cases in flight at once"); + assertTrue(result.isDone()); + assertEquals(caseCount, result.getCasesSucceeded()); + assertEquals(0, result.getCasesFailed()); + } + + @Test + @SneakyThrows + public void maxConcurrencyIsRespected() { + int caseCount = 12; + int limit = 3; + var peak = new AtomicInteger(); + var inFlight = new AtomicInteger(); + + var eval = + testHarness + .braintrust() + .evalBuilder() + .name("max-concurrency-test") + .cases(cases(caseCount)) + .maxConcurrency(limit) + .taskFunction( + input -> { + peak.accumulateAndGet(inFlight.incrementAndGet(), Math::max); + try { + Thread.sleep(25); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + inFlight.decrementAndGet(); + return "fruit"; + }) + .scorers(Scorer.of("s", r -> 1.0)) + .build(); + + var result = eval.run(); + assertTrue(peak.get() > 1, "expected concurrent execution, peak was " + peak.get()); + assertTrue(peak.get() <= limit, "exceeded maxConcurrency, peak was " + peak.get()); + assertEquals(caseCount, result.getCasesSucceeded()); + } + + @Test + @SneakyThrows + public void serialWhenMaxConcurrencyIsOne() { + var peak = new AtomicInteger(); + var inFlight = new AtomicInteger(); + var eval = + testHarness + .braintrust() + .evalBuilder() + .name("serial-test") + .cases(cases(6)) + .maxConcurrency(1) + .taskFunction( + input -> { + peak.accumulateAndGet(inFlight.incrementAndGet(), Math::max); + try { + Thread.sleep(10); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + inFlight.decrementAndGet(); + return "fruit"; + }) + .scorers(Scorer.of("s", r -> 1.0)) + .build(); + eval.run(); + assertEquals(1, peak.get(), "maxConcurrency(1) must evaluate cases one at a time"); + } + + @Test + @SneakyThrows + public void startReturnsBeforeCompletionAndAwaitBlocks() { + var release = new CountDownLatch(1); + var eval = + testHarness + .braintrust() + .evalBuilder() + .name("start-test") + .cases(cases(4)) + .maxConcurrency(4) + .taskFunction( + input -> { + try { + assertTrue(release.await(10, TimeUnit.SECONDS)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + return "fruit"; + }) + .scorers(Scorer.of("s", r -> 1.0)) + .build(); + + var result = eval.start(); + // The experiment link is available immediately, while cases are still running. + assertNotNull(result.getExperimentUrl()); + assertFalse(result.isDone()); + assertFalse( + result.awaitCompletion(Duration.ofMillis(100)), + "awaitCompletion should time out while cases are still running"); + assertTrue(result.createReportString().contains("running")); + // The start timestamp is known immediately; the duration is not known until the run ends. + assertFalse(result.getStartedAt().isAfter(Instant.now())); + assertTrue( + result.getDuration().isEmpty(), "duration must not be set while cases are running"); + + release.countDown(); + result.awaitCompletion(); + assertTrue(result.isDone()); + assertEquals(4, result.getCasesSucceeded()); + assertTrue(result.createReportString().contains("complete")); + + var duration = result.getDuration().orElseThrow(); + assertFalse(duration.isNegative()); + assertTrue( + duration.compareTo(Duration.ofMillis(100)) >= 0, + "the run was gated for at least 100ms, got " + duration); + var report = result.createReportString(); + assertTrue( + report.matches(".*complete in \\d+ minutes?, \\d+ seconds?:.*"), + "report should carry a \"X minutes, Y seconds\" duration, got: " + report); + assertEquals(4, result.getCasesExecuted()); + assertTrue( + report.contains("4 case(s) executed"), + "report should carry the executed case count, got: " + report); + } + + @Test + @SneakyThrows + public void failingTaskIsContainedButJvmErrorAbortsRun() { + var tasksRun = new AtomicInteger(); + var eval = + testHarness + .braintrust() + .evalBuilder() + .name("failing-task-test") + .cases(cases(6)) + .maxConcurrency(1) + .taskFunction( + input -> { + tasksRun.incrementAndGet(); + if ("input-1".equals(input)) { + throw new RuntimeException("case failure"); + } + if ("input-3".equals(input)) { + throw new LinkageError("jvm failure"); + } + return "fruit"; + }) + .scorers(Scorer.of("s", r -> 1.0)) + .build(); + + var thrown = assertThrows(LinkageError.class, eval::run); + assertEquals("jvm failure", thrown.getMessage()); + assertTrue( + tasksRun.get() >= 4, + "the case after the ordinary failure should still run before the JVM error"); + assertTrue( + tasksRun.get() < 6, + "the run should stop after a JVM error, but every case ran: " + tasksRun.get()); + } + + @Test + @SneakyThrows + public void outOfRangeScoreAbortsTheRun() { + // A scorer that scores out of range is broken rather than unlucky, so the run stops + // instead of grinding through every remaining case. + var tasksRun = new AtomicInteger(); + var eval = + testHarness + .braintrust() + .evalBuilder() + .name("bad-score-test") + .cases(cases(5)) + .maxConcurrency(1) + .taskFunction( + input -> { + tasksRun.incrementAndGet(); + return input; + }) + .scorers(Scorer.of("s", r -> "input-1".equals(r.result()) ? 42.0 : 1.0)) + .build(); + + var thrown = assertThrows(RuntimeException.class, eval::run); + assertTrue( + thrown.getMessage().contains("score must be between 0 and 1"), + "the scorer's error should surface to the caller, got: " + thrown.getMessage()); + assertTrue( + tasksRun.get() < 5, + "the run should stop early, but every case ran: " + tasksRun.get()); + } + + @Test + @SneakyThrows + public void throwingScorerFallbackAbortsTheRun() { + // The scorer throws and its own fallback throws too: nothing can score this eval. + Scorer brokenScorer = + new Scorer<>() { + @Override + public String getName() { + return "broken"; + } + + @Override + public java.util.List score(TaskResult taskResult) { + throw new IllegalStateException("scorer boom"); + } + + @Override + public java.util.List scoreForScorerException( + Exception scorerException, TaskResult taskResult) { + throw new IllegalStateException("fallback boom", scorerException); + } + }; + var eval = + testHarness + .braintrust() + .evalBuilder() + // Reuses the recorded experiment name from the test above: cassettes are + // keyed on it, and this test cares about the scorer, not the experiment. + .name("bad-score-test") + .cases(cases(5)) + .maxConcurrency(1) + .taskFunction(input -> input) + .scorers(brokenScorer) + .build(); + + var thrown = assertThrows(IllegalStateException.class, eval::run); + assertEquals("fallback boom", thrown.getMessage()); + } + + @Test + @SneakyThrows + public void callerSuppliedExecutorIsNotShutDown() { + ExecutorService executor = Executors.newFixedThreadPool(4); + try { + var eval = + testHarness + .braintrust() + .evalBuilder() + .name("byo-executor-test") + .cases(cases(8)) + .executor(executor) + .taskFunction(input -> "fruit") + .scorers(Scorer.of("s", r -> 1.0)) + .build(); + var result = eval.run(); + assertEquals(8, result.getCasesSucceeded()); + assertFalse(executor.isShutdown(), "SDK must not shut down a caller-supplied executor"); + } finally { + executor.shutdownNow(); + } + } + + @Test + @SneakyThrows + public void fallsBackToConfigDefaultMaxConcurrency() { + // No maxConcurrency on the builder: the bound must come from the config. + var harness = TestHarness.setup(cfg -> cfg.defaultMaxConcurrency(2)); + var peak = new AtomicInteger(); + var inFlight = new AtomicInteger(); + + var eval = + harness.braintrust() + .evalBuilder() + .name("config-default-concurrency-test") + .cases(cases(8)) + .taskFunction( + input -> { + peak.accumulateAndGet(inFlight.incrementAndGet(), Math::max); + try { + Thread.sleep(25); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + inFlight.decrementAndGet(); + return "fruit"; + }) + .scorers(Scorer.of("s", r -> 1.0)) + .build(); + + var result = eval.run(); + assertEquals(8, result.getCasesSucceeded()); + assertTrue(peak.get() > 1, "expected concurrent execution, peak was " + peak.get()); + assertTrue( + peak.get() <= 2, + "config default of 2 should bound concurrency, peak was " + peak.get()); + } + + @Test + @SneakyThrows + public void suppliedExecutorGovernsConcurrency() { + // The eval no longer bounds in-flight cases itself, so a supplied executor is the only + // thing deciding how much runs at once — maxConcurrency must not sneak back in as a limit. + var executor = Executors.newFixedThreadPool(2); + var peak = new AtomicInteger(); + var inFlight = new AtomicInteger(); + try { + var eval = + testHarness + .braintrust() + .evalBuilder() + .name("supplied-executor-concurrency-test") + .cases(cases(12)) + .maxConcurrency(8) + .executor(executor) + .taskFunction( + input -> { + peak.accumulateAndGet( + inFlight.incrementAndGet(), Math::max); + try { + Thread.sleep(25); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + inFlight.decrementAndGet(); + return "fruit"; + }) + .scorers(Scorer.of("s", r -> 1.0)) + .build(); + + var result = eval.run(); + assertEquals(12, result.getCasesSucceeded()); + assertTrue(peak.get() > 1, "expected concurrent execution, peak was " + peak.get()); + assertTrue( + peak.get() <= 2, + "the supplied 2-thread executor should bound concurrency, not" + + " maxConcurrency(8); peak was " + + peak.get()); + } finally { + executor.shutdownNow(); + } + } + + @Test + @SneakyThrows + public void coordinatorDoesNotRunAheadOfThePool() { + // The default pool backpressures the coordinator: it must not drain the whole cursor into + // the executor's queue while the only worker is stuck on the first case. This is what + // regresses if the default pool is ever swapped back to an unbounded newFixedThreadPool. + int caseCount = 20; + var pulled = new AtomicInteger(); + var firstCaseStarted = new CountDownLatch(1); + var release = new CountDownLatch(1); + + var eval = + testHarness + .braintrust() + .evalBuilder() + .name("backpressure-test") + .dataset(countingCursorDataset(cases(caseCount), pulled)) + .maxConcurrency(1) + .taskFunction( + input -> { + firstCaseStarted.countDown(); + try { + release.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + return "fruit"; + }) + .scorers(Scorer.of("s", r -> 1.0)) + .build(); + + var result = eval.start(); + assertTrue(firstCaseStarted.await(10, TimeUnit.SECONDS), "first case never started"); + // Give the coordinator a chance to run ahead if it is going to. + Thread.sleep(250); + // The blocked case, plus at most one more sitting in the handoff. Anything beyond that + // means the coordinator drained the cursor without waiting for the pool. + assertTrue( + pulled.get() <= 2, + "coordinator ran ahead of the pool: pulled " + pulled.get() + " of " + caseCount); + + release.countDown(); + result.awaitCompletion(); + assertEquals(caseCount, result.getCasesSucceeded()); + assertEquals(caseCount, pulled.get(), "every case should eventually be pulled"); + } + + /** Wraps a dataset so the test can see how far ahead of the executor the coordinator pulls. */ + private Dataset countingCursorDataset( + DatasetCase[] cases, AtomicInteger pulled) { + var delegate = Dataset.of(cases); + return new Dataset<>() { + @Override + public Dataset.Cursor> openCursor() { + var cursor = delegate.openCursor(); + return new Dataset.Cursor<>() { + @Override + public Optional> next() { + var next = cursor.next(); + next.ifPresent(unused -> pulled.incrementAndGet()); + return next; + } + + @Override + public void close() { + cursor.close(); + } + + @Override + public Optional version() { + return cursor.version(); + } + }; + } + + @Override + public String id() { + return delegate.id(); + } + + @Override + public Optional version() { + return delegate.version(); + } + }; + } + + @Test + @SneakyThrows + public void rejectsInvalidMaxConcurrency() { + assertThrows( + IllegalArgumentException.class, + () -> Eval.builder().maxConcurrency(0)); + } +} diff --git a/braintrust-sdk/src/test/java/dev/braintrust/eval/EvalTest.java b/braintrust-sdk/src/test/java/dev/braintrust/eval/EvalTest.java index 9c63549c..412a68ff 100644 --- a/braintrust-sdk/src/test/java/dev/braintrust/eval/EvalTest.java +++ b/braintrust-sdk/src/test/java/dev/braintrust/eval/EvalTest.java @@ -477,6 +477,11 @@ void evalContinuesWhenTaskThrows() { // eval should complete without throwing, even though one case errors var result = eval.run(); assertNotNull(result.getExperimentUrl()); + assertEquals( + 2, + result.getCasesSucceeded(), + "a task exception recovered by scorer fallbacks should succeed"); + assertEquals(0, result.getCasesFailed()); var spans = testHarness.awaitExportedSpans(); diff --git a/jvm-compat-tests/java21/build.gradle b/jvm-compat-tests/java21/build.gradle new file mode 100644 index 00000000..57c1b480 --- /dev/null +++ b/jvm-compat-tests/java21/build.gradle @@ -0,0 +1,51 @@ +// Isolated JVM-compatibility test project pinned to Java 21. +// +// The rest of the build compiles and tests on Java 17. This project exists to run +// SDK code on a *different* JVM so we catch things that only show up at runtime on +// a newer release (class-file version, JDK internals access, agent attach rules, +// stdlib behavior changes, ...). +// +// It is deliberately its own subproject rather than an extra test task so each JVM +// under test can pin its own toolchain and dependency versions independently. +// To cover another JVM, copy this directory to jvm-compat-tests/java and bump +// the languageVersion below. +// +// Run with: ./gradlew :jvm-compat-tests:java21:test +plugins { + id 'java' +} + +java { + toolchain { + languageVersion = JavaLanguageVersion.of(21) + vendor = JvmVendorSpec.ADOPTIUM + } +} + +repositories { + mavenCentral() +} + +ext { + junitVersion = rootProject.ext.junitVersion + slf4jVersion = rootProject.ext.slf4jVersion +} + +dependencies { + // Depend on the SDK the same way an end user would: as a published jar built + // for Java 17, loaded here by a Java 21 runtime. + testImplementation project(":braintrust-sdk") + + testImplementation "org.junit.jupiter:junit-jupiter:${junitVersion}" + testRuntimeOnly 'org.junit.platform:junit-platform-launcher' + testRuntimeOnly "org.slf4j:slf4j-simple:${slf4jVersion}" +} + +test { + useJUnitPlatform() + + testLogging { + events 'passed', 'skipped', 'failed' + showStandardStreams = true + } +} diff --git a/jvm-compat-tests/java21/src/test/java/dev/braintrust/jvmcompat/Java21SmokeTest.java b/jvm-compat-tests/java21/src/test/java/dev/braintrust/jvmcompat/Java21SmokeTest.java new file mode 100644 index 00000000..b5cc7a64 --- /dev/null +++ b/jvm-compat-tests/java21/src/test/java/dev/braintrust/jvmcompat/Java21SmokeTest.java @@ -0,0 +1,97 @@ +package dev.braintrust.jvmcompat; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import dev.braintrust.Braintrust; +import dev.braintrust.BraintrustUtils; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.jupiter.api.Test; + +class Java21SmokeTest { + @Test + void runsOnJava21() { + assertEquals( + 21, + Runtime.version().feature(), + "expected the test JVM to be Java 21, got " + System.getProperty("java.version")); + assertNotNull(Braintrust.class.getName()); + } + + @Test + void usesVirtualThreadsOnJava21() throws Exception { + ExecutorService executor = BraintrustUtils.newExecutor(10, "java-21-smoke-test-"); + try { + var thread = executor.submit(Thread::currentThread).get(10, TimeUnit.SECONDS); + assertTrue( + thread.isVirtual(), + "expected a virtual thread on Java 21, got " + + thread + + " " + + thread.getClass()); + assertTrue( + thread.getName().startsWith("java-21-smoke-test-"), + "expected the thread name prefix to be preserved, got " + thread.getName()); + } finally { + executor.shutdown(); + } + } + + /** + * Virtual threads must not quietly turn {@code maxSize} into "unbounded" — eval concurrency + * depends on it to avoid hammering LLM providers. + */ + @Test + void virtualThreadExecutorStillCapsConcurrency() throws Exception { + int maxSize = 4; + ExecutorService executor = BraintrustUtils.newExecutor(maxSize, "java-21-bounded-"); + var inFlight = new AtomicInteger(); + var peak = new AtomicInteger(); + var release = new CountDownLatch(1); + var done = new CountDownLatch(maxSize); + try { + // Saturate the executor: each task parks until we release it. + for (int i = 0; i < maxSize; i++) { + executor.execute( + () -> { + peak.accumulateAndGet(inFlight.incrementAndGet(), Math::max); + try { + release.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } finally { + inFlight.decrementAndGet(); + done.countDown(); + } + }); + } + + // One more submit must block until a permit frees up rather than spawning task #5. + var submitted = new CountDownLatch(1); + var submitter = + Thread.ofPlatform() + .daemon() + .start( + () -> { + executor.execute(() -> {}); + submitted.countDown(); + }); + assertTrue( + !submitted.await(500, TimeUnit.MILLISECONDS), + "submitting past maxSize should have backpressured the caller"); + + release.countDown(); + assertTrue(done.await(10, TimeUnit.SECONDS), "tasks did not finish"); + assertTrue(submitted.await(10, TimeUnit.SECONDS), "blocked submit never went through"); + submitter.join(); + + assertEquals(maxSize, peak.get(), "concurrency exceeded maxSize"); + } finally { + executor.shutdown(); + } + } +} diff --git a/settings.gradle b/settings.gradle index f04daf27..97fda48f 100644 --- a/settings.gradle +++ b/settings.gradle @@ -46,3 +46,5 @@ include 'btx' include 'braintrust-api' include 'braintrust-otel-extension' include 'perf-tests' +include 'jvm-compat-tests' +include 'jvm-compat-tests:java21'