feat(cortado): add multi-turn intermediate SQL execution, Set Match d…

evalbench/dataset/cortadoinput.py

@@ -27,6 +27,7 @@ def __init__(self, raw_dict: dict, job_id: str = "", trace_id: str = ""):
 
         self.agent_results = []
         self.scoring_results = []
+        self.other = {}
 
     @classmethod
     def init_from_proto(cls, proto):
@@ -39,23 +40,31 @@ def init_from_proto(cls, proto):
 
         raw_dict["id"] = str(getattr(proto, "id", "-1"))
 
-        return cls(
+        obj = cls(
             raw_dict=raw_dict,
             job_id=getattr(proto, "job_id", ""),
             trace_id=getattr(proto, "trace_id", ""),
         )
+        if hasattr(proto, "other"):
+            for k, v in proto.other.items():
+                obj.other[k] = v
+        return obj
 
     def to_proto(self):
         """Packs the object into the Protobuf to send to Google3."""
         # Note: You must import eval_request_pb2 here to prevent circular dependencies
         from evalproto import eval_request_pb2
 
-        return eval_request_pb2.EvalInputRequest(
+        proto_req = eval_request_pb2.EvalInputRequest(
             id=int(self.id) if self.id.isdigit() else 0,
             payload=self.payload_str,
             # We map starting_prompt to nl_prompt for backwards compatibility
             nl_prompt=self.nl_prompt
         )
+        if hasattr(self, "other") and isinstance(self.other, dict):
+            for k, v in self.other.items():
+                proto_req.other[k] = str(v)
+        return proto_req
 
     def copy(self):
         return copy.deepcopy(self)

evalbench/eval_service.py

@@ -411,7 +411,12 @@ async def read_from_client():
                 logging.debug("Read task cancelled as expected.")
 
             # Process final scoring and reporting
-            job_id, run_time, results_tf, scores_tf = orchestrator.process()
+            process_res = orchestrator.process()
+            if len(process_res) == 5:
+                job_id, run_time, results_tf, scores_tf, multi_trial_scores_tf = process_res
+            else:
+                job_id, run_time, results_tf, scores_tf = process_res
+                multi_trial_scores_tf = None
             reporters = get_reporters(config.get(
                 "reporting") or {}, job_id, run_time)
 
@@ -427,7 +432,7 @@ async def read_from_client():
                 run_time,
                 results_tf,
                 scores_tf,
-                None,  # Added None for multi_trial_scores_tf
+                multi_trial_scores_tf,
                 config,
                 model_config,
                 db_configs,