fix(brightdata): fix async Discover API, echo-back fields, and registry ordering

apps/docs/content/docs/en/tools/agiloft.mdx

@@ -7,7 +7,7 @@ import { BlockInfoCard } from "@/components/ui/block-info-card"
 
 <BlockInfoCard 
   type="agiloft"
-  color="#263A5C"
+  color="#FFFFFF"
 />
 
 {/* MANUAL-CONTENT-START:intro */}

apps/sim/blocks/registry.ts

@@ -245,9 +245,9 @@ export const registry: Record<string, BlockConfig> = {
   ashby: AshbyBlock,
   athena: AthenaBlock,
   attio: AttioBlock,
+  box: BoxBlock,
   brandfetch: BrandfetchBlock,
   brightdata: BrightDataBlock,
-  box: BoxBlock,
   browser_use: BrowserUseBlock,
   calcom: CalComBlock,
   calendly: CalendlyBlock,

apps/sim/tools/brightdata/cancel_snapshot.ts

@@ -38,17 +38,17 @@ export const brightDataCancelSnapshotTool: ToolConfig<
     }),
   },
 
-  transformResponse: async (response: Response) => {
+  transformResponse: async (response: Response, params) => {
     if (!response.ok) {
       const errorText = await response.text()
       throw new Error(errorText || `Cancel snapshot failed with status ${response.status}`)
     }
 
-    const data = (await response.json().catch(() => null)) as Record<string, unknown> | null
+    await response.json().catch(() => null)
     return {
       success: true,
       output: {
-        snapshotId: (data?.snapshot_id as string) ?? null,
+        snapshotId: params?.snapshotId ?? null,
         cancelled: true,
       },
     }

apps/sim/tools/brightdata/discover.ts

@@ -1,6 +1,13 @@
+import { createLogger } from '@sim/logger'
+import { DEFAULT_EXECUTION_TIMEOUT_MS } from '@/lib/core/execution-limits'
 import type { BrightDataDiscoverParams, BrightDataDiscoverResponse } from '@/tools/brightdata/types'
 import type { ToolConfig } from '@/tools/types'
 
+const logger = createLogger('tools:brightdata:discover')
+
+const POLL_INTERVAL_MS = 3000
+const MAX_POLL_TIME_MS = DEFAULT_EXECUTION_TIMEOUT_MS
+
 export const brightDataDiscoverTool: ToolConfig<
   BrightDataDiscoverParams,
   BrightDataDiscoverResponse
@@ -84,45 +91,120 @@ export const brightDataDiscoverTool: ToolConfig<
     },
   },
 
-  transformResponse: async (response: Response) => {
+  transformResponse: async (response: Response, params) => {
     if (!response.ok) {
       const errorText = await response.text()
       throw new Error(errorText || `Discover request failed with status ${response.status}`)
     }
 
     const data = await response.json()
 
-    let results: Array<{
-      url: string | null
-      title: string | null
-      description: string | null
-      relevanceScore: number | null
-      content: string | null
-    }> = []
-
-    const items = Array.isArray(data) ? data : (data?.results ?? data?.data ?? [])
-
-    if (Array.isArray(items)) {
-      results = items.map((item: Record<string, unknown>) => ({
-        url: (item.link as string) ?? (item.url as string) ?? null,
-        title: (item.title as string) ?? null,
-        description: (item.description as string) ?? (item.snippet as string) ?? null,
-        relevanceScore: (item.relevance_score as number) ?? null,
-        content:
-          (item.content as string) ?? (item.text as string) ?? (item.markdown as string) ?? null,
-      }))
-    }
-
     return {
       success: true,
       output: {
-        results,
-        query: null,
-        totalResults: results.length,
+        results: [],
+        query: params?.query ?? null,
+        totalResults: 0,
+        taskId: data.task_id ?? null,
       },
     }
   },
 
+  postProcess: async (result, params) => {
+    if (!result.success) return result
+
+    const taskId = result.output.taskId
+    if (!taskId) {
+      return {
+        ...result,
+        success: false,
+        error: 'Discover API did not return a task_id. Cannot poll for results.',
+      }
+    }
+
+    logger.info(`Bright Data Discover task ${taskId} created, polling for results...`)
+
+    let elapsedTime = 0
+
+    while (elapsedTime < MAX_POLL_TIME_MS) {
+      try {
+        const pollResponse = await fetch(
+          `https://api.brightdata.com/discover?task_id=${encodeURIComponent(taskId)}`,
+          {
+            method: 'GET',
+            headers: {
+              Authorization: `Bearer ${params.apiKey}`,
+            },
+          }
+        )
+
+        if (!pollResponse.ok) {
+          return {
+            ...result,
+            success: false,
+            error: `Failed to poll discover results: ${pollResponse.statusText}`,
+          }
+        }
+
+        const data = await pollResponse.json()
+        logger.info(`Bright Data Discover task ${taskId} status: ${data.status}`)
+
+        if (data.status === 'done') {
+          const items = Array.isArray(data.results) ? data.results : []
+
+          const results = items.map((item: Record<string, unknown>) => ({
+            url: (item.link as string) ?? (item.url as string) ?? null,
+            title: (item.title as string) ?? null,
+            description: (item.description as string) ?? (item.snippet as string) ?? null,
+            relevanceScore: (item.relevance_score as number) ?? null,
+            content: (item.content as string) ?? null,
+          }))
+
+          return {
+            success: true,
+            output: {
+              results,
+              query: params.query ?? null,
+              totalResults: results.length,
+            },
+          }
+        }
+
+        if (data.status === 'failed' || data.status === 'error') {
+          return {
+            ...result,
+            success: false,
+            error: `Discover task failed: ${data.error ?? 'Unknown error'}`,
+          }
+        }
+
+        await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS))
+        elapsedTime += POLL_INTERVAL_MS
+      } catch (error) {
+        logger.error('Error polling for discover task:', {
+          message: error instanceof Error ? error.message : String(error),
+          taskId,
+        })
+
+        return {
+          ...result,
+          success: false,
+          error: `Error polling for discover task: ${error instanceof Error ? error.message : String(error)}`,
+        }
+      }
+    }
+
+    logger.warn(
+      `Discover task ${taskId} did not complete within the maximum polling time (${MAX_POLL_TIME_MS / 1000}s)`
+    )
+
+    return {
+      ...result,
+      success: false,
+      error: `Discover task ${taskId} timed out after ${MAX_POLL_TIME_MS / 1000}s. Check status manually.`,
+    }
+  },
+
   outputs: {
     results: {
       type: 'array',

apps/sim/tools/brightdata/download_snapshot.ts

@@ -56,7 +56,7 @@ export const brightDataDownloadSnapshotTool: ToolConfig<
     }),
   },
 
-  transformResponse: async (response: Response) => {
+  transformResponse: async (response: Response, params) => {
     if (response.status === 409) {
       throw new Error(
         'Snapshot is not ready for download. Check the snapshot status first and wait until it is "ready".'
@@ -89,7 +89,7 @@ export const brightDataDownloadSnapshotTool: ToolConfig<
       output: {
         data,
         format: contentType,
-        snapshotId: (data[0]?.snapshot_id as string) ?? null,
+        snapshotId: params?.snapshotId ?? null,
       },
     }
   },

apps/sim/tools/brightdata/scrape_url.ts

@@ -66,7 +66,7 @@ export const brightDataScrapeUrlTool: ToolConfig<
     },
   },
 
-  transformResponse: async (response: Response) => {
+  transformResponse: async (response: Response, params) => {
     const contentType = response.headers.get('content-type') || ''
 
     if (!response.ok) {
@@ -86,7 +86,7 @@ export const brightDataScrapeUrlTool: ToolConfig<
       success: true,
       output: {
         content,
-        url: null,
+        url: params?.url ?? null,
         statusCode: response.status,
       },
     }

apps/sim/tools/brightdata/serp_search.ts

@@ -129,7 +129,7 @@ export const brightDataSerpSearchTool: ToolConfig<
     },
   },
 
-  transformResponse: async (response: Response) => {
+  transformResponse: async (response: Response, params) => {
     if (!response.ok) {
       const errorText = await response.text()
       throw new Error(errorText || `SERP request failed with status ${response.status}`)
@@ -178,9 +178,14 @@ export const brightDataSerpSearchTool: ToolConfig<
       success: true,
       output: {
         results,
-        query: ((data?.general as Record<string, unknown> | undefined)?.query as string) ?? null,
+        query:
+          ((data?.general as Record<string, unknown> | undefined)?.query as string) ??
+          params?.query ??
+          null,
         searchEngine:
-          ((data?.general as Record<string, unknown> | undefined)?.search_engine as string) ?? null,
+          ((data?.general as Record<string, unknown> | undefined)?.search_engine as string) ??
+          params?.searchEngine ??
+          null,
       },
     }
   },

apps/sim/tools/brightdata/types.ts

@@ -131,6 +131,7 @@ export interface BrightDataDiscoverResponse extends ToolResponse {
     }>
     query: string | null
     totalResults: number
+    taskId?: string | null
   }
 }