From 22df446568e8c4b4fa214e0c5ebcfcca544692fb Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 15 Apr 2026 13:16:30 -0700 Subject: [PATCH 1/6] fix(brightdata): use params for echo-back fields in transformResponse transformResponse receives params as its second argument. Use it to return the original url, query, snapshotId, and searchEngine values instead of hardcoding null or extracting from response data that may not contain them. Co-Authored-By: Claude Opus 4.6 --- apps/sim/tools/brightdata/cancel_snapshot.ts | 6 +++--- apps/sim/tools/brightdata/discover.ts | 4 ++-- apps/sim/tools/brightdata/download_snapshot.ts | 4 ++-- apps/sim/tools/brightdata/scrape_url.ts | 4 ++-- apps/sim/tools/brightdata/serp_search.ts | 11 ++++++++--- 5 files changed, 17 insertions(+), 12 deletions(-) diff --git a/apps/sim/tools/brightdata/cancel_snapshot.ts b/apps/sim/tools/brightdata/cancel_snapshot.ts index a5a2328979..8f4823ccc3 100644 --- a/apps/sim/tools/brightdata/cancel_snapshot.ts +++ b/apps/sim/tools/brightdata/cancel_snapshot.ts @@ -38,17 +38,17 @@ export const brightDataCancelSnapshotTool: ToolConfig< }), }, - transformResponse: async (response: Response) => { + transformResponse: async (response: Response, params) => { if (!response.ok) { const errorText = await response.text() throw new Error(errorText || `Cancel snapshot failed with status ${response.status}`) } - const data = (await response.json().catch(() => null)) as Record | null + await response.json().catch(() => null) return { success: true, output: { - snapshotId: (data?.snapshot_id as string) ?? null, + snapshotId: params?.snapshotId ?? null, cancelled: true, }, } diff --git a/apps/sim/tools/brightdata/discover.ts b/apps/sim/tools/brightdata/discover.ts index 8f0f8ced6b..8ccaa25817 100644 --- a/apps/sim/tools/brightdata/discover.ts +++ b/apps/sim/tools/brightdata/discover.ts @@ -84,7 +84,7 @@ export const brightDataDiscoverTool: ToolConfig< }, }, - transformResponse: async (response: Response) => { + transformResponse: async (response: Response, params) => { if (!response.ok) { const errorText = await response.text() throw new Error(errorText || `Discover request failed with status ${response.status}`) @@ -117,7 +117,7 @@ export const brightDataDiscoverTool: ToolConfig< success: true, output: { results, - query: null, + query: params?.query ?? null, totalResults: results.length, }, } diff --git a/apps/sim/tools/brightdata/download_snapshot.ts b/apps/sim/tools/brightdata/download_snapshot.ts index c62cfc4c68..555be41aab 100644 --- a/apps/sim/tools/brightdata/download_snapshot.ts +++ b/apps/sim/tools/brightdata/download_snapshot.ts @@ -56,7 +56,7 @@ export const brightDataDownloadSnapshotTool: ToolConfig< }), }, - transformResponse: async (response: Response) => { + transformResponse: async (response: Response, params) => { if (response.status === 409) { throw new Error( 'Snapshot is not ready for download. Check the snapshot status first and wait until it is "ready".' @@ -89,7 +89,7 @@ export const brightDataDownloadSnapshotTool: ToolConfig< output: { data, format: contentType, - snapshotId: (data[0]?.snapshot_id as string) ?? null, + snapshotId: params?.snapshotId ?? null, }, } }, diff --git a/apps/sim/tools/brightdata/scrape_url.ts b/apps/sim/tools/brightdata/scrape_url.ts index 1fe284cd31..1d62d4c6df 100644 --- a/apps/sim/tools/brightdata/scrape_url.ts +++ b/apps/sim/tools/brightdata/scrape_url.ts @@ -66,7 +66,7 @@ export const brightDataScrapeUrlTool: ToolConfig< }, }, - transformResponse: async (response: Response) => { + transformResponse: async (response: Response, params) => { const contentType = response.headers.get('content-type') || '' if (!response.ok) { @@ -86,7 +86,7 @@ export const brightDataScrapeUrlTool: ToolConfig< success: true, output: { content, - url: null, + url: params?.url ?? null, statusCode: response.status, }, } diff --git a/apps/sim/tools/brightdata/serp_search.ts b/apps/sim/tools/brightdata/serp_search.ts index e9ed8ef1de..3acf3b2f07 100644 --- a/apps/sim/tools/brightdata/serp_search.ts +++ b/apps/sim/tools/brightdata/serp_search.ts @@ -129,7 +129,7 @@ export const brightDataSerpSearchTool: ToolConfig< }, }, - transformResponse: async (response: Response) => { + transformResponse: async (response: Response, params) => { if (!response.ok) { const errorText = await response.text() throw new Error(errorText || `SERP request failed with status ${response.status}`) @@ -178,9 +178,14 @@ export const brightDataSerpSearchTool: ToolConfig< success: true, output: { results, - query: ((data?.general as Record | undefined)?.query as string) ?? null, + query: + ((data?.general as Record | undefined)?.query as string) ?? + params?.query ?? + null, searchEngine: - ((data?.general as Record | undefined)?.search_engine as string) ?? null, + ((data?.general as Record | undefined)?.search_engine as string) ?? + params?.searchEngine ?? + null, }, } }, From 594bc96f8b8263ca3e8e4610f8c9bffca0425b0f Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 15 Apr 2026 13:25:42 -0700 Subject: [PATCH 2/6] fix(brightdata): handle async Discover API with polling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Bright Data Discover API is asynchronous — POST /discover returns a task_id, and results must be polled via GET /discover?task_id=... The previous implementation incorrectly treated it as synchronous, always returning empty results. Uses postProcess (matching Firecrawl crawl pattern) to poll every 3s with a 120s timeout until status is "done". Co-Authored-By: Claude Opus 4.6 --- apps/sim/tools/brightdata/discover.ts | 96 ++++++++++++++++++++------- 1 file changed, 73 insertions(+), 23 deletions(-) diff --git a/apps/sim/tools/brightdata/discover.ts b/apps/sim/tools/brightdata/discover.ts index 8ccaa25817..0b0d3e0c62 100644 --- a/apps/sim/tools/brightdata/discover.ts +++ b/apps/sim/tools/brightdata/discover.ts @@ -1,6 +1,12 @@ +import { createLogger } from '@sim/logger' import type { BrightDataDiscoverParams, BrightDataDiscoverResponse } from '@/tools/brightdata/types' import type { ToolConfig } from '@/tools/types' +const logger = createLogger('tools:brightdata:discover') + +const POLL_INTERVAL_MS = 3000 +const MAX_POLL_TIME_MS = 120000 + export const brightDataDiscoverTool: ToolConfig< BrightDataDiscoverParams, BrightDataDiscoverResponse @@ -92,35 +98,79 @@ export const brightDataDiscoverTool: ToolConfig< const data = await response.json() - let results: Array<{ - url: string | null - title: string | null - description: string | null - relevanceScore: number | null - content: string | null - }> = [] - - const items = Array.isArray(data) ? data : (data?.results ?? data?.data ?? []) - - if (Array.isArray(items)) { - results = items.map((item: Record) => ({ - url: (item.link as string) ?? (item.url as string) ?? null, - title: (item.title as string) ?? null, - description: (item.description as string) ?? (item.snippet as string) ?? null, - relevanceScore: (item.relevance_score as number) ?? null, - content: - (item.content as string) ?? (item.text as string) ?? (item.markdown as string) ?? null, - })) - } - return { success: true, output: { - results, + results: [], query: params?.query ?? null, - totalResults: results.length, + totalResults: 0, + taskId: data.task_id ?? null, }, + } as BrightDataDiscoverResponse + }, + + postProcess: async (result, params) => { + if (!result.success) return result + + const taskId = (result.output as Record).taskId as string | null + if (!taskId) { + throw new Error('Discover API did not return a task_id. Cannot poll for results.') + } + + logger.info(`Bright Data Discover task ${taskId} created, polling for results...`) + + let elapsedTime = 0 + + while (elapsedTime < MAX_POLL_TIME_MS) { + const pollResponse = await fetch( + `https://api.brightdata.com/discover?task_id=${encodeURIComponent(taskId)}`, + { + method: 'GET', + headers: { + Authorization: `Bearer ${params.apiKey}`, + }, + } + ) + + if (!pollResponse.ok) { + throw new Error(`Failed to poll discover results: ${pollResponse.statusText}`) + } + + const data = await pollResponse.json() + logger.info(`Bright Data Discover task ${taskId} status: ${data.status}`) + + if (data.status === 'done') { + const items = Array.isArray(data.results) ? data.results : [] + + const results = items.map((item: Record) => ({ + url: (item.link as string) ?? (item.url as string) ?? null, + title: (item.title as string) ?? null, + description: (item.description as string) ?? (item.snippet as string) ?? null, + relevanceScore: (item.relevance_score as number) ?? null, + content: (item.content as string) ?? null, + })) + + return { + success: true, + output: { + results, + query: params.query ?? null, + totalResults: results.length, + }, + } as BrightDataDiscoverResponse + } + + if (data.status === 'failed' || data.status === 'error') { + throw new Error(`Discover task failed: ${data.error ?? 'Unknown error'}`) + } + + await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS)) + elapsedTime += POLL_INTERVAL_MS } + + throw new Error( + `Discover task ${taskId} timed out after ${MAX_POLL_TIME_MS / 1000}s. Check status manually.` + ) }, outputs: { From 0897a1b3375d2fbafcc9028985dd9f0e5f39d70d Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 15 Apr 2026 13:29:54 -0700 Subject: [PATCH 3/6] fix(brightdata): alphabetize block registry entry Move box before brandfetch/brightdata to maintain alphabetical ordering. Co-Authored-By: Claude Opus 4.6 --- apps/sim/blocks/registry.ts | 2 +- bun.lock | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/sim/blocks/registry.ts b/apps/sim/blocks/registry.ts index 270bea945c..2b2541a4d3 100644 --- a/apps/sim/blocks/registry.ts +++ b/apps/sim/blocks/registry.ts @@ -245,9 +245,9 @@ export const registry: Record = { ashby: AshbyBlock, athena: AthenaBlock, attio: AttioBlock, + box: BoxBlock, brandfetch: BrandfetchBlock, brightdata: BrightDataBlock, - box: BoxBlock, browser_use: BrowserUseBlock, calcom: CalComBlock, calendly: CalendlyBlock, diff --git a/bun.lock b/bun.lock index aaa61ed6da..654302c866 100644 --- a/bun.lock +++ b/bun.lock @@ -1,5 +1,6 @@ { "lockfileVersion": 1, + "configVersion": 0, "workspaces": { "": { "name": "simstudio", From 17827e8eaeca185eaaf03e5dcb0c1d4036412b35 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 15 Apr 2026 13:31:11 -0700 Subject: [PATCH 4/6] lint --- apps/docs/content/docs/en/tools/agiloft.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/docs/content/docs/en/tools/agiloft.mdx b/apps/docs/content/docs/en/tools/agiloft.mdx index b5579d98ff..235300ea25 100644 --- a/apps/docs/content/docs/en/tools/agiloft.mdx +++ b/apps/docs/content/docs/en/tools/agiloft.mdx @@ -7,7 +7,7 @@ import { BlockInfoCard } from "@/components/ui/block-info-card" {/* MANUAL-CONTENT-START:intro */} From b549ee47cd149b60d6a08263f847753b456e6de5 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 15 Apr 2026 14:05:19 -0700 Subject: [PATCH 5/6] fix(brightdata): return error objects instead of throwing in postProcess The executor wraps postProcess in try-catch and falls back to the intermediate transformResponse result on error, which has success: true with empty results. Throwing errors would silently return empty results. Match Firecrawl's pattern: return { ...result, success: false, error } instead of throwing. Also add taskId to BrightDataDiscoverResponse type to eliminate unsafe casts. Co-Authored-By: Claude Opus 4.6 --- apps/sim/tools/brightdata/discover.ts | 115 ++++++++++++++++---------- apps/sim/tools/brightdata/types.ts | 1 + 2 files changed, 74 insertions(+), 42 deletions(-) diff --git a/apps/sim/tools/brightdata/discover.ts b/apps/sim/tools/brightdata/discover.ts index 0b0d3e0c62..f2a4373adb 100644 --- a/apps/sim/tools/brightdata/discover.ts +++ b/apps/sim/tools/brightdata/discover.ts @@ -106,15 +106,19 @@ export const brightDataDiscoverTool: ToolConfig< totalResults: 0, taskId: data.task_id ?? null, }, - } as BrightDataDiscoverResponse + } }, postProcess: async (result, params) => { if (!result.success) return result - const taskId = (result.output as Record).taskId as string | null + const taskId = result.output.taskId if (!taskId) { - throw new Error('Discover API did not return a task_id. Cannot poll for results.') + return { + ...result, + success: false, + error: 'Discover API did not return a task_id. Cannot poll for results.', + } } logger.info(`Bright Data Discover task ${taskId} created, polling for results...`) @@ -122,55 +126,82 @@ export const brightDataDiscoverTool: ToolConfig< let elapsedTime = 0 while (elapsedTime < MAX_POLL_TIME_MS) { - const pollResponse = await fetch( - `https://api.brightdata.com/discover?task_id=${encodeURIComponent(taskId)}`, - { - method: 'GET', - headers: { - Authorization: `Bearer ${params.apiKey}`, - }, + try { + const pollResponse = await fetch( + `https://api.brightdata.com/discover?task_id=${encodeURIComponent(taskId)}`, + { + method: 'GET', + headers: { + Authorization: `Bearer ${params.apiKey}`, + }, + } + ) + + if (!pollResponse.ok) { + return { + ...result, + success: false, + error: `Failed to poll discover results: ${pollResponse.statusText}`, + } } - ) - - if (!pollResponse.ok) { - throw new Error(`Failed to poll discover results: ${pollResponse.statusText}`) - } - const data = await pollResponse.json() - logger.info(`Bright Data Discover task ${taskId} status: ${data.status}`) + const data = await pollResponse.json() + logger.info(`Bright Data Discover task ${taskId} status: ${data.status}`) + + if (data.status === 'done') { + const items = Array.isArray(data.results) ? data.results : [] + + const results = items.map((item: Record) => ({ + url: (item.link as string) ?? (item.url as string) ?? null, + title: (item.title as string) ?? null, + description: (item.description as string) ?? (item.snippet as string) ?? null, + relevanceScore: (item.relevance_score as number) ?? null, + content: (item.content as string) ?? null, + })) + + return { + success: true, + output: { + results, + query: params.query ?? null, + totalResults: results.length, + }, + } + } - if (data.status === 'done') { - const items = Array.isArray(data.results) ? data.results : [] + if (data.status === 'failed' || data.status === 'error') { + return { + ...result, + success: false, + error: `Discover task failed: ${data.error ?? 'Unknown error'}`, + } + } - const results = items.map((item: Record) => ({ - url: (item.link as string) ?? (item.url as string) ?? null, - title: (item.title as string) ?? null, - description: (item.description as string) ?? (item.snippet as string) ?? null, - relevanceScore: (item.relevance_score as number) ?? null, - content: (item.content as string) ?? null, - })) + await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS)) + elapsedTime += POLL_INTERVAL_MS + } catch (error) { + logger.error('Error polling for discover task:', { + message: error instanceof Error ? error.message : String(error), + taskId, + }) return { - success: true, - output: { - results, - query: params.query ?? null, - totalResults: results.length, - }, - } as BrightDataDiscoverResponse - } - - if (data.status === 'failed' || data.status === 'error') { - throw new Error(`Discover task failed: ${data.error ?? 'Unknown error'}`) + ...result, + success: false, + error: `Error polling for discover task: ${error instanceof Error ? error.message : String(error)}`, + } } - - await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS)) - elapsedTime += POLL_INTERVAL_MS } - throw new Error( - `Discover task ${taskId} timed out after ${MAX_POLL_TIME_MS / 1000}s. Check status manually.` + logger.warn( + `Discover task ${taskId} did not complete within the maximum polling time (${MAX_POLL_TIME_MS / 1000}s)` ) + + return { + ...result, + success: false, + error: `Discover task ${taskId} timed out after ${MAX_POLL_TIME_MS / 1000}s. Check status manually.`, + } }, outputs: { diff --git a/apps/sim/tools/brightdata/types.ts b/apps/sim/tools/brightdata/types.ts index 3197826996..d0a3e8aa39 100644 --- a/apps/sim/tools/brightdata/types.ts +++ b/apps/sim/tools/brightdata/types.ts @@ -131,6 +131,7 @@ export interface BrightDataDiscoverResponse extends ToolResponse { }> query: string | null totalResults: number + taskId?: string | null } } From 9e9bf3588730248e71305f95a9553d0707a63ebe Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 15 Apr 2026 16:08:29 -0700 Subject: [PATCH 6/6] fix(brightdata): use platform execution timeout for Discover polling Replace hardcoded 120s timeout with DEFAULT_EXECUTION_TIMEOUT_MS to match Firecrawl and other async polling tools. Respects platform- configured limits (300s free, 3000s paid). Co-Authored-By: Claude Opus 4.6 --- apps/sim/tools/brightdata/discover.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/sim/tools/brightdata/discover.ts b/apps/sim/tools/brightdata/discover.ts index f2a4373adb..153bf465c6 100644 --- a/apps/sim/tools/brightdata/discover.ts +++ b/apps/sim/tools/brightdata/discover.ts @@ -1,11 +1,12 @@ import { createLogger } from '@sim/logger' +import { DEFAULT_EXECUTION_TIMEOUT_MS } from '@/lib/core/execution-limits' import type { BrightDataDiscoverParams, BrightDataDiscoverResponse } from '@/tools/brightdata/types' import type { ToolConfig } from '@/tools/types' const logger = createLogger('tools:brightdata:discover') const POLL_INTERVAL_MS = 3000 -const MAX_POLL_TIME_MS = 120000 +const MAX_POLL_TIME_MS = DEFAULT_EXECUTION_TIMEOUT_MS export const brightDataDiscoverTool: ToolConfig< BrightDataDiscoverParams,