diff --git a/apps/docs/content/docs/en/tools/agiloft.mdx b/apps/docs/content/docs/en/tools/agiloft.mdx index b5579d98ff..235300ea25 100644 --- a/apps/docs/content/docs/en/tools/agiloft.mdx +++ b/apps/docs/content/docs/en/tools/agiloft.mdx @@ -7,7 +7,7 @@ import { BlockInfoCard } from "@/components/ui/block-info-card" {/* MANUAL-CONTENT-START:intro */} diff --git a/apps/sim/blocks/registry.ts b/apps/sim/blocks/registry.ts index 270bea945c..2b2541a4d3 100644 --- a/apps/sim/blocks/registry.ts +++ b/apps/sim/blocks/registry.ts @@ -245,9 +245,9 @@ export const registry: Record = { ashby: AshbyBlock, athena: AthenaBlock, attio: AttioBlock, + box: BoxBlock, brandfetch: BrandfetchBlock, brightdata: BrightDataBlock, - box: BoxBlock, browser_use: BrowserUseBlock, calcom: CalComBlock, calendly: CalendlyBlock, diff --git a/apps/sim/tools/brightdata/cancel_snapshot.ts b/apps/sim/tools/brightdata/cancel_snapshot.ts index a5a2328979..8f4823ccc3 100644 --- a/apps/sim/tools/brightdata/cancel_snapshot.ts +++ b/apps/sim/tools/brightdata/cancel_snapshot.ts @@ -38,17 +38,17 @@ export const brightDataCancelSnapshotTool: ToolConfig< }), }, - transformResponse: async (response: Response) => { + transformResponse: async (response: Response, params) => { if (!response.ok) { const errorText = await response.text() throw new Error(errorText || `Cancel snapshot failed with status ${response.status}`) } - const data = (await response.json().catch(() => null)) as Record | null + await response.json().catch(() => null) return { success: true, output: { - snapshotId: (data?.snapshot_id as string) ?? null, + snapshotId: params?.snapshotId ?? null, cancelled: true, }, } diff --git a/apps/sim/tools/brightdata/discover.ts b/apps/sim/tools/brightdata/discover.ts index 8f0f8ced6b..153bf465c6 100644 --- a/apps/sim/tools/brightdata/discover.ts +++ b/apps/sim/tools/brightdata/discover.ts @@ -1,6 +1,13 @@ +import { createLogger } from '@sim/logger' +import { DEFAULT_EXECUTION_TIMEOUT_MS } from '@/lib/core/execution-limits' import type { BrightDataDiscoverParams, BrightDataDiscoverResponse } from '@/tools/brightdata/types' import type { ToolConfig } from '@/tools/types' +const logger = createLogger('tools:brightdata:discover') + +const POLL_INTERVAL_MS = 3000 +const MAX_POLL_TIME_MS = DEFAULT_EXECUTION_TIMEOUT_MS + export const brightDataDiscoverTool: ToolConfig< BrightDataDiscoverParams, BrightDataDiscoverResponse @@ -84,7 +91,7 @@ export const brightDataDiscoverTool: ToolConfig< }, }, - transformResponse: async (response: Response) => { + transformResponse: async (response: Response, params) => { if (!response.ok) { const errorText = await response.text() throw new Error(errorText || `Discover request failed with status ${response.status}`) @@ -92,37 +99,112 @@ export const brightDataDiscoverTool: ToolConfig< const data = await response.json() - let results: Array<{ - url: string | null - title: string | null - description: string | null - relevanceScore: number | null - content: string | null - }> = [] - - const items = Array.isArray(data) ? data : (data?.results ?? data?.data ?? []) - - if (Array.isArray(items)) { - results = items.map((item: Record) => ({ - url: (item.link as string) ?? (item.url as string) ?? null, - title: (item.title as string) ?? null, - description: (item.description as string) ?? (item.snippet as string) ?? null, - relevanceScore: (item.relevance_score as number) ?? null, - content: - (item.content as string) ?? (item.text as string) ?? (item.markdown as string) ?? null, - })) - } - return { success: true, output: { - results, - query: null, - totalResults: results.length, + results: [], + query: params?.query ?? null, + totalResults: 0, + taskId: data.task_id ?? null, }, } }, + postProcess: async (result, params) => { + if (!result.success) return result + + const taskId = result.output.taskId + if (!taskId) { + return { + ...result, + success: false, + error: 'Discover API did not return a task_id. Cannot poll for results.', + } + } + + logger.info(`Bright Data Discover task ${taskId} created, polling for results...`) + + let elapsedTime = 0 + + while (elapsedTime < MAX_POLL_TIME_MS) { + try { + const pollResponse = await fetch( + `https://api.brightdata.com/discover?task_id=${encodeURIComponent(taskId)}`, + { + method: 'GET', + headers: { + Authorization: `Bearer ${params.apiKey}`, + }, + } + ) + + if (!pollResponse.ok) { + return { + ...result, + success: false, + error: `Failed to poll discover results: ${pollResponse.statusText}`, + } + } + + const data = await pollResponse.json() + logger.info(`Bright Data Discover task ${taskId} status: ${data.status}`) + + if (data.status === 'done') { + const items = Array.isArray(data.results) ? data.results : [] + + const results = items.map((item: Record) => ({ + url: (item.link as string) ?? (item.url as string) ?? null, + title: (item.title as string) ?? null, + description: (item.description as string) ?? (item.snippet as string) ?? null, + relevanceScore: (item.relevance_score as number) ?? null, + content: (item.content as string) ?? null, + })) + + return { + success: true, + output: { + results, + query: params.query ?? null, + totalResults: results.length, + }, + } + } + + if (data.status === 'failed' || data.status === 'error') { + return { + ...result, + success: false, + error: `Discover task failed: ${data.error ?? 'Unknown error'}`, + } + } + + await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS)) + elapsedTime += POLL_INTERVAL_MS + } catch (error) { + logger.error('Error polling for discover task:', { + message: error instanceof Error ? error.message : String(error), + taskId, + }) + + return { + ...result, + success: false, + error: `Error polling for discover task: ${error instanceof Error ? error.message : String(error)}`, + } + } + } + + logger.warn( + `Discover task ${taskId} did not complete within the maximum polling time (${MAX_POLL_TIME_MS / 1000}s)` + ) + + return { + ...result, + success: false, + error: `Discover task ${taskId} timed out after ${MAX_POLL_TIME_MS / 1000}s. Check status manually.`, + } + }, + outputs: { results: { type: 'array', diff --git a/apps/sim/tools/brightdata/download_snapshot.ts b/apps/sim/tools/brightdata/download_snapshot.ts index c62cfc4c68..555be41aab 100644 --- a/apps/sim/tools/brightdata/download_snapshot.ts +++ b/apps/sim/tools/brightdata/download_snapshot.ts @@ -56,7 +56,7 @@ export const brightDataDownloadSnapshotTool: ToolConfig< }), }, - transformResponse: async (response: Response) => { + transformResponse: async (response: Response, params) => { if (response.status === 409) { throw new Error( 'Snapshot is not ready for download. Check the snapshot status first and wait until it is "ready".' @@ -89,7 +89,7 @@ export const brightDataDownloadSnapshotTool: ToolConfig< output: { data, format: contentType, - snapshotId: (data[0]?.snapshot_id as string) ?? null, + snapshotId: params?.snapshotId ?? null, }, } }, diff --git a/apps/sim/tools/brightdata/scrape_url.ts b/apps/sim/tools/brightdata/scrape_url.ts index 1fe284cd31..1d62d4c6df 100644 --- a/apps/sim/tools/brightdata/scrape_url.ts +++ b/apps/sim/tools/brightdata/scrape_url.ts @@ -66,7 +66,7 @@ export const brightDataScrapeUrlTool: ToolConfig< }, }, - transformResponse: async (response: Response) => { + transformResponse: async (response: Response, params) => { const contentType = response.headers.get('content-type') || '' if (!response.ok) { @@ -86,7 +86,7 @@ export const brightDataScrapeUrlTool: ToolConfig< success: true, output: { content, - url: null, + url: params?.url ?? null, statusCode: response.status, }, } diff --git a/apps/sim/tools/brightdata/serp_search.ts b/apps/sim/tools/brightdata/serp_search.ts index e9ed8ef1de..3acf3b2f07 100644 --- a/apps/sim/tools/brightdata/serp_search.ts +++ b/apps/sim/tools/brightdata/serp_search.ts @@ -129,7 +129,7 @@ export const brightDataSerpSearchTool: ToolConfig< }, }, - transformResponse: async (response: Response) => { + transformResponse: async (response: Response, params) => { if (!response.ok) { const errorText = await response.text() throw new Error(errorText || `SERP request failed with status ${response.status}`) @@ -178,9 +178,14 @@ export const brightDataSerpSearchTool: ToolConfig< success: true, output: { results, - query: ((data?.general as Record | undefined)?.query as string) ?? null, + query: + ((data?.general as Record | undefined)?.query as string) ?? + params?.query ?? + null, searchEngine: - ((data?.general as Record | undefined)?.search_engine as string) ?? null, + ((data?.general as Record | undefined)?.search_engine as string) ?? + params?.searchEngine ?? + null, }, } }, diff --git a/apps/sim/tools/brightdata/types.ts b/apps/sim/tools/brightdata/types.ts index 3197826996..d0a3e8aa39 100644 --- a/apps/sim/tools/brightdata/types.ts +++ b/apps/sim/tools/brightdata/types.ts @@ -131,6 +131,7 @@ export interface BrightDataDiscoverResponse extends ToolResponse { }> query: string | null totalResults: number + taskId?: string | null } } diff --git a/bun.lock b/bun.lock index aaa61ed6da..654302c866 100644 --- a/bun.lock +++ b/bun.lock @@ -1,5 +1,6 @@ { "lockfileVersion": 1, + "configVersion": 0, "workspaces": { "": { "name": "simstudio",