From 8627f596ce8d60c1eee7ec91e31ce3eec0059617 Mon Sep 17 00:00:00 2001 From: Zeeshan Adil Date: Tue, 16 Dec 2025 13:03:13 +0500 Subject: [PATCH 1/6] feat(olostep): add Olostep service bubble for web scraping and AI content extraction - Add OlostepBubble with 5 operations: scrape, batch, crawl, map, answer - Add OLOSTEP_API_KEY credential type and configuration - Add 'olostep' to BubbleName type - Register bubble in BubbleFactory - Export from bubble-core index - Add comprehensive unit tests - Add credential UI configuration in bubble-studio --- .../src/pages/CredentialsPage.tsx | 1 + packages/bubble-core/src/bubble-factory.ts | 5 + .../bubbles/service-bubble/olostep.test.ts | 354 +++++++++++ .../src/bubbles/service-bubble/olostep.ts | 591 ++++++++++++++++++ packages/bubble-core/src/index.ts | 2 + .../src/bubble-definition-schema.ts | 1 + .../src/credential-schema.ts | 9 + packages/bubble-shared-schemas/src/types.ts | 4 +- 8 files changed, 966 insertions(+), 1 deletion(-) create mode 100644 packages/bubble-core/src/bubbles/service-bubble/olostep.test.ts create mode 100644 packages/bubble-core/src/bubbles/service-bubble/olostep.ts diff --git a/apps/bubble-studio/src/pages/CredentialsPage.tsx b/apps/bubble-studio/src/pages/CredentialsPage.tsx index a0955e146..338a3f029 100644 --- a/apps/bubble-studio/src/pages/CredentialsPage.tsx +++ b/apps/bubble-studio/src/pages/CredentialsPage.tsx @@ -87,6 +87,7 @@ const getServiceNameForCredentialType = ( [CredentialType.NOTION_API]: 'Notion', [CredentialType.INSFORGE_BASE_URL]: 'InsForge', [CredentialType.INSFORGE_API_KEY]: 'InsForge', + [CredentialType.OLOSTEP_API_KEY]: 'Olostep', [CredentialType.CUSTOM_AUTH_KEY]: 'Custom', [CredentialType.AMAZON_CRED]: 'Amazon', [CredentialType.BROWSERBASE_CRED]: 'BrowserBase', diff --git a/packages/bubble-core/src/bubble-factory.ts b/packages/bubble-core/src/bubble-factory.ts index c5cfc201b..ca7ae5b90 100644 --- a/packages/bubble-core/src/bubble-factory.ts +++ b/packages/bubble-core/src/bubble-factory.ts @@ -195,6 +195,7 @@ export class BubbleFactory { 'memberful', 'luma', 'zoom', + 'olostep', ]; } @@ -476,6 +477,9 @@ export class BubbleFactory { const { ZoomBubble } = await import( './bubbles/service-bubble/zoom/index.js' ); + const { OlostepBubble } = await import( + './bubbles/service-bubble/olostep.js' + ); // Create the default factory instance this.register('hello-world', HelloWorldBubble as BubbleClassWithMetadata); @@ -656,6 +660,7 @@ export class BubbleFactory { this.register('memberful', MemberfulBubble as BubbleClassWithMetadata); this.register('luma', LumaBubble as BubbleClassWithMetadata); this.register('zoom', ZoomBubble as BubbleClassWithMetadata); + this.register('olostep', OlostepBubble as BubbleClassWithMetadata); // After all default bubbles are registered, auto-populate bubbleDependencies if (!BubbleFactory.dependenciesPopulated) { diff --git a/packages/bubble-core/src/bubbles/service-bubble/olostep.test.ts b/packages/bubble-core/src/bubbles/service-bubble/olostep.test.ts new file mode 100644 index 000000000..a08cca659 --- /dev/null +++ b/packages/bubble-core/src/bubbles/service-bubble/olostep.test.ts @@ -0,0 +1,354 @@ +import { describe, it, expect, beforeAll, vi } from 'vitest'; +import { OlostepBubble, type OlostepParamsInput } from './olostep.js'; +import { CredentialType } from '@bubblelab/shared-schemas'; +import { BubbleFactory } from '../../bubble-factory.js'; +import { ZodDiscriminatedUnion, ZodObject } from 'zod'; + +// Helper function to create test credentials +const createTestCredentials = () => ({ + [CredentialType.OLOSTEP_API_KEY]: 'ols_test_0123456789abcdef', +}); + +// Mock fetch for API calls +const mockFetch = vi.fn(); +vi.stubGlobal('fetch', mockFetch); + +const factory = new BubbleFactory(); + +beforeAll(async () => { + await factory.registerDefaults(); +}); + +/** + * Unit tests for Olostep Service Bubble + * + * Tests the Olostep API integration for web scraping, crawling, and AI-powered content extraction + */ +describe('OlostepBubble', () => { + // + // REGISTRATION & SCHEMA + // + describe('Registration & Schema', () => { + it('should be registered in BubbleRegistry', async () => { + const bubbleClass = factory.get('olostep'); + expect(bubbleClass).toBeDefined(); + expect(bubbleClass).toBe(OlostepBubble); + }); + + it('schema should be a Zod discriminated union based on "operation"', () => { + const schema = OlostepBubble.schema; + expect(schema).toBeDefined(); + + // Validate ZodDiscriminatedUnion + expect(schema instanceof ZodDiscriminatedUnion).toBe(true); + + const du = schema as ZodDiscriminatedUnion< + 'operation', + readonly ZodObject[] + >; + expect(du.discriminator).toBe('operation'); + + const operationValues = du.options.map((o) => o.shape.operation.value); + + expect(operationValues).toContain('scrape'); + expect(operationValues).toContain('batch'); + expect(operationValues).toContain('crawl'); + expect(operationValues).toContain('map'); + expect(operationValues).toContain('answer'); + }); + + it('result schema should validate a sample scrape result', () => { + const sample = { + operation: 'scrape', + success: true, + error: '', + markdown_content: '# Hello World', + }; + + const parsed = OlostepBubble.resultSchema.safeParse(sample); + expect(parsed.success).toBe(true); + }); + }); + + // + // METADATA + // + describe('Metadata Tests', () => { + it('should have correct metadata', () => { + const metadata = factory.getMetadata('olostep'); + + expect(metadata).toBeDefined(); + expect(metadata?.name).toBe('olostep'); + expect(metadata?.alias).toBe('web-scraper'); + expect(metadata?.schema).toBeDefined(); + expect(metadata?.resultSchema).toBeDefined(); + expect(metadata?.shortDescription).toContain('Web scraping'); + expect(metadata?.longDescription).toContain('Olostep'); + }); + + it('static properties are correct', () => { + expect(OlostepBubble.bubbleName).toBe('olostep'); + expect(OlostepBubble.alias).toBe('web-scraper'); + expect(OlostepBubble.service).toBe('olostep'); + expect(OlostepBubble.authType).toBe('apikey'); + expect(OlostepBubble.type).toBe('service'); + expect(OlostepBubble.schema).toBeDefined(); + expect(OlostepBubble.resultSchema).toBeDefined(); + expect(OlostepBubble.shortDescription).toContain('Web scraping'); + expect(OlostepBubble.longDescription).toContain('Scrape'); + }); + }); + + // + // CREDENTIAL VALIDATION + // + describe('Credential Validation', () => { + it('should fail testCredential() with missing credentials', async () => { + const bubble = new OlostepBubble({ + operation: 'scrape', + url: 'https://example.com', + }); + + const result = await bubble.testCredential(); + expect(result).toBe(false); + }); + + it('should pass testCredential() with valid credentials', async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + text: async () => JSON.stringify({ markdown_content: '# Test' }), + }); + + const bubble = new OlostepBubble({ + operation: 'scrape', + url: 'https://example.com', + credentials: createTestCredentials(), + }); + + const result = await bubble.testCredential(); + expect(result).toBe(true); + }); + }); + + // + // SCRAPE OPERATION + // + describe('Scrape Operation', () => { + it('should create bubble with scrape operation', () => { + const params: OlostepParamsInput = { + operation: 'scrape', + url: 'https://example.com', + formats: ['markdown'], + }; + + const bubble = new OlostepBubble(params); + expect((bubble as any).params.operation).toBe('scrape'); + expect((bubble as any).params.url).toBe('https://example.com'); + }); + + it('should accept all scrape optional parameters', () => { + const params: OlostepParamsInput = { + operation: 'scrape', + url: 'https://example.com', + formats: ['markdown', 'html'], + country: 'US', + wait_before_scraping: 2000, + parser: '@olostep/product-page', + }; + + const bubble = new OlostepBubble(params); + expect((bubble as any).params.formats).toEqual(['markdown', 'html']); + expect((bubble as any).params.country).toBe('US'); + expect((bubble as any).params.wait_before_scraping).toBe(2000); + expect((bubble as any).params.parser).toBe('@olostep/product-page'); + }); + + it('should return success for scrape', async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + text: async () => + JSON.stringify({ + markdown_content: '# Test Page', + metadata: { title: 'Test', url: 'https://example.com' }, + }), + }); + + const bubble = new OlostepBubble({ + operation: 'scrape', + url: 'https://example.com', + credentials: createTestCredentials(), + }); + + const res = await bubble.action(); + + expect(res.data.operation).toBe('scrape'); + expect(res.success).toBe(true); + expect(res.error).toBe(''); + expect(res.data.markdown_content).toBeDefined(); + }); + }); + + // + // BATCH OPERATION + // + describe('Batch Operation', () => { + it('should create bubble with batch operation', () => { + const params: OlostepParamsInput = { + operation: 'batch', + urls: ['https://example.com/1', 'https://example.com/2'], + }; + + const bubble = new OlostepBubble(params); + expect((bubble as any).params.operation).toBe('batch'); + expect((bubble as any).params.urls).toHaveLength(2); + }); + + it('should return success for batch', async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + text: async () => + JSON.stringify({ + batch_id: 'batch_123', + status: 'processing', + }), + }); + + const bubble = new OlostepBubble({ + operation: 'batch', + urls: ['https://example.com/1', 'https://example.com/2'], + credentials: createTestCredentials(), + }); + + const res = await bubble.action(); + + expect(res.data.operation).toBe('batch'); + expect(res.success).toBe(true); + expect(res.data.batch_id).toBeDefined(); + }); + }); + + // + // CRAWL OPERATION + // + describe('Crawl Operation', () => { + it('should create bubble with crawl operation', () => { + const params: OlostepParamsInput = { + operation: 'crawl', + start_url: 'https://example.com', + max_pages: 50, + }; + + const bubble = new OlostepBubble(params); + expect((bubble as any).params.operation).toBe('crawl'); + expect((bubble as any).params.start_url).toBe('https://example.com'); + expect((bubble as any).params.max_pages).toBe(50); + }); + + it('should return success for crawl', async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + text: async () => + JSON.stringify({ + crawl_id: 'crawl_123', + status: 'completed', + pages_crawled: 10, + }), + }); + + const bubble = new OlostepBubble({ + operation: 'crawl', + start_url: 'https://example.com', + credentials: createTestCredentials(), + }); + + const res = await bubble.action(); + + expect(res.data.operation).toBe('crawl'); + expect(res.success).toBe(true); + expect(res.data.crawl_id).toBeDefined(); + }); + }); + + // + // MAP OPERATION + // + describe('Map Operation', () => { + it('should create bubble with map operation', () => { + const params: OlostepParamsInput = { + operation: 'map', + url: 'https://example.com', + top_n: 200, + }; + + const bubble = new OlostepBubble(params); + expect((bubble as any).params.operation).toBe('map'); + expect((bubble as any).params.url).toBe('https://example.com'); + expect((bubble as any).params.top_n).toBe(200); + }); + + it('should return success for map', async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + text: async () => + JSON.stringify({ + urls: ['https://example.com/page1', 'https://example.com/page2'], + total_urls: 2, + }), + }); + + const bubble = new OlostepBubble({ + operation: 'map', + url: 'https://example.com', + credentials: createTestCredentials(), + }); + + const res = await bubble.action(); + + expect(res.data.operation).toBe('map'); + expect(res.success).toBe(true); + expect(res.data.urls).toBeDefined(); + expect(res.data.urls!.length).toBeGreaterThan(0); + }); + }); + + // + // ANSWER OPERATION + // + describe('Answer Operation', () => { + it('should create bubble with answer operation', () => { + const params: OlostepParamsInput = { + operation: 'answer', + task: 'What is the main topic of this website?', + context_urls: ['https://example.com'], + }; + + const bubble = new OlostepBubble(params); + expect((bubble as any).params.operation).toBe('answer'); + expect((bubble as any).params.task).toContain('main topic'); + }); + + it('should return success for answer', async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + text: async () => + JSON.stringify({ + answer: 'This website is about example content.', + citations: [{ url: 'https://example.com', title: 'Example' }], + sources_used: 1, + }), + }); + + const bubble = new OlostepBubble({ + operation: 'answer', + task: 'What is this website about?', + credentials: createTestCredentials(), + }); + + const res = await bubble.action(); + + expect(res.data.operation).toBe('answer'); + expect(res.success).toBe(true); + expect(res.data.answer).toBeDefined(); + }); + }); +}); diff --git a/packages/bubble-core/src/bubbles/service-bubble/olostep.ts b/packages/bubble-core/src/bubbles/service-bubble/olostep.ts new file mode 100644 index 000000000..e83b8b692 --- /dev/null +++ b/packages/bubble-core/src/bubbles/service-bubble/olostep.ts @@ -0,0 +1,591 @@ +import { z } from 'zod'; +import { ServiceBubble } from '../../types/service-bubble-class.js'; +import type { BubbleContext } from '../../types/bubble.js'; +import { CredentialType, type BubbleName } from '@bubblelab/shared-schemas'; + +// Olostep API base URL +const OLOSTEP_API_URL = 'https://api.olostep.com/v1'; + +// Output format options +const FormatSchema = z.enum(['markdown', 'html', 'json', 'text']); + +// Define the parameters schema for the Olostep bubble +const OlostepParamsSchema = z.discriminatedUnion('operation', [ + // Scrape operation - extract content from a single URL + z.object({ + operation: z + .literal('scrape') + .describe('Extract content from a single URL'), + url: z.string().url().describe('The URL to scrape'), + formats: z + .array(FormatSchema) + .optional() + .default(['markdown']) + .describe('Output formats: markdown, html, json, text'), + country: z + .string() + .length(2) + .optional() + .describe('Two-letter country code for geo-targeting'), + wait_before_scraping: z + .number() + .int() + .min(0) + .max(30000) + .optional() + .describe('Milliseconds to wait before scraping'), + parser: z + .string() + .optional() + .describe( + 'Parser ID for structured extraction (e.g., @olostep/product-page)' + ), + credentials: z + .record(z.nativeEnum(CredentialType), z.string()) + .optional() + .describe('Credentials (injected at runtime)'), + }), + + // Batch scrape operation - scrape multiple URLs at once + z.object({ + operation: z + .literal('batch') + .describe('Scrape multiple URLs in a single request'), + urls: z + .array(z.string().url()) + .min(1) + .max(1000) + .describe('Array of URLs to scrape (max 1000)'), + formats: z + .array(FormatSchema) + .optional() + .default(['markdown']) + .describe('Output formats'), + country: z + .string() + .length(2) + .optional() + .describe('Two-letter country code'), + wait_before_scraping: z + .number() + .int() + .min(0) + .max(30000) + .optional() + .describe('Milliseconds to wait'), + parser: z + .string() + .optional() + .describe('Parser ID for structured extraction'), + credentials: z + .record(z.nativeEnum(CredentialType), z.string()) + .optional() + .describe('Credentials (injected at runtime)'), + }), + + // Crawl operation - crawl a website following links + z.object({ + operation: z + .literal('crawl') + .describe('Crawl a website and extract content from multiple pages'), + start_url: z.string().url().describe('Starting URL for the crawl'), + max_pages: z + .number() + .int() + .min(1) + .max(10000) + .optional() + .default(10) + .describe('Maximum number of pages to crawl'), + follow_links: z + .boolean() + .optional() + .default(true) + .describe('Whether to follow links'), + formats: z + .array(FormatSchema) + .optional() + .default(['markdown']) + .describe('Output formats'), + country: z + .string() + .length(2) + .optional() + .describe('Two-letter country code'), + parser: z + .string() + .optional() + .describe('Parser ID for structured extraction'), + credentials: z + .record(z.nativeEnum(CredentialType), z.string()) + .optional() + .describe('Credentials (injected at runtime)'), + }), + + // Map operation - discover URLs on a website + z.object({ + operation: z.literal('map').describe('Discover all URLs on a website'), + url: z.string().url().describe('Domain URL to map'), + search_query: z + .string() + .optional() + .describe('Optional search query to filter URLs'), + top_n: z + .number() + .int() + .min(1) + .max(10000) + .optional() + .default(100) + .describe('Maximum number of URLs to return'), + include_urls: z + .array(z.string()) + .optional() + .describe('URL patterns to include'), + exclude_urls: z + .array(z.string()) + .optional() + .describe('URL patterns to exclude'), + credentials: z + .record(z.nativeEnum(CredentialType), z.string()) + .optional() + .describe('Credentials (injected at runtime)'), + }), + + // Answer operation - AI-powered question answering + z.object({ + operation: z + .literal('answer') + .describe('Get AI-powered answers from web content'), + task: z.string().min(1).describe('The question or task to answer'), + context_urls: z + .array(z.string().url()) + .optional() + .describe('URLs to use as context for answering'), + format: z + .enum(['markdown', 'text', 'json']) + .optional() + .default('markdown') + .describe('Output format'), + include_citations: z + .boolean() + .optional() + .default(true) + .describe('Include source citations'), + top_k_sources: z + .number() + .int() + .min(1) + .max(20) + .optional() + .default(5) + .describe('Number of sources to consider'), + json_schema: z + .record(z.any()) + .optional() + .describe('JSON schema for structured output'), + credentials: z + .record(z.nativeEnum(CredentialType), z.string()) + .optional() + .describe('Credentials (injected at runtime)'), + }), +]); + +export type OlostepParamsInput = z.input; +type OlostepParams = z.output; + +// Result schemas for each operation +const ScrapeResultSchema = z.object({ + operation: z.literal('scrape'), + markdown_content: z.string().optional(), + html_content: z.string().optional(), + text_content: z.string().optional(), + json_content: z.any().optional(), + metadata: z + .object({ + title: z.string().optional(), + description: z.string().optional(), + url: z.string().optional(), + }) + .optional(), + success: z.boolean(), + error: z.string(), +}); + +const BatchResultSchema = z.object({ + operation: z.literal('batch'), + batch_id: z.string().optional(), + status: z.string().optional(), + items: z.array(z.any()).optional(), + success: z.boolean(), + error: z.string(), +}); + +const CrawlResultSchema = z.object({ + operation: z.literal('crawl'), + crawl_id: z.string().optional(), + status: z.string().optional(), + pages_crawled: z.number().optional(), + pages: z.array(z.any()).optional(), + success: z.boolean(), + error: z.string(), +}); + +const MapResultSchema = z.object({ + operation: z.literal('map'), + urls: z.array(z.string()).optional(), + total_urls: z.number().optional(), + success: z.boolean(), + error: z.string(), +}); + +const AnswerResultSchema = z.object({ + operation: z.literal('answer'), + answer: z.string().optional(), + citations: z + .array( + z.object({ + url: z.string(), + title: z.string().optional(), + snippet: z.string().optional(), + }) + ) + .optional(), + sources_used: z.number().optional(), + success: z.boolean(), + error: z.string(), +}); + +const OlostepResultSchema = z.discriminatedUnion('operation', [ + ScrapeResultSchema, + BatchResultSchema, + CrawlResultSchema, + MapResultSchema, + AnswerResultSchema, +]); + +type OlostepResult = z.output; + +export class OlostepBubble extends ServiceBubble { + static readonly service = 'olostep'; + static readonly authType = 'apikey' as const; + static readonly bubbleName: BubbleName = 'olostep' as BubbleName; + static readonly type = 'service' as const; + static readonly schema = OlostepParamsSchema; + static readonly resultSchema = OlostepResultSchema; + static readonly credentialOptions = [CredentialType.OLOSTEP_API_KEY]; + static readonly shortDescription = + 'Web scraping and AI-powered content extraction'; + static readonly longDescription = ` + Olostep is a powerful web scraping and AI-powered content extraction API. + + Features: + - **Scrape**: Extract content from any URL in markdown, HTML, JSON, or text format + - **Batch**: Scrape up to 1000 URLs in a single request + - **Crawl**: Crawl websites and extract content from multiple pages + - **Map**: Discover all URLs on a website for sitemap generation + - **Answer**: AI-powered question answering with web content as context + + Use cases: + - Content extraction and data collection + - Website monitoring and change detection + - Research and competitive analysis + - Lead generation and data enrichment + - Building AI agents with web access + - Automated content summarization + + Supported parsers for structured extraction: + - Twitter/X profiles and posts + - GitHub repositories and profiles + - Product pages, job listings, and more + `; + static readonly alias = 'web-scraper'; + + constructor( + params: OlostepParamsInput = { + operation: 'scrape', + url: 'https://example.com', + formats: ['markdown'], + }, + context?: BubbleContext + ) { + super(params, context); + } + + protected chooseCredential(): string | undefined { + return this.params.credentials?.[CredentialType.OLOSTEP_API_KEY]; + } + + public async testCredential(): Promise { + const apiKey = this.chooseCredential(); + if (!apiKey) return false; + + try { + // Simple health check with minimal scrape + const response = await fetch(`${OLOSTEP_API_URL}/scrapes`, { + method: 'POST', + headers: { + Authorization: `Bearer ${apiKey}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + url_to_scrape: 'https://example.com', + formats: ['text'], + }), + }); + return response.ok; + } catch { + return false; + } + } + + protected async performAction( + context?: BubbleContext + ): Promise { + const apiKey = this.chooseCredential(); + if (!apiKey) { + return this.createErrorResult('OLOSTEP_API_KEY credential is required'); + } + + const { operation } = this.params; + context?.logger?.info?.(`olostep.${operation}`); + + try { + switch (operation) { + case 'scrape': + return await this.performScrape(apiKey); + case 'batch': + return await this.performBatch(apiKey); + case 'crawl': + return await this.performCrawl(apiKey); + case 'map': + return await this.performMap(apiKey); + case 'answer': + return await this.performAnswer(apiKey); + default: + return this.createErrorResult(`Unknown operation: ${operation}`); + } + } catch (error) { + const message = error instanceof Error ? error.message : 'Unknown error'; + return this.createErrorResult(message); + } + } + + private async performScrape(apiKey: string): Promise { + if (this.params.operation !== 'scrape') { + return this.createErrorResult('Invalid operation'); + } + + const { url, formats, country, wait_before_scraping, parser } = this.params; + + const payload: Record = { + url_to_scrape: url, + formats: formats || ['markdown'], + }; + if (country) payload.country = country; + if (wait_before_scraping) + payload.wait_before_scraping = wait_before_scraping; + if (parser) payload.parser = { id: parser }; + + const response = await this.makeRequest('/scrapes', payload, apiKey); + + return { + operation: 'scrape', + markdown_content: response.markdown_content, + html_content: response.html_content, + text_content: response.text_content, + json_content: response.json_content, + metadata: response.metadata, + success: true, + error: '', + }; + } + + private async performBatch(apiKey: string): Promise { + if (this.params.operation !== 'batch') { + return this.createErrorResult('Invalid operation'); + } + + const { urls, formats, country, wait_before_scraping, parser } = + this.params; + + const items = urls.map((url, i) => ({ url, custom_id: `item_${i}` })); + + const payload: Record = { + items, + formats: formats || ['markdown'], + }; + if (country) payload.country = country; + if (wait_before_scraping) + payload.wait_before_scraping = wait_before_scraping; + if (parser) payload.parser = { id: parser }; + + const response = await this.makeRequest('/batches', payload, apiKey); + + return { + operation: 'batch', + batch_id: response.batch_id || response.id, + status: response.status, + items: response.items, + success: true, + error: '', + }; + } + + private async performCrawl(apiKey: string): Promise { + if (this.params.operation !== 'crawl') { + return this.createErrorResult('Invalid operation'); + } + + const { start_url, max_pages, follow_links, formats, country, parser } = + this.params; + + const payload: Record = { + start_url, + max_pages: max_pages || 10, + follow_links: follow_links ?? true, + formats: formats || ['markdown'], + }; + if (country) payload.country = country; + if (parser) payload.parser = { id: parser }; + + const response = await this.makeRequest('/crawls', payload, apiKey); + + return { + operation: 'crawl', + crawl_id: response.crawl_id || response.id, + status: response.status, + pages_crawled: response.pages_crawled, + pages: response.pages, + success: true, + error: '', + }; + } + + private async performMap(apiKey: string): Promise { + if (this.params.operation !== 'map') { + return this.createErrorResult('Invalid operation'); + } + + const { url, search_query, top_n, include_urls, exclude_urls } = + this.params; + + const payload: Record = { + url, + top_n: top_n || 100, + }; + if (search_query) payload.search_query = search_query; + if (include_urls) payload.include_urls = include_urls; + if (exclude_urls) payload.exclude_urls = exclude_urls; + + const response = await this.makeRequest('/maps', payload, apiKey); + + return { + operation: 'map', + urls: response.urls || response.links, + total_urls: response.total_urls || response.urls?.length, + success: true, + error: '', + }; + } + + private async performAnswer(apiKey: string): Promise { + if (this.params.operation !== 'answer') { + return this.createErrorResult('Invalid operation'); + } + + const { + task, + context_urls, + format, + include_citations, + top_k_sources, + json_schema, + } = this.params; + + const payload: Record = { + task, + format: format || 'markdown', + include_citations: include_citations ?? true, + top_k_sources: top_k_sources || 5, + }; + if (context_urls) payload.context_urls = context_urls; + if (json_schema) payload.json_schema = json_schema; + + const response = await this.makeRequest('/answers', payload, apiKey); + + return { + operation: 'answer', + answer: response.answer || response.result, + citations: response.citations, + sources_used: response.sources_used, + success: true, + error: '', + }; + } + + private async makeRequest( + endpoint: string, + payload: Record, + apiKey: string + ): Promise { + const response = await fetch(`${OLOSTEP_API_URL}${endpoint}`, { + method: 'POST', + headers: { + Authorization: `Bearer ${apiKey}`, + 'Content-Type': 'application/json', + Accept: 'application/json', + }, + body: JSON.stringify(payload), + }); + + const text = await response.text(); + let json: any; + try { + json = text ? JSON.parse(text) : {}; + } catch { + json = { raw: text }; + } + + if (!response.ok) { + throw new Error( + json?.error?.message || json?.message || `HTTP ${response.status}` + ); + } + + return json; + } + + private createErrorResult(error: string): OlostepResult { + const operation = this.params.operation; + const base = { success: false, error }; + + switch (operation) { + case 'scrape': + return { operation: 'scrape', ...base }; + case 'batch': + return { operation: 'batch', ...base }; + case 'crawl': + return { operation: 'crawl', ...base }; + case 'map': + return { operation: 'map', ...base }; + case 'answer': + return { operation: 'answer', ...base }; + default: + return { operation: 'scrape', ...base }; + } + } +} + +// Export types for external usage +export type OlostepScrapeParams = Extract< + OlostepParams, + { operation: 'scrape' } +>; +export type OlostepBatchParams = Extract; +export type OlostepCrawlParams = Extract; +export type OlostepMapParams = Extract; +export type OlostepAnswerParams = Extract< + OlostepParams, + { operation: 'answer' } +>; diff --git a/packages/bubble-core/src/index.ts b/packages/bubble-core/src/index.ts index 873e4df0c..ad64900be 100644 --- a/packages/bubble-core/src/index.ts +++ b/packages/bubble-core/src/index.ts @@ -191,6 +191,8 @@ export { } from './bubbles/service-bubble/s3/index.js'; export type { FirecrawlParamsInput } from './bubbles/service-bubble/firecrawl.js'; export { FirecrawlBubble } from './bubbles/service-bubble/firecrawl.js'; +export { OlostepBubble } from './bubbles/service-bubble/olostep.js'; +export type { OlostepParamsInput } from './bubbles/service-bubble/olostep.js'; export { InsForgeDbBubble } from './bubbles/service-bubble/insforge-db.js'; export type { InsForgeDbParamsInput } from './bubbles/service-bubble/insforge-db.js'; export { diff --git a/packages/bubble-shared-schemas/src/bubble-definition-schema.ts b/packages/bubble-shared-schemas/src/bubble-definition-schema.ts index 04fee0b43..e53bab566 100644 --- a/packages/bubble-shared-schemas/src/bubble-definition-schema.ts +++ b/packages/bubble-shared-schemas/src/bubble-definition-schema.ts @@ -50,6 +50,7 @@ export const CREDENTIAL_CONFIGURATION_MAP: Record< [CredentialType.NOTION_API]: {}, [CredentialType.INSFORGE_BASE_URL]: {}, [CredentialType.INSFORGE_API_KEY]: {}, + [CredentialType.OLOSTEP_API_KEY]: {}, [CredentialType.CUSTOM_AUTH_KEY]: {}, [CredentialType.AMAZON_CRED]: { proxyServer: BubbleParameterType.STRING, diff --git a/packages/bubble-shared-schemas/src/credential-schema.ts b/packages/bubble-shared-schemas/src/credential-schema.ts index 68b4aba9a..15e775f14 100644 --- a/packages/bubble-shared-schemas/src/credential-schema.ts +++ b/packages/bubble-shared-schemas/src/credential-schema.ts @@ -347,6 +347,13 @@ export const CREDENTIAL_TYPE_CONFIG: Record = namePlaceholder: 'My InsForge API Key', credentialConfigurations: {}, }, + [CredentialType.OLOSTEP_API_KEY]: { + label: 'Olostep', + description: 'API key for Olostep web scraping and AI content extraction', + placeholder: 'ols_...', + namePlaceholder: 'My Olostep API Key', + credentialConfigurations: {}, + }, [CredentialType.CRUSTDATA_API_KEY]: { label: 'Crustdata API Key', description: 'API key for your Crustdata backend', @@ -832,6 +839,7 @@ export const CREDENTIAL_ENV_MAP: Record = { [CredentialType.GRANOLA_API_KEY]: 'GRANOLA_API_KEY', [CredentialType.MEMBERFUL_CRED]: '', // Multi-field credential (subdomain + apiKey), no single env var [CredentialType.ZOOM_CRED]: '', // OAuth credential, no env var + [CredentialType.OLOSTEP_API_KEY]: 'OLOSTEP_API_KEY', [CredentialType.CREDENTIAL_WILDCARD]: '', // Wildcard marker, not a real credential }; @@ -3089,6 +3097,7 @@ export const BUBBLE_CREDENTIAL_OPTIONS: Record< memberful: [CredentialType.MEMBERFUL_CRED], luma: [], zoom: [CredentialType.ZOOM_CRED], + olostep: [CredentialType.OLOSTEP_API_KEY], }; export interface CredentialSiblingEntry { diff --git a/packages/bubble-shared-schemas/src/types.ts b/packages/bubble-shared-schemas/src/types.ts index 923ea41f1..13c082313 100644 --- a/packages/bubble-shared-schemas/src/types.ts +++ b/packages/bubble-shared-schemas/src/types.ts @@ -31,6 +31,7 @@ export enum CredentialType { S3_CRED = 'S3_CRED', // Scraping Credentials APIFY_CRED = 'APIFY_CRED', + OLOSTEP_API_KEY = 'OLOSTEP_API_KEY', // Voice Credentials ELEVENLABS_API_KEY = 'ELEVENLABS_API_KEY', @@ -236,4 +237,5 @@ export type BubbleName = | 'granola' | 'memberful' | 'luma' - | 'zoom'; + | 'zoom' + | 'olostep'; From dc0246e70781adcc33d10dbf9752b0a5c990ef0e Mon Sep 17 00:00:00 2001 From: Zeeshan Adil Date: Wed, 17 Dec 2025 11:56:20 +0500 Subject: [PATCH 2/6] Add error handling tests and improve exhaustive type checking --- .../bubbles/service-bubble/olostep.test.ts | 92 +++++++++++++++++++ .../src/bubbles/service-bubble/olostep.ts | 5 +- 2 files changed, 96 insertions(+), 1 deletion(-) diff --git a/packages/bubble-core/src/bubbles/service-bubble/olostep.test.ts b/packages/bubble-core/src/bubbles/service-bubble/olostep.test.ts index a08cca659..ac0da6b3f 100644 --- a/packages/bubble-core/src/bubbles/service-bubble/olostep.test.ts +++ b/packages/bubble-core/src/bubbles/service-bubble/olostep.test.ts @@ -351,4 +351,96 @@ describe('OlostepBubble', () => { expect(res.data.answer).toBeDefined(); }); }); + + // + // ERROR HANDLING + // + describe('Error Handling', () => { + it('should handle API request failures (non-2xx responses)', async () => { + mockFetch.mockResolvedValueOnce({ + ok: false, + status: 401, + text: async () => JSON.stringify({ error: 'Invalid API key' }), + }); + + const bubble = new OlostepBubble({ + operation: 'scrape', + url: 'https://example.com', + credentials: createTestCredentials(), + }); + + const res = await bubble.action(); + + expect(res.success).toBe(false); + expect(res.error).toBeDefined(); + expect(res.error).not.toBe(''); + }); + + it('should handle network errors (fetch throws exception)', async () => { + mockFetch.mockRejectedValueOnce( + new Error('Network error: Failed to fetch') + ); + + const bubble = new OlostepBubble({ + operation: 'scrape', + url: 'https://example.com', + credentials: createTestCredentials(), + }); + + const res = await bubble.action(); + + expect(res.success).toBe(false); + expect(res.error).toContain('Network error'); + }); + + it('should handle malformed response data gracefully', async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + text: async () => 'not valid json {{{', + }); + + const bubble = new OlostepBubble({ + operation: 'scrape', + url: 'https://example.com', + credentials: createTestCredentials(), + }); + + const res = await bubble.action(); + + // Implementation gracefully handles malformed JSON by returning raw text + expect(res.data.operation).toBe('scrape'); + }); + + it('should handle missing credentials gracefully', async () => { + const bubble = new OlostepBubble({ + operation: 'scrape', + url: 'https://example.com', + // No credentials provided + }); + + const res = await bubble.action(); + + expect(res.success).toBe(false); + expect(res.error).toContain('OLOSTEP_API_KEY'); + }); + + it('should handle rate limiting responses', async () => { + mockFetch.mockResolvedValueOnce({ + ok: false, + status: 429, + text: async () => JSON.stringify({ error: 'Rate limit exceeded' }), + }); + + const bubble = new OlostepBubble({ + operation: 'scrape', + url: 'https://example.com', + credentials: createTestCredentials(), + }); + + const res = await bubble.action(); + + expect(res.success).toBe(false); + expect(res.error).toBeDefined(); + }); + }); }); diff --git a/packages/bubble-core/src/bubbles/service-bubble/olostep.ts b/packages/bubble-core/src/bubbles/service-bubble/olostep.ts index e83b8b692..12bb42efc 100644 --- a/packages/bubble-core/src/bubbles/service-bubble/olostep.ts +++ b/packages/bubble-core/src/bubbles/service-bubble/olostep.ts @@ -571,8 +571,11 @@ export class OlostepBubble extends ServiceBubble { return { operation: 'map', ...base }; case 'answer': return { operation: 'answer', ...base }; - default: + default: { + // Exhaustive check: TypeScript will error if a case is missing + const _exhaustiveCheck: never = operation; return { operation: 'scrape', ...base }; + } } } } From 8e680b3dcb0fbccdf7c502c581631548a751b30a Mon Sep 17 00:00:00 2001 From: Zeeshan Adil Date: Mon, 13 Apr 2026 18:32:39 +0000 Subject: [PATCH 3/6] fix(olostep): make unreachable operation branch explicit --- packages/bubble-core/src/bubbles/service-bubble/olostep.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/bubble-core/src/bubbles/service-bubble/olostep.ts b/packages/bubble-core/src/bubbles/service-bubble/olostep.ts index 12bb42efc..f26669ddd 100644 --- a/packages/bubble-core/src/bubbles/service-bubble/olostep.ts +++ b/packages/bubble-core/src/bubbles/service-bubble/olostep.ts @@ -572,9 +572,8 @@ export class OlostepBubble extends ServiceBubble { case 'answer': return { operation: 'answer', ...base }; default: { - // Exhaustive check: TypeScript will error if a case is missing const _exhaustiveCheck: never = operation; - return { operation: 'scrape', ...base }; + throw new Error(`Unhandled Olostep operation: ${_exhaustiveCheck}`); } } } From 75afb7c0597d1184195743864aff41776f71557a Mon Sep 17 00:00:00 2001 From: Zeeshan Adil Date: Mon, 13 Apr 2026 18:44:46 +0000 Subject: [PATCH 4/6] fix(olostep): address BubbleLab review feedback --- .../bubbles/service-bubble/olostep.test.ts | 10 ++++++- .../src/bubbles/service-bubble/olostep.ts | 27 +++++++++++++++---- packages/bubble-core/src/index.ts | 9 ++++++- 3 files changed, 39 insertions(+), 7 deletions(-) diff --git a/packages/bubble-core/src/bubbles/service-bubble/olostep.test.ts b/packages/bubble-core/src/bubbles/service-bubble/olostep.test.ts index ac0da6b3f..1b2892895 100644 --- a/packages/bubble-core/src/bubbles/service-bubble/olostep.test.ts +++ b/packages/bubble-core/src/bubbles/service-bubble/olostep.test.ts @@ -1,4 +1,4 @@ -import { describe, it, expect, beforeAll, vi } from 'vitest'; +import { describe, it, expect, beforeAll, afterAll, afterEach, vi } from 'vitest'; import { OlostepBubble, type OlostepParamsInput } from './olostep.js'; import { CredentialType } from '@bubblelab/shared-schemas'; import { BubbleFactory } from '../../bubble-factory.js'; @@ -19,6 +19,14 @@ beforeAll(async () => { await factory.registerDefaults(); }); +afterEach(() => { + mockFetch.mockReset(); +}); + +afterAll(() => { + vi.unstubAllGlobals(); +}); + /** * Unit tests for Olostep Service Bubble * diff --git a/packages/bubble-core/src/bubbles/service-bubble/olostep.ts b/packages/bubble-core/src/bubbles/service-bubble/olostep.ts index f26669ddd..1ebf4fd2e 100644 --- a/packages/bubble-core/src/bubbles/service-bubble/olostep.ts +++ b/packages/bubble-core/src/bubbles/service-bubble/olostep.ts @@ -316,13 +316,28 @@ export class OlostepBubble extends ServiceBubble { return this.params.credentials?.[CredentialType.OLOSTEP_API_KEY]; } + private async fetchWithTimeout( + input: string, + init: RequestInit, + timeoutMs = 30000 + ): Promise { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), timeoutMs); + + try { + return await fetch(input, { ...init, signal: controller.signal }); + } finally { + clearTimeout(timeout); + } + } + public async testCredential(): Promise { const apiKey = this.chooseCredential(); if (!apiKey) return false; try { // Simple health check with minimal scrape - const response = await fetch(`${OLOSTEP_API_URL}/scrapes`, { + const response = await this.fetchWithTimeout(`${OLOSTEP_API_URL}/scrapes`, { method: 'POST', headers: { Authorization: `Bearer ${apiKey}`, @@ -529,7 +544,7 @@ export class OlostepBubble extends ServiceBubble { payload: Record, apiKey: string ): Promise { - const response = await fetch(`${OLOSTEP_API_URL}${endpoint}`, { + const response = await this.fetchWithTimeout(`${OLOSTEP_API_URL}${endpoint}`, { method: 'POST', headers: { Authorization: `Bearer ${apiKey}`, @@ -548,9 +563,11 @@ export class OlostepBubble extends ServiceBubble { } if (!response.ok) { - throw new Error( - json?.error?.message || json?.message || `HTTP ${response.status}` - ); + const errorMessage = + typeof json?.error === 'string' + ? json.error + : json?.error?.message || json?.message; + throw new Error(errorMessage || `HTTP ${response.status}`); } return json; diff --git a/packages/bubble-core/src/index.ts b/packages/bubble-core/src/index.ts index ad64900be..0da86ecfb 100644 --- a/packages/bubble-core/src/index.ts +++ b/packages/bubble-core/src/index.ts @@ -192,7 +192,14 @@ export { export type { FirecrawlParamsInput } from './bubbles/service-bubble/firecrawl.js'; export { FirecrawlBubble } from './bubbles/service-bubble/firecrawl.js'; export { OlostepBubble } from './bubbles/service-bubble/olostep.js'; -export type { OlostepParamsInput } from './bubbles/service-bubble/olostep.js'; +export type { + OlostepParamsInput, + OlostepScrapeParams, + OlostepBatchParams, + OlostepCrawlParams, + OlostepMapParams, + OlostepAnswerParams, +} from './bubbles/service-bubble/olostep.js'; export { InsForgeDbBubble } from './bubbles/service-bubble/insforge-db.js'; export type { InsForgeDbParamsInput } from './bubbles/service-bubble/insforge-db.js'; export { From 8f41cc7525b18e2164eda1c5b0f2a15566f1827f Mon Sep 17 00:00:00 2001 From: Zeeshan Adil Date: Mon, 13 Apr 2026 18:45:23 +0000 Subject: [PATCH 5/6] fix(olostep): preserve explicit zero wait time --- packages/bubble-core/src/bubbles/service-bubble/olostep.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/bubble-core/src/bubbles/service-bubble/olostep.ts b/packages/bubble-core/src/bubbles/service-bubble/olostep.ts index 1ebf4fd2e..820f57197 100644 --- a/packages/bubble-core/src/bubbles/service-bubble/olostep.ts +++ b/packages/bubble-core/src/bubbles/service-bubble/olostep.ts @@ -398,7 +398,7 @@ export class OlostepBubble extends ServiceBubble { formats: formats || ['markdown'], }; if (country) payload.country = country; - if (wait_before_scraping) + if (wait_before_scraping !== undefined) payload.wait_before_scraping = wait_before_scraping; if (parser) payload.parser = { id: parser }; @@ -431,7 +431,7 @@ export class OlostepBubble extends ServiceBubble { formats: formats || ['markdown'], }; if (country) payload.country = country; - if (wait_before_scraping) + if (wait_before_scraping !== undefined) payload.wait_before_scraping = wait_before_scraping; if (parser) payload.parser = { id: parser }; From fe2ebb0b785215d08538dd75b6f68849c9a6cdaa Mon Sep 17 00:00:00 2001 From: Zeeshan Adil Date: Sat, 2 May 2026 19:19:18 +0000 Subject: [PATCH 6/6] fix(olostep): preserve plain-text error bodies and strengthen malformed JSON test - Include json.raw fallback in makeRequest error extraction so non-JSON error responses from the API are surfaced rather than falling back to the generic HTTP status message - Strengthen malformed-JSON test to assert res.success is true (not just res.data.operation) so regressions in the graceful fallback path are caught --- .../bubble-core/src/bubbles/service-bubble/olostep.test.ts | 3 ++- packages/bubble-core/src/bubbles/service-bubble/olostep.ts | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/bubble-core/src/bubbles/service-bubble/olostep.test.ts b/packages/bubble-core/src/bubbles/service-bubble/olostep.test.ts index 1b2892895..e1ec581fc 100644 --- a/packages/bubble-core/src/bubbles/service-bubble/olostep.test.ts +++ b/packages/bubble-core/src/bubbles/service-bubble/olostep.test.ts @@ -415,8 +415,9 @@ describe('OlostepBubble', () => { const res = await bubble.action(); - // Implementation gracefully handles malformed JSON by returning raw text + // Graceful fallback: parses raw text, returns success with raw content expect(res.data.operation).toBe('scrape'); + expect(res.success).toBe(true); }); it('should handle missing credentials gracefully', async () => { diff --git a/packages/bubble-core/src/bubbles/service-bubble/olostep.ts b/packages/bubble-core/src/bubbles/service-bubble/olostep.ts index 820f57197..fd4f39ab0 100644 --- a/packages/bubble-core/src/bubbles/service-bubble/olostep.ts +++ b/packages/bubble-core/src/bubbles/service-bubble/olostep.ts @@ -566,7 +566,9 @@ export class OlostepBubble extends ServiceBubble { const errorMessage = typeof json?.error === 'string' ? json.error - : json?.error?.message || json?.message; + : json?.error?.message || + json?.message || + (typeof json?.raw === 'string' ? json.raw : undefined); throw new Error(errorMessage || `HTTP ${response.status}`); }