From c3934bb75bfa9dd459214b77aed60dea608fcbd7 Mon Sep 17 00:00:00 2001 From: AIWintermuteAI Date: Sat, 18 Jul 2026 00:18:05 +0200 Subject: [PATCH 1/5] feat: add POST /extract/batch endpoint for WebUI integration --- core/server.go | 1 + core/server_extract.go | 130 +++++++++++++++++++++++++++++++ core/server_extract_test.go | 148 ++++++++++++++++++++++++++++++++++++ 3 files changed, 279 insertions(+) diff --git a/core/server.go b/core/server.go index 6bcd95e..9fa2c7c 100644 --- a/core/server.go +++ b/core/server.go @@ -235,6 +235,7 @@ func NewServerWithOptions(host string, port int, opts ServerOptions, searchEngin serv.app.Get("/mega/engines", serv.handleListEngines) serv.app.Get("/extract", serv.handleExtract) serv.app.Post("/extract", serv.handleExtract) + serv.app.Post("/extract/batch", serv.handleBatchExtract) return &serv } diff --git a/core/server_extract.go b/core/server_extract.go index a3ef971..c057d39 100644 --- a/core/server_extract.go +++ b/core/server_extract.go @@ -418,3 +418,133 @@ func SanitizeExtractError(err error) string { } return msg } + +const maxBatchExtractURLs = 20 + +type batchExtractPayload struct { + URLs []string `json:"urls"` + Mode string `json:"mode"` +} + +// batchExtractItem is the WebUI-compatible response item per URL. +type batchExtractItem struct { + PageContent string `json:"page_content"` + Metadata map[string]string `json:"metadata"` +} + +func (s *Server) handleBatchExtract(c *fiber.Ctx) error { + startedAt := time.Now() + requestCtx := withRequestUsage(c.UserContext(), "extract-batch") + c.SetUserContext(requestCtx) + defer setNetworkBytesHeader(c, requestCtx) + defer setBrowserProfileHeader(c, requestCtx) + + cfg := s.opts.Extract.Normalized() + if !cfg.Enabled { + return &APIError{HTTPStatus: fiber.StatusNotFound, ErrorCode: "not_found", Message: "Extraction is disabled"} + } + + var body batchExtractPayload + if len(c.Body()) == 0 { + return &APIError{HTTPStatus: fiber.StatusBadRequest, ErrorCode: "invalid_request", Message: "request body is required"} + } + if err := c.BodyParser(&body); err != nil { + return &APIError{HTTPStatus: fiber.StatusBadRequest, ErrorCode: "invalid_request", Message: "invalid JSON body"} + } + + // Deduplicate and normalize URLs. + seen := make(map[string]struct{}, len(body.URLs)) + var urls []string + for _, raw := range body.URLs { + u := extractpkg.NormalizeURL(strings.TrimSpace(raw)) + if u == "" { + continue + } + if _, dup := seen[u]; dup { + continue + } + seen[u] = struct{}{} + urls = append(urls, u) + } + if len(urls) == 0 { + return &APIError{HTTPStatus: fiber.StatusBadRequest, ErrorCode: "invalid_request", Message: "urls array is required and must contain at least one valid URL"} + } + if len(urls) > maxBatchExtractURLs { + return &APIError{ + HTTPStatus: fiber.StatusBadRequest, + ErrorCode: "invalid_request", + Message: fmt.Sprintf("urls array exceeds maximum of %d", maxBatchExtractURLs), + } + } + + // Validate all URLs upfront. + for _, u := range urls { + if err := validateExtractTargetURL(c.UserContext(), u, cfg.AllowPrivateNetworks); err != nil { + return &APIError{HTTPStatus: fiber.StatusBadRequest, ErrorCode: "invalid_extract_url", Message: err.Error()} + } + } + + mode := firstNonEmpty(body.Mode, cfg.DefaultMode) + extractor := s.newExtractor() + results := make([]batchExtractItem, len(urls)) + + // Concurrent extraction with bounded parallelism (same pattern as + // EnrichEnvelopeWithExtraction). + ctx, cancel := context.WithTimeout(c.UserContext(), cfg.BatchTimeout(len(urls))) + defer cancel() + + sem := make(chan struct{}, cfg.MaxConcurrent) + var wg sync.WaitGroup + for i, u := range urls { + wg.Add(1) + sem <- struct{}{} + go func(idx int, url string) { + defer wg.Done() + defer func() { <-sem }() + + if err := ctx.Err(); err != nil { + results[idx] = batchExtractItem{ + PageContent: "", + Metadata: map[string]string{"source": url, "error": "batch timeout"}, + } + return + } + + req := extractpkg.ExtractRequest{ + URL: url, + Mode: extractpkg.Mode(mode), + ProxyURL: "", + LangCode: strings.TrimSpace(c.Query("lang")), + Timeout: cfg.Timeout, + MaxBytes: cfg.MaxBytes, + } + result, err := extractor.Extract(ctx, req) + if err != nil { + results[idx] = batchExtractItem{ + PageContent: "", + Metadata: map[string]string{ + "source": url, + "error": SanitizeExtractError(err), + }, + } + return + } + results[idx] = batchExtractItem{ + PageContent: result.Markdown, + Metadata: map[string]string{ + "source": url, + "title": result.Title, + "description": result.Description, + "lang": result.Lang, + "canonical": result.Canonical, + "mode_used": result.Meta.ModeUsed, + "fetched_at": result.Meta.FetchedAt, + }, + } + }(i, u) + } + wg.Wait() + + _ = startedAt + return c.JSON(results) +} diff --git a/core/server_extract_test.go b/core/server_extract_test.go index 3ac26f0..d8eaaa3 100644 --- a/core/server_extract_test.go +++ b/core/server_extract_test.go @@ -2,8 +2,11 @@ package core import ( "context" + "encoding/json" + "fmt" "net/http" "net/http/httptest" + "reflect" "strings" "testing" "time" @@ -109,3 +112,148 @@ func TestValidateExtractTargetURLNormalizesBarePublicIP(t *testing.T) { t.Fatalf("expected bare public IP target to validate after scheme normalization: %v", err) } } + +func TestBatchExtractReturnsWebUIFormat(t *testing.T) { + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`

Test Page

This is a test page with enough content to pass the minimum runes threshold for extraction in batch mode.

`)) + })) + defer target.Close() + + opts := DefaultServerOptions() + opts.Extract = extractpkg.Config{ + Enabled: true, + DefaultMode: string(extractpkg.ModeFast), + Timeout: time.Second, + MaxBytes: 256 * 1024, + MaxConcurrent: 2, + AllowPrivateNetworks: true, + } + s := NewServerWithOptions("127.0.0.1", 0, opts) + + body := fmt.Sprintf(`{"urls":["%s"]}`, target.URL) + req, err := http.NewRequest(http.MethodPost, "/extract/batch", strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Content-Type", "application/json") + + resp, err := s.app.Test(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusOK) + } + + var results []map[string]interface{} + if err := json.NewDecoder(resp.Body).Decode(&results); err != nil { + t.Fatalf("decode response: %v", err) + } + if len(results) != 1 { + t.Fatalf("results count = %d, want 1", len(results)) + } + if _, ok := results[0]["page_content"]; !ok { + t.Fatalf("expected page_content key, got keys: %v", reflect.ValueOf(results[0]).MapKeys()) + } + if _, ok := results[0]["metadata"]; !ok { + t.Fatalf("expected metadata key, got keys: %v", reflect.ValueOf(results[0]).MapKeys()) + } +} + +func TestBatchExtractHandlesMultipleURLs(t *testing.T) { + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`

Multi

Page with sufficient content for batch extraction test that verifies concurrent processing works correctly.

`)) + })) + defer target.Close() + + opts := DefaultServerOptions() + opts.Extract = extractpkg.Config{ + Enabled: true, + DefaultMode: string(extractpkg.ModeFast), + Timeout: time.Second, + MaxBytes: 256 * 1024, + MaxConcurrent: 2, + AllowPrivateNetworks: true, + } + s := NewServerWithOptions("127.0.0.1", 0, opts) + + body := fmt.Sprintf(`{"urls":["%s/1","%s/2","%s/3"]}`, target.URL, target.URL, target.URL) + req, err := http.NewRequest(http.MethodPost, "/extract/batch", strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Content-Type", "application/json") + + resp, err := s.app.Test(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusOK) + } + + var results []map[string]interface{} + if err := json.NewDecoder(resp.Body).Decode(&results); err != nil { + t.Fatalf("decode response: %v", err) + } + if len(results) != 3 { + t.Fatalf("results count = %d, want 3", len(results)) + } +} + +func TestBatchExtractRejectsEmptyURLs(t *testing.T) { + opts := DefaultServerOptions() + opts.Extract = extractpkg.DefaultConfig() + s := NewServerWithOptions("127.0.0.1", 0, opts) + + req, err := http.NewRequest(http.MethodPost, "/extract/batch", strings.NewReader(`{"urls":[]}`)) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Content-Type", "application/json") + + resp, err := s.app.Test(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusBadRequest) + } +} + +func TestBatchExtractRejectsURLsOverLimit(t *testing.T) { + opts := DefaultServerOptions() + opts.Extract = extractpkg.DefaultConfig() + s := NewServerWithOptions("127.0.0.1", 0, opts) + + // Build 21 URLs (limit is 20) + urls := make([]string, 21) + for i := 0; i < 21; i++ { + urls[i] = fmt.Sprintf("https://example.com/%d", i) + } + body, _ := json.Marshal(map[string][]string{"urls": urls}) + + req, err := http.NewRequest(http.MethodPost, "/extract/batch", strings.NewReader(string(body))) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Content-Type", "application/json") + + resp, err := s.app.Test(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusBadRequest) + } +} From 58fdf2b7a67addd2c1f20b3baf8baf133d43498d Mon Sep 17 00:00:00 2001 From: AIWintermuteAI Date: Sat, 18 Jul 2026 00:19:15 +0200 Subject: [PATCH 2/5] docs: add /extract/batch endpoint to OpenAPI spec --- docs/openapi.yaml | 59 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/docs/openapi.yaml b/docs/openapi.yaml index 6fb6449..703f807 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -420,6 +420,8 @@ paths: $ref: "#/components/headers/XRequestID" X-Cache: $ref: "#/components/headers/XCache" + X-Fallback-Engine: + $ref: "#/components/headers/XFallbackEngine" X-Proxy-Mode: $ref: "#/components/headers/XProxyMode" X-Proxy-Tag: @@ -547,6 +549,35 @@ paths: $ref: "#/components/responses/BadRequestError" "502": $ref: "#/components/responses/BadGatewayError" + + /extract/batch: + post: + tags: [Extract] + operationId: extractBatch + summary: Extract content from multiple URLs (WebUI-compatible) + description: > + Accepts an array of URLs and returns extracted page content in the + format expected by Open WebUI's ExternalWebLoader. Each result + contains `page_content` (markdown) and `metadata` (title, source, + lang, etc.). + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/BatchExtractRequest" + responses: + "200": + description: Batch extraction results + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/BatchExtractItem" + "400": + $ref: "#/components/responses/BadRequestError" + /health: get: tags: [Health] @@ -2037,3 +2068,31 @@ components: $ref: "#/components/schemas/MegaEngineInfo" total: type: integer + + # ── Batch extract (WebUI) ──────────────────────────────────────── + BatchExtractRequest: + type: object + required: [urls] + properties: + urls: + type: array + items: + type: string + maxItems: 20 + description: URLs to extract content from (max 20) + mode: + type: string + enum: [auto, fast, rendered] + description: Extraction mode (default: auto) + + BatchExtractItem: + type: object + properties: + page_content: + type: string + description: Extracted markdown content + metadata: + type: object + additionalProperties: + type: string + description: Page metadata (title, source, lang, etc.) From aec0d9c42a685a02027a52985768bc7b1d8b61af Mon Sep 17 00:00:00 2001 From: AIWintermuteAI Date: Sat, 18 Jul 2026 09:49:38 +0200 Subject: [PATCH 3/5] cleanup --- core/server_extract.go | 5 ++--- core/server_extract_test.go | 2 +- docs/openapi.yaml | 13 +++++-------- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/core/server_extract.go b/core/server_extract.go index c057d39..9f77dd3 100644 --- a/core/server_extract.go +++ b/core/server_extract.go @@ -426,14 +426,14 @@ type batchExtractPayload struct { Mode string `json:"mode"` } -// batchExtractItem is the WebUI-compatible response item per URL. +// batchExtractItem is a response item for a single extracted URL, using +// page_content/metadata keys type batchExtractItem struct { PageContent string `json:"page_content"` Metadata map[string]string `json:"metadata"` } func (s *Server) handleBatchExtract(c *fiber.Ctx) error { - startedAt := time.Now() requestCtx := withRequestUsage(c.UserContext(), "extract-batch") c.SetUserContext(requestCtx) defer setNetworkBytesHeader(c, requestCtx) @@ -545,6 +545,5 @@ func (s *Server) handleBatchExtract(c *fiber.Ctx) error { } wg.Wait() - _ = startedAt return c.JSON(results) } diff --git a/core/server_extract_test.go b/core/server_extract_test.go index d8eaaa3..8baad7f 100644 --- a/core/server_extract_test.go +++ b/core/server_extract_test.go @@ -113,7 +113,7 @@ func TestValidateExtractTargetURLNormalizesBarePublicIP(t *testing.T) { } } -func TestBatchExtractReturnsWebUIFormat(t *testing.T) { +func TestBatchExtractSingleURL(t *testing.T) { target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte(`

Test Page

This is a test page with enough content to pass the minimum runes threshold for extraction in batch mode.

`)) diff --git a/docs/openapi.yaml b/docs/openapi.yaml index 703f807..4998eec 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -420,8 +420,6 @@ paths: $ref: "#/components/headers/XRequestID" X-Cache: $ref: "#/components/headers/XCache" - X-Fallback-Engine: - $ref: "#/components/headers/XFallbackEngine" X-Proxy-Mode: $ref: "#/components/headers/XProxyMode" X-Proxy-Tag: @@ -554,12 +552,11 @@ paths: post: tags: [Extract] operationId: extractBatch - summary: Extract content from multiple URLs (WebUI-compatible) + summary: Extract content from multiple URLs description: > - Accepts an array of URLs and returns extracted page content in the - format expected by Open WebUI's ExternalWebLoader. Each result - contains `page_content` (markdown) and `metadata` (title, source, - lang, etc.). + Accepts an array of URLs and returns extracted page content for each + one. Each result contains `page_content` (markdown) and `metadata` + (title, source, lang, etc.). requestBody: required: true content: @@ -2069,7 +2066,7 @@ components: total: type: integer - # ── Batch extract (WebUI) ──────────────────────────────────────── + # ── Batch extract ───────────────────────────────────────────────── BatchExtractRequest: type: object required: [urls] From e430584324dec43eafbdc007a2a3772dc31b34c1 Mon Sep 17 00:00:00 2001 From: AIWintermuteAI Date: Sat, 18 Jul 2026 09:56:20 +0200 Subject: [PATCH 4/5] refactor: use errInvalidParam helper and add per-URL took_ms to batch extract --- core/server_extract.go | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/core/server_extract.go b/core/server_extract.go index 9f77dd3..ac30944 100644 --- a/core/server_extract.go +++ b/core/server_extract.go @@ -446,10 +446,10 @@ func (s *Server) handleBatchExtract(c *fiber.Ctx) error { var body batchExtractPayload if len(c.Body()) == 0 { - return &APIError{HTTPStatus: fiber.StatusBadRequest, ErrorCode: "invalid_request", Message: "request body is required"} + return errInvalidParam("request body is required") } if err := c.BodyParser(&body); err != nil { - return &APIError{HTTPStatus: fiber.StatusBadRequest, ErrorCode: "invalid_request", Message: "invalid JSON body"} + return errInvalidParam("invalid JSON body") } // Deduplicate and normalize URLs. @@ -467,14 +467,10 @@ func (s *Server) handleBatchExtract(c *fiber.Ctx) error { urls = append(urls, u) } if len(urls) == 0 { - return &APIError{HTTPStatus: fiber.StatusBadRequest, ErrorCode: "invalid_request", Message: "urls array is required and must contain at least one valid URL"} + return errInvalidParam("urls array is required and must contain at least one valid URL") } if len(urls) > maxBatchExtractURLs { - return &APIError{ - HTTPStatus: fiber.StatusBadRequest, - ErrorCode: "invalid_request", - Message: fmt.Sprintf("urls array exceeds maximum of %d", maxBatchExtractURLs), - } + return errInvalidParam(fmt.Sprintf("urls array exceeds maximum of %d", maxBatchExtractURLs)) } // Validate all URLs upfront. @@ -539,6 +535,7 @@ func (s *Server) handleBatchExtract(c *fiber.Ctx) error { "canonical": result.Canonical, "mode_used": result.Meta.ModeUsed, "fetched_at": result.Meta.FetchedAt, + "took_ms": fmt.Sprintf("%d", result.Meta.TookMs), }, } }(i, u) From 00edd86c1cc9235207474113cd72a0dcd9e775eb Mon Sep 17 00:00:00 2001 From: Rustem Kamalov Date: Sat, 18 Jul 2026 21:31:46 +0300 Subject: [PATCH 5/5] fix: openapi yaml parse error + /extract/batch timeout exemption --- core/middleware.go | 4 ++-- core/server_timeout_test.go | 7 +++++-- docs/openapi.yaml | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/core/middleware.go b/core/middleware.go index dd407c2..0019ef1 100644 --- a/core/middleware.go +++ b/core/middleware.go @@ -65,10 +65,10 @@ func RequestContextMiddleware() fiber.Handler { // RequestTimeoutMiddleware bounds wall-clock time per request by attaching a // deadline to the user context, which fasthttp never cancels on client -// disconnect. /mega/* (MegaTimeout) and /extract (batch budget) are exempt. +// disconnect. /mega/* (MegaTimeout) and /extract* (batch budget) are exempt. func RequestTimeoutMiddleware(timeout time.Duration) fiber.Handler { return func(c *fiber.Ctx) error { - if strings.HasPrefix(c.Path(), "/mega/") || c.Path() == "/extract" { + if strings.HasPrefix(c.Path(), "/mega/") || c.Path() == "/extract" || c.Path() == "/extract/batch" { return c.Next() } ctx, cancel := context.WithTimeout(c.UserContext(), timeout) diff --git a/core/server_timeout_test.go b/core/server_timeout_test.go index 709103c..a815768 100644 --- a/core/server_timeout_test.go +++ b/core/server_timeout_test.go @@ -12,8 +12,8 @@ import ( ) // FP-2: every endpoint that doesn't manage its own deadline budget must get -// one from RequestTimeoutMiddleware; /mega/* (MegaTimeout) and /extract -// (batch budget) keep theirs. +// one from RequestTimeoutMiddleware; /mega/* (MegaTimeout) and /extract, +// /extract/batch (batch budget) keep theirs. func TestRequestTimeoutMiddlewareSetsDeadlineExceptBudgetedPaths(t *testing.T) { app := fiber.New() app.Use(RequestTimeoutMiddleware(time.Minute)) @@ -32,12 +32,14 @@ func TestRequestTimeoutMiddlewareSetsDeadlineExceptBudgetedPaths(t *testing.T) { app.Post("/google/parse", record("/google/parse")) app.Get("/mega/search", record("/mega/search")) app.Get("/extract", record("/extract")) + app.Post("/extract/batch", record("/extract/batch")) for path, method := range map[string]string{ "/google/search": http.MethodGet, "/google/parse": http.MethodPost, "/mega/search": http.MethodGet, "/extract": http.MethodGet, + "/extract/batch": http.MethodPost, } { req := httptest.NewRequest(method, path, nil) resp, err := app.Test(req, -1) @@ -54,6 +56,7 @@ func TestRequestTimeoutMiddlewareSetsDeadlineExceptBudgetedPaths(t *testing.T) { "/google/parse": true, "/mega/search": false, "/extract": false, + "/extract/batch": false, } { if deadlines[path] != want { t.Errorf("%s: deadline attached = %v, want %v", path, deadlines[path], want) diff --git a/docs/openapi.yaml b/docs/openapi.yaml index 4998eec..9e57353 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -2080,7 +2080,7 @@ components: mode: type: string enum: [auto, fast, rendered] - description: Extraction mode (default: auto) + description: "Extraction mode (default: auto)" BatchExtractItem: type: object