From 0622b1349c909887be995b50d84e3512dcc9bd49 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 25 Aug 2026 23:27:31 +0000 Subject: [PATCH 1/3] Fix silent dropping of oversized attachments; add slow-parse TRACK log Oversized attachments (>10MB) and attachments whose S3 upload failed were silently discarded in the content sanitizer with no log line and no trace in the response. Now they're collected as droppedAttachments (filename, mimeType, sizeBytes, reason), returned in the sanitizer response, logged as a TRACK from the sanitizer, and surfaced as a WARN from the processor when present. Also add a TRACK log when MIME parsing itself takes longer than 10s, as an earlier signal than the existing 50s near-timeout alert. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01V85rptvcLi67TUmaqU1s53 --- src/isolated/content-sanitizer.ts | 46 ++++++- src/processor/content-sanitizer-client.ts | 8 ++ src/processor/processor.ts | 9 ++ .../content-sanitizer-attachments.spec.ts | 123 ++++++++++++++++++ 4 files changed, 182 insertions(+), 4 deletions(-) create mode 100644 tests/isolated/content-sanitizer-attachments.spec.ts diff --git a/src/isolated/content-sanitizer.ts b/src/isolated/content-sanitizer.ts index 2013e634..855de3fa 100644 --- a/src/isolated/content-sanitizer.ts +++ b/src/isolated/content-sanitizer.ts @@ -9,6 +9,11 @@ import type { Logger } from "../logger.js"; // actually failing — mirrors the api.slow_request pattern in src/api/app.ts. const SLOW_INVOCATION_THRESHOLD_MS = 50_000; +// Lower-bar threshold so a message that's merely slow to parse (large/complex MIME, +// many attachments) shows up before it gets anywhere near the 50s near-timeout alert — +// gives us an early signal to look at parsing performance, not just imminent failures. +const SLOW_PARSE_THRESHOLD_MS = 10_000; + // --------------------------------------------------------------------------- // Types // --------------------------------------------------------------------------- @@ -25,6 +30,13 @@ interface AttachmentRef { s3Key: string; } +interface DroppedAttachment { + filename: string; + mimeType: string; + sizeBytes: number; + reason: "too_large" | "upload_failed"; +} + interface ContentSanitizeRequest { presignedGetUrl: string; presignedPost: { @@ -58,6 +70,7 @@ interface ContentSanitizeResponse { sentAt?: string; assets?: ExtractedAsset[]; links?: ExtractedLink[]; + droppedAttachments?: DroppedAttachment[]; }; } @@ -214,10 +227,15 @@ async function processEmail(event: ContentSanitizeRequest, logger?: Logger): Pro // 2. Parse with mailparser logger?.trackPoint("mime_parse_start"); + const parseStart = Date.now(); let parsed; try { parsed = await simpleParser(rawMime); + const parseElapsedMs = Date.now() - parseStart; logger?.trackPoint("mime_parse_complete"); + if (parseElapsedMs > SLOW_PARSE_THRESHOLD_MS) { + logger?.track("MIME parse exceeded 10s.", { code: "content_sanitizer.slow_parse", elapsedMs: parseElapsedMs, sizeBytes: rawMime.length, accountId: event.accountId }); + } } catch (e) { return { success: false, @@ -258,14 +276,21 @@ async function processEmail(event: ContentSanitizeRequest, logger?: Logger): Pro const cidMap: Record = {}; const inlineImages: Array<{ mimeType: string; content: Buffer }> = []; const attachmentsWithBytes: Array<{ filename: string; mimeType: string; content: Buffer; s3Key?: string }> = []; + const droppedAttachments: DroppedAttachment[] = []; for (const attachment of attachments) { + const contentType = attachment.contentType || "application/octet-stream"; + if (attachment.size > MAX_SINGLE_ATTACHMENT_SIZE) { + droppedAttachments.push({ + filename: attachment.filename ?? `attachment-${uploadIndex}`, + mimeType: contentType, + sizeBytes: attachment.size, + reason: "too_large", + }); continue; } - const contentType = attachment.contentType || "application/octet-stream"; - if (attachment.contentId) { // Inline image — embed as data URI, no S3 upload needed cidMap[attachment.contentId] = `data:${contentType};base64,${attachment.content.toString("base64")}`; @@ -278,15 +303,25 @@ async function processEmail(event: ContentSanitizeRequest, logger?: Logger): Pro const s3Key = `${event.keyPrefix}${uploadIndex}`; const uploaded = await uploadViaPresignedPost(event.presignedPost, s3Key, attachment.content, contentType, event.retentionTag); + const filename = attachment.filename ?? `attachment-${uploadIndex}`; if (uploaded) { - const filename = attachment.filename ?? `attachment-${uploadIndex}`; attachmentRefs.push({ filename, mimeType: contentType, sizeBytes: attachment.size, s3Key }); attachmentsWithBytes.push({ filename, mimeType: contentType, content: attachment.content, s3Key }); + } else { + droppedAttachments.push({ filename, mimeType: contentType, sizeBytes: attachment.size, reason: "upload_failed" }); } uploadIndex++; } - logger?.trackPoint("attachments_processed", { attachmentRefCount: attachmentRefs.length, inlineImageCount: inlineImages.length }); + logger?.trackPoint("attachments_processed", { attachmentRefCount: attachmentRefs.length, inlineImageCount: inlineImages.length, droppedCount: droppedAttachments.length }); + if (droppedAttachments.length > 0) { + logger?.track("Attachment(s) dropped from message.", { + code: "content_sanitizer.attachments_dropped", + accountId: event.accountId, + droppedCount: droppedAttachments.length, + dropped: droppedAttachments.map(d => ({ mimeType: d.mimeType, sizeBytes: d.sizeBytes, reason: d.reason })), + }); + } // 6. Sanitize HTML and inline CID images let htmlBody: string | undefined; @@ -402,6 +437,9 @@ async function processEmail(event: ContentSanitizeRequest, logger?: Logger): Pro if (extractedLinks.length > 0) { result.parsed.links = extractedLinks; } + if (droppedAttachments.length > 0) { + result.parsed.droppedAttachments = droppedAttachments; + } logger?.trackPoint("response_built"); return result; diff --git a/src/processor/content-sanitizer-client.ts b/src/processor/content-sanitizer-client.ts index 7f42f9ee..274baa50 100644 --- a/src/processor/content-sanitizer-client.ts +++ b/src/processor/content-sanitizer-client.ts @@ -20,6 +20,13 @@ interface AttachmentRef { s3Key: string; } +export interface DroppedAttachment { + filename: string; + mimeType: string; + sizeBytes: number; + reason: "too_large" | "upload_failed"; +} + export interface ContentSanitizeRequest { presignedGetUrl: string; presignedPost: { @@ -59,6 +66,7 @@ export interface ContentSanitizeResponse { sentAt?: string; assets?: ExtractedAsset[]; links?: ExtractedLink[]; + droppedAttachments?: DroppedAttachment[]; }; } diff --git a/src/processor/processor.ts b/src/processor/processor.ts index 0948e9bb..380c0cda 100644 --- a/src/processor/processor.ts +++ b/src/processor/processor.ts @@ -887,6 +887,15 @@ export class SignalProcessor { const { parsed: sanitizedParsed } = sanitizeResult.value; const sanitizerAssets = sanitizedParsed.assets ?? []; + if (sanitizedParsed.droppedAttachments && sanitizedParsed.droppedAttachments.length > 0) { + this.logger.warn("Message had attachment(s) dropped by content sanitizer", { + code: "processor.attachments_dropped", + accountId, + droppedCount: sanitizedParsed.droppedAttachments.length, + dropped: sanitizedParsed.droppedAttachments.map(d => ({ mimeType: d.mimeType, sizeBytes: d.sizeBytes, reason: d.reason })), + }); + } + // Map sanitized response to ParsedMime for downstream compatibility const parsed: ParsedMime = { from: sanitizedParsed.from, diff --git a/tests/isolated/content-sanitizer-attachments.spec.ts b/tests/isolated/content-sanitizer-attachments.spec.ts new file mode 100644 index 00000000..6c982264 --- /dev/null +++ b/tests/isolated/content-sanitizer-attachments.spec.ts @@ -0,0 +1,123 @@ +import { describe, it, expect, vi, afterEach } from "vitest"; +import { handler } from "../../src/isolated/content-sanitizer.js"; + +// --------------------------------------------------------------------------- +// Oversized attachments were previously silently dropped: filtered out of the +// upload loop with no log line and no trace in the response, so there was no +// way to tell a "message with no attachments" apart from a "message whose +// attachment got silently discarded for being too big." These tests pin the +// current behaviour: dropped attachments must be reported in the response and +// logged as a TRACK. +// --------------------------------------------------------------------------- + +const BOUNDARY = "----=_Part_oversized_boundary"; + +function buildEmailWithOversizedAttachment(attachmentSize: number): string { + const attachment = Buffer.alloc(attachmentSize, "A"); + return [ + "From: sender@example.com", + "To: recipient@example.com", + "Subject: Message with a huge attachment", + "MIME-Version: 1.0", + `Content-Type: multipart/mixed; boundary="${BOUNDARY}"`, + "", + `--${BOUNDARY}`, + 'Content-Type: text/plain; charset="UTF-8"', + "", + "See attached.", + "", + `--${BOUNDARY}`, + "Content-Type: application/octet-stream", + "Content-Transfer-Encoding: base64", + 'Content-Disposition: attachment; filename="huge.bin"', + "", + attachment.toString("base64"), + "", + `--${BOUNDARY}--`, + ].join("\r\n"); +} + +function mockFetch(raw: string) { + const buf = Buffer.from(raw, "utf-8"); + vi.stubGlobal("fetch", vi.fn(async (url: string) => { + if (url === "https://example.com/get") { + return { + ok: true, + status: 200, + arrayBuffer: async () => buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength), + }; + } + // presigned POST upload — shouldn't be reached for an oversized attachment + return { ok: true, status: 204 }; + })); +} + +afterEach(() => { + vi.unstubAllGlobals(); + vi.restoreAllMocks(); +}); + +describe("content-sanitizer — oversized attachments", () => { + it("drops an attachment over the single-attachment size limit and reports it in the response", async () => { + mockFetch(buildEmailWithOversizedAttachment(11 * 1024 * 1024)); // > 10MB MAX_SINGLE_ATTACHMENT_SIZE + + const result = await handler({ + presignedGetUrl: "https://example.com/get", + presignedPost: { url: "https://example.com/post", fields: {} }, + accountId: "acct-test", + senderEtld1: "example.com", + keyPrefix: "emails/msg-oversized/", + retentionTag: null, + }); + + expect(result.success).toBe(true); + if (!result.success) return; + + expect(result.parsed.attachments).toEqual([]); + expect(result.parsed.droppedAttachments).toEqual([ + { filename: "huge.bin", mimeType: "application/octet-stream", sizeBytes: 11 * 1024 * 1024, reason: "too_large" }, + ]); + }); + + it("emits a TRACK log naming the dropped attachment count and reason", async () => { + mockFetch(buildEmailWithOversizedAttachment(11 * 1024 * 1024)); + const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + + await handler({ + presignedGetUrl: "https://example.com/get", + presignedPost: { url: "https://example.com/post", fields: {} }, + accountId: "acct-test", + senderEtld1: "example.com", + keyPrefix: "emails/msg-oversized/", + retentionTag: null, + invocationId: "inv-attachments-dropped", + }); + + const trackEntry = logSpy.mock.calls + .map(call => call[0]) + .find((entry): entry is Record => + typeof entry === "object" && entry !== null && (entry as Record).code === "content_sanitizer.attachments_dropped"); + + expect(trackEntry).toBeDefined(); + expect(trackEntry).toMatchObject({ level: "TRACK", droppedCount: 1 }); + }); + + it("does not drop attachments within the single-attachment size limit", async () => { + mockFetch(buildEmailWithOversizedAttachment(1024)); // well under the 10MB limit + + const result = await handler({ + presignedGetUrl: "https://example.com/get", + presignedPost: { url: "https://example.com/post", fields: {} }, + accountId: "acct-test", + senderEtld1: "example.com", + keyPrefix: "emails/msg-small/", + retentionTag: null, + }); + + expect(result.success).toBe(true); + if (!result.success) return; + + expect(result.parsed.attachments).toHaveLength(1); + expect(result.parsed.droppedAttachments).toBeUndefined(); + }); +}); From 2c380225c90c61e7e87b91cf65366273285262e1 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 26 Aug 2026 00:17:49 +0000 Subject: [PATCH 2/3] Cap inline CID image size/count to keep htmlBody out of DynamoDB's item limit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Inline CID images were embedded as unbounded base64 data URIs directly in htmlBody, which is a stored DynamoDB attribute subject to the table's 400KB item cap. Now only small images (<=100KB, first 3 per message) are inlined; anything past either cap is uploaded to S3 like a regular attachment via a new InboundEmailSignalData.inlineImages list (s3Key only, never the bytes), and its cid: reference is left unresolved in htmlBody until API read time. Along the way, discovered and fixed two pre-existing issues this surfaced: - mailparser's default keepCidLinks:false silently base64-embeds every CID image into parsed.html before our own code ever runs, which is why the existing size-aware logic could never have worked without keepCidLinks: true — now set explicitly so the sanitizer's own budget decides. - The sanitizer's cid resolution keyed off attachment.contentId (the raw, angle-bracketed Content-ID header value, e.g. "") instead of attachment.cid (the bracket-stripped form actually referenced by bare `cid:` links in the HTML) — a latent mismatch that never surfaced because mailparser's own auto-embed ran first and masked it. signalsApi.ts and threadsApi.ts each had an identical local withAttachmentUrls() helper; consolidated into one shared withResolvedContentUrls() in signal-transforms.ts that both computes Attachment.url from s3Key (as before) and now also resolves any leftover cid: reference in htmlBody using inlineImages — computed lazily at read time, never persisted as a baked-in URL, matching the existing convention. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01V85rptvcLi67TUmaqU1s53 --- src/api/signal-transforms.ts | 38 ++++++ src/api/signalsApi.ts | 13 +- src/api/threadsApi.ts | 13 +- src/isolated/content-sanitizer.ts | 74 ++++++++-- src/processor/content-sanitizer-client.ts | 8 ++ src/processor/mime.ts | 3 +- src/processor/processor.ts | 2 + src/types/index.ts | 15 ++ tests/api/signal-transforms.spec.ts | 61 ++++++++- .../content-sanitizer-inline-images.spec.ts | 128 ++++++++++++++++++ 10 files changed, 327 insertions(+), 28 deletions(-) create mode 100644 tests/isolated/content-sanitizer-inline-images.spec.ts diff --git a/src/api/signal-transforms.ts b/src/api/signal-transforms.ts index 251937ec..4f990b6c 100644 --- a/src/api/signal-transforms.ts +++ b/src/api/signal-transforms.ts @@ -4,11 +4,14 @@ import type { Thread as DbThread, AnySignal, + Attachment, EmailSignalData, + InboundEmailSignalData, DeliverabilitySignalData, MatchedRuleResult, Signal as DbSignal, } from "../types/index.js"; +import { isEmailSignal } from "../types/index.js"; import type * as Api from "./schemas.js"; // matchedRules is an append-only trace (e.g. a signal dismissed from quarantine gets a second @@ -21,6 +24,41 @@ function collapseMatchedRules(rules: MatchedRuleResult[]): MatchedRuleResult[] { return [...byRuleId.values()]; } +function escapeRegExp(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +// Resolves what the content sanitizer left as s3Key-only references (never a baked-in URL, +// so CDN config can change without a data migration) into CDN urls, at API read time: +// - Attachment.s3Key -> Attachment.url, as before. +// - Any `cid:{contentId}` left unresolved in htmlBody (an inline image too large, or past +// the per-message budget, to embed as a data URI — see MAX_INLINE_DATA_URI_SIZE / +// MAX_INLINE_DATA_URI_COUNT in the content sanitizer) -> the matching InlineImageRef's CDN url. +// inlineImages itself is intentionally not included on the returned signal — it's write-side +// plumbing for this substitution, not something the client needs. +export function withResolvedContentUrls(signal: T, cdnBase: string): T { + if (!isEmailSignal(signal)) return signal; + + const attachments = signal.data.attachments.map((a: Attachment) => ({ ...a, url: `${cdnBase}/${a.s3Key}` })); + + const inlineImages = (signal.data as InboundEmailSignalData).inlineImages; + let htmlBody = (signal.data as InboundEmailSignalData).htmlBody; + if (htmlBody && inlineImages && inlineImages.length > 0) { + for (const ref of inlineImages) { + htmlBody = htmlBody.replace(new RegExp(`cid:${escapeRegExp(ref.contentId)}`, "g"), `${cdnBase}/${ref.s3Key}`); + } + } + + return { + ...signal, + data: { + ...signal.data, + attachments, + ...(htmlBody !== undefined ? { htmlBody } : {}), + }, + } as T; +} + export function toApiThread(thread: DbThread): Api.Thread { return { threadId: thread.id, diff --git a/src/api/signalsApi.ts b/src/api/signalsApi.ts index 7ab55438..a0b20f38 100644 --- a/src/api/signalsApi.ts +++ b/src/api/signalsApi.ts @@ -3,14 +3,14 @@ import type { OpenAPIHono } from "@hono/zod-openapi"; import { DateTime } from "luxon"; import { getDomain } from "tldts"; import { zParse } from "./validate.js"; -import { toApiThread, toApiSignal } from "./signal-transforms.js"; +import { toApiThread, toApiSignal, withResolvedContentUrls } from "./signal-transforms.js"; import { deriveGroupingKey } from "../grouping-key.js"; import { handlePostApprovalCalendar } from "../processor/calendar/post-approval-handler.js"; import { resolveRetention } from "../retention.js"; import { buildActiveThread } from "../thread-factory.js"; import { isEmailSignal } from "../types/index.js"; import type { Result } from "neverthrow"; -import type { Thread, Signal, AnySignal, Attachment, MatchedRuleResult, PageParams } from "../types/index.js"; +import type { Thread, Signal, MatchedRuleResult, PageParams } from "../types/index.js"; import type { Pagination } from "../types/index.js"; import type { ThreadDatabase } from "../database/thread-database.js"; import type { AccountDatabase } from "../database/account-database.js"; @@ -29,11 +29,6 @@ function page(key: K, items: T[], nextCursor?: string): Rec return { [key]: items, pagination: { cursor: nextCursor ?? null } } as Record & { pagination: Pagination }; } -function withAttachmentUrls(signal: T, cdnBase: string): T { - if (!isEmailSignal(signal)) return signal; - return { ...signal, data: { ...signal.data, attachments: signal.data.attachments.map((a: Attachment) => ({ ...a, url: `${cdnBase}/${a.s3Key}` })) } }; -} - export class SignalsApi { constructor( private readonly threadDb: ThreadDatabase, @@ -99,7 +94,7 @@ export class SignalsApi { items = items.filter(s => isEmailSignal(s) && s.data.from.address.toLowerCase().includes(senderLower)); } - const itemsWithUrls = contentCdnBaseUrl ? items.map(s => withAttachmentUrls(s, contentCdnBaseUrl)) : items; + const itemsWithUrls = contentCdnBaseUrl ? items.map(s => withResolvedContentUrls(s, contentCdnBaseUrl)) : items; return c.json(page("signals", itemsWithUrls.map(toApiSignal), result.value.nextCursor), 200); }); @@ -238,7 +233,7 @@ export class SignalsApi { } } - const signalWithUrls = contentCdnBaseUrl ? withAttachmentUrls(signal, contentCdnBaseUrl) : signal; + const signalWithUrls = contentCdnBaseUrl ? withResolvedContentUrls(signal, contentCdnBaseUrl) : signal; logger.info("Signal activated", { code: "api.signals.activated", accountId, signalId, threadId: thread.id }); return c.json({ thread: toApiThread(thread), signal: toApiSignal({ ...signalWithUrls, status: "active", threadId: thread.id }) }, 200); }); diff --git a/src/api/threadsApi.ts b/src/api/threadsApi.ts index e9bf551c..550ad748 100644 --- a/src/api/threadsApi.ts +++ b/src/api/threadsApi.ts @@ -6,12 +6,12 @@ import { getDomain } from "tldts"; import { validateRecipientMx } from "../dns/mx-validator.js"; import { computeUndoWindowSeconds } from "./undo-window.js"; import { zParse } from "./validate.js"; -import { toApiThread, toApiSignal } from "./signal-transforms.js"; +import { toApiThread, toApiSignal, withResolvedContentUrls } from "./signal-transforms.js"; import { buildScheduleName } from "../scheduler/schedule-name.js"; import { durationToSeconds } from "../retention.js"; import { isCalendarEventSignal, isEmailSignal } from "../types/index.js"; import type { EmailContentStore } from "./content-store.js"; -import type { Signal, AnySignal, Attachment, PageParams, ThreadStatus, Workflow } from "../types/index.js"; +import type { Signal, AnySignal, PageParams, ThreadStatus, Workflow } from "../types/index.js"; import type { CalendarResponseData, DomainMisconfigurationData, Pagination } from "../types/index.js"; import type { UpdateThreadFields, ThreadDatabase } from "../database/thread-database.js"; import type { AccountDatabase } from "../database/account-database.js"; @@ -51,11 +51,6 @@ function page(key: K, items: T[], nextCursor?: string): Rec return { [key]: items, pagination: { cursor: nextCursor ?? null } } as Record & { pagination: Pagination }; } -function withAttachmentUrls(signal: T, cdnBase: string): T { - if (!isEmailSignal(signal)) return signal; - return { ...signal, data: { ...signal.data, attachments: signal.data.attachments.map((a: Attachment) => ({ ...a, url: `${cdnBase}/${a.s3Key}` })) } }; -} - export class ThreadsApi { constructor( private readonly threadDb: ThreadDatabase, @@ -334,7 +329,7 @@ export class ThreadsApi { } const enrichedSignals = signals.map(signal => { - const withUrls = contentCdnBaseUrl ? withAttachmentUrls(signal, contentCdnBaseUrl) : signal; + const withUrls = contentCdnBaseUrl ? withResolvedContentUrls(signal, contentCdnBaseUrl) : signal; const apiSignal = toApiSignal(withUrls); if (isCalendarEventSignal(withUrls) && enrichments.has(withUrls.data.veventUid)) { return { ...apiSignal, latestResponse: enrichments.get(withUrls.data.veventUid) }; @@ -771,7 +766,7 @@ export class ThreadsApi { } const signal = signalResult.value; if (!signal) return err(c, 404, "Signal not found", "SIGNAL_NOT_FOUND"); - const withUrls = contentCdnBaseUrl ? withAttachmentUrls(signal, contentCdnBaseUrl) : signal; + const withUrls = contentCdnBaseUrl ? withResolvedContentUrls(signal, contentCdnBaseUrl) : signal; return c.json(toApiSignal(withUrls), 200); }); diff --git a/src/isolated/content-sanitizer.ts b/src/isolated/content-sanitizer.ts index 855de3fa..09a9e6d3 100644 --- a/src/isolated/content-sanitizer.ts +++ b/src/isolated/content-sanitizer.ts @@ -37,6 +37,13 @@ interface DroppedAttachment { reason: "too_large" | "upload_failed"; } +interface InlineImageRef { + contentId: string; + mimeType: string; + sizeBytes: number; + s3Key: string; +} + interface ContentSanitizeRequest { presignedGetUrl: string; presignedPost: { @@ -71,6 +78,7 @@ interface ContentSanitizeResponse { assets?: ExtractedAsset[]; links?: ExtractedLink[]; droppedAttachments?: DroppedAttachment[]; + inlineImages?: InlineImageRef[]; }; } @@ -90,6 +98,16 @@ const MAX_ATTACHMENTS = 50; const MAX_TOTAL_ATTACHMENT_SIZE = 25 * 1024 * 1024; // 25MB const MAX_SINGLE_ATTACHMENT_SIZE = 10 * 1024 * 1024; // 10MB +// DynamoDB's per-item cap is 400KB; MAX_HTML_BODY_BYTES in processor.ts already reserves +// 300KB of that for htmlBody as a last-resort truncation guard. These two caps keep most +// messages from ever needing that guard: a data URI runs ~33% larger than its source bytes, +// so 3 images at 100KB each tops out around 400KB of embedded text — big enough for a +// typical logo/signature image, small enough that inline images can't silently balloon +// htmlBody the way an unbounded embed would. Anything past either cap is uploaded to S3 +// instead (see InlineImageRef) and resolved to a CDN url at API read time. +const MAX_INLINE_DATA_URI_SIZE = 100 * 1024; // 100KB +const MAX_INLINE_DATA_URI_COUNT = 3; + // --------------------------------------------------------------------------- // Helpers // --------------------------------------------------------------------------- @@ -230,7 +248,11 @@ async function processEmail(event: ContentSanitizeRequest, logger?: Logger): Pro const parseStart = Date.now(); let parsed; try { - parsed = await simpleParser(rawMime); + // keepCidLinks: mailparser's default behaviour is to eagerly replace every `cid:` + // reference in the HTML with a base64 data URI itself, before this file ever sees the + // parsed message — which would silently bypass the inline-image size/count budget below. + // With this set, `cid:` references are left as-is in parsed.html for us to resolve. + parsed = await simpleParser(rawMime, { keepCidLinks: true }); const parseElapsedMs = Date.now() - parseStart; logger?.trackPoint("mime_parse_complete"); if (parseElapsedMs > SLOW_PARSE_THRESHOLD_MS) { @@ -277,6 +299,8 @@ async function processEmail(event: ContentSanitizeRequest, logger?: Logger): Pro const inlineImages: Array<{ mimeType: string; content: Buffer }> = []; const attachmentsWithBytes: Array<{ filename: string; mimeType: string; content: Buffer; s3Key?: string }> = []; const droppedAttachments: DroppedAttachment[] = []; + const inlineImageRefs: InlineImageRef[] = []; + let inlinedDataUriCount = 0; for (const attachment of attachments) { const contentType = attachment.contentType || "application/octet-stream"; @@ -291,11 +315,37 @@ async function processEmail(event: ContentSanitizeRequest, logger?: Logger): Pro continue; } - if (attachment.contentId) { - // Inline image — embed as data URI, no S3 upload needed - cidMap[attachment.contentId] = `data:${contentType};base64,${attachment.content.toString("base64")}`; - if (contentType.startsWith("image/")) { - inlineImages.push({ mimeType: contentType, content: attachment.content }); + // `cid` (not `contentId`) is the bracket-stripped form mailparser itself matches + // `cid:` references against — it's what appears bare in ``. + if (attachment.cid) { + const cid = attachment.cid; + const fitsInlineBudget = attachment.size <= MAX_INLINE_DATA_URI_SIZE && inlinedDataUriCount < MAX_INLINE_DATA_URI_COUNT; + + if (fitsInlineBudget) { + // Small inline image, still within the per-message budget — embed as a data URI, no S3 upload needed + cidMap[cid] = `data:${contentType};base64,${attachment.content.toString("base64")}`; + inlinedDataUriCount++; + if (contentType.startsWith("image/")) { + inlineImages.push({ mimeType: contentType, content: attachment.content }); + } + } else { + // Too large, or past the per-message inline budget — upload to S3 like a regular + // attachment instead of embedding. Its `cid:` reference is deliberately left + // unresolved below; the API layer resolves it to a CDN url at read time. + const s3Key = `${event.keyPrefix}${uploadIndex}`; + const uploaded = await uploadViaPresignedPost(event.presignedPost, s3Key, attachment.content, contentType, event.retentionTag); + const filename = attachment.filename ?? `inline-${uploadIndex}`; + + if (uploaded) { + inlineImageRefs.push({ contentId: cid, mimeType: contentType, sizeBytes: attachment.size, s3Key }); + // Still eligible for QR scanning via the attachment-image path in extractAssets + if (contentType.startsWith("image/")) { + attachmentsWithBytes.push({ filename, mimeType: contentType, content: attachment.content, s3Key }); + } + } else { + droppedAttachments.push({ filename, mimeType: contentType, sizeBytes: attachment.size, reason: "upload_failed" }); + } + uploadIndex++; } continue; } @@ -313,7 +363,7 @@ async function processEmail(event: ContentSanitizeRequest, logger?: Logger): Pro uploadIndex++; } - logger?.trackPoint("attachments_processed", { attachmentRefCount: attachmentRefs.length, inlineImageCount: inlineImages.length, droppedCount: droppedAttachments.length }); + logger?.trackPoint("attachments_processed", { attachmentRefCount: attachmentRefs.length, inlineImageCount: inlineImages.length, inlineImageRefCount: inlineImageRefs.length, droppedCount: droppedAttachments.length }); if (droppedAttachments.length > 0) { logger?.track("Attachment(s) dropped from message.", { code: "content_sanitizer.attachments_dropped", @@ -330,7 +380,12 @@ async function processEmail(event: ContentSanitizeRequest, logger?: Logger): Pro if (parsed.html) { const htmlInput = typeof parsed.html === "string" ? parsed.html : ""; const sanitized = sanitizeHtml(htmlInput); - htmlBody = sanitized.html.replace(/cid:([^"'\s>]+)/g, (_, id: string) => cidMap[id] ?? ""); + const inlineImageContentIds = new Set(inlineImageRefs.map(ref => ref.contentId)); + htmlBody = sanitized.html.replace(/cid:([^"'\s>]+)/g, (match, id: string) => { + if (id in cidMap) return cidMap[id] ?? ""; + if (inlineImageContentIds.has(id)) return match; // resolved to a CDN url at API read time + return ""; + }); logger?.trackPoint("html_sanitized", { htmlLength: htmlInput.length }); // Extract links from raw HTML using a DOM parser (before sanitization strips tracking pixels etc.) @@ -440,6 +495,9 @@ async function processEmail(event: ContentSanitizeRequest, logger?: Logger): Pro if (droppedAttachments.length > 0) { result.parsed.droppedAttachments = droppedAttachments; } + if (inlineImageRefs.length > 0) { + result.parsed.inlineImages = inlineImageRefs; + } logger?.trackPoint("response_built"); return result; diff --git a/src/processor/content-sanitizer-client.ts b/src/processor/content-sanitizer-client.ts index 274baa50..8e8b46e2 100644 --- a/src/processor/content-sanitizer-client.ts +++ b/src/processor/content-sanitizer-client.ts @@ -27,6 +27,13 @@ export interface DroppedAttachment { reason: "too_large" | "upload_failed"; } +export interface InlineImageRef { + contentId: string; + mimeType: string; + sizeBytes: number; + s3Key: string; +} + export interface ContentSanitizeRequest { presignedGetUrl: string; presignedPost: { @@ -67,6 +74,7 @@ export interface ContentSanitizeResponse { assets?: ExtractedAsset[]; links?: ExtractedLink[]; droppedAttachments?: DroppedAttachment[]; + inlineImages?: InlineImageRef[]; }; } diff --git a/src/processor/mime.ts b/src/processor/mime.ts index 1623305d..80f866e9 100644 --- a/src/processor/mime.ts +++ b/src/processor/mime.ts @@ -1,4 +1,4 @@ -import type { EmailAddress, Attachment } from "../types/index.js"; +import type { EmailAddress, Attachment, InlineImageRef } from "../types/index.js"; import type { DbError, Result } from "../errors.js"; export interface ParsedMime { @@ -12,6 +12,7 @@ export interface ParsedMime { attachments: Attachment[]; headers: Record; sentAt?: string; + inlineImages?: InlineImageRef[]; } export interface MimeParser { diff --git a/src/processor/processor.ts b/src/processor/processor.ts index 380c0cda..34425dc1 100644 --- a/src/processor/processor.ts +++ b/src/processor/processor.ts @@ -913,6 +913,7 @@ export class SignalProcessor { ...(sanitizedParsed.textBody !== undefined ? { textBody: sanitizedParsed.textBody } : {}), ...(sanitizedParsed.htmlBody !== undefined ? { htmlBody: sanitizedParsed.htmlBody } : {}), ...(sanitizedParsed.sentAt !== undefined ? { sentAt: sanitizedParsed.sentAt } : {}), + ...(sanitizedParsed.inlineImages ? { inlineImages: sanitizedParsed.inlineImages } : {}), }; this.logger.trackPoint("email_parsed"); @@ -2106,6 +2107,7 @@ function buildSignal(opts: { ...(htmlBodyTruncated ? { htmlBodyTruncated: true } : {}), ...(parsed.sentAt !== undefined ? { sentAt: parsed.sentAt } : {}), ...(unsubscribe !== undefined ? { unsubscribe } : {}), + ...(parsed.inlineImages && parsed.inlineImages.length > 0 ? { inlineImages: parsed.inlineImages } : {}), }, }; diff --git a/src/types/index.ts b/src/types/index.ts index 127e07eb..28940cd5 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -372,6 +372,19 @@ export interface Attachment { s3Key: string; } +// A CID-referenced inline image too large (or too numerous — see MAX_INLINE_DATA_URI_COUNT +// in the content sanitizer) to embed as a base64 data URI in htmlBody without risking the +// DynamoDB 400KB item cap. Uploaded to S3 like a regular Attachment instead; only the s3Key +// is stored, never the bytes. Its `cid:{contentId}` reference is left unresolved in htmlBody +// at write time and swapped for a CDN url at API read time (see withResolvedContentUrls in +// src/api/signal-transforms.ts), mirroring how Attachment.url is computed lazily from s3Key. +export interface InlineImageRef { + contentId: string; + mimeType: string; + sizeBytes: number; + s3Key: string; +} + // --------------------------------------------------------------------------- // MatchedRuleResult — per-rule trace written to Signal.matchedRules // --------------------------------------------------------------------------- @@ -445,6 +458,8 @@ export interface InboundEmailSignalData extends EmailSignalDataBase { htmlBody?: string; /** Set to true when htmlBody was truncated before storage. Full content recoverable from S3 via s3Key. */ htmlBodyTruncated?: boolean; + /** Inline CID images stored via S3 (not embedded as data URIs) — see InlineImageRef. */ + inlineImages?: InlineImageRef[]; } // --------------------------------------------------------------------------- diff --git a/tests/api/signal-transforms.spec.ts b/tests/api/signal-transforms.spec.ts index f9ee3b45..4293092b 100644 --- a/tests/api/signal-transforms.spec.ts +++ b/tests/api/signal-transforms.spec.ts @@ -1,5 +1,5 @@ import { describe, it, expect } from "vitest"; -import { toApiSignal } from "../../src/api/signal-transforms.js"; +import { toApiSignal, withResolvedContentUrls } from "../../src/api/signal-transforms.js"; import type { Signal } from "../../src/types/index.js"; function makeInboundSignal(overrides: { data?: Partial } & Partial> = {}): Signal { @@ -67,3 +67,62 @@ describe("toApiSignal — matchedRules collapse", () => { expect(data.matchedRules).toEqual([{ ruleId: "SR-02", actions: [{ type: "quarantine_visible" }], labelsAdded: [], statusChange: "quarantine_visible" }]); }); }); + +// --------------------------------------------------------------------------- +// withResolvedContentUrls — s3Key -> CDN url resolution, computed lazily at read +// time (never stored). Covers both Attachment.url and the inline-image cid: +// substitution for images the sanitizer routed to S3 instead of embedding as a +// data URI (see MAX_INLINE_DATA_URI_SIZE / MAX_INLINE_DATA_URI_COUNT). +// --------------------------------------------------------------------------- +describe("withResolvedContentUrls", () => { + it("adds a CDN url to each attachment computed from its s3Key", () => { + const signal = makeInboundSignal({ + data: { + attachments: [ + { filename: "doc.pdf", mimeType: "application/pdf", sizeBytes: 1024, s3Key: "emails/msg-001/0" }, + ], + }, + }); + + const result = withResolvedContentUrls(signal, "https://cdn.example.com"); + const attachments = result.data.attachments as Array<{ url?: string }>; + expect(attachments[0]?.url).toBe("https://cdn.example.com/emails/msg-001/0"); + }); + + it("replaces a leftover cid: reference in htmlBody with the matching inline image's CDN url", () => { + const signal = makeInboundSignal({ + data: { + htmlBody: '

Logo:

', + inlineImages: [{ contentId: "logo123", mimeType: "image/png", sizeBytes: 500_000, s3Key: "emails/msg-001/inline-0" }], + }, + }); + + const result = withResolvedContentUrls(signal, "https://cdn.example.com"); + expect((result.data as { htmlBody?: string }).htmlBody).toBe( + '

Logo:

', + ); + }); + + it("does not touch htmlBody when there are no inline images to resolve", () => { + const signal = makeInboundSignal({ data: { htmlBody: "

Hello

" } }); + const result = withResolvedContentUrls(signal, "https://cdn.example.com"); + expect((result.data as { htmlBody?: string }).htmlBody).toBe("

Hello

"); + }); + + it("resolves multiple inline images independently", () => { + const signal = makeInboundSignal({ + data: { + htmlBody: '', + inlineImages: [ + { contentId: "a", mimeType: "image/png", sizeBytes: 500_000, s3Key: "emails/msg-001/inline-0" }, + { contentId: "b", mimeType: "image/jpeg", sizeBytes: 600_000, s3Key: "emails/msg-001/inline-1" }, + ], + }, + }); + + const result = withResolvedContentUrls(signal, "https://cdn.example.com"); + expect((result.data as { htmlBody?: string }).htmlBody).toBe( + '', + ); + }); +}); diff --git a/tests/isolated/content-sanitizer-inline-images.spec.ts b/tests/isolated/content-sanitizer-inline-images.spec.ts new file mode 100644 index 00000000..f466ae71 --- /dev/null +++ b/tests/isolated/content-sanitizer-inline-images.spec.ts @@ -0,0 +1,128 @@ +import { describe, it, expect, vi, afterEach } from "vitest"; +import { handler } from "../../src/isolated/content-sanitizer.js"; + +// --------------------------------------------------------------------------- +// Inline CID images were previously embedded as base64 data URIs directly in +// htmlBody with no size/count limit — a single large inline logo could bloat +// the stored signal well past DynamoDB's 400KB item cap. Now only small images +// (<=100KB, first 3 per message) are inlined; anything past either cap is +// uploaded to S3 like a regular attachment, with its `cid:` reference left +// unresolved in htmlBody (see MAX_INLINE_DATA_URI_SIZE / MAX_INLINE_DATA_URI_COUNT). +// --------------------------------------------------------------------------- + +const BOUNDARY = "----=_Part_inline_boundary"; + +function buildEmailWithInlineImages(images: Array<{ contentId: string; size: number }>): string { + const parts = images.map(({ contentId, size }) => [ + `--${BOUNDARY}`, + "Content-Type: image/png", + "Content-Transfer-Encoding: base64", + `Content-ID: <${contentId}>`, + `Content-Disposition: inline; filename="${contentId}.png"`, + "", + Buffer.alloc(size, "A").toString("base64"), + "", + ].join("\r\n")); + + const imgTags = images.map(({ contentId }) => ``).join(""); + + return [ + "From: sender@example.com", + "To: recipient@example.com", + "Subject: Message with inline images", + "MIME-Version: 1.0", + `Content-Type: multipart/related; boundary="${BOUNDARY}"`, + "", + `--${BOUNDARY}`, + 'Content-Type: text/html; charset="UTF-8"', + "", + `${imgTags}`, + "", + ...parts, + `--${BOUNDARY}--`, + ].join("\r\n"); +} + +function mockFetch(raw: string) { + const buf = Buffer.from(raw, "utf-8"); + vi.stubGlobal("fetch", vi.fn(async (url: string) => { + if (url === "https://example.com/get") { + return { ok: true, status: 200, arrayBuffer: async () => buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength) }; + } + return { ok: true, status: 204 }; + })); +} + +afterEach(() => { + vi.unstubAllGlobals(); +}); + +describe("content-sanitizer — inline image size/count cap", () => { + it("embeds a small inline image as a data URI, no S3 upload / inlineImages entry", async () => { + mockFetch(buildEmailWithInlineImages([{ contentId: "logo", size: 1024 }])); + + const result = await handler({ + presignedGetUrl: "https://example.com/get", + presignedPost: { url: "https://example.com/post", fields: {} }, + accountId: "acct-test", + senderEtld1: "example.com", + keyPrefix: "emails/msg-small-inline/", + retentionTag: null, + }); + + expect(result.success).toBe(true); + if (!result.success) return; + expect(result.parsed.htmlBody).toContain("data:image/png;base64,"); + expect(result.parsed.htmlBody).not.toContain("cid:logo"); + expect(result.parsed.inlineImages).toBeUndefined(); + }); + + it("uploads an inline image over 100KB to S3 instead of embedding it, leaving cid: unresolved", async () => { + mockFetch(buildEmailWithInlineImages([{ contentId: "logo", size: 200 * 1024 }])); + + const result = await handler({ + presignedGetUrl: "https://example.com/get", + presignedPost: { url: "https://example.com/post", fields: {} }, + accountId: "acct-test", + senderEtld1: "example.com", + keyPrefix: "emails/msg-big-inline/", + retentionTag: null, + }); + + expect(result.success).toBe(true); + if (!result.success) return; + expect(result.parsed.htmlBody).toContain("cid:logo"); + expect(result.parsed.htmlBody).not.toContain("data:image/png;base64,"); + expect(result.parsed.inlineImages).toEqual([ + { contentId: "logo", mimeType: "image/png", sizeBytes: 200 * 1024, s3Key: "emails/msg-big-inline/0" }, + ]); + }); + + it("inlines only the first 3 small images per message, routing the 4th to S3", async () => { + mockFetch(buildEmailWithInlineImages([ + { contentId: "img1", size: 1024 }, + { contentId: "img2", size: 1024 }, + { contentId: "img3", size: 1024 }, + { contentId: "img4", size: 1024 }, + ])); + + const result = await handler({ + presignedGetUrl: "https://example.com/get", + presignedPost: { url: "https://example.com/post", fields: {} }, + accountId: "acct-test", + senderEtld1: "example.com", + keyPrefix: "emails/msg-many-inline/", + retentionTag: null, + }); + + expect(result.success).toBe(true); + if (!result.success) return; + expect(result.parsed.htmlBody).not.toContain("cid:img1"); + expect(result.parsed.htmlBody).not.toContain("cid:img2"); + expect(result.parsed.htmlBody).not.toContain("cid:img3"); + expect(result.parsed.htmlBody).toContain("cid:img4"); + expect(result.parsed.inlineImages).toEqual([ + { contentId: "img4", mimeType: "image/png", sizeBytes: 1024, s3Key: "emails/msg-many-inline/0" }, + ]); + }); +}); From 7bef51506f8fc2cf805fbe2a5f2d51d8be714d42 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 26 Aug 2026 08:45:47 +0000 Subject: [PATCH 3/3] Strip attachments from the raw email view in the backend, not the frontend MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The "view original email" feature previously served the fully original raw MIME (including base64 attachment bodies) and had the frontend strip attachments for display only — Copy/Download still shipped the full raw bytes to the client. Move this to the backend instead: the content sanitizer now builds a display-safe copy of the raw MIME at ingestion time (attachments fully stripped; small inline images kept intact so the .eml still renders in a real mail client if downloaded, capped at 100KB per image and 300KB cumulative per message — additional images beyond either cap are truncated), uploads it alongside extracted attachments, and the processor persists its s3Key as InboundEmailSignalData.displayRawS3Key. The GET .../signals/:id/raw endpoint now redirects to this display-safe copy when available, falling back to the true original for signals processed before this feature existed. The true original is never served through that path anymore — it stays available server-side only (e.g. for reprocessing). Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01V85rptvcLi67TUmaqU1s53 --- src/api/threadsApi.ts | 11 +- src/isolated/content-sanitizer.ts | 22 +++ src/isolated/raw-email-display.ts | 122 +++++++++++++++++ src/processor/content-sanitizer-client.ts | 1 + src/processor/mime.ts | 1 + src/processor/processor.ts | 2 + src/types/index.ts | 7 + .../content-sanitizer-display-raw.spec.ts | 78 +++++++++++ tests/isolated/raw-email-display.spec.ts | 127 ++++++++++++++++++ 9 files changed, 370 insertions(+), 1 deletion(-) create mode 100644 src/isolated/raw-email-display.ts create mode 100644 tests/isolated/content-sanitizer-display-raw.spec.ts create mode 100644 tests/isolated/raw-email-display.spec.ts diff --git a/src/api/threadsApi.ts b/src/api/threadsApi.ts index 550ad748..97a5a99b 100644 --- a/src/api/threadsApi.ts +++ b/src/api/threadsApi.ts @@ -11,7 +11,7 @@ import { buildScheduleName } from "../scheduler/schedule-name.js"; import { durationToSeconds } from "../retention.js"; import { isCalendarEventSignal, isEmailSignal } from "../types/index.js"; import type { EmailContentStore } from "./content-store.js"; -import type { Signal, AnySignal, PageParams, ThreadStatus, Workflow } from "../types/index.js"; +import type { Signal, AnySignal, InboundEmailSignalData, PageParams, ThreadStatus, Workflow } from "../types/index.js"; import type { CalendarResponseData, DomainMisconfigurationData, Pagination } from "../types/index.js"; import type { UpdateThreadFields, ThreadDatabase } from "../database/thread-database.js"; import type { AccountDatabase } from "../database/account-database.js"; @@ -793,6 +793,15 @@ export class ThreadsApi { if (!isEmailSignal(signal)) return err(c, 400, "Signal is not an email", "SIGNAL_NOT_FOUND"); if (!signal.data.s3Key) return err(c, 404, "Raw email not available", "SIGNAL_NOT_FOUND"); + // The display-safe copy (attachments stripped, small inline images kept — built by + // the content sanitizer, see raw-email-display.ts) is what this endpoint serves. + // The true raw original at signal.data.s3Key is never exposed through this path — + // it stays available server-side only, e.g. for reprocessing. + const displayRawS3Key = (signal.data as InboundEmailSignalData).displayRawS3Key; + if (displayRawS3Key && contentCdnBaseUrl) { + return c.redirect(`${contentCdnBaseUrl}/${displayRawS3Key}`, 307); + } + const url = await emailContentStore.getRawEmailUrl(signal); return c.redirect(url, 307); }); diff --git a/src/isolated/content-sanitizer.ts b/src/isolated/content-sanitizer.ts index 09a9e6d3..504a89e0 100644 --- a/src/isolated/content-sanitizer.ts +++ b/src/isolated/content-sanitizer.ts @@ -2,6 +2,7 @@ import { simpleParser } from "mailparser"; import { Window } from "happy-dom"; import { sanitizeHtml } from "./html-sanitizer.js"; import { extractAssets, type ExtractedAsset } from "./asset-extractor.js"; +import { buildDisplayRawEmail } from "./raw-email-display.js"; import type { Logger } from "../logger.js"; // Lambda timeout is 60s (see deploy/compute.tf). Emitting a TRACK log past this @@ -79,6 +80,7 @@ interface ContentSanitizeResponse { links?: ExtractedLink[]; droppedAttachments?: DroppedAttachment[]; inlineImages?: InlineImageRef[]; + displayRawS3Key?: string; }; } @@ -243,6 +245,23 @@ async function processEmail(event: ContentSanitizeRequest, logger?: Logger): Pro }; } + // 1b. Build a display-safe copy of the raw MIME (attachments stripped, small inline + // images kept — see raw-email-display.ts) and upload it. This is what the "view original + // email" / download-as-.eml feature serves; the true original is never exposed through + // that path. Best-effort — a failure here must not fail the whole sanitize, the feature + // just falls back to unavailable for this message. + logger?.trackPoint("display_raw_build_start"); + let displayRawS3Key: string | undefined; + try { + const displayRaw = buildDisplayRawEmail(rawMime.toString("latin1")); + const key = `${event.keyPrefix}raw-display.eml`; + const uploaded = await uploadViaPresignedPost(event.presignedPost, key, Buffer.from(displayRaw, "latin1"), "message/rfc822", event.retentionTag); + if (uploaded) displayRawS3Key = key; + logger?.trackPoint("display_raw_build_complete", { uploaded }); + } catch { + logger?.trackPoint("display_raw_build_failed"); + } + // 2. Parse with mailparser logger?.trackPoint("mime_parse_start"); const parseStart = Date.now(); @@ -498,6 +517,9 @@ async function processEmail(event: ContentSanitizeRequest, logger?: Logger): Pro if (inlineImageRefs.length > 0) { result.parsed.inlineImages = inlineImageRefs; } + if (displayRawS3Key) { + result.parsed.displayRawS3Key = displayRawS3Key; + } logger?.trackPoint("response_built"); return result; diff --git a/src/isolated/raw-email-display.ts b/src/isolated/raw-email-display.ts new file mode 100644 index 00000000..ba28e35e --- /dev/null +++ b/src/isolated/raw-email-display.ts @@ -0,0 +1,122 @@ +/** + * Builds a display-safe copy of the raw MIME source: a byte-faithful copy of the + * original with attachment bodies stripped out entirely, except for small inline + * images (kept so the message still renders correctly if the .eml is opened in a + * real mail client — those clients resolve `cid:` references against the file's + * own MIME parts, unlike our app's htmlBody rendering, which resolves them + * against a CDN url instead). This is the copy served by the "view original + * email" / download-as-.eml feature; the true, unmodified original stays in S3 + * for internal use (e.g. reprocessing) and is never served through that path. + * + * This is a textual boundary walk, not a semantic MIME parse — it never decodes + * or interprets attachment bytes, only locates part boundaries/headers to decide + * what to keep. Operates on the raw MIME text as fetched, inside the sanitizer's + * security boundary (see docs/adr/011-content-sanitizer-security-boundary.md). + */ + +// Per-image cap: an inline image bigger than this is truncated regardless of +// how much of the cumulative budget remains. +const MAX_DISPLAY_INLINE_IMAGE_SIZE = 100 * 1024; // 100KB + +// Cumulative cap across all inline images kept in one message — once reached, +// every remaining inline image is truncated even if individually under the +// per-image cap. +const MAX_DISPLAY_INLINE_TOTAL_SIZE = 300 * 1024; // 300KB + +function escapeRegExp(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +function formatBytes(bytes: number): string { + if (bytes < 1024) return `${bytes} B`; + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`; + return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; +} + +export function buildDisplayRawEmail(raw: string): string { + const boundaries = new Set(); + const boundaryRegex = /boundary\s*=\s*"?([^";\r\n]+)"?/gi; + let boundaryMatch: RegExpExecArray | null; + while ((boundaryMatch = boundaryRegex.exec(raw))) { + boundaries.add(boundaryMatch[1]!); + } + if (boundaries.size === 0) return raw; // not multipart — nothing to strip + + const boundaryLineRegex = new RegExp( + `^--(${[...boundaries].map(escapeRegExp).join("|")})(--)?\\s*$`, + ); + + const lines = raw.split(/\r\n|\n/); + const output: string[] = []; + let i = 0; + let inlineBudgetRemaining = MAX_DISPLAY_INLINE_TOTAL_SIZE; + + // Preamble / top-level headers before the first boundary — unchanged + while (i < lines.length && !boundaryLineRegex.test(lines[i]!)) { + output.push(lines[i]!); + i++; + } + + while (i < lines.length) { + const boundaryLine = lines[i]!; + output.push(boundaryLine); + i++; + if (/--\s*$/.test(boundaryLine)) break; // closing boundary — no part follows + + const headerLines: string[] = []; + while (i < lines.length && lines[i] !== "" && !boundaryLineRegex.test(lines[i]!)) { + headerLines.push(lines[i]!); + output.push(lines[i]!); + i++; + } + if (i < lines.length && lines[i] === "") { + output.push(lines[i]!); + i++; + } + + const bodyStart = i; + while (i < lines.length && !boundaryLineRegex.test(lines[i]!)) { + i++; + } + const bodyLines = lines.slice(bodyStart, i); + const bodyHasContent = bodyLines.some(line => line.trim() !== ""); + + if (!bodyHasContent) { + output.push(...bodyLines); + continue; + } + + const headerText = headerLines.join("\n"); + const isInlineImage = /Content-Disposition:\s*inline/i.test(headerText) && /Content-Type:\s*image\//i.test(headerText); + const isFilePart = + /Content-Disposition:\s*attachment/i.test(headerText) || + /Content-Disposition:[^\r\n]*\bfilename\*?=/i.test(headerText) || + /Content-Type:[^\r\n]*\bname\s*=/i.test(headerText); + + if (isInlineImage) { + const approxBytes = Math.floor(bodyLines.join("").length * 0.75); // base64 -> raw bytes estimate + if (approxBytes <= MAX_DISPLAY_INLINE_IMAGE_SIZE && approxBytes <= inlineBudgetRemaining) { + output.push(...bodyLines); + inlineBudgetRemaining -= approxBytes; + } else { + output.push(`[inline image omitted: exceeds display size limit (~${formatBytes(approxBytes)})]`); + output.push(""); + } + continue; + } + + if (isFilePart) { + const filenameMatch = + headerText.match(/filename\*?=\s*"?([^";\r\n]+)"?/i) ?? headerText.match(/name\s*=\s*"?([^";\r\n]+)"?/i); + const filename = filenameMatch ? filenameMatch[1] : null; + const approxBytes = Math.floor(bodyLines.join("").length * 0.75); + output.push(`[attachment content omitted${filename ? `: ${filename}` : ""} (~${formatBytes(approxBytes)})]`); + output.push(""); + continue; + } + + output.push(...bodyLines); + } + + return output.join("\r\n"); +} diff --git a/src/processor/content-sanitizer-client.ts b/src/processor/content-sanitizer-client.ts index 8e8b46e2..a6886d95 100644 --- a/src/processor/content-sanitizer-client.ts +++ b/src/processor/content-sanitizer-client.ts @@ -75,6 +75,7 @@ export interface ContentSanitizeResponse { links?: ExtractedLink[]; droppedAttachments?: DroppedAttachment[]; inlineImages?: InlineImageRef[]; + displayRawS3Key?: string; }; } diff --git a/src/processor/mime.ts b/src/processor/mime.ts index 80f866e9..4e50c96d 100644 --- a/src/processor/mime.ts +++ b/src/processor/mime.ts @@ -13,6 +13,7 @@ export interface ParsedMime { headers: Record; sentAt?: string; inlineImages?: InlineImageRef[]; + displayRawS3Key?: string; } export interface MimeParser { diff --git a/src/processor/processor.ts b/src/processor/processor.ts index 34425dc1..c3bce11b 100644 --- a/src/processor/processor.ts +++ b/src/processor/processor.ts @@ -914,6 +914,7 @@ export class SignalProcessor { ...(sanitizedParsed.htmlBody !== undefined ? { htmlBody: sanitizedParsed.htmlBody } : {}), ...(sanitizedParsed.sentAt !== undefined ? { sentAt: sanitizedParsed.sentAt } : {}), ...(sanitizedParsed.inlineImages ? { inlineImages: sanitizedParsed.inlineImages } : {}), + ...(sanitizedParsed.displayRawS3Key ? { displayRawS3Key: sanitizedParsed.displayRawS3Key } : {}), }; this.logger.trackPoint("email_parsed"); @@ -2108,6 +2109,7 @@ function buildSignal(opts: { ...(parsed.sentAt !== undefined ? { sentAt: parsed.sentAt } : {}), ...(unsubscribe !== undefined ? { unsubscribe } : {}), ...(parsed.inlineImages && parsed.inlineImages.length > 0 ? { inlineImages: parsed.inlineImages } : {}), + ...(parsed.displayRawS3Key ? { displayRawS3Key: parsed.displayRawS3Key } : {}), }, }; diff --git a/src/types/index.ts b/src/types/index.ts index 28940cd5..de1ad178 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -460,6 +460,13 @@ export interface InboundEmailSignalData extends EmailSignalDataBase { htmlBodyTruncated?: boolean; /** Inline CID images stored via S3 (not embedded as data URIs) — see InlineImageRef. */ inlineImages?: InlineImageRef[]; + /** + * S3 key (in the content bucket, alongside extracted attachments) of a display-safe + * copy of the raw MIME source — attachments stripped, small inline images kept. Served + * by the "view original email" / download-as-.eml feature in place of the true raw + * original (which stays at `s3Key` for internal use only, e.g. reprocessing). + */ + displayRawS3Key?: string; } // --------------------------------------------------------------------------- diff --git a/tests/isolated/content-sanitizer-display-raw.spec.ts b/tests/isolated/content-sanitizer-display-raw.spec.ts new file mode 100644 index 00000000..33c30e7a --- /dev/null +++ b/tests/isolated/content-sanitizer-display-raw.spec.ts @@ -0,0 +1,78 @@ +import { describe, it, expect, vi, afterEach } from "vitest"; +import { handler } from "../../src/isolated/content-sanitizer.js"; + +// --------------------------------------------------------------------------- +// The sanitizer uploads a display-safe copy of the raw MIME (see +// raw-email-display.ts) alongside the extracted attachments, and returns its +// s3Key so the processor can persist it. This is what "view original email" / +// download-as-.eml serves — never the true raw original. +// --------------------------------------------------------------------------- + +const RAW_EMAIL_WITH_ATTACHMENT = [ + "From: sender@example.com", + "To: recipient@example.com", + "Subject: Test email with attachment", + "MIME-Version: 1.0", + 'Content-Type: multipart/mixed; boundary="----=_Part_boundary"', + "", + "------=_Part_boundary", + 'Content-Type: text/plain; charset="UTF-8"', + "", + "Body text.", + "", + "------=_Part_boundary", + "Content-Type: application/pdf", + "Content-Transfer-Encoding: base64", + 'Content-Disposition: attachment; filename="document.pdf"', + "", + "QQ==".repeat(50), + "", + "------=_Part_boundary--", +].join("\r\n"); + +function mockFetch(raw: string, uploads: Array<{ key: string; body: string }>) { + const buf = Buffer.from(raw, "utf-8"); + vi.stubGlobal("fetch", vi.fn(async (url: unknown, init?: RequestInit) => { + if (url === "https://example.com/get") { + return { ok: true, status: 200, arrayBuffer: async () => buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength) }; + } + if (url === "https://example.com/post" && init?.body instanceof FormData) { + const key = init.body.get("key") as string; + const file = init.body.get("file") as File; + uploads.push({ key, body: await file.text() }); + return { ok: true, status: 204 }; + } + return { ok: true, status: 204 }; + })); +} + +afterEach(() => { + vi.unstubAllGlobals(); +}); + +describe("content-sanitizer — display-safe raw email upload", () => { + it("uploads a display-safe raw copy with attachments stripped and returns its s3Key", async () => { + const uploads: Array<{ key: string; body: string }> = []; + mockFetch(RAW_EMAIL_WITH_ATTACHMENT, uploads); + + const result = await handler({ + presignedGetUrl: "https://example.com/get", + presignedPost: { url: "https://example.com/post", fields: {} }, + accountId: "acct-test", + senderEtld1: "example.com", + keyPrefix: "emails/msg-display-raw/", + retentionTag: null, + }); + + expect(result.success).toBe(true); + if (!result.success) return; + + expect(result.parsed.displayRawS3Key).toBe("emails/msg-display-raw/raw-display.eml"); + + const displayUpload = uploads.find(u => u.key === "emails/msg-display-raw/raw-display.eml"); + expect(displayUpload).toBeDefined(); + expect(displayUpload!.body).toContain("Body text."); + expect(displayUpload!.body).toContain("[attachment content omitted: document.pdf"); + expect(displayUpload!.body).not.toContain("QQ==QQ=="); + }); +}); diff --git a/tests/isolated/raw-email-display.spec.ts b/tests/isolated/raw-email-display.spec.ts new file mode 100644 index 00000000..27733c66 --- /dev/null +++ b/tests/isolated/raw-email-display.spec.ts @@ -0,0 +1,127 @@ +import { describe, it, expect } from "vitest"; +import { buildDisplayRawEmail } from "../../src/isolated/raw-email-display.js"; + +// --------------------------------------------------------------------------- +// buildDisplayRawEmail produces the copy served by "view original email" / +// download-as-.eml: attachments fully stripped, small inline images kept +// (needed for the .eml to still render if opened in a real mail client), +// bounded by a per-image cap and a cumulative budget across the message. +// --------------------------------------------------------------------------- + +function buildRawEmail(parts: Array<{ headers: string[]; body: string }>, boundary = "----=_Part_test_boundary"): string { + const body = parts.map(p => [`--${boundary}`, ...p.headers, "", p.body, ""].join("\r\n")).join(""); + return [ + "From: sender@example.com", + "To: recipient@example.com", + "Subject: Test", + "MIME-Version: 1.0", + `Content-Type: multipart/mixed; boundary="${boundary}"`, + "", + body + `--${boundary}--`, + ].join("\r\n"); +} + +describe("buildDisplayRawEmail", () => { + it("strips a regular attachment body, keeping its headers", () => { + const raw = buildRawEmail([ + { headers: ['Content-Type: text/plain; charset="UTF-8"'], body: "Body text." }, + { + headers: [ + "Content-Type: application/pdf", + "Content-Transfer-Encoding: base64", + 'Content-Disposition: attachment; filename="document.pdf"', + ], + body: "QQ==".repeat(50), + }, + ]); + + const result = buildDisplayRawEmail(raw); + expect(result).toContain('Content-Disposition: attachment; filename="document.pdf"'); + expect(result).toContain("[attachment content omitted: document.pdf"); + expect(result).not.toContain("QQ==QQ=="); + expect(result).toContain("Body text."); + }); + + it("keeps a small inline image's bytes untouched", () => { + const raw = buildRawEmail([ + { headers: ['Content-Type: text/html; charset="UTF-8"'], body: '' }, + { + headers: [ + "Content-Type: image/png", + "Content-Transfer-Encoding: base64", + "Content-ID: ", + 'Content-Disposition: inline; filename="logo.png"', + ], + body: "aGVsbG8=".repeat(10), // well under 100KB + }, + ]); + + const result = buildDisplayRawEmail(raw); + expect(result).toContain("aGVsbG8=".repeat(10)); + expect(result).not.toContain("[inline image omitted"); + }); + + it("truncates an inline image over the 100KB per-image cap", () => { + const bigBody = "A".repeat(140_000); // ~105KB decoded + const raw = buildRawEmail([ + { + headers: [ + "Content-Type: image/png", + "Content-Transfer-Encoding: base64", + "Content-ID: ", + 'Content-Disposition: inline; filename="logo.png"', + ], + body: bigBody, + }, + ]); + + const result = buildDisplayRawEmail(raw); + expect(result).toContain("[inline image omitted: exceeds display size limit"); + expect(result).not.toContain(bigBody); + }); + + it("truncates inline images once the cumulative 300KB budget is exhausted, even if individually under the per-image cap", () => { + // Each image ~90KB decoded (under the 100KB per-image cap), but 4 of them exceed + // the 300KB cumulative budget, so the 4th must be truncated. + const imageBody = "A".repeat(120_000); // ~90KB decoded + const raw = buildRawEmail([ + { headers: ["Content-Type: image/png", "Content-Transfer-Encoding: base64", "Content-ID: ", 'Content-Disposition: inline; filename="1.png"'], body: imageBody }, + { headers: ["Content-Type: image/png", "Content-Transfer-Encoding: base64", "Content-ID: ", 'Content-Disposition: inline; filename="2.png"'], body: imageBody }, + { headers: ["Content-Type: image/png", "Content-Transfer-Encoding: base64", "Content-ID: ", 'Content-Disposition: inline; filename="3.png"'], body: imageBody }, + { headers: ["Content-Type: image/png", "Content-Transfer-Encoding: base64", "Content-ID: ", 'Content-Disposition: inline; filename="4.png"'], body: imageBody }, + ]); + + const result = buildDisplayRawEmail(raw); + const omittedCount = (result.match(/\[inline image omitted/g) ?? []).length; + expect(omittedCount).toBeGreaterThanOrEqual(1); + // The kept images' bytes must still be present for at least the first ones + expect(result).toContain(imageBody.slice(0, 100)); + }); + + it("leaves non-multipart plain-text emails unchanged", () => { + const raw = [ + "From: sender@example.com", + "To: recipient@example.com", + "Subject: Plain text", + "Content-Type: text/plain; charset=\"UTF-8\"", + "MIME-Version: 1.0", + "", + "Just a plain message.", + ].join("\r\n"); + + expect(buildDisplayRawEmail(raw)).toBe(raw); + }); + + it("leaves the visible text/html body untouched", () => { + const raw = buildRawEmail([ + { headers: ['Content-Type: text/html; charset="UTF-8"'], body: "

Hello world

" }, + { + headers: ["Content-Type: application/pdf", "Content-Transfer-Encoding: base64", 'Content-Disposition: attachment; filename="a.pdf"'], + body: "QQ==".repeat(20), + }, + ]); + + const result = buildDisplayRawEmail(raw); + expect(result).toContain("

Hello world

"); + }); +});