diff --git a/src/api/signal-transforms.ts b/src/api/signal-transforms.ts index 251937ec..4f990b6c 100644 --- a/src/api/signal-transforms.ts +++ b/src/api/signal-transforms.ts @@ -4,11 +4,14 @@ import type { Thread as DbThread, AnySignal, + Attachment, EmailSignalData, + InboundEmailSignalData, DeliverabilitySignalData, MatchedRuleResult, Signal as DbSignal, } from "../types/index.js"; +import { isEmailSignal } from "../types/index.js"; import type * as Api from "./schemas.js"; // matchedRules is an append-only trace (e.g. a signal dismissed from quarantine gets a second @@ -21,6 +24,41 @@ function collapseMatchedRules(rules: MatchedRuleResult[]): MatchedRuleResult[] { return [...byRuleId.values()]; } +function escapeRegExp(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +// Resolves what the content sanitizer left as s3Key-only references (never a baked-in URL, +// so CDN config can change without a data migration) into CDN urls, at API read time: +// - Attachment.s3Key -> Attachment.url, as before. +// - Any `cid:{contentId}` left unresolved in htmlBody (an inline image too large, or past +// the per-message budget, to embed as a data URI — see MAX_INLINE_DATA_URI_SIZE / +// MAX_INLINE_DATA_URI_COUNT in the content sanitizer) -> the matching InlineImageRef's CDN url. +// inlineImages itself is intentionally not included on the returned signal — it's write-side +// plumbing for this substitution, not something the client needs. +export function withResolvedContentUrls(signal: T, cdnBase: string): T { + if (!isEmailSignal(signal)) return signal; + + const attachments = signal.data.attachments.map((a: Attachment) => ({ ...a, url: `${cdnBase}/${a.s3Key}` })); + + const inlineImages = (signal.data as InboundEmailSignalData).inlineImages; + let htmlBody = (signal.data as InboundEmailSignalData).htmlBody; + if (htmlBody && inlineImages && inlineImages.length > 0) { + for (const ref of inlineImages) { + htmlBody = htmlBody.replace(new RegExp(`cid:${escapeRegExp(ref.contentId)}`, "g"), `${cdnBase}/${ref.s3Key}`); + } + } + + return { + ...signal, + data: { + ...signal.data, + attachments, + ...(htmlBody !== undefined ? { htmlBody } : {}), + }, + } as T; +} + export function toApiThread(thread: DbThread): Api.Thread { return { threadId: thread.id, diff --git a/src/api/signalsApi.ts b/src/api/signalsApi.ts index 7ab55438..a0b20f38 100644 --- a/src/api/signalsApi.ts +++ b/src/api/signalsApi.ts @@ -3,14 +3,14 @@ import type { OpenAPIHono } from "@hono/zod-openapi"; import { DateTime } from "luxon"; import { getDomain } from "tldts"; import { zParse } from "./validate.js"; -import { toApiThread, toApiSignal } from "./signal-transforms.js"; +import { toApiThread, toApiSignal, withResolvedContentUrls } from "./signal-transforms.js"; import { deriveGroupingKey } from "../grouping-key.js"; import { handlePostApprovalCalendar } from "../processor/calendar/post-approval-handler.js"; import { resolveRetention } from "../retention.js"; import { buildActiveThread } from "../thread-factory.js"; import { isEmailSignal } from "../types/index.js"; import type { Result } from "neverthrow"; -import type { Thread, Signal, AnySignal, Attachment, MatchedRuleResult, PageParams } from "../types/index.js"; +import type { Thread, Signal, MatchedRuleResult, PageParams } from "../types/index.js"; import type { Pagination } from "../types/index.js"; import type { ThreadDatabase } from "../database/thread-database.js"; import type { AccountDatabase } from "../database/account-database.js"; @@ -29,11 +29,6 @@ function page(key: K, items: T[], nextCursor?: string): Rec return { [key]: items, pagination: { cursor: nextCursor ?? null } } as Record & { pagination: Pagination }; } -function withAttachmentUrls(signal: T, cdnBase: string): T { - if (!isEmailSignal(signal)) return signal; - return { ...signal, data: { ...signal.data, attachments: signal.data.attachments.map((a: Attachment) => ({ ...a, url: `${cdnBase}/${a.s3Key}` })) } }; -} - export class SignalsApi { constructor( private readonly threadDb: ThreadDatabase, @@ -99,7 +94,7 @@ export class SignalsApi { items = items.filter(s => isEmailSignal(s) && s.data.from.address.toLowerCase().includes(senderLower)); } - const itemsWithUrls = contentCdnBaseUrl ? items.map(s => withAttachmentUrls(s, contentCdnBaseUrl)) : items; + const itemsWithUrls = contentCdnBaseUrl ? items.map(s => withResolvedContentUrls(s, contentCdnBaseUrl)) : items; return c.json(page("signals", itemsWithUrls.map(toApiSignal), result.value.nextCursor), 200); }); @@ -238,7 +233,7 @@ export class SignalsApi { } } - const signalWithUrls = contentCdnBaseUrl ? withAttachmentUrls(signal, contentCdnBaseUrl) : signal; + const signalWithUrls = contentCdnBaseUrl ? withResolvedContentUrls(signal, contentCdnBaseUrl) : signal; logger.info("Signal activated", { code: "api.signals.activated", accountId, signalId, threadId: thread.id }); return c.json({ thread: toApiThread(thread), signal: toApiSignal({ ...signalWithUrls, status: "active", threadId: thread.id }) }, 200); }); diff --git a/src/api/threadsApi.ts b/src/api/threadsApi.ts index e9bf551c..97a5a99b 100644 --- a/src/api/threadsApi.ts +++ b/src/api/threadsApi.ts @@ -6,12 +6,12 @@ import { getDomain } from "tldts"; import { validateRecipientMx } from "../dns/mx-validator.js"; import { computeUndoWindowSeconds } from "./undo-window.js"; import { zParse } from "./validate.js"; -import { toApiThread, toApiSignal } from "./signal-transforms.js"; +import { toApiThread, toApiSignal, withResolvedContentUrls } from "./signal-transforms.js"; import { buildScheduleName } from "../scheduler/schedule-name.js"; import { durationToSeconds } from "../retention.js"; import { isCalendarEventSignal, isEmailSignal } from "../types/index.js"; import type { EmailContentStore } from "./content-store.js"; -import type { Signal, AnySignal, Attachment, PageParams, ThreadStatus, Workflow } from "../types/index.js"; +import type { Signal, AnySignal, InboundEmailSignalData, PageParams, ThreadStatus, Workflow } from "../types/index.js"; import type { CalendarResponseData, DomainMisconfigurationData, Pagination } from "../types/index.js"; import type { UpdateThreadFields, ThreadDatabase } from "../database/thread-database.js"; import type { AccountDatabase } from "../database/account-database.js"; @@ -51,11 +51,6 @@ function page(key: K, items: T[], nextCursor?: string): Rec return { [key]: items, pagination: { cursor: nextCursor ?? null } } as Record & { pagination: Pagination }; } -function withAttachmentUrls(signal: T, cdnBase: string): T { - if (!isEmailSignal(signal)) return signal; - return { ...signal, data: { ...signal.data, attachments: signal.data.attachments.map((a: Attachment) => ({ ...a, url: `${cdnBase}/${a.s3Key}` })) } }; -} - export class ThreadsApi { constructor( private readonly threadDb: ThreadDatabase, @@ -334,7 +329,7 @@ export class ThreadsApi { } const enrichedSignals = signals.map(signal => { - const withUrls = contentCdnBaseUrl ? withAttachmentUrls(signal, contentCdnBaseUrl) : signal; + const withUrls = contentCdnBaseUrl ? withResolvedContentUrls(signal, contentCdnBaseUrl) : signal; const apiSignal = toApiSignal(withUrls); if (isCalendarEventSignal(withUrls) && enrichments.has(withUrls.data.veventUid)) { return { ...apiSignal, latestResponse: enrichments.get(withUrls.data.veventUid) }; @@ -771,7 +766,7 @@ export class ThreadsApi { } const signal = signalResult.value; if (!signal) return err(c, 404, "Signal not found", "SIGNAL_NOT_FOUND"); - const withUrls = contentCdnBaseUrl ? withAttachmentUrls(signal, contentCdnBaseUrl) : signal; + const withUrls = contentCdnBaseUrl ? withResolvedContentUrls(signal, contentCdnBaseUrl) : signal; return c.json(toApiSignal(withUrls), 200); }); @@ -798,6 +793,15 @@ export class ThreadsApi { if (!isEmailSignal(signal)) return err(c, 400, "Signal is not an email", "SIGNAL_NOT_FOUND"); if (!signal.data.s3Key) return err(c, 404, "Raw email not available", "SIGNAL_NOT_FOUND"); + // The display-safe copy (attachments stripped, small inline images kept — built by + // the content sanitizer, see raw-email-display.ts) is what this endpoint serves. + // The true raw original at signal.data.s3Key is never exposed through this path — + // it stays available server-side only, e.g. for reprocessing. + const displayRawS3Key = (signal.data as InboundEmailSignalData).displayRawS3Key; + if (displayRawS3Key && contentCdnBaseUrl) { + return c.redirect(`${contentCdnBaseUrl}/${displayRawS3Key}`, 307); + } + const url = await emailContentStore.getRawEmailUrl(signal); return c.redirect(url, 307); }); diff --git a/src/isolated/content-sanitizer.ts b/src/isolated/content-sanitizer.ts index 2013e634..504a89e0 100644 --- a/src/isolated/content-sanitizer.ts +++ b/src/isolated/content-sanitizer.ts @@ -2,6 +2,7 @@ import { simpleParser } from "mailparser"; import { Window } from "happy-dom"; import { sanitizeHtml } from "./html-sanitizer.js"; import { extractAssets, type ExtractedAsset } from "./asset-extractor.js"; +import { buildDisplayRawEmail } from "./raw-email-display.js"; import type { Logger } from "../logger.js"; // Lambda timeout is 60s (see deploy/compute.tf). Emitting a TRACK log past this @@ -9,6 +10,11 @@ import type { Logger } from "../logger.js"; // actually failing — mirrors the api.slow_request pattern in src/api/app.ts. const SLOW_INVOCATION_THRESHOLD_MS = 50_000; +// Lower-bar threshold so a message that's merely slow to parse (large/complex MIME, +// many attachments) shows up before it gets anywhere near the 50s near-timeout alert — +// gives us an early signal to look at parsing performance, not just imminent failures. +const SLOW_PARSE_THRESHOLD_MS = 10_000; + // --------------------------------------------------------------------------- // Types // --------------------------------------------------------------------------- @@ -25,6 +31,20 @@ interface AttachmentRef { s3Key: string; } +interface DroppedAttachment { + filename: string; + mimeType: string; + sizeBytes: number; + reason: "too_large" | "upload_failed"; +} + +interface InlineImageRef { + contentId: string; + mimeType: string; + sizeBytes: number; + s3Key: string; +} + interface ContentSanitizeRequest { presignedGetUrl: string; presignedPost: { @@ -58,6 +78,9 @@ interface ContentSanitizeResponse { sentAt?: string; assets?: ExtractedAsset[]; links?: ExtractedLink[]; + droppedAttachments?: DroppedAttachment[]; + inlineImages?: InlineImageRef[]; + displayRawS3Key?: string; }; } @@ -77,6 +100,16 @@ const MAX_ATTACHMENTS = 50; const MAX_TOTAL_ATTACHMENT_SIZE = 25 * 1024 * 1024; // 25MB const MAX_SINGLE_ATTACHMENT_SIZE = 10 * 1024 * 1024; // 10MB +// DynamoDB's per-item cap is 400KB; MAX_HTML_BODY_BYTES in processor.ts already reserves +// 300KB of that for htmlBody as a last-resort truncation guard. These two caps keep most +// messages from ever needing that guard: a data URI runs ~33% larger than its source bytes, +// so 3 images at 100KB each tops out around 400KB of embedded text — big enough for a +// typical logo/signature image, small enough that inline images can't silently balloon +// htmlBody the way an unbounded embed would. Anything past either cap is uploaded to S3 +// instead (see InlineImageRef) and resolved to a CDN url at API read time. +const MAX_INLINE_DATA_URI_SIZE = 100 * 1024; // 100KB +const MAX_INLINE_DATA_URI_COUNT = 3; + // --------------------------------------------------------------------------- // Helpers // --------------------------------------------------------------------------- @@ -212,12 +245,38 @@ async function processEmail(event: ContentSanitizeRequest, logger?: Logger): Pro }; } + // 1b. Build a display-safe copy of the raw MIME (attachments stripped, small inline + // images kept — see raw-email-display.ts) and upload it. This is what the "view original + // email" / download-as-.eml feature serves; the true original is never exposed through + // that path. Best-effort — a failure here must not fail the whole sanitize, the feature + // just falls back to unavailable for this message. + logger?.trackPoint("display_raw_build_start"); + let displayRawS3Key: string | undefined; + try { + const displayRaw = buildDisplayRawEmail(rawMime.toString("latin1")); + const key = `${event.keyPrefix}raw-display.eml`; + const uploaded = await uploadViaPresignedPost(event.presignedPost, key, Buffer.from(displayRaw, "latin1"), "message/rfc822", event.retentionTag); + if (uploaded) displayRawS3Key = key; + logger?.trackPoint("display_raw_build_complete", { uploaded }); + } catch { + logger?.trackPoint("display_raw_build_failed"); + } + // 2. Parse with mailparser logger?.trackPoint("mime_parse_start"); + const parseStart = Date.now(); let parsed; try { - parsed = await simpleParser(rawMime); + // keepCidLinks: mailparser's default behaviour is to eagerly replace every `cid:` + // reference in the HTML with a base64 data URI itself, before this file ever sees the + // parsed message — which would silently bypass the inline-image size/count budget below. + // With this set, `cid:` references are left as-is in parsed.html for us to resolve. + parsed = await simpleParser(rawMime, { keepCidLinks: true }); + const parseElapsedMs = Date.now() - parseStart; logger?.trackPoint("mime_parse_complete"); + if (parseElapsedMs > SLOW_PARSE_THRESHOLD_MS) { + logger?.track("MIME parse exceeded 10s.", { code: "content_sanitizer.slow_parse", elapsedMs: parseElapsedMs, sizeBytes: rawMime.length, accountId: event.accountId }); + } } catch (e) { return { success: false, @@ -258,19 +317,54 @@ async function processEmail(event: ContentSanitizeRequest, logger?: Logger): Pro const cidMap: Record = {}; const inlineImages: Array<{ mimeType: string; content: Buffer }> = []; const attachmentsWithBytes: Array<{ filename: string; mimeType: string; content: Buffer; s3Key?: string }> = []; + const droppedAttachments: DroppedAttachment[] = []; + const inlineImageRefs: InlineImageRef[] = []; + let inlinedDataUriCount = 0; for (const attachment of attachments) { + const contentType = attachment.contentType || "application/octet-stream"; + if (attachment.size > MAX_SINGLE_ATTACHMENT_SIZE) { + droppedAttachments.push({ + filename: attachment.filename ?? `attachment-${uploadIndex}`, + mimeType: contentType, + sizeBytes: attachment.size, + reason: "too_large", + }); continue; } - const contentType = attachment.contentType || "application/octet-stream"; - - if (attachment.contentId) { - // Inline image — embed as data URI, no S3 upload needed - cidMap[attachment.contentId] = `data:${contentType};base64,${attachment.content.toString("base64")}`; - if (contentType.startsWith("image/")) { - inlineImages.push({ mimeType: contentType, content: attachment.content }); + // `cid` (not `contentId`) is the bracket-stripped form mailparser itself matches + // `cid:` references against — it's what appears bare in ``. + if (attachment.cid) { + const cid = attachment.cid; + const fitsInlineBudget = attachment.size <= MAX_INLINE_DATA_URI_SIZE && inlinedDataUriCount < MAX_INLINE_DATA_URI_COUNT; + + if (fitsInlineBudget) { + // Small inline image, still within the per-message budget — embed as a data URI, no S3 upload needed + cidMap[cid] = `data:${contentType};base64,${attachment.content.toString("base64")}`; + inlinedDataUriCount++; + if (contentType.startsWith("image/")) { + inlineImages.push({ mimeType: contentType, content: attachment.content }); + } + } else { + // Too large, or past the per-message inline budget — upload to S3 like a regular + // attachment instead of embedding. Its `cid:` reference is deliberately left + // unresolved below; the API layer resolves it to a CDN url at read time. + const s3Key = `${event.keyPrefix}${uploadIndex}`; + const uploaded = await uploadViaPresignedPost(event.presignedPost, s3Key, attachment.content, contentType, event.retentionTag); + const filename = attachment.filename ?? `inline-${uploadIndex}`; + + if (uploaded) { + inlineImageRefs.push({ contentId: cid, mimeType: contentType, sizeBytes: attachment.size, s3Key }); + // Still eligible for QR scanning via the attachment-image path in extractAssets + if (contentType.startsWith("image/")) { + attachmentsWithBytes.push({ filename, mimeType: contentType, content: attachment.content, s3Key }); + } + } else { + droppedAttachments.push({ filename, mimeType: contentType, sizeBytes: attachment.size, reason: "upload_failed" }); + } + uploadIndex++; } continue; } @@ -278,15 +372,25 @@ async function processEmail(event: ContentSanitizeRequest, logger?: Logger): Pro const s3Key = `${event.keyPrefix}${uploadIndex}`; const uploaded = await uploadViaPresignedPost(event.presignedPost, s3Key, attachment.content, contentType, event.retentionTag); + const filename = attachment.filename ?? `attachment-${uploadIndex}`; if (uploaded) { - const filename = attachment.filename ?? `attachment-${uploadIndex}`; attachmentRefs.push({ filename, mimeType: contentType, sizeBytes: attachment.size, s3Key }); attachmentsWithBytes.push({ filename, mimeType: contentType, content: attachment.content, s3Key }); + } else { + droppedAttachments.push({ filename, mimeType: contentType, sizeBytes: attachment.size, reason: "upload_failed" }); } uploadIndex++; } - logger?.trackPoint("attachments_processed", { attachmentRefCount: attachmentRefs.length, inlineImageCount: inlineImages.length }); + logger?.trackPoint("attachments_processed", { attachmentRefCount: attachmentRefs.length, inlineImageCount: inlineImages.length, inlineImageRefCount: inlineImageRefs.length, droppedCount: droppedAttachments.length }); + if (droppedAttachments.length > 0) { + logger?.track("Attachment(s) dropped from message.", { + code: "content_sanitizer.attachments_dropped", + accountId: event.accountId, + droppedCount: droppedAttachments.length, + dropped: droppedAttachments.map(d => ({ mimeType: d.mimeType, sizeBytes: d.sizeBytes, reason: d.reason })), + }); + } // 6. Sanitize HTML and inline CID images let htmlBody: string | undefined; @@ -295,7 +399,12 @@ async function processEmail(event: ContentSanitizeRequest, logger?: Logger): Pro if (parsed.html) { const htmlInput = typeof parsed.html === "string" ? parsed.html : ""; const sanitized = sanitizeHtml(htmlInput); - htmlBody = sanitized.html.replace(/cid:([^"'\s>]+)/g, (_, id: string) => cidMap[id] ?? ""); + const inlineImageContentIds = new Set(inlineImageRefs.map(ref => ref.contentId)); + htmlBody = sanitized.html.replace(/cid:([^"'\s>]+)/g, (match, id: string) => { + if (id in cidMap) return cidMap[id] ?? ""; + if (inlineImageContentIds.has(id)) return match; // resolved to a CDN url at API read time + return ""; + }); logger?.trackPoint("html_sanitized", { htmlLength: htmlInput.length }); // Extract links from raw HTML using a DOM parser (before sanitization strips tracking pixels etc.) @@ -402,6 +511,15 @@ async function processEmail(event: ContentSanitizeRequest, logger?: Logger): Pro if (extractedLinks.length > 0) { result.parsed.links = extractedLinks; } + if (droppedAttachments.length > 0) { + result.parsed.droppedAttachments = droppedAttachments; + } + if (inlineImageRefs.length > 0) { + result.parsed.inlineImages = inlineImageRefs; + } + if (displayRawS3Key) { + result.parsed.displayRawS3Key = displayRawS3Key; + } logger?.trackPoint("response_built"); return result; diff --git a/src/isolated/raw-email-display.ts b/src/isolated/raw-email-display.ts new file mode 100644 index 00000000..ba28e35e --- /dev/null +++ b/src/isolated/raw-email-display.ts @@ -0,0 +1,122 @@ +/** + * Builds a display-safe copy of the raw MIME source: a byte-faithful copy of the + * original with attachment bodies stripped out entirely, except for small inline + * images (kept so the message still renders correctly if the .eml is opened in a + * real mail client — those clients resolve `cid:` references against the file's + * own MIME parts, unlike our app's htmlBody rendering, which resolves them + * against a CDN url instead). This is the copy served by the "view original + * email" / download-as-.eml feature; the true, unmodified original stays in S3 + * for internal use (e.g. reprocessing) and is never served through that path. + * + * This is a textual boundary walk, not a semantic MIME parse — it never decodes + * or interprets attachment bytes, only locates part boundaries/headers to decide + * what to keep. Operates on the raw MIME text as fetched, inside the sanitizer's + * security boundary (see docs/adr/011-content-sanitizer-security-boundary.md). + */ + +// Per-image cap: an inline image bigger than this is truncated regardless of +// how much of the cumulative budget remains. +const MAX_DISPLAY_INLINE_IMAGE_SIZE = 100 * 1024; // 100KB + +// Cumulative cap across all inline images kept in one message — once reached, +// every remaining inline image is truncated even if individually under the +// per-image cap. +const MAX_DISPLAY_INLINE_TOTAL_SIZE = 300 * 1024; // 300KB + +function escapeRegExp(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +function formatBytes(bytes: number): string { + if (bytes < 1024) return `${bytes} B`; + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`; + return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; +} + +export function buildDisplayRawEmail(raw: string): string { + const boundaries = new Set(); + const boundaryRegex = /boundary\s*=\s*"?([^";\r\n]+)"?/gi; + let boundaryMatch: RegExpExecArray | null; + while ((boundaryMatch = boundaryRegex.exec(raw))) { + boundaries.add(boundaryMatch[1]!); + } + if (boundaries.size === 0) return raw; // not multipart — nothing to strip + + const boundaryLineRegex = new RegExp( + `^--(${[...boundaries].map(escapeRegExp).join("|")})(--)?\\s*$`, + ); + + const lines = raw.split(/\r\n|\n/); + const output: string[] = []; + let i = 0; + let inlineBudgetRemaining = MAX_DISPLAY_INLINE_TOTAL_SIZE; + + // Preamble / top-level headers before the first boundary — unchanged + while (i < lines.length && !boundaryLineRegex.test(lines[i]!)) { + output.push(lines[i]!); + i++; + } + + while (i < lines.length) { + const boundaryLine = lines[i]!; + output.push(boundaryLine); + i++; + if (/--\s*$/.test(boundaryLine)) break; // closing boundary — no part follows + + const headerLines: string[] = []; + while (i < lines.length && lines[i] !== "" && !boundaryLineRegex.test(lines[i]!)) { + headerLines.push(lines[i]!); + output.push(lines[i]!); + i++; + } + if (i < lines.length && lines[i] === "") { + output.push(lines[i]!); + i++; + } + + const bodyStart = i; + while (i < lines.length && !boundaryLineRegex.test(lines[i]!)) { + i++; + } + const bodyLines = lines.slice(bodyStart, i); + const bodyHasContent = bodyLines.some(line => line.trim() !== ""); + + if (!bodyHasContent) { + output.push(...bodyLines); + continue; + } + + const headerText = headerLines.join("\n"); + const isInlineImage = /Content-Disposition:\s*inline/i.test(headerText) && /Content-Type:\s*image\//i.test(headerText); + const isFilePart = + /Content-Disposition:\s*attachment/i.test(headerText) || + /Content-Disposition:[^\r\n]*\bfilename\*?=/i.test(headerText) || + /Content-Type:[^\r\n]*\bname\s*=/i.test(headerText); + + if (isInlineImage) { + const approxBytes = Math.floor(bodyLines.join("").length * 0.75); // base64 -> raw bytes estimate + if (approxBytes <= MAX_DISPLAY_INLINE_IMAGE_SIZE && approxBytes <= inlineBudgetRemaining) { + output.push(...bodyLines); + inlineBudgetRemaining -= approxBytes; + } else { + output.push(`[inline image omitted: exceeds display size limit (~${formatBytes(approxBytes)})]`); + output.push(""); + } + continue; + } + + if (isFilePart) { + const filenameMatch = + headerText.match(/filename\*?=\s*"?([^";\r\n]+)"?/i) ?? headerText.match(/name\s*=\s*"?([^";\r\n]+)"?/i); + const filename = filenameMatch ? filenameMatch[1] : null; + const approxBytes = Math.floor(bodyLines.join("").length * 0.75); + output.push(`[attachment content omitted${filename ? `: ${filename}` : ""} (~${formatBytes(approxBytes)})]`); + output.push(""); + continue; + } + + output.push(...bodyLines); + } + + return output.join("\r\n"); +} diff --git a/src/processor/content-sanitizer-client.ts b/src/processor/content-sanitizer-client.ts index 7f42f9ee..a6886d95 100644 --- a/src/processor/content-sanitizer-client.ts +++ b/src/processor/content-sanitizer-client.ts @@ -20,6 +20,20 @@ interface AttachmentRef { s3Key: string; } +export interface DroppedAttachment { + filename: string; + mimeType: string; + sizeBytes: number; + reason: "too_large" | "upload_failed"; +} + +export interface InlineImageRef { + contentId: string; + mimeType: string; + sizeBytes: number; + s3Key: string; +} + export interface ContentSanitizeRequest { presignedGetUrl: string; presignedPost: { @@ -59,6 +73,9 @@ export interface ContentSanitizeResponse { sentAt?: string; assets?: ExtractedAsset[]; links?: ExtractedLink[]; + droppedAttachments?: DroppedAttachment[]; + inlineImages?: InlineImageRef[]; + displayRawS3Key?: string; }; } diff --git a/src/processor/mime.ts b/src/processor/mime.ts index 1623305d..4e50c96d 100644 --- a/src/processor/mime.ts +++ b/src/processor/mime.ts @@ -1,4 +1,4 @@ -import type { EmailAddress, Attachment } from "../types/index.js"; +import type { EmailAddress, Attachment, InlineImageRef } from "../types/index.js"; import type { DbError, Result } from "../errors.js"; export interface ParsedMime { @@ -12,6 +12,8 @@ export interface ParsedMime { attachments: Attachment[]; headers: Record; sentAt?: string; + inlineImages?: InlineImageRef[]; + displayRawS3Key?: string; } export interface MimeParser { diff --git a/src/processor/processor.ts b/src/processor/processor.ts index 0948e9bb..c3bce11b 100644 --- a/src/processor/processor.ts +++ b/src/processor/processor.ts @@ -887,6 +887,15 @@ export class SignalProcessor { const { parsed: sanitizedParsed } = sanitizeResult.value; const sanitizerAssets = sanitizedParsed.assets ?? []; + if (sanitizedParsed.droppedAttachments && sanitizedParsed.droppedAttachments.length > 0) { + this.logger.warn("Message had attachment(s) dropped by content sanitizer", { + code: "processor.attachments_dropped", + accountId, + droppedCount: sanitizedParsed.droppedAttachments.length, + dropped: sanitizedParsed.droppedAttachments.map(d => ({ mimeType: d.mimeType, sizeBytes: d.sizeBytes, reason: d.reason })), + }); + } + // Map sanitized response to ParsedMime for downstream compatibility const parsed: ParsedMime = { from: sanitizedParsed.from, @@ -904,6 +913,8 @@ export class SignalProcessor { ...(sanitizedParsed.textBody !== undefined ? { textBody: sanitizedParsed.textBody } : {}), ...(sanitizedParsed.htmlBody !== undefined ? { htmlBody: sanitizedParsed.htmlBody } : {}), ...(sanitizedParsed.sentAt !== undefined ? { sentAt: sanitizedParsed.sentAt } : {}), + ...(sanitizedParsed.inlineImages ? { inlineImages: sanitizedParsed.inlineImages } : {}), + ...(sanitizedParsed.displayRawS3Key ? { displayRawS3Key: sanitizedParsed.displayRawS3Key } : {}), }; this.logger.trackPoint("email_parsed"); @@ -2097,6 +2108,8 @@ function buildSignal(opts: { ...(htmlBodyTruncated ? { htmlBodyTruncated: true } : {}), ...(parsed.sentAt !== undefined ? { sentAt: parsed.sentAt } : {}), ...(unsubscribe !== undefined ? { unsubscribe } : {}), + ...(parsed.inlineImages && parsed.inlineImages.length > 0 ? { inlineImages: parsed.inlineImages } : {}), + ...(parsed.displayRawS3Key ? { displayRawS3Key: parsed.displayRawS3Key } : {}), }, }; diff --git a/src/types/index.ts b/src/types/index.ts index 127e07eb..de1ad178 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -372,6 +372,19 @@ export interface Attachment { s3Key: string; } +// A CID-referenced inline image too large (or too numerous — see MAX_INLINE_DATA_URI_COUNT +// in the content sanitizer) to embed as a base64 data URI in htmlBody without risking the +// DynamoDB 400KB item cap. Uploaded to S3 like a regular Attachment instead; only the s3Key +// is stored, never the bytes. Its `cid:{contentId}` reference is left unresolved in htmlBody +// at write time and swapped for a CDN url at API read time (see withResolvedContentUrls in +// src/api/signal-transforms.ts), mirroring how Attachment.url is computed lazily from s3Key. +export interface InlineImageRef { + contentId: string; + mimeType: string; + sizeBytes: number; + s3Key: string; +} + // --------------------------------------------------------------------------- // MatchedRuleResult — per-rule trace written to Signal.matchedRules // --------------------------------------------------------------------------- @@ -445,6 +458,15 @@ export interface InboundEmailSignalData extends EmailSignalDataBase { htmlBody?: string; /** Set to true when htmlBody was truncated before storage. Full content recoverable from S3 via s3Key. */ htmlBodyTruncated?: boolean; + /** Inline CID images stored via S3 (not embedded as data URIs) — see InlineImageRef. */ + inlineImages?: InlineImageRef[]; + /** + * S3 key (in the content bucket, alongside extracted attachments) of a display-safe + * copy of the raw MIME source — attachments stripped, small inline images kept. Served + * by the "view original email" / download-as-.eml feature in place of the true raw + * original (which stays at `s3Key` for internal use only, e.g. reprocessing). + */ + displayRawS3Key?: string; } // --------------------------------------------------------------------------- diff --git a/tests/api/signal-transforms.spec.ts b/tests/api/signal-transforms.spec.ts index f9ee3b45..4293092b 100644 --- a/tests/api/signal-transforms.spec.ts +++ b/tests/api/signal-transforms.spec.ts @@ -1,5 +1,5 @@ import { describe, it, expect } from "vitest"; -import { toApiSignal } from "../../src/api/signal-transforms.js"; +import { toApiSignal, withResolvedContentUrls } from "../../src/api/signal-transforms.js"; import type { Signal } from "../../src/types/index.js"; function makeInboundSignal(overrides: { data?: Partial } & Partial> = {}): Signal { @@ -67,3 +67,62 @@ describe("toApiSignal — matchedRules collapse", () => { expect(data.matchedRules).toEqual([{ ruleId: "SR-02", actions: [{ type: "quarantine_visible" }], labelsAdded: [], statusChange: "quarantine_visible" }]); }); }); + +// --------------------------------------------------------------------------- +// withResolvedContentUrls — s3Key -> CDN url resolution, computed lazily at read +// time (never stored). Covers both Attachment.url and the inline-image cid: +// substitution for images the sanitizer routed to S3 instead of embedding as a +// data URI (see MAX_INLINE_DATA_URI_SIZE / MAX_INLINE_DATA_URI_COUNT). +// --------------------------------------------------------------------------- +describe("withResolvedContentUrls", () => { + it("adds a CDN url to each attachment computed from its s3Key", () => { + const signal = makeInboundSignal({ + data: { + attachments: [ + { filename: "doc.pdf", mimeType: "application/pdf", sizeBytes: 1024, s3Key: "emails/msg-001/0" }, + ], + }, + }); + + const result = withResolvedContentUrls(signal, "https://cdn.example.com"); + const attachments = result.data.attachments as Array<{ url?: string }>; + expect(attachments[0]?.url).toBe("https://cdn.example.com/emails/msg-001/0"); + }); + + it("replaces a leftover cid: reference in htmlBody with the matching inline image's CDN url", () => { + const signal = makeInboundSignal({ + data: { + htmlBody: '

Logo:

', + inlineImages: [{ contentId: "logo123", mimeType: "image/png", sizeBytes: 500_000, s3Key: "emails/msg-001/inline-0" }], + }, + }); + + const result = withResolvedContentUrls(signal, "https://cdn.example.com"); + expect((result.data as { htmlBody?: string }).htmlBody).toBe( + '

Logo:

', + ); + }); + + it("does not touch htmlBody when there are no inline images to resolve", () => { + const signal = makeInboundSignal({ data: { htmlBody: "

Hello

" } }); + const result = withResolvedContentUrls(signal, "https://cdn.example.com"); + expect((result.data as { htmlBody?: string }).htmlBody).toBe("

Hello

"); + }); + + it("resolves multiple inline images independently", () => { + const signal = makeInboundSignal({ + data: { + htmlBody: '', + inlineImages: [ + { contentId: "a", mimeType: "image/png", sizeBytes: 500_000, s3Key: "emails/msg-001/inline-0" }, + { contentId: "b", mimeType: "image/jpeg", sizeBytes: 600_000, s3Key: "emails/msg-001/inline-1" }, + ], + }, + }); + + const result = withResolvedContentUrls(signal, "https://cdn.example.com"); + expect((result.data as { htmlBody?: string }).htmlBody).toBe( + '', + ); + }); +}); diff --git a/tests/isolated/content-sanitizer-attachments.spec.ts b/tests/isolated/content-sanitizer-attachments.spec.ts new file mode 100644 index 00000000..6c982264 --- /dev/null +++ b/tests/isolated/content-sanitizer-attachments.spec.ts @@ -0,0 +1,123 @@ +import { describe, it, expect, vi, afterEach } from "vitest"; +import { handler } from "../../src/isolated/content-sanitizer.js"; + +// --------------------------------------------------------------------------- +// Oversized attachments were previously silently dropped: filtered out of the +// upload loop with no log line and no trace in the response, so there was no +// way to tell a "message with no attachments" apart from a "message whose +// attachment got silently discarded for being too big." These tests pin the +// current behaviour: dropped attachments must be reported in the response and +// logged as a TRACK. +// --------------------------------------------------------------------------- + +const BOUNDARY = "----=_Part_oversized_boundary"; + +function buildEmailWithOversizedAttachment(attachmentSize: number): string { + const attachment = Buffer.alloc(attachmentSize, "A"); + return [ + "From: sender@example.com", + "To: recipient@example.com", + "Subject: Message with a huge attachment", + "MIME-Version: 1.0", + `Content-Type: multipart/mixed; boundary="${BOUNDARY}"`, + "", + `--${BOUNDARY}`, + 'Content-Type: text/plain; charset="UTF-8"', + "", + "See attached.", + "", + `--${BOUNDARY}`, + "Content-Type: application/octet-stream", + "Content-Transfer-Encoding: base64", + 'Content-Disposition: attachment; filename="huge.bin"', + "", + attachment.toString("base64"), + "", + `--${BOUNDARY}--`, + ].join("\r\n"); +} + +function mockFetch(raw: string) { + const buf = Buffer.from(raw, "utf-8"); + vi.stubGlobal("fetch", vi.fn(async (url: string) => { + if (url === "https://example.com/get") { + return { + ok: true, + status: 200, + arrayBuffer: async () => buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength), + }; + } + // presigned POST upload — shouldn't be reached for an oversized attachment + return { ok: true, status: 204 }; + })); +} + +afterEach(() => { + vi.unstubAllGlobals(); + vi.restoreAllMocks(); +}); + +describe("content-sanitizer — oversized attachments", () => { + it("drops an attachment over the single-attachment size limit and reports it in the response", async () => { + mockFetch(buildEmailWithOversizedAttachment(11 * 1024 * 1024)); // > 10MB MAX_SINGLE_ATTACHMENT_SIZE + + const result = await handler({ + presignedGetUrl: "https://example.com/get", + presignedPost: { url: "https://example.com/post", fields: {} }, + accountId: "acct-test", + senderEtld1: "example.com", + keyPrefix: "emails/msg-oversized/", + retentionTag: null, + }); + + expect(result.success).toBe(true); + if (!result.success) return; + + expect(result.parsed.attachments).toEqual([]); + expect(result.parsed.droppedAttachments).toEqual([ + { filename: "huge.bin", mimeType: "application/octet-stream", sizeBytes: 11 * 1024 * 1024, reason: "too_large" }, + ]); + }); + + it("emits a TRACK log naming the dropped attachment count and reason", async () => { + mockFetch(buildEmailWithOversizedAttachment(11 * 1024 * 1024)); + const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + + await handler({ + presignedGetUrl: "https://example.com/get", + presignedPost: { url: "https://example.com/post", fields: {} }, + accountId: "acct-test", + senderEtld1: "example.com", + keyPrefix: "emails/msg-oversized/", + retentionTag: null, + invocationId: "inv-attachments-dropped", + }); + + const trackEntry = logSpy.mock.calls + .map(call => call[0]) + .find((entry): entry is Record => + typeof entry === "object" && entry !== null && (entry as Record).code === "content_sanitizer.attachments_dropped"); + + expect(trackEntry).toBeDefined(); + expect(trackEntry).toMatchObject({ level: "TRACK", droppedCount: 1 }); + }); + + it("does not drop attachments within the single-attachment size limit", async () => { + mockFetch(buildEmailWithOversizedAttachment(1024)); // well under the 10MB limit + + const result = await handler({ + presignedGetUrl: "https://example.com/get", + presignedPost: { url: "https://example.com/post", fields: {} }, + accountId: "acct-test", + senderEtld1: "example.com", + keyPrefix: "emails/msg-small/", + retentionTag: null, + }); + + expect(result.success).toBe(true); + if (!result.success) return; + + expect(result.parsed.attachments).toHaveLength(1); + expect(result.parsed.droppedAttachments).toBeUndefined(); + }); +}); diff --git a/tests/isolated/content-sanitizer-display-raw.spec.ts b/tests/isolated/content-sanitizer-display-raw.spec.ts new file mode 100644 index 00000000..33c30e7a --- /dev/null +++ b/tests/isolated/content-sanitizer-display-raw.spec.ts @@ -0,0 +1,78 @@ +import { describe, it, expect, vi, afterEach } from "vitest"; +import { handler } from "../../src/isolated/content-sanitizer.js"; + +// --------------------------------------------------------------------------- +// The sanitizer uploads a display-safe copy of the raw MIME (see +// raw-email-display.ts) alongside the extracted attachments, and returns its +// s3Key so the processor can persist it. This is what "view original email" / +// download-as-.eml serves — never the true raw original. +// --------------------------------------------------------------------------- + +const RAW_EMAIL_WITH_ATTACHMENT = [ + "From: sender@example.com", + "To: recipient@example.com", + "Subject: Test email with attachment", + "MIME-Version: 1.0", + 'Content-Type: multipart/mixed; boundary="----=_Part_boundary"', + "", + "------=_Part_boundary", + 'Content-Type: text/plain; charset="UTF-8"', + "", + "Body text.", + "", + "------=_Part_boundary", + "Content-Type: application/pdf", + "Content-Transfer-Encoding: base64", + 'Content-Disposition: attachment; filename="document.pdf"', + "", + "QQ==".repeat(50), + "", + "------=_Part_boundary--", +].join("\r\n"); + +function mockFetch(raw: string, uploads: Array<{ key: string; body: string }>) { + const buf = Buffer.from(raw, "utf-8"); + vi.stubGlobal("fetch", vi.fn(async (url: unknown, init?: RequestInit) => { + if (url === "https://example.com/get") { + return { ok: true, status: 200, arrayBuffer: async () => buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength) }; + } + if (url === "https://example.com/post" && init?.body instanceof FormData) { + const key = init.body.get("key") as string; + const file = init.body.get("file") as File; + uploads.push({ key, body: await file.text() }); + return { ok: true, status: 204 }; + } + return { ok: true, status: 204 }; + })); +} + +afterEach(() => { + vi.unstubAllGlobals(); +}); + +describe("content-sanitizer — display-safe raw email upload", () => { + it("uploads a display-safe raw copy with attachments stripped and returns its s3Key", async () => { + const uploads: Array<{ key: string; body: string }> = []; + mockFetch(RAW_EMAIL_WITH_ATTACHMENT, uploads); + + const result = await handler({ + presignedGetUrl: "https://example.com/get", + presignedPost: { url: "https://example.com/post", fields: {} }, + accountId: "acct-test", + senderEtld1: "example.com", + keyPrefix: "emails/msg-display-raw/", + retentionTag: null, + }); + + expect(result.success).toBe(true); + if (!result.success) return; + + expect(result.parsed.displayRawS3Key).toBe("emails/msg-display-raw/raw-display.eml"); + + const displayUpload = uploads.find(u => u.key === "emails/msg-display-raw/raw-display.eml"); + expect(displayUpload).toBeDefined(); + expect(displayUpload!.body).toContain("Body text."); + expect(displayUpload!.body).toContain("[attachment content omitted: document.pdf"); + expect(displayUpload!.body).not.toContain("QQ==QQ=="); + }); +}); diff --git a/tests/isolated/content-sanitizer-inline-images.spec.ts b/tests/isolated/content-sanitizer-inline-images.spec.ts new file mode 100644 index 00000000..f466ae71 --- /dev/null +++ b/tests/isolated/content-sanitizer-inline-images.spec.ts @@ -0,0 +1,128 @@ +import { describe, it, expect, vi, afterEach } from "vitest"; +import { handler } from "../../src/isolated/content-sanitizer.js"; + +// --------------------------------------------------------------------------- +// Inline CID images were previously embedded as base64 data URIs directly in +// htmlBody with no size/count limit — a single large inline logo could bloat +// the stored signal well past DynamoDB's 400KB item cap. Now only small images +// (<=100KB, first 3 per message) are inlined; anything past either cap is +// uploaded to S3 like a regular attachment, with its `cid:` reference left +// unresolved in htmlBody (see MAX_INLINE_DATA_URI_SIZE / MAX_INLINE_DATA_URI_COUNT). +// --------------------------------------------------------------------------- + +const BOUNDARY = "----=_Part_inline_boundary"; + +function buildEmailWithInlineImages(images: Array<{ contentId: string; size: number }>): string { + const parts = images.map(({ contentId, size }) => [ + `--${BOUNDARY}`, + "Content-Type: image/png", + "Content-Transfer-Encoding: base64", + `Content-ID: <${contentId}>`, + `Content-Disposition: inline; filename="${contentId}.png"`, + "", + Buffer.alloc(size, "A").toString("base64"), + "", + ].join("\r\n")); + + const imgTags = images.map(({ contentId }) => ``).join(""); + + return [ + "From: sender@example.com", + "To: recipient@example.com", + "Subject: Message with inline images", + "MIME-Version: 1.0", + `Content-Type: multipart/related; boundary="${BOUNDARY}"`, + "", + `--${BOUNDARY}`, + 'Content-Type: text/html; charset="UTF-8"', + "", + `${imgTags}`, + "", + ...parts, + `--${BOUNDARY}--`, + ].join("\r\n"); +} + +function mockFetch(raw: string) { + const buf = Buffer.from(raw, "utf-8"); + vi.stubGlobal("fetch", vi.fn(async (url: string) => { + if (url === "https://example.com/get") { + return { ok: true, status: 200, arrayBuffer: async () => buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength) }; + } + return { ok: true, status: 204 }; + })); +} + +afterEach(() => { + vi.unstubAllGlobals(); +}); + +describe("content-sanitizer — inline image size/count cap", () => { + it("embeds a small inline image as a data URI, no S3 upload / inlineImages entry", async () => { + mockFetch(buildEmailWithInlineImages([{ contentId: "logo", size: 1024 }])); + + const result = await handler({ + presignedGetUrl: "https://example.com/get", + presignedPost: { url: "https://example.com/post", fields: {} }, + accountId: "acct-test", + senderEtld1: "example.com", + keyPrefix: "emails/msg-small-inline/", + retentionTag: null, + }); + + expect(result.success).toBe(true); + if (!result.success) return; + expect(result.parsed.htmlBody).toContain("data:image/png;base64,"); + expect(result.parsed.htmlBody).not.toContain("cid:logo"); + expect(result.parsed.inlineImages).toBeUndefined(); + }); + + it("uploads an inline image over 100KB to S3 instead of embedding it, leaving cid: unresolved", async () => { + mockFetch(buildEmailWithInlineImages([{ contentId: "logo", size: 200 * 1024 }])); + + const result = await handler({ + presignedGetUrl: "https://example.com/get", + presignedPost: { url: "https://example.com/post", fields: {} }, + accountId: "acct-test", + senderEtld1: "example.com", + keyPrefix: "emails/msg-big-inline/", + retentionTag: null, + }); + + expect(result.success).toBe(true); + if (!result.success) return; + expect(result.parsed.htmlBody).toContain("cid:logo"); + expect(result.parsed.htmlBody).not.toContain("data:image/png;base64,"); + expect(result.parsed.inlineImages).toEqual([ + { contentId: "logo", mimeType: "image/png", sizeBytes: 200 * 1024, s3Key: "emails/msg-big-inline/0" }, + ]); + }); + + it("inlines only the first 3 small images per message, routing the 4th to S3", async () => { + mockFetch(buildEmailWithInlineImages([ + { contentId: "img1", size: 1024 }, + { contentId: "img2", size: 1024 }, + { contentId: "img3", size: 1024 }, + { contentId: "img4", size: 1024 }, + ])); + + const result = await handler({ + presignedGetUrl: "https://example.com/get", + presignedPost: { url: "https://example.com/post", fields: {} }, + accountId: "acct-test", + senderEtld1: "example.com", + keyPrefix: "emails/msg-many-inline/", + retentionTag: null, + }); + + expect(result.success).toBe(true); + if (!result.success) return; + expect(result.parsed.htmlBody).not.toContain("cid:img1"); + expect(result.parsed.htmlBody).not.toContain("cid:img2"); + expect(result.parsed.htmlBody).not.toContain("cid:img3"); + expect(result.parsed.htmlBody).toContain("cid:img4"); + expect(result.parsed.inlineImages).toEqual([ + { contentId: "img4", mimeType: "image/png", sizeBytes: 1024, s3Key: "emails/msg-many-inline/0" }, + ]); + }); +}); diff --git a/tests/isolated/raw-email-display.spec.ts b/tests/isolated/raw-email-display.spec.ts new file mode 100644 index 00000000..27733c66 --- /dev/null +++ b/tests/isolated/raw-email-display.spec.ts @@ -0,0 +1,127 @@ +import { describe, it, expect } from "vitest"; +import { buildDisplayRawEmail } from "../../src/isolated/raw-email-display.js"; + +// --------------------------------------------------------------------------- +// buildDisplayRawEmail produces the copy served by "view original email" / +// download-as-.eml: attachments fully stripped, small inline images kept +// (needed for the .eml to still render if opened in a real mail client), +// bounded by a per-image cap and a cumulative budget across the message. +// --------------------------------------------------------------------------- + +function buildRawEmail(parts: Array<{ headers: string[]; body: string }>, boundary = "----=_Part_test_boundary"): string { + const body = parts.map(p => [`--${boundary}`, ...p.headers, "", p.body, ""].join("\r\n")).join(""); + return [ + "From: sender@example.com", + "To: recipient@example.com", + "Subject: Test", + "MIME-Version: 1.0", + `Content-Type: multipart/mixed; boundary="${boundary}"`, + "", + body + `--${boundary}--`, + ].join("\r\n"); +} + +describe("buildDisplayRawEmail", () => { + it("strips a regular attachment body, keeping its headers", () => { + const raw = buildRawEmail([ + { headers: ['Content-Type: text/plain; charset="UTF-8"'], body: "Body text." }, + { + headers: [ + "Content-Type: application/pdf", + "Content-Transfer-Encoding: base64", + 'Content-Disposition: attachment; filename="document.pdf"', + ], + body: "QQ==".repeat(50), + }, + ]); + + const result = buildDisplayRawEmail(raw); + expect(result).toContain('Content-Disposition: attachment; filename="document.pdf"'); + expect(result).toContain("[attachment content omitted: document.pdf"); + expect(result).not.toContain("QQ==QQ=="); + expect(result).toContain("Body text."); + }); + + it("keeps a small inline image's bytes untouched", () => { + const raw = buildRawEmail([ + { headers: ['Content-Type: text/html; charset="UTF-8"'], body: '' }, + { + headers: [ + "Content-Type: image/png", + "Content-Transfer-Encoding: base64", + "Content-ID: ", + 'Content-Disposition: inline; filename="logo.png"', + ], + body: "aGVsbG8=".repeat(10), // well under 100KB + }, + ]); + + const result = buildDisplayRawEmail(raw); + expect(result).toContain("aGVsbG8=".repeat(10)); + expect(result).not.toContain("[inline image omitted"); + }); + + it("truncates an inline image over the 100KB per-image cap", () => { + const bigBody = "A".repeat(140_000); // ~105KB decoded + const raw = buildRawEmail([ + { + headers: [ + "Content-Type: image/png", + "Content-Transfer-Encoding: base64", + "Content-ID: ", + 'Content-Disposition: inline; filename="logo.png"', + ], + body: bigBody, + }, + ]); + + const result = buildDisplayRawEmail(raw); + expect(result).toContain("[inline image omitted: exceeds display size limit"); + expect(result).not.toContain(bigBody); + }); + + it("truncates inline images once the cumulative 300KB budget is exhausted, even if individually under the per-image cap", () => { + // Each image ~90KB decoded (under the 100KB per-image cap), but 4 of them exceed + // the 300KB cumulative budget, so the 4th must be truncated. + const imageBody = "A".repeat(120_000); // ~90KB decoded + const raw = buildRawEmail([ + { headers: ["Content-Type: image/png", "Content-Transfer-Encoding: base64", "Content-ID: ", 'Content-Disposition: inline; filename="1.png"'], body: imageBody }, + { headers: ["Content-Type: image/png", "Content-Transfer-Encoding: base64", "Content-ID: ", 'Content-Disposition: inline; filename="2.png"'], body: imageBody }, + { headers: ["Content-Type: image/png", "Content-Transfer-Encoding: base64", "Content-ID: ", 'Content-Disposition: inline; filename="3.png"'], body: imageBody }, + { headers: ["Content-Type: image/png", "Content-Transfer-Encoding: base64", "Content-ID: ", 'Content-Disposition: inline; filename="4.png"'], body: imageBody }, + ]); + + const result = buildDisplayRawEmail(raw); + const omittedCount = (result.match(/\[inline image omitted/g) ?? []).length; + expect(omittedCount).toBeGreaterThanOrEqual(1); + // The kept images' bytes must still be present for at least the first ones + expect(result).toContain(imageBody.slice(0, 100)); + }); + + it("leaves non-multipart plain-text emails unchanged", () => { + const raw = [ + "From: sender@example.com", + "To: recipient@example.com", + "Subject: Plain text", + "Content-Type: text/plain; charset=\"UTF-8\"", + "MIME-Version: 1.0", + "", + "Just a plain message.", + ].join("\r\n"); + + expect(buildDisplayRawEmail(raw)).toBe(raw); + }); + + it("leaves the visible text/html body untouched", () => { + const raw = buildRawEmail([ + { headers: ['Content-Type: text/html; charset="UTF-8"'], body: "

Hello world

" }, + { + headers: ["Content-Type: application/pdf", "Content-Transfer-Encoding: base64", 'Content-Disposition: attachment; filename="a.pdf"'], + body: "QQ==".repeat(20), + }, + ]); + + const result = buildDisplayRawEmail(raw); + expect(result).toContain("

Hello world

"); + }); +});