From 180c4a3472633cf0d18f3a08b188243b03ba46b4 Mon Sep 17 00:00:00 2001 From: Ildar Nurislamov Date: Tue, 4 Aug 2026 16:06:23 +0400 Subject: [PATCH 1/5] fix(console): explicit types + zod boundaries on config exports (JITSU-158 #2) Annotate every paginated findMany in the export endpoints with explicit Prisma GetPayload types. The cursor back-edge previously made inference self-referential and TypeScript silently collapsed the results to any, which is how the 2026-07-30 blank-options typo ({ data_ } for { data: data_ }) compiled. With the annotations the same typo now fails with TS2339 (verified). All Prisma Json column reads now go through tolerant zod parsers (field-level catch drops only the offending field) instead of implicit any. Raw queries and the billing pg query get typed results. object-hash and stable-hash imports are pinned to explicit signatures - their inferred types differ between tsc and the type-aware lint program. getCoreDestinationTypeNonStrict accepts string | undefined - call sites were always able to pass undefined; the signature just could not say so. Behavior deliberately preserved except three crash-to-skip fixes on malformed rows: a domain config without a string name, a classic-mapping without a string value, and a canceled subscription with null period_end no longer throw mid-export. Co-Authored-By: Claude Fable 5 --- webapps/console/lib/schema/destinations.tsx | 4 +- .../pages/api/admin/export/[name]/index.ts | 361 +++++++++++------- 2 files changed, 232 insertions(+), 133 deletions(-) diff --git a/webapps/console/lib/schema/destinations.tsx b/webapps/console/lib/schema/destinations.tsx index 7aed78d3c..915bc955f 100644 --- a/webapps/console/lib/schema/destinations.tsx +++ b/webapps/console/lib/schema/destinations.tsx @@ -264,8 +264,8 @@ export function getCoreDestinationType(typeId: string): DestinationType { return destinationType; } -export function getCoreDestinationTypeNonStrict(typeId: string): DestinationType | undefined { - return coreDestinationsMap[typeId]; +export function getCoreDestinationTypeNonStrict(typeId: string | undefined): DestinationType | undefined { + return typeId ? coreDestinationsMap[typeId] : undefined; } export const ClickhouseCredentials = z.object({ diff --git a/webapps/console/pages/api/admin/export/[name]/index.ts b/webapps/console/pages/api/admin/export/[name]/index.ts index 5a016a62a..e572abbbe 100644 --- a/webapps/console/pages/api/admin/export/[name]/index.ts +++ b/webapps/console/pages/api/admin/export/[name]/index.ts @@ -8,9 +8,9 @@ import omit from "lodash/omit"; import { NextApiRequest } from "next"; import hash from "object-hash"; import { default as stableHash } from "stable-hash"; -import { WorkspaceDbModel, FunctionsServerDbModel } from "../../../../../prisma/schema"; -import { ProfileBuilder } from "@jitsu/destination-functions"; +import { FunctionsServerDbModel } from "../../../../../prisma/schema"; import { getServerEnv } from "../../../../../lib/server/serverEnv"; +import { Prisma } from "@prisma/client"; const serverEnv = getServerEnv(); const defaultFunctionsClass = serverEnv.DEFAULT_FUNCTIONS_CLASS; @@ -48,6 +48,76 @@ const batchSize = 1000; const safeLastModified = new Date(2024, 0, 1, 0, 0, 0, 0); +// Explicit result types for the paginated findMany loops. Without them the +// cursor back-edge (`cursor:` <- `lastId` <- `objects[last].id`) makes +// inference self-referential and the checker silently collapses the whole +// result to `any` - the blind spot that let the 2026-07-30 blank-options bug +// compile (JITSU-158). +type LinkRow = Prisma.ConfigurationObjectLinkGetPayload<{ include: { from: true; to: true; workspace: true } }>; +type ObjectRow = Prisma.ConfigurationObjectGetPayload<{}>; +type ObjectRowWithWorkspace = Prisma.ConfigurationObjectGetPayload<{ include: { workspace: true } }>; +type StreamRow = Prisma.ConfigurationObjectGetPayload<{ + include: { toLinks: { include: { to: true } }; workspace: true }; +}>; +type WorkspaceRow = Prisma.WorkspaceGetPayload<{}>; +type WorkspaceWithProfilesRow = Prisma.WorkspaceGetPayload<{ + include: { profileBuilders: { include: { functions: { include: { function: true } } } } }; +}>; + +// object-hash ships no type declarations (its type is inferred from JS via +// allowJs) and stable-hash's `exports` map has no `types` condition - the +// inferred import type differs between tsc and the type-aware lint program. +// Pin one explicit signature so every call site type-checks identically. +const hashValue = hash as (value: unknown) => string; +const stableHashValue = stableHash as unknown as (value: unknown) => string; + +// Prisma `Json` columns surface as `JsonValue` - reads must go through an +// explicit parse instead of implicit `any`. Parsers are tolerant on purpose: +// a junk field must not fail the export (see logExportEntityError), so +// field-level `catch` drops only the offending field and object-level `catch` +// normalizes a non-object root to {}. +const JsonRecord = z.record(z.unknown()); +function asRecord(v: unknown): Record { + const parsed = JsonRecord.safeParse(v); + return parsed.success ? parsed.data : {}; +} +// Connection options (ConfigurationObjectLink.data). +const LinkData = z + .object({ + disabled: z.unknown().optional(), + clickhouseSettings: z.unknown().optional(), + functionsEnv: z.record(z.unknown()).optional().catch(undefined), + }) + .passthrough() + .catch({}); +// ConfigurationObject.config - the fields the exports read by name. +const ObjectConfig = z + .object({ + destinationType: z.string().optional().catch(undefined), + name: z.string().optional().catch(undefined), + }) + .passthrough() + .catch({}); +// ProfileBuilder.connectionOptions. +const PbConnectionOptions = z + .object({ + profileWindow: z.unknown().optional(), + variables: z.unknown().optional(), + functions: z.array(z.unknown()).optional().catch(undefined), + }) + .passthrough() + .catch({}); +// ConfigurationObject.config for streams. +const StreamConfig = z + .object({ + shard: z.unknown().optional(), + publicKeys: z.array(z.unknown()).optional().catch(undefined), + privateKeys: z.array(z.unknown()).optional().catch(undefined), + domains: z.array(z.string()).optional().catch(undefined), + }) + .passthrough() + .catch({}); + function dateMax(...dates: (Date | undefined)[]): Date | undefined { return dates.reduce((acc, d) => (d && (!acc || d.getTime() > acc.getTime()) ? d : acc), undefined); } @@ -134,8 +204,7 @@ function selectFunctionsServer( } async function getLastUpdated(): Promise { - return ( - (await db.prisma().$queryRaw` + const rows = await db.prisma().$queryRaw<{ last_updated: Date | null }[]>` select greatest( (select max("updatedAt") from newjitsu."ConfigurationObjectLink"), @@ -143,8 +212,8 @@ async function getLastUpdated(): Promise { (select max("updatedAt") from newjitsu."ConfigurationObject"), (select max("updatedAt") from newjitsu."FunctionsServer"), (select max("updatedAt") from newjitsu."Workspace") - ) as "last_updated"`) as any - )[0]["last_updated"]; + ) as "last_updated"`; + return rows[0]?.last_updated ?? undefined; } async function exportBulkerConnections(writer: Writer) { @@ -153,7 +222,7 @@ async function exportBulkerConnections(writer: Writer) { let lastId: string | undefined = undefined; let needComma = false; while (true) { - const objects = await db.prisma().configurationObjectLink.findMany({ + const objects: LinkRow[] = await db.prisma().configurationObjectLink.findMany({ where: { deleted: false, OR: [{ type: "push" }, { type: null }], @@ -174,24 +243,25 @@ async function exportBulkerConnections(writer: Writer) { for (const { data: data_, from, id, to, updatedAt, workspace } of objects) { let payload: string | undefined; try { - const data = data_ || {}; - if (data?.disabled) { + const data = LinkData.parse(data_); + if (data.disabled) { continue; // skip disabled connections } - const destinationType = to.config.destinationType; + const toConfig = ObjectConfig.parse(to.config); + const destinationType = toConfig.destinationType; const coreDestinationType = getCoreDestinationTypeNonStrict(destinationType); if (coreDestinationType?.usesBulker || coreDestinationType?.hybrid) { - const credentials = omit(to.config, "destinationType", "type", "name"); + const credentials: Record = omit(toConfig, "destinationType", "type", "name"); if (destinationType === "clickhouse") { - if (typeof (data as any).clickhouseSettings === "string") { + if (typeof data.clickhouseSettings === "string") { const extraParams = Object.fromEntries( - ((data as any).clickhouseSettings as string) + data.clickhouseSettings .split("\n") .filter(s => s.includes("=")) .map(s => s.split("=")) .map(([k, v]) => [k.trim(), v.trim()]) ); - credentials.parameters = { ...(credentials.parameters || {}), ...extraParams }; + credentials.parameters = { ...asRecord(credentials.parameters), ...extraParams }; } if (!credentials.provisioned) { credentials.loadAsJson = false; @@ -208,7 +278,7 @@ async function exportBulkerConnections(writer: Writer) { }, id: id, type: destinationType, - options: omit(data as any, "clickhouseSettings"), + options: omit(data, "clickhouseSettings"), updatedAt: dateMax(updatedAt, to.updatedAt), credentials: credentials, }); @@ -235,7 +305,7 @@ async function exportBulkerConnections(writer: Writer) { } lastId = undefined; while (true) { - const objects = await db.prisma().configurationObject.findMany({ + const objects: ObjectRowWithWorkspace[] = await db.prisma().configurationObject.findMany({ where: { deleted: false, type: "destination", workspace: { deleted: false } }, include: { workspace: true }, take: batchSize, @@ -250,7 +320,8 @@ async function exportBulkerConnections(writer: Writer) { for (const { id, workspace, config, updatedAt } of objects) { let payload: string | undefined; try { - const destinationType = config.destinationType; + const parsedConfig = ObjectConfig.parse(config); + const destinationType = parsedConfig.destinationType; const coreDestinationType = getCoreDestinationTypeNonStrict(destinationType); if (coreDestinationType?.usesBulker || coreDestinationType?.hybrid) { payload = JSON.stringify({ @@ -265,7 +336,7 @@ async function exportBulkerConnections(writer: Writer) { deduplicate: true, }, updatedAt: updatedAt, - credentials: omit(config, "destinationType", "type", "name"), + credentials: omit(parsedConfig, "destinationType", "type", "name"), }); } } catch (e) { @@ -294,13 +365,16 @@ async function exportBulkerConnections(writer: Writer) { //the static service token. const url = `${getEeConnection().host}api/s3-connections`; try { - const backupConnections = await rpc(url, { + const backupConnections: unknown = await rpc(url, { method: "GET", headers: { "Content-Type": "application/json", ...serviceTokenHeaders(), }, }); + if (!Array.isArray(backupConnections)) { + throw new Error(`Expected an array of backup connections, got: ${typeof backupConnections}`); + } for (const conn of backupConnections) { if (needComma) { writer.write(","); @@ -321,7 +395,7 @@ async function exportBulkerConnections(writer: Writer) { async function exportRotorConnections(writer: Writer) { const workspacesWithClasses = await functionsClassByWorkspace(); - const functionsClassFunc = (workspace: any) => + const functionsClassFunc = (workspace: { id: string; featuresEnabled?: string[] | null }) => extractFunctionsClasses(workspace.featuresEnabled ?? [])[0] || workspacesWithClasses.get(workspace.id)?.class || defaultFunctionsClass; @@ -344,7 +418,7 @@ async function exportRotorConnections(writer: Writer) { orderBy: { id: "asc" }, }); while (true) { - const objects = await db.prisma().configurationObjectLink.findMany({ + const objects: LinkRow[] = await db.prisma().configurationObjectLink.findMany({ where: { deleted: false, NOT: { type: "sync" }, @@ -365,15 +439,16 @@ async function exportRotorConnections(writer: Writer) { for (const { data: data_, from, id, to, updatedAt, workspace } of objects) { let payload: string | undefined; try { - const data = data_ || {}; - if (data?.disabled) { + const data = LinkData.parse(data_); + if (data.disabled) { continue; // skip disabled connections } - const destinationType = to.config.destinationType; + const destinationType = ObjectConfig.parse(to.config).destinationType; const coreDestinationType = getCoreDestinationTypeNonStrict(destinationType); if (!coreDestinationType) { getLog().atError().log(`Unknown destination type: ${destinationType} for connection ${id}`); } + const credentials = omit(asRecord(to.config), "destinationType", "type", "name"); payload = JSON.stringify({ __debug: { workspace: { id: workspace.id, name: workspace.slug }, @@ -382,23 +457,23 @@ async function exportRotorConnections(writer: Writer) { type: destinationType, workspaceId: workspace.id, streamId: from.id, - streamName: from.config?.name, + streamName: ObjectConfig.parse(from.config).name, destinationId: to.id, usesBulker: !!coreDestinationType?.usesBulker, options: { ...data, ...((workspace.featuresEnabled ?? []).includes("nofetchlogs") && - data?.functionsEnv?.FETCH_LOGS_ENABLED !== "true" + data.functionsEnv?.FETCH_LOGS_ENABLED !== "true" ? { fetchLogLevel: "debug" } : {}), ...((workspace.featuresEnabled ?? []).includes("fastFunctions") ? { fastFunctions: true } : {}), functionsServer: selectFunctionsServer(functionsServers, workspace.id, id, functionsClassFunc(workspace)), workspaceUpdatedAt: workspace.updatedAt, }, - optionsHash: hash(data), + optionsHash: hashValue(data), updatedAt: dateMax(updatedAt, to.updatedAt), - credentials: omit(to.config, "destinationType", "type", "name"), - credentialsHash: hash(omit(to.config, "destinationType", "type", "name")), + credentials: credentials, + credentialsHash: hashValue(credentials), }); } catch (e) { // Only entity materialization/serialization is guarded: one malformed row @@ -422,7 +497,7 @@ async function exportRotorConnections(writer: Writer) { } lastId = undefined; while (true) { - const objects = await db.prisma().configurationObject.findMany({ + const objects: ObjectRowWithWorkspace[] = await db.prisma().configurationObject.findMany({ where: { deleted: false, type: "destination", workspace: { deleted: false } }, include: { workspace: true }, take: batchSize, @@ -437,20 +512,22 @@ async function exportRotorConnections(writer: Writer) { for (const { id, workspace, config, updatedAt } of objects) { let payload: string | undefined; try { - const destinationType = config?.destinationType; + const parsedConfig = ObjectConfig.parse(config); + const destinationType = parsedConfig.destinationType; const coreDestinationType = getCoreDestinationTypeNonStrict(destinationType); if (coreDestinationType?.usesBulker || coreDestinationType?.hybrid) { + const credentials = omit(parsedConfig, "destinationType", "type", "name"); payload = JSON.stringify({ id: id, type: destinationType, workspaceId: workspace.id, streamId: id, - streamName: config?.name, + streamName: parsedConfig.name, destinationId: id, usesBulker: !!coreDestinationType?.usesBulker, updatedAt: updatedAt, - credentials: omit(config, "destinationType", "type", "name"), - credentialsHash: hash(omit(config, "destinationType", "type", "name")), + credentials: credentials, + credentialsHash: hashValue(credentials), }); } } catch (e) { @@ -476,20 +553,21 @@ async function exportRotorConnections(writer: Writer) { for (const pb of profileBuilders) { let payload: string | undefined; try { + const connectionOptions = PbConnectionOptions.parse(pb.connectionOptions); const cred = { - ...(pb.intermediateStorageCredentials ?? ({} as any)), - profileWindowDays: (pb.connectionOptions ?? ({} as any)).profileWindow, + ...asRecord(pb.intermediateStorageCredentials), + profileWindowDays: connectionOptions.profileWindow, profileBuilderId: pb.id, eventsCollectionName: `profiles-raw-${pb.workspace.id}-${pb.id}`, traitsCollectionName: `profiles-traits-${pb.workspace.id}-${pb.id}`, }; const opts = { - functionsEnv: (pb.connectionOptions ?? ({} as any)).variables, + functionsEnv: connectionOptions.variables, functions: [ { functionId: "builtin.transformation.user-recognition", }, - ...((pb.connectionOptions ?? ({} as any)).functions || []), + ...(connectionOptions.functions ?? []), ], functionsServer: selectFunctionsServer( functionsServers, @@ -511,10 +589,10 @@ async function exportRotorConnections(writer: Writer) { destinationId: pb.destinationId, usesBulker: false, options: opts, - optionsHash: hash(opts), + optionsHash: hashValue(opts), updatedAt: pb.updatedAt, credentials: cred, - credentialsHash: hash(cred), + credentialsHash: hashValue(cred), }); } catch (e) { // Only entity materialization/serialization is guarded: one malformed row @@ -541,7 +619,7 @@ async function exportFunctions(writer: Writer) { let lastId: string | undefined = undefined; let needComma = false; while (true) { - const objects = await db.prisma().configurationObject.findMany({ + const objects: ObjectRow[] = await db.prisma().configurationObject.findMany({ where: { deleted: false, type: "function", @@ -559,10 +637,11 @@ async function exportFunctions(writer: Writer) { for (const row of objects) { let payload: string | undefined; try { + const config = asRecord(row.config); payload = JSON.stringify({ ...omit(row, "deleted", "config"), - ...row.config, - codeHash: hash(row.config?.code || row.config?.draft || ""), + ...config, + codeHash: hashValue(config.code || config.draft || ""), }); } catch (e) { // Only entity materialization/serialization is guarded: one malformed row @@ -589,7 +668,7 @@ async function exportFunctions(writer: Writer) { async function exportStreamsWithDestinations(writer: Writer) { const workspacesWithClasses = await functionsClassByWorkspace(); - const functionsClassFunc = (workspace: any) => + const functionsClassFunc = (workspace: { id: string; featuresEnabled?: string[] | null }) => extractFunctionsClasses(workspace.featuresEnabled ?? [])[0] || workspacesWithClasses.get(workspace.id)?.class || defaultFunctionsClass; @@ -603,10 +682,10 @@ async function exportStreamsWithDestinations(writer: Writer) { }); const domainsMap = new Map(); for (const domain of domains) { - const name = (domain.config as any).name; - if (!name.includes("*")) { + const name = ObjectConfig.parse(domain.config).name; + if (name && !name.includes("*")) { const d = domainsMap.get(domain.workspaceId) || []; - domainsMap.set(domain.workspaceId, [...d, (domain.config as any).name]); + domainsMap.set(domain.workspaceId, [...d, name]); } } const classicMappings = await db.prisma().configurationObject.findMany({ @@ -619,8 +698,9 @@ async function exportStreamsWithDestinations(writer: Writer) { }); const classicKeysMap: Record = {}; classicMappings - .filter(c => c.config && c.config["value"]) - .flatMap(c => c.config!["value"].split("\n")) + .map(c => asRecord(c.config).value) + .filter((value): value is string => typeof value === "string" && value !== "") + .flatMap(value => value.split("\n")) .forEach(line => { const [source, apikey] = line.split(/=(.*)/s).map((s: string) => s.trim()); if (source && apikey) { @@ -641,17 +721,17 @@ async function exportStreamsWithDestinations(writer: Writer) { }, orderBy: { id: "asc" }, }); - const pbMap = new Map(); + const pbMap = new Map(); for (const pb of profileBuilders) { const pbs = pbMap.get(pb.workspaceId) || []; - pbMap.set(pb.workspaceId, [...pbs, pb as unknown as ProfileBuilder]); + pbMap.set(pb.workspaceId, [...pbs, pb]); } writer.write("["); let lastId: string | undefined = undefined; let needComma = false; while (true) { - const objects = await db.prisma().configurationObject.findMany({ + const objects: StreamRow[] = await db.prisma().configurationObject.findMany({ where: { deleted: false, type: "stream", workspace: { deleted: false } }, include: { toLinks: { include: { to: true } }, workspace: true }, take: batchSize, @@ -666,11 +746,12 @@ async function exportStreamsWithDestinations(writer: Writer) { for (const obj of objects) { let payload: string | undefined; try { + const streamConfig = StreamConfig.parse(obj.config); const throttlePercent = workspacesWithClasses.get(obj.workspace.id)?.status !== "active" ? getNumericOption("throttle", obj.workspace) : undefined; - const shardNumber = obj.config.shard || getNumericOption("shard", obj.workspace); + const shardNumber = streamConfig.shard || getNumericOption("shard", obj.workspace); const classicKeys = classicKeysMap[obj.id] || ({} as ClassicKeys); payload = JSON.stringify({ __debug: { @@ -680,10 +761,10 @@ async function exportStreamsWithDestinations(writer: Writer) { stream: { ...omit(obj, "type", "workspaceId", "config", "toLinks", "deleted", "createdAt", "updatedAt", "workspace"), ...{ - ...omit(obj.config, "shard"), - publicKeys: [classicKeys.publicKeys ?? [], obj.config.publicKeys ?? []].flat(), - privateKeys: [classicKeys.privateKeys ?? [], obj.config.privateKeys ?? []].flat(), - domains: [...new Set([...(domainsMap.get(obj.workspace.id) ?? []), ...(obj.config.domains ?? [])])], + ...omit(streamConfig, "shard"), + publicKeys: [classicKeys.publicKeys ?? [], streamConfig.publicKeys ?? []].flat(), + privateKeys: [classicKeys.privateKeys ?? [], streamConfig.privateKeys ?? []].flat(), + domains: [...new Set([...(domainsMap.get(obj.workspace.id) ?? []), ...(streamConfig.domains ?? [])])], }, workspaceId: obj.workspace.id, }, @@ -695,40 +776,46 @@ async function exportStreamsWithDestinations(writer: Writer) { captureHeaders: (obj.workspace.featuresEnabled || []).includes("captureHeaders"), destinations: [ ...obj.toLinks - .filter(l => !l.deleted && l.type === "push" && !l.data?.disabled && !l.to.deleted) - .map(l => ({ - id: l.to.id, - connectionId: l.id, - destinationType: (l.to.config ?? {}).destinationType, - name: (l.to.config ?? {}).name, - credentials: omit(l.to.config, "destinationType", "type", "name"), + .filter(l => !l.deleted && l.type === "push" && !LinkData.parse(l.data).disabled && !l.to.deleted) + .map(l => { + const toConfig = ObjectConfig.parse(l.to.config); + return { + id: l.to.id, + connectionId: l.id, + destinationType: toConfig.destinationType, + name: toConfig.name, + credentials: omit(toConfig, "destinationType", "type", "name"), + options: { + ...LinkData.parse(l.data), + functionsServer: selectFunctionsServer( + functionsServers, + obj.workspace.id, + l.id, + functionsClassFunc(obj.workspace) + ), + }, + }; + }), + ...(pbMap.get(obj.workspace.id) ?? []).map(pb => { + const connectionOptions = PbConnectionOptions.parse(pb.connectionOptions); + return { + id: pb.id, + connectionId: pb.id, + destinationType: "profiles", + name: "profiles", + credentials: { + ...asRecord(pb.intermediateStorageCredentials), + profileWindowDays: connectionOptions.profileWindow, + profileBuilderId: pb.id, + eventsCollectionName: `profiles-raw-${obj.workspace.id}-${pb.id}`, + traitsCollectionName: `profiles-traits-${obj.workspace.id}-${pb.id}`, + }, options: { - ...(l.data ?? {}), - functionsServer: selectFunctionsServer( - functionsServers, - obj.workspace.id, - l.id, - functionsClassFunc(obj.workspace) - ), + functionsEnv: connectionOptions.variables, + functions: connectionOptions.functions, }, - })), - ...(pbMap.get(obj.workspace.id) ?? []).map(pb => ({ - id: pb.id, - connectionId: pb.id, - destinationType: "profiles", - name: "profiles", - credentials: { - ...pb.intermediateStorageCredentials, - profileWindowDays: pb.connectionOptions.profileWindow, - profileBuilderId: pb.id, - eventsCollectionName: `profiles-raw-${obj.workspace.id}-${pb.id}`, - traitsCollectionName: `profiles-traits-${obj.workspace.id}-${pb.id}`, - }, - options: { - functionsEnv: pb.connectionOptions?.variables, - functions: pb.connectionOptions?.functions, - }, - })), + }; + }), ], }); } catch (e) { @@ -755,12 +842,13 @@ async function exportStreamsWithDestinations(writer: Writer) { } async function exportWorkspacesLastModified(): Promise { - const lastUpdated = ( - (await db.prisma().$queryRaw`select max("updatedAt") as "last_updated" from newjitsu."Workspace"`) as any - )[0]["last_updated"] as Date; + const rows = await db.prisma().$queryRaw< + { last_updated: Date | null }[] + >`select max("updatedAt") as "last_updated" from newjitsu."Workspace"`; + const lastUpdated = rows[0]?.last_updated ?? undefined; // force refresh every 5 minute to actualize possible subscription status changes or expirations const forceRefreshEveryMs = 5 * 60 * 1000; - if (lastUpdated.getTime() < Date.now() - forceRefreshEveryMs) { + if (!lastUpdated || lastUpdated.getTime() < Date.now() - forceRefreshEveryMs) { return new Date(Math.floor(Date.now() / forceRefreshEveryMs) * forceRefreshEveryMs); } return lastUpdated; @@ -772,7 +860,11 @@ async function functionsClassByWorkspace(): Promise(); - const rows = await db.pgPool().query(`with customers as (select obj -> 'customer' ->> 'id' as customer_id, + const rows = await db.pgPool().query<{ + id: string; + status: string; + period_end: Date | null; + }>(`with customers as (select obj -> 'customer' ->> 'id' as customer_id, obj -> 'subscription' ->> 'status' as status, (obj -> 'subscription' -> 'current_period_end')::int as period_end from newjitsuee.kvstore @@ -792,7 +884,7 @@ async function functionsClassByWorkspace(): Promise now) { + if (row.period_end && row.period_end.getTime() > now) { workspacesWithClasses.set(row.id, { class: "dedicated", status: "active" }); } } @@ -809,7 +901,7 @@ async function exportWorkspaces(writer: Writer) { let lastId: string | undefined = undefined; let needComma = false; while (true) { - const objects = await db.prisma().workspace.findMany({ + const objects: WorkspaceRow[] = await db.prisma().workspace.findMany({ where: { deleted: false, }, @@ -851,19 +943,18 @@ async function exportWorkspaces(writer: Writer) { } async function exportWorkspacesWithProfilesLastModified(): Promise { - const lastUpdated = ( - (await db.prisma().$queryRaw` + const rows = await db.prisma().$queryRaw<{ last_updated: Date | null }[]>` select greatest( (select max("updatedAt") from newjitsu."ConfigurationObject" where type='function'), (select max("updatedAt") from newjitsu."ProfileBuilder"), (select max("updatedAt") from newjitsu."ProfileBuilderFunction"), (select max("updatedAt") from newjitsu."Workspace") - ) as "last_updated"`) as any - )[0]["last_updated"]; + ) as "last_updated"`; + const lastUpdated = rows[0]?.last_updated ?? undefined; // force refresh every 5 minute to actualize possible subscription status changes or expirations const forceRefreshEveryMs = 5 * 60 * 1000; - if (lastUpdated?.getTime() < Date.now() - forceRefreshEveryMs) { + if (!lastUpdated || lastUpdated.getTime() < Date.now() - forceRefreshEveryMs) { return new Date(Math.floor(Date.now() / forceRefreshEveryMs) * forceRefreshEveryMs); } return lastUpdated; @@ -891,7 +982,7 @@ async function exportWorkspacesWithProfiles(writer: Writer) { let lastId: string | undefined = undefined; let needComma = false; while (true) { - const objects = await db.prisma().workspace.findMany({ + const objects: WorkspaceWithProfilesRow[] = await db.prisma().workspace.findMany({ where: { deleted: false, }, @@ -908,26 +999,27 @@ async function exportWorkspacesWithProfiles(writer: Writer) { for (const row of objects) { let payload: string | undefined; try { - row.featuresEnabled = addFunctionsClass(row.featuresEnabled ?? [], functionsClassFunc(row.id)); - row.profileBuilders = row.profileBuilders - .filter(pb => pb.version > 0) - .map(pb => { - pb.functions = pb.functions.map(f => { - return { + const workspacePayload = { + ...row, + featuresEnabled: addFunctionsClass(row.featuresEnabled ?? [], functionsClassFunc(row.id)), + profileBuilders: row.profileBuilders + .filter(pb => pb.version > 0) + .map(pb => ({ + ...pb, + functions: pb.functions.map(f => ({ ...omit(f.function, "config"), - ...f.function.config, - }; - }); - // Add functionsServer routing info for profile builder - (pb as any).functionsServer = selectProfileBuilderFunctionsServer( - functionsServers, - row.id, - pb.id, - functionsClassFunc(row.id) - ); - return pb; - }); - payload = JSON.stringify(row); + ...asRecord(f.function.config), + })), + // functionsServer routing info for profile builder + functionsServer: selectProfileBuilderFunctionsServer( + functionsServers, + row.id, + pb.id, + functionsClassFunc(row.id) + ), + })), + }; + payload = JSON.stringify(workspacePayload); } catch (e) { // Only entity materialization/serialization is guarded: one malformed row // must not take down the whole export. Writes happen OUTSIDE the try so a @@ -981,7 +1073,7 @@ async function exportSyncs(writer: Writer) { let lastId: string | undefined = undefined; let needComma = false; while (true) { - const objects = await db.prisma().configurationObjectLink.findMany({ + const objects: LinkRow[] = await db.prisma().configurationObjectLink.findMany({ where: { deleted: false, type: "sync", @@ -1036,9 +1128,16 @@ async function exportSyncs(writer: Writer) { writer.write("]"); } -function exportSyncEntity({ data, from, id, to, updatedAt, workspace }: any) { - let destinationConfig: any = { ...(to.config as any) }; - const destinationType = destinationConfig.destinationType; +function exportSyncEntity({ + data, + from, + id, + to, + updatedAt, + workspace, +}: Pick) { + let destinationConfig: Record = { ...asRecord(to.config) }; + const destinationType = ObjectConfig.parse(to.config).destinationType; const coreDestinationType = getCoreDestinationTypeNonStrict(destinationType); if (!coreDestinationType) { getLog() @@ -1060,15 +1159,15 @@ function exportSyncEntity({ data, from, id, to, updatedAt, workspace }: any) { ); return []; } - const syncData = (data ?? {}) as Record; - let serviceConfig: any = { ...(from.config as any) }; + const syncData = asRecord(data); + let serviceConfig: Record = { ...asRecord(from.config) }; // versionHash MUST be derived from the raw persisted credentials — // matches the formula used by scheduleSync and sources/discover when // they store catalog rows in source_catalog. Hashing post-mutation or // post-OAuth-refresh creds would make sidecar's catalog lookup miss // the rows that scheduleSync wrote. - const versionHash = `${workspace.id}_${from.id}_${juavaHash("md5", stableHash(serviceConfig.credentials))}`; + const versionHash = `${workspace.id}_${from.id}_${juavaHash("md5", stableHashValue(serviceConfig.credentials))}`; // scheduleSync applies this default for these packages — apply it to // a separate `credentials` value used only in the runtime source.config @@ -1080,7 +1179,7 @@ function exportSyncEntity({ data, from, id, to, updatedAt, workspace }: any) { ) { serviceConfig = { ...serviceConfig, - credentials: { ...serviceConfig.credentials, sync_checkpoint_records: 200000 }, + credentials: { ...asRecord(serviceConfig.credentials), sync_checkpoint_records: 200000 }, }; } @@ -1195,7 +1294,7 @@ const exports: Export[] = [ }, ]; -const exportsMap = exports.reduce((acc, e) => ({ ...acc, [e.name]: e }), {}); +const exportsMap: Record = exports.reduce((acc, e) => ({ ...acc, [e.name]: e }), {}); export function getExport(name: string): Export { return requireDefined(exportsMap[name], `Export ${name} not found`); @@ -1232,7 +1331,7 @@ export function notModified(ifModifiedSince: Date | undefined, lastModified: Dat return ifModifiedSince.getTime() >= lastModifiedCopy.getTime(); } -function getNumericOption(name: string, workspace: z.infer, defaultValue?: number) { +function getNumericOption(name: string, workspace: { featuresEnabled?: string[] | null }, defaultValue?: number) { const opt = (workspace.featuresEnabled ?? []).find(f => f.startsWith(name)); if (opt) { //remove all non-numeric From 6507f935cb079943f02b2d600586fa0e5750cf99 Mon Sep 17 00:00:00 2001 From: Ildar Nurislamov Date: Tue, 4 Aug 2026 16:06:24 +0400 Subject: [PATCH 2/5] feat(console): type-aware unsafe-any lint gate on pages/api/admin (JITSU-158 #3) Enable @typescript-eslint/no-unsafe-assignment/-member-access/-argument as errors on pages/api/admin/**, type-aware via project tsconfig. The export endpoints pass clean; the eight admin files with pre-existing implicit-any debt carry an explicit file-top exemption header scoped to the rules they actually violate - remove the header when fixing a file, never add new ones. Runs as part of pnpm lint, which is already in the required Lint & Test check. Co-Authored-By: Claude Fable 5 --- webapps/console/eslint.config.mjs | 19 +++++++++++++++++++ webapps/console/pages/api/admin/become.ts | 4 ++++ .../pages/api/admin/catalog-refresh.ts | 4 ++++ .../console/pages/api/admin/domains-report.ts | 4 ++++ .../pages/api/admin/events-log-trim.ts | 4 ++++ .../console/pages/api/admin/notifications.ts | 4 ++++ .../console/pages/api/admin/sync-logs-trim.ts | 4 ++++ .../pages/api/admin/sync-quota-check.ts | 4 ++++ webapps/console/pages/api/admin/users.ts | 4 ++++ 9 files changed, 51 insertions(+) diff --git a/webapps/console/eslint.config.mjs b/webapps/console/eslint.config.mjs index 522027112..4cdafb08d 100644 --- a/webapps/console/eslint.config.mjs +++ b/webapps/console/eslint.config.mjs @@ -49,6 +49,25 @@ const eslintConfig = defineConfig([ "no-restricted-properties": "off", }, }, + // Type-aware unsafe-`any` rules on the admin API surface. The config export + // endpoints feed bulker/rotor/syncctl directly; an implicit `any` here is how + // the 2026-07-30 blank-options incident compiled (JITSU-158) — a + // destructuring typo on a Prisma result the checker had silently collapsed + // to `any`. These rules make any such collapse a lint failure instead. + { + files: ["pages/api/admin/**/*.ts"], + languageOptions: { + parserOptions: { + project: "./tsconfig.json", + tsconfigRootDir: import.meta.dirname, + }, + }, + rules: { + "@typescript-eslint/no-unsafe-assignment": "error", + "@typescript-eslint/no-unsafe-member-access": "error", + "@typescript-eslint/no-unsafe-argument": "error", + }, + }, ]); export default eslintConfig; diff --git a/webapps/console/pages/api/admin/become.ts b/webapps/console/pages/api/admin/become.ts index 8835ae44e..6585a530d 100644 --- a/webapps/console/pages/api/admin/become.ts +++ b/webapps/console/pages/api/admin/become.ts @@ -1,3 +1,7 @@ +/* eslint-disable @typescript-eslint/no-unsafe-assignment -- + * Pre-existing implicit-`any` debt, exempted when the unsafe-any gate was + * introduced for pages/api/admin (JITSU-158 action item 3). Fix the `any` + * flows in this file, then remove this header - do not add new ones. */ import { createRoute } from "../../../lib/api"; import { z } from "zod"; import { assertDefined, assertTrue, requireDefined } from "juava"; diff --git a/webapps/console/pages/api/admin/catalog-refresh.ts b/webapps/console/pages/api/admin/catalog-refresh.ts index 3f7b630e7..c5a20519b 100644 --- a/webapps/console/pages/api/admin/catalog-refresh.ts +++ b/webapps/console/pages/api/admin/catalog-refresh.ts @@ -1,3 +1,7 @@ +/* eslint-disable @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-argument -- + * Pre-existing implicit-`any` debt, exempted when the unsafe-any gate was + * introduced for pages/api/admin (JITSU-158 action item 3). Fix the `any` + * flows in this file, then remove this header - do not add new ones. */ import { createRoute, verifyAdmin } from "../../../lib/api"; import { db } from "../../../lib/server/db"; import { rpc } from "juava"; diff --git a/webapps/console/pages/api/admin/domains-report.ts b/webapps/console/pages/api/admin/domains-report.ts index e3c53fcf2..1f5c3c8f6 100644 --- a/webapps/console/pages/api/admin/domains-report.ts +++ b/webapps/console/pages/api/admin/domains-report.ts @@ -1,3 +1,7 @@ +/* eslint-disable @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-argument -- + * Pre-existing implicit-`any` debt, exempted when the unsafe-any gate was + * introduced for pages/api/admin (JITSU-158 action item 3). Fix the `any` + * flows in this file, then remove this header - do not add new ones. */ import { createRoute } from "../../../lib/api"; import { db } from "../../../lib/server/db"; import { assertDefined, assertTrue } from "juava"; diff --git a/webapps/console/pages/api/admin/events-log-trim.ts b/webapps/console/pages/api/admin/events-log-trim.ts index 1f507f26e..cc3f22a49 100644 --- a/webapps/console/pages/api/admin/events-log-trim.ts +++ b/webapps/console/pages/api/admin/events-log-trim.ts @@ -1,3 +1,7 @@ +/* eslint-disable @typescript-eslint/no-unsafe-member-access -- + * Pre-existing implicit-`any` debt, exempted when the unsafe-any gate was + * introduced for pages/api/admin (JITSU-158 action item 3). Fix the `any` + * flows in this file, then remove this header - do not add new ones. */ import { createRoute, getUser, verifyAdmin } from "../../../lib/api"; import { stopwatch, getClickhouseConfig } from "juava"; import { clickhouse } from "../../../lib/server/clickhouse"; diff --git a/webapps/console/pages/api/admin/notifications.ts b/webapps/console/pages/api/admin/notifications.ts index 7e9a79b8b..af6f60d31 100644 --- a/webapps/console/pages/api/admin/notifications.ts +++ b/webapps/console/pages/api/admin/notifications.ts @@ -1,3 +1,7 @@ +/* eslint-disable @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-argument -- + * Pre-existing implicit-`any` debt, exempted when the unsafe-any gate was + * introduced for pages/api/admin (JITSU-158 action item 3). Fix the `any` + * flows in this file, then remove this header - do not add new ones. */ import { createRoute, verifyAdmin } from "../../../lib/api"; import { clickhouse, dateToClickhouse } from "../../../lib/server/clickhouse"; import { db } from "../../../lib/server/db"; diff --git a/webapps/console/pages/api/admin/sync-logs-trim.ts b/webapps/console/pages/api/admin/sync-logs-trim.ts index b71f23060..bb78a095e 100644 --- a/webapps/console/pages/api/admin/sync-logs-trim.ts +++ b/webapps/console/pages/api/admin/sync-logs-trim.ts @@ -1,3 +1,7 @@ +/* eslint-disable @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-argument -- + * Pre-existing implicit-`any` debt, exempted when the unsafe-any gate was + * introduced for pages/api/admin (JITSU-158 action item 3). Fix the `any` + * flows in this file, then remove this header - do not add new ones. */ import { createRoute, verifyAdmin } from "../../../lib/api"; import { z } from "zod"; import { getServerLog } from "../../../lib/server/log"; diff --git a/webapps/console/pages/api/admin/sync-quota-check.ts b/webapps/console/pages/api/admin/sync-quota-check.ts index 7ee012d9d..fe4f010e6 100644 --- a/webapps/console/pages/api/admin/sync-quota-check.ts +++ b/webapps/console/pages/api/admin/sync-quota-check.ts @@ -1,3 +1,7 @@ +/* eslint-disable @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access -- + * Pre-existing implicit-`any` debt, exempted when the unsafe-any gate was + * introduced for pages/api/admin (JITSU-158 action item 3). Fix the `any` + * flows in this file, then remove this header - do not add new ones. */ import { z } from "zod"; import { createRoute } from "../../../lib/api"; import { getServerEnv } from "../../../lib/server/serverEnv"; diff --git a/webapps/console/pages/api/admin/users.ts b/webapps/console/pages/api/admin/users.ts index 2c89f6d9c..9d6148abf 100644 --- a/webapps/console/pages/api/admin/users.ts +++ b/webapps/console/pages/api/admin/users.ts @@ -1,3 +1,7 @@ +/* eslint-disable @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-argument -- + * Pre-existing implicit-`any` debt, exempted when the unsafe-any gate was + * introduced for pages/api/admin (JITSU-158 action item 3). Fix the `any` + * flows in this file, then remove this header - do not add new ones. */ import { getUser } from "../../../lib/api"; import { z } from "zod"; import { assertDefined, assertTrue, getErrorMessage, requireDefined } from "juava"; From af9b959fc572196757c3eab6a55ff9f4ccd5c000 Mon Sep 17 00:00:00 2001 From: Ildar Nurislamov Date: Tue, 4 Aug 2026 16:21:37 +0400 Subject: [PATCH 3/5] feat(console): parse connection options with the destination-type schema Connection options in the three connection exports now parse through the destination type's own connectionOptions schema (destinations.tsx) with .passthrough(), falling back to the generic tolerant parse only when the type is unknown or the stored data does not conform (logged at warn). Absent fields therefore materialize to the console defaults - the export emits explicit deduplicate: true, mode: batch, frequency: 60, primaryKey: message_id, ... instead of leaving consumers to re-default absent options themselves, differently and unsafely (JITSU-136). A blank-data row - the 2026-07-30 failure shape - now exports safe defaults rather than nothing. Stored values always win over defaults; unknown keys pass through untouched. Note: one-time optionsHash churn on deploy, and connections that never persisted frequency move from bulker's fast absent-default to the 60m the UI has always shown. Co-Authored-By: Claude Fable 5 --- .../pages/api/admin/export/[name]/index.ts | 82 +++++++++++++------ 1 file changed, 58 insertions(+), 24 deletions(-) diff --git a/webapps/console/pages/api/admin/export/[name]/index.ts b/webapps/console/pages/api/admin/export/[name]/index.ts index e572abbbe..f02267987 100644 --- a/webapps/console/pages/api/admin/export/[name]/index.ts +++ b/webapps/console/pages/api/admin/export/[name]/index.ts @@ -81,7 +81,8 @@ function asRecord(v: unknown): Record { const parsed = JsonRecord.safeParse(v); return parsed.success ? parsed.data : {}; } -// Connection options (ConfigurationObjectLink.data). +// Connection options (ConfigurationObjectLink.data) - the generic fallback +// shape when the destination-type schema can't be applied. const LinkData = z .object({ disabled: z.unknown().optional(), @@ -90,6 +91,37 @@ const LinkData = z }) .passthrough() .catch({}); +type LinkDataParsed = z.infer; + +// Parses connection options with the destination type's own connectionOptions +// schema (lib/schema/destinations.tsx), so absent fields materialize to the +// console defaults (deduplicate: true, mode: batch, ...) instead of being +// omitted and re-defaulted - differently and unsafely - by bulker/rotor +// (JITSU-136 / JITSU-158). `.passthrough()` is essential: the schemas strip +// unknown keys by default, and a field consumers understand but the console +// schema doesn't list yet must still flow through. Falls back to the generic +// tolerant parse when the type is unknown or the stored data doesn't conform. +const linkDataSchemaCache = new Map(); +function parseLinkData(destinationType: string | undefined, data: unknown): LinkDataParsed { + const coreType = getCoreDestinationTypeNonStrict(destinationType); + if (coreType) { + let schema = linkDataSchemaCache.get(coreType.id); + if (!schema) { + schema = coreType.connectionOptions.passthrough(); + linkDataSchemaCache.set(coreType.id, schema); + } + const parsed = schema.safeParse(data ?? {}); + if (parsed.success) { + return parsed.data as LinkDataParsed; + } + getLog() + .atWarn() + .log( + `Connection options do not conform to the '${destinationType}' schema, exporting stored fields as-is: ${parsed.error.message}` + ); + } + return LinkData.parse(data); +} // ConfigurationObject.config - the fields the exports read by name. const ObjectConfig = z .object({ @@ -243,12 +275,12 @@ async function exportBulkerConnections(writer: Writer) { for (const { data: data_, from, id, to, updatedAt, workspace } of objects) { let payload: string | undefined; try { - const data = LinkData.parse(data_); + const toConfig = ObjectConfig.parse(to.config); + const destinationType = toConfig.destinationType; + const data = parseLinkData(destinationType, data_); if (data.disabled) { continue; // skip disabled connections } - const toConfig = ObjectConfig.parse(to.config); - const destinationType = toConfig.destinationType; const coreDestinationType = getCoreDestinationTypeNonStrict(destinationType); if (coreDestinationType?.usesBulker || coreDestinationType?.hybrid) { const credentials: Record = omit(toConfig, "destinationType", "type", "name"); @@ -439,11 +471,11 @@ async function exportRotorConnections(writer: Writer) { for (const { data: data_, from, id, to, updatedAt, workspace } of objects) { let payload: string | undefined; try { - const data = LinkData.parse(data_); + const destinationType = ObjectConfig.parse(to.config).destinationType; + const data = parseLinkData(destinationType, data_); if (data.disabled) { continue; // skip disabled connections } - const destinationType = ObjectConfig.parse(to.config).destinationType; const coreDestinationType = getCoreDestinationTypeNonStrict(destinationType); if (!coreDestinationType) { getLog().atError().log(`Unknown destination type: ${destinationType} for connection ${id}`); @@ -776,26 +808,28 @@ async function exportStreamsWithDestinations(writer: Writer) { captureHeaders: (obj.workspace.featuresEnabled || []).includes("captureHeaders"), destinations: [ ...obj.toLinks - .filter(l => !l.deleted && l.type === "push" && !LinkData.parse(l.data).disabled && !l.to.deleted) + .filter(l => !l.deleted && l.type === "push" && !l.to.deleted) .map(l => { const toConfig = ObjectConfig.parse(l.to.config); - return { - id: l.to.id, - connectionId: l.id, - destinationType: toConfig.destinationType, - name: toConfig.name, - credentials: omit(toConfig, "destinationType", "type", "name"), - options: { - ...LinkData.parse(l.data), - functionsServer: selectFunctionsServer( - functionsServers, - obj.workspace.id, - l.id, - functionsClassFunc(obj.workspace) - ), - }, - }; - }), + return { l, toConfig, data: parseLinkData(toConfig.destinationType, l.data) }; + }) + .filter(({ data }) => !data.disabled) + .map(({ l, toConfig, data }) => ({ + id: l.to.id, + connectionId: l.id, + destinationType: toConfig.destinationType, + name: toConfig.name, + credentials: omit(toConfig, "destinationType", "type", "name"), + options: { + ...data, + functionsServer: selectFunctionsServer( + functionsServers, + obj.workspace.id, + l.id, + functionsClassFunc(obj.workspace) + ), + }, + })), ...(pbMap.get(obj.workspace.id) ?? []).map(pb => { const connectionOptions = PbConnectionOptions.parse(pb.connectionOptions); return { From 699b1a7e91683cff0f48d0b5d838b7576b2bf830 Mon Sep 17 00:00:00 2001 From: Ildar Nurislamov Date: Tue, 4 Aug 2026 16:23:11 +0400 Subject: [PATCH 4/5] fix(console): keep frequency absent unless actually stored (back compat) The console default (60m) differs from bulker's absent-option default, so materializing it would silently change the batch cadence of every connection that never persisted a frequency. All other defaults still materialize; frequency exports only when stored (explicit null included). Co-Authored-By: Claude Fable 5 --- webapps/console/pages/api/admin/export/[name]/index.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/webapps/console/pages/api/admin/export/[name]/index.ts b/webapps/console/pages/api/admin/export/[name]/index.ts index f02267987..b9c5114a1 100644 --- a/webapps/console/pages/api/admin/export/[name]/index.ts +++ b/webapps/console/pages/api/admin/export/[name]/index.ts @@ -112,7 +112,15 @@ function parseLinkData(destinationType: string | undefined, data: unknown): Link } const parsed = schema.safeParse(data ?? {}); if (parsed.success) { - return parsed.data as LinkDataParsed; + const result = parsed.data as LinkDataParsed; + // Back-compat: frequency's console default (60m) differs from bulker's + // absent-option default, so materializing it would change the batch + // cadence of every connection that never persisted it. Keep it absent + // unless actually stored. + if (!(data != null && typeof data === "object" && "frequency" in data)) { + delete result.frequency; + } + return result; } getLog() .atWarn() From 461c867a4363ed2e3a0c253645978f64b44649f4 Mon Sep 17 00:00:00 2001 From: Ildar Nurislamov Date: Tue, 4 Aug 2026 16:53:06 +0400 Subject: [PATCH 5/5] fix(console): apply passthrough at every schema level, not just the root .passthrough() only affects the object it is called on - unknown keys inside nested declared objects were still stripped by the per-type parse. Prod data has real instances (an enabled flag on functions[] entries of 3 live connections) that the old spread-raw-data export preserved. deepPassthrough rebuilds the connectionOptions schema with passthrough on every nested object; defaults and validation semantics are unchanged (verified). Co-Authored-By: Claude Fable 5 --- .../pages/api/admin/export/[name]/index.ts | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/webapps/console/pages/api/admin/export/[name]/index.ts b/webapps/console/pages/api/admin/export/[name]/index.ts index b9c5114a1..26408e5ee 100644 --- a/webapps/console/pages/api/admin/export/[name]/index.ts +++ b/webapps/console/pages/api/admin/export/[name]/index.ts @@ -101,13 +101,39 @@ type LinkDataParsed = z.infer; // unknown keys by default, and a field consumers understand but the console // schema doesn't list yet must still flow through. Falls back to the generic // tolerant parse when the type is unknown or the stored data doesn't conform. +// `.passthrough()` only affects the object it is applied to - unknown keys +// inside nested declared objects (e.g. an `enabled` flag on a functions[] +// entry) would still be stripped. Rebuild the schema with passthrough at +// every object level so stored fields are never silently dropped. +function deepPassthrough(schema: z.ZodTypeAny): z.ZodTypeAny { + if (schema instanceof z.ZodObject) { + const shape = Object.fromEntries( + Object.entries(schema.shape as z.ZodRawShape).map(([k, v]) => [k, deepPassthrough(v)]) + ); + return z.object(shape).passthrough(); + } + if (schema instanceof z.ZodArray) { + return new z.ZodArray({ ...schema._def, type: deepPassthrough(schema._def.type as z.ZodTypeAny) }); + } + if (schema instanceof z.ZodOptional) { + return new z.ZodOptional({ ...schema._def, innerType: deepPassthrough(schema._def.innerType as z.ZodTypeAny) }); + } + if (schema instanceof z.ZodNullable) { + return new z.ZodNullable({ ...schema._def, innerType: deepPassthrough(schema._def.innerType as z.ZodTypeAny) }); + } + if (schema instanceof z.ZodDefault) { + return new z.ZodDefault({ ...schema._def, innerType: deepPassthrough(schema._def.innerType as z.ZodTypeAny) }); + } + return schema; +} + const linkDataSchemaCache = new Map(); function parseLinkData(destinationType: string | undefined, data: unknown): LinkDataParsed { const coreType = getCoreDestinationTypeNonStrict(destinationType); if (coreType) { let schema = linkDataSchemaCache.get(coreType.id); if (!schema) { - schema = coreType.connectionOptions.passthrough(); + schema = deepPassthrough(coreType.connectionOptions); linkDataSchemaCache.set(coreType.id, schema); } const parsed = schema.safeParse(data ?? {});