From d9833582f56243c55fe3b5d8f388307b18123dc4 Mon Sep 17 00:00:00 2001 From: Vigneshraj Sekar Babu Date: Wed, 13 May 2026 11:13:14 -0700 Subject: [PATCH 1/6] feat: add static sites hosting gateway --- Tiltfile | 3 + helm/environments/default/lifecycle.yaml | 48 ++ helm/environments/local/lifecycle.yaml | 50 +- .../api/v2/sites/[siteId]/content/route.ts | 46 ++ src/app/api/v2/sites/[siteId]/extend/route.ts | 39 ++ src/app/api/v2/sites/[siteId]/route.ts | 50 ++ src/app/api/v2/sites/route.ts | 51 ++ .../db/migrations/027_add_sites_hosting.ts | 143 ++++++ src/server/jobs/index.ts | 6 + src/server/lib/sites/config.test.ts | 44 ++ src/server/lib/sites/config.ts | 212 ++++++++ src/server/lib/sites/contentType.ts | 54 ++ src/server/lib/sites/routeHelpers.ts | 40 ++ src/server/lib/sites/storage.ts | 157 ++++++ src/server/lib/sites/validation.test.ts | 184 +++++++ src/server/lib/sites/validation.ts | 334 +++++++++++++ src/server/models/Site.ts | 51 ++ src/server/models/SiteVersion.ts | 54 ++ src/server/models/index.ts | 6 + src/server/services/index.ts | 2 + src/server/services/sites.ts | 467 ++++++++++++++++++ src/server/services/types/globalConfig.ts | 36 ++ src/server/services/types/index.ts | 2 + src/shared/config.ts | 1 + ws-server.ts | 64 +++ 25 files changed, 2143 insertions(+), 1 deletion(-) create mode 100644 src/app/api/v2/sites/[siteId]/content/route.ts create mode 100644 src/app/api/v2/sites/[siteId]/extend/route.ts create mode 100644 src/app/api/v2/sites/[siteId]/route.ts create mode 100644 src/app/api/v2/sites/route.ts create mode 100644 src/server/db/migrations/027_add_sites_hosting.ts create mode 100644 src/server/lib/sites/config.test.ts create mode 100644 src/server/lib/sites/config.ts create mode 100644 src/server/lib/sites/contentType.ts create mode 100644 src/server/lib/sites/routeHelpers.ts create mode 100644 src/server/lib/sites/storage.ts create mode 100644 src/server/lib/sites/validation.test.ts create mode 100644 src/server/lib/sites/validation.ts create mode 100644 src/server/models/Site.ts create mode 100644 src/server/models/SiteVersion.ts create mode 100644 src/server/services/sites.ts diff --git a/Tiltfile b/Tiltfile index ba13ec89..34b323a3 100644 --- a/Tiltfile +++ b/Tiltfile @@ -428,6 +428,9 @@ for r in patched_deploy: if "web" in name: labels = ["web"] port_forwards = ['5001:80'] + elif "gateway" in name: + labels = ["gateway"] + port_forwards = ['5002:80'] elif "worker" in name: labels = ["worker"] resource_deps.append('lifecycle-web') diff --git a/helm/environments/default/lifecycle.yaml b/helm/environments/default/lifecycle.yaml index 988d8114..f205f811 100644 --- a/helm/environments/default/lifecycle.yaml +++ b/helm/environments/default/lifecycle.yaml @@ -158,6 +158,54 @@ components: periodSeconds: 5 failureThreshold: 3 + gateway: + enabled: false + fullnameOverride: 'lifecycle-gateway' + + service: + enabled: true + type: ClusterIP + port: 80 + targetPort: 80 + + ingress: + enabled: true + ingressClassName: nginx + annotations: {} + hosts: + - host: '*.sites.' + paths: ['/'] + + deployment: + replicaCount: 2 + resources: + requests: + cpu: 100m + memory: 200Mi + limits: + cpu: 500m + memory: 512Mi + extraEnv: + - name: LIFECYCLE_MODE + value: gateway + ports: + - name: http + containerPort: 80 + protocol: TCP + livenessProbe: + httpGet: + path: /api/health + port: http + initialDelaySeconds: 30 + periodSeconds: 10 + failureThreshold: 6 + readinessProbe: + httpGet: + path: /api/health + port: http + periodSeconds: 5 + failureThreshold: 3 + postgresql: enabled: false diff --git a/helm/environments/local/lifecycle.yaml b/helm/environments/local/lifecycle.yaml index 8bac805e..6196d07b 100644 --- a/helm/environments/local/lifecycle.yaml +++ b/helm/environments/local/lifecycle.yaml @@ -179,6 +179,54 @@ components: failureThreshold: 10 timeoutSeconds: 10 + gateway: + enabled: true + fullnameOverride: 'lifecycle-gateway' + service: + enabled: true + type: ClusterIP + port: 80 + targetPort: 80 + ingress: + enabled: false + deployment: + replicaCount: 1 + extraEnv: + - name: JOB_VERSION + value: default + - name: ENVIRONMENT + value: dev + - name: APP_ENV + value: dev + - name: NODE_ENV + value: development + - name: LIFECYCLE_MODE + value: gateway + - name: PORT + value: '80' + - name: DD_TRACE_ENABLED + value: 'false' + ports: + - name: http + containerPort: 80 + protocol: TCP + livenessProbe: + httpGet: + path: /api/health + port: 80 + initialDelaySeconds: 5 + periodSeconds: 30 + failureThreshold: 6 + timeoutSeconds: 60 + readinessProbe: + httpGet: + path: /api/health + port: 80 + initialDelaySeconds: 5 + periodSeconds: 30 + failureThreshold: 12 + timeoutSeconds: 60 + postgresql: enabled: false @@ -187,7 +235,7 @@ redis: minio: enabled: false - defaultBuckets: 'lifecycle-logs' + defaultBuckets: 'lifecycle-logs,lifecycle-sites' auth: rootUser: minioadmin rootPassword: minioadmin diff --git a/src/app/api/v2/sites/[siteId]/content/route.ts b/src/app/api/v2/sites/[siteId]/content/route.ts new file mode 100644 index 00000000..93c15560 --- /dev/null +++ b/src/app/api/v2/sites/[siteId]/content/route.ts @@ -0,0 +1,46 @@ +/** + * Copyright 2026 GoodRx, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { NextRequest } from 'next/server'; +import { createApiHandler } from 'server/lib/createApiHandler'; +import { getRequestUserIdentity } from 'server/lib/get-user'; +import { successResponse } from 'server/lib/response'; +import { readUploadFile, sitesErrorResponse } from 'server/lib/sites/routeHelpers'; +import SitesService from 'server/services/sites'; + +export const runtime = 'nodejs'; + +type RouteContext = { + params: { + siteId: string; + }; +}; + +const putHandler = async (req: NextRequest, { params }: RouteContext) => { + try { + const upload = await readUploadFile(req); + const service = new SitesService(); + const site = await service.replaceSiteContent(params.siteId, { + ...upload, + user: getRequestUserIdentity(req), + }); + return successResponse({ site }, { status: 200 }, req); + } catch (error) { + return sitesErrorResponse(error, req); + } +}; + +export const PUT = createApiHandler(putHandler); diff --git a/src/app/api/v2/sites/[siteId]/extend/route.ts b/src/app/api/v2/sites/[siteId]/extend/route.ts new file mode 100644 index 00000000..3e1e355c --- /dev/null +++ b/src/app/api/v2/sites/[siteId]/extend/route.ts @@ -0,0 +1,39 @@ +/** + * Copyright 2026 GoodRx, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { NextRequest } from 'next/server'; +import { createApiHandler } from 'server/lib/createApiHandler'; +import { successResponse } from 'server/lib/response'; +import { sitesErrorResponse } from 'server/lib/sites/routeHelpers'; +import SitesService from 'server/services/sites'; + +type RouteContext = { + params: { + siteId: string; + }; +}; + +const postHandler = async (req: NextRequest, { params }: RouteContext) => { + try { + const service = new SitesService(); + const site = await service.extendSite(params.siteId); + return successResponse({ site }, { status: 200 }, req); + } catch (error) { + return sitesErrorResponse(error, req); + } +}; + +export const POST = createApiHandler(postHandler); diff --git a/src/app/api/v2/sites/[siteId]/route.ts b/src/app/api/v2/sites/[siteId]/route.ts new file mode 100644 index 00000000..3b1255d9 --- /dev/null +++ b/src/app/api/v2/sites/[siteId]/route.ts @@ -0,0 +1,50 @@ +/** + * Copyright 2026 GoodRx, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { NextRequest } from 'next/server'; +import { createApiHandler } from 'server/lib/createApiHandler'; +import { successResponse } from 'server/lib/response'; +import { sitesErrorResponse } from 'server/lib/sites/routeHelpers'; +import SitesService from 'server/services/sites'; + +type RouteContext = { + params: { + siteId: string; + }; +}; + +const getHandler = async (req: NextRequest, { params }: RouteContext) => { + try { + const service = new SitesService(); + const site = await service.getSite(params.siteId); + return successResponse({ site }, { status: 200 }, req); + } catch (error) { + return sitesErrorResponse(error, req); + } +}; + +const deleteHandler = async (req: NextRequest, { params }: RouteContext) => { + try { + const service = new SitesService(); + const site = await service.deleteSite(params.siteId); + return successResponse({ site }, { status: 200 }, req); + } catch (error) { + return sitesErrorResponse(error, req); + } +}; + +export const GET = createApiHandler(getHandler); +export const DELETE = createApiHandler(deleteHandler); diff --git a/src/app/api/v2/sites/route.ts b/src/app/api/v2/sites/route.ts new file mode 100644 index 00000000..c0b55e34 --- /dev/null +++ b/src/app/api/v2/sites/route.ts @@ -0,0 +1,51 @@ +/** + * Copyright 2026 GoodRx, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { NextRequest } from 'next/server'; +import { createApiHandler } from 'server/lib/createApiHandler'; +import { getRequestUserIdentity } from 'server/lib/get-user'; +import { successResponse } from 'server/lib/response'; +import { readUploadFile, sitesErrorResponse } from 'server/lib/sites/routeHelpers'; +import SitesService from 'server/services/sites'; + +export const runtime = 'nodejs'; + +const getHandler = async (req: NextRequest) => { + try { + const service = new SitesService(); + const sites = await service.listSites(); + return successResponse({ sites }, { status: 200 }, req); + } catch (error) { + return sitesErrorResponse(error, req); + } +}; + +const postHandler = async (req: NextRequest) => { + try { + const upload = await readUploadFile(req); + const service = new SitesService(); + const site = await service.createSite({ + ...upload, + user: getRequestUserIdentity(req), + }); + return successResponse({ site }, { status: 201 }, req); + } catch (error) { + return sitesErrorResponse(error, req); + } +}; + +export const GET = createApiHandler(getHandler); +export const POST = createApiHandler(postHandler); diff --git a/src/server/db/migrations/027_add_sites_hosting.ts b/src/server/db/migrations/027_add_sites_hosting.ts new file mode 100644 index 00000000..ba973373 --- /dev/null +++ b/src/server/db/migrations/027_add_sites_hosting.ts @@ -0,0 +1,143 @@ +/** + * Copyright 2026 GoodRx, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Knex } from 'knex'; + +export const config = { + transaction: true, +}; + +const SITES_TABLE = 'sites'; +const SITE_VERSIONS_TABLE = 'site_versions'; +const SITES_CONFIG_KEY = 'sites'; + +const DEFAULT_SITES_CONFIG = { + enabled: false, + domain: 'localhost', + port: null, + hostPrefix: 'site', + ttl: { + enabled: true, + defaultDays: 7, + extensionDays: 7, + }, + upload: { + maxUploadBytes: 10 * 1024 * 1024, + maxExtractedBytes: 10 * 1024 * 1024, + maxFiles: 500, + allowedExtensions: [ + 'html', + 'zip', + 'json', + 'md', + 'markdown', + 'txt', + 'css', + 'js', + 'mjs', + 'map', + 'csv', + 'xml', + 'svg', + 'png', + 'jpg', + 'jpeg', + 'gif', + 'webp', + 'avif', + 'ico', + 'webmanifest', + 'wasm', + 'woff', + 'woff2', + 'ttf', + 'otf', + 'pdf', + ], + }, + storage: { + backend: 'minio', + bucket: 'lifecycle-sites', + prefix: 'sites', + region: 'us-west-2', + endpoint: null, + forcePathStyle: true, + }, + cleanup: { + enabled: true, + intervalMinutes: 15, + }, +}; + +export async function up(knex: Knex): Promise { + await knex.schema.createTable(SITES_TABLE, (table) => { + table.increments('id').primary(); + table.string('siteId', 32).notNullable(); + table.string('name', 255).notNullable(); + table.string('status', 32).notNullable().defaultTo('active'); + table.string('activeVersionId', 32).nullable(); + table.integer('fileCount').notNullable().defaultTo(0); + table.bigInteger('sizeBytes').notNullable().defaultTo(0); + table.timestamp('expiresAt').nullable(); + table.string('createdByUserId', 255).nullable(); + table.string('createdByDisplayName', 255).nullable(); + table.string('updatedByUserId', 255).nullable(); + table.string('updatedByDisplayName', 255).nullable(); + table.timestamp('createdAt').notNullable().defaultTo(knex.fn.now()); + table.timestamp('updatedAt').notNullable().defaultTo(knex.fn.now()); + table.timestamp('deletedAt').nullable(); + + table.unique(['siteId']); + table.index(['status', 'expiresAt']); + table.index(['deletedAt']); + }); + + await knex.schema.createTable(SITE_VERSIONS_TABLE, (table) => { + table.increments('id').primary(); + table.string('siteId', 32).notNullable(); + table.string('versionId', 32).notNullable(); + table.string('storagePrefix', 512).notNullable(); + table.string('entrypoint', 255).notNullable().defaultTo('index.html'); + table.integer('fileCount').notNullable(); + table.bigInteger('sizeBytes').notNullable(); + table.jsonb('manifest').notNullable().defaultTo('[]'); + table.timestamp('createdAt').notNullable().defaultTo(knex.fn.now()); + table.timestamp('updatedAt').notNullable().defaultTo(knex.fn.now()); + table.timestamp('deletedAt').nullable(); + + table.unique(['siteId', 'versionId']); + table.index(['siteId']); + table.foreign('siteId').references('siteId').inTable(SITES_TABLE).onDelete('CASCADE'); + }); + + const existingConfig = await knex('global_config').where('key', SITES_CONFIG_KEY).first(); + if (!existingConfig) { + await knex('global_config').insert({ + key: SITES_CONFIG_KEY, + config: DEFAULT_SITES_CONFIG, + createdAt: knex.fn.now(), + updatedAt: knex.fn.now(), + deletedAt: null, + description: 'Sites hosting configuration for uploaded static files and ZIP sites.', + }); + } +} + +export async function down(knex: Knex): Promise { + await knex('global_config').where('key', SITES_CONFIG_KEY).delete(); + await knex.schema.dropTableIfExists(SITE_VERSIONS_TABLE); + await knex.schema.dropTableIfExists(SITES_TABLE); +} diff --git a/src/server/jobs/index.ts b/src/server/jobs/index.ts index 539bf9c5..fd847d56 100644 --- a/src/server/jobs/index.ts +++ b/src/server/jobs/index.ts @@ -63,12 +63,18 @@ export default function bootstrapJobs(services: IServices) { /* Setup TTL cleanup job */ services.TTLCleanupService.setupTTLCleanupJob(); + services.SitesService.setupSitesCleanupJob(); queueManager.registerWorker(QUEUE_NAMES.TTL_CLEANUP, services.TTLCleanupService.processTTLCleanupQueue, { connection: redisClient.getConnection(), concurrency: 1, }); + queueManager.registerWorker(QUEUE_NAMES.SITES_CLEANUP, services.SitesService.processSitesCleanupQueue, { + connection: redisClient.getConnection(), + concurrency: 1, + }); + queueManager.registerWorker(QUEUE_NAMES.INGRESS_MANIFEST, services.Ingress.createOrUpdateIngressForBuild, { connection: redisClient.getConnection(), concurrency: 1, diff --git a/src/server/lib/sites/config.test.ts b/src/server/lib/sites/config.test.ts new file mode 100644 index 00000000..8b1c89f6 --- /dev/null +++ b/src/server/lib/sites/config.test.ts @@ -0,0 +1,44 @@ +/** + * Copyright 2026 GoodRx, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +jest.mock('shared/config', () => ({ + OBJECT_STORE_ACCESS_KEY: 'minioadmin', + OBJECT_STORE_ENDPOINT: 'minio', + OBJECT_STORE_PORT: '9000', + OBJECT_STORE_REGION: 'us-west-2', + OBJECT_STORE_SECRET_KEY: 'minioadmin', + OBJECT_STORE_TYPE: 'minio', + OBJECT_STORE_USE_SSL: 'false', +})); + +import { buildSiteUrl, parseSiteIdFromHost, resolveSitesConfig } from './config'; + +describe('sites config host prefix', () => { + it('defaults to disabled when sites config is missing', () => { + expect(resolveSitesConfig().enabled).toBe(false); + }); + + it('uses the configured host prefix for site URLs and host parsing', () => { + const config = resolveSitesConfig({ + domain: 'sites.example.com', + hostPrefix: 'artifact', + }); + + expect(buildSiteUrl('abc123', config)).toBe('https://artifact-abc123.sites.example.com'); + expect(parseSiteIdFromHost('artifact-abc123.sites.example.com', config)).toBe('abc123'); + expect(parseSiteIdFromHost('site-abc123.sites.example.com', config)).toBeNull(); + }); +}); diff --git a/src/server/lib/sites/config.ts b/src/server/lib/sites/config.ts new file mode 100644 index 00000000..6b67080a --- /dev/null +++ b/src/server/lib/sites/config.ts @@ -0,0 +1,212 @@ +/** + * Copyright 2026 GoodRx, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + OBJECT_STORE_ACCESS_KEY, + OBJECT_STORE_ENDPOINT, + OBJECT_STORE_PORT, + OBJECT_STORE_REGION, + OBJECT_STORE_SECRET_KEY, + OBJECT_STORE_TYPE, + OBJECT_STORE_USE_SSL, +} from 'shared/config'; +import type { SitesConfig } from 'server/services/types/globalConfig'; + +export const TEN_MIB = 10 * 1024 * 1024; +export const DEFAULT_HOST_PREFIX = 'site'; +export const DEFAULT_ALLOWED_EXTENSIONS = [ + 'html', + 'zip', + 'json', + 'md', + 'markdown', + 'txt', + 'css', + 'js', + 'mjs', + 'map', + 'csv', + 'xml', + 'svg', + 'png', + 'jpg', + 'jpeg', + 'gif', + 'webp', + 'avif', + 'ico', + 'webmanifest', + 'wasm', + 'woff', + 'woff2', + 'ttf', + 'otf', + 'pdf', +]; + +export type ResolvedSitesConfig = { + enabled: boolean; + domain: string; + port: number | null; + hostPrefix: string; + ttl: { + enabled: boolean; + defaultDays: number; + extensionDays: number; + }; + upload: { + maxUploadBytes: number; + maxExtractedBytes: number; + maxFiles: number; + allowedExtensions: string[]; + }; + storage: { + backend: 's3' | 'minio'; + bucket: string; + prefix: string; + region: string; + endpoint: string | null; + forcePathStyle: boolean; + accessKeyId?: string; + secretAccessKey?: string; + }; + cleanup: { + enabled: boolean; + intervalMinutes: number; + }; +}; + +function normalizeDomain(domain?: string | null): { domain: string; port: number | null } { + const rawDomain = (domain || 'localhost') + .trim() + .replace(/^https?:\/\//, '') + .replace(/\/.*$/, '') + .replace(/^\*\./, '') + .replace(/\.$/, ''); + const portMatch = rawDomain.match(/:(\d+)$/); + const parsedPort = portMatch ? Number(portMatch[1]) : null; + const normalizedDomain = portMatch ? rawDomain.slice(0, -portMatch[0].length) : rawDomain; + + return { + domain: normalizedDomain || 'localhost', + port: + parsedPort !== null && Number.isInteger(parsedPort) && parsedPort > 0 && parsedPort <= 65535 ? parsedPort : null, + }; +} + +function normalizePort(port?: number | string | null): number | null { + const value = typeof port === 'string' ? Number(port) : port; + return Number.isInteger(value) && Number(value) > 0 && Number(value) <= 65535 ? Number(value) : null; +} + +function normalizeHostPrefix(prefix?: string | null): string { + const normalized = (prefix || DEFAULT_HOST_PREFIX) + .trim() + .toLowerCase() + .replace(/[^a-z0-9-]/g, '-') + .replace(/^-+|-+$/g, '') + .replace(/-{2,}/g, '-'); + + return normalized || DEFAULT_HOST_PREFIX; +} + +function normalizeAllowedExtensions(config?: SitesConfig | null): string[] { + const extensions = config?.upload?.allowedExtensions || config?.upload?.allowedTypes || DEFAULT_ALLOWED_EXTENSIONS; + return Array.from( + new Set(extensions.map((extension) => extension.trim().toLowerCase().replace(/^\./, '')).filter(Boolean)) + ); +} + +function resolveEndpoint(config: SitesConfig | undefined): string | null { + if (config?.storage?.endpoint) { + return config.storage.endpoint; + } + + if ((config?.storage?.backend || OBJECT_STORE_TYPE) === 's3') { + return null; + } + + const protocol = OBJECT_STORE_USE_SSL === 'true' ? 'https' : 'http'; + return `${protocol}://${OBJECT_STORE_ENDPOINT}:${OBJECT_STORE_PORT}`; +} + +export function resolveSitesConfig(config?: SitesConfig | null): ResolvedSitesConfig { + const backend = (config?.storage?.backend || OBJECT_STORE_TYPE || 'minio') === 's3' ? 's3' : 'minio'; + const domain = normalizeDomain(config?.domain); + + return { + enabled: config?.enabled ?? false, + domain: domain.domain, + port: normalizePort(config?.port) ?? domain.port, + hostPrefix: normalizeHostPrefix(config?.hostPrefix), + ttl: { + enabled: config?.ttl?.enabled ?? true, + defaultDays: config?.ttl?.defaultDays ?? 7, + extensionDays: config?.ttl?.extensionDays ?? 7, + }, + upload: { + maxUploadBytes: config?.upload?.maxUploadBytes ?? TEN_MIB, + maxExtractedBytes: config?.upload?.maxExtractedBytes ?? TEN_MIB, + maxFiles: config?.upload?.maxFiles ?? 500, + allowedExtensions: normalizeAllowedExtensions(config), + }, + storage: { + backend, + bucket: config?.storage?.bucket || 'lifecycle-sites', + prefix: (config?.storage?.prefix || 'sites').replace(/^\/+|\/+$/g, ''), + region: config?.storage?.region || OBJECT_STORE_REGION || 'us-west-2', + endpoint: resolveEndpoint(config || undefined), + forcePathStyle: config?.storage?.forcePathStyle ?? backend === 'minio', + accessKeyId: backend === 'minio' ? OBJECT_STORE_ACCESS_KEY : undefined, + secretAccessKey: backend === 'minio' ? OBJECT_STORE_SECRET_KEY : undefined, + }, + cleanup: { + enabled: config?.cleanup?.enabled ?? true, + intervalMinutes: config?.cleanup?.intervalMinutes ?? 15, + }, + }; +} + +export function buildSiteUrl(siteId: string, config: ResolvedSitesConfig): string { + const protocol = config.domain === 'localhost' || config.domain.endsWith('.localhost') ? 'http' : 'https'; + const port = config.port ? `:${config.port}` : ''; + return `${protocol}://${config.hostPrefix}-${siteId}.${config.domain}${port}`; +} + +export function parseSiteIdFromHost(hostHeader: string | undefined, config: ResolvedSitesConfig): string | null { + if (!hostHeader) { + return null; + } + + const host = hostHeader.split(':')[0]?.toLowerCase(); + if (!host) { + return null; + } + + const suffix = `.${config.domain.toLowerCase()}`; + if (!host.endsWith(suffix)) { + return null; + } + + const label = host.slice(0, -suffix.length); + const prefix = `${config.hostPrefix}-`; + if (!label.startsWith(prefix)) { + return null; + } + + const siteId = label.slice(prefix.length); + return /^[a-z0-9-]+$/.test(siteId) ? siteId : null; +} diff --git a/src/server/lib/sites/contentType.ts b/src/server/lib/sites/contentType.ts new file mode 100644 index 00000000..a5c12aca --- /dev/null +++ b/src/server/lib/sites/contentType.ts @@ -0,0 +1,54 @@ +/** + * Copyright 2026 GoodRx, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const CONTENT_TYPES: Record = { + '.avif': 'image/avif', + '.css': 'text/css; charset=utf-8', + '.csv': 'text/csv; charset=utf-8', + '.gif': 'image/gif', + '.html': 'text/html; charset=utf-8', + '.ico': 'image/x-icon', + '.jpeg': 'image/jpeg', + '.jpg': 'image/jpeg', + '.js': 'text/javascript; charset=utf-8', + '.json': 'application/json; charset=utf-8', + '.map': 'application/json; charset=utf-8', + '.markdown': 'text/markdown; charset=utf-8', + '.md': 'text/markdown; charset=utf-8', + '.mjs': 'text/javascript; charset=utf-8', + '.otf': 'font/otf', + '.pdf': 'application/pdf', + '.png': 'image/png', + '.svg': 'image/svg+xml', + '.txt': 'text/plain; charset=utf-8', + '.ttf': 'font/ttf', + '.wasm': 'application/wasm', + '.webmanifest': 'application/manifest+json', + '.webp': 'image/webp', + '.woff': 'font/woff', + '.woff2': 'font/woff2', + '.xml': 'application/xml; charset=utf-8', + '.zip': 'application/zip', +}; + +export function getContentType(filePath: string): string { + const lastDot = filePath.lastIndexOf('.'); + if (lastDot < 0) { + return 'application/octet-stream'; + } + + return CONTENT_TYPES[filePath.slice(lastDot).toLowerCase()] || 'application/octet-stream'; +} diff --git a/src/server/lib/sites/routeHelpers.ts b/src/server/lib/sites/routeHelpers.ts new file mode 100644 index 00000000..32bbeaaa --- /dev/null +++ b/src/server/lib/sites/routeHelpers.ts @@ -0,0 +1,40 @@ +/** + * Copyright 2026 GoodRx, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { NextRequest, NextResponse } from 'next/server'; +import { errorResponse } from 'server/lib/response'; +import { SitesServiceError } from 'server/services/sites'; + +export async function readUploadFile(req: NextRequest): Promise<{ fileName: string; content: Buffer; name?: string }> { + const formData = await req.formData(); + const file = formData.get('file'); + if (!file || typeof file === 'string') { + throw new SitesServiceError('A file upload is required.', 400); + } + + const nameValue = formData.get('name'); + const name = typeof nameValue === 'string' ? nameValue : undefined; + const arrayBuffer = await file.arrayBuffer(); + return { + fileName: file.name || 'upload', + content: Buffer.from(arrayBuffer), + name, + }; +} + +export function sitesErrorResponse(error: unknown, req: NextRequest): NextResponse { + return errorResponse(error, { status: error instanceof SitesServiceError ? error.statusCode : 500 }, req); +} diff --git a/src/server/lib/sites/storage.ts b/src/server/lib/sites/storage.ts new file mode 100644 index 00000000..f87a02a8 --- /dev/null +++ b/src/server/lib/sites/storage.ts @@ -0,0 +1,157 @@ +/** + * Copyright 2026 GoodRx, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + CreateBucketCommand, + DeleteObjectsCommand, + GetObjectCommand, + HeadBucketCommand, + ListObjectsV2Command, + PutObjectCommand, + S3Client, +} from '@aws-sdk/client-s3'; +import type { Readable } from 'stream'; +import { getLogger } from 'server/lib/logger'; +import type { ResolvedSitesConfig } from './config'; +import type { SiteUploadFile } from './validation'; + +export type SitesObject = { + body: Readable; + contentType: string; + contentLength?: number; +}; + +export class SitesObjectNotFoundError extends Error { + statusCode = 404; +} + +export class SitesStorage { + private client: S3Client; + private bucketVerified = false; + + constructor(private config: ResolvedSitesConfig) { + this.client = new S3Client({ + region: config.storage.region, + endpoint: config.storage.endpoint || undefined, + forcePathStyle: config.storage.forcePathStyle, + credentials: + config.storage.accessKeyId && config.storage.secretAccessKey + ? { + accessKeyId: config.storage.accessKeyId, + secretAccessKey: config.storage.secretAccessKey, + } + : undefined, + }); + } + + objectKey(storagePrefix: string, filePath: string): string { + return `${storagePrefix.replace(/\/+$/g, '')}/${filePath.replace(/^\/+/g, '')}`; + } + + versionPrefix(siteId: string, versionId: string): string { + return [this.config.storage.prefix, siteId, 'versions', versionId].filter(Boolean).join('/').replace(/\/+/g, '/'); + } + + sitePrefix(siteId: string): string { + return [this.config.storage.prefix, siteId].filter(Boolean).join('/').replace(/\/+/g, '/'); + } + + async ensureBucket(): Promise { + if (this.bucketVerified) return; + + try { + await this.client.send(new HeadBucketCommand({ Bucket: this.config.storage.bucket })); + this.bucketVerified = true; + } catch (error) { + if (this.config.storage.backend === 's3') { + getLogger().warn( + { error }, + `SitesStorage: bucket=${this.config.storage.bucket} not verified; ensure it is provisioned` + ); + return; + } + + await this.client.send(new CreateBucketCommand({ Bucket: this.config.storage.bucket })); + this.bucketVerified = true; + } + } + + async putFiles(storagePrefix: string, files: SiteUploadFile[]): Promise { + await this.ensureBucket(); + await Promise.all( + files.map((file) => + this.client.send( + new PutObjectCommand({ + Bucket: this.config.storage.bucket, + Key: this.objectKey(storagePrefix, file.path), + Body: file.content, + ContentType: file.contentType, + }) + ) + ) + ); + } + + async getObject(storagePrefix: string, filePath: string): Promise { + const key = this.objectKey(storagePrefix, filePath); + try { + const result = await this.client.send(new GetObjectCommand({ Bucket: this.config.storage.bucket, Key: key })); + if (!result.Body) { + throw new SitesObjectNotFoundError(`Object not found: ${filePath}`); + } + + return { + body: result.Body as Readable, + contentType: result.ContentType || 'application/octet-stream', + contentLength: result.ContentLength, + }; + } catch (error) { + if (error?.name === 'NoSuchKey' || error?.name === 'NotFound') { + throw new SitesObjectNotFoundError(`Object not found: ${filePath}`); + } + throw error; + } + } + + async deletePrefix(prefix: string): Promise { + let continuationToken: string | undefined; + do { + const result = await this.client.send( + new ListObjectsV2Command({ + Bucket: this.config.storage.bucket, + Prefix: `${prefix.replace(/\/+$/g, '')}/`, + ContinuationToken: continuationToken, + }) + ); + + const objects = (result.Contents || []) + .map((object) => object.Key) + .filter((key): key is string => Boolean(key)) + .map((Key) => ({ Key })); + + if (objects.length > 0) { + await this.client.send( + new DeleteObjectsCommand({ + Bucket: this.config.storage.bucket, + Delete: { Objects: objects, Quiet: true }, + }) + ); + } + + continuationToken = result.IsTruncated ? result.NextContinuationToken : undefined; + } while (continuationToken); + } +} diff --git a/src/server/lib/sites/validation.test.ts b/src/server/lib/sites/validation.test.ts new file mode 100644 index 00000000..61961409 --- /dev/null +++ b/src/server/lib/sites/validation.test.ts @@ -0,0 +1,184 @@ +/** + * Copyright 2026 GoodRx, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import zlib from 'zlib'; +import { normalizeGatewayPath, validateSiteUpload } from './validation'; + +const DEFAULT_OPTIONS = { + maxUploadBytes: 10 * 1024 * 1024, + maxExtractedBytes: 10 * 1024 * 1024, + maxFiles: 500, + allowedExtensions: ['html', 'zip', 'json', 'md', 'markdown', 'txt', 'js'], +}; + +function zip(entries: Record): Buffer { + const localParts: Buffer[] = []; + const centralParts: Buffer[] = []; + let offset = 0; + + for (const [entryPath, value] of Object.entries(entries)) { + const name = Buffer.from(entryPath); + const content = Buffer.from(value); + const compressed = zlib.deflateRawSync(content); + + const local = Buffer.alloc(30); + local.writeUInt32LE(0x04034b50, 0); + local.writeUInt16LE(20, 4); + local.writeUInt16LE(0x800, 6); + local.writeUInt16LE(8, 8); + local.writeUInt32LE(0, 10); + local.writeUInt32LE(0, 14); + local.writeUInt32LE(compressed.length, 18); + local.writeUInt32LE(content.length, 22); + local.writeUInt16LE(name.length, 26); + local.writeUInt16LE(0, 28); + + localParts.push(local, name, compressed); + + const central = Buffer.alloc(46); + central.writeUInt32LE(0x02014b50, 0); + central.writeUInt16LE(20, 4); + central.writeUInt16LE(20, 6); + central.writeUInt16LE(0x800, 8); + central.writeUInt16LE(8, 10); + central.writeUInt32LE(0, 12); + central.writeUInt32LE(0, 16); + central.writeUInt32LE(compressed.length, 20); + central.writeUInt32LE(content.length, 24); + central.writeUInt16LE(name.length, 28); + central.writeUInt16LE(0, 30); + central.writeUInt16LE(0, 32); + central.writeUInt16LE(0, 34); + central.writeUInt16LE(0, 36); + central.writeUInt32LE(0o100644 * 0x10000, 38); + central.writeUInt32LE(offset, 42); + centralParts.push(central, name); + + offset += local.length + name.length + compressed.length; + } + + const centralDirectory = Buffer.concat(centralParts); + const eocd = Buffer.alloc(22); + eocd.writeUInt32LE(0x06054b50, 0); + eocd.writeUInt16LE(0, 4); + eocd.writeUInt16LE(0, 6); + eocd.writeUInt16LE(Object.keys(entries).length, 8); + eocd.writeUInt16LE(Object.keys(entries).length, 10); + eocd.writeUInt32LE(centralDirectory.length, 12); + eocd.writeUInt32LE(offset, 16); + eocd.writeUInt16LE(0, 20); + + return Buffer.concat([...localParts, centralDirectory, eocd]); +} + +describe('validateSiteUpload', () => { + it('accepts a single html file as index.html', () => { + const result = validateSiteUpload({ + ...DEFAULT_OPTIONS, + fileName: 'demo.html', + content: Buffer.from('ok'), + }); + + expect(result.fileCount).toBe(1); + expect(result.files[0].path).toBe('index.html'); + expect(result.entrypoint).toBe('index.html'); + }); + + it('accepts safe single-file document uploads as the root entrypoint', () => { + const result = validateSiteUpload({ + ...DEFAULT_OPTIONS, + fileName: 'data.json', + content: Buffer.from('{"ok":true}'), + }); + + expect(result.fileCount).toBe(1); + expect(result.files[0].path).toBe('index.json'); + expect(result.entrypoint).toBe('index.json'); + }); + + it('accepts and strips a single top-level zip folder', () => { + const result = validateSiteUpload({ + ...DEFAULT_OPTIONS, + fileName: 'demo.zip', + content: zip({ + 'dist/index.html': 'ok', + 'dist/assets/app.js': 'console.log("ok")', + }), + }); + + expect(result.files.map((file) => file.path).sort()).toEqual(['assets/app.js', 'index.html']); + expect(result.entrypoint).toBe('index.html'); + }); + + it('rejects traversal entries', () => { + expect(() => + validateSiteUpload({ + ...DEFAULT_OPTIONS, + fileName: 'demo.zip', + content: zip({ + 'dist/index.html': 'ok', + '../secret.txt': 'no', + }), + }) + ).toThrow('path traversal'); + }); + + it('enforces upload and extracted size limits', () => { + expect(() => + validateSiteUpload({ + ...DEFAULT_OPTIONS, + fileName: 'demo.html', + maxUploadBytes: 2, + content: Buffer.from('too large'), + }) + ).toThrow('Upload size'); + + expect(() => + validateSiteUpload({ + ...DEFAULT_OPTIONS, + fileName: 'demo.zip', + maxExtractedBytes: 4, + content: zip({ 'index.html': 'too large' }), + }) + ).toThrow('Extracted site size'); + }); + + it('rejects unsupported single-file and zip entry extensions', () => { + expect(() => + validateSiteUpload({ + ...DEFAULT_OPTIONS, + fileName: 'demo.sh', + content: Buffer.from('echo no'), + }) + ).toThrow('Only these file extensions'); + + expect(() => + validateSiteUpload({ + ...DEFAULT_OPTIONS, + fileName: 'demo.zip', + content: zip({ 'index.html': 'ok', 'run.sh': 'echo no' }), + }) + ).toThrow('File extension is not supported'); + }); +}); + +describe('normalizeGatewayPath', () => { + it('normalizes root and rejects traversal paths', () => { + expect(normalizeGatewayPath('/')).toBe('index.html'); + expect(normalizeGatewayPath('/docs/')).toBe('docs/index.html'); + expect(() => normalizeGatewayPath('/../secret.txt')).toThrow('path traversal'); + }); +}); diff --git a/src/server/lib/sites/validation.ts b/src/server/lib/sites/validation.ts new file mode 100644 index 00000000..d6d7590d --- /dev/null +++ b/src/server/lib/sites/validation.ts @@ -0,0 +1,334 @@ +/** + * Copyright 2026 GoodRx, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import path from 'path'; +import zlib from 'zlib'; +import { getContentType } from './contentType'; + +export type SiteUploadFile = { + path: string; + content: Buffer; + sizeBytes: number; + contentType: string; +}; + +export type ValidatedSiteUpload = { + files: SiteUploadFile[]; + fileCount: number; + sizeBytes: number; + entrypoint: string; +}; + +export type SiteUploadValidationOptions = { + fileName: string; + content: Buffer; + maxUploadBytes: number; + maxExtractedBytes: number; + maxFiles: number; + allowedExtensions: string[]; +}; + +export class SiteUploadValidationError extends Error { + statusCode = 400 as const; +} + +function reject(message: string): never { + throw new SiteUploadValidationError(message); +} + +function getFileExtension(fileName: string): string { + return fileName.split('/').pop()?.split('.').pop()?.toLowerCase() || ''; +} + +function getAllowedExtension(fileName: string, allowedExtensions: string[]): string { + const ext = getFileExtension(fileName); + const allowed = new Set(allowedExtensions.map((extension) => extension.toLowerCase().replace(/^\./, ''))); + + if (ext && allowed.has(ext)) { + return ext; + } + + reject(`Only these file extensions are supported: ${Array.from(allowed).sort().join(', ')}.`); +} + +function normalizeArchivePath(input: string): string { + const candidate = input.replace(/\\/g, '/'); + if (!candidate || candidate.includes('\0')) { + reject('Archive contains an invalid path.'); + } + + if (candidate.startsWith('/') || /^[A-Za-z]:\//.test(candidate)) { + reject('Archive contains an absolute path.'); + } + + const normalized = path.posix.normalize(candidate); + if (!normalized || normalized === '.' || normalized.startsWith('../') || normalized.includes('/../')) { + reject('Archive contains a path traversal entry.'); + } + + return normalized; +} + +type ZipEntry = { + path: string; + content: Buffer; + sizeBytes: number; +}; + +function findEndOfCentralDirectory(buffer: Buffer): number { + const minOffset = Math.max(0, buffer.length - 0xffff - 22); + for (let offset = buffer.length - 22; offset >= minOffset; offset--) { + if (buffer.readUInt32LE(offset) === 0x06054b50) { + return offset; + } + } + reject('Invalid zip: central directory was not found.'); +} + +function isSymlink(externalAttributes: number): boolean { + return (((externalAttributes >>> 16) & 0o170000) as number) === 0o120000; +} + +function parseZipEntries(buffer: Buffer, maxExtractedBytes: number, maxFiles: number): ZipEntry[] { + const eocdOffset = findEndOfCentralDirectory(buffer); + const entryCount = buffer.readUInt16LE(eocdOffset + 10); + const centralDirectorySize = buffer.readUInt32LE(eocdOffset + 12); + const centralDirectoryOffset = buffer.readUInt32LE(eocdOffset + 16); + + if (entryCount === 0) { + reject('Zip upload is empty.'); + } + + if (centralDirectoryOffset + centralDirectorySize > buffer.length) { + reject('Invalid zip: central directory is out of bounds.'); + } + + const entries: ZipEntry[] = []; + let offset = centralDirectoryOffset; + let extractedSize = 0; + + for (let i = 0; i < entryCount; i++) { + if (offset + 46 > buffer.length || buffer.readUInt32LE(offset) !== 0x02014b50) { + reject('Invalid zip: central directory entry is malformed.'); + } + + const flags = buffer.readUInt16LE(offset + 8); + const method = buffer.readUInt16LE(offset + 10); + const compressedSize = buffer.readUInt32LE(offset + 20); + const uncompressedSize = buffer.readUInt32LE(offset + 24); + const fileNameLength = buffer.readUInt16LE(offset + 28); + const extraLength = buffer.readUInt16LE(offset + 30); + const commentLength = buffer.readUInt16LE(offset + 32); + const externalAttributes = buffer.readUInt32LE(offset + 38); + const localHeaderOffset = buffer.readUInt32LE(offset + 42); + const nameStart = offset + 46; + const nameEnd = nameStart + fileNameLength; + + if (nameEnd > buffer.length) { + reject('Invalid zip: filename is out of bounds.'); + } + + if (compressedSize === 0xffffffff || uncompressedSize === 0xffffffff) { + reject('Zip64 uploads are not supported in v1.'); + } + + if ((flags & 0x1) === 0x1) { + reject('Encrypted zip uploads are not supported.'); + } + + const rawName = buffer.slice(nameStart, nameEnd); + const entryName = rawName.toString((flags & 0x800) === 0x800 ? 'utf8' : 'utf8'); + const normalizedPath = normalizeArchivePath(entryName); + const isDirectory = normalizedPath.endsWith('/') || entryName.endsWith('/'); + + if (isSymlink(externalAttributes)) { + reject('Zip uploads cannot contain symlinks.'); + } + + if (!isDirectory) { + if (entries.length + 1 > maxFiles) { + reject(`Zip upload cannot contain more than ${maxFiles} files.`); + } + + extractedSize += uncompressedSize; + if (extractedSize > maxExtractedBytes) { + reject(`Extracted site size must be ${maxExtractedBytes} bytes or less.`); + } + + if (localHeaderOffset + 30 > buffer.length || buffer.readUInt32LE(localHeaderOffset) !== 0x04034b50) { + reject('Invalid zip: local file header is malformed.'); + } + + const localNameLength = buffer.readUInt16LE(localHeaderOffset + 26); + const localExtraLength = buffer.readUInt16LE(localHeaderOffset + 28); + const dataStart = localHeaderOffset + 30 + localNameLength + localExtraLength; + const dataEnd = dataStart + compressedSize; + + if (dataEnd > buffer.length) { + reject('Invalid zip: compressed file data is out of bounds.'); + } + + const compressed = buffer.slice(dataStart, dataEnd); + const content = + method === 0 ? Buffer.from(compressed) : method === 8 ? zlib.inflateRawSync(compressed) : undefined; + + if (!content) { + reject('Zip upload contains an unsupported compression method.'); + } + + if (content.length !== uncompressedSize) { + reject('Invalid zip: extracted file size does not match metadata.'); + } + + entries.push({ + path: normalizedPath, + content, + sizeBytes: content.length, + }); + } + + offset = nameEnd + extraLength + commentLength; + } + + return entries; +} + +function stripSingleTopLevelFolder(entries: ZipEntry[]): ZipEntry[] { + const paths = entries.map((entry) => entry.path); + if (paths.includes('index.html')) { + return entries; + } + + const topLevel = new Set(paths.map((entryPath) => entryPath.split('/')[0])); + if (topLevel.size !== 1) { + reject('Zip upload must contain index.html at the root or inside one top-level folder.'); + } + + const [folder] = Array.from(topLevel); + const prefix = `${folder}/`; + const stripped = entries.map((entry) => ({ + ...entry, + path: entry.path.startsWith(prefix) ? entry.path.slice(prefix.length) : entry.path, + })); + + if (!stripped.some((entry) => entry.path === 'index.html')) { + reject('Zip upload must contain index.html at the root or inside one top-level folder.'); + } + + return stripped; +} + +function assertAllowedFilePath(filePath: string, allowedExtensions: string[]) { + const ext = getFileExtension(filePath); + const allowed = new Set(allowedExtensions.map((extension) => extension.toLowerCase().replace(/^\./, ''))); + if (!ext || !allowed.has(ext)) { + reject(`File extension is not supported: ${filePath}`); + } +} + +function singleFileEntrypoint(extension: string): string { + return extension === 'markdown' ? 'index.markdown' : `index.${extension}`; +} + +function finalizeFiles( + entries: ZipEntry[], + maxFiles: number, + maxExtractedBytes: number, + allowedExtensions: string[], + entrypoint = 'index.html' +): ValidatedSiteUpload { + if (entries.length === 0) { + reject('Upload does not contain any files.'); + } + + if (entries.length > maxFiles) { + reject(`Site cannot contain more than ${maxFiles} files.`); + } + + const seen = new Set(); + let sizeBytes = 0; + const files = entries.map((entry) => { + const normalizedPath = normalizeArchivePath(entry.path); + assertAllowedFilePath(normalizedPath, allowedExtensions); + if (seen.has(normalizedPath)) { + reject(`Upload contains a duplicate file path: ${normalizedPath}`); + } + seen.add(normalizedPath); + sizeBytes += entry.sizeBytes; + return { + path: normalizedPath, + content: entry.content, + sizeBytes: entry.sizeBytes, + contentType: getContentType(normalizedPath), + }; + }); + + if (!seen.has(entrypoint)) { + reject(`Site must include ${entrypoint}.`); + } + + if (sizeBytes > maxExtractedBytes) { + reject(`Extracted site size must be ${maxExtractedBytes} bytes or less.`); + } + + return { + files, + fileCount: files.length, + sizeBytes, + entrypoint, + }; +} + +export function validateSiteUpload(options: SiteUploadValidationOptions): ValidatedSiteUpload { + const { fileName, content, maxUploadBytes, maxExtractedBytes, maxFiles, allowedExtensions } = options; + if (!content.length) { + reject('Upload file is empty.'); + } + + if (content.length > maxUploadBytes) { + reject(`Upload size must be ${maxUploadBytes} bytes or less.`); + } + + const uploadType = getAllowedExtension(fileName, allowedExtensions); + + if (uploadType !== 'zip') { + const entrypoint = singleFileEntrypoint(uploadType); + return finalizeFiles( + [ + { + path: entrypoint, + content, + sizeBytes: content.length, + }, + ], + maxFiles, + maxExtractedBytes, + allowedExtensions, + entrypoint + ); + } + + const entries = stripSingleTopLevelFolder(parseZipEntries(content, maxExtractedBytes, maxFiles)); + return finalizeFiles(entries, maxFiles, maxExtractedBytes, allowedExtensions); +} + +export function normalizeGatewayPath(pathname: string): string { + const rawPath = pathname.split('?')[0] || '/'; + const decoded = decodeURIComponent(rawPath); + const withoutLeadingSlash = decoded.replace(/^\/+/, '') || 'index.html'; + const normalized = normalizeArchivePath(withoutLeadingSlash); + return normalized.endsWith('/') ? `${normalized}index.html` : normalized; +} diff --git a/src/server/models/Site.ts b/src/server/models/Site.ts new file mode 100644 index 00000000..c32b2a31 --- /dev/null +++ b/src/server/models/Site.ts @@ -0,0 +1,51 @@ +/** + * Copyright 2026 GoodRx, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Model from './_Model'; + +export type SiteStatus = 'active' | 'deleted' | 'expired'; + +export default class Site extends Model { + siteId!: string; + name!: string; + status!: SiteStatus; + activeVersionId?: string | null; + fileCount!: number; + sizeBytes!: number | string; + expiresAt?: string | null; + createdByUserId?: string | null; + createdByDisplayName?: string | null; + updatedByUserId?: string | null; + updatedByDisplayName?: string | null; + + static tableName = 'sites'; + static timestamps = true; + static deleteable = true; + + static get relationMappings() { + const SiteVersion = require('./SiteVersion').default; + return { + versions: { + relation: Model.HasManyRelation, + modelClass: SiteVersion, + join: { + from: 'sites.siteId', + to: 'site_versions.siteId', + }, + }, + }; + } +} diff --git a/src/server/models/SiteVersion.ts b/src/server/models/SiteVersion.ts new file mode 100644 index 00000000..5f61a738 --- /dev/null +++ b/src/server/models/SiteVersion.ts @@ -0,0 +1,54 @@ +/** + * Copyright 2026 GoodRx, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Model from './_Model'; +import Site from './Site'; + +export type SiteVersionManifestEntry = { + path: string; + sizeBytes: number; + contentType: string; +}; + +export default class SiteVersion extends Model { + siteId!: string; + versionId!: string; + storagePrefix!: string; + entrypoint!: string; + fileCount!: number; + sizeBytes!: number | string; + manifest!: SiteVersionManifestEntry[]; + + static tableName = 'site_versions'; + static timestamps = true; + + static get relationMappings() { + return { + site: { + relation: Model.BelongsToOneRelation, + modelClass: Site, + join: { + from: 'site_versions.siteId', + to: 'sites.siteId', + }, + }, + }; + } + + static get jsonAttributes() { + return ['manifest']; + } +} diff --git a/src/server/models/index.ts b/src/server/models/index.ts index 2a831c7e..5675e96b 100644 --- a/src/server/models/index.ts +++ b/src/server/models/index.ts @@ -41,6 +41,8 @@ import AgentMessage from './AgentMessage'; import AgentPendingAction from './AgentPendingAction'; import AgentToolExecution from './AgentToolExecution'; import UserMcpConnection from './UserMcpConnection'; +import Site from './Site'; +import SiteVersion from './SiteVersion'; export interface IModels { Build: typeof Build; @@ -70,6 +72,8 @@ export interface IModels { AgentPendingAction: typeof AgentPendingAction; AgentToolExecution: typeof AgentToolExecution; UserMcpConnection: typeof UserMcpConnection; + Site: typeof Site; + SiteVersion: typeof SiteVersion; } export { @@ -100,4 +104,6 @@ export { AgentPendingAction, AgentToolExecution, UserMcpConnection, + Site, + SiteVersion, }; diff --git a/src/server/services/index.ts b/src/server/services/index.ts index 6c791ab1..a98d9464 100644 --- a/src/server/services/index.ts +++ b/src/server/services/index.ts @@ -31,6 +31,7 @@ import GlobalConfig from 'server/services/globalConfig'; import LabelService from 'server/services/label'; import TTLCleanupService from 'server/services/ttlCleanup'; import DeployCleanupService from 'server/services/deployCleanup'; +import SitesService from 'server/services/sites'; import { IServices } from 'server/services/types'; export default function createAndBindServices(): IServices { @@ -52,5 +53,6 @@ export default function createAndBindServices(): IServices { LabelService: new LabelService(), TTLCleanupService: new TTLCleanupService(), DeployCleanupService: new DeployCleanupService(), + SitesService: new SitesService(), }; } diff --git a/src/server/services/sites.ts b/src/server/services/sites.ts new file mode 100644 index 00000000..4e9b5e55 --- /dev/null +++ b/src/server/services/sites.ts @@ -0,0 +1,467 @@ +/** + * Copyright 2026 GoodRx, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Queue, Job } from 'bullmq'; +import { customAlphabet } from 'nanoid'; +import { Transaction } from 'objection'; +import Service from './_service'; +import GlobalConfigService from './globalConfig'; +import { QUEUE_NAMES } from 'shared/config'; +import { redisClient } from 'server/lib/dependencies'; +import { getLogger } from 'server/lib/logger'; +import { buildSiteUrl, parseSiteIdFromHost, resolveSitesConfig, ResolvedSitesConfig } from 'server/lib/sites/config'; +import { SitesObjectNotFoundError, SitesStorage } from 'server/lib/sites/storage'; +import { + normalizeGatewayPath, + SiteUploadValidationError, + validateSiteUpload, + ValidatedSiteUpload, +} from 'server/lib/sites/validation'; +import { getContentType } from 'server/lib/sites/contentType'; +import type Site from 'server/models/Site'; +import type SiteVersion from 'server/models/SiteVersion'; +import type { RequestUserIdentity } from 'server/lib/get-user'; + +const createSiteId = customAlphabet('abcdefghijklmnopqrstuvwxyz0123456789', 10); +const createVersionId = customAlphabet('abcdefghijklmnopqrstuvwxyz0123456789', 12); +const DAY_MS = 24 * 60 * 60 * 1000; +type SitesErrorStatusCode = 400 | 401 | 403 | 404 | 409 | 500 | 502 | 503; + +export class SitesServiceError extends Error { + constructor(message: string, public statusCode: SitesErrorStatusCode = 500) { + super(message); + } +} + +export type CreateOrReplaceSiteInput = { + fileName: string; + content: Buffer; + name?: string | null; + user?: RequestUserIdentity | null; +}; + +export type SiteResponse = { + id: string; + name: string; + url: string; + status: string; + createdAt: string | null; + updatedAt: string | null; + expiresAt: string | null; + fileCount: number; + sizeBytes: number; + createdByDisplayName: string | null; + updatedByDisplayName: string | null; +}; + +export type GatewayObjectResponse = { + body: NodeJS.ReadableStream; + contentType: string; + contentLength?: number; + statusCode: number; +}; + +export default class SitesService extends Service { + sitesCleanupQueue: Queue = this.queueManager.registerQueue(QUEUE_NAMES.SITES_CLEANUP, { + connection: redisClient.getConnection(), + defaultJobOptions: { + attempts: 1, + removeOnComplete: true, + removeOnFail: false, + }, + }); + + private async getConfig(): Promise { + const globalConfig = await GlobalConfigService.getInstance().getAllConfigs(); + return resolveSitesConfig(globalConfig.sites); + } + + private assertEnabled(config: ResolvedSitesConfig) { + if (!config.enabled) { + throw new SitesServiceError('Sites hosting is disabled.', 404); + } + } + + private serialize(site: Site, config: ResolvedSitesConfig): SiteResponse { + return { + id: site.siteId, + name: site.name, + url: buildSiteUrl(site.siteId, config), + status: site.status, + createdAt: site.createdAt || null, + updatedAt: site.updatedAt || null, + expiresAt: site.expiresAt || null, + fileCount: Number(site.fileCount || 0), + sizeBytes: Number(site.sizeBytes || 0), + createdByDisplayName: site.createdByDisplayName || null, + updatedByDisplayName: site.updatedByDisplayName || null, + }; + } + + private defaultSiteName(siteId: string, config: ResolvedSitesConfig): string { + return `${config.hostPrefix}-${siteId}`; + } + + private expirationForNewSite(config: ResolvedSitesConfig): string | null { + if (!config.ttl.enabled) { + return null; + } + return new Date(Date.now() + config.ttl.defaultDays * DAY_MS).toISOString(); + } + + private async createVersion( + siteId: string, + upload: ValidatedSiteUpload, + config: ResolvedSitesConfig, + uploadedStoragePrefixes: string[], + trx?: Transaction + ): Promise { + const versionId = createVersionId(); + const storage = new SitesStorage(config); + const storagePrefix = storage.versionPrefix(siteId, versionId); + + uploadedStoragePrefixes.push(storagePrefix); + await storage.putFiles(storagePrefix, upload.files); + + return this.db.models.SiteVersion.query(trx).insert({ + siteId, + versionId, + storagePrefix, + entrypoint: upload.entrypoint, + fileCount: upload.fileCount, + sizeBytes: upload.sizeBytes, + manifest: upload.files.map(({ path, sizeBytes, contentType }) => ({ path, sizeBytes, contentType })), + }) as unknown as Promise; + } + + private validateUpload(input: CreateOrReplaceSiteInput, config: ResolvedSitesConfig): ValidatedSiteUpload { + try { + return validateSiteUpload({ + fileName: input.fileName, + content: input.content, + maxUploadBytes: config.upload.maxUploadBytes, + maxExtractedBytes: config.upload.maxExtractedBytes, + maxFiles: config.upload.maxFiles, + allowedExtensions: config.upload.allowedExtensions, + }); + } catch (error) { + if (error instanceof SiteUploadValidationError) { + throw new SitesServiceError(error.message, error.statusCode); + } + throw error; + } + } + + private async cleanupStoragePrefixes(config: ResolvedSitesConfig, storagePrefixes: string[]) { + await Promise.all( + storagePrefixes.map((storagePrefix) => + new SitesStorage(config).deletePrefix(storagePrefix).catch((error) => { + getLogger().warn({ error, storagePrefix }, 'Sites: failed to clean up uploaded prefix after error'); + }) + ) + ); + } + + private async withUploadRollback( + config: ResolvedSitesConfig, + operation: (uploadedStoragePrefixes: string[]) => Promise + ): Promise { + const uploadedStoragePrefixes: string[] = []; + try { + return await operation(uploadedStoragePrefixes); + } catch (error) { + await this.cleanupStoragePrefixes(config, uploadedStoragePrefixes); + throw error; + } + } + + async createSite(input: CreateOrReplaceSiteInput): Promise { + const config = await this.getConfig(); + this.assertEnabled(config); + const upload = this.validateUpload(input, config); + const siteId = createSiteId(); + const siteName = input.name?.trim() || this.defaultSiteName(siteId, config); + const expiresAt = this.expirationForNewSite(config); + + const site = await this.withUploadRollback(config, (uploadedStoragePrefixes) => + this.db.models.Site.transact(async (trx) => { + const created = (await this.db.models.Site.query(trx).insert({ + siteId, + name: siteName, + status: 'active', + activeVersionId: null, + fileCount: 0, + sizeBytes: 0, + expiresAt, + createdByUserId: input.user?.userId || null, + createdByDisplayName: input.user?.displayName || null, + updatedByUserId: input.user?.userId || null, + updatedByDisplayName: input.user?.displayName || null, + })) as Site; + + const version = await this.createVersion(siteId, upload, config, uploadedStoragePrefixes, trx); + return created.$query(trx).patchAndFetch({ + activeVersionId: version.versionId, + fileCount: upload.fileCount, + sizeBytes: upload.sizeBytes, + }) as unknown as Promise; + }) + ); + + return this.serialize(site, config); + } + + async listSites(): Promise { + const config = await this.getConfig(); + this.assertEnabled(config); + + const sites = (await this.db.models.Site.query() + .whereNull('deletedAt') + .orderBy('updatedAt', 'desc')) as unknown as Site[]; + return sites.map((site) => this.serialize(site, config)); + } + + async getSite(siteId: string): Promise { + const config = await this.getConfig(); + this.assertEnabled(config); + + const site = (await this.db.models.Site.query().findOne({ siteId }).whereNull('deletedAt')) as unknown as + | Site + | undefined; + if (!site) { + throw new SitesServiceError('Site not found.', 404); + } + + return this.serialize(site, config); + } + + private async getActiveSite(siteId: string): Promise<{ site: Site; config: ResolvedSitesConfig }> { + const config = await this.getConfig(); + this.assertEnabled(config); + + const site = (await this.db.models.Site.query().findOne({ siteId }).whereNull('deletedAt')) as unknown as + | Site + | undefined; + if (!site || site.status !== 'active' || !site.activeVersionId) { + throw new SitesServiceError('Site not found.', 404); + } + + if (config.ttl.enabled && site.expiresAt && new Date(site.expiresAt).getTime() <= Date.now()) { + throw new SitesServiceError('Site not found.', 404); + } + + return { site, config }; + } + + async replaceSiteContent(siteId: string, input: CreateOrReplaceSiteInput): Promise { + const { site, config } = await this.getActiveSite(siteId); + const upload = this.validateUpload(input, config); + + let previousVersions: SiteVersion[] = []; + const updated = await this.withUploadRollback(config, (uploadedStoragePrefixes) => + this.db.models.Site.transact(async (trx) => { + previousVersions = (await this.db.models.SiteVersion.query(trx) + .where({ siteId }) + .whereNull('deletedAt')) as unknown as SiteVersion[]; + const version = await this.createVersion(siteId, upload, config, uploadedStoragePrefixes, trx); + const patched = (await site.$query(trx).patchAndFetch({ + activeVersionId: version.versionId, + fileCount: upload.fileCount, + sizeBytes: upload.sizeBytes, + updatedByUserId: input.user?.userId || site.updatedByUserId || null, + updatedByDisplayName: input.user?.displayName || site.updatedByDisplayName || null, + })) as Site; + + if (previousVersions.length > 0) { + await this.db.models.SiteVersion.query(trx) + .whereIn( + 'versionId', + previousVersions.map((item) => item.versionId) + ) + .patch({ deletedAt: new Date().toISOString() }); + } + + return patched; + }) + ); + + await Promise.all( + previousVersions.map((item) => new SitesStorage(config).deletePrefix(item.storagePrefix).catch(() => undefined)) + ); + + return this.serialize(updated, config); + } + + async extendSite(siteId: string): Promise { + const { site, config } = await this.getActiveSite(siteId); + if (!config.ttl.enabled) { + throw new SitesServiceError('TTL is disabled for hosted sites.', 400); + } + + const base = site.expiresAt ? Math.max(new Date(site.expiresAt).getTime(), Date.now()) : Date.now(); + const expiresAt = new Date(base + config.ttl.extensionDays * DAY_MS).toISOString(); + const updated = (await site.$query().patchAndFetch({ expiresAt })) as Site; + return this.serialize(updated, config); + } + + async deleteSite(siteId: string): Promise { + const config = await this.getConfig(); + this.assertEnabled(config); + + const site = (await this.db.models.Site.query().findOne({ siteId }).whereNull('deletedAt')) as unknown as + | Site + | undefined; + if (!site) { + throw new SitesServiceError('Site not found.', 404); + } + + const versions = (await this.db.models.SiteVersion.query().where({ siteId })) as unknown as SiteVersion[]; + const deleted = (await this.db.models.Site.transact(async (trx) => { + const timestamp = new Date().toISOString(); + const patched = (await site.$query(trx).patchAndFetch({ + status: 'deleted', + deletedAt: timestamp, + })) as Site; + await this.db.models.SiteVersion.query(trx) + .where({ siteId }) + .whereNull('deletedAt') + .patch({ deletedAt: timestamp }); + return patched; + })) as Site; + + await Promise.all(versions.map((version) => new SitesStorage(config).deletePrefix(version.storagePrefix))); + return this.serialize(deleted, config); + } + + async getGatewayObject(hostHeader: string | undefined, pathname: string): Promise { + const config = await this.getConfig(); + this.assertEnabled(config); + + const siteId = parseSiteIdFromHost(hostHeader, config); + if (!siteId) { + throw new SitesServiceError('Site not found.', 404); + } + + const { site } = await this.getActiveSite(siteId); + const version = (await this.db.models.SiteVersion.query().findOne({ + siteId, + versionId: site.activeVersionId, + })) as unknown as SiteVersion | undefined; + + if (!version) { + throw new SitesServiceError('Site not found.', 404); + } + + const storage = new SitesStorage(config); + let requestedPath: string; + try { + requestedPath = + pathname === '/' || pathname === '' ? version.entrypoint || 'index.html' : normalizeGatewayPath(pathname); + } catch (error) { + if (error instanceof SiteUploadValidationError || error instanceof URIError) { + throw new SitesServiceError('Site not found.', 404); + } + throw error; + } + + try { + const object = await storage.getObject(version.storagePrefix, requestedPath); + return { + body: object.body, + contentType: object.contentType || getContentType(requestedPath), + contentLength: object.contentLength, + statusCode: 200, + }; + } catch (error) { + throw error instanceof SitesObjectNotFoundError ? new SitesServiceError('Site not found.', 404) : error; + } + } + + async matchesGatewayHost(hostHeader: string | undefined): Promise { + const config = await this.getConfig(); + if (!config.enabled) { + return false; + } + return Boolean(parseSiteIdFromHost(hostHeader, config)); + } + + async cleanupExpiredSites(): Promise<{ expired: number; cleaned: number; errors: number }> { + const config = await this.getConfig(); + if (!config.enabled || !config.ttl.enabled || !config.cleanup.enabled) { + return { expired: 0, cleaned: 0, errors: 0 }; + } + + const expiredSites = (await this.db.models.Site.query() + .whereNull('deletedAt') + .where('status', 'active') + .whereNotNull('expiresAt') + .where('expiresAt', '<=', new Date().toISOString()) + .limit(100)) as unknown as Site[]; + + let cleaned = 0; + let errors = 0; + + for (const site of expiredSites) { + try { + const versions = (await this.db.models.SiteVersion.query() + .where({ siteId: site.siteId }) + .whereNull('deletedAt')) as unknown as SiteVersion[]; + const timestamp = new Date().toISOString(); + await this.db.models.Site.transact(async (trx) => { + await site.$query(trx).patch({ status: 'expired', deletedAt: timestamp }); + await this.db.models.SiteVersion.query(trx) + .where({ siteId: site.siteId }) + .whereNull('deletedAt') + .patch({ deletedAt: timestamp }); + }); + await Promise.all(versions.map((version) => new SitesStorage(config).deletePrefix(version.storagePrefix))); + cleaned++; + } catch (error) { + errors++; + getLogger().error({ error, siteId: site.siteId }, 'Sites: cleanup failed'); + } + } + + return { expired: expiredSites.length, cleaned, errors }; + } + + processSitesCleanupQueue = async (_job: Job) => { + const result = await this.cleanupExpiredSites(); + getLogger().info( + `Sites: cleanup complete expired=${result.expired} cleaned=${result.cleaned} errors=${result.errors}` + ); + return result; + }; + + async setupSitesCleanupJob() { + const config = await this.getConfig(); + if (!config.enabled || !config.ttl.enabled || !config.cleanup.enabled) { + getLogger().debug('Sites: cleanup disabled'); + return; + } + + await this.sitesCleanupQueue.add( + 'sites-cleanup', + {}, + { + jobId: 'sites-cleanup', + repeat: { + every: config.cleanup.intervalMinutes * 60 * 1000, + }, + } + ); + } +} diff --git a/src/server/services/types/globalConfig.ts b/src/server/services/types/globalConfig.ts index cb18fbff..d640ae6c 100644 --- a/src/server/services/types/globalConfig.ts +++ b/src/server/services/types/globalConfig.ts @@ -22,6 +22,7 @@ export type GlobalConfig = { helmDefaults: HelmDefaults; buildDefaults?: BuildDefaults; agentSessionDefaults?: AgentSessionDefaults; + sites?: SitesConfig; postgresql: Helm; mysql: Helm; redis: Helm; @@ -135,6 +136,41 @@ export type AgentSessionDefaults = { controlPlane?: AgentSessionControlPlaneConfig; }; +export type SitesStorageBackend = 's3' | 'minio'; + +export type SitesStorageConfig = { + backend?: SitesStorageBackend; + bucket?: string; + prefix?: string; + region?: string; + endpoint?: string | null; + forcePathStyle?: boolean | null; +}; + +export type SitesConfig = { + enabled?: boolean; + domain?: string; + port?: number | string | null; + hostPrefix?: string | null; + ttl?: { + enabled?: boolean; + defaultDays?: number; + extensionDays?: number; + }; + upload?: { + maxUploadBytes?: number; + maxExtractedBytes?: number; + maxFiles?: number; + allowedExtensions?: string[]; + allowedTypes?: string[]; + }; + storage?: SitesStorageConfig; + cleanup?: { + enabled?: boolean; + intervalMinutes?: number; + }; +}; + export type RoleSettings = { role: string; name?: string; diff --git a/src/server/services/types/index.ts b/src/server/services/types/index.ts index 6699a87b..57c928ac 100644 --- a/src/server/services/types/index.ts +++ b/src/server/services/types/index.ts @@ -31,6 +31,7 @@ import GithubService from 'server/services/github'; import LabelService from 'server/services/label'; import TTLCleanupService from 'server/services/ttlCleanup'; import DeployCleanupService from 'server/services/deployCleanup'; +import SitesService from 'server/services/sites'; export interface IServices { BuildService: BuildService; @@ -50,6 +51,7 @@ export interface IServices { LabelService: LabelService; TTLCleanupService: TTLCleanupService; DeployCleanupService: DeployCleanupService; + SitesService: SitesService; } export * from 'server/services/types/github'; diff --git a/src/shared/config.ts b/src/shared/config.ts index b8f9eebc..74a814ff 100644 --- a/src/shared/config.ts +++ b/src/shared/config.ts @@ -106,6 +106,7 @@ export const QUEUE_NAMES = { CLEANUP: `cleanup_${JOB_VERSION}`, // NOTE: No version suffix - singleton queue shared across app instances TTL_CLEANUP: 'ttl_cleanup', + SITES_CLEANUP: 'sites_cleanup', GLOBAL_CONFIG_CACHE_REFRESH: 'global_config_cache_refresh', GITHUB_CLIENT_TOKEN_CACHE_REFRESH: 'github_client_token_cache_refresh', INGRESS_MANIFEST: `ingress_manifest_${JOB_VERSION}`, diff --git a/ws-server.ts b/ws-server.ts index 8265a181..50387d3e 100644 --- a/ws-server.ts +++ b/ws-server.ts @@ -33,7 +33,9 @@ import { parse, URL } from 'url'; import next from 'next'; import { WebSocketServer, WebSocket } from 'ws'; import { rootLogger } from './src/server/lib/logger'; +import { LIFECYCLE_MODE } from './src/shared/config'; import { streamK8sLogs, AbortHandle } from './src/server/lib/k8sStreamer'; +import SitesService from './src/server/services/sites'; import { buildWorkspaceEditorProxyHeaders, serializeSocketHttpResponse, @@ -52,6 +54,7 @@ const SESSION_WORKSPACE_EDITOR_PATH_PREFIX = '/api/agent-session/workspace-edito const SESSION_WORKSPACE_EDITOR_COOKIE_NAME = 'lfc_session_workspace_editor_auth'; const SESSION_WORKSPACE_EDITOR_PORT = parseInt(process.env.AGENT_SESSION_WORKSPACE_EDITOR_PORT || '13337', 10); const logger = rootLogger.child({ filename: __filename }); +let sitesGatewayService: SitesService | null = null; const HOP_BY_HOP_HEADERS = new Set([ 'connection', 'keep-alive', @@ -87,6 +90,14 @@ function parseCookieHeader(cookieHeader: string | string[] | undefined): Record< type SessionWorkspaceEditorPathMatch = { sessionId: string; forwardPath: string }; +function getSitesGatewayService(): SitesService { + if (!sitesGatewayService) { + sitesGatewayService = new SitesService(); + } + + return sitesGatewayService; +} + function parseSessionWorkspaceEditorPath(pathname: string | null | undefined): SessionWorkspaceEditorPathMatch | null { const safePathname = pathname || ''; if (safePathname.startsWith(SESSION_WORKSPACE_EDITOR_PATH_PREFIX)) { @@ -490,10 +501,63 @@ async function handleSessionWorkspaceEditorHttp( } } +async function handleSitesGatewayHttp(req: IncomingMessage, res: ServerResponse, pathname: string) { + if (LIFECYCLE_MODE !== 'gateway' && LIFECYCLE_MODE !== 'all') { + return false; + } + + const service = getSitesGatewayService(); + if (!(await service.matchesGatewayHost(req.headers.host))) { + return false; + } + + if (!req.method || !['GET', 'HEAD'].includes(req.method.toUpperCase())) { + res.statusCode = 404; + res.end('not found'); + return true; + } + + try { + const object = await service.getGatewayObject(req.headers.host, pathname); + + res.statusCode = object.statusCode; + res.setHeader('Content-Type', object.contentType); + res.setHeader('X-Content-Type-Options', 'nosniff'); + res.setHeader('Cache-Control', 'private, max-age=60'); + if (object.contentLength !== undefined) { + res.setHeader('Content-Length', object.contentLength.toString()); + } + + if (req.method.toUpperCase() === 'HEAD') { + res.end(); + return true; + } + + object.body.on('error', (error) => { + logger.error({ error, path: pathname }, 'SitesGateway: stream failed'); + if (!res.headersSent) { + res.statusCode = 502; + } + res.end(); + }); + object.body.pipe(res); + return true; + } catch (error: any) { + const statusCode = typeof error?.statusCode === 'number' ? error.statusCode : 500; + logger.warn({ error, path: pathname, statusCode }, 'SitesGateway: request failed'); + res.statusCode = statusCode === 404 ? 404 : 500; + res.end(statusCode === 404 ? 'not found' : 'internal server error'); + return true; + } +} + app.prepare().then(() => { const httpServer = createServer(async (req: IncomingMessage, res: ServerResponse) => { try { const parsedUrl = parse(req.url!, true); + if (parsedUrl.pathname && (await handleSitesGatewayHttp(req, res, parsedUrl.pathname))) { + return; + } if ( parsedUrl.pathname && (await handleSessionWorkspaceEditorHttp( From ed3e787911ea639d859d295380c829b737d233f4 Mon Sep 17 00:00:00 2001 From: vmelikyan Date: Wed, 13 May 2026 12:00:49 -0700 Subject: [PATCH 2/6] fixes - Capped ZIP decompression by actual inflated bytes to prevent upload OOM bypasses. - Made object cleanup retryable by only marking rows deleted after storage deletion succeeds. - Closed gateway object streams on HEAD requests. - Added OpenAPI coverage for /api/v2/sites* and regenerated UI API types. - Verified create, fetch, HEAD, replace, extend, and delete end to end in Tilt. --- .../api/v2/sites/[siteId]/content/route.ts | 41 +++++++++ src/app/api/v2/sites/[siteId]/extend/route.ts | 34 +++++++ src/app/api/v2/sites/[siteId]/route.ts | 52 +++++++++++ src/app/api/v2/sites/route.ts | 54 ++++++++++++ src/server/lib/sites/validation.test.ts | 15 +++- src/server/lib/sites/validation.ts | 27 +++++- src/server/services/sites.ts | 48 ++++++---- src/shared/openApiSpec.ts | 88 +++++++++++++++++++ ws-server.ts | 1 + 9 files changed, 338 insertions(+), 22 deletions(-) diff --git a/src/app/api/v2/sites/[siteId]/content/route.ts b/src/app/api/v2/sites/[siteId]/content/route.ts index 93c15560..b46c6ca7 100644 --- a/src/app/api/v2/sites/[siteId]/content/route.ts +++ b/src/app/api/v2/sites/[siteId]/content/route.ts @@ -29,6 +29,47 @@ type RouteContext = { }; }; +/** + * @openapi + * /api/v2/sites/{siteId}/content: + * put: + * summary: Replace hosted static site content + * description: Uploads a new static file or ZIP archive and makes it the active content for the site. + * tags: + * - Sites + * operationId: replaceSiteContent + * parameters: + * - in: path + * name: siteId + * required: true + * schema: + * type: string + * requestBody: + * required: true + * content: + * multipart/form-data: + * schema: + * $ref: '#/components/schemas/SiteUploadRequest' + * responses: + * '200': + * description: Hosted static site content replaced. + * content: + * application/json: + * schema: + * $ref: '#/components/schemas/SiteSuccessResponse' + * '400': + * description: Invalid upload. + * content: + * application/json: + * schema: + * $ref: '#/components/schemas/ApiErrorResponse' + * '404': + * description: Site not found or sites hosting is disabled. + * content: + * application/json: + * schema: + * $ref: '#/components/schemas/ApiErrorResponse' + */ const putHandler = async (req: NextRequest, { params }: RouteContext) => { try { const upload = await readUploadFile(req); diff --git a/src/app/api/v2/sites/[siteId]/extend/route.ts b/src/app/api/v2/sites/[siteId]/extend/route.ts index 3e1e355c..00b0f52e 100644 --- a/src/app/api/v2/sites/[siteId]/extend/route.ts +++ b/src/app/api/v2/sites/[siteId]/extend/route.ts @@ -26,6 +26,40 @@ type RouteContext = { }; }; +/** + * @openapi + * /api/v2/sites/{siteId}/extend: + * post: + * summary: Extend a hosted static site's expiration + * tags: + * - Sites + * operationId: extendSite + * parameters: + * - in: path + * name: siteId + * required: true + * schema: + * type: string + * responses: + * '200': + * description: Hosted static site expiration extended. + * content: + * application/json: + * schema: + * $ref: '#/components/schemas/SiteSuccessResponse' + * '400': + * description: TTL is disabled for hosted sites. + * content: + * application/json: + * schema: + * $ref: '#/components/schemas/ApiErrorResponse' + * '404': + * description: Site not found or sites hosting is disabled. + * content: + * application/json: + * schema: + * $ref: '#/components/schemas/ApiErrorResponse' + */ const postHandler = async (req: NextRequest, { params }: RouteContext) => { try { const service = new SitesService(); diff --git a/src/app/api/v2/sites/[siteId]/route.ts b/src/app/api/v2/sites/[siteId]/route.ts index 3b1255d9..1aa7d1cb 100644 --- a/src/app/api/v2/sites/[siteId]/route.ts +++ b/src/app/api/v2/sites/[siteId]/route.ts @@ -26,6 +26,58 @@ type RouteContext = { }; }; +/** + * @openapi + * /api/v2/sites/{siteId}: + * get: + * summary: Get a hosted static site + * tags: + * - Sites + * operationId: getSite + * parameters: + * - in: path + * name: siteId + * required: true + * schema: + * type: string + * responses: + * '200': + * description: Hosted static site. + * content: + * application/json: + * schema: + * $ref: '#/components/schemas/SiteSuccessResponse' + * '404': + * description: Site not found or sites hosting is disabled. + * content: + * application/json: + * schema: + * $ref: '#/components/schemas/ApiErrorResponse' + * delete: + * summary: Delete a hosted static site + * tags: + * - Sites + * operationId: deleteSite + * parameters: + * - in: path + * name: siteId + * required: true + * schema: + * type: string + * responses: + * '200': + * description: Hosted static site deleted. + * content: + * application/json: + * schema: + * $ref: '#/components/schemas/SiteSuccessResponse' + * '404': + * description: Site not found or sites hosting is disabled. + * content: + * application/json: + * schema: + * $ref: '#/components/schemas/ApiErrorResponse' + */ const getHandler = async (req: NextRequest, { params }: RouteContext) => { try { const service = new SitesService(); diff --git a/src/app/api/v2/sites/route.ts b/src/app/api/v2/sites/route.ts index c0b55e34..99f74c9f 100644 --- a/src/app/api/v2/sites/route.ts +++ b/src/app/api/v2/sites/route.ts @@ -23,6 +23,60 @@ import SitesService from 'server/services/sites'; export const runtime = 'nodejs'; +/** + * @openapi + * /api/v2/sites: + * get: + * summary: List hosted static sites + * description: Returns all non-deleted hosted static sites. + * tags: + * - Sites + * operationId: listSites + * responses: + * '200': + * description: Hosted static sites. + * content: + * application/json: + * schema: + * $ref: '#/components/schemas/SitesListSuccessResponse' + * '404': + * description: Sites hosting is disabled. + * content: + * application/json: + * schema: + * $ref: '#/components/schemas/ApiErrorResponse' + * post: + * summary: Create a hosted static site + * description: Uploads a static file or ZIP archive and publishes it as a hosted static site. + * tags: + * - Sites + * operationId: createSite + * requestBody: + * required: true + * content: + * multipart/form-data: + * schema: + * $ref: '#/components/schemas/SiteUploadRequest' + * responses: + * '201': + * description: Hosted static site created. + * content: + * application/json: + * schema: + * $ref: '#/components/schemas/SiteSuccessResponse' + * '400': + * description: Invalid upload. + * content: + * application/json: + * schema: + * $ref: '#/components/schemas/ApiErrorResponse' + * '404': + * description: Sites hosting is disabled. + * content: + * application/json: + * schema: + * $ref: '#/components/schemas/ApiErrorResponse' + */ const getHandler = async (req: NextRequest) => { try { const service = new SitesService(); diff --git a/src/server/lib/sites/validation.test.ts b/src/server/lib/sites/validation.test.ts index 61961409..daa70c1f 100644 --- a/src/server/lib/sites/validation.test.ts +++ b/src/server/lib/sites/validation.test.ts @@ -24,7 +24,7 @@ const DEFAULT_OPTIONS = { allowedExtensions: ['html', 'zip', 'json', 'md', 'markdown', 'txt', 'js'], }; -function zip(entries: Record): Buffer { +function zip(entries: Record, declaredSizeByPath: Record = {}): Buffer { const localParts: Buffer[] = []; const centralParts: Buffer[] = []; let offset = 0; @@ -57,7 +57,7 @@ function zip(entries: Record): Buffer { central.writeUInt32LE(0, 12); central.writeUInt32LE(0, 16); central.writeUInt32LE(compressed.length, 20); - central.writeUInt32LE(content.length, 24); + central.writeUInt32LE(declaredSizeByPath[entryPath] ?? content.length, 24); central.writeUInt16LE(name.length, 28); central.writeUInt16LE(0, 30); central.writeUInt16LE(0, 32); @@ -156,6 +156,17 @@ describe('validateSiteUpload', () => { ).toThrow('Extracted site size'); }); + it('caps actual inflated content even when zip metadata understates size', () => { + expect(() => + validateSiteUpload({ + ...DEFAULT_OPTIONS, + fileName: 'demo.zip', + maxExtractedBytes: 4, + content: zip({ 'index.html': 'too large' }, { 'index.html': 1 }), + }) + ).toThrow(/Extracted site size|Invalid zip/); + }); + it('rejects unsupported single-file and zip entry extensions', () => { expect(() => validateSiteUpload({ diff --git a/src/server/lib/sites/validation.ts b/src/server/lib/sites/validation.ts index d6d7590d..75832026 100644 --- a/src/server/lib/sites/validation.ts +++ b/src/server/lib/sites/validation.ts @@ -102,6 +102,19 @@ function isSymlink(externalAttributes: number): boolean { return (((externalAttributes >>> 16) & 0o170000) as number) === 0o120000; } +function inflateRawWithLimit(compressed: Buffer, maxOutputBytes: number, maxExtractedBytes: number): Buffer { + try { + return zlib.inflateRawSync(compressed, { maxOutputLength: maxOutputBytes } as zlib.ZlibOptions); + } catch (error) { + const message = error instanceof Error ? error.message : ''; + if (message.includes('maxOutputLength') || message.includes('Cannot create a Buffer larger')) { + reject(`Extracted site size must be ${maxExtractedBytes} bytes or less.`); + } + + reject('Invalid zip: compressed file data could not be extracted.'); + } +} + function parseZipEntries(buffer: Buffer, maxExtractedBytes: number, maxFiles: number): ZipEntry[] { const eocdOffset = findEndOfCentralDirectory(buffer); const entryCount = buffer.readUInt16LE(eocdOffset + 10); @@ -163,8 +176,7 @@ function parseZipEntries(buffer: Buffer, maxExtractedBytes: number, maxFiles: nu reject(`Zip upload cannot contain more than ${maxFiles} files.`); } - extractedSize += uncompressedSize; - if (extractedSize > maxExtractedBytes) { + if (extractedSize + uncompressedSize > maxExtractedBytes) { reject(`Extracted site size must be ${maxExtractedBytes} bytes or less.`); } @@ -183,12 +195,21 @@ function parseZipEntries(buffer: Buffer, maxExtractedBytes: number, maxFiles: nu const compressed = buffer.slice(dataStart, dataEnd); const content = - method === 0 ? Buffer.from(compressed) : method === 8 ? zlib.inflateRawSync(compressed) : undefined; + method === 0 + ? Buffer.from(compressed) + : method === 8 + ? inflateRawWithLimit(compressed, maxExtractedBytes - extractedSize, maxExtractedBytes) + : undefined; if (!content) { reject('Zip upload contains an unsupported compression method.'); } + extractedSize += content.length; + if (extractedSize > maxExtractedBytes) { + reject(`Extracted site size must be ${maxExtractedBytes} bytes or less.`); + } + if (content.length !== uncompressedSize) { reject('Invalid zip: extracted file size does not match metadata.'); } diff --git a/src/server/services/sites.ts b/src/server/services/sites.ts index 4e9b5e55..2e19d78c 100644 --- a/src/server/services/sites.ts +++ b/src/server/services/sites.ts @@ -188,6 +188,29 @@ export default class SitesService extends Service { } } + private async cleanupSupersededVersions(config: ResolvedSitesConfig, siteId: string, versions: SiteVersion[]) { + const deletedVersionIds: string[] = []; + + for (const version of versions) { + try { + await new SitesStorage(config).deletePrefix(version.storagePrefix); + deletedVersionIds.push(version.versionId); + } catch (error) { + getLogger().warn( + { error, siteId, versionId: version.versionId, storagePrefix: version.storagePrefix }, + 'Sites: version cleanup deferred' + ); + } + } + + if (deletedVersionIds.length > 0) { + await this.db.models.SiteVersion.query() + .where({ siteId }) + .whereIn('versionId', deletedVersionIds) + .patch({ deletedAt: new Date().toISOString() }); + } + } + async createSite(input: CreateOrReplaceSiteInput): Promise { const config = await this.getConfig(); this.assertEnabled(config); @@ -285,22 +308,11 @@ export default class SitesService extends Service { updatedByDisplayName: input.user?.displayName || site.updatedByDisplayName || null, })) as Site; - if (previousVersions.length > 0) { - await this.db.models.SiteVersion.query(trx) - .whereIn( - 'versionId', - previousVersions.map((item) => item.versionId) - ) - .patch({ deletedAt: new Date().toISOString() }); - } - return patched; }) ); - await Promise.all( - previousVersions.map((item) => new SitesStorage(config).deletePrefix(item.storagePrefix).catch(() => undefined)) - ); + await this.cleanupSupersededVersions(config, siteId, previousVersions); return this.serialize(updated, config); } @@ -329,6 +341,8 @@ export default class SitesService extends Service { } const versions = (await this.db.models.SiteVersion.query().where({ siteId })) as unknown as SiteVersion[]; + await Promise.all(versions.map((version) => new SitesStorage(config).deletePrefix(version.storagePrefix))); + const deleted = (await this.db.models.Site.transact(async (trx) => { const timestamp = new Date().toISOString(); const patched = (await site.$query(trx).patchAndFetch({ @@ -342,7 +356,6 @@ export default class SitesService extends Service { return patched; })) as Site; - await Promise.all(versions.map((version) => new SitesStorage(config).deletePrefix(version.storagePrefix))); return this.serialize(deleted, config); } @@ -416,9 +429,11 @@ export default class SitesService extends Service { for (const site of expiredSites) { try { - const versions = (await this.db.models.SiteVersion.query() - .where({ siteId: site.siteId }) - .whereNull('deletedAt')) as unknown as SiteVersion[]; + const versions = (await this.db.models.SiteVersion.query().where({ + siteId: site.siteId, + })) as unknown as SiteVersion[]; + await Promise.all(versions.map((version) => new SitesStorage(config).deletePrefix(version.storagePrefix))); + const timestamp = new Date().toISOString(); await this.db.models.Site.transact(async (trx) => { await site.$query(trx).patch({ status: 'expired', deletedAt: timestamp }); @@ -427,7 +442,6 @@ export default class SitesService extends Service { .whereNull('deletedAt') .patch({ deletedAt: timestamp }); }); - await Promise.all(versions.map((version) => new SitesStorage(config).deletePrefix(version.storagePrefix))); cleaned++; } catch (error) { errors++; diff --git a/src/shared/openApiSpec.ts b/src/shared/openApiSpec.ts index b11bcf07..7180bc5e 100644 --- a/src/shared/openApiSpec.ts +++ b/src/shared/openApiSpec.ts @@ -171,6 +171,94 @@ export const openApiSpecificationForV2Api: OAS3Options = { required: ['message'], }, + Site: { + type: 'object', + properties: { + id: { type: 'string', example: 'abc123def4' }, + name: { type: 'string', example: 'sample-site' }, + url: { type: 'string', format: 'uri', example: 'http://site-abc123def4.localhost:5002' }, + status: { type: 'string', enum: ['active', 'deleted', 'expired'] }, + createdAt: { type: 'string', format: 'date-time', nullable: true }, + updatedAt: { type: 'string', format: 'date-time', nullable: true }, + expiresAt: { type: 'string', format: 'date-time', nullable: true }, + fileCount: { type: 'integer', minimum: 0 }, + sizeBytes: { type: 'integer', format: 'int64', minimum: 0 }, + createdByDisplayName: { type: 'string', nullable: true }, + updatedByDisplayName: { type: 'string', nullable: true }, + }, + required: [ + 'id', + 'name', + 'url', + 'status', + 'createdAt', + 'updatedAt', + 'expiresAt', + 'fileCount', + 'sizeBytes', + 'createdByDisplayName', + 'updatedByDisplayName', + ], + }, + + SiteUploadRequest: { + type: 'object', + properties: { + file: { + type: 'string', + format: 'binary', + description: 'Static HTML file, supported asset file, or ZIP archive containing index.html.', + }, + name: { + type: 'string', + description: 'Optional display name for the hosted site.', + example: 'sample-site', + }, + }, + required: ['file'], + }, + + SiteSuccessResponse: { + allOf: [ + { $ref: '#/components/schemas/SuccessApiResponse' }, + { + type: 'object', + properties: { + data: { + type: 'object', + properties: { + site: { $ref: '#/components/schemas/Site' }, + }, + required: ['site'], + }, + }, + required: ['data'], + }, + ], + }, + + SitesListSuccessResponse: { + allOf: [ + { $ref: '#/components/schemas/SuccessApiResponse' }, + { + type: 'object', + properties: { + data: { + type: 'object', + properties: { + sites: { + type: 'array', + items: { $ref: '#/components/schemas/Site' }, + }, + }, + required: ['sites'], + }, + }, + required: ['data'], + }, + ], + }, + /** * @description Container for response metadata, including pagination. */ diff --git a/ws-server.ts b/ws-server.ts index 50387d3e..896ec692 100644 --- a/ws-server.ts +++ b/ws-server.ts @@ -529,6 +529,7 @@ async function handleSitesGatewayHttp(req: IncomingMessage, res: ServerResponse, } if (req.method.toUpperCase() === 'HEAD') { + (object.body as NodeJS.ReadableStream & { destroy?: () => void }).destroy?.(); res.end(); return true; } From d0af655d366b4ffe83b133ee050a51d1178be52d Mon Sep 17 00:00:00 2001 From: Vigneshraj Sekar Babu Date: Thu, 14 May 2026 10:25:46 -0700 Subject: [PATCH 3/6] fix: store site attribution as emails --- .../db/migrations/027_add_sites_hosting.ts | 6 ++---- src/server/models/Site.ts | 6 ++---- src/server/services/sites.ts | 17 +++++++---------- 3 files changed, 11 insertions(+), 18 deletions(-) diff --git a/src/server/db/migrations/027_add_sites_hosting.ts b/src/server/db/migrations/027_add_sites_hosting.ts index ba973373..f7180ed9 100644 --- a/src/server/db/migrations/027_add_sites_hosting.ts +++ b/src/server/db/migrations/027_add_sites_hosting.ts @@ -92,10 +92,8 @@ export async function up(knex: Knex): Promise { table.integer('fileCount').notNullable().defaultTo(0); table.bigInteger('sizeBytes').notNullable().defaultTo(0); table.timestamp('expiresAt').nullable(); - table.string('createdByUserId', 255).nullable(); - table.string('createdByDisplayName', 255).nullable(); - table.string('updatedByUserId', 255).nullable(); - table.string('updatedByDisplayName', 255).nullable(); + table.string('createdBy', 255).nullable(); + table.string('updatedBy', 255).nullable(); table.timestamp('createdAt').notNullable().defaultTo(knex.fn.now()); table.timestamp('updatedAt').notNullable().defaultTo(knex.fn.now()); table.timestamp('deletedAt').nullable(); diff --git a/src/server/models/Site.ts b/src/server/models/Site.ts index c32b2a31..34383ae0 100644 --- a/src/server/models/Site.ts +++ b/src/server/models/Site.ts @@ -26,10 +26,8 @@ export default class Site extends Model { fileCount!: number; sizeBytes!: number | string; expiresAt?: string | null; - createdByUserId?: string | null; - createdByDisplayName?: string | null; - updatedByUserId?: string | null; - updatedByDisplayName?: string | null; + createdBy?: string | null; + updatedBy?: string | null; static tableName = 'sites'; static timestamps = true; diff --git a/src/server/services/sites.ts b/src/server/services/sites.ts index 2e19d78c..db3cb6a6 100644 --- a/src/server/services/sites.ts +++ b/src/server/services/sites.ts @@ -63,8 +63,8 @@ export type SiteResponse = { expiresAt: string | null; fileCount: number; sizeBytes: number; - createdByDisplayName: string | null; - updatedByDisplayName: string | null; + createdBy: string | null; + updatedBy: string | null; }; export type GatewayObjectResponse = { @@ -106,8 +106,8 @@ export default class SitesService extends Service { expiresAt: site.expiresAt || null, fileCount: Number(site.fileCount || 0), sizeBytes: Number(site.sizeBytes || 0), - createdByDisplayName: site.createdByDisplayName || null, - updatedByDisplayName: site.updatedByDisplayName || null, + createdBy: site.createdBy || null, + updatedBy: site.updatedBy || null, }; } @@ -229,10 +229,8 @@ export default class SitesService extends Service { fileCount: 0, sizeBytes: 0, expiresAt, - createdByUserId: input.user?.userId || null, - createdByDisplayName: input.user?.displayName || null, - updatedByUserId: input.user?.userId || null, - updatedByDisplayName: input.user?.displayName || null, + createdBy: input.user?.email || null, + updatedBy: input.user?.email || null, })) as Site; const version = await this.createVersion(siteId, upload, config, uploadedStoragePrefixes, trx); @@ -304,8 +302,7 @@ export default class SitesService extends Service { activeVersionId: version.versionId, fileCount: upload.fileCount, sizeBytes: upload.sizeBytes, - updatedByUserId: input.user?.userId || site.updatedByUserId || null, - updatedByDisplayName: input.user?.displayName || site.updatedByDisplayName || null, + updatedBy: input.user?.email || site.updatedBy || null, })) as Site; return patched; From 5d07f703bf7a0d2133ac3079fb36ca58b0b0bcbd Mon Sep 17 00:00:00 2001 From: Vigneshraj Sekar Babu Date: Thu, 14 May 2026 10:52:31 -0700 Subject: [PATCH 4/6] docs: update sites api attribution fields --- src/shared/openApiSpec.ts | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/shared/openApiSpec.ts b/src/shared/openApiSpec.ts index 7180bc5e..4d2e9a37 100644 --- a/src/shared/openApiSpec.ts +++ b/src/shared/openApiSpec.ts @@ -183,8 +183,16 @@ export const openApiSpecificationForV2Api: OAS3Options = { expiresAt: { type: 'string', format: 'date-time', nullable: true }, fileCount: { type: 'integer', minimum: 0 }, sizeBytes: { type: 'integer', format: 'int64', minimum: 0 }, - createdByDisplayName: { type: 'string', nullable: true }, - updatedByDisplayName: { type: 'string', nullable: true }, + createdBy: { + type: 'string', + nullable: true, + description: 'Email address of the user who created the site.', + }, + updatedBy: { + type: 'string', + nullable: true, + description: 'Email address of the user who last updated the site.', + }, }, required: [ 'id', @@ -196,8 +204,8 @@ export const openApiSpecificationForV2Api: OAS3Options = { 'expiresAt', 'fileCount', 'sizeBytes', - 'createdByDisplayName', - 'updatedByDisplayName', + 'createdBy', + 'updatedBy', ], }, From a76f3bd7432133d0ba4f459e051f6c4b9cd9ed0c Mon Sep 17 00:00:00 2001 From: Vigneshraj Sekar Babu Date: Thu, 14 May 2026 12:01:56 -0700 Subject: [PATCH 5/6] feat(sites): paginate hosted sites list --- src/app/api/v2/sites/route.ts | 31 ++- src/server/lib/sites/routeHelpers.ts | 13 + src/server/services/__tests__/sites.test.ts | 253 ++++++++++++++++++++ src/server/services/sites.ts | 49 +++- src/shared/openApiSpec.test.ts | 40 ++++ 5 files changed, 378 insertions(+), 8 deletions(-) create mode 100644 src/server/services/__tests__/sites.test.ts diff --git a/src/app/api/v2/sites/route.ts b/src/app/api/v2/sites/route.ts index 99f74c9f..18a3128d 100644 --- a/src/app/api/v2/sites/route.ts +++ b/src/app/api/v2/sites/route.ts @@ -18,7 +18,7 @@ import { NextRequest } from 'next/server'; import { createApiHandler } from 'server/lib/createApiHandler'; import { getRequestUserIdentity } from 'server/lib/get-user'; import { successResponse } from 'server/lib/response'; -import { readUploadFile, sitesErrorResponse } from 'server/lib/sites/routeHelpers'; +import { readSitesListFilters, readUploadFile, sitesErrorResponse } from 'server/lib/sites/routeHelpers'; import SitesService from 'server/services/sites'; export const runtime = 'nodejs'; @@ -32,6 +32,31 @@ export const runtime = 'nodejs'; * tags: * - Sites * operationId: listSites + * parameters: + * - name: user + * in: query + * required: false + * description: Filters to sites created or last updated by the supplied user email. + * schema: + * type: string + * example: user@example.com + * - name: page + * in: query + * required: false + * description: Page number for pagination. + * schema: + * type: integer + * default: 1 + * minimum: 1 + * - name: limit + * in: query + * required: false + * description: Number of sites per page. + * schema: + * type: integer + * default: 25 + * minimum: 1 + * maximum: 100 * responses: * '200': * description: Hosted static sites. @@ -80,8 +105,8 @@ export const runtime = 'nodejs'; const getHandler = async (req: NextRequest) => { try { const service = new SitesService(); - const sites = await service.listSites(); - return successResponse({ sites }, { status: 200 }, req); + const result = await service.listSites(readSitesListFilters(req.nextUrl.searchParams)); + return successResponse({ sites: result.sites }, { status: 200, metadata: { pagination: result.pagination } }, req); } catch (error) { return sitesErrorResponse(error, req); } diff --git a/src/server/lib/sites/routeHelpers.ts b/src/server/lib/sites/routeHelpers.ts index 32bbeaaa..0bebddae 100644 --- a/src/server/lib/sites/routeHelpers.ts +++ b/src/server/lib/sites/routeHelpers.ts @@ -17,6 +17,7 @@ import { NextRequest, NextResponse } from 'next/server'; import { errorResponse } from 'server/lib/response'; import { SitesServiceError } from 'server/services/sites'; +import type { ListSitesFilters } from 'server/services/sites'; export async function readUploadFile(req: NextRequest): Promise<{ fileName: string; content: Buffer; name?: string }> { const formData = await req.formData(); @@ -35,6 +36,18 @@ export async function readUploadFile(req: NextRequest): Promise<{ fileName: stri }; } +export function readSitesListFilters(searchParams: URLSearchParams): ListSitesFilters { + const user = searchParams.get('user')?.trim(); + const page = Number.parseInt(searchParams.get('page') || '', 10); + const limit = Number.parseInt(searchParams.get('limit') || '', 10); + + return { + ...(user ? { user } : {}), + ...(Number.isNaN(page) ? {} : { page }), + ...(Number.isNaN(limit) ? {} : { limit }), + }; +} + export function sitesErrorResponse(error: unknown, req: NextRequest): NextResponse { return errorResponse(error, { status: error instanceof SitesServiceError ? error.statusCode : 500 }, req); } diff --git a/src/server/services/__tests__/sites.test.ts b/src/server/services/__tests__/sites.test.ts new file mode 100644 index 00000000..a101c70d --- /dev/null +++ b/src/server/services/__tests__/sites.test.ts @@ -0,0 +1,253 @@ +/** + * Copyright 2026 GoodRx, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import mockRedisClient from 'server/lib/__mocks__/redisClientMock'; + +mockRedisClient(); + +const mockGetAllConfigs = jest.fn(); + +jest.mock('shared/config', () => ({ + QUEUE_NAMES: { + SITES_CLEANUP: 'sites-cleanup', + }, + OBJECT_STORE_ACCESS_KEY: 'minio', + OBJECT_STORE_ENDPOINT: 'localhost', + OBJECT_STORE_PORT: '9000', + OBJECT_STORE_REGION: 'us-west-2', + OBJECT_STORE_SECRET_KEY: 'minio', + OBJECT_STORE_TYPE: 'minio', + OBJECT_STORE_USE_SSL: 'false', +})); + +jest.mock('server/lib/logger', () => ({ + getLogger: jest.fn(() => ({ + warn: jest.fn(), + })), +})); + +jest.mock('server/lib/dependencies', () => ({ + defaultDb: {}, + defaultRedis: {}, + defaultRedlock: {}, + defaultQueueManager: { + registerQueue: jest.fn(() => ({ add: jest.fn() })), + }, + redisClient: { + getConnection: jest.fn(() => ({ + duplicate: jest.fn(), + })), + }, +})); + +jest.mock('server/services/globalConfig', () => ({ + __esModule: true, + default: { + getInstance: jest.fn(() => ({ + getAllConfigs: (...args: any[]) => mockGetAllConfigs(...args), + })), + }, +})); + +import SitesService from 'server/services/sites'; + +type SiteRow = { + siteId: string; + name: string; + status: string; + fileCount: number; + sizeBytes: number; + createdAt: string; + updatedAt: string; + expiresAt: string | null; + createdBy: string | null; + updatedBy: string | null; + deletedAt: string | null; +}; + +class SiteQuery { + private filters: Array<(row: SiteRow) => boolean> = []; + private sortBy: { field: keyof SiteRow; direction: string } | null = null; + + constructor(private readonly rows: SiteRow[]) {} + + whereNull(field: keyof SiteRow) { + this.filters.push((row) => row[field] == null); + return this; + } + + whereRaw(sql: string, values: string[]) { + if (sql.includes('createdBy') && sql.includes('updatedBy')) { + const user = values[0]; + this.filters.push((row) => row.createdBy?.toLowerCase() === user || row.updatedBy?.toLowerCase() === user); + } + return this; + } + + orderBy(field: keyof SiteRow, direction: string) { + this.sortBy = { field, direction }; + return this; + } + + async page(pageIndex: number, pageSize: number) { + const rows = this.filteredRows(); + const start = pageIndex * pageSize; + return { + results: rows.slice(start, start + pageSize), + total: rows.length, + }; + } + + then(resolve: (value: SiteRow[]) => unknown, reject?: (reason: unknown) => unknown) { + return Promise.resolve(this.filteredRows()).then(resolve, reject); + } + + private filteredRows() { + const rows = this.rows.filter((row) => this.filters.every((filter) => filter(row))); + if (!this.sortBy) return rows; + + const sortBy = this.sortBy; + return [...rows].sort((a, b) => { + const compared = String(a[sortBy.field] || '').localeCompare(String(b[sortBy.field] || '')); + return sortBy.direction === 'desc' ? -compared : compared; + }); + } +} + +function createSiteRow(overrides: Partial = {}): SiteRow { + return { + siteId: 'site-1', + name: 'site', + status: 'active', + fileCount: 1, + sizeBytes: 128, + createdAt: '2026-05-01T00:00:00.000Z', + updatedAt: '2026-05-01T00:00:00.000Z', + expiresAt: null, + createdBy: null, + updatedBy: null, + deletedAt: null, + ...overrides, + }; +} + +describe('SitesService', () => { + let rows: SiteRow[]; + let service: SitesService; + + beforeEach(() => { + rows = []; + mockGetAllConfigs.mockResolvedValue({ + sites: { + enabled: true, + domain: 'sites.example.com', + hostPrefix: 'site', + }, + }); + + service = new SitesService( + { + models: { + Site: { + query: jest.fn(() => new SiteQuery(rows)), + }, + }, + } as any, + {} as any, + {} as any, + { registerQueue: jest.fn(() => ({ add: jest.fn() })) } as any + ); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + describe('listSites', () => { + it('returns all non-deleted sites without a user filter', async () => { + rows.push( + createSiteRow({ siteId: 'old', updatedAt: '2026-05-01T00:00:00.000Z' }), + createSiteRow({ + siteId: 'deleted', + updatedAt: '2026-05-03T00:00:00.000Z', + deletedAt: '2026-05-04T00:00:00.000Z', + }), + createSiteRow({ siteId: 'new', updatedAt: '2026-05-02T00:00:00.000Z' }) + ); + + await expect(service.listSites()).resolves.toMatchObject({ + sites: [{ id: 'new' }, { id: 'old' }], + pagination: { + current: 1, + total: 1, + items: 2, + limit: 25, + }, + }); + }); + + it('filters to sites created or last updated by the supplied user email', async () => { + rows.push( + createSiteRow({ + siteId: 'created-by-user', + createdBy: 'ALICE@example.com', + updatedBy: 'other@example.com', + updatedAt: '2026-05-01T00:00:00.000Z', + }), + createSiteRow({ + siteId: 'updated-by-user', + createdBy: 'other@example.com', + updatedBy: 'alice@example.com', + updatedAt: '2026-05-03T00:00:00.000Z', + }), + createSiteRow({ + siteId: 'not-touched-by-user', + createdBy: 'other@example.com', + updatedBy: 'other@example.com', + updatedAt: '2026-05-04T00:00:00.000Z', + }) + ); + + await expect(service.listSites({ user: ' Alice@Example.com ' })).resolves.toMatchObject({ + sites: [{ id: 'updated-by-user' }, { id: 'created-by-user' }], + pagination: { + current: 1, + total: 1, + items: 2, + limit: 25, + }, + }); + }); + + it('paginates sites after sorting and filtering', async () => { + rows.push( + createSiteRow({ siteId: 'oldest', updatedAt: '2026-05-01T00:00:00.000Z' }), + createSiteRow({ siteId: 'middle', updatedAt: '2026-05-02T00:00:00.000Z' }), + createSiteRow({ siteId: 'newest', updatedAt: '2026-05-03T00:00:00.000Z' }) + ); + + await expect(service.listSites({ page: 2, limit: 1 })).resolves.toMatchObject({ + sites: [{ id: 'middle' }], + pagination: { + current: 2, + total: 3, + items: 3, + limit: 1, + }, + }); + }); + }); +}); diff --git a/src/server/services/sites.ts b/src/server/services/sites.ts index db3cb6a6..b2411252 100644 --- a/src/server/services/sites.ts +++ b/src/server/services/sites.ts @@ -33,11 +33,15 @@ import { import { getContentType } from 'server/lib/sites/contentType'; import type Site from 'server/models/Site'; import type SiteVersion from 'server/models/SiteVersion'; +import type { PaginationMetadata } from 'server/lib/paginate'; import type { RequestUserIdentity } from 'server/lib/get-user'; const createSiteId = customAlphabet('abcdefghijklmnopqrstuvwxyz0123456789', 10); const createVersionId = customAlphabet('abcdefghijklmnopqrstuvwxyz0123456789', 12); const DAY_MS = 24 * 60 * 60 * 1000; +const DEFAULT_LIST_PAGE = 1; +const DEFAULT_LIST_LIMIT = 25; +const MAX_LIST_LIMIT = 100; type SitesErrorStatusCode = 400 | 401 | 403 | 404 | 409 | 500 | 502 | 503; export class SitesServiceError extends Error { @@ -53,6 +57,17 @@ export type CreateOrReplaceSiteInput = { user?: RequestUserIdentity | null; }; +export type ListSitesFilters = { + user?: string; + page?: number; + limit?: number; +}; + +export type ListSitesResult = { + sites: SiteResponse[]; + pagination: PaginationMetadata; +}; + export type SiteResponse = { id: string; name: string; @@ -188,6 +203,17 @@ export default class SitesService extends Service { } } + private normalizePagination(filters: ListSitesFilters): { page: number; limit: number } { + const page = + Number.isFinite(filters.page) && Number(filters.page) > 0 ? Math.floor(Number(filters.page)) : DEFAULT_LIST_PAGE; + const limit = + Number.isFinite(filters.limit) && Number(filters.limit) > 0 + ? Math.min(Math.floor(Number(filters.limit)), MAX_LIST_LIMIT) + : DEFAULT_LIST_LIMIT; + + return { page, limit }; + } + private async cleanupSupersededVersions(config: ResolvedSitesConfig, siteId: string, versions: SiteVersion[]) { const deletedVersionIds: string[] = []; @@ -245,14 +271,27 @@ export default class SitesService extends Service { return this.serialize(site, config); } - async listSites(): Promise { + async listSites(filters: ListSitesFilters = {}): Promise { const config = await this.getConfig(); this.assertEnabled(config); + const user = filters.user?.trim().toLowerCase(); + const { page, limit } = this.normalizePagination(filters); - const sites = (await this.db.models.Site.query() - .whereNull('deletedAt') - .orderBy('updatedAt', 'desc')) as unknown as Site[]; - return sites.map((site) => this.serialize(site, config)); + const query = this.db.models.Site.query().whereNull('deletedAt'); + if (user) { + query.whereRaw('(lower("createdBy") = ? or lower("updatedBy") = ?)', [user, user]); + } + + const result = await query.orderBy('updatedAt', 'desc').page(page - 1, limit); + return { + sites: (result.results as Site[]).map((site) => this.serialize(site, config)), + pagination: { + current: page, + total: Math.max(Math.ceil(result.total / limit), 1), + items: result.total, + limit, + }, + }; } async getSite(siteId: string): Promise { diff --git a/src/shared/openApiSpec.test.ts b/src/shared/openApiSpec.test.ts index 135eae91..85cbda05 100644 --- a/src/shared/openApiSpec.test.ts +++ b/src/shared/openApiSpec.test.ts @@ -16,6 +16,46 @@ function getOperation(path: string, method: string) { return swaggerSpec.paths[path]?.[method]; } +describe('OpenAPI v2 sites contract', () => { + it('documents sites list filters and pagination', () => { + expect(getOperation('/api/v2/sites', 'get')?.parameters).toEqual([ + { + name: 'user', + in: 'query', + required: false, + description: 'Filters to sites created or last updated by the supplied user email.', + schema: { + type: 'string', + }, + example: 'user@example.com', + }, + { + name: 'page', + in: 'query', + required: false, + description: 'Page number for pagination.', + schema: { + type: 'integer', + default: 1, + minimum: 1, + }, + }, + { + name: 'limit', + in: 'query', + required: false, + description: 'Number of sites per page.', + schema: { + type: 'integer', + default: 25, + minimum: 1, + maximum: 100, + }, + }, + ]); + }); +}); + describe('OpenAPI v2 agent session contract', () => { it('documents build metadata routes and link schemas', () => { expect(getOperation('/api/v2/builds/{uuid}/metadata', 'get')?.tags).toEqual(['Builds']); From de30d292b161b4e03770154fbc19e3b0edaeedd0 Mon Sep 17 00:00:00 2001 From: Vigneshraj Sekar Babu Date: Mon, 18 May 2026 14:37:58 -0700 Subject: [PATCH 6/6] fix(sites): wait for upload attempts before rollback --- src/server/lib/sites/storage.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/server/lib/sites/storage.ts b/src/server/lib/sites/storage.ts index f87a02a8..a193cd78 100644 --- a/src/server/lib/sites/storage.ts +++ b/src/server/lib/sites/storage.ts @@ -91,7 +91,7 @@ export class SitesStorage { async putFiles(storagePrefix: string, files: SiteUploadFile[]): Promise { await this.ensureBucket(); - await Promise.all( + const results = await Promise.allSettled( files.map((file) => this.client.send( new PutObjectCommand({ @@ -103,6 +103,11 @@ export class SitesStorage { ) ) ); + + const failedUpload = results.find((result): result is PromiseRejectedResult => result.status === 'rejected'); + if (failedUpload) { + throw failedUpload.reason; + } } async getObject(storagePrefix: string, filePath: string): Promise {