diff --git a/packages/liveblocks-server/CHANGELOG.md b/packages/liveblocks-server/CHANGELOG.md index 93ac605540..d25e5c1998 100644 --- a/packages/liveblocks-server/CHANGELOG.md +++ b/packages/liveblocks-server/CHANGELOG.md @@ -1,5 +1,9 @@ ## vNEXT (not yet released) +## v1.8.0 + +- Add full support for LiveFile + ## v1.7.0 - Add `--skip-install` (`-s`) flag to `liveblocks upgrade` to update diff --git a/packages/liveblocks-server/package.json b/packages/liveblocks-server/package.json index 97b62a35be..95535b9492 100644 --- a/packages/liveblocks-server/package.json +++ b/packages/liveblocks-server/package.json @@ -1,6 +1,6 @@ { "name": "@liveblocks/server", - "version": "1.7.0", + "version": "1.8.0", "description": "Liveblocks backend server foundation.", "type": "module", "main": "./dist/index.js", diff --git a/packages/liveblocks-server/src/index.ts b/packages/liveblocks-server/src/index.ts index 7c72663b7f..6cfd86ff14 100644 --- a/packages/liveblocks-server/src/index.ts +++ b/packages/liveblocks-server/src/index.ts @@ -29,6 +29,10 @@ export { snapshotToPlainLson_eager, snapshotToPlainLson_lazy, } from "~/formats/PlainLson"; +export { + hasUploadedLivefiles, + hasUploadedLivefilesInPlainLson, +} from "~/livefiles"; export { makeInMemorySnapshot } from "~/makeInMemorySnapshot"; export type { MetadataDB } from "~/MetadataDB"; export { makeMetadataDB } from "~/MetadataDB"; diff --git a/packages/liveblocks-server/src/interfaces/IBlobStore.ts b/packages/liveblocks-server/src/interfaces/IBlobStore.ts new file mode 100644 index 0000000000..7c4d757272 --- /dev/null +++ b/packages/liveblocks-server/src/interfaces/IBlobStore.ts @@ -0,0 +1,112 @@ +/** + * Copyright (c) Liveblocks Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** Bytes accepted by the store. Uploads stream; tests usually pass a buffer. */ +export type BlobBody = ReadableStream | Uint8Array; + +/** + * Everything the store knows about an object. + * + * There is deliberately no separate metadata record: a LiveFile's name and + * MIME type round-trip through `contentDisposition` and `contentType` on the + * object itself, exactly as they do in S3/R2, and its size is a property of the + * stored bytes. + */ +export type BlobMeta = { + size: number; + contentType: string; + contentDisposition: string; +}; + +/** Metadata supplied at write time. Size isn't known until the bytes land. */ +export type BlobMetaInput = Omit; + +export type UploadedPart = { + partNumber: number; + etag: string; +}; + +/** + * Blob storage for bytes that live outside the CRDT tree. + * + * Requirements for implementors: + * - Writes are immediately visible to head()/get(). + * - put() and completeMultipart() report the size of the bytes actually + * stored, never a caller-supplied figure. + */ +export interface IBlobStore { + /** + * Store `body` under `key`, replacing anything already there, and return the + * metadata of what was written — including the observed size. + */ + put(key: string, body: BlobBody, meta: BlobMetaInput): Promise; + + /** + * Return the metadata for `key`, or undefined if no such object exists. + * Callers rely on this to distinguish "uploaded" from "never uploaded", so + * implementations must not report partially-written objects. + */ + head(key: string): Promise; + + /** Stream the bytes stored at `key`, or undefined if no such object exists. */ + get(key: string): Promise | undefined>; + + /** Delete `key`. No-op if it doesn't exist. */ + delete(key: string): Promise; + + /** + * Begin a multipart upload for `key` and return its upload ID. The object at + * `key` is not touched until completeMultipart() succeeds. + */ + createMultipart(key: string, meta: BlobMetaInput): Promise; + + /** + * Store one part. Parts may arrive in any order and may be re-uploaded; the + * last write for a given part number wins. The returned ETag identifies the + * part's contents. + */ + uploadPart( + key: string, + uploadId: string, + partNumber: number, + body: BlobBody + ): Promise; + + /** + * Assemble the named parts, in ascending part-number order, into the object + * at `key`, and discard the upload. Parts not named here are dropped, so the + * resulting object may be smaller than everything uploaded. + */ + completeMultipart( + key: string, + uploadId: string, + parts: UploadedPart[] + ): Promise; + + /** Discard an in-progress multipart upload and its parts. */ + abortMultipart(key: string, uploadId: string): Promise; + + /** + * Return a URL that grants read access to `key` for roughly `ttlSeconds`, + * with no further credentials. The SDK hands this straight to the browser, + * which fetches it without an Authorization header, so whatever authority it + * carries has to be inside the URL. + * + * The value is opaque to callers: nothing may assume S3 URL shape. + */ + signedGetUrl(key: string, ttlSeconds: number): Promise; +} diff --git a/packages/liveblocks-server/src/interfaces/IStorageDriver.ts b/packages/liveblocks-server/src/interfaces/IStorageDriver.ts index c7ce20d72a..1346eb678a 100644 --- a/packages/liveblocks-server/src/interfaces/IStorageDriver.ts +++ b/packages/liveblocks-server/src/interfaces/IStorageDriver.ts @@ -267,6 +267,27 @@ export interface IStorageDriver { */ raw_iter_nodes(): Iterable<[string, SerializedCrdt]>; + // --------------------------------------------------------------------------- + // LiveFile upload receipt APIs + // --------------------------------------------------------------------------- + + /** + * Record that the bytes for `fileId` have been uploaded, and how many there + * are. + * + * A receipt is what makes a LiveFile referenceable: the Room layer refuses + * CREATE_FILE ops for files with no receipt, and implementations MUST use the + * recorded size in place of the client-supplied one whenever a FILE node is + * written. + */ + put_livefile_upload(fileId: string, size: number): void; + + /** + * Return the recorded size for `fileId`, or undefined if no upload has been + * recorded. + */ + get_livefile_upload_size(fileId: string): number | undefined; + // --------------------------------------------------------------------------- // Metadata APIs (key-value store, isolated from nodes) // --------------------------------------------------------------------------- diff --git a/packages/liveblocks-server/src/interfaces/index.ts b/packages/liveblocks-server/src/interfaces/index.ts index 46cce78e96..aa055a0e05 100644 --- a/packages/liveblocks-server/src/interfaces/index.ts +++ b/packages/liveblocks-server/src/interfaces/index.ts @@ -15,6 +15,13 @@ * along with this program. If not, see . */ +export type { + BlobBody, + BlobMeta, + BlobMetaInput, + IBlobStore, + UploadedPart, +} from "./IBlobStore"; export type { IServerWebSocket } from "./IServerWebSocket"; export type { IReadableSnapshot, diff --git a/packages/liveblocks-server/src/livefiles.ts b/packages/liveblocks-server/src/livefiles.ts new file mode 100644 index 0000000000..2f363530b4 --- /dev/null +++ b/packages/liveblocks-server/src/livefiles.ts @@ -0,0 +1,81 @@ +/** + * Copyright (c) Liveblocks Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import type { PlainLsonObject } from "@liveblocks/core"; +import { ClientMsgCode, CrdtType, OpCode } from "@liveblocks/core"; + +import { plainLsonToNodeStream } from "~/formats/PlainLson"; +import type { IStorageDriver } from "~/interfaces"; +import type { Op } from "~/protocol"; + +/** + * A LiveFile may only be referenced from Storage once its bytes have actually + * been uploaded — otherwise clients would sync a node pointing at nothing, and + * the server would have no way to know the file's real size. + * + * Both checks below are the same rule applied to the two shapes a new file can + * arrive in: ops on the wire, and a whole document being installed at once. + */ + +/** + * The part of a client message this check reads. Deliberately structural: + * callers hold different client-message unions (the Cloudflare worker's also + * covers feed messages), and all this needs is "does it carry storage ops". + */ +type MessageWithMaybeOps = { + readonly type: number; + readonly ops?: readonly Op[]; +}; + +/** True unless some CREATE_FILE op references a file that was never uploaded. */ +export function hasUploadedLivefiles( + driver: IStorageDriver, + messages: readonly MessageWithMaybeOps[] +): boolean { + for (const message of messages) { + if (message.type !== ClientMsgCode.UPDATE_STORAGE) { + continue; + } + + for (const op of message.ops ?? []) { + if (op.type !== OpCode.CREATE_FILE) { + continue; + } + + if (driver.get_livefile_upload_size(op.data.id) === undefined) { + return false; + } + } + } + return true; +} + +/** True unless the document contains a LiveFile that was never uploaded. */ +export function hasUploadedLivefilesInPlainLson( + driver: IStorageDriver, + document: PlainLsonObject +): boolean { + for (const [, node] of plainLsonToNodeStream(document)) { + if ( + node.type === CrdtType.FILE && + driver.get_livefile_upload_size(node.data.id) === undefined + ) { + return false; + } + } + return true; +} diff --git a/packages/liveblocks-server/src/plugins/InMemoryDriver.ts b/packages/liveblocks-server/src/plugins/InMemoryDriver.ts index 3a9e5c4508..a4a52dd794 100644 --- a/packages/liveblocks-server/src/plugins/InMemoryDriver.ts +++ b/packages/liveblocks-server/src/plugins/InMemoryDriver.ts @@ -162,6 +162,7 @@ export class InMemoryDriver implements IStorageDriver { private _leasedSessions: Map; private _feeds: Map; private _feedMessages: Map; // Key: `${feedId}:${messageId}` + private _livefileUploads: Map; // Key: fileId, value: size constructor(options?: { initialActor?: number; @@ -173,6 +174,7 @@ export class InMemoryDriver implements IStorageDriver { this._leasedSessions = new Map(); this._feeds = new Map(); this._feedMessages = new Map(); + this._livefileUploads = new Map(); this._nextActor = options?.initialActor ?? -1; @@ -197,10 +199,31 @@ export class InMemoryDriver implements IStorageDriver { this._nodes.clear(); for (const [id, node] of plainLsonToNodeStream(doc)) { - this._nodes.set(id, node); + this._nodes.set(id, this._withUploadedSize(node)); } } + put_livefile_upload(fileId: string, size: number): void { + this._livefileUploads.set(fileId, size); + } + + get_livefile_upload_size(fileId: string): number | undefined { + return this._livefileUploads.get(fileId); + } + + /** + * A FILE node's size comes from its upload receipt, never from whatever the + * client claimed. Nodes of any other type pass through untouched, as do + * files with no receipt — refusing those is the Room layer's job. + */ + private _withUploadedSize(node: N): N { + if (node.type !== CrdtType.FILE) return node; + const size = this._livefileUploads.get(node.data.id); + return size === undefined + ? node + : { ...node, data: { ...node.data, size } }; + } + get_meta(key: string) { return this._metadb.get(key); } @@ -553,7 +576,7 @@ export class InMemoryDriver implements IStorageDriver { } set_child(id: string, node: SerializedChild, allowOverwrite?: boolean): void { - this.loadedApi.set_child(id, node, allowOverwrite); + this.loadedApi.set_child(id, this._withUploadedSize(node), allowOverwrite); } move_sibling(id: string, newPos: Pos): void { diff --git a/packages/liveblocks-server/test/plugins/_generateFullTestSuite.ts b/packages/liveblocks-server/test/plugins/_generateFullTestSuite.ts index 3d1f55bd18..e5b9bd6513 100644 --- a/packages/liveblocks-server/test/plugins/_generateFullTestSuite.ts +++ b/packages/liveblocks-server/test/plugins/_generateFullTestSuite.ts @@ -2579,6 +2579,138 @@ export function generateFullTestSuite(config: { })); }); + describe("livefile upload receipt API impl", () => { + const FILE_ID = "fl_iN9WvpTnFO4qXbLXpZ2Kr"; + const OTHER_FILE_ID = "fl_Bq7zMk1RsW0dYvLc3TnAe"; + + function fileData(size: number, id = FILE_ID): LiveFileData { + return { id, name: "hello.txt", size, mimeType: "text/plain" }; + } + + /** All FILE node sizes currently in storage, in iteration order. */ + function fileSizes(driver: TDriver): number[] { + const sizes = []; + for (const [, node] of driver.iter_nodes()) { + if (node.type === CrdtType.FILE) { + sizes.push(node.data.size); + } + } + return sizes; + } + + test("get_livefile_upload_size is undefined for an unrecorded file", () => + runTest((driver) => { + expect(driver.get_livefile_upload_size(FILE_ID)).toEqual(undefined); + })); + + test("put_livefile_upload records a size that reads back", () => + runTest((driver) => { + driver.put_livefile_upload(FILE_ID, 11); + expect(driver.get_livefile_upload_size(FILE_ID)).toEqual(11); + + // Receipts are per-file, not global + expect(driver.get_livefile_upload_size(OTHER_FILE_ID)).toEqual( + undefined + ); + })); + + test("put_livefile_upload overwrites an existing receipt", () => + runTest((driver) => { + driver.put_livefile_upload(FILE_ID, 11); + driver.put_livefile_upload(FILE_ID, 22); + expect(driver.get_livefile_upload_size(FILE_ID)).toEqual(22); + })); + + test("a zero-byte upload is recorded, and is not the same as undefined", () => + runTest((driver) => { + driver.put_livefile_upload(FILE_ID, 0); + expect(driver.get_livefile_upload_size(FILE_ID)).toEqual(0); + })); + + test("set_child replaces a client-claimed size with the recorded one", () => + runTest((driver) => { + driver.put_livefile_upload(FILE_ID, 11); + + // The client claims this file is 1 byte. It is not. + driver.set_child("1:0", { + type: CrdtType.FILE, + parentId: "root", + parentKey: "file", + data: fileData(1), + }); + + expect(driver.get_node("1:0")).toEqual({ + type: CrdtType.FILE, + parentId: "root", + parentKey: "file", + data: fileData(11), + }); + })); + + test("set_child replaces a client-claimed size even when the truth is 0", () => + runTest((driver) => { + driver.put_livefile_upload(FILE_ID, 0); + + driver.set_child("1:0", { + type: CrdtType.FILE, + parentId: "root", + parentKey: "file", + data: fileData(999), + }); + + expect(fileSizes(driver)).toEqual([0]); + })); + + test("set_child leaves the claimed size alone when there is no receipt", () => + runTest((driver) => { + // Refusing unreferenced files is the Room layer's job, not the + // driver's. With no receipt the driver has nothing better to say. + driver.set_child("1:0", { + type: CrdtType.FILE, + parentId: "root", + parentKey: "file", + data: fileData(1), + }); + + expect(fileSizes(driver)).toEqual([1]); + })); + + test("DANGEROUSLY_reset_nodes replaces claimed sizes with recorded ones", () => + runTest((driver) => { + driver.put_livefile_upload(FILE_ID, 11); + + driver.DANGEROUSLY_reset_nodes({ + liveblocksType: "LiveObject", + data: { + file: { liveblocksType: "LiveFile", data: fileData(1) }, + }, + }); + + expect(fileSizes(driver)).toEqual([11]); + })); + + test("DANGEROUSLY_reset_nodes leaves unrecorded files alone", () => + runTest((driver) => { + driver.DANGEROUSLY_reset_nodes({ + liveblocksType: "LiveObject", + data: { + file: { liveblocksType: "LiveFile", data: fileData(1) }, + }, + }); + + expect(fileSizes(driver)).toEqual([1]); + })); + + test("receipts survive DANGEROUSLY_reset_nodes", () => + runTest((driver) => { + // Resetting the document wipes nodes, but upload history is not part + // of the document — a file that was uploaded stays uploaded. + driver.put_livefile_upload(FILE_ID, 11); + driver.DANGEROUSLY_reset_nodes(EMPTY_DOC); + expect(driver.get_livefile_upload_size(FILE_ID)).toEqual(11); + })); + }); + describe("meta API impl", () => { test("get_meta with empty store is undefined", () => runTest((driver) => diff --git a/tools/liveblocks-cli/package.json b/tools/liveblocks-cli/package.json index 25236f8f3f..9f107f05d2 100644 --- a/tools/liveblocks-cli/package.json +++ b/tools/liveblocks-cli/package.json @@ -1,6 +1,6 @@ { "name": "liveblocks", - "version": "1.7.0", + "version": "1.8.0", "description": "Liveblocks command line interface", "type": "module", "bin": { @@ -52,6 +52,7 @@ "@liveblocks/zenrouter": "^1.2.0", "decoders": "^2.9.0", "js-base64": "^3.7.5", + "mime": "^4.0.4", "yjs": "^13.6.10" } } diff --git a/tools/liveblocks-cli/src/dev-server/blobs/FsBlobStore.ts b/tools/liveblocks-cli/src/dev-server/blobs/FsBlobStore.ts new file mode 100644 index 0000000000..81ce1acc63 --- /dev/null +++ b/tools/liveblocks-cli/src/dev-server/blobs/FsBlobStore.ts @@ -0,0 +1,320 @@ +/** + * Copyright (c) Liveblocks Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import { createHmac, randomUUID, timingSafeEqual } from "node:crypto"; +import { mkdirSync, renameSync, rmSync } from "node:fs"; +import { join } from "node:path"; + +import type { + BlobBody, + BlobMeta, + BlobMetaInput, + IBlobStore, + UploadedPart, +} from "@liveblocks/server"; + +/** Path segment under the store root holding completed objects. */ +const BLOBS_DIR = "blobs"; + +/** Path segment under the store root holding in-progress multipart uploads. */ +const PARTIAL_DIR = "partial"; + +/** Suffix of the sidecar file carrying an object's content type/disposition. */ +const META_SUFFIX = ".meta"; + +type StoredMeta = BlobMetaInput; + +type MultipartMeta = BlobMetaInput & { key: string }; + +export type FsBlobStoreOptions = { + /** Directory to keep blobs in. Created if it doesn't exist. */ + root: string; + /** Origin that signed URLs should point at, e.g. "http://localhost:1153". */ + baseUrl: string; + /** Secret used to sign download URLs. */ + secret: string; +}; + +/** + * A filesystem-backed IBlobStore for the dev server. + * + * Two things a real object store gives away for free have to be built by hand + * here. Object metadata (content type and disposition, which is where a + * LiveFile's MIME type and name live) has no home on a filesystem, so it goes + * in a sidecar next to the bytes. And there is no notion of a credentialed URL, + * so signedGetUrl() mints an HMAC-signed link that the dev server's own + * download route validates — see verifySignedGetUrl(), which is the half of + * that exchange an S3-backed implementation would have no use for. + */ +export class FsBlobStore implements IBlobStore { + readonly #root: string; + readonly #baseUrl: string; + readonly #secret: string; + + constructor(options: FsBlobStoreOptions) { + this.#root = options.root; + this.#baseUrl = options.baseUrl.replace(/\/$/, ""); + this.#secret = options.secret; + + mkdirSync(join(this.#root, BLOBS_DIR), { recursive: true }); + mkdirSync(join(this.#root, PARTIAL_DIR), { recursive: true }); + } + + /** + * Keys are opaque and routinely contain "/" (they look like + * "/"), but the interface says to treat them as flat + * names, so each one becomes a single percent-encoded filename. + */ + #objectPath(key: string): string { + return join(this.#root, BLOBS_DIR, asFilename(key)); + } + + #uploadDir(uploadId: string): string { + return join(this.#root, PARTIAL_DIR, asFilename(uploadId)); + } + + async #readMeta(path: string): Promise { + const file = Bun.file(path + META_SUFFIX); + if (!(await file.exists())) return undefined; + return (await file.json()) as StoredMeta; + } + + async put( + key: string, + body: BlobBody, + meta: BlobMetaInput + ): Promise { + const path = this.#objectPath(key); + + // Write the bytes to one side first, then publish by rename, so that a + // half-written upload is never visible to head(). + const tmpPath = `${path}.${randomUUID()}.partial`; + const size = await Bun.write(tmpPath, await toBytes(body)); + await Bun.write(path + META_SUFFIX, JSON.stringify(meta)); + renameSync(tmpPath, path); + + return { ...meta, size }; + } + + async head(key: string): Promise { + const path = this.#objectPath(key); + const file = Bun.file(path); + if (!(await file.exists())) return undefined; + + const meta = await this.#readMeta(path); + return { + size: file.size, + contentType: meta?.contentType ?? "", + contentDisposition: meta?.contentDisposition ?? "", + }; + } + + async get(key: string): Promise | undefined> { + const file = Bun.file(this.#objectPath(key)); + if (!(await file.exists())) return undefined; + return file.stream(); + } + + async delete(key: string): Promise { + const path = this.#objectPath(key); + await Bun.file(path).unlink().catch(NOOP); + await Bun.file(path + META_SUFFIX) + .unlink() + .catch(NOOP); + } + + async createMultipart(key: string, meta: BlobMetaInput): Promise { + const uploadId = randomUUID(); + mkdirSync(this.#uploadDir(uploadId), { recursive: true }); + + const multipartMeta: MultipartMeta = { ...meta, key }; + await Bun.write( + join(this.#uploadDir(uploadId), "meta"), + JSON.stringify(multipartMeta) + ); + return uploadId; + } + + async uploadPart( + key: string, + uploadId: string, + partNumber: number, + body: BlobBody + ): Promise { + await this.#requireUpload(key, uploadId); + + const bytes = await toBytes(body); + await Bun.write(this.#partPath(uploadId, partNumber), bytes); + return { partNumber, etag: etagOf(bytes) }; + } + + async completeMultipart( + key: string, + uploadId: string, + parts: UploadedPart[] + ): Promise { + const meta = await this.#requireUpload(key, uploadId); + + // Only the named parts are assembled, and always in part-number order, no + // matter what order the caller listed or uploaded them in. + const ordered = [...parts].sort((a, b) => a.partNumber - b.partNumber); + + const chunks: Uint8Array[] = []; + for (const part of ordered) { + const file = Bun.file(this.#partPath(uploadId, part.partNumber)); + if (!(await file.exists())) { + throw new Error(`No such part ${part.partNumber} in upload ${uploadId}`); // prettier-ignore + } + + const bytes = new Uint8Array(await file.arrayBuffer()); + if (etagOf(bytes) !== part.etag) { + throw new Error(`ETag mismatch for part ${part.partNumber}`); + } + chunks.push(bytes); + } + + const result = await this.put(key, concat(chunks), { + contentType: meta.contentType, + contentDisposition: meta.contentDisposition, + }); + + await this.abortMultipart(key, uploadId); + return result; + } + + async abortMultipart(key: string, uploadId: string): Promise { + // Aborting an upload that isn't there is fine: the SDK aborts on failure + // paths where it may never have been created. Aborting one belonging to a + // different object is not — that would throw away someone else's parts. + const meta = await this.#readUpload(uploadId); + if (meta === undefined) { + return; + } + if (meta.key !== key) { + throw new Error(`Upload ${uploadId} does not belong to ${key}`); + } + + rmSync(this.#uploadDir(uploadId), { recursive: true, force: true }); + } + + async signedGetUrl(key: string, ttlSeconds: number): Promise { + const expiresAt = Date.now() + ttlSeconds * 1000; + const url = new URL(`${this.#baseUrl}${DOWNLOAD_PATH}`); + url.searchParams.set("key", key); + url.searchParams.set("exp", String(expiresAt)); + url.searchParams.set("sig", this.#sign(key, expiresAt)); + return Promise.resolve(url.toString()); + } + + /** + * Validate a download URL's query params and return the key it grants access + * to, or undefined if the signature doesn't match or the link has expired. + * + * Not part of IBlobStore: an object store that mints real presigned URLs + * verifies them itself, and would never be asked this. + */ + verifySignedGetUrl(params: URLSearchParams): string | undefined { + const key = params.get("key"); + const exp = Number(params.get("exp")); + const sig = params.get("sig"); + if (key === null || sig === null || !Number.isFinite(exp)) return undefined; + if (Date.now() > exp) return undefined; + + const expected = Buffer.from(this.#sign(key, exp)); + const actual = Buffer.from(sig); + if (expected.length !== actual.length) return undefined; + return timingSafeEqual(expected, actual) ? key : undefined; + } + + #sign(key: string, expiresAt: number): string { + return createHmac("sha256", this.#secret) + .update(`${key}\n${expiresAt}`) + .digest("hex"); + } + + #partPath(uploadId: string, partNumber: number): string { + if (!Number.isSafeInteger(partNumber) || partNumber < 1) { + throw new Error(`Invalid part number: ${String(partNumber)}`); + } + return join(this.#uploadDir(uploadId), `part-${partNumber}`); + } + + /** Load an upload's metadata, or undefined if there is no such upload. */ + async #readUpload(uploadId: string): Promise { + const file = Bun.file(join(this.#uploadDir(uploadId), "meta")); + if (!(await file.exists())) { + return undefined; + } + return (await file.json()) as MultipartMeta; + } + + /** Load an upload's metadata, throwing if it isn't an upload for `key`. */ + async #requireUpload(key: string, uploadId: string): Promise { + const meta = await this.#readUpload(uploadId); + if (meta === undefined) { + throw new Error(`No such multipart upload ${uploadId}`); + } + if (meta.key !== key) { + throw new Error(`Upload ${uploadId} does not belong to ${key}`); + } + return meta; + } +} + +/** Path the dev server serves signed downloads from. */ +export const DOWNLOAD_PATH = "/blob"; + +const NOOP = () => {}; + +/** + * Turn an opaque identifier into exactly one filename. + * + * Percent-encoding gets almost all of the way there — "/" and "\" both come + * out escaped, so no input can name a nested path. What it does NOT escape is + * ".", so `encodeURIComponent("..")` is still ".." and joining that onto a + * directory silently walks up out of it. The three inputs that survive + * encoding as path syntax are refused here, which keeps the promise that one + * identifier is one file inside the store. + */ +function asFilename(value: string): string { + const encoded = encodeURIComponent(value); + if (encoded === "" || encoded === "." || encoded === "..") { + throw new Error(`Unsafe blob store name: ${JSON.stringify(value)}`); + } + return encoded; +} + +async function toBytes(body: BlobBody): Promise { + if (body instanceof Uint8Array) return body; + return new Uint8Array(await new Response(body).arrayBuffer()); +} + +function concat(chunks: Uint8Array[]): Uint8Array { + const total = chunks.reduce((sum, chunk) => sum + chunk.byteLength, 0); + const result = new Uint8Array(total); + let offset = 0; + for (const chunk of chunks) { + result.set(chunk, offset); + offset += chunk.byteLength; + } + return result; +} + +/** Any stable content hash works; S3 uses MD5, we don't need to match it. */ +function etagOf(bytes: Uint8Array): string { + return new Bun.CryptoHasher("sha256").update(bytes).digest("hex"); +} diff --git a/tools/liveblocks-cli/src/dev-server/blobs/store.ts b/tools/liveblocks-cli/src/dev-server/blobs/store.ts new file mode 100644 index 0000000000..9a4eaddcfb --- /dev/null +++ b/tools/liveblocks-cli/src/dev-server/blobs/store.ts @@ -0,0 +1,77 @@ +/** + * Copyright (c) Liveblocks Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import { randomUUID } from "node:crypto"; +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; + +import { FsBlobStore } from "./FsBlobStore"; + +const DEFAULT_BASE_URL = "http://localhost:1153"; + +let _store: FsBlobStore | null = null; +let _baseUrl = DEFAULT_BASE_URL; +let _root = ".liveblocks/v3/files"; + +/** + * Point the store at a directory. Called by the rooms DB as it initializes, so + * that blobs follow the same base path as everything else — including into a + * temp dir in ephemeral mode. + */ +export function setBlobsRoot(root: string): void { + if (root !== _root) { + _root = root; + _store = null; + } +} + +/** + * Point signed download URLs at the origin the server actually bound to. The + * dev server calls this once it knows its port; tests can leave the default. + */ +export function setBlobStoreBaseUrl(baseUrl: string): void { + if (baseUrl !== _baseUrl) { + _baseUrl = baseUrl; + _store = null; + } +} + +export function getBlobStore(): FsBlobStore { + return (_store ??= new FsBlobStore({ + root: _root, + baseUrl: _baseUrl, + secret: readOrCreateSecret(_root), + })); +} + +/** + * Keep the URL-signing secret alongside the blobs, so that links handed out + * before a restart still work afterwards. Dev-only: a local file, protected by + * nothing but its permissions. + */ +function readOrCreateSecret(root: string): string { + mkdirSync(root, { recursive: true }); + + const path = join(root, ".signing-secret"); + if (existsSync(path)) { + return readFileSync(path, "utf8"); + } + + const secret = randomUUID(); + writeFileSync(path, secret, { mode: 0o600 }); + return secret; +} diff --git a/tools/liveblocks-cli/src/dev-server/db/BunSQLiteDriver.ts b/tools/liveblocks-cli/src/dev-server/db/BunSQLiteDriver.ts index 77909628b9..5564012898 100644 --- a/tools/liveblocks-cli/src/dev-server/db/BunSQLiteDriver.ts +++ b/tools/liveblocks-cli/src/dev-server/db/BunSQLiteDriver.ts @@ -601,6 +601,16 @@ export class BunSQLiteDriver implements IStorageDriver { )` ); + // Create a table recording which LiveFile blobs have been uploaded, and + // how large they are. This is the only trustworthy source for a FILE + // node's size — see put_livefile_upload(). + db.run( + `CREATE TABLE IF NOT EXISTS livefile_uploads ( + file_id TEXT NOT NULL PRIMARY KEY, + size INTEGER NOT NULL CHECK (size >= 0) + ) STRICT` + ); + // Create a table to read/write BINARY values db.run( `CREATE TABLE IF NOT EXISTS ydocs ( @@ -710,7 +720,37 @@ export class BunSQLiteDriver implements IStorageDriver { } set_child(id: string, node: SerializedChild, allowOverwrite?: boolean): void { - this.loadedApi.set_child(id, node, allowOverwrite); + this.loadedApi.set_child(id, this._withUploadedSize(node), allowOverwrite); + } + + put_livefile_upload(fileId: string, size: number): void { + this.db.run( + `INSERT INTO livefile_uploads (file_id, size) VALUES (?, ?) + ON CONFLICT (file_id) DO UPDATE SET size = excluded.size`, + [fileId, size] + ); + } + + get_livefile_upload_size(fileId: string): number | undefined { + return this.db + .query< + { size: number }, + [string] + >("SELECT size FROM livefile_uploads WHERE file_id = ?") + .get(fileId)?.size; + } + + /** + * A FILE node's size comes from its upload receipt, never from whatever the + * client claimed. Nodes of any other type pass through untouched, as do + * files with no receipt — refusing those is the Room layer's job. + */ + private _withUploadedSize(node: N): N { + if (node.type !== CrdtType.FILE) return node; + const size = this.get_livefile_upload_size(node.data.id); + return size === undefined + ? node + : { ...node, data: { ...node.data, size } }; } move_sibling(id: string, newPos: Pos): void { @@ -793,7 +833,8 @@ export class BunSQLiteDriver implements IStorageDriver { // doesn't transiently violate the self-referencing parent_id FK. this.db.run("PRAGMA defer_foreign_keys = ON"); this.db.run("DELETE FROM nodes"); - for (const [id, node] of plainLsonToNodeStream(doc)) { + for (const [id, rawNode] of plainLsonToNodeStream(doc)) { + const node = this._withUploadedSize(rawNode); const parentId = id === "root" ? null : (node.parentId ?? null); const parentKey = id === "root" ? null : (node.parentKey ?? null); const jdata = diff --git a/tools/liveblocks-cli/src/dev-server/db/rooms.ts b/tools/liveblocks-cli/src/dev-server/db/rooms.ts index 62645537cd..1f617d06e0 100644 --- a/tools/liveblocks-cli/src/dev-server/db/rooms.ts +++ b/tools/liveblocks-cli/src/dev-server/db/rooms.ts @@ -18,12 +18,13 @@ import type { JsonObject } from "@liveblocks/core"; import { nanoid, Permission, WebsocketCloseCodes } from "@liveblocks/core"; import type { Millis } from "@liveblocks/server"; -import { DefaultMap, Room } from "@liveblocks/server"; +import { DefaultMap, hasUploadedLivefiles, Room } from "@liveblocks/server"; import { Database } from "bun:sqlite"; import { mkdirSync, mkdtempSync, rmSync } from "fs"; import { tmpdir } from "os"; import { dirname, join, resolve } from "path"; +import { setBlobsRoot } from "../blobs/store"; import { BunSQLiteDriver } from "./BunSQLiteDriver"; // --------------------------------------------------------------------------- @@ -65,9 +66,16 @@ function roomsDir(): string { return join(basePath, "rooms"); } +/** Where LiveFile blobs live, alongside the per-room storage files. */ +function blobsDir(): string { + return join(basePath, "files"); +} + function ensureInit(): void { if (_initializedDb) return; + setBlobsRoot(blobsDir()); + const dbPath = join(basePath, "db.sql"); mkdirSync(dirname(dbPath), { recursive: true }); @@ -331,6 +339,15 @@ const instances = new DefaultMap< const storage = new BunSQLiteDriver(getStoragePath(record.internalId)); const room = new Room(roomId, { storage, + hooks: { + // A client may not reference a LiveFile it hasn't uploaded yet. Without + // this the room would happily sync a node pointing at nothing, and its + // size would be whatever the client felt like claiming. + isClientMsgAllowed: (msg) => + hasUploadedLivefiles(storage, [msg]) + ? { allowed: true } + : { allowed: false, reason: "Storage file has not been uploaded" }, + }, }); return room; }); @@ -352,6 +369,7 @@ export function useEphemeralStorage(): string { const root = mkdtempSync(join(tmpdir(), "liveblocks-dev-")); basePath = join(root, "data"); isEphemeral = true; + setBlobsRoot(blobsDir()); return root; } diff --git a/tools/liveblocks-cli/src/dev-server/index.ts b/tools/liveblocks-cli/src/dev-server/index.ts index d4b0df2452..3d78faf997 100644 --- a/tools/liveblocks-cli/src/dev-server/index.ts +++ b/tools/liveblocks-cli/src/dev-server/index.ts @@ -38,6 +38,7 @@ import { } from "~/lib/term-colors"; import { authorizeWebSocket } from "./auth"; +import { setBlobStoreBaseUrl } from "./blobs/store"; import type { ClientMeta, RoomMeta, SessionMeta } from "./db/rooms"; import * as Rooms from "./db/rooms"; import { @@ -387,6 +388,10 @@ const dev: SubCommand = { }, }); listenPort = newServer.port!; + + // Signed download URLs have to point back at the port we actually bound, + // which isn't known until now (--random-port picks it at bind time). + setBlobStoreBaseUrl(`http://${newServer.hostname}:${newServer.port}`); return newServer; } diff --git a/tools/liveblocks-cli/src/dev-server/lib/decoders.ts b/tools/liveblocks-cli/src/dev-server/lib/decoders.ts index ed9f5e7d1b..55785e77ee 100644 --- a/tools/liveblocks-cli/src/dev-server/lib/decoders.ts +++ b/tools/liveblocks-cli/src/dev-server/lib/decoders.ts @@ -17,7 +17,34 @@ import type { IUserInfo } from "@liveblocks/core"; import type { Decoder } from "decoders"; -import { inexact, optional, string } from "decoders"; +import { + array, + inexact, + optional, + regex, + sized, + startsWith, + string, +} from "decoders"; + +// Mirrors `storageFileIdDecoder` in @shared/common, which isn't reachable from +// here: that package is backend-only and this one is mirrored to the public +// repo. The shape is "fl_" plus 21 nanoid characters. +export const storageFileId = sized(startsWith("fl_"), { size: 24 }); + +export const storageFileIds = array(storageFileId).refine( + (value) => value.length <= 500, + "Too many file ids, max 500" +); + +// Multipart upload ids are minted by the blob store as UUIDs. Unlike a real +// object store, which treats them as opaque tokens, the dev server's +// filesystem store turns them into a directory name — so what the URL says has +// to be checked before it gets anywhere near a path. +export const uploadId = regex( + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i, + "Must be a valid upload id" +); // A IUserInfo shape is any JSON object, but with the only requirement that // `name` and `avatar` keys are strings (if present). diff --git a/tools/liveblocks-cli/src/dev-server/lib/storage-files.ts b/tools/liveblocks-cli/src/dev-server/lib/storage-files.ts new file mode 100644 index 0000000000..0c113b69a3 --- /dev/null +++ b/tools/liveblocks-cli/src/dev-server/lib/storage-files.ts @@ -0,0 +1,353 @@ +/** + * Copyright (c) Liveblocks Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import { CrdtType } from "@liveblocks/core"; +import type { BlobMeta, UploadedPart } from "@liveblocks/server"; +import { abort } from "@liveblocks/zenrouter"; +import mime from "mime"; + +import { getBlobStore } from "~/dev-server/blobs/store"; +import * as Rooms from "~/dev-server/db/rooms"; + +export const SIGNED_URL_EXPIRES_IN_SECONDS = 3600; + +/** + * Per-file size cap. Production reads `paywall.fileMaxSizeInBytes` from the + * project's features config; the dev server has no such notion, so it's + * effectively unlimited. The enforcement points below are kept in place so + * that introducing a real limit is a config change rather than a rewrite. + */ +const FILE_MAX_SIZE_IN_BYTES = Number.POSITIVE_INFINITY; + +export type StorageFileData = { + id: string; + name: string; + size: number; + mimeType: string; +}; + +export type StorageFileWithSignedUrl = StorageFileData & { + url: string; + expiresAt: string; +}; + +/** + * Blobs are keyed by internal room ID, so renaming a room doesn't strand its + * files and two rooms can't collide on a file ID. + */ +export function storageFileObjectKey( + internalRoomId: string, + fileId: string +): string { + return `${internalRoomId}/${fileId}`; +} + +/** Resolve a room's internal ID, 404ing if the room doesn't exist. */ +export function requireInternalRoomId(roomId: string): string { + const record = Rooms.getRoom(roomId); + if (!record) { + abort(404); + } + return record.internalId; +} + +export function signedUrlExpiresAt(): string { + return new Date( + Date.now() + SIGNED_URL_EXPIRES_IN_SECONDS * 1000 + ).toISOString(); +} + +/** + * Upload a whole file in one request. + * + * Re-uploading the same file ID is tolerated rather than rejected, because the + * SDK retries uploads: if the object is already there, the request body is + * drained and the existing metadata returned. It's only a conflict if the + * caller is describing a *different* file under an ID that's already taken. + */ +export async function uploadStorageFile( + internalRoomId: string, + fileId: string, + name: string, + body: ReadableStream, + fileSize?: number +): Promise { + const store = getBlobStore(); + const key = storageFileObjectKey(internalRoomId, fileId); + + const existing = await store.head(key); + if (existing) { + await consume(body); + + const existingData = storageFileDataFrom(fileId, existing); + if ( + existingData.name !== name || + (fileSize !== undefined && existingData.size !== fileSize) + ) { + abort(409); + } + return existingData; + } + + // Pre-flight on the advisory size the client claims, so an over-limit upload + // can be refused before its bytes are streamed anywhere. + if (fileSize !== undefined && fileSize > FILE_MAX_SIZE_IN_BYTES) { + await consume(body); + abort(413); + } + + const mimeType = getMimeType(name); + const written = await store.put(key, body, { + contentType: mimeType, + contentDisposition: createContentDisposition(name), + }); + + // ...and again on the real size, which is the only one that counts. + if (written.size > FILE_MAX_SIZE_IN_BYTES) { + await store.delete(key); + abort(413); + } + + return { id: fileId, name, size: written.size, mimeType }; +} + +export async function createStorageFileMultipartUpload( + internalRoomId: string, + fileId: string, + name: string, + fileSize?: number +): Promise<{ fileId: string; uploadId: string }> { + if (fileSize !== undefined && fileSize > FILE_MAX_SIZE_IN_BYTES) { + abort(413); + } + + const store = getBlobStore(); + const key = storageFileObjectKey(internalRoomId, fileId); + + if (await store.head(key)) { + abort(409); + } + + const uploadId = await store.createMultipart(key, { + contentType: getMimeType(name), + contentDisposition: createContentDisposition(name), + }); + return { fileId, uploadId }; +} + +export async function uploadStorageFileMultipartPart( + internalRoomId: string, + fileId: string, + uploadId: string, + partNumber: number, + body: ReadableStream +): Promise { + return await getBlobStore().uploadPart( + storageFileObjectKey(internalRoomId, fileId), + uploadId, + partNumber, + body + ); +} + +/** + * Assemble a multipart upload. Like the single-shot path, an already-present + * object short-circuits, so that a retried completion is not an error. + */ +export async function completeStorageFileMultipartUpload( + internalRoomId: string, + fileId: string, + uploadId: string, + parts: UploadedPart[] +): Promise { + const store = getBlobStore(); + const key = storageFileObjectKey(internalRoomId, fileId); + + const existing = await store.head(key); + if (existing) { + return storageFileDataFrom(fileId, existing); + } + + const completed = await store.completeMultipart(key, uploadId, parts); + if (completed.size > FILE_MAX_SIZE_IN_BYTES) { + await store.delete(key); + abort(413); + } + + const uploaded = await store.head(key); + if (!uploaded) { + abort(404); + } + return storageFileDataFrom(fileId, uploaded); +} + +export async function abortStorageFileMultipartUpload( + internalRoomId: string, + fileId: string, + uploadId: string +): Promise { + await getBlobStore().abortMultipart( + storageFileObjectKey(internalRoomId, fileId), + uploadId + ); +} + +export async function getStorageFileWithSignedUrl( + internalRoomId: string, + fileId: string +): Promise { + const store = getBlobStore(); + const key = storageFileObjectKey(internalRoomId, fileId); + + const meta = await store.head(key); + if (!meta) { + return null; + } + + return { + ...storageFileDataFrom(fileId, meta), + url: await store.signedGetUrl(key, SIGNED_URL_EXPIRES_IN_SECONDS), + expiresAt: signedUrlExpiresAt(), + }; +} + +/** + * Resolve a batch of file IDs to download URLs. + * + * The three outcomes are distinguishable on purpose, and the client acts on + * them differently: + * - a URL: referenced in Storage, and the bytes are there + * - `false`: uploaded, but nothing in Storage points at it yet — the client + * retries, since this usually means an op is still in flight + * - `null`: nothing we know about, and retrying won't help + */ +export async function getStorageFileSignedUrls( + internalRoomId: string, + fileIds: string[], + referencedFileIds: ReadonlySet, + uploadedFileIds: ReadonlySet +): Promise<{ urls: (string | false | null)[]; expiresAt: string }> { + const store = getBlobStore(); + const expiresAt = signedUrlExpiresAt(); + + const resolved = new Map(); + for (const fileId of new Set(fileIds)) { + if (!referencedFileIds.has(fileId)) { + resolved.set(fileId, uploadedFileIds.has(fileId) ? false : null); + continue; + } + + const key = storageFileObjectKey(internalRoomId, fileId); + resolved.set( + fileId, + (await store.head(key)) + ? await store.signedGetUrl(key, SIGNED_URL_EXPIRES_IN_SECONDS) + : null + ); + } + + return { urls: fileIds.map((id) => resolved.get(id) ?? null), expiresAt }; +} + +/** + * Which of `fileIds` are referenced by a LiveFile node, and which have been + * uploaded. Production answers this inside the Durable Object with two SQL + * queries; here both live in the same process. + */ +export function partitionStorageFileIds( + roomId: string, + fileIds: string[] +): { referenced: Set; uploaded: Set } { + const room = Rooms.getRoomInstance(roomId); + const wanted = new Set(fileIds); + + const referenced = new Set(); + for (const [, node] of room.driver.iter_nodes()) { + if (node.type === CrdtType.FILE && wanted.has(node.data.id)) { + referenced.add(node.data.id); + } + } + + const uploaded = new Set(); + for (const fileId of wanted) { + if (room.driver.get_livefile_upload_size(fileId) !== undefined) { + uploaded.add(fileId); + } + } + + return { referenced, uploaded }; +} + +/** Record that a file's bytes have landed, so it may be referenced. */ +export function recordLivefileUpload( + roomId: string, + file: StorageFileData +): void { + Rooms.getRoomInstance(roomId).driver.put_livefile_upload(file.id, file.size); +} + +function storageFileDataFrom(fileId: string, meta: BlobMeta): StorageFileData { + return { + id: fileId, + name: filenameFromContentDisposition(meta.contentDisposition), + size: meta.size, + mimeType: meta.contentType, + }; +} + +async function consume(body: ReadableStream): Promise { + await new Response(body).arrayBuffer(); +} + +function getMimeType(name: string): string { + return mime.getType(name) ?? ""; +} + +export function createContentDisposition( + fileName: string, + type: "inline" | "attachment" = "inline" +): string { + const escaped = fileName.replace(/\\/g, "\\\\").replace(/"/g, '\\"'); + return `${type}; filename="${escaped}"`; +} + +/** Inverse of createContentDisposition(), including its backslash escaping. */ +function filenameFromContentDisposition(contentDisposition: string): string { + const prefix = 'filename="'; + const start = contentDisposition.indexOf(prefix); + if (start === -1) { + return ""; + } + + let fileName = ""; + for (let i = start + prefix.length; i < contentDisposition.length; i++) { + const char = contentDisposition[i]; + if (char === '"') { + return fileName; + } + + if (char === "\\" && i + 1 < contentDisposition.length) { + fileName += contentDisposition[i + 1]; + i++; + continue; + } + + fileName += char; + } + + return ""; +} diff --git a/tools/liveblocks-cli/src/dev-server/routes/client-api.ts b/tools/liveblocks-cli/src/dev-server/routes/client-api.ts index b99a926feb..d126243f2a 100644 --- a/tools/liveblocks-cli/src/dev-server/routes/client-api.ts +++ b/tools/liveblocks-cli/src/dev-server/routes/client-api.ts @@ -15,11 +15,40 @@ * along with this program. If not, see . */ -import { ZenRouter } from "@liveblocks/zenrouter"; +import { abort, ZenRouter } from "@liveblocks/zenrouter"; +import { array, number, numeric, object, string } from "decoders"; +import { + storageFileId, + storageFileIds, + uploadId, +} from "~/dev-server/lib/decoders"; import { verifyJwtLite } from "~/dev-server/lib/jwt-lite"; +import { + abortStorageFileMultipartUpload, + completeStorageFileMultipartUpload, + createStorageFileMultipartUpload, + getStorageFileSignedUrls, + partitionStorageFileIds, + recordLivefileUpload, + requireInternalRoomId, + uploadStorageFile, + uploadStorageFileMultipartPart, +} from "~/dev-server/lib/storage-files"; import { DUMMY, NOT_IMPLEMENTED } from "~/dev-server/responses"; +/** The client's advisory `?fileSize=`, used only for pre-flight limit checks. */ +function optionalFileSize(url: URL): number | undefined { + const raw = url.searchParams.get("fileSize"); + if (raw === null) return undefined; + + const size = Number(raw); + if (!Number.isSafeInteger(size) || size < 0) { + abort(400); + } + return size; +} + export const zen = new ZenRouter({ cors: { allowCredentials: true, @@ -34,6 +63,11 @@ export const zen = new ZenRouter({ const acessToken = verifyJwtLite(token); return acessToken !== null; }, + params: { + fileId: storageFileId, + partNumber: numeric, + uploadId, + }, }); /** @@ -135,6 +169,118 @@ zen.route("POST /v2/c/rooms//text-metadata", () => { }); }); +/** + * ------------------------------------------------------------ + * LIVEFILE (Storage files) + * ------------------------------------------------------------ + * + * Mirrors the secret-key routes, minus the single-file GET and plus the batch + * presigned-urls endpoint — the same asymmetry production has. + * + * TODO: Verify the authenticated user's write permission for this room. The + * dev server currently only checks that the token is valid, like the other + * room-scoped client routes here. + */ + +zen.route( + "PUT /v2/c/rooms//storage/files//upload/", + async ({ req, url, p }) => { + const internalRoomId = requireInternalRoomId(p.roomId); + if (!req.body) { + abort(400); + } + + const file = await uploadStorageFile( + internalRoomId, + p.fileId, + p.name, + req.body, + optionalFileSize(url) + ); + recordLivefileUpload(p.roomId, file); + return file; + } +); + +zen.route( + "POST /v2/c/rooms//storage/files//multipart/", + async ({ url, p }) => { + const internalRoomId = requireInternalRoomId(p.roomId); + return await createStorageFileMultipartUpload( + internalRoomId, + p.fileId, + p.name, + optionalFileSize(url) + ); + } +); + +zen.route( + "PUT /v2/c/rooms//storage/files//multipart//", + async ({ req, p }) => { + const internalRoomId = requireInternalRoomId(p.roomId); + if (!req.body) { + abort(400); + } + + return await uploadStorageFileMultipartPart( + internalRoomId, + p.fileId, + p.uploadId, + p.partNumber, + req.body + ); + } +); + +zen.route( + "POST /v2/c/rooms//storage/files//multipart//complete", + + object({ parts: array(object({ partNumber: number, etag: string })) }), + + async ({ p, body }) => { + const internalRoomId = requireInternalRoomId(p.roomId); + const file = await completeStorageFileMultipartUpload( + internalRoomId, + p.fileId, + p.uploadId, + body.parts + ); + recordLivefileUpload(p.roomId, file); + return file; + } +); + +zen.route( + "DELETE /v2/c/rooms//storage/files//multipart/", + async ({ p }) => { + const internalRoomId = requireInternalRoomId(p.roomId); + await abortStorageFileMultipartUpload(internalRoomId, p.fileId, p.uploadId); + return new Response(null, { status: 200 }); + } +); + +zen.route( + "POST /v2/c/rooms//storage/files/presigned-urls", + + object({ fileIds: storageFileIds }), + + async ({ p, body }) => { + const internalRoomId = requireInternalRoomId(p.roomId); + const { referenced, uploaded } = partitionStorageFileIds( + p.roomId, + body.fileIds + ); + + return await getStorageFileSignedUrls( + internalRoomId, + body.fileIds, + referenced, + uploaded + ); + } +); + /** * ------------------------------------------------------------ * NOT IMPLEMENTED ROUTES @@ -149,12 +295,6 @@ zen.route("POST /v2/c/rooms//text-metadata", () => { zen.route("POST /v2/c/rooms//attachments//multipart//complete", () => NOT_IMPLEMENTED()); zen.route("DELETE /v2/c/rooms//attachments//multipart/", () => NOT_IMPLEMENTED()); zen.route("POST /v2/c/rooms//attachments/presigned-urls", () => NOT_IMPLEMENTED()); - zen.route("PUT /v2/c/rooms//storage/files//upload/", () => NOT_IMPLEMENTED()); - zen.route("POST /v2/c/rooms//storage/files//multipart/", () => NOT_IMPLEMENTED()); - zen.route("PUT /v2/c/rooms//storage/files//multipart//", () => NOT_IMPLEMENTED()); - zen.route("POST /v2/c/rooms//storage/files//multipart//complete", () => NOT_IMPLEMENTED()); - zen.route("DELETE /v2/c/rooms//storage/files//multipart/", () => NOT_IMPLEMENTED()); - zen.route("POST /v2/c/rooms//storage/files/presigned-urls", () => NOT_IMPLEMENTED()); zen.route("POST /v2/c/rooms//send-message", () => NOT_IMPLEMENTED()); zen.route("GET /v2/c/rooms//storage", () => NOT_IMPLEMENTED()); zen.route("GET /v2/c/rooms//versions", () => NOT_IMPLEMENTED()); diff --git a/tools/liveblocks-cli/src/dev-server/routes/public.ts b/tools/liveblocks-cli/src/dev-server/routes/public.ts index 824237db22..5d8399f137 100644 --- a/tools/liveblocks-cli/src/dev-server/routes/public.ts +++ b/tools/liveblocks-cli/src/dev-server/routes/public.ts @@ -17,6 +17,7 @@ import { abort, html, json, ZenRouter } from "@liveblocks/zenrouter"; +import { getBlobStore } from "~/dev-server/blobs/store"; import welcomeHtml from "~/dev-server/static/welcome.html"; export const zen = new ZenRouter({ @@ -37,3 +38,38 @@ zen.route("GET /", () => ) ) ); + +/** + * Serve a LiveFile blob to whoever holds a valid signed link. + * + * Unauthenticated by design: this is the dev-server stand-in for an object + * store's presigned URL, and the browser fetches it as an src or similar, + * with no opportunity to attach a header. All the authority is in the query + * string, and the store checks it. + * + * Dev-server-only, so it's listed in DEVSERVER_ONLY_ROUTES in the route-parity + * check — production hands out R2 URLs, which never come back to us. + */ +zen.route("GET /blob", async ({ url }) => { + const store = getBlobStore(); + + const key = store.verifySignedGetUrl(url.searchParams); + if (key === undefined) { + abort(403); + } + + const meta = await store.head(key); + const body = await store.get(key); + if (!meta || !body) { + abort(404); + } + + return new Response(body, { + status: 200, + headers: { + "Content-Type": meta.contentType || "application/octet-stream", + "Content-Disposition": meta.contentDisposition, + "Content-Length": String(meta.size), + }, + }); +}); diff --git a/tools/liveblocks-cli/src/dev-server/routes/rest-api.ts b/tools/liveblocks-cli/src/dev-server/routes/rest-api.ts index 78b90973f1..88b2b21eea 100644 --- a/tools/liveblocks-cli/src/dev-server/routes/rest-api.ts +++ b/tools/liveblocks-cli/src/dev-server/routes/rest-api.ts @@ -26,6 +26,8 @@ import { QueryParser } from "@liveblocks/query-parser"; import type { Guid, YDocId } from "@liveblocks/server"; import { ConsoleTarget, + hasUploadedLivefiles, + hasUploadedLivefilesInPlainLson, jsonObjectYolo, Logger, ROOT_YDOC_ID, @@ -34,13 +36,15 @@ import { snapshotToPlainLson_eager, transientClientMsgDecoder, } from "@liveblocks/server"; -import { json, ndjsonStream, ZenRouter } from "@liveblocks/zenrouter"; +import { abort, json, ndjsonStream, ZenRouter } from "@liveblocks/zenrouter"; import { array, constant, either, enum_, nullable, + number, + numeric, object, optional, record, @@ -53,6 +57,17 @@ import * as Y from "yjs"; import type { DbRoom, RoomFilters } from "~/dev-server/db/rooms"; import * as Rooms from "~/dev-server/db/rooms"; import { authorizeSecretKey } from "~/dev-server/lib/auth"; +import { storageFileId, uploadId } from "~/dev-server/lib/decoders"; +import { + abortStorageFileMultipartUpload, + completeStorageFileMultipartUpload, + createStorageFileMultipartUpload, + getStorageFileWithSignedUrl, + recordLivefileUpload, + requireInternalRoomId, + uploadStorageFile, + uploadStorageFileMultipartPart, +} from "~/dev-server/lib/storage-files"; import { yDocToJson } from "~/dev-server/lib/ydoc"; import { DUMMY, NOT_IMPLEMENTED } from "~/dev-server/responses"; @@ -71,8 +86,25 @@ const roomMetadata = record( export const zen = new ZenRouter({ authorize: ({ req }) => authorizeSecretKey(req), + params: { + fileId: storageFileId, + partNumber: numeric, + uploadId, + }, }); +/** The client's advisory `?fileSize=`, used only for pre-flight limit checks. */ +function optionalFileSize(url: URL): number | undefined { + const raw = url.searchParams.get("fileSize"); + if (raw === null) return undefined; + + const size = Number(raw); + if (!Number.isSafeInteger(size) || size < 0) { + abort(400); + } + return size; +} + function ROOM_NOT_FOUND(roomId: string): Response { return json( { error: "ROOM_NOT_FOUND", message: `Room with id "${roomId}" not found.` }, @@ -336,6 +368,16 @@ zen.route( ); } + if (!hasUploadedLivefilesInPlainLson(room.driver, body)) { + return json( + { + error: "UNPROCESSABLE_ENTITY", + message: "Storage file has not been uploaded", + }, + 422 + ); + } + // Initialize storage room.driver.DANGEROUSLY_reset_nodes(body); room.unload(); @@ -663,6 +705,18 @@ zen.route( const room = Rooms.getRoomInstance(p.roomId); + // Backend sessions deliberately skip the isClientMsgAllowed hook, so the + // upload-before-reference rule has to be enforced here explicitly. + if (!hasUploadedLivefiles(room.driver, body.messages)) { + return json( + { + error: "UNPROCESSABLE_ENTITY", + message: "Storage file has not been uploaded", + }, + 422 + ); + } + const [session, capturedServerMsgs] = room.createBackendSession_experimental(); @@ -675,6 +729,112 @@ zen.route( } ); +/** + * ------------------------------------------------------------ + * LIVEFILE (Storage files) + * ------------------------------------------------------------ + * + * Uploads are proxied: the request body streams straight into the blob store, + * and only downloads are handed out as signed URLs. Every successful upload is + * followed by recording a receipt, which is what makes the file referenceable + * from Storage at all. + */ + +zen.route( + "PUT /v2/rooms//storage/files//upload/", + async ({ req, url, p }) => { + const internalRoomId = requireInternalRoomId(p.roomId); + if (!req.body) { + abort(400); + } + + const file = await uploadStorageFile( + internalRoomId, + p.fileId, + p.name, + req.body, + optionalFileSize(url) + ); + recordLivefileUpload(p.roomId, file); + return file; + } +); + +zen.route( + "POST /v2/rooms//storage/files//multipart/", + async ({ url, p }) => { + const internalRoomId = requireInternalRoomId(p.roomId); + return await createStorageFileMultipartUpload( + internalRoomId, + p.fileId, + p.name, + optionalFileSize(url) + ); + } +); + +zen.route( + "PUT /v2/rooms//storage/files//multipart//", + async ({ req, p }) => { + const internalRoomId = requireInternalRoomId(p.roomId); + if (!req.body) { + abort(400); + } + + return await uploadStorageFileMultipartPart( + internalRoomId, + p.fileId, + p.uploadId, + p.partNumber, + req.body + ); + } +); + +zen.route( + "POST /v2/rooms//storage/files//multipart//complete", + + object({ parts: array(object({ partNumber: number, etag: string })) }), + + async ({ p, body }) => { + const internalRoomId = requireInternalRoomId(p.roomId); + const file = await completeStorageFileMultipartUpload( + internalRoomId, + p.fileId, + p.uploadId, + body.parts + ); + recordLivefileUpload(p.roomId, file); + return file; + } +); + +zen.route( + "DELETE /v2/rooms//storage/files//multipart/", + async ({ p }) => { + const internalRoomId = requireInternalRoomId(p.roomId); + await abortStorageFileMultipartUpload(internalRoomId, p.fileId, p.uploadId); + return new Response(null, { status: 200 }); + } +); + +zen.route("GET /v2/rooms//storage/files/", async ({ p }) => { + const internalRoomId = requireInternalRoomId(p.roomId); + const file = await getStorageFileWithSignedUrl(internalRoomId, p.fileId); + + if (!file) { + return json( + { + error: "STORAGE_FILE_NOT_FOUND", + message: "Storage file not found", + suggestion: "Please verify the file ID and room ID are correct", + }, + 404 + ); + } + return file; +}); + /** * ------------------------------------------------------------ * NOT IMPLEMENTED ROUTES @@ -724,12 +884,6 @@ zen.route( zen.route("POST /v2/rooms//attachments//multipart//complete", () => NOT_IMPLEMENTED()); zen.route("DELETE /v2/rooms//attachments//multipart/", () => NOT_IMPLEMENTED()); zen.route("GET /v2/rooms//attachments/", () => NOT_IMPLEMENTED()); - zen.route("PUT /v2/rooms//storage/files//upload/", () => NOT_IMPLEMENTED()); - zen.route("POST /v2/rooms//storage/files//multipart/", () => NOT_IMPLEMENTED()); - zen.route("PUT /v2/rooms//storage/files//multipart//", () => NOT_IMPLEMENTED()); - zen.route("POST /v2/rooms//storage/files//multipart//complete", () => NOT_IMPLEMENTED()); - zen.route("DELETE /v2/rooms//storage/files//multipart/", () => NOT_IMPLEMENTED()); - zen.route("GET /v2/rooms//storage/files/", () => NOT_IMPLEMENTED()); zen.route("GET /v2/rooms//users//notification-settings", () => NOT_IMPLEMENTED()); zen.route("GET /v2/rooms//users//subscription-settings", () => NOT_IMPLEMENTED()); zen.route("POST /v2/rooms//users//notification-settings", () => NOT_IMPLEMENTED()); diff --git a/tools/liveblocks-cli/src/upgrade/index.ts b/tools/liveblocks-cli/src/upgrade/index.ts index cc8d477aa7..d9939e3544 100644 --- a/tools/liveblocks-cli/src/upgrade/index.ts +++ b/tools/liveblocks-cli/src/upgrade/index.ts @@ -197,13 +197,9 @@ const upgrade: SubCommand = { // Uninstall renamed packages first if (depsToUninstall.length > 0) { - execFileSync( - pm, - [uninstallCmd, ...skipInstallArgs, ...depsToUninstall], - { - stdio: "inherit", - } - ); + execFileSync(pm, [uninstallCmd, ...skipInstallArgs, ...depsToUninstall], { + stdio: "inherit", + }); } // Install/upgrade packages diff --git a/tools/liveblocks-cli/test/devserver/_helpers.ts b/tools/liveblocks-cli/test/devserver/_helpers.ts index 89a8b46cb2..bf0d8c4ce6 100644 --- a/tools/liveblocks-cli/test/devserver/_helpers.ts +++ b/tools/liveblocks-cli/test/devserver/_helpers.ts @@ -15,7 +15,9 @@ * along with this program. If not, see . */ -import { nanoid } from "@liveblocks/core"; +import { nanoid, Permission } from "@liveblocks/core"; + +import { createJwtLite } from "~/dev-server/lib/jwt-lite"; // TODO Reinstate URL-unsafe characters (`/`, `+`, `?`) like the production // helpers in our real production app. Doing so requires the dev-server test @@ -24,3 +26,14 @@ import { nanoid } from "@liveblocks/core"; export function makeExternalRoomId(): string { return `room-${nanoid()}`; } + +/** A `Bearer ` value granting write access to one room. */ +export function makeAccessToken(roomId: string): string { + const token = createJwtLite({ + k: "acc", + pid: "localdev", + uid: `user-${nanoid()}`, + perms: { [roomId]: [Permission.RoomWrite] }, + }); + return `Bearer ${token}`; +} diff --git a/tools/liveblocks-cli/test/devserver/blobs/FsBlobStore.test.ts b/tools/liveblocks-cli/test/devserver/blobs/FsBlobStore.test.ts new file mode 100644 index 0000000000..f1952f6fe8 --- /dev/null +++ b/tools/liveblocks-cli/test/devserver/blobs/FsBlobStore.test.ts @@ -0,0 +1,396 @@ +/** + * Copyright (c) Liveblocks Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { describe, expect, test } from "bun:test"; + +import { FsBlobStore } from "~/dev-server/blobs/FsBlobStore"; + +// bun:test's expect().rejects is awaitable, but not typed as a Thenable, so +// the rule misfires here. Same disable as in _generateFullTestSuite. +/* eslint-disable @typescript-eslint/await-thenable */ + +const BASE_URL = "http://localhost:1153"; +const SECRET = "test-secret"; + +const TEXT = { contentType: "text/plain", contentDisposition: 'inline; filename="hello.txt"' }; // prettier-ignore + +/** A store in a fresh temp dir, cleaned up when the test ends. */ +function makeStore(): FsBlobStore { + const root = mkdtempSync(join(tmpdir(), "lb-blobs-")); + // bun:test has no per-test teardown hook we can register from a helper, so + // clean up on process exit instead — these are tiny temp dirs. + process.on("exit", () => rmSync(root, { recursive: true, force: true })); + return new FsBlobStore({ root, baseUrl: BASE_URL, secret: SECRET }); +} + +function bytes(text: string): Uint8Array { + return new TextEncoder().encode(text); +} + +async function readAll( + stream: ReadableStream | undefined +): Promise { + expect(stream).toBeDefined(); + return await new Response(stream).text(); +} + +// A key of the shape the routes actually use, plus one that would escape the +// store root if keys were ever treated as paths. +const KEY = "room_abc123/fl_iN9WvpTnFO4qXbLXpZ2Kr"; +const OTHER_KEY = "room_abc123/fl_someOtherFile00000000"; +const HOSTILE_KEY = "../../etc/passwd"; + +describe("FsBlobStore", () => { + describe("put / head / get / delete", () => { + test("put reports the observed size and head reads it back", async () => { + const store = makeStore(); + const meta = await store.put(KEY, bytes("hello world"), TEXT); + + expect(meta).toEqual({ ...TEXT, size: 11 }); + expect(await store.head(KEY)).toEqual({ ...TEXT, size: 11 }); + }); + + test("head is undefined for an object that was never written", async () => { + const store = makeStore(); + expect(await store.head(KEY)).toBeUndefined(); + }); + + test("get streams back exactly what was written", async () => { + const store = makeStore(); + await store.put(KEY, bytes("hello world"), TEXT); + expect(await readAll(await store.get(KEY))).toEqual("hello world"); + }); + + test("get is undefined for an object that was never written", async () => { + const store = makeStore(); + expect(await store.get(KEY)).toBeUndefined(); + }); + + test("put accepts a stream, not just a buffer", async () => { + const store = makeStore(); + const meta = await store.put(KEY, streamOf("streamed"), TEXT); + + expect(meta.size).toEqual(8); + expect(await readAll(await store.get(KEY))).toEqual("streamed"); + }); + + test("a zero-byte object is stored and is not the same as absent", async () => { + const store = makeStore(); + await store.put(KEY, bytes(""), TEXT); + + expect(await store.head(KEY)).toEqual({ ...TEXT, size: 0 }); + expect(await readAll(await store.get(KEY))).toEqual(""); + }); + + test("put replaces an existing object and its metadata", async () => { + const store = makeStore(); + await store.put(KEY, bytes("first"), TEXT); + await store.put(KEY, bytes("second version"), { + contentType: "application/json", + contentDisposition: 'inline; filename="other.json"', + }); + + expect(await store.head(KEY)).toEqual({ + contentType: "application/json", + contentDisposition: 'inline; filename="other.json"', + size: 14, + }); + }); + + test("delete removes the object", async () => { + const store = makeStore(); + await store.put(KEY, bytes("hello world"), TEXT); + await store.delete(KEY); + + expect(await store.head(KEY)).toBeUndefined(); + expect(await store.get(KEY)).toBeUndefined(); + }); + + test("delete is a no-op for an object that doesn't exist", async () => { + const store = makeStore(); + await store.delete(KEY); + expect(await store.head(KEY)).toBeUndefined(); + }); + + test("keys are flat names, so path-like keys can't escape the store", async () => { + const store = makeStore(); + await store.put(HOSTILE_KEY, bytes("nope"), TEXT); + + // Round-trips as an ordinary key, and is not confusable with any other + expect(await store.head(HOSTILE_KEY)).toEqual({ ...TEXT, size: 4 }); + expect(await store.head("etc/passwd")).toBeUndefined(); + }); + + // Percent-encoding escapes "/" but not ".", so these are the only inputs + // that would still read as path syntax once encoded. + test.each(["..", ".", ""])( + "a key of %p is refused rather than resolving to a directory", + async (key) => { + const store = makeStore(); + await expect(store.head(key)).rejects.toThrow(/Unsafe blob store name/); + await expect(store.get(key)).rejects.toThrow(/Unsafe blob store name/); + await expect(store.put(key, bytes("x"), TEXT)).rejects.toThrow( + /Unsafe blob store name/ + ); + } + ); + }); + + describe("multipart", () => { + test("parts uploaded out of order are assembled in part-number order", async () => { + const store = makeStore(); + const uploadId = await store.createMultipart(KEY, TEXT); + + // Deliberately uploaded and listed back-to-front + const third = await store.uploadPart(KEY, uploadId, 3, bytes("ccc")); + const first = await store.uploadPart(KEY, uploadId, 1, bytes("aaa")); + const second = await store.uploadPart(KEY, uploadId, 2, bytes("bbb")); + + const meta = await store.completeMultipart(KEY, uploadId, [ + third, + second, + first, + ]); + + expect(meta).toEqual({ ...TEXT, size: 9 }); + expect(await readAll(await store.get(KEY))).toEqual("aaabbbccc"); + }); + + test("the object does not exist until the upload is completed", async () => { + const store = makeStore(); + const uploadId = await store.createMultipart(KEY, TEXT); + await store.uploadPart(KEY, uploadId, 1, bytes("aaa")); + + expect(await store.head(KEY)).toBeUndefined(); + }); + + test("re-uploading a part replaces it", async () => { + const store = makeStore(); + const uploadId = await store.createMultipart(KEY, TEXT); + await store.uploadPart(KEY, uploadId, 1, bytes("aaa")); + const replaced = await store.uploadPart(KEY, uploadId, 1, bytes("zzz")); + + await store.completeMultipart(KEY, uploadId, [replaced]); + expect(await readAll(await store.get(KEY))).toEqual("zzz"); + }); + + test("etags identify part contents", async () => { + const store = makeStore(); + const uploadId = await store.createMultipart(KEY, TEXT); + + const a = await store.uploadPart(KEY, uploadId, 1, bytes("same")); + const b = await store.uploadPart(KEY, uploadId, 2, bytes("same")); + const c = await store.uploadPart(KEY, uploadId, 3, bytes("different")); + + expect(a.etag).toEqual(b.etag); + expect(a.etag).not.toEqual(c.etag); + }); + + test("completing with a mismatched etag is refused", async () => { + const store = makeStore(); + const uploadId = await store.createMultipart(KEY, TEXT); + const part = await store.uploadPart(KEY, uploadId, 1, bytes("aaa")); + + await expect( + store.completeMultipart(KEY, uploadId, [ + { ...part, etag: "not-the-right-etag" }, + ]) + ).rejects.toThrow(/ETag mismatch/); + }); + + test("completing with a part that was never uploaded is refused", async () => { + const store = makeStore(); + const uploadId = await store.createMultipart(KEY, TEXT); + + await expect( + store.completeMultipart(KEY, uploadId, [{ partNumber: 7, etag: "x" }]) + ).rejects.toThrow(/No such part/); + }); + + test("abort discards the upload, leaving no object behind", async () => { + const store = makeStore(); + const uploadId = await store.createMultipart(KEY, TEXT); + const part = await store.uploadPart(KEY, uploadId, 1, bytes("aaa")); + + await store.abortMultipart(KEY, uploadId); + + expect(await store.head(KEY)).toBeUndefined(); + await expect( + store.completeMultipart(KEY, uploadId, [part]) + ).rejects.toThrow(/No such multipart upload/); + }); + + test("an upload can't be driven against a different key", async () => { + const store = makeStore(); + const uploadId = await store.createMultipart(KEY, TEXT); + + await expect( + store.uploadPart("room_abc123/fl_someOtherFile00000000", uploadId, 1, bytes("aaa")) // prettier-ignore + ).rejects.toThrow(/does not belong to/); + }); + + test("an upload can't be aborted against a different key", async () => { + const store = makeStore(); + const uploadId = await store.createMultipart(KEY, TEXT); + const part = await store.uploadPart(KEY, uploadId, 1, bytes("aaa")); + + await expect(store.abortMultipart(OTHER_KEY, uploadId)).rejects.toThrow( + /does not belong to/ + ); + + // ...and the upload it named is left intact + await store.completeMultipart(KEY, uploadId, [part]); + expect(await readAll(await store.get(KEY))).toEqual("aaa"); + }); + + // The routes take straight from the URL with no decoder, so + // this is the one identifier a caller fully controls. Left unchecked, ".." + // resolves to the store root — where abort's recursive delete would land. + test.each(["..", ".", ""])( + "an upload id of %p is refused rather than resolving to the store root", + async (uploadId) => { + const store = makeStore(); + await store.put(KEY, bytes("untouched"), TEXT); + + await expect(store.abortMultipart(KEY, uploadId)).rejects.toThrow( + /Unsafe blob store name/ + ); + await expect( + store.uploadPart(KEY, uploadId, 1, bytes("x")) + ).rejects.toThrow(/Unsafe blob store name/); + + // The store is still standing + expect(await store.head(KEY)).toEqual({ ...TEXT, size: 9 }); + } + ); + + // A JS number can never interpolate into a "/", so this isn't a traversal + // guard — it's the same "one identifier, one file" invariant as the names + // above, and it stops nonsense part numbers becoming odd little files. + test.each([0, -1, 1.5, Number.MAX_SAFE_INTEGER + 1])( + "a part number of %p is refused", + async (partNumber) => { + const store = makeStore(); + const uploadId = await store.createMultipart(KEY, TEXT); + + await expect( + store.uploadPart(KEY, uploadId, partNumber, bytes("x")) + ).rejects.toThrow(/Invalid part number/); + } + ); + + test("aborting an upload that doesn't exist is a no-op", async () => { + // The SDK aborts on failure paths, where the upload may never have been + // created in the first place. + const store = makeStore(); + await store.abortMultipart(KEY, "no-such-upload-id"); + }); + + test("completing twice is refused, since the upload is gone", async () => { + const store = makeStore(); + const uploadId = await store.createMultipart(KEY, TEXT); + const part = await store.uploadPart(KEY, uploadId, 1, bytes("aaa")); + await store.completeMultipart(KEY, uploadId, [part]); + + await expect( + store.completeMultipart(KEY, uploadId, [part]) + ).rejects.toThrow(/No such multipart upload/); + }); + }); + + describe("signed URLs", () => { + test("a freshly signed URL verifies and yields its key back", async () => { + const store = makeStore(); + const url = new URL(await store.signedGetUrl(KEY, 3600)); + + expect(url.origin).toEqual(BASE_URL); + expect(store.verifySignedGetUrl(url.searchParams)).toEqual(KEY); + }); + + test("a tampered key is refused", async () => { + const store = makeStore(); + const url = new URL(await store.signedGetUrl(KEY, 3600)); + url.searchParams.set("key", "room_abc123/fl_someOtherFile00000000"); + + expect(store.verifySignedGetUrl(url.searchParams)).toBeUndefined(); + }); + + test("a tampered signature is refused", async () => { + const store = makeStore(); + const url = new URL(await store.signedGetUrl(KEY, 3600)); + url.searchParams.set("sig", "0".repeat(64)); + + expect(store.verifySignedGetUrl(url.searchParams)).toBeUndefined(); + }); + + test("extending the expiry without re-signing is refused", async () => { + const store = makeStore(); + const url = new URL(await store.signedGetUrl(KEY, 3600)); + url.searchParams.set("exp", String(Date.now() + 999_999_999)); + + expect(store.verifySignedGetUrl(url.searchParams)).toBeUndefined(); + }); + + test("an expired URL is refused even though it is correctly signed", async () => { + const store = makeStore(); + const url = new URL(await store.signedGetUrl(KEY, -1)); + + expect(store.verifySignedGetUrl(url.searchParams)).toBeUndefined(); + }); + + test("a URL signed with a different secret is refused", async () => { + const store = makeStore(); + const url = new URL(await store.signedGetUrl(KEY, 3600)); + + const otherStore = new FsBlobStore({ + root: mkdtempSync(join(tmpdir(), "lb-blobs-")), + baseUrl: BASE_URL, + secret: "a-different-secret", + }); + expect(otherStore.verifySignedGetUrl(url.searchParams)).toBeUndefined(); + }); + + test("missing params are refused rather than throwing", () => { + const store = makeStore(); + expect(store.verifySignedGetUrl(new URLSearchParams())).toBeUndefined(); + expect( + store.verifySignedGetUrl(new URLSearchParams({ key: KEY })) + ).toBeUndefined(); + }); + + test("signing does not require the object to exist", async () => { + // head() is the existence check; signing is pure string work. The routes + // rely on being able to check existence separately. + const store = makeStore(); + const url = new URL(await store.signedGetUrl(KEY, 3600)); + expect(store.verifySignedGetUrl(url.searchParams)).toEqual(KEY); + }); + }); +}); + +function streamOf(text: string): ReadableStream { + return new ReadableStream({ + start(controller) { + controller.enqueue(bytes(text)); + controller.close(); + }, + }); +} diff --git a/tools/liveblocks-cli/test/devserver/rest-api/livefile-hook.test.ts b/tools/liveblocks-cli/test/devserver/rest-api/livefile-hook.test.ts new file mode 100644 index 0000000000..8bbbc5bb06 --- /dev/null +++ b/tools/liveblocks-cli/test/devserver/rest-api/livefile-hook.test.ts @@ -0,0 +1,241 @@ +/** + * Copyright (c) Liveblocks Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import type { Json } from "@liveblocks/core"; +import { ClientMsgCode, nanoid, OpCode, ServerMsgCode } from "@liveblocks/core"; +import type { IServerWebSocket } from "@liveblocks/server"; +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; + +import * as Rooms from "~/dev-server/db/rooms"; +import { zen as restApi } from "~/dev-server/routes/rest-api"; + +import { makeExternalRoomId } from "../_helpers"; + +const BASE = "http://localhost"; +const SECRET = { Authorization: "Bearer sk_localdev" }; + +function makeFileId(): string { + return `fl_${nanoid()}`; +} + +/** + * Collects everything the server sends down a session's socket. + * + * The whole point of these tests is the isClientMsgAllowed hook, which only + * runs for browser sessions. The REST routes all use backend sessions, which + * deliberately skip it, so they can't reach this code path at all. + */ +class FakeSocket implements IServerWebSocket { + readonly sent: string[] = []; + closed: { code: number; reason?: string } | undefined; + + send(msg: string | ArrayBuffer): number { + this.sent.push(typeof msg === "string" ? msg : ""); + return 1; + } + + close(code: number, reason?: string): void { + this.closed = { code, reason }; + } + + /** Every server message received, parsed. */ + messages(): { type: number; reason?: string }[] { + return this.sent.flatMap((raw) => { + const parsed = JSON.parse(raw) as + | { type: number; reason?: string } + | { type: number; reason?: string }[]; + return Array.isArray(parsed) ? parsed : [parsed]; + }); + } +} + +/** Start a real browser session on a room, and return its socket. */ +function connect(roomId: string): { + socket: FakeSocket; + send: (msgs: Json[]) => Promise; +} { + const room = Rooms.getRoomInstance(roomId); + const socket = new FakeSocket(); + const ticket = room.createTicket(); + + // Side effects from hooks are fire-and-forget here; nothing under test + // depends on them completing. + const defer = (p: Promise) => void p; + + room.startBrowserSession(ticket, socket, undefined, defer); + return { + socket, + send: (msgs) => + room.handleData( + ticket.sessionKey, + JSON.stringify(msgs), + undefined, + defer + ), + }; +} + +function createFileOp(fileId: string, size: number) { + return { + type: OpCode.CREATE_FILE, + id: "1:0", + parentId: "root", + parentKey: "file", + data: { id: fileId, name: "hello.txt", size, mimeType: "text/plain" }, + opId: "1:1", + }; +} + +async function api(method: string, path: string, body?: unknown) { + const headers: Record = { ...SECRET }; + const init: RequestInit = { method, headers }; + if (body !== undefined) { + headers["Content-Type"] = "application/json"; + init.body = JSON.stringify(body); + } + return restApi.fetch(new Request(`${BASE}${path}`, init)); +} + +async function makeRoom(): Promise { + const roomId = makeExternalRoomId(); + await api("POST", "/v2/rooms", { id: roomId }); + return roomId; +} + +async function upload(roomId: string, fileId: string, contents: string) { + return restApi.fetch( + new Request( + `${BASE}/v2/rooms/${roomId}/storage/files/${fileId}/upload/hello.txt`, + { method: "PUT", headers: SECRET, body: contents } + ) + ); +} + +async function serverStorage(roomId: string) { + const resp = await api("GET", `/v2/rooms/${roomId}/storage`); + return (await resp.json()) as { + data: Record; + }; +} + +describe("isClientMsgAllowed hook (WebSocket path)", () => { + beforeAll(() => Rooms.useEphemeralStorage()); + afterAll(() => Rooms.cleanup()); // Needed in bun:test (unlike in Vitest) + + test("rejects a CREATE_FILE op for a file that was never uploaded", async () => { + const roomId = await makeRoom(); + const { socket, send } = connect(roomId); + + await send([ + { type: ClientMsgCode.UPDATE_STORAGE, ops: [createFileOp(makeFileId(), 1)] }, // prettier-ignore + ]); + + const rejection = socket + .messages() + .find((msg) => msg.type === ServerMsgCode.REJECT_STORAGE_OP); + + expect(rejection).toBeDefined(); + expect(rejection?.reason).toBe("Storage file has not been uploaded"); + + // ...and the node never made it into Storage + expect((await serverStorage(roomId)).data.file).toBeUndefined(); + }); + + test("allows a CREATE_FILE op once the file has been uploaded", async () => { + const roomId = await makeRoom(); + const fileId = makeFileId(); + await upload(roomId, fileId, "hello world"); + + const { socket, send } = connect(roomId); + await send([ + { type: ClientMsgCode.UPDATE_STORAGE, ops: [createFileOp(fileId, 11)] }, + ]); + + expect( + socket.messages().find((m) => m.type === ServerMsgCode.REJECT_STORAGE_OP) + ).toBeUndefined(); + expect((await serverStorage(roomId)).data.file?.data.size).toBe(11); + }); + + test("the uploaded size wins over the size the client claims", async () => { + const roomId = await makeRoom(); + const fileId = makeFileId(); + await upload(roomId, fileId, "hello world"); + + const { send } = connect(roomId); + // Uploaded 11 bytes, claiming 1 + await send([ + { type: ClientMsgCode.UPDATE_STORAGE, ops: [createFileOp(fileId, 1)] }, + ]); + + expect((await serverStorage(roomId)).data.file?.data.size).toBe(11); + }); + + test("one bad file in a batch rejects the whole message", async () => { + const roomId = await makeRoom(); + const uploadedId = makeFileId(); + await upload(roomId, uploadedId, "hello world"); + + const { socket, send } = connect(roomId); + await send([ + { + type: ClientMsgCode.UPDATE_STORAGE, + ops: [ + createFileOp(uploadedId, 11), + { + ...createFileOp(makeFileId(), 1), + id: "1:2", + parentKey: "ghost", + opId: "1:3", + }, + ], + }, + ]); + + expect( + socket.messages().find((m) => m.type === ServerMsgCode.REJECT_STORAGE_OP) + ).toBeDefined(); + + // Neither op applied: the check is per-message, not per-op + const storage = await serverStorage(roomId); + expect(storage.data.file).toBeUndefined(); + expect(storage.data.ghost).toBeUndefined(); + }); + + test("ops that don't create files are unaffected", async () => { + const roomId = await makeRoom(); + const { socket, send } = connect(roomId); + + await send([ + { + type: ClientMsgCode.UPDATE_STORAGE, + ops: [ + { + type: OpCode.UPDATE_OBJECT, + id: "root", + data: { greeting: "hello" }, + opId: "1:1", + }, + ], + }, + ]); + + expect( + socket.messages().find((m) => m.type === ServerMsgCode.REJECT_STORAGE_OP) + ).toBeUndefined(); + }); +}); diff --git a/tools/liveblocks-cli/test/devserver/rest-api/storage-files.test.ts b/tools/liveblocks-cli/test/devserver/rest-api/storage-files.test.ts new file mode 100644 index 0000000000..5573a10246 --- /dev/null +++ b/tools/liveblocks-cli/test/devserver/rest-api/storage-files.test.ts @@ -0,0 +1,688 @@ +/** + * Copyright (c) Liveblocks Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import { ClientMsgCode, nanoid, OpCode } from "@liveblocks/core"; +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; + +import * as Rooms from "~/dev-server/db/rooms"; +import { zen as clientApi } from "~/dev-server/routes/client-api"; +import { zen as publicApi } from "~/dev-server/routes/public"; +import { zen as restApi } from "~/dev-server/routes/rest-api"; + +import { makeAccessToken, makeExternalRoomId } from "../_helpers"; + +const BASE = "http://localhost"; +const SECRET = { Authorization: "Bearer sk_localdev" }; + +function makeFileId(): string { + return `fl_${nanoid()}`; +} + +async function api( + method: string, + path: string, + body?: unknown +): Promise { + const headers: Record = { ...SECRET }; + const init: RequestInit = { method, headers }; + if (body !== undefined) { + headers["Content-Type"] = "application/json"; + init.body = JSON.stringify(body); + } + return restApi.fetch(new Request(`${BASE}${path}`, init)); +} + +/** PUT a raw body (the upload routes take octet-stream, not JSON). */ +async function putBlob( + path: string, + body: string, + token = SECRET.Authorization +): Promise { + const router = path.startsWith("/v2/c/") ? clientApi : restApi; + return router.fetch( + new Request(`${BASE}${path}`, { + method: "PUT", + headers: { Authorization: token }, + body, + }) + ); +} + +async function clientApiCall( + method: string, + path: string, + token: string, + body?: unknown +): Promise { + const headers: Record = { Authorization: token }; + const init: RequestInit = { method, headers }; + if (body !== undefined) { + headers["Content-Type"] = "application/json"; + init.body = JSON.stringify(body); + } + return clientApi.fetch(new Request(`${BASE}${path}`, init)); +} + +type FileData = { + id: string; + name: string; + size: number; + mimeType: string; +}; +type SignedFile = FileData & { url: string; expiresAt: string }; +type Part = { partNumber: number; etag: string }; +type ErrorBody = { error: string; message: string }; +type StorageDoc = { data: { file: { data: { size: number } } } }; + +async function readJson(resp: Response): Promise { + return (await resp.json()) as T; +} + +/** Create a room and return its ID. */ +async function makeRoom(): Promise { + const roomId = makeExternalRoomId(); + await api("POST", "/v2/rooms", { id: roomId }); + return roomId; +} + +describe("REST API - storage files (LiveFile)", () => { + beforeAll(() => Rooms.useEphemeralStorage()); + afterAll(() => Rooms.cleanup()); // Needed in bun:test (unlike in Vitest) + + describe("PUT /v2/rooms//storage/files//upload/", () => { + test("uploads a file and reports its server-measured metadata", async () => { + const roomId = await makeRoom(); + const fileId = makeFileId(); + + const resp = await putBlob( + `/v2/rooms/${roomId}/storage/files/${fileId}/upload/hello.txt`, + "hello world" + ); + + expect(resp.status).toBe(200); + expect(await readJson(resp)).toEqual({ + id: fileId, + name: "hello.txt", + size: 11, + mimeType: "text/plain", + }); + }); + + test("returns 404 for a room that doesn't exist", async () => { + const resp = await putBlob( + `/v2/rooms/${makeExternalRoomId()}/storage/files/${makeFileId()}/upload/hello.txt`, + "hello world" + ); + expect(resp.status).toBe(404); + }); + + test("returns 400 for a malformed file id", async () => { + const roomId = await makeRoom(); + const resp = await putBlob( + `/v2/rooms/${roomId}/storage/files/not-a-file-id/upload/hello.txt`, + "hello world" + ); + expect(resp.status).toBe(400); + }); + + test("repeating the same upload returns the existing file", async () => { + const roomId = await makeRoom(); + const fileId = makeFileId(); + const path = `/v2/rooms/${roomId}/storage/files/${fileId}/upload/hello.txt`; + + const first = await putBlob(path, "hello world"); + const second = await putBlob(path, "hello world"); + + expect(second.status).toBe(200); + expect(await readJson(second)).toEqual( + await readJson(first) + ); + }); + + test("reusing a file id for a different name is a conflict", async () => { + const roomId = await makeRoom(); + const fileId = makeFileId(); + + await putBlob( + `/v2/rooms/${roomId}/storage/files/${fileId}/upload/hello.txt`, + "hello world" + ); + const resp = await putBlob( + `/v2/rooms/${roomId}/storage/files/${fileId}/upload/other.txt`, + "hello world" + ); + + expect(resp.status).toBe(409); + }); + + test("repeating an upload with a mismatched fileSize is a conflict", async () => { + const roomId = await makeRoom(); + const fileId = makeFileId(); + const path = `/v2/rooms/${roomId}/storage/files/${fileId}/upload/hello.txt`; + + await putBlob(path, "hello world"); + const resp = await putBlob(`${path}?fileSize=999`, "hello world"); + + expect(resp.status).toBe(409); + }); + + test("filenames with quotes survive the content-disposition round-trip", async () => { + const roomId = await makeRoom(); + const fileId = makeFileId(); + const name = 'we"ird\\name.txt'; + + await putBlob( + `/v2/rooms/${roomId}/storage/files/${fileId}/upload/${encodeURIComponent(name)}`, + "hello world" + ); + + const resp = await api( + "GET", + `/v2/rooms/${roomId}/storage/files/${fileId}` + ); + expect((await readJson(resp)).name).toEqual(name); + }); + + test("a zero-byte file uploads and is recorded", async () => { + const roomId = await makeRoom(); + const fileId = makeFileId(); + + const resp = await putBlob( + `/v2/rooms/${roomId}/storage/files/${fileId}/upload/empty.txt`, + "" + ); + + expect(resp.status).toBe(200); + expect((await readJson(resp)).size).toBe(0); + + // A zero-byte file is uploaded, so referencing it must be allowed + expect( + Rooms.getRoomInstance(roomId).driver.get_livefile_upload_size(fileId) + ).toBe(0); + }); + }); + + describe("GET /v2/rooms//storage/files/", () => { + test("returns metadata plus a signed URL", async () => { + const roomId = await makeRoom(); + const fileId = makeFileId(); + await putBlob( + `/v2/rooms/${roomId}/storage/files/${fileId}/upload/hello.txt`, + "hello world" + ); + + const resp = await api( + "GET", + `/v2/rooms/${roomId}/storage/files/${fileId}` + ); + expect(resp.status).toBe(200); + + const body = await readJson(resp); + expect(body).toMatchObject({ + id: fileId, + name: "hello.txt", + size: 11, + mimeType: "text/plain", + }); + expect(typeof body.url).toBe("string"); + expect(Date.parse(body.expiresAt)).toBeGreaterThan(Date.now()); + }); + + test("returns 404 for a file that was never uploaded", async () => { + const roomId = await makeRoom(); + const resp = await api( + "GET", + `/v2/rooms/${roomId}/storage/files/${makeFileId()}` + ); + expect(resp.status).toBe(404); + }); + + test("a file uploaded to another room is not visible", async () => { + const roomA = await makeRoom(); + const roomB = await makeRoom(); + const fileId = makeFileId(); + await putBlob( + `/v2/rooms/${roomA}/storage/files/${fileId}/upload/hello.txt`, + "hello world" + ); + + const resp = await api("GET", `/v2/rooms/${roomB}/storage/files/${fileId}`); // prettier-ignore + expect(resp.status).toBe(404); + }); + }); + + describe("GET /blob", () => { + test("a signed URL serves the bytes with their metadata", async () => { + const roomId = await makeRoom(); + const fileId = makeFileId(); + await putBlob( + `/v2/rooms/${roomId}/storage/files/${fileId}/upload/hello.txt`, + "hello world" + ); + + const { url } = await readJson( + await api("GET", `/v2/rooms/${roomId}/storage/files/${fileId}`) + ); + + // No Authorization header: the browser fetching this URL has none + const resp = await publicApi.fetch(new Request(url)); + + expect(resp.status).toBe(200); + expect(await resp.text()).toEqual("hello world"); + expect(resp.headers.get("content-type")).toBe("text/plain"); + expect(resp.headers.get("content-disposition")).toBe( + 'inline; filename="hello.txt"' + ); + }); + + test("a tampered signature is refused", async () => { + const roomId = await makeRoom(); + const fileId = makeFileId(); + await putBlob( + `/v2/rooms/${roomId}/storage/files/${fileId}/upload/hello.txt`, + "hello world" + ); + + const { url } = await readJson( + await api("GET", `/v2/rooms/${roomId}/storage/files/${fileId}`) + ); + + const tampered = new URL(url); + tampered.searchParams.set("sig", "0".repeat(64)); + + const resp = await publicApi.fetch(new Request(tampered)); + expect(resp.status).toBe(403); + }); + + test("an unsigned request is refused", async () => { + const resp = await publicApi.fetch( + new Request(`${BASE}/blob?key=whatever`) + ); + expect(resp.status).toBe(403); + }); + }); + + describe("multipart upload", () => { + test("uploads a file across parts, assembled in order", async () => { + const roomId = await makeRoom(); + const fileId = makeFileId(); + + const created = await api( + "POST", + `/v2/rooms/${roomId}/storage/files/${fileId}/multipart/hello.txt` + ); + expect(created.status).toBe(200); + const { uploadId } = await readJson<{ uploadId: string }>(created); + expect(typeof uploadId).toBe("string"); + + // Uploaded back-to-front on purpose + const second = await putBlob( + `/v2/rooms/${roomId}/storage/files/${fileId}/multipart/${uploadId}/2`, + "world" + ); + const first = await putBlob( + `/v2/rooms/${roomId}/storage/files/${fileId}/multipart/${uploadId}/1`, + "hello " + ); + + const completed = await api( + "POST", + `/v2/rooms/${roomId}/storage/files/${fileId}/multipart/${uploadId}/complete`, + { + parts: [await readJson(second), await readJson(first)], + } + ); + + expect(completed.status).toBe(200); + expect(await readJson(completed)).toEqual({ + id: fileId, + name: "hello.txt", + size: 11, + mimeType: "text/plain", + }); + + const { url } = await readJson( + await api("GET", `/v2/rooms/${roomId}/storage/files/${fileId}`) + ); + expect(await (await publicApi.fetch(new Request(url))).text()).toEqual( + "hello world" + ); + }); + + test("a malformed upload id is rejected at the router", async () => { + // The dev server's blob store turns the upload id into a directory name, + // so it's checked before it reaches any path building. Production hands + // it to R2 as an opaque token and has no such concern. + // + // Note there's no case for ".." here: the URL parser collapses dot + // segments — even percent-encoded ones — so one can never arrive as a + // path parameter in the first place. FsBlobStore refuses it anyway, for + // callers that aren't routes. + const roomId = await makeRoom(); + const fileId = makeFileId(); + + const resp = await api( + "DELETE", + `/v2/rooms/${roomId}/storage/files/${fileId}/multipart/not-a-uuid` + ); + expect(resp.status).toBe(400); + }); + + test("creating an upload for an existing file is a conflict", async () => { + const roomId = await makeRoom(); + const fileId = makeFileId(); + await putBlob( + `/v2/rooms/${roomId}/storage/files/${fileId}/upload/hello.txt`, + "hello world" + ); + + const resp = await api( + "POST", + `/v2/rooms/${roomId}/storage/files/${fileId}/multipart/hello.txt` + ); + expect(resp.status).toBe(409); + }); + + test("aborting discards the upload, leaving no file behind", async () => { + const roomId = await makeRoom(); + const fileId = makeFileId(); + + const { uploadId } = await readJson<{ uploadId: string }>( + await api( + "POST", + `/v2/rooms/${roomId}/storage/files/${fileId}/multipart/hello.txt` + ) + ); + await putBlob( + `/v2/rooms/${roomId}/storage/files/${fileId}/multipart/${uploadId}/1`, + "hello" + ); + + const aborted = await api( + "DELETE", + `/v2/rooms/${roomId}/storage/files/${fileId}/multipart/${uploadId}` + ); + expect(aborted.status).toBe(200); + + const resp = await api( + "GET", + `/v2/rooms/${roomId}/storage/files/${fileId}` + ); + expect(resp.status).toBe(404); + }); + + test("the file is not readable until the upload completes", async () => { + const roomId = await makeRoom(); + const fileId = makeFileId(); + + const { uploadId } = await readJson<{ uploadId: string }>( + await api( + "POST", + `/v2/rooms/${roomId}/storage/files/${fileId}/multipart/hello.txt` + ) + ); + await putBlob( + `/v2/rooms/${roomId}/storage/files/${fileId}/multipart/${uploadId}/1`, + "hello" + ); + + const resp = await api( + "GET", + `/v2/rooms/${roomId}/storage/files/${fileId}` + ); + expect(resp.status).toBe(404); + }); + }); + + describe("POST /v2/c/rooms//storage/files/presigned-urls", () => { + test("distinguishes referenced, uploaded-but-unreferenced, and unknown", async () => { + const roomId = await makeRoom(); + const token = makeAccessToken(roomId); + + const referencedId = makeFileId(); + const uploadedOnlyId = makeFileId(); + const unknownId = makeFileId(); + + // One file uploaded and referenced from Storage... + await putBlob( + `/v2/rooms/${roomId}/storage/files/${referencedId}/upload/a.txt`, + "aaa" + ); + await api("POST", `/v2/rooms/${roomId}/storage`, { + liveblocksType: "LiveObject", + data: { + file: { + liveblocksType: "LiveFile", + data: { + id: referencedId, + name: "a.txt", + size: 3, + mimeType: "text/plain", + }, + }, + }, + }); + + // ...one uploaded but not referenced... + await putBlob( + `/v2/rooms/${roomId}/storage/files/${uploadedOnlyId}/upload/b.txt`, + "bbb" + ); + + // ...and one we've never heard of. + const resp = await clientApiCall( + "POST", + `/v2/c/rooms/${roomId}/storage/files/presigned-urls`, + token, + { fileIds: [referencedId, uploadedOnlyId, unknownId] } + ); + + expect(resp.status).toBe(200); + const { urls } = await readJson<{ urls: (string | false | null)[] }>( + resp + ); + expect(typeof urls[0]).toBe("string"); + expect(urls[1]).toBe(false); + expect(urls[2]).toBe(null); + }); + + test("returns one entry per requested id, including duplicates", async () => { + const roomId = await makeRoom(); + const token = makeAccessToken(roomId); + const fileId = makeFileId(); + + const resp = await clientApiCall( + "POST", + `/v2/c/rooms/${roomId}/storage/files/presigned-urls`, + token, + { fileIds: [fileId, fileId] } + ); + + const { urls } = await readJson<{ urls: (string | false | null)[] }>( + resp + ); + expect(urls).toEqual([null, null]); + }); + + test("rejects a malformed file id", async () => { + const roomId = await makeRoom(); + const token = makeAccessToken(roomId); + + const resp = await clientApiCall( + "POST", + `/v2/c/rooms/${roomId}/storage/files/presigned-urls`, + token, + { fileIds: ["nope"] } + ); + expect(resp.status).toBe(422); + }); + }); + + describe("the client-api routes mirror the secret-key ones", () => { + test("upload works through /v2/c with an access token", async () => { + const roomId = await makeRoom(); + const token = makeAccessToken(roomId); + const fileId = makeFileId(); + + const resp = await putBlob( + `/v2/c/rooms/${roomId}/storage/files/${fileId}/upload/hello.txt`, + "hello world", + token + ); + + expect(resp.status).toBe(200); + expect((await readJson(resp)).size).toBe(11); + }); + + test("upload through /v2/c is refused without a token", async () => { + const roomId = await makeRoom(); + const resp = await clientApi.fetch( + new Request( + `${BASE}/v2/c/rooms/${roomId}/storage/files/${makeFileId()}/upload/hello.txt`, + { method: "PUT", body: "hello world" } + ) + ); + expect(resp.status).toBe(403); + }); + }); + + describe("upload-before-reference", () => { + test("send-message refuses a LiveFile that was never uploaded", async () => { + const roomId = await makeRoom(); + + const resp = await api("POST", `/v2/rooms/${roomId}/send-message`, { + messages: [ + { + type: ClientMsgCode.UPDATE_STORAGE, + ops: [ + { + type: OpCode.CREATE_FILE, + id: "1:0", + parentId: "root", + parentKey: "file", + data: { + id: makeFileId(), + name: "ghost.txt", + size: 1, + mimeType: "text/plain", + }, + opId: "1:1", + }, + ], + }, + ], + }); + + expect(resp.status).toBe(422); + expect((await readJson(resp)).message).toBe( + "Storage file has not been uploaded" + ); + }); + + test("send-message accepts an uploaded LiveFile, and overrides its size", async () => { + const roomId = await makeRoom(); + const fileId = makeFileId(); + await putBlob( + `/v2/rooms/${roomId}/storage/files/${fileId}/upload/hello.txt`, + "hello world" + ); + + const resp = await api("POST", `/v2/rooms/${roomId}/send-message`, { + messages: [ + { + type: ClientMsgCode.UPDATE_STORAGE, + ops: [ + { + type: OpCode.CREATE_FILE, + id: "1:0", + parentId: "root", + parentKey: "file", + // The client claims 1 byte. It uploaded 11. + data: { + id: fileId, + name: "hello.txt", + size: 1, + mimeType: "text/plain", + }, + opId: "1:1", + }, + ], + }, + ], + }); + expect(resp.status).toBe(200); + + const storage = await readJson( + await api("GET", `/v2/rooms/${roomId}/storage`) + ); + expect(storage.data.file.data.size).toBe(11); + }); + + test("storage init refuses a LiveFile that was never uploaded", async () => { + const roomId = await makeRoom(); + + const resp = await api("POST", `/v2/rooms/${roomId}/storage`, { + liveblocksType: "LiveObject", + data: { + file: { + liveblocksType: "LiveFile", + data: { + id: makeFileId(), + name: "ghost.txt", + size: 1, + mimeType: "text/plain", + }, + }, + }, + }); + + expect(resp.status).toBe(422); + expect((await readJson(resp)).message).toBe( + "Storage file has not been uploaded" + ); + }); + + test("storage init overrides a claimed size with the uploaded one", async () => { + const roomId = await makeRoom(); + const fileId = makeFileId(); + await putBlob( + `/v2/rooms/${roomId}/storage/files/${fileId}/upload/hello.txt`, + "hello world" + ); + + await api("POST", `/v2/rooms/${roomId}/storage`, { + liveblocksType: "LiveObject", + data: { + file: { + liveblocksType: "LiveFile", + data: { + id: fileId, + name: "hello.txt", + size: 1, + mimeType: "text/plain", + }, + }, + }, + }); + + const storage = await readJson( + await api("GET", `/v2/rooms/${roomId}/storage`) + ); + expect(storage.data.file.data.size).toBe(11); + }); + }); +}); diff --git a/tools/liveblocks-cli/test/plugins/_generateFullTestSuite.ts b/tools/liveblocks-cli/test/plugins/_generateFullTestSuite.ts index df18e5b060..d32465961c 100644 --- a/tools/liveblocks-cli/test/plugins/_generateFullTestSuite.ts +++ b/tools/liveblocks-cli/test/plugins/_generateFullTestSuite.ts @@ -2584,6 +2584,138 @@ export function generateFullTestSuite(config: { })); }); + describe("livefile upload receipt API impl", () => { + const FILE_ID = "fl_iN9WvpTnFO4qXbLXpZ2Kr"; + const OTHER_FILE_ID = "fl_Bq7zMk1RsW0dYvLc3TnAe"; + + function fileData(size: number, id = FILE_ID): LiveFileData { + return { id, name: "hello.txt", size, mimeType: "text/plain" }; + } + + /** All FILE node sizes currently in storage, in iteration order. */ + function fileSizes(driver: TDriver): number[] { + const sizes = []; + for (const [, node] of driver.iter_nodes()) { + if (node.type === CrdtType.FILE) { + sizes.push(node.data.size); + } + } + return sizes; + } + + test("get_livefile_upload_size is undefined for an unrecorded file", () => + runTest((driver) => { + expect(driver.get_livefile_upload_size(FILE_ID)).toEqual(undefined); + })); + + test("put_livefile_upload records a size that reads back", () => + runTest((driver) => { + driver.put_livefile_upload(FILE_ID, 11); + expect(driver.get_livefile_upload_size(FILE_ID)).toEqual(11); + + // Receipts are per-file, not global + expect(driver.get_livefile_upload_size(OTHER_FILE_ID)).toEqual( + undefined + ); + })); + + test("put_livefile_upload overwrites an existing receipt", () => + runTest((driver) => { + driver.put_livefile_upload(FILE_ID, 11); + driver.put_livefile_upload(FILE_ID, 22); + expect(driver.get_livefile_upload_size(FILE_ID)).toEqual(22); + })); + + test("a zero-byte upload is recorded, and is not the same as undefined", () => + runTest((driver) => { + driver.put_livefile_upload(FILE_ID, 0); + expect(driver.get_livefile_upload_size(FILE_ID)).toEqual(0); + })); + + test("set_child replaces a client-claimed size with the recorded one", () => + runTest((driver) => { + driver.put_livefile_upload(FILE_ID, 11); + + // The client claims this file is 1 byte. It is not. + driver.set_child("1:0", { + type: CrdtType.FILE, + parentId: "root", + parentKey: "file", + data: fileData(1), + }); + + expect(driver.get_node("1:0")).toEqual({ + type: CrdtType.FILE, + parentId: "root", + parentKey: "file", + data: fileData(11), + }); + })); + + test("set_child replaces a client-claimed size even when the truth is 0", () => + runTest((driver) => { + driver.put_livefile_upload(FILE_ID, 0); + + driver.set_child("1:0", { + type: CrdtType.FILE, + parentId: "root", + parentKey: "file", + data: fileData(999), + }); + + expect(fileSizes(driver)).toEqual([0]); + })); + + test("set_child leaves the claimed size alone when there is no receipt", () => + runTest((driver) => { + // Refusing unreferenced files is the Room layer's job, not the + // driver's. With no receipt the driver has nothing better to say. + driver.set_child("1:0", { + type: CrdtType.FILE, + parentId: "root", + parentKey: "file", + data: fileData(1), + }); + + expect(fileSizes(driver)).toEqual([1]); + })); + + test("DANGEROUSLY_reset_nodes replaces claimed sizes with recorded ones", () => + runTest((driver) => { + driver.put_livefile_upload(FILE_ID, 11); + + driver.DANGEROUSLY_reset_nodes({ + liveblocksType: "LiveObject", + data: { + file: { liveblocksType: "LiveFile", data: fileData(1) }, + }, + }); + + expect(fileSizes(driver)).toEqual([11]); + })); + + test("DANGEROUSLY_reset_nodes leaves unrecorded files alone", () => + runTest((driver) => { + driver.DANGEROUSLY_reset_nodes({ + liveblocksType: "LiveObject", + data: { + file: { liveblocksType: "LiveFile", data: fileData(1) }, + }, + }); + + expect(fileSizes(driver)).toEqual([1]); + })); + + test("receipts survive DANGEROUSLY_reset_nodes", () => + runTest((driver) => { + // Resetting the document wipes nodes, but upload history is not part + // of the document — a file that was uploaded stays uploaded. + driver.put_livefile_upload(FILE_ID, 11); + driver.DANGEROUSLY_reset_nodes(EMPTY_DOC); + expect(driver.get_livefile_upload_size(FILE_ID)).toEqual(11); + })); + }); + describe("meta API impl", () => { test("get_meta with empty store is undefined", () => runTest((driver) =>