From c0602c4979722a1fb0541062c78a308b9b3d0cc0 Mon Sep 17 00:00:00 2001 From: Marc Bouchenoire Date: Thu, 23 Jul 2026 17:20:25 +0200 Subject: [PATCH] LiveFile (PR 1796) Co-authored-by: Olivier Foucherot Original commit: 11be08a347eb2f9a87e226157e3ff8c64ecf59f3 --- packages/liveblocks-server/package.json | 2 +- packages/liveblocks-server/src/Storage.ts | 12 +- packages/liveblocks-server/src/decoders/Op.ts | 26 ++++ .../src/formats/LossyJson.ts | 4 + .../src/formats/PlainLson.ts | 21 ++- .../src/plugins/InMemoryDriver.ts | 10 +- .../liveblocks-server/src/protocol/vNEXT.ts | 1 + .../test/decoders/ClientMsg.test.ts | 115 ++++++++++++++++ .../test/formats/LossyJson.test.ts | 19 +++ .../test/formats/PlainLson.test.ts | 28 ++++ .../test/plugins/_generateFullTestSuite.ts | 124 +++++++++++++++++- .../test/storage/live-file.test.ts | 79 +++++++++++ .../test/storage/model-based/storage-model.ts | 6 +- tools/liveblocks-cli/package.json | 2 +- .../src/dev-server/db/BunSQLiteDriver.ts | 39 +++++- .../liveblocks-cli/src/dev-server/db/rooms.ts | 6 +- .../src/dev-server/routes/client-api.ts | 6 + .../src/dev-server/routes/rest-api.ts | 6 + .../test/plugins/BunSQLiteDriver.test.ts | 4 +- .../test/plugins/_generateFullTestSuite.ts | 124 +++++++++++++++++- 20 files changed, 606 insertions(+), 28 deletions(-) create mode 100644 packages/liveblocks-server/test/decoders/ClientMsg.test.ts create mode 100644 packages/liveblocks-server/test/storage/live-file.test.ts diff --git a/packages/liveblocks-server/package.json b/packages/liveblocks-server/package.json index 9563580942b..66e090271e9 100644 --- a/packages/liveblocks-server/package.json +++ b/packages/liveblocks-server/package.json @@ -69,7 +69,7 @@ }, "sideEffects": false, "dependencies": { - "@liveblocks/core": "v3.21.0-private3", + "@liveblocks/core": "3.23.0-file1", "async-mutex": "^0.4.0", "decoders": "^2.9.0", "itertools": "^2.7.1", diff --git a/packages/liveblocks-server/src/Storage.ts b/packages/liveblocks-server/src/Storage.ts index 1cbac87c591..9f57bdf830f 100644 --- a/packages/liveblocks-server/src/Storage.ts +++ b/packages/liveblocks-server/src/Storage.ts @@ -128,6 +128,14 @@ function nodeFromCreateChildOp(op: CreateOp): SerializedChild { data: op.data, }; + case OpCode.CREATE_FILE: + return { + type: CrdtType.FILE, + parentId: op.parentId, + parentKey: op.parentKey, + data: op.data, + }; + // istanbul ignore next default: return assertNever(op, "Unknown op code"); @@ -176,6 +184,7 @@ export class Storage { case OpCode.CREATE_MAP: case OpCode.CREATE_REGISTER: case OpCode.CREATE_OBJECT: + case OpCode.CREATE_FILE: return this.applyCreateOp(op); case OpCode.UPDATE_OBJECT: @@ -255,7 +264,8 @@ export class Storage { return this.createChildAsListItem(op, node); case CrdtType.REGISTER: - // It's illegal for registers to have children + case CrdtType.FILE: + // It's illegal for leaf nodes to have children return ignore(op); // istanbul ignore next diff --git a/packages/liveblocks-server/src/decoders/Op.ts b/packages/liveblocks-server/src/decoders/Op.ts index 7cd2e212883..4977ac471e3 100644 --- a/packages/liveblocks-server/src/decoders/Op.ts +++ b/packages/liveblocks-server/src/decoders/Op.ts @@ -19,15 +19,19 @@ import { OpCode } from "@liveblocks/core"; import type { Decoder } from "decoders"; import { constant, + number, object, oneOf, optional, + sized, + startsWith, string, taggedUnion, } from "decoders"; import type { ClientWireOp, + CreateFileOp, CreateListOp, CreateMapOp, CreateObjectOp, @@ -43,6 +47,11 @@ import { jsonObjectYolo, jsonYolo } from "./jsonYolo"; type HasOpId = { opId: string }; const intent = oneOf(["set", "push"] as const); +const storageFileId = sized(startsWith("fl_"), { size: 24 }); +const fileSize = number.refine( + (value) => Number.isSafeInteger(value) && value >= 0, + "Must be a valid file size" +); const updateObjectOp: Decoder = object({ type: constant(OpCode.UPDATE_OBJECT), @@ -93,6 +102,22 @@ const createRegisterOp: Decoder = object({ deletedId: optional(string), }); +const createFileOp: Decoder = object({ + type: constant(OpCode.CREATE_FILE), + opId: string, + id: string, + parentId: string, + parentKey: string, + data: object({ + id: storageFileId, + name: string, + size: fileSize, + mimeType: string, + }), + intent: optional(intent), + deletedId: optional(string), +}); + const deleteCrdtOp: Decoder = object({ type: constant(OpCode.DELETE_CRDT), opId: string, @@ -119,6 +144,7 @@ export const op: Decoder = taggedUnion("type", { [OpCode.CREATE_LIST]: createListOp, [OpCode.CREATE_MAP]: createMapOp, [OpCode.CREATE_REGISTER]: createRegisterOp, + [OpCode.CREATE_FILE]: createFileOp, [OpCode.DELETE_CRDT]: deleteCrdtOp, [OpCode.SET_PARENT_KEY]: setParentKeyOp, [OpCode.DELETE_OBJECT_KEY]: deleteObjectKeyOp, diff --git a/packages/liveblocks-server/src/formats/LossyJson.ts b/packages/liveblocks-server/src/formats/LossyJson.ts index 8748ffdc1cb..80cd6824afd 100644 --- a/packages/liveblocks-server/src/formats/LossyJson.ts +++ b/packages/liveblocks-server/src/formats/LossyJson.ts @@ -49,6 +49,8 @@ function buildNode(snapshot: IReadableSnapshot, id: string): Json { return buildList(snapshot, id); } else if (node.type === CrdtType.MAP) { return buildMap(snapshot, id); + } else if (node.type === CrdtType.FILE) { + return node.data; } else { return node.data; } @@ -120,6 +122,8 @@ function* emit(snapshot: IReadableSnapshot, id: string): StringGen { yield* emitMap(snapshot, id); } else if (node.type === CrdtType.REGISTER) { yield JSON.stringify(node.data); + } else if (node.type === CrdtType.FILE) { + yield JSON.stringify(node.data); } } diff --git a/packages/liveblocks-server/src/formats/PlainLson.ts b/packages/liveblocks-server/src/formats/PlainLson.ts index 26c34169444..d06c32046af 100644 --- a/packages/liveblocks-server/src/formats/PlainLson.ts +++ b/packages/liveblocks-server/src/formats/PlainLson.ts @@ -20,6 +20,7 @@ import type { ObjectStorageNode, PlainLson, PlainLsonFields, + PlainLsonFile, PlainLsonList, PlainLsonMap, PlainLsonObject, @@ -44,7 +45,7 @@ function generateId(state: { clock: number }) { function isSpecialPlainLsonValue( value: PlainLson -): value is PlainLsonObject | PlainLsonMap | PlainLsonList { +): value is PlainLsonObject | PlainLsonMap | PlainLsonList | PlainLsonFile { return isJsonObject(value) && value.liveblocksType !== undefined; } @@ -72,6 +73,18 @@ function* iterJson( yield* iterMap(key, data.data, parent, state); return; + case "LiveFile": + yield [ + generateId(state), + { + type: CrdtType.FILE, + data: data.data, + parentId: parent[0], + parentKey: key, + }, + ]; + return; + // istanbul ignore next default: assertNever(data, "Unknown `liveblocksType` field"); @@ -231,6 +244,8 @@ function buildNode(snapshot: IReadableSnapshot, id: string): PlainLson { return buildList(snapshot, id); } else if (node.type === CrdtType.MAP) { return buildMap(snapshot, id); + } else if (node.type === CrdtType.FILE) { + return { liveblocksType: "LiveFile", data: node.data }; } else { // TEMPORARY: `?? null` is only here to project legacy KV rooms that // contain data-less registers (under a LiveMap, representing `null` @@ -308,6 +323,10 @@ function* emit(snapshot: IReadableSnapshot, id: string): StringGen { // TEMPORARY: see buildNode — remove `?? null` once all rooms are // migrated to SQLite. yield JSON.stringify(node.data ?? null); + } else if (node.type === CrdtType.FILE) { + yield '{"liveblocksType":"LiveFile","data":'; + yield JSON.stringify(node.data); + yield "}"; } } diff --git a/packages/liveblocks-server/src/plugins/InMemoryDriver.ts b/packages/liveblocks-server/src/plugins/InMemoryDriver.ts index bef786b293d..3a9e5c45087 100644 --- a/packages/liveblocks-server/src/plugins/InMemoryDriver.ts +++ b/packages/liveblocks-server/src/plugins/InMemoryDriver.ts @@ -88,9 +88,11 @@ function buildReverseLookup(nodes: NodeMap) { } } - if (node.type !== CrdtType.REGISTER) { + const isLeafNode = + node.type === CrdtType.REGISTER || node.type === CrdtType.FILE; + if (!isLeafNode) { queue.push(...revNodes.valuesAt(nodeId)); - } else { + } else if (node.type === CrdtType.REGISTER) { const parent = nodes.get(node.parentId); if (parent?.type === CrdtType.OBJECT) { continue; @@ -637,6 +639,10 @@ export class InMemoryDriver implements IStorageDriver { throw new Error("Cannot add register under object"); } + if (parentNode.type === CrdtType.FILE) { + throw new Error("Cannot add child under file"); + } + const conflictingSiblingId = revNodes.get(node.parentId, node.parentKey); if (conflictingSiblingId !== id) { // Conflict! diff --git a/packages/liveblocks-server/src/protocol/vNEXT.ts b/packages/liveblocks-server/src/protocol/vNEXT.ts index 1066e5b6a42..5cfe12af6c8 100644 --- a/packages/liveblocks-server/src/protocol/vNEXT.ts +++ b/packages/liveblocks-server/src/protocol/vNEXT.ts @@ -37,6 +37,7 @@ export type { RoomStateServerMsg, ServerMsg } from "@liveblocks/core"; // All Op types export type { ClientWireOp, + CreateFileOp, CreateListOp, CreateMapOp, CreateObjectOp, diff --git a/packages/liveblocks-server/test/decoders/ClientMsg.test.ts b/packages/liveblocks-server/test/decoders/ClientMsg.test.ts new file mode 100644 index 00000000000..3a416e3b235 --- /dev/null +++ b/packages/liveblocks-server/test/decoders/ClientMsg.test.ts @@ -0,0 +1,115 @@ +/** + * Copyright (c) Liveblocks Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import { ClientMsgCode, OpCode } from "@liveblocks/core"; +import { describe, expect, test } from "vitest"; + +import { clientMsgDecoder } from "~/decoders"; + +describe("clientMsgDecoder", () => { + test("accepts valid CREATE_FILE storage file ids", () => { + expect(() => + clientMsgDecoder.verify({ + type: ClientMsgCode.UPDATE_STORAGE, + ops: [ + { + type: OpCode.CREATE_FILE, + opId: "op-1", + id: "1:0", + parentId: "root", + parentKey: "cover", + data: { + id: "fl_123456789012345678901", + name: "cover.png", + size: 123, + mimeType: "image/png", + }, + }, + ], + }) + ).not.toThrow(); + }); + + test("accepts a zero-byte CREATE_FILE", () => { + expect(() => + clientMsgDecoder.verify({ + type: ClientMsgCode.UPDATE_STORAGE, + ops: [ + { + type: OpCode.CREATE_FILE, + opId: "op-1", + id: "1:0", + parentId: "root", + parentKey: "empty", + data: { + id: "fl_123456789012345678901", + name: "empty.txt", + size: 0, + mimeType: "text/plain", + }, + }, + ], + }) + ).not.toThrow(); + }); + + test("rejects invalid CREATE_FILE storage file ids", () => { + expect(() => + clientMsgDecoder.verify({ + type: ClientMsgCode.UPDATE_STORAGE, + ops: [ + { + type: OpCode.CREATE_FILE, + opId: "op-1", + id: "1:0", + parentId: "root", + parentKey: "cover", + data: { + id: "file_123", + name: "cover.png", + size: 123, + mimeType: "image/png", + }, + }, + ], + }) + ).toThrow(); + }); + + test.each([-1, 1.5])("rejects invalid CREATE_FILE size %s", (size) => { + expect(() => + clientMsgDecoder.verify({ + type: ClientMsgCode.UPDATE_STORAGE, + ops: [ + { + type: OpCode.CREATE_FILE, + opId: "op-1", + id: "1:0", + parentId: "root", + parentKey: "cover", + data: { + id: "fl_123456789012345678901", + name: "cover.png", + size, + mimeType: "image/png", + }, + }, + ], + }) + ).toThrow(); + }); +}); diff --git a/packages/liveblocks-server/test/formats/LossyJson.test.ts b/packages/liveblocks-server/test/formats/LossyJson.test.ts index ad8d895303b..e85b674b27a 100644 --- a/packages/liveblocks-server/test/formats/LossyJson.test.ts +++ b/packages/liveblocks-server/test/formats/LossyJson.test.ts @@ -47,6 +47,25 @@ describe("Serialization of nodes (to LossyJson format)", () => { expect(json).toEqual({}); }); + test("LiveFile", () => { + const file = { + id: "fl_123", + name: "brief.pdf", + size: 42, + mimeType: "application/pdf", + }; + // prettier-ignore + const snapshot = makeSnapshot([ + ["root", { data: {}, type: CrdtType.OBJECT }], + ["si:1", { data: file, parentId: "root", parentKey: "file", type: CrdtType.FILE }], + ]); + + const json = snapshotToLossyJson(snapshot); + expect(json).toEqual({ + file, + }); + }); + test("With root node", () => { // prettier-ignore const snapshot = makeSnapshot([ diff --git a/packages/liveblocks-server/test/formats/PlainLson.test.ts b/packages/liveblocks-server/test/formats/PlainLson.test.ts index 2296fd4ce9d..da82727a361 100644 --- a/packages/liveblocks-server/test/formats/PlainLson.test.ts +++ b/packages/liveblocks-server/test/formats/PlainLson.test.ts @@ -69,6 +69,34 @@ describe("Serialization of nodes (to PlainLson format)", () => { ); }); + test("LiveFile", () => { + const file = { + id: "fl_123", + name: "brief.pdf", + size: 42, + mimeType: "application/pdf", + }; + // prettier-ignore + const nodes: StorageNode[] = [ + ["root", { data: {}, type: CrdtType.OBJECT }], + ["si:1", { data: file, parentId: "root", parentKey: "file", type: CrdtType.FILE }], + ]; + + const plainLson = snapshotToPlainLson(makeSnapshot(nodes)); + expect(plainLson).toEqual({ + liveblocksType: "LiveObject", + data: { + file: { + liveblocksType: "LiveFile", + data: file, + }, + }, + }); + + const convertedNodes = plainLsonToNodeMap(plainLson); + expect(convertedNodes).toEqual(new Map(nodes)); + }); + test("With root node", () => { // prettier-ignore const nodes: StorageNode[] = [ diff --git a/packages/liveblocks-server/test/plugins/_generateFullTestSuite.ts b/packages/liveblocks-server/test/plugins/_generateFullTestSuite.ts index 5674b1561c3..3d1f55bd184 100644 --- a/packages/liveblocks-server/test/plugins/_generateFullTestSuite.ts +++ b/packages/liveblocks-server/test/plugins/_generateFullTestSuite.ts @@ -38,14 +38,17 @@ import type { Awaitable, ChildStorageNode, + FileStorageNode, Json, JsonObject, ListStorageNode, + LiveFileData, MapStorageNode, NodeMap, NodeStream, ObjectStorageNode, PlainLson, + PlainLsonFile, PlainLsonList, PlainLsonMap, PlainLsonObject, @@ -79,6 +82,7 @@ import { Logger as LoggerImpl, LogLevel, LogTarget } from "~/lib/Logger"; import { quote } from "~/lib/text"; import type { ClientWireOp, + CreateFileOp, CreateListOp, CreateMapOp, CreateObjectOp, @@ -208,6 +212,7 @@ export function selfCheck(storage: Storage): void { assert(driver.get_node(parentId) !== undefined, `Node ${quote(id)} points to ${quote(parentId)}, but no such node exists`); // prettier-ignore assert(driver.get_node(parentId)?.type !== CrdtType.REGISTER, `Node ${quote(parentId)} has children (e.g. ${quote(id)}), but is a register node`); // prettier-ignore + assert(driver.get_node(parentId)?.type !== CrdtType.FILE, `Node ${quote(parentId)} has children (e.g. ${quote(id)}), but is a file node`); // prettier-ignore assert(node.type !== CrdtType.REGISTER || driver.get_node(parentId)?.type !== CrdtType.OBJECT, `Node ${quote(id)} is a REGISTER with value ${JSON.stringify((node as SerializedRegister).data)}, but appears as a child under OBJECT node ${quote(parentId)}. Would have expected OBJECT to have this value as a static data attribute under key ${quote(parentKey)}.`); // prettier-ignore @@ -271,17 +276,18 @@ type InfiniteStream = Omit, "next"> & { next(): { value: T; done?: false }; }; -// Check if a PlainLson value is a container (LiveObject/LiveList/LiveMap) +// Check if a PlainLson value is a container (LiveObject/LiveList/LiveMap/LiveFile) function isPlainLsonContainer( value: PlainLson -): value is PlainLsonObject | PlainLsonList | PlainLsonMap { +): value is PlainLsonObject | PlainLsonList | PlainLsonMap | PlainLsonFile { return ( typeof value === "object" && value !== null && "liveblocksType" in value && (value.liveblocksType === "LiveObject" || value.liveblocksType === "LiveList" || - value.liveblocksType === "LiveMap") + value.liveblocksType === "LiveMap" || + value.liveblocksType === "LiveFile") ); } @@ -350,6 +356,16 @@ function plainLsonTreeToNodeMap( for (const [key, child] of Object.entries(plainLson.data)) { recurse(child, id, key); } + } else if (plainLson.liveblocksType === "LiveFile") { + result.push([ + id, + { + type: CrdtType.FILE, + parentId, + parentKey, + data: plainLson.data, + }, + ]); } } @@ -415,6 +431,15 @@ export function register( return [id, { type: CrdtType.REGISTER, parentId, parentKey, data }]; } +export function file( + id: string, + parentId: string, + parentKey: string, + data: LiveFileData +): FileStorageNode { + return [id, { type: CrdtType.FILE, parentId, parentKey, data }]; +} + export function updateObjectOp( id: string, data: Partial, @@ -484,6 +509,27 @@ export function createRegisterOp( }; } +export function createFileOp( + id: string, + parentId: string, + parentKey: string, + data: LiveFileData, + intent?: "set" | "push", + deletedId?: string, + opId = nanoid() +): CreateFileOp & HasOpId { + return { + opId, + id, + type: OpCode.CREATE_FILE, + parentId, + parentKey, + data, + intent, + deletedId, + }; +} + export function createMapOp( id: string, parentId: string, @@ -594,6 +640,14 @@ export function generateArbitraries() { ) ), + liveFileData: (): fc.Arbitrary => + fc.record({ + id: fc.stringMatching(/^fl_[0-9A-Za-z_-]{21}$/), + name: fc.string({ minLength: 1 }), + size: fc.nat(), + mimeType: fc.string({ minLength: 1 }), + }), + rootNodeTuple: () => fc.tuple<["root", SerializedRootObject]>( fc.constant("root"), @@ -643,6 +697,12 @@ export function generateArbitraries() { data: arb.json(), parentId: nonObjectParentId, parentKey: arb.key(), + }), + fc.record({ + type: fc.constant(CrdtType.FILE), + data: arb.liveFileData(), + parentId, + parentKey: arb.key(), }) ) .filter(wouldNotOverwriteDefaultDoc); @@ -807,13 +867,15 @@ export function generateArbitraries() { PlainLsonObject: PlainLsonObject; PlainLsonList: PlainLsonList; PlainLsonMap: PlainLsonMap; + PlainLsonFile: PlainLsonFile; Json: Json; }>((tie) => ({ PlainLson: fc.oneof( { arbitrary: tie("Json"), weight: 1, depthIdentifier, maxDepth, depthSize }, // prettier-ignore { arbitrary: tie("PlainLsonObject"), weight: 2, depthIdentifier, maxDepth, depthSize }, // prettier-ignore { arbitrary: tie("PlainLsonMap"), weight: 1, depthIdentifier, maxDepth, depthSize }, // prettier-ignore - { arbitrary: tie("PlainLsonList"), weight: 1, depthIdentifier, maxDepth, depthSize } // prettier-ignore + { arbitrary: tie("PlainLsonList"), weight: 1, depthIdentifier, maxDepth, depthSize }, // prettier-ignore + { arbitrary: tie("PlainLsonFile"), weight: 1, depthIdentifier, maxDepth, depthSize } // prettier-ignore ), PlainLsonObject: fc.record({ liveblocksType: fc.constant("LiveObject" as const), @@ -827,6 +889,10 @@ export function generateArbitraries() { liveblocksType: fc.constant("LiveList" as const), data: fc.array(tie("PlainLson"), { maxLength: options?.maxLength ?? 5 }), // prettier-ignore }), + PlainLsonFile: fc.record({ + liveblocksType: fc.constant("LiveFile" as const), + data: arb.liveFileData(), + }), Json: arb.json(), })).PlainLsonObject; }, @@ -911,6 +977,27 @@ export function generateArbitraries() { fc.option(arb.key(), { freq: 10, nil: undefined }), }), + createFileOp: (options?: { + id?: fc.Arbitrary; + parentId?: fc.Arbitrary; + parentKey?: fc.Arbitrary; + data?: fc.Arbitrary; + intent?: fc.Arbitrary<"set" | undefined>; + deletedId?: fc.Arbitrary; + }) => + fc.record({ + type: fc.constant(OpCode.CREATE_FILE), + opId: arb.opId(), + id: options?.id ?? arb.key(), + parentId: options?.parentId ?? arb.key(), + parentKey: options?.parentKey ?? arb.parentKey(), + data: options?.data ?? arb.liveFileData(), + intent: options?.intent ?? arb.intent(), + deletedId: + options?.deletedId ?? + fc.option(arb.key(), { freq: 10, nil: undefined }), + }), + createOp: (options?: { id?: fc.Arbitrary; parentId?: fc.Arbitrary; @@ -922,7 +1009,8 @@ export function generateArbitraries() { arb.createListOp(options), arb.createMapOp(options), arb.createObjectOp(options), - arb.createRegisterOp(options) + arb.createRegisterOp(options), + arb.createFileOp(options) ), deleteCrdtOpArb: () => @@ -1319,6 +1407,32 @@ export function generateFullTestSuite(config: { ); })); + test("set_child: throws if adding a child under a file", () => + runTest((driver) => { + resetToDefaultNodes(driver); + + const [, fileNode] = file("0:file", "root", "file", { + id: "fl_123456789012345678901", + name: "file.txt", + size: 5, + mimeType: "text/plain", + }); + + driver.set_child("0:file", fileNode); + + expectToThrow( + () => + driver.set_child("0:child", { + type: CrdtType.LIST, + parentId: "0:file", + parentKey: "child", + }), + /cannot add child under file/i + ); + + expect(driver.get_node("0:child")).toBe(undefined); + })); + test("get_child_at: returns child id after set", () => runTest((driver) => { driver.DANGEROUSLY_reset_nodes(EMPTY_DOC); diff --git a/packages/liveblocks-server/test/storage/live-file.test.ts b/packages/liveblocks-server/test/storage/live-file.test.ts new file mode 100644 index 00000000000..2e9d9534ba3 --- /dev/null +++ b/packages/liveblocks-server/test/storage/live-file.test.ts @@ -0,0 +1,79 @@ +/** + * Copyright (c) Liveblocks Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import { CrdtType } from "@liveblocks/core"; +import { describe, expect, test } from "vitest"; + +import { + createFileOp, + createObjectOp, +} from "~test/plugins/_generateFullTestSuite"; + +import { rootObj, runWithStorage } from "./utils"; + +const FILE_DATA = { + id: "fl_123", + name: "brief.pdf", + size: 42, + mimeType: "application/pdf", +}; + +describe("LiveFile", () => { + test("creates a file node under an object", () => + runWithStorage([rootObj()], ({ storage, driver }) => { + const op = createFileOp("1:0", "root", "file", FILE_DATA); + const [result] = storage.applyOps([op]); + + expect(result).toEqual({ + action: "accepted", + op, + fix: undefined, + }); + expect(driver.get_node("1:0")).toEqual({ + type: CrdtType.FILE, + parentId: "root", + parentKey: "file", + data: FILE_DATA, + }); + })); + + test("ignores children created under a file node", () => + runWithStorage( + [ + rootObj(), + [ + "1:0", + { + type: CrdtType.FILE, + parentId: "root", + parentKey: "file", + data: FILE_DATA, + }, + ], + ], + ({ storage, driver }) => { + const op = createObjectOp("2:0", "1:0", "child", {}); + const [result] = storage.applyOps([op]); + + expect(result).toEqual({ + action: "ignored", + ignoredOpId: op.opId, + }); + expect(driver.get_node("2:0")).toBeUndefined(); + } + )); +}); diff --git a/packages/liveblocks-server/test/storage/model-based/storage-model.ts b/packages/liveblocks-server/test/storage/model-based/storage-model.ts index 2f51abcf8fd..c674b954495 100644 --- a/packages/liveblocks-server/test/storage/model-based/storage-model.ts +++ b/packages/liveblocks-server/test/storage/model-based/storage-model.ts @@ -70,6 +70,7 @@ class ApplyOpCommand implements fc.Command { case OpCode.CREATE_LIST: case OpCode.CREATE_MAP: case OpCode.CREATE_REGISTER: + case OpCode.CREATE_FILE: return model.availableParentNodeIds.has(this.op.parentId); case OpCode.DELETE_CRDT: @@ -104,7 +105,8 @@ class ApplyOpCommand implements fc.Command { break; case OpCode.CREATE_REGISTER: - // Don't register the register as a potential parent ID + case OpCode.CREATE_FILE: + // Don't register leaf nodes as potential parent IDs break; case OpCode.CREATE_LIST: @@ -152,6 +154,8 @@ class ApplyOpCommand implements fc.Command { return "CreateMapOp"; case OpCode.CREATE_REGISTER: return "CreateRegisterOp"; + case OpCode.CREATE_FILE: + return "CreateFileOp"; case OpCode.SET_PARENT_KEY: return "SetParentKeyOp"; case OpCode.UPDATE_OBJECT: diff --git a/tools/liveblocks-cli/package.json b/tools/liveblocks-cli/package.json index 17b15535193..ed19c4feff5 100644 --- a/tools/liveblocks-cli/package.json +++ b/tools/liveblocks-cli/package.json @@ -46,7 +46,7 @@ "typescript": "^5.9.3" }, "dependencies": { - "@liveblocks/core": "v3.21.0-private3", + "@liveblocks/core": "3.23.0-file1", "@liveblocks/query-parser": "workspace:^", "@liveblocks/server": "workspace:^", "@liveblocks/zenrouter": "^1.2.0", diff --git a/tools/liveblocks-cli/src/dev-server/db/BunSQLiteDriver.ts b/tools/liveblocks-cli/src/dev-server/db/BunSQLiteDriver.ts index 34f85ac1954..77909628b98 100644 --- a/tools/liveblocks-cli/src/dev-server/db/BunSQLiteDriver.ts +++ b/tools/liveblocks-cli/src/dev-server/db/BunSQLiteDriver.ts @@ -21,6 +21,7 @@ import type { IUserInfo, Json, JsonObject, + LiveFileData, NodeStream, PlainLsonObject, Relax, @@ -76,7 +77,7 @@ type NodeRow = } | { id: string; - type: CrdtType.OBJECT | CrdtType.REGISTER; + type: CrdtType.OBJECT | CrdtType.REGISTER | CrdtType.FILE; parent_id: string; parent_key: string; jdata: jstring; @@ -100,6 +101,14 @@ function rowToSerializedCrdt(row: NodeRow): SerializedCrdt { case CrdtType.REGISTER: return { type, parentId, parentKey, data: parseJson(jdata) }; + case CrdtType.FILE: + return { + type, + parentId, + parentKey, + data: parseJson(jdata), + }; + case CrdtType.LIST: case CrdtType.MAP: return { type, parentId, parentKey }; @@ -199,7 +208,7 @@ function sanitize_illegalNodes(db: Database): void { WHERE (c.type = ${CrdtType.REGISTER} AND p.type = ${CrdtType.OBJECT}) OR - p.type = ${CrdtType.REGISTER}` + p.type IN (${CrdtType.REGISTER}, ${CrdtType.FILE})` ) .all(); @@ -348,7 +357,9 @@ function upsert_node(db: Database, id: string, node: SerializedCrdt): void { const parentId = id === "root" ? null : (node.parentId ?? null); const parentKey = id === "root" ? null : (node.parentKey ?? null); const jdata = - node.type === CrdtType.OBJECT || node.type === CrdtType.REGISTER + node.type === CrdtType.OBJECT || + node.type === CrdtType.REGISTER || + node.type === CrdtType.FILE ? JSON.stringify(node.data) : null; @@ -426,6 +437,10 @@ function set_child( throw new Error("Cannot add register under object"); } + if (parentNode.type === CrdtType.FILE) { + throw new Error("Cannot add child under file"); + } + const conflictingSiblingId = get_child_at(db, node.parentId, node.parentKey); if (conflictingSiblingId !== id) { const hasConflictingData = hasStaticDataAt(parentNode, node.parentKey); @@ -549,11 +564,11 @@ export class BunSQLiteDriver implements IStorageDriver { `CREATE TABLE IF NOT EXISTS nodes ( id TEXT NOT NULL PRIMARY KEY, - type INTEGER NOT NULL CHECK (type >= 0 AND type <= 3), - -- ^^^^^^^ 0=LiveObject, 1=LiveList, 2=LiveMap, 3=Register + type INTEGER NOT NULL CHECK (type >= 0 AND type <= 5), + -- ^^^^^^^ 0=LiveObject, 1=LiveList, 2=LiveMap, 3=Register, 4=LiveText, 5=LiveFile parent_id TEXT, -- NULL only for root parent_key TEXT, -- NULL only for root - jdata TEXT, -- JSON data for LiveObject and Register; NULL for LiveList/LiveMap + jdata TEXT, -- JSON data for LiveObject, Register, LiveText, and LiveFile; NULL for LiveList/LiveMap UNIQUE (parent_id, parent_key), @@ -564,6 +579,14 @@ export class BunSQLiteDriver implements IStorageDriver { CHECK (id != 'root' OR (parent_id IS NULL AND parent_key IS NULL)), CHECK (id = 'root' OR (parent_id IS NOT NULL AND parent_key IS NOT NULL)), + -- Types must have the correct/expected jdata + CHECK (type != 0 OR jdata IS NOT NULL), -- LiveObject must have jdata + CHECK (type != 1 OR jdata IS NULL), -- LiveList must NOT have jdata + CHECK (type != 2 OR jdata IS NULL), -- LiveMap must NOT have jdata + CHECK (type != 3 OR jdata IS NOT NULL), -- Register must have jdata (even "null" is stored as JSON string) + CHECK (type != 4 OR jdata IS NOT NULL), -- LiveText must have jdata + CHECK (type != 5 OR jdata IS NOT NULL), -- LiveFile must have jdata + -- Foreign key: parent_id must reference an existing node FOREIGN KEY (parent_id) REFERENCES nodes (id) ON DELETE RESTRICT ) STRICT` @@ -774,7 +797,9 @@ export class BunSQLiteDriver implements IStorageDriver { const parentId = id === "root" ? null : (node.parentId ?? null); const parentKey = id === "root" ? null : (node.parentKey ?? null); const jdata = - node.type === CrdtType.OBJECT || node.type === CrdtType.REGISTER + node.type === CrdtType.OBJECT || + node.type === CrdtType.REGISTER || + node.type === CrdtType.FILE ? JSON.stringify(node.data) : null; insertStm.run(id, node.type, parentId, parentKey, jdata); diff --git a/tools/liveblocks-cli/src/dev-server/db/rooms.ts b/tools/liveblocks-cli/src/dev-server/db/rooms.ts index 32a0e4b02fb..62645537cdd 100644 --- a/tools/liveblocks-cli/src/dev-server/db/rooms.ts +++ b/tools/liveblocks-cli/src/dev-server/db/rooms.ts @@ -54,9 +54,9 @@ export type ClientMeta = JsonObject; // Module state // --------------------------------------------------------------------------- -// Bumped to v2 when the per-room node storage moved to a normalized SQLite -// schema. Old v1 data is left untouched on disk. -const DEFAULT_BASE_PATH = ".liveblocks/v2"; +// Bumped to v3 when the per-room node storage schema added new CRDT node +// types. Old v1/v2 data is left untouched on disk. +const DEFAULT_BASE_PATH = ".liveblocks/v3"; let basePath = DEFAULT_BASE_PATH; let isEphemeral = false; let _initializedDb: Database | null = null; diff --git a/tools/liveblocks-cli/src/dev-server/routes/client-api.ts b/tools/liveblocks-cli/src/dev-server/routes/client-api.ts index 97c27e8b89f..b99a926feb4 100644 --- a/tools/liveblocks-cli/src/dev-server/routes/client-api.ts +++ b/tools/liveblocks-cli/src/dev-server/routes/client-api.ts @@ -149,6 +149,12 @@ zen.route("POST /v2/c/rooms//text-metadata", () => { zen.route("POST /v2/c/rooms//attachments//multipart//complete", () => NOT_IMPLEMENTED()); zen.route("DELETE /v2/c/rooms//attachments//multipart/", () => NOT_IMPLEMENTED()); zen.route("POST /v2/c/rooms//attachments/presigned-urls", () => NOT_IMPLEMENTED()); + zen.route("PUT /v2/c/rooms//storage/files//upload/", () => NOT_IMPLEMENTED()); + zen.route("POST /v2/c/rooms//storage/files//multipart/", () => NOT_IMPLEMENTED()); + zen.route("PUT /v2/c/rooms//storage/files//multipart//", () => NOT_IMPLEMENTED()); + zen.route("POST /v2/c/rooms//storage/files//multipart//complete", () => NOT_IMPLEMENTED()); + zen.route("DELETE /v2/c/rooms//storage/files//multipart/", () => NOT_IMPLEMENTED()); + zen.route("POST /v2/c/rooms//storage/files/presigned-urls", () => NOT_IMPLEMENTED()); zen.route("POST /v2/c/rooms//send-message", () => NOT_IMPLEMENTED()); zen.route("GET /v2/c/rooms//storage", () => NOT_IMPLEMENTED()); zen.route("GET /v2/c/rooms//versions", () => NOT_IMPLEMENTED()); diff --git a/tools/liveblocks-cli/src/dev-server/routes/rest-api.ts b/tools/liveblocks-cli/src/dev-server/routes/rest-api.ts index 9f894f86943..c6436e5bccc 100644 --- a/tools/liveblocks-cli/src/dev-server/routes/rest-api.ts +++ b/tools/liveblocks-cli/src/dev-server/routes/rest-api.ts @@ -719,6 +719,12 @@ zen.route( zen.route("POST /v2/rooms//threads//comments//add-reaction", () => NOT_IMPLEMENTED()); zen.route("POST /v2/rooms//threads//comments//remove-reaction", () => NOT_IMPLEMENTED()); zen.route("GET /v2/rooms//attachments/", () => NOT_IMPLEMENTED()); + zen.route("PUT /v2/rooms//storage/files//upload/", () => NOT_IMPLEMENTED()); + zen.route("POST /v2/rooms//storage/files//multipart/", () => NOT_IMPLEMENTED()); + zen.route("PUT /v2/rooms//storage/files//multipart//", () => NOT_IMPLEMENTED()); + zen.route("POST /v2/rooms//storage/files//multipart//complete", () => NOT_IMPLEMENTED()); + zen.route("DELETE /v2/rooms//storage/files//multipart/", () => NOT_IMPLEMENTED()); + zen.route("GET /v2/rooms//storage/files/", () => NOT_IMPLEMENTED()); zen.route("GET /v2/rooms//users//notification-settings", () => NOT_IMPLEMENTED()); zen.route("GET /v2/rooms//users//subscription-settings", () => NOT_IMPLEMENTED()); zen.route("POST /v2/rooms//users//notification-settings", () => NOT_IMPLEMENTED()); diff --git a/tools/liveblocks-cli/test/plugins/BunSQLiteDriver.test.ts b/tools/liveblocks-cli/test/plugins/BunSQLiteDriver.test.ts index 9074700a09a..5c5204d7e9b 100644 --- a/tools/liveblocks-cli/test/plugins/BunSQLiteDriver.test.ts +++ b/tools/liveblocks-cli/test/plugins/BunSQLiteDriver.test.ts @@ -40,7 +40,9 @@ function initBunSQLite(dbPath: string, rawNodes: NodeMap): void { const parentId = id === "root" ? null : (node.parentId ?? null); const parentKey = id === "root" ? null : (node.parentKey ?? null); const jdata = - node.type === CrdtType.OBJECT || node.type === CrdtType.REGISTER + node.type === CrdtType.OBJECT || + node.type === CrdtType.REGISTER || + node.type === CrdtType.FILE ? JSON.stringify(node.data) : null; diff --git a/tools/liveblocks-cli/test/plugins/_generateFullTestSuite.ts b/tools/liveblocks-cli/test/plugins/_generateFullTestSuite.ts index 2f84926841a..df18e5b0602 100644 --- a/tools/liveblocks-cli/test/plugins/_generateFullTestSuite.ts +++ b/tools/liveblocks-cli/test/plugins/_generateFullTestSuite.ts @@ -38,14 +38,17 @@ import type { Awaitable, ChildStorageNode, + FileStorageNode, Json, JsonObject, ListStorageNode, + LiveFileData, MapStorageNode, NodeMap, NodeStream, ObjectStorageNode, PlainLson, + PlainLsonFile, PlainLsonList, PlainLsonMap, PlainLsonObject, @@ -68,6 +71,7 @@ import { } from "@liveblocks/core"; import type { ClientWireOp, + CreateFileOp, CreateListOp, CreateMapOp, CreateObjectOp, @@ -213,6 +217,7 @@ export function selfCheck(storage: Storage): void { assert(driver.get_node(parentId) !== undefined, `Node ${quote(id)} points to ${quote(parentId)}, but no such node exists`); // prettier-ignore assert(driver.get_node(parentId)?.type !== CrdtType.REGISTER, `Node ${quote(parentId)} has children (e.g. ${quote(id)}), but is a register node`); // prettier-ignore + assert(driver.get_node(parentId)?.type !== CrdtType.FILE, `Node ${quote(parentId)} has children (e.g. ${quote(id)}), but is a file node`); // prettier-ignore assert(node.type !== CrdtType.REGISTER || driver.get_node(parentId)?.type !== CrdtType.OBJECT, `Node ${quote(id)} is a REGISTER with value ${JSON.stringify((node as SerializedRegister).data)}, but appears as a child under OBJECT node ${quote(parentId)}. Would have expected OBJECT to have this value as a static data attribute under key ${quote(parentKey)}.`); // prettier-ignore @@ -276,17 +281,18 @@ type InfiniteStream = Omit, "next"> & { next(): { value: T; done?: false }; }; -// Check if a PlainLson value is a container (LiveObject/LiveList/LiveMap) +// Check if a PlainLson value is a container (LiveObject/LiveList/LiveMap/LiveFile) function isPlainLsonContainer( value: PlainLson -): value is PlainLsonObject | PlainLsonList | PlainLsonMap { +): value is PlainLsonObject | PlainLsonList | PlainLsonMap | PlainLsonFile { return ( typeof value === "object" && value !== null && "liveblocksType" in value && (value.liveblocksType === "LiveObject" || value.liveblocksType === "LiveList" || - value.liveblocksType === "LiveMap") + value.liveblocksType === "LiveMap" || + value.liveblocksType === "LiveFile") ); } @@ -355,6 +361,16 @@ function plainLsonTreeToNodeMap( for (const [key, child] of Object.entries(plainLson.data)) { recurse(child, id, key); } + } else if (plainLson.liveblocksType === "LiveFile") { + result.push([ + id, + { + type: CrdtType.FILE, + parentId, + parentKey, + data: plainLson.data, + }, + ]); } } @@ -420,6 +436,15 @@ export function register( return [id, { type: CrdtType.REGISTER, parentId, parentKey, data }]; } +export function file( + id: string, + parentId: string, + parentKey: string, + data: LiveFileData +): FileStorageNode { + return [id, { type: CrdtType.FILE, parentId, parentKey, data }]; +} + export function updateObjectOp( id: string, data: Partial, @@ -489,6 +514,27 @@ export function createRegisterOp( }; } +export function createFileOp( + id: string, + parentId: string, + parentKey: string, + data: LiveFileData, + intent?: "set" | "push", + deletedId?: string, + opId = nanoid() +): CreateFileOp & HasOpId { + return { + opId, + id, + type: OpCode.CREATE_FILE, + parentId, + parentKey, + data, + intent, + deletedId, + }; +} + export function createMapOp( id: string, parentId: string, @@ -599,6 +645,14 @@ export function generateArbitraries() { ) ), + liveFileData: (): fc.Arbitrary => + fc.record({ + id: fc.stringMatching(/^fl_[0-9A-Za-z_-]{21}$/), + name: fc.string({ minLength: 1 }), + size: fc.nat(), + mimeType: fc.string({ minLength: 1 }), + }), + rootNodeTuple: () => fc.tuple<["root", SerializedRootObject]>( fc.constant("root"), @@ -648,6 +702,12 @@ export function generateArbitraries() { data: arb.json(), parentId: nonObjectParentId, parentKey: arb.key(), + }), + fc.record({ + type: fc.constant(CrdtType.FILE), + data: arb.liveFileData(), + parentId, + parentKey: arb.key(), }) ) .filter(wouldNotOverwriteDefaultDoc); @@ -812,13 +872,15 @@ export function generateArbitraries() { PlainLsonObject: PlainLsonObject; PlainLsonList: PlainLsonList; PlainLsonMap: PlainLsonMap; + PlainLsonFile: PlainLsonFile; Json: Json; }>((tie) => ({ PlainLson: fc.oneof( { arbitrary: tie("Json"), weight: 1, depthIdentifier, maxDepth, depthSize }, // prettier-ignore { arbitrary: tie("PlainLsonObject"), weight: 2, depthIdentifier, maxDepth, depthSize }, // prettier-ignore { arbitrary: tie("PlainLsonMap"), weight: 1, depthIdentifier, maxDepth, depthSize }, // prettier-ignore - { arbitrary: tie("PlainLsonList"), weight: 1, depthIdentifier, maxDepth, depthSize } // prettier-ignore + { arbitrary: tie("PlainLsonList"), weight: 1, depthIdentifier, maxDepth, depthSize }, // prettier-ignore + { arbitrary: tie("PlainLsonFile"), weight: 1, depthIdentifier, maxDepth, depthSize } // prettier-ignore ), PlainLsonObject: fc.record({ liveblocksType: fc.constant("LiveObject" as const), @@ -832,6 +894,10 @@ export function generateArbitraries() { liveblocksType: fc.constant("LiveList" as const), data: fc.array(tie("PlainLson"), { maxLength: options?.maxLength ?? 5 }), // prettier-ignore }), + PlainLsonFile: fc.record({ + liveblocksType: fc.constant("LiveFile" as const), + data: arb.liveFileData(), + }), Json: arb.json(), })).PlainLsonObject; }, @@ -916,6 +982,27 @@ export function generateArbitraries() { fc.option(arb.key(), { freq: 10, nil: undefined }), }), + createFileOp: (options?: { + id?: fc.Arbitrary; + parentId?: fc.Arbitrary; + parentKey?: fc.Arbitrary; + data?: fc.Arbitrary; + intent?: fc.Arbitrary<"set" | undefined>; + deletedId?: fc.Arbitrary; + }) => + fc.record({ + type: fc.constant(OpCode.CREATE_FILE), + opId: arb.opId(), + id: options?.id ?? arb.key(), + parentId: options?.parentId ?? arb.key(), + parentKey: options?.parentKey ?? arb.parentKey(), + data: options?.data ?? arb.liveFileData(), + intent: options?.intent ?? arb.intent(), + deletedId: + options?.deletedId ?? + fc.option(arb.key(), { freq: 10, nil: undefined }), + }), + createOp: (options?: { id?: fc.Arbitrary; parentId?: fc.Arbitrary; @@ -927,7 +1014,8 @@ export function generateArbitraries() { arb.createListOp(options), arb.createMapOp(options), arb.createObjectOp(options), - arb.createRegisterOp(options) + arb.createRegisterOp(options), + arb.createFileOp(options) ), deleteCrdtOpArb: () => @@ -1324,6 +1412,32 @@ export function generateFullTestSuite(config: { ); })); + test("set_child: throws if adding a child under a file", () => + runTest((driver) => { + resetToDefaultNodes(driver); + + const [, fileNode] = file("0:file", "root", "file", { + id: "fl_123456789012345678901", + name: "file.txt", + size: 5, + mimeType: "text/plain", + }); + + driver.set_child("0:file", fileNode); + + expectToThrow( + () => + driver.set_child("0:child", { + type: CrdtType.LIST, + parentId: "0:file", + parentKey: "child", + }), + /cannot add child under file/i + ); + + expect(driver.get_node("0:child")).toBe(undefined); + })); + test("get_child_at: returns child id after set", () => runTest((driver) => { driver.DANGEROUSLY_reset_nodes(EMPTY_DOC);