diff --git a/packages/liveblocks-server/package.json b/packages/liveblocks-server/package.json index 796f831ea60..17e204feabf 100644 --- a/packages/liveblocks-server/package.json +++ b/packages/liveblocks-server/package.json @@ -60,8 +60,7 @@ "tsd": "^0.33.0", "tsup": "^8.5.1", "typescript": "^5.9.3", - "vite-tsconfig-paths": "^6.1.1", - "vitest": "^3.2.4" + "vitest": "^4.1.4" }, "repository": { "type": "git", diff --git a/packages/liveblocks-server/src/Room.ts b/packages/liveblocks-server/src/Room.ts index ed36f73e5f1..bb9fcd4d9ac 100644 --- a/packages/liveblocks-server/src/Room.ts +++ b/packages/liveblocks-server/src/Room.ts @@ -1739,8 +1739,8 @@ export class Room { const isV2 = msg.v2; const [update, stateVector, snapshotHash] = await Promise.all([ this.yjsStorage.getYDocUpdate(this.logger, vector, guid, isV2), - this.yjsStorage.getYStateVector(guid), - this.yjsStorage.getSnapshotHash({ guid, isV2 }), + this.yjsStorage.getYStateVector(this.logger, guid), + this.yjsStorage.getSnapshotHash(this.logger, { guid, isV2 }), ]); if (update !== null && snapshotHash !== null) { diff --git a/packages/liveblocks-server/src/YjsStorage.ts b/packages/liveblocks-server/src/YjsStorage.ts index 86ba5b90a6b..b256e366a1e 100644 --- a/packages/liveblocks-server/src/YjsStorage.ts +++ b/packages/liveblocks-server/src/YjsStorage.ts @@ -27,6 +27,24 @@ import type { Logger } from "~/lib/Logger"; // How many updates to store before compacting const UPDATE_COUNT_THRESHOLD = 1_000; +// log 10% of the time a merge would shrink +const MERGE_SHRINK_WARN_SAMPLE_RATE = 0.1; +const MERGE_SHRINK_THRESHOLD = 0.8; + +// Writes a log if the merge would shrink significantly +function warnIfYjsMergeShrunk( + logger: Logger, + oldSize: number, + newSize: number +): void { + if ( + oldSize * MERGE_SHRINK_THRESHOLD > newSize && + Math.random() < MERGE_SHRINK_WARN_SAMPLE_RATE + ) { + logger.warn(`merged < 80% of sum: ${oldSize} -> ${newSize}`); + } +} + export class YjsStorage { private readonly driver: IStorageDriver; private readonly updateCountThreshold: number; @@ -53,8 +71,8 @@ export class YjsStorage { // Public API // ------------------------------------------------------------------------------------ - public async getYDoc(docId: YDocId): Promise { - const doc = await this.loadDocByIdIfNotAlreadyLoaded(docId); + public async getYDoc(logger: Logger, docId: YDocId): Promise { + const doc = await this.loadDocByIdIfNotAlreadyLoaded(logger, docId); return doc; } @@ -86,7 +104,8 @@ export class YjsStorage { guid?: Guid, isV2: boolean = false ): Promise | null> { - const doc = guid !== undefined ? await this.getYSubdoc(guid) : this.doc; + const doc = + guid !== undefined ? await this.getYSubdoc(logger, guid) : this.doc; if (!doc) { return null; } @@ -106,21 +125,28 @@ export class YjsStorage { return Y.encodeStateAsUpdate(doc, encodedTargetVector); } - public async getYStateVector(guid?: Guid): Promise { - const doc = guid !== undefined ? await this.getYSubdoc(guid) : this.doc; + public async getYStateVector( + logger: Logger, + guid?: Guid + ): Promise { + const doc = + guid !== undefined ? await this.getYSubdoc(logger, guid) : this.doc; if (!doc) { return null; } return Base64.fromUint8Array(Y.encodeStateVector(doc)); } - public async getSnapshotHash(options: { - guid?: Guid; - isV2?: boolean; - }): Promise { + public async getSnapshotHash( + logger: Logger, + options: { + guid?: Guid; + isV2?: boolean; + } + ): Promise { const doc = options.guid !== undefined - ? await this.getYSubdoc(options.guid) + ? await this.getYSubdoc(logger, options.guid) : this.doc; if (!doc) { return null; @@ -141,7 +167,8 @@ export class YjsStorage { guid?: Guid, isV2?: boolean ): Promise<{ isUpdated: boolean; snapshotHash: string }> { - const doc = guid !== undefined ? await this.getYSubdoc(guid) : this.doc; + const doc = + guid !== undefined ? await this.getYSubdoc(logger, guid) : this.doc; if (!doc) { throw new Error(`YDoc with guid ${guid} not found`); } @@ -174,7 +201,10 @@ export class YjsStorage { } } - public loadDocByIdIfNotAlreadyLoaded(docId: YDocId): Promise { + public loadDocByIdIfNotAlreadyLoaded( + logger: Logger, + docId: YDocId + ): Promise { let loaded$ = this.initPromisesById.get(docId); let doc = docId === ROOT_YDOC_ID ? this.doc : this.findYSubdocByGuid(docId); if (!doc) { @@ -182,14 +212,14 @@ export class YjsStorage { doc = new Y.Doc(); } if (loaded$ === undefined) { - loaded$ = this._loadYDocFromDurableStorage(doc, docId); + loaded$ = this._loadYDocFromDurableStorage(logger, doc, docId); this.initPromisesById.set(docId, loaded$); } return loaded$; } - public async load(_logger: Logger): Promise { - await this.loadDocByIdIfNotAlreadyLoaded(ROOT_YDOC_ID); + public async load(logger: Logger): Promise { + await this.loadDocByIdIfNotAlreadyLoaded(logger, ROOT_YDOC_ID); } /** @@ -245,6 +275,7 @@ export class YjsStorage { }; private _loadYDocFromDurableStorage = async ( + logger: Logger, doc: Y.Doc, docId: YDocId ): Promise => { @@ -252,8 +283,10 @@ export class YjsStorage { await this.driver.iter_y_updates(docId) ); const updates = Object.values(docUpdates); + const beforeSize = updates.reduce((acc, update) => acc + update.length, 0); const newupdate = Y.mergeUpdates(updates); const storedKeys = Object.keys(docUpdates); + warnIfYjsMergeShrunk(logger, beforeSize, newupdate.length); Y.applyUpdate(doc, newupdate); // after compaction, there will only be one unique key. if (this.shouldCompact(storedKeys)) { @@ -290,12 +323,12 @@ export class YjsStorage { } // gets a subdoc, it will be loaded if not already loaded - private async getYSubdoc(guid: Guid): Promise { + private async getYSubdoc(logger: Logger, guid: Guid): Promise { const subdoc = this.findYSubdocByGuid(guid); if (!subdoc) { return null; } - await this.loadDocByIdIfNotAlreadyLoaded(guid); + await this.loadDocByIdIfNotAlreadyLoaded(logger, guid); return subdoc; } diff --git a/packages/liveblocks-server/test/plugins/_generateFullTestSuite.ts b/packages/liveblocks-server/test/plugins/_generateFullTestSuite.ts index bed1561d27b..1dfd6e79269 100644 --- a/packages/liveblocks-server/test/plugins/_generateFullTestSuite.ts +++ b/packages/liveblocks-server/test/plugins/_generateFullTestSuite.ts @@ -5500,7 +5500,7 @@ export function generateFullTestSuite(config: { const yjsStorage = new YjsStorage(driver); // Load the root doc - await yjsStorage.loadDocByIdIfNotAlreadyLoaded(guid); + await yjsStorage.loadDocByIdIfNotAlreadyLoaded(blackHole, guid); return await callback({ yjsStorage }); } diff --git a/packages/liveblocks-server/vitest.config.ts b/packages/liveblocks-server/vitest.config.ts index 8f902abc115..e273ee62f1a 100644 --- a/packages/liveblocks-server/vitest.config.ts +++ b/packages/liveblocks-server/vitest.config.ts @@ -1,8 +1,7 @@ import { defineConfig } from "vitest/config"; -import tsconfigPaths from "vite-tsconfig-paths"; export default defineConfig({ - plugins: [tsconfigPaths()], + resolve: { tsconfigPaths: true }, test: { // Will avoid having to put import `describe`, `test`, `expect`, etc in diff --git a/tools/liveblocks-cli/src/dev-server/routes/rest-api.ts b/tools/liveblocks-cli/src/dev-server/routes/rest-api.ts index 53efd562dbf..aa8cb25c2a9 100644 --- a/tools/liveblocks-cli/src/dev-server/routes/rest-api.ts +++ b/tools/liveblocks-cli/src/dev-server/routes/rest-api.ts @@ -17,9 +17,11 @@ import type { Json, JsonObject, PlainLsonObject } from "@liveblocks/core"; import { QueryParser } from "@liveblocks/query-parser"; -import type { Guid, Logger, YDocId } from "@liveblocks/server"; +import type { Guid, YDocId } from "@liveblocks/server"; import { + ConsoleTarget, jsonObjectYolo, + Logger, ROOT_YDOC_ID, snapshotToLossyJson_eager, snapshotToNodeStream, @@ -371,8 +373,8 @@ zen.route("GET /v2/rooms//ydoc", async ({ url, p }) => { const type = url.searchParams.get("type") ?? ""; const ydocId = (url.searchParams.get("guid") ?? ROOT_YDOC_ID) as YDocId; const formatting = url.searchParams.get("formatting") !== null; - - const doc = await room.yjsStorage.getYDoc(ydocId); + const logger = new Logger(new ConsoleTarget("warning")); + const doc = await room.yjsStorage.getYDoc(logger, ydocId); const result = yDocToJson(doc, key, formatting, type); return new Response(JSON.stringify(result), { @@ -449,7 +451,8 @@ zen.route("GET /v2/rooms//ydoc-binary", async ({ url, p }) => { const ydocId = (url.searchParams.get("guid") ?? ROOT_YDOC_ID) as YDocId; const encoder = url.searchParams.get("encoder"); - const doc = await room.yjsStorage.getYDoc(ydocId); + const logger = new Logger(new ConsoleTarget("warning")); + const doc = await room.yjsStorage.getYDoc(logger, ydocId); const update = encoder === "v2" ? Y.encodeStateAsUpdateV2(doc) diff --git a/tools/liveblocks-cli/test/plugins/_generateFullTestSuite.ts b/tools/liveblocks-cli/test/plugins/_generateFullTestSuite.ts index 135c6e189f0..b5739edb9fc 100644 --- a/tools/liveblocks-cli/test/plugins/_generateFullTestSuite.ts +++ b/tools/liveblocks-cli/test/plugins/_generateFullTestSuite.ts @@ -5506,7 +5506,7 @@ export function generateFullTestSuite(config: { const yjsStorage = new YjsStorage(driver); // Load the root doc - await yjsStorage.loadDocByIdIfNotAlreadyLoaded(guid); + await yjsStorage.loadDocByIdIfNotAlreadyLoaded(blackHole, guid); return await callback({ yjsStorage }); }