[pull] main from liveblocks:main

packages/liveblocks-server/package.json

@@ -60,8 +60,7 @@
     "tsd": "^0.33.0",
     "tsup": "^8.5.1",
     "typescript": "^5.9.3",
-    "vite-tsconfig-paths": "^6.1.1",
-    "vitest": "^3.2.4"
+    "vitest": "^4.1.4"
   },
   "repository": {
     "type": "git",

packages/liveblocks-server/src/Room.ts

@@ -1739,8 +1739,8 @@ export class Room<RM, SM, CM extends JsonObject, C = undefined> {
         const isV2 = msg.v2;
         const [update, stateVector, snapshotHash] = await Promise.all([
           this.yjsStorage.getYDocUpdate(this.logger, vector, guid, isV2),
-          this.yjsStorage.getYStateVector(guid),
-          this.yjsStorage.getSnapshotHash({ guid, isV2 }),
+          this.yjsStorage.getYStateVector(this.logger, guid),
+          this.yjsStorage.getSnapshotHash(this.logger, { guid, isV2 }),
         ]);
 
         if (update !== null && snapshotHash !== null) {

packages/liveblocks-server/src/YjsStorage.ts

@@ -27,6 +27,24 @@ import type { Logger } from "~/lib/Logger";
 // How many updates to store before compacting
 const UPDATE_COUNT_THRESHOLD = 1_000;
 
+// log 10% of the time a merge would shrink
+const MERGE_SHRINK_WARN_SAMPLE_RATE = 0.1;
+const MERGE_SHRINK_THRESHOLD = 0.8;
+
+// Writes a log if the merge would shrink significantly
+function warnIfYjsMergeShrunk(
+  logger: Logger,
+  oldSize: number,
+  newSize: number
+): void {
+  if (
+    oldSize * MERGE_SHRINK_THRESHOLD > newSize &&
+    Math.random() < MERGE_SHRINK_WARN_SAMPLE_RATE
+  ) {
+    logger.warn(`merged < 80% of sum: ${oldSize} -> ${newSize}`);
+  }
+}
+
 export class YjsStorage {
   private readonly driver: IStorageDriver;
   private readonly updateCountThreshold: number;
@@ -53,8 +71,8 @@ export class YjsStorage {
   // Public API
   // ------------------------------------------------------------------------------------
 
-  public async getYDoc(docId: YDocId): Promise<Y.Doc> {
-    const doc = await this.loadDocByIdIfNotAlreadyLoaded(docId);
+  public async getYDoc(logger: Logger, docId: YDocId): Promise<Y.Doc> {
+    const doc = await this.loadDocByIdIfNotAlreadyLoaded(logger, docId);
     return doc;
   }
 
@@ -86,7 +104,8 @@ export class YjsStorage {
     guid?: Guid,
     isV2: boolean = false
   ): Promise<Uint8Array<ArrayBuffer> | null> {
-    const doc = guid !== undefined ? await this.getYSubdoc(guid) : this.doc;
+    const doc =
+      guid !== undefined ? await this.getYSubdoc(logger, guid) : this.doc;
     if (!doc) {
       return null;
     }
@@ -106,21 +125,28 @@ export class YjsStorage {
     return Y.encodeStateAsUpdate(doc, encodedTargetVector);
   }
 
-  public async getYStateVector(guid?: Guid): Promise<string | null> {
-    const doc = guid !== undefined ? await this.getYSubdoc(guid) : this.doc;
+  public async getYStateVector(
+    logger: Logger,
+    guid?: Guid
+  ): Promise<string | null> {
+    const doc =
+      guid !== undefined ? await this.getYSubdoc(logger, guid) : this.doc;
     if (!doc) {
       return null;
     }
     return Base64.fromUint8Array(Y.encodeStateVector(doc));
   }
 
-  public async getSnapshotHash(options: {
-    guid?: Guid;
-    isV2?: boolean;
-  }): Promise<string | null> {
+  public async getSnapshotHash(
+    logger: Logger,
+    options: {
+      guid?: Guid;
+      isV2?: boolean;
+    }
+  ): Promise<string | null> {
     const doc =
       options.guid !== undefined
-        ? await this.getYSubdoc(options.guid)
+        ? await this.getYSubdoc(logger, options.guid)
         : this.doc;
     if (!doc) {
       return null;
@@ -141,7 +167,8 @@ export class YjsStorage {
     guid?: Guid,
     isV2?: boolean
   ): Promise<{ isUpdated: boolean; snapshotHash: string }> {
-    const doc = guid !== undefined ? await this.getYSubdoc(guid) : this.doc;
+    const doc =
+      guid !== undefined ? await this.getYSubdoc(logger, guid) : this.doc;
     if (!doc) {
       throw new Error(`YDoc with guid ${guid} not found`);
     }
@@ -174,22 +201,25 @@ export class YjsStorage {
     }
   }
 
-  public loadDocByIdIfNotAlreadyLoaded(docId: YDocId): Promise<Y.Doc> {
+  public loadDocByIdIfNotAlreadyLoaded(
+    logger: Logger,
+    docId: YDocId
+  ): Promise<Y.Doc> {
     let loaded$ = this.initPromisesById.get(docId);
     let doc = docId === ROOT_YDOC_ID ? this.doc : this.findYSubdocByGuid(docId);
     if (!doc) {
       // An API call can load a subdoc without the root doc (this._doc) being loaded, we account for that by just instantiating a doc here.
       doc = new Y.Doc();
     }
     if (loaded$ === undefined) {
-      loaded$ = this._loadYDocFromDurableStorage(doc, docId);
+      loaded$ = this._loadYDocFromDurableStorage(logger, doc, docId);
       this.initPromisesById.set(docId, loaded$);
     }
     return loaded$;
   }
 
-  public async load(_logger: Logger): Promise<void> {
-    await this.loadDocByIdIfNotAlreadyLoaded(ROOT_YDOC_ID);
+  public async load(logger: Logger): Promise<void> {
+    await this.loadDocByIdIfNotAlreadyLoaded(logger, ROOT_YDOC_ID);
   }
 
   /**
@@ -245,15 +275,18 @@ export class YjsStorage {
   };
 
   private _loadYDocFromDurableStorage = async (
+    logger: Logger,
     doc: Y.Doc,
     docId: YDocId
   ): Promise<Y.Doc> => {
     const docUpdates = Object.fromEntries(
       await this.driver.iter_y_updates(docId)
     );
     const updates = Object.values(docUpdates);
+    const beforeSize = updates.reduce((acc, update) => acc + update.length, 0);
     const newupdate = Y.mergeUpdates(updates);
     const storedKeys = Object.keys(docUpdates);
+    warnIfYjsMergeShrunk(logger, beforeSize, newupdate.length);
     Y.applyUpdate(doc, newupdate);
     // after compaction, there will only be one unique key.
     if (this.shouldCompact(storedKeys)) {
@@ -290,12 +323,12 @@ export class YjsStorage {
   }
 
   // gets a subdoc, it will be loaded if not already loaded
-  private async getYSubdoc(guid: Guid): Promise<Y.Doc | null> {
+  private async getYSubdoc(logger: Logger, guid: Guid): Promise<Y.Doc | null> {
     const subdoc = this.findYSubdocByGuid(guid);
     if (!subdoc) {
       return null;
     }
-    await this.loadDocByIdIfNotAlreadyLoaded(guid);
+    await this.loadDocByIdIfNotAlreadyLoaded(logger, guid);
     return subdoc;
   }
 

packages/liveblocks-server/test/plugins/_generateFullTestSuite.ts

@@ -5500,7 +5500,7 @@ export function generateFullTestSuite<TDriver extends IStorageDriver>(config: {
       const yjsStorage = new YjsStorage(driver);
 
       // Load the root doc
-      await yjsStorage.loadDocByIdIfNotAlreadyLoaded(guid);
+      await yjsStorage.loadDocByIdIfNotAlreadyLoaded(blackHole, guid);
 
       return await callback({ yjsStorage });
     }

packages/liveblocks-server/vitest.config.ts

@@ -1,8 +1,7 @@
 import { defineConfig } from "vitest/config";
-import tsconfigPaths from "vite-tsconfig-paths";
 
 export default defineConfig({
-  plugins: [tsconfigPaths()],
+  resolve: { tsconfigPaths: true },
 
   test: {
     // Will avoid having to put import `describe`, `test`, `expect`, etc in

tools/liveblocks-cli/src/dev-server/routes/rest-api.ts

@@ -17,9 +17,11 @@
 
 import type { Json, JsonObject, PlainLsonObject } from "@liveblocks/core";
 import { QueryParser } from "@liveblocks/query-parser";
-import type { Guid, Logger, YDocId } from "@liveblocks/server";
+import type { Guid, YDocId } from "@liveblocks/server";
 import {
+  ConsoleTarget,
   jsonObjectYolo,
+  Logger,
   ROOT_YDOC_ID,
   snapshotToLossyJson_eager,
   snapshotToNodeStream,
@@ -371,8 +373,8 @@ zen.route("GET /v2/rooms/<roomId>/ydoc", async ({ url, p }) => {
   const type = url.searchParams.get("type") ?? "";
   const ydocId = (url.searchParams.get("guid") ?? ROOT_YDOC_ID) as YDocId;
   const formatting = url.searchParams.get("formatting") !== null;
-
-  const doc = await room.yjsStorage.getYDoc(ydocId);
+  const logger = new Logger(new ConsoleTarget("warning"));
+  const doc = await room.yjsStorage.getYDoc(logger, ydocId);
   const result = yDocToJson(doc, key, formatting, type);
 
   return new Response(JSON.stringify(result), {
@@ -449,7 +451,8 @@ zen.route("GET /v2/rooms/<roomId>/ydoc-binary", async ({ url, p }) => {
   const ydocId = (url.searchParams.get("guid") ?? ROOT_YDOC_ID) as YDocId;
   const encoder = url.searchParams.get("encoder");
 
-  const doc = await room.yjsStorage.getYDoc(ydocId);
+  const logger = new Logger(new ConsoleTarget("warning"));
+  const doc = await room.yjsStorage.getYDoc(logger, ydocId);
   const update =
     encoder === "v2"
       ? Y.encodeStateAsUpdateV2(doc)

tools/liveblocks-cli/test/plugins/_generateFullTestSuite.ts

@@ -5506,7 +5506,7 @@ export function generateFullTestSuite<TDriver extends IStorageDriver>(config: {
       const yjsStorage = new YjsStorage(driver);
 
       // Load the root doc
-      await yjsStorage.loadDocByIdIfNotAlreadyLoaded(guid);
+      await yjsStorage.loadDocByIdIfNotAlreadyLoaded(blackHole, guid);
 
       return await callback({ yjsStorage });
     }