[pull] main from liveblocks:main

packages/liveblocks-server/CHANGELOG.md

@@ -1,5 +1,9 @@
 ## vNEXT (not yet released)
 
+## v1.8.0
+
+- Add full support for LiveFile
+
 ## v1.7.0
 
 - Add `--skip-install` (`-s`) flag to `liveblocks upgrade` to update

packages/liveblocks-server/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@liveblocks/server",
-  "version": "1.7.0",
+  "version": "1.8.0",
   "description": "Liveblocks backend server foundation.",
   "type": "module",
   "main": "./dist/index.js",

packages/liveblocks-server/src/index.ts

@@ -29,6 +29,10 @@ export {
   snapshotToPlainLson_eager,
   snapshotToPlainLson_lazy,
 } from "~/formats/PlainLson";
+export {
+  hasUploadedLivefiles,
+  hasUploadedLivefilesInPlainLson,
+} from "~/livefiles";
 export { makeInMemorySnapshot } from "~/makeInMemorySnapshot";
 export type { MetadataDB } from "~/MetadataDB";
 export { makeMetadataDB } from "~/MetadataDB";

packages/liveblocks-server/src/interfaces/IBlobStore.ts

@@ -0,0 +1,112 @@
+/**
+ * Copyright (c) Liveblocks Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/** Bytes accepted by the store. Uploads stream; tests usually pass a buffer. */
+export type BlobBody = ReadableStream<Uint8Array> | Uint8Array;
+
+/**
+ * Everything the store knows about an object.
+ *
+ * There is deliberately no separate metadata record: a LiveFile's name and
+ * MIME type round-trip through `contentDisposition` and `contentType` on the
+ * object itself, exactly as they do in S3/R2, and its size is a property of the
+ * stored bytes.
+ */
+export type BlobMeta = {
+  size: number;
+  contentType: string;
+  contentDisposition: string;
+};
+
+/** Metadata supplied at write time. Size isn't known until the bytes land. */
+export type BlobMetaInput = Omit<BlobMeta, "size">;
+
+export type UploadedPart = {
+  partNumber: number;
+  etag: string;
+};
+
+/**
+ * Blob storage for bytes that live outside the CRDT tree.
+ *
+ * Requirements for implementors:
+ * - Writes are immediately visible to head()/get().
+ * - put() and completeMultipart() report the size of the bytes actually
+ *   stored, never a caller-supplied figure.
+ */
+export interface IBlobStore {
+  /**
+   * Store `body` under `key`, replacing anything already there, and return the
+   * metadata of what was written — including the observed size.
+   */
+  put(key: string, body: BlobBody, meta: BlobMetaInput): Promise<BlobMeta>;
+
+  /**
+   * Return the metadata for `key`, or undefined if no such object exists.
+   * Callers rely on this to distinguish "uploaded" from "never uploaded", so
+   * implementations must not report partially-written objects.
+   */
+  head(key: string): Promise<BlobMeta | undefined>;
+
+  /** Stream the bytes stored at `key`, or undefined if no such object exists. */
+  get(key: string): Promise<ReadableStream<Uint8Array> | undefined>;
+
+  /** Delete `key`. No-op if it doesn't exist. */
+  delete(key: string): Promise<void>;
+
+  /**
+   * Begin a multipart upload for `key` and return its upload ID. The object at
+   * `key` is not touched until completeMultipart() succeeds.
+   */
+  createMultipart(key: string, meta: BlobMetaInput): Promise<string>;
+
+  /**
+   * Store one part. Parts may arrive in any order and may be re-uploaded; the
+   * last write for a given part number wins. The returned ETag identifies the
+   * part's contents.
+   */
+  uploadPart(
+    key: string,
+    uploadId: string,
+    partNumber: number,
+    body: BlobBody
+  ): Promise<UploadedPart>;
+
+  /**
+   * Assemble the named parts, in ascending part-number order, into the object
+   * at `key`, and discard the upload. Parts not named here are dropped, so the
+   * resulting object may be smaller than everything uploaded.
+   */
+  completeMultipart(
+    key: string,
+    uploadId: string,
+    parts: UploadedPart[]
+  ): Promise<BlobMeta>;
+
+  /** Discard an in-progress multipart upload and its parts. */
+  abortMultipart(key: string, uploadId: string): Promise<void>;
+
+  /**
+   * Return a URL that grants read access to `key` for roughly `ttlSeconds`,
+   * with no further credentials. The SDK hands this straight to the browser,
+   * which fetches it without an Authorization header, so whatever authority it
+   * carries has to be inside the URL.
+   *
+   * The value is opaque to callers: nothing may assume S3 URL shape.
+   */
+  signedGetUrl(key: string, ttlSeconds: number): Promise<string>;
+}

packages/liveblocks-server/src/interfaces/IStorageDriver.ts

@@ -267,6 +267,27 @@ export interface IStorageDriver {
    */
   raw_iter_nodes(): Iterable<[string, SerializedCrdt]>;
 
+  // ---------------------------------------------------------------------------
+  // LiveFile upload receipt APIs
+  // ---------------------------------------------------------------------------
+
+  /**
+   * Record that the bytes for `fileId` have been uploaded, and how many there
+   * are.
+   *
+   * A receipt is what makes a LiveFile referenceable: the Room layer refuses
+   * CREATE_FILE ops for files with no receipt, and implementations MUST use the
+   * recorded size in place of the client-supplied one whenever a FILE node is
+   * written.
+   */
+  put_livefile_upload(fileId: string, size: number): void;
+
+  /**
+   * Return the recorded size for `fileId`, or undefined if no upload has been
+   * recorded.
+   */
+  get_livefile_upload_size(fileId: string): number | undefined;
+
   // ---------------------------------------------------------------------------
   // Metadata APIs (key-value store, isolated from nodes)
   // ---------------------------------------------------------------------------

packages/liveblocks-server/src/interfaces/index.ts

@@ -15,6 +15,13 @@
  * along with this program. If not, see <https://www.gnu.org/licenses/>.
  */
 
+export type {
+  BlobBody,
+  BlobMeta,
+  BlobMetaInput,
+  IBlobStore,
+  UploadedPart,
+} from "./IBlobStore";
 export type { IServerWebSocket } from "./IServerWebSocket";
 export type {
   IReadableSnapshot,

packages/liveblocks-server/src/livefiles.ts

@@ -0,0 +1,81 @@
+/**
+ * Copyright (c) Liveblocks Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import type { PlainLsonObject } from "@liveblocks/core";
+import { ClientMsgCode, CrdtType, OpCode } from "@liveblocks/core";
+
+import { plainLsonToNodeStream } from "~/formats/PlainLson";
+import type { IStorageDriver } from "~/interfaces";
+import type { Op } from "~/protocol";
+
+/**
+ * A LiveFile may only be referenced from Storage once its bytes have actually
+ * been uploaded — otherwise clients would sync a node pointing at nothing, and
+ * the server would have no way to know the file's real size.
+ *
+ * Both checks below are the same rule applied to the two shapes a new file can
+ * arrive in: ops on the wire, and a whole document being installed at once.
+ */
+
+/**
+ * The part of a client message this check reads. Deliberately structural:
+ * callers hold different client-message unions (the Cloudflare worker's also
+ * covers feed messages), and all this needs is "does it carry storage ops".
+ */
+type MessageWithMaybeOps = {
+  readonly type: number;
+  readonly ops?: readonly Op[];
+};
+
+/** True unless some CREATE_FILE op references a file that was never uploaded. */
+export function hasUploadedLivefiles(
+  driver: IStorageDriver,
+  messages: readonly MessageWithMaybeOps[]
+): boolean {
+  for (const message of messages) {
+    if (message.type !== ClientMsgCode.UPDATE_STORAGE) {
+      continue;
+    }
+
+    for (const op of message.ops ?? []) {
+      if (op.type !== OpCode.CREATE_FILE) {
+        continue;
+      }
+
+      if (driver.get_livefile_upload_size(op.data.id) === undefined) {
+        return false;
+      }
+    }
+  }
+  return true;
+}
+
+/** True unless the document contains a LiveFile that was never uploaded. */
+export function hasUploadedLivefilesInPlainLson(
+  driver: IStorageDriver,
+  document: PlainLsonObject
+): boolean {
+  for (const [, node] of plainLsonToNodeStream(document)) {
+    if (
+      node.type === CrdtType.FILE &&
+      driver.get_livefile_upload_size(node.data.id) === undefined
+    ) {
+      return false;
+    }
+  }
+  return true;
+}

packages/liveblocks-server/src/plugins/InMemoryDriver.ts

@@ -162,6 +162,7 @@ export class InMemoryDriver implements IStorageDriver {
   private _leasedSessions: Map<string, LeasedSession>;
   private _feeds: Map<string, Feed>;
   private _feedMessages: Map<string, FeedMessage>; // Key: `${feedId}:${messageId}`
+  private _livefileUploads: Map<string, number>; // Key: fileId, value: size
 
   constructor(options?: {
     initialActor?: number;
@@ -173,6 +174,7 @@ export class InMemoryDriver implements IStorageDriver {
     this._leasedSessions = new Map();
     this._feeds = new Map();
     this._feedMessages = new Map();
+    this._livefileUploads = new Map();
 
     this._nextActor = options?.initialActor ?? -1;
 
@@ -197,10 +199,31 @@ export class InMemoryDriver implements IStorageDriver {
 
     this._nodes.clear();
     for (const [id, node] of plainLsonToNodeStream(doc)) {
-      this._nodes.set(id, node);
+      this._nodes.set(id, this._withUploadedSize(node));
     }
   }
 
+  put_livefile_upload(fileId: string, size: number): void {
+    this._livefileUploads.set(fileId, size);
+  }
+
+  get_livefile_upload_size(fileId: string): number | undefined {
+    return this._livefileUploads.get(fileId);
+  }
+
+  /**
+   * A FILE node's size comes from its upload receipt, never from whatever the
+   * client claimed. Nodes of any other type pass through untouched, as do
+   * files with no receipt — refusing those is the Room layer's job.
+   */
+  private _withUploadedSize<N extends SerializedCrdt>(node: N): N {
+    if (node.type !== CrdtType.FILE) return node;
+    const size = this._livefileUploads.get(node.data.id);
+    return size === undefined
+      ? node
+      : { ...node, data: { ...node.data, size } };
+  }
+
   get_meta(key: string) {
     return this._metadb.get(key);
   }
@@ -553,7 +576,7 @@ export class InMemoryDriver implements IStorageDriver {
   }
 
   set_child(id: string, node: SerializedChild, allowOverwrite?: boolean): void {
-    this.loadedApi.set_child(id, node, allowOverwrite);
+    this.loadedApi.set_child(id, this._withUploadedSize(node), allowOverwrite);
   }
 
   move_sibling(id: string, newPos: Pos): void {