feat(export): Add client for /export endpoint

.github/workflows/test.yaml

@@ -92,7 +92,7 @@ jobs:
       fail-fast: false
       matrix:
         WEAVIATE_VERSION:
-          ["1.32.24", "1.33.11", "1.34.7", "1.35.2", "1.36.9", "1.37.2"]
+          ["1.32.24", "1.33.11", "1.34.7", "1.35.2", "1.36.9", "1.37.7"]
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
 

src/it/java/io/weaviate/containers/Weaviate.java

@@ -45,7 +45,7 @@ public enum Version {
     V134(1, 34, 7),
     V135(1, 35, 2),
     V136(1, 36, 9),
-    V137(1, 37, 2);
+    V137(1, 37, 7);
 
     public final SemanticVersion semver;
 
@@ -228,6 +228,13 @@ public Builder withFilesystemBackup(String fsPath) {
       return this;
     }
 
+    public Builder withExportPath(String fsPath) {
+      withFilesystemBackup("/tmp/backups");
+      environment.put("EXPORT_ENABLED", "true");
+      environment.put("EXPORT_DEFAULT_PATH", fsPath);
+      return this;
+    }
+
     public Builder withAdminUsers(String... admins) {
       adminUsers.addAll(Arrays.asList(admins));
       return this;

src/it/java/io/weaviate/integration/CollectionsITest.java

@@ -18,9 +18,9 @@
 import io.weaviate.client6.v1.api.collections.Quantization;
 import io.weaviate.client6.v1.api.collections.ReferenceProperty;
 import io.weaviate.client6.v1.api.collections.Replication;
+import io.weaviate.client6.v1.api.collections.Replication.AsyncReplicationConfig;
 import io.weaviate.client6.v1.api.collections.TextAnalyzer;
 import io.weaviate.client6.v1.api.collections.Tokenization;
-import io.weaviate.client6.v1.api.collections.Replication.AsyncReplicationConfig;
 import io.weaviate.client6.v1.api.collections.VectorConfig;
 import io.weaviate.client6.v1.api.collections.VectorIndex;
 import io.weaviate.client6.v1.api.collections.config.PropertyIndexType;
@@ -464,7 +464,7 @@ public void testTextAnalyzer() throws Exception {
 
   @Test
   public void test_asyncReplicationConfig() throws IOException {
-    Weaviate.Version.latest().orSkip();
+    Weaviate.Version.V137.orSkip();
 
     // Arrange
     var nsThings = ns("Things");
@@ -496,10 +496,10 @@ public void test_asyncReplicationConfig() throws IOException {
         .extracting(CollectionConfig::replication)
         .extracting(Replication::asyncReplicationConfig)
         .returns(1, AsyncReplicationConfig::hashTreeHeight)
-        .returns(2, AsyncReplicationConfig::maxWorkers)
+        .returns(null, AsyncReplicationConfig::maxWorkers) // Deprecated
         .returns(3, AsyncReplicationConfig::frequencyMillis)
         .returns(4, AsyncReplicationConfig::frequencyMillisWhilePropagating)
-        .returns(5, AsyncReplicationConfig::aliveNodesCheckingFrequencyMillis)
+        .returns(null, AsyncReplicationConfig::aliveNodesCheckingFrequencyMillis)
         .returns(6, AsyncReplicationConfig::loggingFrequencySeconds)
         .returns(7, AsyncReplicationConfig::diffBatchSize)
         .returns(8, AsyncReplicationConfig::diffPerNodeTimeoutSeconds)

src/it/java/io/weaviate/integration/ExportITest.java

@@ -0,0 +1,85 @@
+package io.weaviate.integration;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeoutException;
+
+import org.assertj.core.api.Assertions;
+import org.assertj.core.api.InstanceOfAssertFactories;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import io.weaviate.ConcurrentTest;
+import io.weaviate.client6.v1.api.WeaviateClient;
+import io.weaviate.client6.v1.api.export.Export;
+import io.weaviate.client6.v1.api.export.ExportStatus;
+import io.weaviate.client6.v1.api.export.FileFormat;
+import io.weaviate.client6.v1.api.export.ShardExportProgress;
+import io.weaviate.containers.Weaviate;
+
+public class ExportITest extends ConcurrentTest {
+  private static final WeaviateClient client = Weaviate.custom()
+      .withExportPath("/tmp/export").build()
+      .getClient();
+
+  @BeforeClass
+  public static void __() {
+    Weaviate.Version.V137.orSkip();
+  }
+
+  @Test
+  public void test_lifecycle() throws IOException, TimeoutException {
+    // Arrange
+    String nsA = ns("A"), nsB = ns("B"), nsC = ns("C");
+    String exportId = ns("export_1").toLowerCase();
+    String backend = "filesystem";
+
+    var collectionA = client.collections.create(nsA);
+    var collectionB = client.collections.create(nsB);
+    var collectionC = client.collections.create(nsC);
+
+    // Insert some data
+    for (var c : List.of(collectionA, collectionB, collectionC)) {
+      var resp = c.data.insertMany(Map.of(), Map.of(), Map.of());
+      Assertions.assertThat(resp.errors()).isEmpty();
+    }
+
+    // Act: start export
+    var started = client.export.create(exportId, backend, FileFormat.PARQUET,
+        export -> export
+            .includeCollections(nsA, nsB));
+
+    // Assert
+    Assertions.assertThat(started)
+        .as("created export operation")
+        .returns(exportId, Export::id)
+        .returns(backend, Export::backend)
+        .returns(ExportStatus.STARTED, Export::status)
+        .returns(null, Export::error)
+        .extracting(Export::includesCollections, InstanceOfAssertFactories.list(String.class))
+        .containsOnly(nsA, nsB);
+
+    // Act: await export competion
+    var completed = started.waitForCompletion(client);
+
+    // Assert
+    Assertions.assertThat(completed)
+        .as("await export completion")
+        .returns(exportId, Export::id)
+        .returns(backend, Export::backend)
+        .returns(ExportStatus.SUCCESS, Export::status)
+        .returns(null, Export::error)
+        .extracting(Export::includesCollections, InstanceOfAssertFactories.list(String.class))
+        .containsOnly(nsA, nsB);
+
+    Assertions.assertThat(completed)
+        .extracting(Export::shardStatus, InstanceOfAssertFactories.map(String.class,
+            Object.class))
+        .allSatisfy((__, shards) -> {
+          Assertions.assertThat(shards)
+              .asInstanceOf(InstanceOfAssertFactories.map(String.class, ShardExportProgress.class))
+              .isNotEmpty();
+        });
+  }
+}

src/main/java/io/weaviate/client6/v1/api/WeaviateClient.java

@@ -7,6 +7,7 @@
 import io.weaviate.client6.v1.api.backup.WeaviateBackupClient;
 import io.weaviate.client6.v1.api.cluster.WeaviateClusterClient;
 import io.weaviate.client6.v1.api.collections.WeaviateCollectionsClient;
+import io.weaviate.client6.v1.api.export.WeaviateExportClient;
 import io.weaviate.client6.v1.api.rbac.groups.WeaviateGroupsClient;
 import io.weaviate.client6.v1.api.rbac.roles.WeaviateRolesClient;
 import io.weaviate.client6.v1.api.rbac.users.WeaviateUsersClient;
@@ -42,6 +43,9 @@ public class WeaviateClient implements AutoCloseable {
   /** Client for {@code /backups} endpoints for managing backups. */
   public final WeaviateBackupClient backup;
 
+  /** Client for {@code /export} endpoints for managing exports. */
+  public final WeaviateExportClient export;
+
   /**
    * Client for {@code /authz/roles} endpoints for managing RBAC roles.
    */
@@ -124,6 +128,7 @@ public WeaviateClient(Config config) {
     this.grpcTransport = new DefaultGrpcTransport(grpcOpt);
     this.alias = new WeaviateAliasClient(restTransport);
     this.backup = new WeaviateBackupClient(restTransport);
+    this.export = new WeaviateExportClient(restTransport);
     this.tokenize = new WeaviateTokenizeClient(restTransport);
     this.collections = new WeaviateCollectionsClient(restTransport, grpcTransport);
     this.roles = new WeaviateRolesClient(restTransport);

src/main/java/io/weaviate/client6/v1/api/WeaviateClientAsync.java

@@ -9,6 +9,7 @@
 import io.weaviate.client6.v1.api.cluster.WeaviateClusterClientAsync;
 import io.weaviate.client6.v1.api.collections.WeaviateCollectionsClient;
 import io.weaviate.client6.v1.api.collections.WeaviateCollectionsClientAsync;
+import io.weaviate.client6.v1.api.export.WeaviateExportClientAsync;
 import io.weaviate.client6.v1.api.rbac.groups.WeaviateGroupsClientAsync;
 import io.weaviate.client6.v1.api.rbac.roles.WeaviateRolesClientAsync;
 import io.weaviate.client6.v1.api.rbac.users.WeaviateUsersClientAsync;
@@ -56,6 +57,9 @@ public class WeaviateClientAsync implements AutoCloseable {
   /** Client for {@code /backups} endpoints for managing backups. */
   public final WeaviateBackupClientAsync backup;
 
+  /** Client for {@code /export} endpoints for managing exports. */
+  public final WeaviateExportClientAsync export;
+
   /**
    * Client for {@code /nodes} and {@code /replication} endpoints
    * for managing replication and sharding.
@@ -128,6 +132,7 @@ public WeaviateClientAsync(Config config) {
     this.grpcTransport = new DefaultGrpcTransport(grpcOpt);
     this.alias = new WeaviateAliasClientAsync(restTransport);
     this.backup = new WeaviateBackupClientAsync(restTransport);
+    this.export = new WeaviateExportClientAsync(restTransport);
     this.tokenize = new WeaviateTokenizeClientAsync(restTransport);
     this.roles = new WeaviateRolesClientAsync(restTransport);
     this.groups = new WeaviateGroupsClientAsync(restTransport);

src/main/java/io/weaviate/client6/v1/api/collections/Replication.java

@@ -93,7 +93,11 @@ public Builder hashTreeHeight(int hashTreeHeight) {
         return this;
       }
 
-      /** Maximum number of async replication workers. */
+      /**
+       * Maximum number of async replication workers.
+       *
+       * @deprecated This paramter should be controled server-side.
+       */
       public Builder maxWorkers(int maxWorkers) {
         this.maxWorkers = maxWorkers;
         return this;
@@ -117,7 +121,11 @@ public Builder frequencyMillisWhilePropagating(int frequencyMillisWhilePropagati
         return this;
       }
 
-      /** Interval in milliseconds at which liveness of target nodes is checked." */
+      /**
+       * Interval in milliseconds at which liveness of target nodes is checked.
+       *
+       * @deprecated This parameter should be controled server-side.
+       */
       public Builder aliveNodesCheckingFrequencyMillis(int aliveNodesCheckingFrequencyMillis) {
         this.aliveNodesCheckingFrequencyMillis = aliveNodesCheckingFrequencyMillis;
         return this;

src/main/java/io/weaviate/client6/v1/api/export/CancelExportRequest.java

@@ -0,0 +1,18 @@
+package io.weaviate.client6.v1.api.export;
+
+import java.util.Collections;
+
+import io.weaviate.client6.v1.internal.rest.Endpoint;
+import io.weaviate.client6.v1.internal.rest.SimpleEndpoint;
+
+public record CancelExportRequest(String exportId, String backend) {
+
+  public static Endpoint<CancelExportRequest, Void> _ENDPOINT = SimpleEndpoint.sideEffect(
+      __ -> "DELETE",
+      request -> {
+        var cancel = (CancelExportRequest) request;
+        return "/export/" + cancel.backend + "/" + cancel.exportId;
+      },
+      request -> Collections.emptyMap())
+      .allowStatus(409);
+}

src/main/java/io/weaviate/client6/v1/api/export/CreateExportRequest.java

@@ -0,0 +1,88 @@
+package io.weaviate.client6.v1.api.export;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.function.Function;
+
+import com.google.gson.annotations.SerializedName;
+
+import io.weaviate.client6.v1.internal.ObjectBuilder;
+import io.weaviate.client6.v1.internal.json.JSON;
+import io.weaviate.client6.v1.internal.rest.Endpoint;
+import io.weaviate.client6.v1.internal.rest.SimpleEndpoint;
+
+public record CreateExportRequest(ExportCreate config, String backend) {
+
+  public static Endpoint<CreateExportRequest, Export> _ENDPOINT = new SimpleEndpoint<>(
+      request -> "POST",
+      request -> "/export/" + request.backend,
+      request -> Collections.emptyMap(),
+      request -> JSON.serialize(request.config),
+      (statusCode, response) -> JSON.deserialize(response, Export.class));
+
+  public static record ExportCreate(
+      @SerializedName("id") String id,
+      @SerializedName("file_format") FileFormat fileFormat,
+      @SerializedName("include") List<String> includeCollections,
+      @SerializedName("exclude") List<String> excludeCollections) {
+
+    public static ExportCreate of(String exportId, FileFormat fileFormat) {
+      return of(exportId, fileFormat, ObjectBuilder.identity());
+    }
+
+    public static ExportCreate of(String exportId, FileFormat fileFormat,
+        Function<Builder, ObjectBuilder<ExportCreate>> fn) {
+      return fn.apply(new Builder(exportId, fileFormat)).build();
+    }
+
+    public ExportCreate(Builder builder) {
+      this(
+          builder.exportId,
+          builder.fileFormat,
+          builder.includeCollections,
+          builder.excludeCollections);
+    }
+
+    public static class Builder implements ObjectBuilder<ExportCreate> {
+      private final String exportId;
+      private FileFormat fileFormat;
+
+      private final List<String> includeCollections = new ArrayList<>();
+      private final List<String> excludeCollections = new ArrayList<>();
+
+      public Builder(String exportId, FileFormat fileFormat) {
+        this.exportId = exportId;
+        this.fileFormat = fileFormat;
+      }
+
+      /** Collection that should be included in the export. */
+      public Builder includeCollections(String... includeCollections) {
+        return includeCollections(Arrays.asList(includeCollections));
+      }
+
+      /** Collection that should be included in the export. */
+      public Builder includeCollections(List<String> includeCollections) {
+        this.includeCollections.addAll(includeCollections);
+        return this;
+      }
+
+      /** Collection that should be excluded from the export. */
+      public Builder excludeCollections(String... excludeCollections) {
+        return excludeCollections(Arrays.asList(excludeCollections));
+      }
+
+      /** Collection that should be excluded from the export. */
+      public Builder excludeCollections(List<String> excludeCollections) {
+        this.excludeCollections.addAll(excludeCollections);
+        return this;
+      }
+
+      @Override
+      public ExportCreate build() {
+        return new ExportCreate(this);
+      }
+    }
+  }
+}