[1/2] ci: stabilize GitHub Actions build (ANTLR jar + flusher shutdown)

.github/workflows/ci.yml

@@ -42,14 +42,17 @@ jobs:
           restore-keys: |
             nuget-${{ runner.os }}-
 
+      - name: Verify vendored ANTLR tool JAR
+        run: test -f build/antlr/antlr4-4.13.1-complete.jar
+
       - name: Restore
         run: dotnet restore OutWit.slnx
 
       - name: Build
         run: dotnet build OutWit.slnx --configuration Release --no-restore
 
       - name: Test
-        run: dotnet test OutWit.slnx --configuration Release --no-build --verbosity normal --logger "trx;LogFileName=test-results.trx" --results-directory TestResults
+        run: dotnet test OutWit.slnx --configuration Release --no-build --verbosity normal --filter "Category!=Performance" --logger "trx;LogFileName=test-results.trx" --results-directory TestResults
 
       - name: Upload Test Results
         if: always()

OutWit.slnx

@@ -21,6 +21,7 @@
     <Project Path="Sources/Core/OutWit.Database.Core.BouncyCastle/OutWit.Database.Core.BouncyCastle.csproj" Id="e244e45c-4d29-4c06-9bbb-68f33d803880" />
     <Project Path="Sources/Core/OutWit.Database.Core.IndexedDb/OutWit.Database.Core.IndexedDb.csproj" Id="0B08187A-589F-4B18-9840-F4F3940F9DA8" />
     <Project Path="Sources/Core/OutWit.Database.Core.IndexedDb.Tests/OutWit.Database.Core.IndexedDb.Tests.csproj" Id="6CB5FC26-DE05-488F-90AC-A6B7AA4955BC" />
+    <Project Path="Sources/Core/OutWit.Database.Native/OutWit.Database.Native.csproj" Id="A1B2C3D4-E5F6-7890-ABCD-EF1234567890" />
   </Folder>
   <Folder Name="/Sources/Engine/">
     <Project Path="Sources/Engine/OutWit.Database.Parser/OutWit.Database.Parser.csproj" Id="5d480879-e75e-4d6a-bd07-09be725177a5" />

Sources/Core/OutWit.Database.Core/Indexes/IndexMetadataJsonContext.cs

@@ -0,0 +1,7 @@
+using System.Text.Json.Serialization;
+
+namespace OutWit.Database.Core.Indexes;
+
+[JsonSerializable(typeof(IndexMetadata))]
+[JsonSerializable(typeof(List<string>))]
+internal sealed partial class IndexMetadataJsonContext : JsonSerializerContext;

Sources/Core/OutWit.Database.Core/Indexes/IndexMetadataStore.cs

@@ -56,7 +56,7 @@ public void SaveIndex(string name, bool isUnique)
 
             var metadata = new IndexMetadata { Name = name, IsUnique = isUnique };
             var key = CreateKey(name);
-            var value = JsonSerializer.SerializeToUtf8Bytes(metadata);
+            var value = JsonSerializer.SerializeToUtf8Bytes(metadata, IndexMetadataJsonContext.Default.IndexMetadata);
 
             m_store.Put(key, value);
 
@@ -84,7 +84,7 @@ public void SaveIndex(string name, bool isUnique)
             if (value == null)
                 return null;
 
-            return JsonSerializer.Deserialize<IndexMetadata>(value);
+            return JsonSerializer.Deserialize(value, IndexMetadataJsonContext.Default.IndexMetadata);
         }
 
         /// <summary>
@@ -150,7 +150,7 @@ public async ValueTask SaveIndexAsync(string name, bool isUnique, CancellationTo
 
             var metadata = new IndexMetadata { Name = name, IsUnique = isUnique };
             var key = CreateKey(name);
-            var value = JsonSerializer.SerializeToUtf8Bytes(metadata);
+            var value = JsonSerializer.SerializeToUtf8Bytes(metadata, IndexMetadataJsonContext.Default.IndexMetadata);
 
             await m_store.PutAsync(key, value, cancellationToken).ConfigureAwait(false);
 
@@ -177,7 +177,7 @@ public async ValueTask SaveIndexAsync(string name, bool isUnique, CancellationTo
             if (value == null)
                 return null;
 
-            return JsonSerializer.Deserialize<IndexMetadata>(value);
+            return JsonSerializer.Deserialize(value, IndexMetadataJsonContext.Default.IndexMetadata);
         }
 
         /// <summary>
@@ -244,7 +244,7 @@ private List<string> LoadCatalog()
 
             try
             {
-                return JsonSerializer.Deserialize<List<string>>(value) ?? [];
+                return JsonSerializer.Deserialize(value, IndexMetadataJsonContext.Default.ListString) ?? [];
             }
             catch
             {
@@ -254,7 +254,7 @@ private List<string> LoadCatalog()
 
         private void SaveCatalog(List<string> catalog)
         {
-            var value = JsonSerializer.SerializeToUtf8Bytes(catalog);
+            var value = JsonSerializer.SerializeToUtf8Bytes(catalog, IndexMetadataJsonContext.Default.ListString);
             m_store.Put(CATALOG_KEY, value);
         }
 
@@ -271,7 +271,7 @@ private async ValueTask<List<string>> LoadCatalogAsync(CancellationToken cancell
 
             try
             {
-                return JsonSerializer.Deserialize<List<string>>(value) ?? [];
+                return JsonSerializer.Deserialize(value, IndexMetadataJsonContext.Default.ListString) ?? [];
             }
             catch
             {
@@ -281,7 +281,7 @@ private async ValueTask<List<string>> LoadCatalogAsync(CancellationToken cancell
 
         private async ValueTask SaveCatalogAsync(List<string> catalog, CancellationToken cancellationToken = default)
         {
-            var value = JsonSerializer.SerializeToUtf8Bytes(catalog);
+            var value = JsonSerializer.SerializeToUtf8Bytes(catalog, IndexMetadataJsonContext.Default.ListString);
             await m_store.PutAsync(CATALOG_KEY, value, cancellationToken).ConfigureAwait(false);
         }
 

Sources/Core/OutWit.Database.Core/LSM/LsmMemTableFlusher.cs

@@ -270,10 +270,9 @@ public void Dispose()
         m_disposed = true;
 
         m_flushChannel.Writer.Complete();
-        m_cts.Cancel();
-
         Task.WaitAll(m_flushTasks, TimeSpan.FromSeconds(10));
 
+        m_cts.Cancel();
         m_cts.Dispose();
     }
 
@@ -287,10 +286,9 @@ public async ValueTask DisposeAsync()
         m_disposed = true;
 
         m_flushChannel.Writer.Complete();
-        await m_cts.CancelAsync();
-
         await Task.WhenAll(m_flushTasks).WaitAsync(TimeSpan.FromSeconds(10));
 
+        await m_cts.CancelAsync();
         m_cts.Dispose();
     }
 

Sources/Core/OutWit.Database.Core/LSM/LsmParallelCompactor.cs

@@ -272,10 +272,9 @@ public void Dispose()
         m_disposed = true;
 
         m_jobChannel.Writer.Complete();
-        m_cts.Cancel();
-
         Task.WaitAll(m_workerTasks, TimeSpan.FromSeconds(30));
 
+        m_cts.Cancel();
         m_cts.Dispose();
     }
 
@@ -289,10 +288,9 @@ public async ValueTask DisposeAsync()
         m_disposed = true;
 
         m_jobChannel.Writer.Complete();
-        await m_cts.CancelAsync();
-
         await Task.WhenAll(m_workerTasks).WaitAsync(TimeSpan.FromSeconds(30));
 
+        await m_cts.CancelAsync();
         m_cts.Dispose();
     }
 

Sources/Core/OutWit.Database.Core/LSM/LsmParallelWriter.cs

@@ -322,6 +322,20 @@ private async Task MergeLoopAsync()
         {
             // Channel closed during shutdown
         }
+        finally
+        {
+            DrainPendingBuffers(reader);
+        }
+    }
+
+    private void DrainPendingBuffers(ChannelReader<(LsmWriteBuffer Buffer, TaskCompletionSource<bool>? Completion)> reader)
+    {
+        var buffersToMerge = new List<(LsmWriteBuffer Buffer, TaskCompletionSource<bool>? Completion)>();
+        while (reader.TryRead(out var item))
+            buffersToMerge.Add(item);
+
+        if (buffersToMerge.Count > 0)
+            MergeBuffersBatch(buffersToMerge);
     }
 
     /// <summary>
@@ -482,7 +496,6 @@ public void Dispose()
         m_bufferChannel.Writer.Complete();
         m_cts.Cancel();
 
-        // Wait for merge task
         m_mergeTask.Wait(TimeSpan.FromSeconds(5));
 
         // Dispose thread-local buffers
@@ -508,7 +521,6 @@ public async ValueTask DisposeAsync()
         m_bufferChannel.Writer.Complete();
         await m_cts.CancelAsync();
 
-        // Wait for merge task
         await m_mergeTask.WaitAsync(TimeSpan.FromSeconds(5));
 
         // Dispose thread-local buffers

Sources/Core/OutWit.Database.Native.Smoke/OutWit.Database.Native.Smoke.csproj

@@ -0,0 +1,14 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net10.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+    <IsPackable>false</IsPackable>
+    <GeneratePackageOnBuild>false</GeneratePackageOnBuild>
+  </PropertyGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\OutWit.Database.Native\OutWit.Database.Native.csproj" />
+  </ItemGroup>
+</Project>

Sources/Core/OutWit.Database.Native.Smoke/Program.cs

@@ -0,0 +1,65 @@
+using System.Runtime.InteropServices;
+using OutWit.Database.Native;
+
+var path = Path.Combine(Path.GetTempPath(), $"witdb-smoke-{Guid.NewGuid():N}.witdb");
+var mode = args.FirstOrDefault() ?? "pinvoke";
+
+if (mode == "managed")
+{
+    Console.WriteLine($"[managed] Opening {path}");
+    var status = WitDbInterop.Open(path, null, createIfMissing: true, out var handle);
+    Console.WriteLine($"open={status} handle={handle}");
+    if (status != WitDbStatusCode.Ok)
+    {
+        Console.WriteLine(WitDbLastError.GetMessage());
+        return 1;
+    }
+
+    WitDbInterop.Close(handle);
+    Console.WriteLine("ok");
+    return 0;
+}
+
+var publishDll = Path.GetFullPath(Path.Combine(
+    AppContext.BaseDirectory,
+    "..", "..", "..", "..",
+    "OutWit.Database.Native",
+    "bin", "Release", "net10.0", "win-x64", "publish", "witdb.dll"));
+if (!File.Exists(publishDll))
+{
+    publishDll = Environment.GetEnvironmentVariable("WITDB_NATIVE_PATH") ?? publishDll;
+}
+
+Console.WriteLine($"[pinvoke] dll={publishDll}");
+NativeLibrary.SetDllImportResolver(
+    typeof(WitDbNative).Assembly,
+    (_, _, _) => NativeLibrary.Load(publishDll));
+Console.WriteLine($"[pinvoke] Opening {path}");
+var code = WitDbNative.witdb_open(path, null, 1, out var pinvokeHandle);
+Console.WriteLine($"open={code} handle={pinvokeHandle}");
+if (code != 0)
+{
+    var msg = WitDbNative.witdb_last_error_message();
+    Console.WriteLine(Marshal.PtrToStringUTF8(msg));
+    return 1;
+}
+
+WitDbNative.witdb_close(pinvokeHandle);
+Console.WriteLine("ok");
+return 0;
+
+internal static partial class WitDbNative
+{
+    [LibraryImport("witdb", StringMarshalling = StringMarshalling.Utf8)]
+    internal static partial uint witdb_open(
+        string path,
+        string? password,
+        byte create_if_missing,
+        out UIntPtr out_db);
+
+    [LibraryImport("witdb")]
+    internal static partial uint witdb_close(UIntPtr db);
+
+    [LibraryImport("witdb", StringMarshalling = StringMarshalling.Utf8)]
+    internal static partial IntPtr witdb_last_error_message();
+}

Sources/Core/OutWit.Database.Native/OutWit.Database.Native.csproj

@@ -0,0 +1,37 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net10.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+    <PublishAot>true</PublishAot>
+    <NativeLib>Shared</NativeLib>
+    <IsPackable>false</IsPackable>
+    <GeneratePackageOnBuild>false</GeneratePackageOnBuild>
+    <IsAotCompatible>true</IsAotCompatible>
+    <AssemblyName>witdb</AssemblyName>
+    <RootNamespace>OutWit.Database.Native</RootNamespace>
+    <Description>NativeAOT C ABI exports for WitDatabase (pywitdb).</Description>
+    <NoWarn>$(NoWarn);CA2255</NoWarn>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\OutWit.Database.Core\OutWit.Database.Core.csproj" />
+    <ProjectReference Include="..\OutWit.Database.Core.BouncyCastle\OutWit.Database.Core.BouncyCastle.csproj" />
+    <ProjectReference Include="..\..\Engine\OutWit.Database\OutWit.Database.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <None Include="include\witdb.h" Pack="false" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <InternalsVisibleTo Include="OutWit.Database.Native.Smoke" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <TrimmerRootDescriptor Include="trimming.xml" />
+  </ItemGroup>
+
+</Project>

Sources/Core/OutWit.Database.Native/README.md

@@ -0,0 +1,15 @@
+# OutWit.Database.Native
+
+NativeAOT shared library exposing the WitDatabase C ABI (`include/witdb.h`) for **pywitdb**.
+
+## Build
+
+```bash
+dotnet publish Sources/Core/OutWit.Database.Native/OutWit.Database.Native.csproj -c Release -r win-x64
+```
+
+Artifact: `bin/Release/net9.0/win-x64/publish/witdb.dll`
+
+## Consumer
+
+- [AI-Guiders/pywitdb](https://github.com/AI-Guiders/pywitdb) — `ctypes` via `WITDB_NATIVE_PATH` or packaged `native/<rid>/`.