Run DirectTls handshake user callbacks off the epoll pump thread

src/Servers/Kestrel/Core/src/DirectTlsEndpointOptions.cs

@@ -38,8 +38,10 @@ public sealed class DirectTlsEndpointOptions
     /// allocated so it carries the same connection id that will later serve the request); the second is the
     /// requested SNI host name, or <see langword="null"/> when the client did not send one.
     /// <para>
-    /// This callback runs synchronously on the epoll worker thread that owns the connection and must not block.
-    /// A blocking or long-running callback stalls the handshake and I/O of every connection assigned to that worker.
+    /// The callback is invoked on the <see cref="ThreadPool"/>, so a slow one does not hold up the handshakes or
+    /// I/O of other connections. It does still delay this connection, and the time spent in it counts against
+    /// <see cref="HandshakeTimeout"/> - a callback that overruns that budget costs this connection its handshake.
+    /// Prefer fast, non-blocking work regardless: each concurrent invocation occupies a thread-pool thread.
     /// </para>
     /// </remarks>
     public Func<ConnectionContext?, string?, X509Certificate2?>? ServerCertificateSelector { get; set; }
@@ -65,8 +67,10 @@ public sealed class DirectTlsEndpointOptions
     /// when it produced no <see cref="SslPolicyErrors"/>.
     /// </summary>
     /// <remarks>
-    /// This callback runs synchronously on the epoll worker thread that owns the connection and must not block.
-    /// A blocking or long-running callback stalls the handshake and I/O of every connection assigned to that worker.
+    /// The callback is invoked on the <see cref="ThreadPool"/>, so a slow one does not hold up the handshakes or
+    /// I/O of other connections. It does still delay this connection, and the time spent in it counts against
+    /// <see cref="HandshakeTimeout"/> - a callback that overruns that budget costs this connection its handshake.
+    /// Prefer fast, non-blocking work regardless: each concurrent invocation occupies a thread-pool thread.
     /// </remarks>
     public Func<X509Certificate2, X509Chain?, SslPolicyErrors, bool>? ClientCertificateValidation { get; set; }
 
@@ -79,8 +83,10 @@ public sealed class DirectTlsEndpointOptions
     /// (for example with <c>ToArray()</c>) if they must outlive the call. The first argument is the
     /// <see cref="ConnectionContext"/> for the connection being negotiated.
     /// <para>
-    /// This callback runs synchronously on the epoll worker thread that owns the connection and must not block.
-    /// A blocking or long-running callback stalls the handshake and I/O of every connection assigned to that worker.
+    /// The callback is invoked on the <see cref="ThreadPool"/>, so a slow one does not hold up the handshakes or
+    /// I/O of other connections. It does still delay this connection, and the time spent in it counts against
+    /// <see cref="HandshakeTimeout"/> - a callback that overruns that budget costs this connection its handshake.
+    /// Prefer fast, non-blocking work regardless: each concurrent invocation occupies a thread-pool thread.
     /// </para>
     /// </remarks>
     public Action<ConnectionContext, ReadOnlySequence<byte>>? TlsClientHelloBytesCallback { get; set; }
@@ -137,7 +143,7 @@ public TimeSpan HandshakeTimeout
 
     /// <summary>
     /// The HTTP protocols (ALPN) advertised for this endpoint,
-    /// sourced from <see cref="ListenOptions.Protocols"/> after the endpoint has been configured. 
+    /// sourced from <see cref="ListenOptions.Protocols"/> after the endpoint has been configured.
     /// </summary>
     internal HttpProtocols HttpProtocols { get; set; } = HttpProtocols.Http1AndHttp2;
 }

src/Servers/Kestrel/Transport.DirectTls/src/ClientCertificateValidator.cs

@@ -21,17 +21,17 @@ internal static class ClientCertificateValidator
     /// <see cref="SslStream"/>'s default server-side client-certificate validation policy.
     /// </summary>
     /// <remarks>
-    /// The returned chain is configured for use on the pump thread:
+    /// The chain build and the endpoint callback run on the thread pool, never on the pump thread.
     /// <list type="bullet">
     /// <item>
-    /// <see cref="X509RevocationMode.NoCheck"/> avoids blocking the pump thread on CRL/OCSP network I/O and
-    /// matches the transport default (<c>CheckCertificateRevocation == false</c>).
+    /// <see cref="X509RevocationMode.NoCheck"/> avoids blocking on CRL/OCSP network I/O and matches the
+    /// transport default (<c>CheckCertificateRevocation == false</c>).
     /// </item>
     /// <item>
     /// <see cref="X509ChainPolicy.DisableCertificateDownloads"/> is <see langword="true"/> so the chain
-    /// engine never makes synchronous AIA fetches for missing intermediates. <see cref="X509Chain.Build"/>
-    /// runs on the pump thread, so an attacker-supplied leaf whose Authority Information Access extension
-    /// points at an unreachable or slow URL could otherwise stall every connection this pump owns.
+    /// engine never makes synchronous AIA fetches for missing intermediates. Otherwise a supplied
+    /// leaf whose Authority Information Access extension points at an unreachable or slow URL would occupy a
+    /// thread pool thread for the lifetime of that fetch, one per connection.
     /// <see cref="SslStream"/> sets the same flag on the server side for this reason.
     /// Legitimate clients send their intermediates in the handshake, which are supplied here via <paramref name="intermediates"/>.
     /// </item>

src/Servers/Kestrel/Transport.DirectTls/src/Connection/DirectTlsConnection.cs

@@ -199,7 +199,7 @@ internal void AbortBeforeStart()
             _logger.LogDebug(ex, "Failed to dispose half-open connection backend for fd={Fd}", _connectionState.Fd);
         }
 
-        _connectionClosedTokenSource.Cancel();
+        CancelConnectionClosedToken();
 
         // A half-open handshake never reached mTLS validation, so _ownedClientCertificate is normally null
         // here; dispose defensively (no-op when null) to keep both teardown paths symmetric.
@@ -404,6 +404,19 @@ private void OnTlsFatalError(Exception ex)
         Abort(new ConnectionAbortedException("TLS connection error", ex));
     }
 
+    private void CancelConnectionClosedToken()
+    {
+        try
+        {
+            _connectionClosedTokenSource.Cancel();
+        }
+        catch (Exception ex)
+        {
+            // a throwing callback must not escape and abandon the rest of the teardown
+            _logger.LogError(0, ex, $"Unexpected exception in {nameof(DirectTlsConnection)}.{nameof(CancelConnectionClosedToken)}.");
+        }
+    }
+
     public override async ValueTask DisposeAsync()
     {
         // Thread-safe check: only one call to DisposeAsync proceeds
@@ -452,7 +465,7 @@ public override async ValueTask DisposeAsync()
         }
 
         // 7. Signal connection closed
-        _connectionClosedTokenSource.Cancel();
+        CancelConnectionClosedToken();
         _connectionClosedTokenSource.Dispose();
 
         // 8. Dispose the accepted client certificate. The runtime's TlsSession transferred ownership of the

src/Servers/Kestrel/Transport.DirectTls/src/Connection/DirectTlsConnectionListener.cs

@@ -31,6 +31,9 @@ internal sealed class DirectTlsConnectionListener : IConnectionListener
     private readonly TlsEventPumpPool _pumpPool;
     private readonly Action<ConnectionContext, ReadOnlySequence<byte>>? _clientHelloCallback;
 
+    // Whether the endpoint supplied a ServerCertificateSelector, i.e. whether resolving the TLS context can run user code
+    private readonly bool _serverCertificateSelectorConfigured;
+
     // Native OpenSSL server credentials (bootstrap + per-SNI contexts) owned by this listener. Disposed once,
     // at the end of DisposeAsync, after the pump threads are joined. Null only in tests that don't wire them.
     private readonly IDisposable? _ownedServerContexts;
@@ -48,7 +51,6 @@ internal sealed class DirectTlsConnectionListener : IConnectionListener
     private readonly IHostApplicationLifetime _appLifetime;
 
     private int _fatalErrorReported;
-    private Exception? _fatalError;
 
     public EndPoint EndPoint { get; private set; }
 
@@ -62,7 +64,8 @@ public DirectTlsConnectionListener(
         MemoryPool<byte> memoryPool,
         IHostApplicationLifetime applicationLifetime,
         Action<ConnectionContext, ReadOnlySequence<byte>>? clientHelloCallback = null,
-        IDisposable? ownedServerContexts = null)
+        IDisposable? ownedServerContexts = null,
+        bool serverCertificateSelectorConfigured = true)
     {
         ArgumentNullException.ThrowIfNull(tlsContext);
         ArgumentNullException.ThrowIfNull(applicationLifetime);
@@ -75,6 +78,7 @@ public DirectTlsConnectionListener(
         _tlsContext = tlsContext;
         _contextResolver = contextResolver;
         _clientHelloCallback = clientHelloCallback;
+        _serverCertificateSelectorConfigured = serverCertificateSelectorConfigured;
         _ownedServerContexts = ownedServerContexts;
         _appLifetime = applicationLifetime;
         EndPoint = endpoint;
@@ -133,7 +137,8 @@ internal void Bind()
             _options.MaxWriteBufferSize ?? 0,
             OnPumpFatalError,
             _clientHelloCallback,
-            _connectionTracker);
+            _connectionTracker,
+            _serverCertificateSelectorConfigured);
 
         _logger.LogInformation("DirectTls listener started with EPOLLEXCLUSIVE worker accept");
     }
@@ -149,8 +154,6 @@ internal void OnPumpFatalError(Exception error)
             return;
         }
 
-        // Publish the error before completing the channel so AcceptAsync observes it when it wakes.
-        _fatalError = error;
         _logger.LogCritical(error, "A DirectTls pump thread failed unrecoverably; stopping the application.");
 
         _readyConnections.Writer.TryComplete(error);
@@ -159,26 +162,16 @@ internal void OnPumpFatalError(Exception error)
 
     public async ValueTask<ConnectionContext?> AcceptAsync(CancellationToken cancellationToken = default)
     {
-        try
-        {
-            // Wait for a connection that has completed handshake
-            var connection = await _readyConnections.Reader.ReadAsync(cancellationToken);
-            _connectionTracker.ReleaseHandshake();
-            return connection;
-        }
-        catch (ChannelClosedException)
+        while (await _readyConnections.Reader.WaitToReadAsync(cancellationToken))
         {
-            if (_fatalError is not null)
+            if (_readyConnections.Reader.TryRead(out var connection))
             {
-                throw _fatalError;
+                _connectionTracker.ReleaseHandshake();
+                return connection;
             }
-
-            return null;
-        }
-        catch (OperationCanceledException)
-        {
-            return null;
         }
+
+        return null;
     }
 
     public async ValueTask DisposeAsync()
@@ -201,14 +194,7 @@ public async ValueTask DisposeAsync()
         // Drain any remaining connections from the channel
         while (_readyConnections.Reader.TryRead(out var connection))
         {
-            try
-            {
-                await connection.DisposeAsync();
-            }
-            catch
-            {
-                // Ignore errors during cleanup
-            }
+            await connection.DisposeAsync();
         }
 
         // This listener owns its pump pool; stop the pump threads and release their epoll fds. Bound the wait so

src/Servers/Kestrel/Transport.DirectTls/src/DirectTlsTransportFactory.cs

@@ -118,11 +118,11 @@ public async ValueTask<IConnectionListener> BindAsync(EndPoint endpoint, Cancell
                         $"No server certificate was resolved for SNI host name '{hostName}'.");
                 }
 
-                var context = contextCache.GetOrAdd(certificate, cert =>
+                if (!contextCache.TryGetValue(certificate, out var context))
                 {
                     var authenticationOptions = new SslServerAuthenticationOptions
                     {
-                        ServerCertificate = cert,
+                        ServerCertificate = certificate,
                         EnabledSslProtocols = endpointOptions.SslProtocols,
                         ApplicationProtocols = applicationProtocols,
                         ClientCertificateRequired = requireClientCertificate,
@@ -133,8 +133,16 @@ public async ValueTask<IConnectionListener> BindAsync(EndPoint endpoint, Cancell
                         authenticationOptions.RemoteCertificateValidationCallback = clientCertificateValidation;
                     }
 
-                    return TlsContext.CreateServer(authenticationOptions);
-                });
+                    var candidate = TlsContext.CreateServer(authenticationOptions);
+                    context = contextCache.GetOrAdd(certificate, candidate);
+
+                    // Multiple threads racing to add the same certificate can create multiple candidates,
+                    // so make sure we dont leak TlsContext (with cert/key handles) by disposing the non-cached candidate.
+                    if (!ReferenceEquals(context, candidate))
+                    {
+                        candidate.Dispose();
+                    }
+                }
 
                 return (context, clientCertificateValidation);
             };
@@ -163,7 +171,8 @@ public async ValueTask<IConnectionListener> BindAsync(EndPoint endpoint, Cancell
             memoryPool,
             _applicationLifetime,
             clientHelloCallback,
-            ownedServerContexts);
+            ownedServerContexts,
+            serverCertificateSelectorConfigured: endpointOptions.ServerCertificateSelector is not null);
 
         _logger.LogInformation("DirectTls listener bound for endpoint {Endpoint}.", endpoint);
 

src/Servers/Kestrel/Transport.DirectTls/src/Interop/NativeTls.cs

@@ -32,6 +32,14 @@ internal static partial class NativeTls
     // Reporting EAGAIN/EBADF/EINTR as errno return values lets the accept loop drain without exceptions.
     [LibraryImport(LIBC, SetLastError = true)] public static partial int accept4(int sockfd, IntPtr addr, IntPtr addrlen, int flags);
 
+    // eventfd: the pump's cross-thread wakeup. A thread pool thread may write to same fd and make it wake up on epoll_wait.
+    // The counter is drained with a single 8-byte read (EFD_NONBLOCK makes an empty read return EAGAIN rather than block the pump).
+    [LibraryImport(LIBC, SetLastError = true)] public static partial int eventfd(uint initval, int flags);
+
+    [LibraryImport(LIBC, SetLastError = true)] public static partial nint read(int fd, ref long buf, nuint count);
+
+    [LibraryImport(LIBC, SetLastError = true)] public static partial nint write(int fd, ref long buf, nuint count);
+
     [LibraryImport(LIBC, SetLastError = true)]
     private static partial int epoll_ctl(int epfd, int op, int fd, ref EpollEventPacked ev);
 
@@ -109,6 +117,10 @@ public EpollEvent this[int index]
     public const int SOCK_CLOEXEC = 0x80000;   // Linux O_CLOEXEC
     public const int EPOLL_CLOEXEC = 0x80000;  // Linux O_CLOEXEC
 
+    // eventfd flags (Linux eventfd2). Same values as O_NONBLOCK / O_CLOEXEC.
+    public const int EFD_NONBLOCK = 0x800;
+    public const int EFD_CLOEXEC = 0x80000;
+
     // errno values the accept loop distinguishes (Linux asm-generic/errno-base.h). EWOULDBLOCK == EAGAIN on Linux.
     public const int EINTR = 4;    // interrupted by a signal before a connection was accepted - retry
     public const int EBADF = 9;    // listen fd closed underneath the pump during shutdown