Skip to content
Merged
16 changes: 14 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,8 @@ $ ffmpeg -re -f lavfi -i testsrc2=size=640x360:rate=30 -f lavfi -i sine=frequenc
```

Configuration lives in `src/Spangle.MediaServer/spanglesettings.yaml`
(ports, segment format TS/fMP4, LL-HLS, SRT passphrase), overridable with
`SMS_`-prefixed environment variables.
(ports, segment format TS/fMP4, LL-HLS, SRT passphrase, TLS per listener,
publish allowlist), overridable with `SMS_`-prefixed environment variables.

Family
------
Expand Down Expand Up @@ -83,6 +83,18 @@ Roadmap
errors (CI builds Release, so the gate holds there too). The deliberate
exceptions live in `.editorconfig`, each with a written reason —
spec-mirroring enums, numbered protocol-flow files, wire-struct fields
- [x] TLS on every listener that lacked it: HTTPS for HLS/DASH delivery
(`Http.Tls`), HTTPS for the management port so the Bearer token never
travels in cleartext (`Management.Tls`), and RTMPS (`Rtmp.Tls`) —
PKCS#12 or PEM cert/key, plaintext stays the default
- [x] Publish allowlist out of the box: `Publish.AllowedStreamNames` switches
the built-in authorizer from allow-all to an exact-match allowlist
(a custom `IPublishAuthorizer` in DI still overrides everything); the
metadata-injection endpoint on the public port takes an optional Bearer
token (`Http.MetadataInjectionToken`)
- [x] Ended streams no longer hold memory forever: memory storage frees a
stream's final window after `Hls.EndedStreamTtlSeconds` (default 300;
0 restores keep-until-republish), skipping keys a publisher still owns

### Mid term — features

Expand Down
25 changes: 25 additions & 0 deletions src/Spangle.Core/AllowListPublishAuthorizer.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
using System.Collections.Frozen;

namespace Spangle;

/// <summary>
/// Allows only the configured stream names, matched exactly against the raw name the
/// publisher presented (RTMP publish name / SRT streamid). Contested names keep the
/// last-wins takeover policy of <see cref="DefaultPublishAuthorizer"/>, so a reconnect
/// by a holder of a valid name still just works.
/// </summary>
public sealed class AllowListPublishAuthorizer(IEnumerable<string> allowedStreamNames) : IPublishAuthorizer
{
private readonly FrozenSet<string> _allowed = allowedStreamNames.ToFrozenSet(StringComparer.Ordinal);

public ValueTask<PublishDecision> AuthorizeAsync(PublishRequest request, CancellationToken ct)
{
ArgumentNullException.ThrowIfNull(request);
return new ValueTask<PublishDecision>(
!_allowed.Contains(request.StreamName)
? PublishDecision.Deny
: request.ExistingSession is null
? PublishDecision.Allow
: PublishDecision.Takeover);
}
}
3 changes: 2 additions & 1 deletion src/Spangle.Core/Codecs/NALFileFormat.cs
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,8 @@ public static uint ReadNALULength(ref ReadOnlySequence<byte> buff, int lengthSiz
1 => BufferMarshal.As<byte>(lenBuff),
2 => BufferMarshal.As<BigEndianUInt16>(lenBuff).HostValue,
4 => BufferMarshal.As<BigEndianUInt32>(lenBuff).HostValue,
_ => throw new InvalidDataException(),
_ => throw new InvalidDataException(
$"NALU length size must be 1, 2 or 4 bytes; the decoder configuration declared {lengthSize}"),
};

buff = buff.Slice(lenBuff.End);
Expand Down
3 changes: 3 additions & 0 deletions src/Spangle.Core/PublishSessionRegistry.cs
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,9 @@ internal void Release(string streamName, string sessionId)
}
}

/// <summary>True while a publish session owns the given (sanitized) stream key.</summary>
public bool IsLive(string streamKey) => _sessions.ContainsKey(streamKey);

/// <summary>
/// Snapshots every live publish session for monitoring. Codec and byte counters
/// come from the session's receiver context when it is still alive.
Expand Down
77 changes: 72 additions & 5 deletions src/Spangle.Core/Transport/HLS/HLSStorage.cs
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,21 @@ internal LiveBlobReader(Func<int, (byte[]?, bool, Task)> read)
}
}

/// <summary>
/// Optional capability of a storage backend: dropping streams nobody writes anymore.
/// Only the memory backend implements this — file storage is an archive by design
/// and is never cleaned.
/// </summary>
public interface IEvictingHLSStorage
{
/// <summary>
/// Frees every stream without a write for at least <paramref name="idleFor"/>,
/// except those <paramref name="isLive"/> claims a publisher still owns.
/// Returns the number of streams evicted.
/// </summary>
int EvictIdleStreams(TimeSpan idleFor, Func<string, bool> isLive);
}

/// <summary>The storage area of a single stream (one playlist and its media blobs).</summary>
public interface IHLSStreamStorage
{
Expand All @@ -94,9 +109,10 @@ public interface IHLSStreamStorage
/// <summary>
/// Keeps the live window in process memory. Blobs the playlist trims are freed, so
/// a stream holds roughly its sliding window (a few MB); after the stream ends its
/// final window stays servable until the same stream key publishes again.
/// final window stays servable until it is evicted (see <see cref="EvictIdleStreams"/>)
/// or the same stream key publishes again.
/// </summary>
public sealed class MemoryHLSStorage : IHLSStorage
public sealed class MemoryHLSStorage : IHLSStorage, IEvictingHLSStorage
{
private readonly ConcurrentDictionary<string, IHLSStreamStorage> _streams = new(StringComparer.Ordinal);

Expand All @@ -106,15 +122,59 @@ public IHLSStreamStorage GetStream(string streamKey) =>
public bool TryGetStream(string streamKey, out IHLSStreamStorage stream) =>
_streams.TryGetValue(streamKey, out stream!);

public int EvictIdleStreams(TimeSpan idleFor, Func<string, bool> isLive)
{
ArgumentNullException.ThrowIfNull(isLive);
var evicted = 0;
long now = Environment.TickCount64;
foreach ((string key, IHLSStreamStorage stream) in _streams)
{
if (stream is not StreamStorage storage
|| now - storage.LastWriteTicks < idleFor.TotalMilliseconds
|| isLive(key))
{
continue;
}
// value-checked removal: a concurrent re-publish replaced the entry, leave it be
if (_streams.TryRemove(new KeyValuePair<string, IHLSStreamStorage>(key, stream)))
{
storage.Drop();
evicted++;
}
}
return evicted;
}

public override string ToString() => "memory";

private sealed class StreamStorage : IHLSStreamStorage, ILiveBlobStreamStorage
{
private readonly ConcurrentDictionary<string, byte[]> _blobs = new(StringComparer.Ordinal);
private readonly ConcurrentDictionary<string, GrowingBlob> _growing = new(StringComparer.Ordinal);
private volatile string? _playlist;
private long _lastWriteTicks = Environment.TickCount64;

internal long LastWriteTicks => Volatile.Read(ref _lastWriteTicks);

public void WriteBlob(string name, ReadOnlySpan<byte> content) => _blobs[name] = content.ToArray();
private void Touch() => Volatile.Write(ref _lastWriteTicks, Environment.TickCount64);

/// <summary>Frees everything after eviction; blocked LL-DASH readers are released.</summary>
internal void Drop()
{
foreach (GrowingBlob blob in _growing.Values)
{
blob.Complete();
}
_growing.Clear();
_blobs.Clear();
_playlist = null;
}

public void WriteBlob(string name, ReadOnlySpan<byte> content)
{
Touch();
_blobs[name] = content.ToArray();
}

public void DeleteBlob(string name)
{
Expand All @@ -125,7 +185,11 @@ public void DeleteBlob(string name)
}
}

public void PublishPlaylist(string text) => _playlist = text;
public void PublishPlaylist(string text)
{
Touch();
_playlist = text;
}

public bool TryReadBlob(string name, out ReadOnlyMemory<byte> content)
{
Expand All @@ -142,8 +206,11 @@ public bool TryReadBlob(string name, out ReadOnlyMemory<byte> content)

// ---- growing blobs (LL-DASH) ----

public void AppendBlob(string name, ReadOnlySpan<byte> content) =>
public void AppendBlob(string name, ReadOnlySpan<byte> content)
{
Touch();
_growing.GetOrAdd(name, static _ => new GrowingBlob()).Append(content);
}

public void CompleteBlob(string name)
{
Expand Down
5 changes: 4 additions & 1 deletion src/Spangle.Core/Transport/Rtmp/RtmpReceiverContext.cs
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,10 @@ public sealed class RtmpReceiverContext : ReceiverContextBase<RtmpReceiverContex

// =======================================================================

// TODO 設定とかからもらう
/// <summary>
/// Announced to the peer as WindowAcknowledgementSize / SetPeerBandwidth during
/// the connect sequence. Hosts override this from configuration (Rtmp.Bandwidth).
/// </summary>
public uint Bandwidth = 1500000;

/// <summary>
Expand Down
2 changes: 1 addition & 1 deletion src/Spangle.Core/Transport/SRT/SRTReceiverContext.cs
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,7 @@ private static byte PeekByte(in ReadOnlySequence<byte> buff)
/// found; <paramref name="needMoreData"/> distinguishes "a candidate exists but
/// there aren't enough bytes to verify it yet" (keep the tail for the next read).
/// </summary>
private static bool TryResync(ref ReadOnlySequence<byte> buff, out bool needMoreData)
internal static bool TryResync(ref ReadOnlySequence<byte> buff, out bool needMoreData)
{
ReadOnlySequence<byte> search = buff.Slice(1);
Span<byte> next = stackalloc byte[1]; // hoisted: stackalloc in a loop grows the stack per iteration
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,14 +27,29 @@ public static void AddSpangle(this IServiceCollection services)
&& IPAddress.IsLoopback(address);
},
"Management.Token is required when Management.BindAddress is not a loopback address")
.Validate(static options =>
{
// a TLS block without a certificate is a misconfiguration, not "plaintext please"
static bool Ok(TlsOptions tls) => !tls.Enabled || !string.IsNullOrEmpty(tls.CertificatePath);
return Ok(options.Rtmp.Tls) && Ok(options.Http.Tls) && Ok(options.Management.Tls);
},
"Tls.CertificatePath is required wherever Tls.Enabled is set")
.ValidateOnStart();
services.AddSingleton<RtmpConnectionHandler>();
services.AddSingleton<HLSStreamRegistry>();
services.AddSingleton<PublishSessionRegistry>();
services.AddSingleton<Spangle.Spinner.TimedMetadataHub>();
// apps replace this to implement their own publish policy (deny lists,
// key validation, first-wins, ...); the default is allow-all + last-wins
services.TryAddSingleton<IPublishAuthorizer, DefaultPublishAuthorizer>();
// key validation, first-wins, ...); the built-in policy is allow-all +
// last-wins, or an exact-match allowlist when Publish.AllowedStreamNames is set
services.TryAddSingleton<IPublishAuthorizer>(static provider =>
{
PublishOptions publish =
provider.GetRequiredService<IOptions<SpangleMediaServerOptions>>().Value.Publish;
return publish.AllowedStreamNames.Count > 0
? new AllowListPublishAuthorizer(publish.AllowedStreamNames)
: new DefaultPublishAuthorizer();
});
// Memory (default) serves the live window without touching disk;
// File makes the output an on-disk archive as well
services.TryAddSingleton<IHLSStorage>(static provider =>
Expand All @@ -45,6 +60,7 @@ public static void AddSpangle(this IServiceCollection services)
: new MemoryHLSStorage();
});
services.AddHostedService<SrtIngestService>();
services.AddHostedService<HlsEvictionService>();

// Management surface: log capture, delivery counters, and the stats join
services.AddSingleton<SpangleLogBuffer>();
Expand Down
39 changes: 39 additions & 0 deletions src/Spangle.Extensions.Kestrel/HlsEvictionService.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
using Microsoft.Extensions.Options;
using Spangle.Transport.HLS;
using ZLogger;

namespace Spangle.Extensions.Kestrel;

/// <summary>
/// Frees ended streams from evicting storage backends (memory) once they have been
/// idle for <c>Hls.EndedStreamTtlSeconds</c>. Without this, every distinct stream key
/// ever published would hold its final window in memory for the process lifetime.
/// </summary>
public sealed class HlsEvictionService(
IHLSStorage storage,
PublishSessionRegistry sessions,
IOptions<SpangleMediaServerOptions> options,
ILogger<HlsEvictionService> logger) : BackgroundService
{
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
int ttlSeconds = options.Value.Hls.EndedStreamTtlSeconds;
if (ttlSeconds <= 0 || storage is not IEvictingHLSStorage evicting)
{
return; // eviction disabled, or the backend (file archive) never evicts
}

var ttl = TimeSpan.FromSeconds(ttlSeconds);
// sweeping a few times per TTL keeps the overshoot small without busy-looping
var period = TimeSpan.FromSeconds(Math.Clamp(ttlSeconds / 4.0, 5.0, 60.0));
using var timer = new PeriodicTimer(period);
while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false))
{
int evicted = evicting.EvictIdleStreams(ttl, sessions.IsLive);
if (evicted > 0)
{
logger.ZLogInformation($"Freed {evicted} ended stream(s) idle for {ttl}");
}
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,6 @@
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using Microsoft.Extensions.Options;
using Spangle.Spinner;
Expand Down Expand Up @@ -107,25 +105,14 @@ static async (string streamKey, HttpRequest request, TimedMetadataHub hub) =>
return null; // loopback-only bind is enforced by options validation
}

const string prefix = "Bearer ";
string auth = ctx.Request.Headers.Authorization.ToString();
if (auth.StartsWith(prefix, StringComparison.Ordinal)
&& FixedTimeEquals(auth.AsSpan(prefix.Length).Trim(), opt.Token))
if (TokenGate.Matches(ctx.Request.Headers.Authorization.ToString(), opt.Token))
{
return null;
}
ctx.Response.Headers.WWWAuthenticate = "Bearer";
return Results.Unauthorized();
}

private static bool FixedTimeEquals(ReadOnlySpan<char> provided, string expected)
{
Span<byte> providedUtf8 = provided.Length <= 64 ? stackalloc byte[256] : new byte[provided.Length * 4];
int written = Encoding.UTF8.GetBytes(provided, providedUtf8);
byte[] expectedUtf8 = Encoding.UTF8.GetBytes(expected);
return CryptographicOperations.FixedTimeEquals(providedUtf8[..written], expectedUtf8);
}

private static async IAsyncEnumerable<IReadOnlyList<StreamStatsDto>> StatsFeedAsync(
ManagementStatsService stats, [EnumeratorCancellation] CancellationToken ct)
{
Expand Down
1 change: 1 addition & 0 deletions src/Spangle.Extensions.Kestrel/RtmpConnectionHandler.cs
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ public override async Task OnConnectedAsync(ConnectionContext connection)
var ct = cts.Token;

var receiver = connection.CreateRtmpReceiverContext(ct);
receiver.Bandwidth = options.Value.Rtmp.Bandwidth;
var hlsOptions = options.Value.Hls;
var segmentFormat = hlsOptions.SegmentFormat.ToLowerInvariant() switch
{
Expand Down
Loading