feat(metrics): report scheduler utilization in runtime metrics

lib/sentry/metrics.ex

@@ -37,6 +37,8 @@ defmodule Sentry.Metrics do
 
   Once enabled, a supervised collector reports these gauges every 30 seconds:
 
+    * `elixir.runtime.scheduler.utilization` — busy fraction of scheduler time,
+      as a ratio between `0.0` and `1.0`
     * `elixir.runtime.mem.total`, `elixir.runtime.mem.processes`,
       `elixir.runtime.mem.binary`, `elixir.runtime.mem.ets`,
       `elixir.runtime.mem.atom` — in bytes

lib/sentry/metrics/runtime.ex

@@ -10,7 +10,7 @@ defmodule Sentry.Metrics.Runtime do
   @memory_gauges [:total, :processes, :binary, :ets, :atom]
   @min_interval 1_000
 
-  defstruct [:interval, :attributes]
+  defstruct [:interval, :attributes, :scheduler_sample]
 
   @spec start_link(keyword()) :: GenServer.on_start()
   def start_link(opts) when is_list(opts) do
@@ -22,19 +22,35 @@ defmodule Sentry.Metrics.Runtime do
     interval = normalize_interval(Keyword.fetch!(opts, :interval))
     attributes = version_attributes(Keyword.get(opts, :version_attributes, true))
 
+    _ = :erlang.system_flag(:scheduler_wall_time, true)
+
     schedule_tick(interval)
 
-    {:ok, %__MODULE__{interval: interval, attributes: attributes}}
+    {:ok,
+     %__MODULE__{
+       interval: interval,
+       attributes: attributes,
+       scheduler_sample: scheduler_sample()
+     }}
   end
 
   @impl true
   def handle_info(:tick, %__MODULE__{} = state) do
-    collect_and_emit(state)
+    state = collect_and_emit(state)
     schedule_tick(state.interval)
     {:noreply, state}
   end
 
   defp collect_and_emit(%__MODULE__{} = state) do
+    sample = scheduler_sample()
+
+    gauge(
+      state,
+      "elixir.runtime.scheduler.utilization",
+      utilization(state.scheduler_sample, sample),
+      unit: "ratio"
+    )
+
     memory = :erlang.memory()
 
     Enum.each(@memory_gauges, fn key ->
@@ -45,6 +61,25 @@ defmodule Sentry.Metrics.Runtime do
     # metric buffer's batch size, so without an explicit flush it would sit
     # buffered until unrelated telemetry happened to signal the scheduler.
     TelemetryProcessor.flush()
+
+    %{state | scheduler_sample: sample}
+  end
+
+  defp scheduler_sample do
+    case :erlang.statistics(:scheduler_wall_time) do
+      :undefined -> []
+      sample -> Enum.sort(sample)
+    end
+  end
+
+  defp utilization(previous, current) do
+    {active, total} =
+      Enum.zip(previous, current)
+      |> Enum.reduce({0, 0}, fn {{_, active0, total0}, {_, active1, total1}}, {active, total} ->
+        {active + (active1 - active0), total + (total1 - total0)}
+      end)
+
+    if total > 0, do: active / total, else: 0.0
   end
 
   defp gauge(%__MODULE__{} = state, name, value, opts) do

test/sentry/metrics/runtime_test.exs

@@ -56,11 +56,23 @@ defmodule Sentry.Metrics.RuntimeTest do
     end
   end
 
+  describe "scheduler utilization" do
+    test "reports scheduler utilization as a ratio", %{ref: ref} do
+      collect_once()
+
+      assert metric = find_metric(ref, "elixir.runtime.scheduler.utilization")
+      assert metric["type"] == "gauge"
+      assert metric["unit"] == "ratio"
+      assert metric["value"] >= 0.0
+      assert metric["value"] <= 1.0
+    end
+  end
+
   describe "delivery" do
     test "delivers a whole snapshot from a single collection", %{ref: ref} do
       collect_once()
 
-      assert length(snapshot(ref)) == 5
+      assert length(snapshot(ref)) == 6
     end
   end