feat(metrics): report run queue depth in runtime metrics

lib/sentry/metrics.ex

@@ -39,6 +39,8 @@ defmodule Sentry.Metrics do
 
     * `elixir.runtime.scheduler.utilization` — busy fraction of scheduler time,
       as a ratio between `0.0` and `1.0`
+    * `elixir.runtime.run_queue.total` and `elixir.runtime.run_queue.cpu` — how many
+      processes are waiting to run
     * `elixir.runtime.mem.total`, `elixir.runtime.mem.processes`,
       `elixir.runtime.mem.binary`, `elixir.runtime.mem.ets`,
       `elixir.runtime.mem.atom` — in bytes

lib/sentry/metrics/runtime.ex

@@ -51,6 +51,14 @@ defmodule Sentry.Metrics.Runtime do
       unit: "ratio"
     )
 
+    gauge(
+      state,
+      "elixir.runtime.run_queue.total",
+      :erlang.statistics(:total_run_queue_lengths_all)
+    )
+
+    gauge(state, "elixir.runtime.run_queue.cpu", :erlang.statistics(:total_run_queue_lengths))
+
     memory = :erlang.memory()
 
     Enum.each(@memory_gauges, fn key ->
@@ -82,6 +90,8 @@ defmodule Sentry.Metrics.Runtime do
     if total > 0, do: active / total, else: 0.0
   end
 
+  defp gauge(state, name, value, opts \\ [])
+
   defp gauge(%__MODULE__{} = state, name, value, opts) do
     Metrics.gauge(name, value, Keyword.put(opts, :attributes, state.attributes))
   end

test/sentry/metrics/runtime_test.exs

@@ -27,7 +27,7 @@ defmodule Sentry.Metrics.RuntimeTest do
     test "reports memory in bytes", %{ref: ref} do
       collect_once()
 
-      assert metric = find_metric(ref, "elixir.runtime.mem.total")
+      assert metric = find_metric(snapshot(ref), "elixir.runtime.mem.total")
       assert metric["unit"] == "byte"
       assert metric["type"] == "gauge"
       assert metric["value"] > 0
@@ -60,19 +60,35 @@ defmodule Sentry.Metrics.RuntimeTest do
     test "reports scheduler utilization as a ratio", %{ref: ref} do
       collect_once()
 
-      assert metric = find_metric(ref, "elixir.runtime.scheduler.utilization")
+      assert metric = find_metric(snapshot(ref), "elixir.runtime.scheduler.utilization")
       assert metric["type"] == "gauge"
       assert metric["unit"] == "ratio"
       assert metric["value"] >= 0.0
       assert metric["value"] <= 1.0
     end
   end
 
+  describe "run queue metrics" do
+    test "reports total and CPU-bound run queue depth", %{ref: ref} do
+      collect_once()
+
+      metrics = snapshot(ref)
+
+      assert total = find_metric(metrics, "elixir.runtime.run_queue.total")
+      assert cpu = find_metric(metrics, "elixir.runtime.run_queue.cpu")
+
+      assert total["type"] == "gauge"
+      assert cpu["type"] == "gauge"
+      assert total["value"] >= 0
+      assert cpu["value"] >= 0
+    end
+  end
+
   describe "delivery" do
     test "delivers a whole snapshot from a single collection", %{ref: ref} do
       collect_once()
 
-      assert length(snapshot(ref)) == 6
+      assert length(snapshot(ref)) == 8
     end
   end
 
@@ -107,7 +123,7 @@ defmodule Sentry.Metrics.RuntimeTest do
     items
   end
 
-  defp find_metric(ref, name) do
-    Enum.find(snapshot(ref), &(&1["name"] == name))
+  defp find_metric(metrics, name) when is_list(metrics) do
+    Enum.find(metrics, &(&1["name"] == name))
   end
 end