Skip to content

feat(sdk): enrich collected metrics — granular status codes, TTFB, request size, inflight #12

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Fabien83560 merged 2 commits into from
May 29, 2026
Merged

feat(sdk): enrich collected metrics — granular status codes, TTFB, request size, inflight #12

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e5e98bc
feat(sdk): add granular status codes, TTFB, request size, inflight me…
Fabien83560 May 29, 2026
46e2953
fix(tests): add missing headers and write/end to smoke test req/res m…
Fabien83560 May 29, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "apiforgejs",
"version": "2.1.2",
"version": "2.2.0",
"description": "API observability & intelligence SDK for Express.js — local-first, privacy-first",
"main": "src/index.js",
"keywords": [
Expand Down
49 changes: 46 additions & 3 deletions src/aggregator.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,21 +34,32 @@ class Aggregator {
release: event.release,
is_ghost: event.is_ghost,
durations: [],
ttfb_durations: [],
response_sizes: [],
request_sizes: [],
inflight_samples: [],
status_2xx: 0,
status_3xx: 0,
status_4xx: 0,
status_5xx: 0,
status_map: new Map(),
};
this.buffer.set(key, bucket);
}

bucket.durations.push(event.duration_ms);
if (event.ttfb_ms != null) bucket.ttfb_durations.push(event.ttfb_ms);
if (event.response_size != null) bucket.response_sizes.push(event.response_size);
if (event.request_size != null) bucket.request_sizes.push(event.request_size);
if (event.inflight != null) bucket.inflight_samples.push(event.inflight);

const s = event.status;
if (s >= 200 && s < 300) bucket.status_2xx++;
if (s >= 200 && s < 300) bucket.status_2xx++;
else if (s >= 300 && s < 400) bucket.status_3xx++;
else if (s >= 400 && s < 500) bucket.status_4xx++;
else if (s >= 500) bucket.status_5xx++;
else if (s >= 500) bucket.status_5xx++;

bucket.status_map.set(s, (bucket.status_map.get(s) ?? 0) + 1);
}

_flush() {
Expand All @@ -59,17 +70,41 @@ class Aggregator {
const rows = [];

for (const bucket of this.buffer.values()) {
const sorted = bucket.durations.slice().sort((a, b) => a - b);
const sorted = bucket.durations.slice().sort((a, b) => a - b);
const sortedTtfb = bucket.ttfb_durations.slice().sort((a, b) => a - b);
const n = sorted.length;

const sizes = bucket.response_sizes;
const bytes_avg = sizes.length > 0
? sizes.reduce((a, b) => a + b, 0) / sizes.length
: null;

const reqSizes = bucket.request_sizes;
const request_size_avg = reqSizes.length > 0
? reqSizes.reduce((a, b) => a + b, 0) / reqSizes.length
: null;

const lat_avg = n > 0
? bucket.durations.reduce((a, b) => a + b, 0) / n
: null;

const inflight = bucket.inflight_samples;
const inflight_avg = inflight.length > 0
? inflight.reduce((a, b) => a + b, 0) / inflight.length
: null;
const inflight_max = inflight.length > 0
? Math.max(...inflight)
: null;

// Granular distribution — sorted by count desc, all observed codes
const status_dist = bucket.status_map.size > 0
? JSON.stringify(
Object.fromEntries(
[...bucket.status_map.entries()].sort((a, b) => b[1] - a[1])
)
)
: null;

rows.push({
bucket_ts: bucketTs,
route: bucket.route,
Expand All @@ -78,16 +113,24 @@ class Aggregator {
release_tag: bucket.release,
is_ghost: bucket.is_ghost ? 1 : 0,
status_2xx: bucket.status_2xx,
status_3xx: bucket.status_3xx,
status_4xx: bucket.status_4xx,
status_5xx: bucket.status_5xx,
status_dist,
total_calls: n,
lat_p50: percentile(sorted, 0.50),
lat_p90: percentile(sorted, 0.90),
lat_p99: percentile(sorted, 0.99),
lat_avg,
lat_min: sorted[0] ?? 0,
lat_max: sorted[n - 1] ?? 0,
lat_ttfb_p50: sortedTtfb.length > 0 ? percentile(sortedTtfb, 0.50) : null,
lat_ttfb_p90: sortedTtfb.length > 0 ? percentile(sortedTtfb, 0.90) : null,
lat_ttfb_p99: sortedTtfb.length > 0 ? percentile(sortedTtfb, 0.99) : null,
bytes_avg,
request_size_avg,
inflight_avg,
inflight_max,
});
}

Expand Down
112 changes: 75 additions & 37 deletions src/database.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,39 +30,67 @@ class ApiForgeDatabase {
);

CREATE TABLE IF NOT EXISTS api_metrics (
id INTEGER PRIMARY KEY AUTOINCREMENT,
bucket_ts INTEGER NOT NULL,
route TEXT NOT NULL,
method TEXT NOT NULL,
env TEXT NOT NULL DEFAULT 'production',
release_tag TEXT,
status_2xx INTEGER NOT NULL DEFAULT 0,
status_4xx INTEGER NOT NULL DEFAULT 0,
status_5xx INTEGER NOT NULL DEFAULT 0,
total_calls INTEGER NOT NULL DEFAULT 0,
lat_p50 REAL,
lat_p90 REAL,
lat_p99 REAL,
lat_min REAL,
lat_max REAL,
bytes_avg REAL,
is_ghost INTEGER NOT NULL DEFAULT 0
id INTEGER PRIMARY KEY AUTOINCREMENT,
bucket_ts INTEGER NOT NULL,
route TEXT NOT NULL,
method TEXT NOT NULL,
env TEXT NOT NULL DEFAULT 'production',
release_tag TEXT,
status_2xx INTEGER NOT NULL DEFAULT 0,
status_3xx INTEGER NOT NULL DEFAULT 0,
status_4xx INTEGER NOT NULL DEFAULT 0,
status_5xx INTEGER NOT NULL DEFAULT 0,
status_dist TEXT,
total_calls INTEGER NOT NULL DEFAULT 0,
lat_p50 REAL,
lat_p90 REAL,
lat_p99 REAL,
lat_avg REAL,
lat_min REAL,
lat_max REAL,
lat_ttfb_p50 REAL,
lat_ttfb_p90 REAL,
lat_ttfb_p99 REAL,
bytes_avg REAL,
request_size_avg REAL,
inflight_avg REAL,
inflight_max INTEGER,
is_ghost INTEGER NOT NULL DEFAULT 0
);
CREATE INDEX IF NOT EXISTS idx_route_ts ON api_metrics (route, method, bucket_ts);
CREATE INDEX IF NOT EXISTS idx_bucket_ts ON api_metrics (bucket_ts);
CREATE INDEX IF NOT EXISTS idx_release ON api_metrics (release_tag) WHERE release_tag IS NOT NULL;
`);

// Migrations for databases created before these columns were introduced
try { this.db.exec('ALTER TABLE api_metrics ADD COLUMN bytes_avg REAL'); } catch (_) {}
try { this.db.exec('ALTER TABLE api_metrics ADD COLUMN is_ghost INTEGER NOT NULL DEFAULT 0'); } catch (_) {}
const migrations = [
'ALTER TABLE api_metrics ADD COLUMN bytes_avg REAL',
'ALTER TABLE api_metrics ADD COLUMN is_ghost INTEGER NOT NULL DEFAULT 0',
'ALTER TABLE api_metrics ADD COLUMN status_3xx INTEGER NOT NULL DEFAULT 0',
'ALTER TABLE api_metrics ADD COLUMN status_dist TEXT',
'ALTER TABLE api_metrics ADD COLUMN lat_avg REAL',
'ALTER TABLE api_metrics ADD COLUMN lat_ttfb_p50 REAL',
'ALTER TABLE api_metrics ADD COLUMN lat_ttfb_p90 REAL',
'ALTER TABLE api_metrics ADD COLUMN lat_ttfb_p99 REAL',
'ALTER TABLE api_metrics ADD COLUMN request_size_avg REAL',
'ALTER TABLE api_metrics ADD COLUMN inflight_avg REAL',
'ALTER TABLE api_metrics ADD COLUMN inflight_max INTEGER',
];
for (const sql of migrations) {
try { this.db.exec(sql); } catch (_) {}
}

this._stmtInsert = this.db.prepare(`
INSERT INTO api_metrics
(bucket_ts, route, method, env, release_tag,
status_2xx, status_4xx, status_5xx, total_calls,
lat_p50, lat_p90, lat_p99, lat_min, lat_max, bytes_avg, is_ghost)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
status_2xx, status_3xx, status_4xx, status_5xx, status_dist,
total_calls,
lat_p50, lat_p90, lat_p99, lat_avg, lat_min, lat_max,
lat_ttfb_p50, lat_ttfb_p90, lat_ttfb_p99,
bytes_avg, request_size_avg,
inflight_avg, inflight_max,
is_ghost)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`);

this._begin = this.db.prepare('BEGIN');
Expand All @@ -76,8 +104,12 @@ class ApiForgeDatabase {
for (const r of rows) {
this._stmtInsert.run(
r.bucket_ts, r.route, r.method, r.env, r.release_tag ?? null,
r.status_2xx, r.status_4xx, r.status_5xx, r.total_calls,
r.lat_p50, r.lat_p90, r.lat_p99, r.lat_min, r.lat_max, r.bytes_avg ?? null,
r.status_2xx, r.status_3xx ?? 0, r.status_4xx, r.status_5xx, r.status_dist ?? null,
r.total_calls,
r.lat_p50, r.lat_p90, r.lat_p99, r.lat_avg ?? null, r.lat_min, r.lat_max,
r.lat_ttfb_p50 ?? null, r.lat_ttfb_p90 ?? null, r.lat_ttfb_p99 ?? null,
r.bytes_avg ?? null, r.request_size_avg ?? null,
r.inflight_avg ?? null, r.inflight_max ?? null,
r.is_ghost ?? 0
);
}
Expand All @@ -96,6 +128,7 @@ class ApiForgeDatabase {
SELECT
SUM(total_calls) as calls_total,
SUM(status_2xx) as calls_2xx,
SUM(status_3xx) as calls_3xx,
SUM(status_4xx) as calls_4xx,
SUM(status_5xx) as calls_5xx,
AVG(lat_p90) as avg_p90,
Expand Down Expand Up @@ -131,15 +164,19 @@ class ApiForgeDatabase {
return this.db.prepare(`
SELECT
route, method, is_ghost,
SUM(total_calls) as calls,
SUM(status_2xx) as calls_2xx,
SUM(status_4xx) as calls_4xx,
SUM(status_5xx) as calls_5xx,
AVG(lat_p50) as p50,
AVG(lat_p90) as p90,
AVG(lat_p99) as p99,
MAX(lat_max) as lat_max,
AVG(bytes_avg) as bytes_avg
SUM(total_calls) as calls,
SUM(status_2xx) as calls_2xx,
SUM(status_3xx) as calls_3xx,
SUM(status_4xx) as calls_4xx,
SUM(status_5xx) as calls_5xx,
AVG(lat_p50) as p50,
AVG(lat_p90) as p90,
AVG(lat_p99) as p99,
MAX(lat_max) as lat_max,
AVG(bytes_avg) as bytes_avg,
AVG(request_size_avg) as request_size_avg,
AVG(inflight_avg) as inflight_avg,
MAX(inflight_max) as inflight_max
FROM api_metrics
WHERE bucket_ts >= ?
GROUP BY route, method, is_ghost
Expand All @@ -154,10 +191,11 @@ class ApiForgeDatabase {
SELECT
bucket_ts,
SUM(total_calls) as calls,
AVG(lat_p50) as p50,
AVG(lat_p90) as p90,
AVG(lat_p99) as p99,
SUM(status_5xx) as errors
AVG(lat_p50) as p50,
AVG(lat_p90) as p90,
AVG(lat_p99) as p99,
SUM(status_5xx) as errors,
SUM(status_3xx) as redirects
FROM api_metrics
WHERE route = ? AND method = ? AND bucket_ts >= ?
GROUP BY bucket_ts
Expand Down
32 changes: 32 additions & 0 deletions src/interceptor.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ function createInterceptor(aggregator, storeRoutes, config) {
const ignoreSet = new Set(ignorePaths);

let routesScanned = false;
let inflightCount = 0;

function scanRoutes(app) {
try {
Expand All @@ -68,8 +69,36 @@ function createInterceptor(aggregator, storeRoutes, config) {
if (sampling < 1.0 && Math.random() > sampling) return next();

const startHr = process.hrtime.bigint();
inflightCount++;
const inflightSnapshot = inflightCount;

// Request body size — size only, never the content
const requestSize = req.headers['content-length']
? parseInt(req.headers['content-length'], 10)
: null;

// Patch res.write / res.end to capture Time To First Byte
let ttfbMs = null;
const origWrite = res.write.bind(res);
const origEnd = res.end.bind(res);

function captureTtfb() {
if (ttfbMs === null) {
ttfbMs = Number(process.hrtime.bigint() - startHr) / 1_000_000;
}
}

res.write = function patchedWrite(...args) {
captureTtfb();
return origWrite(...args);
};
res.end = function patchedEnd(...args) {
captureTtfb();
return origEnd(...args);
};

res.on('finish', () => {
inflightCount--;
try {
const durationMs = Number(process.hrtime.bigint() - startHr) / 1_000_000;

Expand All @@ -85,12 +114,15 @@ function createInterceptor(aggregator, storeRoutes, config) {
method: req.method,
status: res.statusCode,
duration_ms: durationMs,
ttfb_ms: ttfbMs ?? durationMs,
timestamp: new Date().toISOString(),
env,
release: release || null,
service,
response_size: contentLength ? parseInt(contentLength, 10) : null,
request_size: requestSize,
is_ghost: !req.route,
inflight: inflightSnapshot,
});
} catch (_) {
// Never let instrumentation crash the host application
Expand Down
Loading
Loading