Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 17 additions & 8 deletions packages/loopover-miner/lib/calibration-cli.d.ts
Original file line number Diff line number Diff line change
@@ -1,9 +1,18 @@
import type { PredictedVerdictRecord, ObservedOutcomeRecord } from "./calibration-types.js";
import type { PredictionLedgerEntry } from "./prediction-ledger.js";
import type { LedgerEntry } from "./event-ledger.js";

export function runCalibrationCli(args?: string[], env?: Record<string, string | undefined>): number;

export function toPredictionRecords(rows: PredictionLedgerEntry[]): PredictedVerdictRecord[];

export function toOutcomeRecords(events: LedgerEntry[]): ObservedOutcomeRecord[];
import type { PredictionLedgerEntry } from "./prediction-ledger.js";
import type { PredictedVerdictRecord, ObservedOutcomeRecord } from "./calibration-types.js";
/** Map prediction-ledger rows to predicted-verdict records: the target id becomes a string key and the recorded
* prediction verdict is the `conclusion`. Exported so callers other than this CLI (the MCP calibration-report
* tool, #5821) can build the identical join without re-implementing the mapping. */
export declare function toPredictionRecords(rows: PredictionLedgerEntry[]): PredictedVerdictRecord[];
/** Reduce the append-only `pr_outcome` event stream to the LATEST observed outcome per (repo, PR), as
* observed-outcome records. `recordedAt` comes from the event's own timestamp (always present), so an outcome is
* never dropped for lacking a `closedAt`. Malformed payloads are skipped. Exported for the same reason as
* {@link toPredictionRecords} above. */
export declare function toOutcomeRecords(events: LedgerEntry[]): ObservedOutcomeRecord[];
/**
* Run `loopover-miner calibration [--json]`. Reads the prediction ledger + PR-outcome events, joins them into a
* calibration report, and prints it (a JSON dump under `--json`, else a per-project text summary). Returns the
* process exit code: 0 on success, 1 on an unknown option.
*/
export declare function runCalibrationCli(args?: string[], env?: Record<string, string | undefined>): number;
130 changes: 63 additions & 67 deletions packages/loopover-miner/lib/calibration-cli.js

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

100 changes: 100 additions & 0 deletions packages/loopover-miner/lib/calibration-cli.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
// `loopover-miner calibration [--json]` (#4849): a read-only report joining the miner's own predicted gate
// verdicts (prediction-ledger) with the realized PR outcomes it later observed (event-ledger `pr_outcome`
// events), via the pure buildCalibrationReport join. Opens both local stores, maps their rows to the
// calibration record shapes, renders, and closes. Never modifies the live scoring/calibration logic.
import { buildCalibrationReport } from "./calibration.js";
import { initEventLedger, resolveEventLedgerDbPath } from "./event-ledger.js";
import type { LedgerEntry } from "./event-ledger.js";
import { MINER_PR_OUTCOME_EVENT } from "./pr-outcome.js";
import { initPredictionLedger, resolvePredictionLedgerDbPath } from "./prediction-ledger.js";
import type { PredictionLedgerEntry } from "./prediction-ledger.js";
import type { PredictedVerdictRecord, ObservedOutcomeRecord, CalibrationReport } from "./calibration-types.js";
import { reportCliFailure, describeCliError } from "./cli-error.js";

const CALIBRATION_USAGE = "Usage: loopover-miner calibration [--json]";

/** Map prediction-ledger rows to predicted-verdict records: the target id becomes a string key and the recorded
* prediction verdict is the `conclusion`. Exported so callers other than this CLI (the MCP calibration-report
* tool, #5821) can build the identical join without re-implementing the mapping. */
export function toPredictionRecords(rows: PredictionLedgerEntry[]): PredictedVerdictRecord[] {
return rows.map((row) => ({
project: row.repoFullName,
targetId: String(row.targetId),
predictedDecision: row.conclusion,
recordedAt: row.ts,
}));
}

/** Reduce the append-only `pr_outcome` event stream to the LATEST observed outcome per (repo, PR), as
* observed-outcome records. `recordedAt` comes from the event's own timestamp (always present), so an outcome is
* never dropped for lacking a `closedAt`. Malformed payloads are skipped. Exported for the same reason as
* {@link toPredictionRecords} above. */
export function toOutcomeRecords(events: LedgerEntry[]): ObservedOutcomeRecord[] {
const latest = new Map<string, ObservedOutcomeRecord>();
for (const event of events) {
if (event?.type !== MINER_PR_OUTCOME_EVENT) continue;
const payload = event.payload;
if (!payload || !Number.isInteger(payload.prNumber) || typeof payload.decision !== "string") continue;
latest.set(`${event.repoFullName}:${payload.prNumber}`, {
// ObservedOutcomeRecord.project is declared non-nullable, but LedgerEntry.repoFullName is `string | null`
// for other event kinds; a pr_outcome event always carries a real repoFullName in practice, so this passes
// the value through unchanged rather than substituting a fallback that would be a behavior change.
project: event.repoFullName as string,
targetId: String(payload.prNumber),
outcomeDecision: payload.decision,
recordedAt: event.createdAt,
});
}
return [...latest.values()];
}

function renderReportText(report: CalibrationReport): void {
if (!report.hasSignal) {
console.log("calibration: no decided predictions yet (predictions need a realized merge/close outcome).");
return;
}
for (const row of report.rows) {
const merge = row.mergePrecision === null ? "n/a" : `${Math.round(row.mergePrecision * 100)}%`;
const close = row.closePrecision === null ? "n/a" : `${Math.round(row.closePrecision * 100)}%`;
console.log(
`${row.project}: ${row.decided} decided | ` +
`merge ${row.mergeConfirmed}/${row.wouldMerge} (${merge}) | ` +
`close ${row.closeConfirmed}/${row.wouldClose} (${close}) | hold ${row.hold}`,
);
}
}

/**
* Run `loopover-miner calibration [--json]`. Reads the prediction ledger + PR-outcome events, joins them into a
* calibration report, and prints it (a JSON dump under `--json`, else a per-project text summary). Returns the
* process exit code: 0 on success, 1 on an unknown option.
*/
export function runCalibrationCli(args: string[] = [], env: Record<string, string | undefined> = process.env): number {
const json = args.includes("--json");
// This command takes no positional arguments, so anything that is not `--json` is a mistake -- including a
// bare positional (`calibration foo`), which a `startsWith("-")` check silently let through (#5834). Mirrors
// the strict zero-positional discipline `ledger list` (event-ledger-cli.js) already applies.
const unknown = args.find((token) => token !== "--json");
if (unknown) {
return reportCliFailure(json, `Unknown option: ${unknown}. ${CALIBRATION_USAGE}`, 1);
}

let predictionStore;
let eventLedger;
try {
predictionStore = initPredictionLedger(resolvePredictionLedgerDbPath(env));
eventLedger = initEventLedger(resolveEventLedgerDbPath(env));
const report = buildCalibrationReport(
toPredictionRecords(predictionStore.readPredictions()),
toOutcomeRecords(eventLedger.readEvents()),
);
if (json) console.log(JSON.stringify(report, null, 2));
else renderReportText(report);
return 0;
} catch (error) {
return reportCliFailure(json, describeCliError(error));
} finally {
predictionStore?.close();
eventLedger?.close();
}
}
36 changes: 13 additions & 23 deletions packages/loopover-miner/lib/feasibility-cli.d.ts
Original file line number Diff line number Diff line change
@@ -1,25 +1,15 @@
import type {
FeasibilityClaimStatus,
FeasibilityDuplicateClusterRisk,
FeasibilityGateInput,
FeasibilityGateResult,
FeasibilityIssueStatus,
} from "@loopover/engine";

export type ParsedFeasibilityArgs =
| {
claimStatus: FeasibilityClaimStatus;
duplicateClusterRisk: FeasibilityDuplicateClusterRisk;
issueStatus: FeasibilityIssueStatus;
found: boolean;
json: boolean;
}
| { error: string };

import type { FeasibilityClaimStatus, FeasibilityDuplicateClusterRisk, FeasibilityGateInput, FeasibilityGateResult, FeasibilityIssueStatus } from "@loopover/engine";
export type ParsedFeasibilityArgs = {
claimStatus: FeasibilityClaimStatus;
duplicateClusterRisk: FeasibilityDuplicateClusterRisk;
issueStatus: FeasibilityIssueStatus;
found: boolean;
json: boolean;
} | {
error: string;
};
export declare function parseFeasibilityArgs(args: string[]): ParsedFeasibilityArgs;
export type RunFeasibilityCliOptions = {
buildFeasibilityVerdict?: (input: FeasibilityGateInput) => FeasibilityGateResult;
buildFeasibilityVerdict?: (input: FeasibilityGateInput) => FeasibilityGateResult;
};

export function parseFeasibilityArgs(args: string[]): ParsedFeasibilityArgs;

export function runFeasibilityCli(args: string[], options?: RunFeasibilityCliOptions): number;
export declare function runFeasibilityCli(args: string[], options?: RunFeasibilityCliOptions): number;
Loading