-
Notifications
You must be signed in to change notification settings - Fork 10
fix(research): reuse a recent identical capture instead of re-measuring (chat#1912 row 4) #800
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
786bea0
235dd63
0e8ff03
2618d2f
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,82 @@ | ||
| import { describe, it, expect } from "vitest"; | ||
| import { findReusableSnapshot } from "@/lib/research/playcounts/findReusableSnapshot"; | ||
| import type { Tables } from "@/types/database.types"; | ||
|
|
||
| const NOW = new Date("2026-07-30T14:33:32.000Z"); | ||
|
|
||
| const snapshot = (over: Partial<Tables<"playcount_snapshots">>) => | ||
| ({ | ||
| id: "snap-1", | ||
| account: "acc-1", | ||
| catalog: null, | ||
| album_ids: ["a1", "a2"], | ||
| isrcs: null, | ||
| platforms: ["spotify"], | ||
| schedule: "once", | ||
| state: "done", | ||
| album_count: 2, | ||
| estimated_cost_usd: 0.006, | ||
| created_at: "2026-07-30T14:30:35.000Z", | ||
| updated_at: "2026-07-30T14:30:35.000Z", | ||
| ...over, | ||
| }) as Tables<"playcount_snapshots">; | ||
|
|
||
| const find = (rows: Tables<"playcount_snapshots">[], albumIds = ["a1", "a2"]) => | ||
| findReusableSnapshot({ | ||
| snapshots: rows, | ||
| albumIds, | ||
| platforms: ["spotify"], | ||
| schedule: "once", | ||
| windowMinutes: 15, | ||
| now: NOW, | ||
| }); | ||
|
|
||
| describe("findReusableSnapshot", () => { | ||
| // chat#1912 row 4: one pass through /setup measured the same 24 albums twice | ||
| // — the seeding valuation at 14:30:35, then the first-task pre-run at | ||
| // 14:33:32 — billing the scraper twice for an identical capture. | ||
| it("reuses a capture of the same albums taken minutes ago", () => { | ||
| expect(find([snapshot({})])?.id).toBe("snap-1"); | ||
| }); | ||
|
|
||
| it("ignores album order when comparing scope", () => { | ||
| expect(find([snapshot({ album_ids: ["a2", "a1"] })])?.id).toBe("snap-1"); | ||
| }); | ||
|
|
||
| it("does not reuse a capture of a different album set", () => { | ||
| expect(find([snapshot({ album_ids: ["a1"] })])).toBeNull(); | ||
| expect(find([snapshot({ album_ids: ["a1", "a2", "a3"] })])).toBeNull(); | ||
| }); | ||
|
|
||
| it("does not reuse a capture older than the window", () => { | ||
| expect(find([snapshot({ created_at: "2026-07-30T14:00:00.000Z" })])).toBeNull(); | ||
| }); | ||
|
|
||
| it("does not reuse a failed capture", () => { | ||
| expect(find([snapshot({ state: "failed" })])).toBeNull(); | ||
| }); | ||
|
|
||
| // A queued capture is still the right thing to hand back: the caller polls | ||
| // for measurements either way, and starting a second scrape is the waste. | ||
| it("reuses a capture that is still in flight", () => { | ||
| expect(find([snapshot({ state: "queued" })])?.id).toBe("snap-1"); | ||
| }); | ||
|
|
||
| it("does not reuse across platforms", () => { | ||
| expect(find([snapshot({ platforms: ["apple"] })])).toBeNull(); | ||
| }); | ||
|
|
||
| it("does not reuse a scheduled run for a one-off request", () => { | ||
| expect(find([snapshot({ schedule: "monthly" })])).toBeNull(); | ||
| }); | ||
|
|
||
| it("prefers the newest eligible capture", () => { | ||
| const older = snapshot({ id: "old", created_at: "2026-07-30T14:25:00.000Z" }); | ||
| const newer = snapshot({ id: "new", created_at: "2026-07-30T14:31:00.000Z" }); | ||
| expect(find([older, newer])?.id).toBe("new"); | ||
| }); | ||
|
|
||
| it("returns null when nothing matches", () => { | ||
| expect(find([])).toBeNull(); | ||
| }); | ||
| }); |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,19 @@ | ||
| import type { Tables } from "@/types/database.types"; | ||
|
|
||
| /** | ||
| * The 202-shaped payload for a request satisfied by an existing capture. | ||
| * `estimated_cost_usd` is 0 because nothing was scraped, and `reused` lets | ||
| * callers tell a fresh capture from a handed-back one (chat#1912 row 4). | ||
| */ | ||
| export function buildReusedSnapshotResult(snapshot: Tables<"playcount_snapshots">) { | ||
| return { | ||
| data: { | ||
| status: "success", | ||
| snapshot_id: snapshot.id, | ||
| state: snapshot.state, | ||
| album_count: snapshot.album_count, | ||
| estimated_cost_usd: 0, | ||
| reused: true, | ||
| }, | ||
| }; | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -3,11 +3,20 @@ import { resolveSnapshotAlbums } from "@/lib/research/playcounts/resolveSnapshot | |
| import { selectPlaycountSnapshots } from "@/lib/supabase/playcount_snapshots/selectPlaycountSnapshots"; | ||
| import { insertPlaycountSnapshot } from "@/lib/supabase/playcount_snapshots/insertPlaycountSnapshot"; | ||
| import { playcountSnapshotWorkflow } from "@/app/workflows/playcountSnapshotWorkflow"; | ||
| import { findReusableSnapshot } from "@/lib/research/playcounts/findReusableSnapshot"; | ||
| import { buildReusedSnapshotResult } from "@/lib/research/playcounts/buildReusedSnapshotResult"; | ||
| import { getMonthlySpendUsd } from "@/lib/research/playcounts/getMonthlySpendUsd"; | ||
| import { CreateSnapshotBody } from "@/lib/research/playcounts/validateCreateSnapshotRequest"; | ||
|
|
||
| /** Actor pricing: ~$3 per 1k album URLs. */ | ||
| const COST_PER_ALBUM_USD = 0.003; | ||
| const DEFAULT_MONTHLY_CAP_USD = 25; | ||
| /** | ||
| * Play counts do not move meaningfully inside an hour, so an identical capture | ||
| * requested within this window reuses the earlier one rather than re-scraping | ||
| * (chat#1912 row 4). | ||
| */ | ||
| const REUSE_WINDOW_MINUTES = 60; | ||
|
|
||
| export type CreateSnapshotResult = { data: unknown } | { error: string; status: number }; | ||
|
|
||
|
|
@@ -32,15 +41,35 @@ export async function createSnapshot(params: { | |
| }; | ||
| } | ||
|
|
||
| const monthStart = new Date(); | ||
| const now = new Date(); | ||
| const monthStart = new Date(now); | ||
| monthStart.setUTCDate(1); | ||
| monthStart.setUTCHours(0, 0, 0, 0); | ||
| const monthSnapshots = await selectPlaycountSnapshots({ | ||
| const reuseCutoff = new Date(now.getTime() - REUSE_WINDOW_MINUTES * 60 * 1000); | ||
|
|
||
| // One query serves both needs, so the lower bound is whichever reaches | ||
| // further back. Without this, a request in the first hour of a UTC month | ||
| // could not see the previous month's captures and re-scraped them. | ||
| const lookbackStart = reuseCutoff < monthStart ? reuseCutoff : monthStart; | ||
| const snapshots = await selectPlaycountSnapshots({ | ||
| account: params.accountId, | ||
| createdAfter: monthStart.toISOString(), | ||
| createdAfter: lookbackStart.toISOString(), | ||
| }); | ||
| const spentUsd = monthSnapshots.reduce((sum, row) => sum + (row.estimated_cost_usd ?? 0), 0); | ||
|
|
||
| const spentUsd = getMonthlySpendUsd(snapshots, monthStart); | ||
| const estimatedCostUsd = Number((albumIds.length * COST_PER_ALBUM_USD).toFixed(4)); | ||
|
|
||
| // Hand back an identical recent capture rather than scraping the same albums | ||
| // twice. | ||
| const reusable = findReusableSnapshot({ | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P2: Simultaneous identical requests can still create and scrape two snapshots: the read/check/insert sequence is non-atomic. Use a database-backed lock or atomic find-or-create/unique-claim flow so one request returns the row created by the other. Prompt for AI agents
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Correct, and deliberately not fixed here. Recording the reasoning rather than silently leaving it. The read/check/insert sequence is not atomic, so two simultaneous identical requests can both miss and both scrape. What this PR targets is the sequential duplicate that is actually happening in production: the seeding valuation and the Closing the concurrent window properly needs a database-level guard, and the obvious shapes do not fit cheaply:
So the residual is: two requests landing in the same instant still both scrape. That is strictly better than today, where requests three minutes apart both scrape, and it is not the observed failure. Flagging it on the tracking issue as a follow-up rather than expanding this PR is the honest trade. If you would rather it land here, say so and I will do the advisory-lock version with its own tests. |
||
| snapshots, | ||
| albumIds, | ||
| platforms: params.body.platforms, | ||
| schedule: params.body.schedule, | ||
| windowMinutes: REUSE_WINDOW_MINUTES, | ||
| now, | ||
| }); | ||
| if (reusable) return buildReusedSnapshotResult(reusable); | ||
| const capUsd = Number(process.env.SNAPSHOT_MONTHLY_CAP_USD) || DEFAULT_MONTHLY_CAP_USD; | ||
| if (spentUsd + estimatedCostUsd > capUsd) { | ||
| return { error: "Per-organization monthly snapshot cap reached", status: 429 }; | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,17 @@ | ||
| import type { Tables } from "@/types/database.types"; | ||
|
|
||
| /** | ||
| * Scraper spend for the current calendar month, in USD. | ||
| * | ||
| * The caller's snapshot lookup can reach into the previous month (the reuse | ||
| * window crosses the boundary), so rows are filtered by `monthStart` here | ||
| * rather than relying on the query's lower bound (chat#1912 row 4). | ||
| */ | ||
| export function getMonthlySpendUsd( | ||
| snapshots: Tables<"playcount_snapshots">[], | ||
| monthStart: Date, | ||
| ): number { | ||
| return snapshots | ||
| .filter(row => !!row.created_at && new Date(row.created_at) >= monthStart) | ||
| .reduce((sum, row) => sum + (row.estimated_cost_usd ?? 0), 0); | ||
| } |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
P3: The existing monthly-cap test now fails because its mock row has no
created_at, which this new calculation filters out. Include a current-month timestamp in that fixture (and add a direct helper test) so the cap behavior remains covered.Prompt for AI agents