From 0cfebfed9b041fe3599f0608f3b967b10148032e Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Thu, 21 May 2026 10:21:52 +0000 Subject: [PATCH 1/3] test: add metering E2E integration tests + Ollama CI tier Test #3 from constructive-planning#911: full metering pipeline tests. Metering E2E tests (15 tests): - Config resolution: getLlmBillingConfig reads metaschema tables - Billing functions: check_billing_quota + record_usage SQL stubs - Inference log: logInferenceUsage writes correct fields - meteredEmbed with mock embedder: quota check, record_usage, quota exceeded, request_id propagation - meteredEmbed with real Ollama: live inference + billing pipeline (skipped when Ollama unavailable) CI Tier 4 (ollama-tests): - PostgreSQL + Ollama service containers - Pulls nomic-embed-text model before test run - Runs graphile-llm full test suite including Ollama E2E --- .github/workflows/run-tests.yaml | 81 +++ .../src/__tests__/metering-e2e.test.ts | 536 ++++++++++++++++++ .../src/__tests__/metering-setup.sql | 213 +++++++ 3 files changed, 830 insertions(+) create mode 100644 graphile/graphile-llm/src/__tests__/metering-e2e.test.ts create mode 100644 graphile/graphile-llm/src/__tests__/metering-setup.sql diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index 24130707c..5e2bfbede 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -337,3 +337,84 @@ jobs: - name: Test ${{ matrix.package }} run: cd ./${{ matrix.package }} && pnpm test env: ${{ matrix.env }} + + # ========================================================================= + # TIER 4 – Ollama tests (PostgreSQL + Ollama) + # ========================================================================= + ollama-tests: + needs: build + runs-on: blacksmith-4vcpu-ubuntu-2404 + timeout-minutes: 15 + + env: + PGHOST: localhost + PGPORT: 5432 + PGUSER: postgres + PGPASSWORD: password + OLLAMA_BASE_URL: http://localhost:11434 + + services: + pg_db: + image: ghcr.io/constructive-io/docker/postgres-plus:18 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: password + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + + ollama: + image: ollama/ollama:latest + ports: + - 11434:11434 + options: >- + --health-cmd "curl -f http://localhost:11434/ || exit 1" + --health-interval 15s + --health-timeout 10s + --health-retries 10 + --health-start-period 30s + + steps: + - name: Download workspace + uses: actions/download-artifact@v4 + with: + name: workspace-build + + - name: Extract workspace + run: tar -xzf workspace.tar.gz && rm workspace.tar.gz + + - name: Configure Git (for tests) + run: | + git config --global user.name "CI Test User" + git config --global user.email "ci@example.com" + + - name: Setup pnpm + uses: pnpm/action-setup@v2 + with: + version: 10 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '22' + cache: 'pnpm' + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Seed app_user + run: | + pnpm --filter pgpm exec node dist/index.js admin-users bootstrap --yes + pnpm --filter pgpm exec node dist/index.js admin-users add --test --yes + + - name: Pull Ollama models + run: | + curl -s http://localhost:11434/api/pull -d '{"name":"nomic-embed-text:latest"}' | tail -1 + echo "nomic-embed-text pulled" + + - name: Test graphile-llm + run: cd ./graphile/graphile-llm && pnpm test diff --git a/graphile/graphile-llm/src/__tests__/metering-e2e.test.ts b/graphile/graphile-llm/src/__tests__/metering-e2e.test.ts new file mode 100644 index 000000000..3cbb9c06a --- /dev/null +++ b/graphile/graphile-llm/src/__tests__/metering-e2e.test.ts @@ -0,0 +1,536 @@ +/** + * Metering E2E Integration Tests + * + * Full pipeline: Ollama embedding → metering check_billing_quota → + * record_usage → inference_log INSERT → billing ledger write. + * + * Requires: + * - PostgreSQL (via pgsql-test) + * - Ollama running at OLLAMA_BASE_URL (default: http://127.0.0.1:11434) + * - nomic-embed-text model pulled + * + * Skip condition: set SKIP_OLLAMA_TESTS=1 to skip when Ollama is unavailable. + * + * Run: + * pnpm test -- --testPathPattern=metering-e2e + */ + +import { join } from 'path'; +import OllamaClient from '@agentic-kit/ollama'; +import { getConnections, seed } from 'graphile-test'; +import type { PgTestClient } from 'pgsql-test'; +import { buildEmbedder } from '../../src/embedder'; +import { meteredEmbed, logInferenceUsage } from '../../src/metering'; +import { invalidateLlmBillingConfig, getLlmBillingConfig } from '../../src/config-cache'; +import type { MeteringContext, WithPgClient } from '../../src/metering'; +import type { BillingConfig, InferenceLogConfig, PgClient } from '../../src/config-cache'; +import type { EmbedderFunction } from '../../src/types'; + +// ─── Constants ────────────────────────────────────────────────────────────── + +const OLLAMA_BASE_URL = process.env.OLLAMA_BASE_URL ?? 'http://127.0.0.1:11434'; +const SKIP_OLLAMA = process.env.SKIP_OLLAMA_TESTS === '1'; +const DATABASE_ID = '00000000-0000-0000-0000-000000000001'; +const ENTITY_ID = '00000000-0000-0000-0000-000000000099'; +const ACTOR_ID = '00000000-0000-0000-0000-000000000099'; +const REQUEST_ID = '00000000-0000-0000-0000-00000000e001'; + +// ─── Ollama availability check ────────────────────────────────────────────── + +let ollamaAvailable = false; + +async function checkOllama(): Promise { + if (SKIP_OLLAMA) return false; + try { + const client = new OllamaClient(OLLAMA_BASE_URL); + const models = await client.listModels(); + return models.some((m: string) => m.includes('nomic-embed-text')); + } catch { + return false; + } +} + +// ─── Helpers ──────────────────────────────────────────────────────────────── + +function createWithPgClient(pg: PgTestClient): WithPgClient { + return async (_pgSettings, callback) => { + await callback(pg as unknown as PgClient); + }; +} + +async function buildMeteringCtx( + pg: PgTestClient, + overrides: Partial = {}, +): Promise { + const billing: BillingConfig = { + publicSchema: 'billing_public', + privateSchema: 'billing_private', + recordUsageFunction: 'record_usage', + checkBillingQuotaFunction: 'check_billing_quota', + }; + + const inferenceLog: InferenceLogConfig = { + schema: 'usage_public', + tableName: 'usage_log_inferences', + }; + + return { + withPgClient: createWithPgClient(pg), + pgSettings: { + 'jwt.claims.user_id': ENTITY_ID, + 'jwt.claims.database_id': DATABASE_ID, + 'request.id': REQUEST_ID, + }, + billing, + entityId: ENTITY_ID, + requestId: REQUEST_ID, + databaseId: DATABASE_ID, + actorId: ACTOR_ID, + inferenceLog, + ...overrides, + }; +} + +// ============================================================================= +// TEST SUITE +// ============================================================================= + +describe('Metering E2E', () => { + let pg: PgTestClient; + let teardown: () => Promise; + + jest.setTimeout(60_000); + + beforeAll(async () => { + ollamaAvailable = await checkOllama(); + + const connections = await getConnections( + { + schemas: ['billing_public', 'billing_private', 'usage_public', 'metaschema_public', 'metaschema_modules_public'], + useRoot: true, + authRole: 'postgres', + }, + [seed.sqlfile([join(__dirname, './metering-setup.sql')])], + ); + + pg = connections.db; + teardown = connections.teardown; + }); + + afterAll(async () => { + invalidateLlmBillingConfig(); + if (teardown) await teardown(); + }); + + beforeEach(async () => { + await pg.beforeEach(); + invalidateLlmBillingConfig(); + }); + + afterEach(async () => { + await pg.afterEach(); + }); + + // =========================================================================== + // 1. Config resolution — getLlmBillingConfig reads metaschema tables + // =========================================================================== + + describe('config resolution', () => { + it('resolves billing config from metaschema_modules_public', async () => { + const entry = await getLlmBillingConfig(pg as unknown as PgClient, DATABASE_ID); + expect(entry.billing).not.toBeNull(); + expect(entry.billing!.publicSchema).toBe('billing_public'); + expect(entry.billing!.privateSchema).toBe('billing_private'); + expect(entry.billing!.recordUsageFunction).toBe('record_usage'); + }); + + it('resolves inference log config from metaschema_modules_public', async () => { + const entry = await getLlmBillingConfig(pg as unknown as PgClient, DATABASE_ID); + expect(entry.inferenceLog).not.toBeNull(); + expect(entry.inferenceLog!.schema).toBe('usage_public'); + expect(entry.inferenceLog!.tableName).toBe('usage_log_inferences'); + }); + + it('returns null billing for unknown database_id', async () => { + const entry = await getLlmBillingConfig( + pg as unknown as PgClient, + 'ffffffff-ffff-ffff-ffff-ffffffffffff', + ); + expect(entry.billing).toBeNull(); + expect(entry.inferenceLog).toBeNull(); + }); + }); + + // =========================================================================== + // 2. Billing function stubs — check_billing_quota + record_usage + // =========================================================================== + + describe('billing functions', () => { + it('check_billing_quota returns true when no credits row exists (unlimited)', async () => { + const result = await pg.any<{ allowed: boolean }>( + `SELECT billing_private.check_billing_quota('nomic-embed-text', $1, 100) AS allowed`, + [ENTITY_ID], + ); + expect(result[0].allowed).toBe(true); + }); + + it('check_billing_quota returns true when under limit', async () => { + await pg.query( + `INSERT INTO billing_public.meter_credits (entity_id, meter_slug, credit_amount) VALUES ($1, 'nomic-embed-text', 1000)`, + [ENTITY_ID], + ); + const result = await pg.any<{ allowed: boolean }>( + `SELECT billing_private.check_billing_quota('nomic-embed-text', $1, 100) AS allowed`, + [ENTITY_ID], + ); + expect(result[0].allowed).toBe(true); + }); + + it('check_billing_quota returns false when over limit', async () => { + await pg.query( + `INSERT INTO billing_public.meter_credits (entity_id, meter_slug, credit_amount) VALUES ($1, 'nomic-embed-text', 50)`, + [ENTITY_ID], + ); + await pg.query( + `INSERT INTO billing_public.balances (entity_id, meter_slug, balance) VALUES ($1, 'nomic-embed-text', 45)`, + [ENTITY_ID], + ); + const result = await pg.any<{ allowed: boolean }>( + `SELECT billing_private.check_billing_quota('nomic-embed-text', $1, 10) AS allowed`, + [ENTITY_ID], + ); + expect(result[0].allowed).toBe(false); + }); + + it('record_usage writes to balance and ledger', async () => { + await pg.query( + `SELECT billing_private.record_usage('nomic-embed-text', $1, 42, '{"request_id":"test"}'::jsonb)`, + [ENTITY_ID], + ); + + const balances = await pg.any<{ balance: string }>( + `SELECT balance FROM billing_public.balances WHERE entity_id = $1 AND meter_slug = 'nomic-embed-text'`, + [ENTITY_ID], + ); + expect(Number(balances[0].balance)).toBe(42); + + const ledger = await pg.any<{ amount: string; metadata: Record }>( + `SELECT amount, metadata FROM billing_public.ledger WHERE entity_id = $1 AND meter_slug = 'nomic-embed-text'`, + [ENTITY_ID], + ); + expect(ledger.length).toBe(1); + expect(Number(ledger[0].amount)).toBe(42); + expect(ledger[0].metadata).toEqual({ request_id: 'test' }); + }); + }); + + // =========================================================================== + // 3. Inference log — logInferenceUsage writes to usage_log_inferences + // =========================================================================== + + describe('inference log', () => { + it('logInferenceUsage inserts a row with all fields', async () => { + const ctx = await buildMeteringCtx(pg); + + await logInferenceUsage(ctx, { + databaseId: DATABASE_ID, + entityId: ENTITY_ID, + actorId: ACTOR_ID, + model: 'nomic-embed-text', + provider: 'ollama', + service: 'embedding', + operation: 'create', + inputTokens: 100, + outputTokens: 0, + totalTokens: 100, + cacheReadTokens: null, + cacheWriteTokens: null, + latencyMs: 50, + ragEnabled: false, + chunksRetrieved: null, + embeddingModel: 'nomic-embed-text', + embeddingLatencyMs: 50, + status: 'success', + errorType: null, + rawUsage: null, + }); + + const rows = await pg.any<{ + model: string; + service: string; + operation: string; + entity_id: string; + input_tokens: string; + status: string; + }>( + `SELECT model, service, operation, entity_id, input_tokens, status + FROM usage_public.usage_log_inferences + WHERE entity_id = $1`, + [ENTITY_ID], + ); + + expect(rows.length).toBe(1); + expect(rows[0].model).toBe('nomic-embed-text'); + expect(rows[0].service).toBe('embedding'); + expect(rows[0].operation).toBe('create'); + expect(rows[0].status).toBe('success'); + expect(Number(rows[0].input_tokens)).toBe(100); + }); + }); + + // =========================================================================== + // 4. meteredEmbed — full pipeline with mock embedder + // =========================================================================== + + describe('meteredEmbed with mock embedder', () => { + const mockEmbedder: EmbedderFunction = async (_text: string) => [1, 0, 0]; + + it('calls embedder and records usage when quota allows', async () => { + const ctx = await buildMeteringCtx(pg); + const result = await meteredEmbed(mockEmbedder, 'hello world', ctx, { + embeddingMeterSlug: 'nomic-embed-text', + embeddingModel: 'nomic-embed-text', + }); + + expect(result.metered).toBe(true); + expect(result.quotaExceeded).toBe(false); + expect(result.result).toEqual([1, 0, 0]); + + // Wait for async billing writes + await new Promise((r) => setTimeout(r, 200)); + + // Verify ledger entry + const ledger = await pg.any<{ meter_slug: string; metadata: Record }>( + `SELECT meter_slug, metadata FROM billing_public.ledger WHERE entity_id = $1`, + [ENTITY_ID], + ); + expect(ledger.length).toBe(1); + expect(ledger[0].meter_slug).toBe('nomic-embed-text'); + expect(ledger[0].metadata).toHaveProperty('request_id', REQUEST_ID); + + // Verify inference log entry + const logs = await pg.any<{ model: string; service: string; status: string }>( + `SELECT model, service, status FROM usage_public.usage_log_inferences WHERE entity_id = $1`, + [ENTITY_ID], + ); + expect(logs.length).toBe(1); + expect(logs[0].model).toBe('nomic-embed-text'); + expect(logs[0].service).toBe('embedding'); + expect(logs[0].status).toBe('success'); + }); + + it('returns quotaExceeded when limit is reached', async () => { + // Set a low credit limit + await pg.query( + `INSERT INTO billing_public.meter_credits (entity_id, meter_slug, credit_amount) VALUES ($1, 'nomic-embed-text', 5)`, + [ENTITY_ID], + ); + await pg.query( + `INSERT INTO billing_public.balances (entity_id, meter_slug, balance) VALUES ($1, 'nomic-embed-text', 5)`, + [ENTITY_ID], + ); + + const ctx = await buildMeteringCtx(pg); + const result = await meteredEmbed(mockEmbedder, 'hello world', ctx, { + embeddingMeterSlug: 'nomic-embed-text', + embeddingModel: 'nomic-embed-text', + }); + + expect(result.metered).toBe(true); + expect(result.quotaExceeded).toBe(true); + expect(result.result).toBeNull(); + + // Wait for async inference log write + await new Promise((r) => setTimeout(r, 200)); + + // Verify inference log records the quota_exceeded status + const logs = await pg.any<{ status: string }>( + `SELECT status FROM usage_public.usage_log_inferences WHERE entity_id = $1`, + [ENTITY_ID], + ); + expect(logs.length).toBe(1); + expect(logs[0].status).toBe('quota_exceeded'); + }); + + it('passes through unmetered when no meter slug configured', async () => { + const ctx = await buildMeteringCtx(pg); + const result = await meteredEmbed(mockEmbedder, 'hello', ctx, {}); + + expect(result.metered).toBe(false); + expect(result.quotaExceeded).toBe(false); + expect(result.result).toEqual([1, 0, 0]); + }); + + it('request_id propagates to billing ledger metadata', async () => { + const custom_request_id = '11111111-1111-1111-1111-111111111111'; + const ctx = await buildMeteringCtx(pg, { + requestId: custom_request_id, + pgSettings: { + 'jwt.claims.user_id': ENTITY_ID, + 'jwt.claims.database_id': DATABASE_ID, + 'request.id': custom_request_id, + }, + }); + + await meteredEmbed(mockEmbedder, 'test text', ctx, { + embeddingMeterSlug: 'nomic-embed-text', + embeddingModel: 'nomic-embed-text', + }); + + await new Promise((r) => setTimeout(r, 200)); + + const ledger = await pg.any<{ metadata: Record }>( + `SELECT metadata FROM billing_public.ledger WHERE entity_id = $1`, + [ENTITY_ID], + ); + expect(ledger.length).toBe(1); + expect(ledger[0].metadata).toHaveProperty('request_id', custom_request_id); + }); + }); + + // =========================================================================== + // 5. Real Ollama E2E — full pipeline with live inference + // Skipped when Ollama is not available. + // =========================================================================== + + const describeOllama = ollamaAvailable ? describe : describe.skip; + + describeOllama('meteredEmbed with real Ollama', () => { + let embedder: EmbedderFunction; + + beforeAll(() => { + const built = buildEmbedder({ + provider: 'ollama', + model: 'nomic-embed-text', + baseUrl: OLLAMA_BASE_URL, + }); + if (!built) throw new Error('Failed to build Ollama embedder'); + embedder = built; + }); + + it('embeds text, records usage, and writes inference log', async () => { + const ctx = await buildMeteringCtx(pg); + const result = await meteredEmbed(embedder, 'PostgreSQL is a powerful database', ctx, { + embeddingMeterSlug: 'nomic-embed-text', + embeddingModel: 'nomic-embed-text', + provider: 'ollama', + }); + + expect(result.metered).toBe(true); + expect(result.quotaExceeded).toBe(false); + expect(result.result).not.toBeNull(); + expect(Array.isArray(result.result)).toBe(true); + expect(result.result!.length).toBeGreaterThan(0); + expect(result.latencyMs).toBeGreaterThan(0); + + // Wait for async billing writes + await new Promise((r) => setTimeout(r, 500)); + + // Verify billing ledger + const ledger = await pg.any<{ meter_slug: string; amount: string; metadata: Record }>( + `SELECT meter_slug, amount, metadata FROM billing_public.ledger WHERE entity_id = $1`, + [ENTITY_ID], + ); + expect(ledger.length).toBe(1); + expect(ledger[0].meter_slug).toBe('nomic-embed-text'); + expect(Number(ledger[0].amount)).toBeGreaterThan(0); + expect(ledger[0].metadata).toHaveProperty('request_id', REQUEST_ID); + expect(ledger[0].metadata).toHaveProperty('dims'); + expect(ledger[0].metadata).toHaveProperty('latency_ms'); + + // Verify inference log + const logs = await pg.any<{ + model: string; + provider: string; + service: string; + operation: string; + status: string; + input_tokens: string; + total_tokens: string; + latency_ms: string; + embedding_model: string; + }>( + `SELECT model, provider, service, operation, status, + input_tokens, total_tokens, latency_ms, embedding_model + FROM usage_public.usage_log_inferences + WHERE entity_id = $1`, + [ENTITY_ID], + ); + expect(logs.length).toBe(1); + expect(logs[0].model).toBe('nomic-embed-text'); + expect(logs[0].provider).toBe('ollama'); + expect(logs[0].service).toBe('embedding'); + expect(logs[0].operation).toBe('create'); + expect(logs[0].status).toBe('success'); + expect(Number(logs[0].input_tokens)).toBeGreaterThan(0); + expect(Number(logs[0].latency_ms)).toBeGreaterThan(0); + expect(logs[0].embedding_model).toBe('nomic-embed-text'); + + // Verify balance was deducted + const balances = await pg.any<{ balance: string }>( + `SELECT balance FROM billing_public.balances WHERE entity_id = $1 AND meter_slug = 'nomic-embed-text'`, + [ENTITY_ID], + ); + expect(balances.length).toBe(1); + expect(Number(balances[0].balance)).toBeGreaterThan(0); + }); + + it('quota enforcement blocks real Ollama call', async () => { + // Exhaust quota + await pg.query( + `INSERT INTO billing_public.meter_credits (entity_id, meter_slug, credit_amount) VALUES ($1, 'nomic-embed-text', 1)`, + [ENTITY_ID], + ); + await pg.query( + `INSERT INTO billing_public.balances (entity_id, meter_slug, balance) VALUES ($1, 'nomic-embed-text', 1)`, + [ENTITY_ID], + ); + + const ctx = await buildMeteringCtx(pg); + const result = await meteredEmbed(embedder, 'This should be blocked', ctx, { + embeddingMeterSlug: 'nomic-embed-text', + embeddingModel: 'nomic-embed-text', + provider: 'ollama', + }); + + expect(result.quotaExceeded).toBe(true); + expect(result.result).toBeNull(); + }); + + it('multiple metered calls accumulate in balance and ledger', async () => { + const ctx = await buildMeteringCtx(pg); + const opts = { + embeddingMeterSlug: 'nomic-embed-text', + embeddingModel: 'nomic-embed-text', + provider: 'ollama', + }; + + await meteredEmbed(embedder, 'First call', ctx, opts); + await meteredEmbed(embedder, 'Second call', ctx, opts); + + // Wait for async writes + await new Promise((r) => setTimeout(r, 500)); + + const ledger = await pg.any<{ amount: string }>( + `SELECT amount FROM billing_public.ledger WHERE entity_id = $1 ORDER BY created_at`, + [ENTITY_ID], + ); + expect(ledger.length).toBe(2); + + const balance = await pg.any<{ balance: string }>( + `SELECT balance FROM billing_public.balances WHERE entity_id = $1 AND meter_slug = 'nomic-embed-text'`, + [ENTITY_ID], + ); + expect(Number(balance[0].balance)).toBe( + Number(ledger[0].amount) + Number(ledger[1].amount), + ); + + // Inference log should have 2 entries + const logs = await pg.any( + `SELECT id FROM usage_public.usage_log_inferences WHERE entity_id = $1`, + [ENTITY_ID], + ); + expect(logs.length).toBe(2); + }); + }); +}); diff --git a/graphile/graphile-llm/src/__tests__/metering-setup.sql b/graphile/graphile-llm/src/__tests__/metering-setup.sql new file mode 100644 index 000000000..845ec46ed --- /dev/null +++ b/graphile/graphile-llm/src/__tests__/metering-setup.sql @@ -0,0 +1,213 @@ +-- Metering E2E test seed — minimal billing + inference log infrastructure +-- +-- Creates stub schemas and tables that match what the metering plugin +-- queries via config-cache.ts (metaschema_modules_public.billing_module, +-- metaschema_modules_public.inference_log_module, metaschema_public.schema). +-- +-- Also creates simplified billing functions (check_billing_quota, record_usage) +-- that operate on a real meters/balances/ledger schema so the test can verify +-- actual quota enforcement and ledger writes. + +-- ============================================================================ +-- 1. Metaschema lookup infrastructure +-- ============================================================================ +CREATE SCHEMA IF NOT EXISTS metaschema_public; +CREATE SCHEMA IF NOT EXISTS metaschema_modules_public; + +CREATE TABLE metaschema_public.schema ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + schema_name text NOT NULL UNIQUE +); + +-- billing_module config table (queried by config-cache.ts BILLING_MODULE_SQL) +CREATE TABLE metaschema_modules_public.billing_module ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + database_id uuid NOT NULL, + schema_id uuid NOT NULL REFERENCES metaschema_public.schema(id), + private_schema_id uuid NOT NULL REFERENCES metaschema_public.schema(id), + record_usage_function text NOT NULL DEFAULT 'record_usage' +); + +-- inference_log_module config table (queried by config-cache.ts INFERENCE_LOG_MODULE_SQL) +CREATE TABLE metaschema_modules_public.inference_log_module ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + database_id uuid NOT NULL, + schema_id uuid NOT NULL REFERENCES metaschema_public.schema(id), + inference_log_table_name text NOT NULL DEFAULT 'usage_log_inferences' +); + +-- ============================================================================ +-- 2. Billing schemas + tables +-- ============================================================================ +CREATE SCHEMA IF NOT EXISTS billing_public; +CREATE SCHEMA IF NOT EXISTS billing_private; + +-- Meters table +CREATE TABLE billing_public.meters ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + slug text NOT NULL UNIQUE, + display_name text NOT NULL DEFAULT '', + meter_type text NOT NULL DEFAULT 'quota', + aggregation text NOT NULL DEFAULT 'cumulative', + credit_cost numeric NOT NULL DEFAULT 1, + category_meter text, + period_interval interval, + unit text +); + +-- Balances table (one row per entity+meter) +CREATE TABLE billing_public.balances ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + entity_id uuid NOT NULL, + meter_slug text NOT NULL REFERENCES billing_public.meters(slug), + balance numeric NOT NULL DEFAULT 0, + UNIQUE(entity_id, meter_slug) +); + +-- Ledger table (append-only log of all usage) +CREATE TABLE billing_public.ledger ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + entity_id uuid NOT NULL, + meter_slug text NOT NULL, + amount numeric NOT NULL, + metadata jsonb, + created_at timestamptz NOT NULL DEFAULT now() +); + +-- Meter credits (quota limits per entity+meter) +CREATE TABLE billing_public.meter_credits ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + entity_id uuid NOT NULL, + meter_slug text NOT NULL REFERENCES billing_public.meters(slug), + credit_amount numeric NOT NULL DEFAULT 0, + UNIQUE(entity_id, meter_slug) +); + +-- ============================================================================ +-- 3. Billing functions (simplified but functional) +-- ============================================================================ + +-- check_billing_quota: returns TRUE if entity has remaining quota +CREATE FUNCTION billing_private.check_billing_quota( + p_meter_slug text, + p_entity_id uuid, + p_amount numeric +) RETURNS boolean AS $$ +DECLARE + v_balance numeric; + v_credit numeric; +BEGIN + SELECT COALESCE(balance, 0) INTO v_balance + FROM billing_public.balances + WHERE entity_id = p_entity_id AND meter_slug = p_meter_slug; + + SELECT COALESCE(credit_amount, 0) INTO v_credit + FROM billing_public.meter_credits + WHERE entity_id = p_entity_id AND meter_slug = p_meter_slug; + + -- No credit row means unlimited + IF v_credit IS NULL THEN RETURN TRUE; END IF; + IF v_credit = 0 THEN RETURN TRUE; END IF; + + RETURN COALESCE(v_balance, 0) + p_amount <= v_credit; +END; +$$ LANGUAGE plpgsql; + +-- record_usage: deducts from balance and writes to ledger +CREATE FUNCTION billing_private.record_usage( + p_meter_slug text, + p_entity_id uuid, + p_amount numeric, + p_metadata jsonb DEFAULT NULL +) RETURNS void AS $$ +BEGIN + -- Upsert balance + INSERT INTO billing_public.balances (entity_id, meter_slug, balance) + VALUES (p_entity_id, p_meter_slug, p_amount) + ON CONFLICT (entity_id, meter_slug) + DO UPDATE SET balance = billing_public.balances.balance + p_amount; + + -- Append to ledger + INSERT INTO billing_public.ledger (entity_id, meter_slug, amount, metadata) + VALUES (p_entity_id, p_meter_slug, p_amount, p_metadata); +END; +$$ LANGUAGE plpgsql; + +-- ============================================================================ +-- 4. Inference log schema + table +-- ============================================================================ +CREATE SCHEMA IF NOT EXISTS usage_public; + +CREATE TABLE usage_public.usage_log_inferences ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + database_id uuid, + entity_id uuid NOT NULL, + actor_id uuid, + model text NOT NULL, + provider text, + service text NOT NULL, + operation text NOT NULL, + input_tokens bigint NOT NULL DEFAULT 0, + output_tokens bigint NOT NULL DEFAULT 0, + total_tokens bigint NOT NULL DEFAULT 0, + cache_read_tokens bigint, + cache_write_tokens bigint, + latency_ms bigint NOT NULL DEFAULT 0, + rag_enabled boolean NOT NULL DEFAULT false, + chunks_retrieved integer, + embedding_model text, + embedding_latency_ms bigint, + status text NOT NULL DEFAULT 'success', + error_type text, + raw_usage jsonb, + request_id uuid, + created_at timestamptz NOT NULL DEFAULT now() +); + +-- ============================================================================ +-- 5. Seed metaschema config entries +-- database_id = '00000000-0000-0000-0000-000000000001' (test constant) +-- ============================================================================ +INSERT INTO metaschema_public.schema (id, schema_name) VALUES + ('a0000000-0000-0000-0000-000000000001', 'billing_public'), + ('a0000000-0000-0000-0000-000000000002', 'billing_private'), + ('a0000000-0000-0000-0000-000000000003', 'usage_public'); + +INSERT INTO metaschema_modules_public.billing_module (database_id, schema_id, private_schema_id, record_usage_function) +VALUES ( + '00000000-0000-0000-0000-000000000001', + 'a0000000-0000-0000-0000-000000000001', + 'a0000000-0000-0000-0000-000000000002', + 'record_usage' +); + +INSERT INTO metaschema_modules_public.inference_log_module (database_id, schema_id, inference_log_table_name) +VALUES ( + '00000000-0000-0000-0000-000000000001', + 'a0000000-0000-0000-0000-000000000003', + 'usage_log_inferences' +); + +-- ============================================================================ +-- 6. Seed meters for embedding + chat +-- ============================================================================ +INSERT INTO billing_public.meters (slug, display_name, unit, credit_cost) VALUES + ('nomic-embed-text', 'Nomic Embed', 'characters', 1), + ('inference', 'Inference Pool', 'credits', 1); + +-- ============================================================================ +-- 7. Grant access to all roles (tests run as various roles) +-- ============================================================================ +GRANT USAGE ON SCHEMA metaschema_public TO PUBLIC; +GRANT USAGE ON SCHEMA metaschema_modules_public TO PUBLIC; +GRANT USAGE ON SCHEMA billing_public TO PUBLIC; +GRANT USAGE ON SCHEMA billing_private TO PUBLIC; +GRANT USAGE ON SCHEMA usage_public TO PUBLIC; + +GRANT ALL ON ALL TABLES IN SCHEMA metaschema_public TO PUBLIC; +GRANT ALL ON ALL TABLES IN SCHEMA metaschema_modules_public TO PUBLIC; +GRANT ALL ON ALL TABLES IN SCHEMA billing_public TO PUBLIC; +GRANT ALL ON ALL TABLES IN SCHEMA billing_private TO PUBLIC; +GRANT ALL ON ALL TABLES IN SCHEMA usage_public TO PUBLIC; + +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA billing_private TO PUBLIC; From 38d4506570d9185193e9f9356db86eaeb2504384 Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Thu, 21 May 2026 10:38:57 +0000 Subject: [PATCH 2/3] fix: use bash /dev/tcp health check for Ollama container (curl not available) --- .github/workflows/run-tests.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index 5e2bfbede..bf21395dd 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -372,11 +372,11 @@ jobs: ports: - 11434:11434 options: >- - --health-cmd "curl -f http://localhost:11434/ || exit 1" - --health-interval 15s - --health-timeout 10s + --health-cmd "bash -c 'cat < /dev/null > /dev/tcp/localhost/11434'" + --health-interval 10s + --health-timeout 5s --health-retries 10 - --health-start-period 30s + --health-start-period 10s steps: - name: Download workspace From 8cf39389addf001c038ad41c3d421214c83e5a1c Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Thu, 21 May 2026 10:44:07 +0000 Subject: [PATCH 3/3] fix: run only metering-e2e tests in Ollama tier (not full graphile-llm suite) --- .github/workflows/run-tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index bf21395dd..14eeda744 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -416,5 +416,5 @@ jobs: curl -s http://localhost:11434/api/pull -d '{"name":"nomic-embed-text:latest"}' | tail -1 echo "nomic-embed-text pulled" - - name: Test graphile-llm - run: cd ./graphile/graphile-llm && pnpm test + - name: Test graphile-llm metering + run: cd ./graphile/graphile-llm && npx jest metering-e2e --verbose