From 8b0096d8709e90a7706b14e74627a4533f930ab9 Mon Sep 17 00:00:00 2001 From: Justin Butterfield Date: Thu, 2 Jul 2026 11:44:48 -0400 Subject: [PATCH 1/2] feat: add file-based Devin provider to fix ENAMETOOLONG on Windows Add devinProvider.js as a promptfoo custom provider that writes prompts to a temp file and passes them via --prompt-file instead of as a CLI argument. This avoids the Windows ENAMETOOLONG error when prompts exceed the 32k character command-line limit. The provider also uses async spawn and parses token usage from Devin's --export output. The provider entry in promptfooconfig.yaml is added but commented out, ready to be enabled when needed. --- devinProvider.js | 119 +++++++++++++++++++++++++++++++++++++++++++ promptfooconfig.yaml | 6 +++ 2 files changed, 125 insertions(+) create mode 100644 devinProvider.js diff --git a/devinProvider.js b/devinProvider.js new file mode 100644 index 0000000..30b1239 --- /dev/null +++ b/devinProvider.js @@ -0,0 +1,119 @@ +const { spawn } = require('child_process'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); + +class DevinProvider { + constructor(options) { + this.providerId = options.id || 'devin-custom'; + this.config = options.config || {}; + } + + id() { + return this.providerId; + } + + async callApi(prompt, context, options) { + const model = this.config.model || 'SWE-1.6'; + + let isGraderMode = false; + try { + const parsed = JSON.parse(prompt); + if (Array.isArray(parsed) && parsed.length > 0 && parsed[0].role) { + isGraderMode = true; + } + } catch (e) { + // Not JSON, provider mode + } + + let fullPrompt; + + if (isGraderMode) { + let systemMsg, userMsg; + try { + const messages = JSON.parse(prompt); + const systemMessage = messages.find(m => m.role === 'system'); + const userMessage = messages.find(m => m.role === 'user'); + + if (systemMessage && userMessage) { + systemMsg = systemMessage.content; + userMsg = userMessage.content; + } else { + throw new Error('Missing system or user message'); + } + } catch (e) { + systemMsg = 'You are an evaluator. Respond with only valid JSON: {"pass": bool, "score": 0.0-1.0, "reason": "string"}'; + userMsg = prompt; + } + fullPrompt = `${systemMsg}\n\n${userMsg}`; + } else { + fullPrompt = prompt; + } + + const tmpFile = path.join(os.tmpdir(), `devin-prompt-${Date.now()}-${process.pid}.txt`); + const exportFile = path.join(os.tmpdir(), `devin-export-${Date.now()}-${process.pid}.json`); + + try { + fs.writeFileSync(tmpFile, fullPrompt, 'utf8'); + + const args = ['-p', '--prompt-file', tmpFile, '--model', model, '--export', exportFile]; + if (!isGraderMode) { + args.push('--permission-mode', 'auto'); + } + + const result = await new Promise((resolve, reject) => { + const child = spawn('devin', args, { + encoding: 'utf8', + stdio: ['pipe', 'pipe', 'pipe'], + }); + + let stdout = ''; + let stderr = ''; + + child.stdout.on('data', (data) => { stdout += data; }); + child.stderr.on('data', (data) => { stderr += data; }); + + child.on('error', (err) => { + resolve({ error: err.message, tokenUsage: parseTokenUsage(exportFile) }); + }); + + child.on('close', (code) => { + if (code !== 0) { + resolve({ error: stdout || stderr || `devin exited with code ${code}`, tokenUsage: parseTokenUsage(exportFile) }); + } else { + resolve({ output: stdout, tokenUsage: parseTokenUsage(exportFile) }); + } + }); + }); + + return result; + } finally { + try { + fs.unlinkSync(tmpFile); + fs.unlinkSync(exportFile); + } catch (e) { + // Ignore cleanup errors + } + } + } +} + +function parseTokenUsage(exportFilePath) { + try { + const data = JSON.parse(fs.readFileSync(exportFilePath, 'utf8')); + const agentSteps = (data.steps || []).filter(s => s.source === 'agent'); + let promptTokens = 0, completionTokens = 0; + for (const step of agentSteps) { + const m = step.metadata?.metrics; + if (m) { + promptTokens += (m.input_tokens || 0) + (m.cache_read_tokens || 0) + (m.cache_creation_tokens || 0); + completionTokens += (m.output_tokens || 0); + } + } + return { total: promptTokens + completionTokens, prompt: promptTokens, completion: completionTokens }; + } catch { + return { total: 0, prompt: 0, completion: 0 }; + } +} + +module.exports = DevinProvider; diff --git a/promptfooconfig.yaml b/promptfooconfig.yaml index dca66be..4bd1046 100644 --- a/promptfooconfig.yaml +++ b/promptfooconfig.yaml @@ -46,6 +46,12 @@ providers: config: model: 'SWE-1.6' + # Devin CLI provider — file-based (fixes ENAMETOOLONG on Windows for prompts >32k) + #- id: 'file://devinProvider.js' + # label: 'Devin SWE-1.6 (file provider)' + # config: + # model: 'SWE-1.6' + # Devin CLI provider — For Testing a Second Model # MUST Use Label to differentiate providers with same ID (first in wins otherwise) # - id: 'exec: node ./devin.js' From 4b8e031b91054e1b68d0a8c2304b1ea4407adf6e Mon Sep 17 00:00:00 2001 From: Justin Butterfield Date: Wed, 5 Aug 2026 09:07:33 -0400 Subject: [PATCH 2/2] Updated Token usage to support the new ATIF v1.7 transcripts from newr version of devin --- devinProvider.js | 38 +++++++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/devinProvider.js b/devinProvider.js index 30b1239..a1e0da6 100644 --- a/devinProvider.js +++ b/devinProvider.js @@ -101,18 +101,38 @@ class DevinProvider { function parseTokenUsage(exportFilePath) { try { const data = JSON.parse(fs.readFileSync(exportFilePath, 'utf8')); - const agentSteps = (data.steps || []).filter(s => s.source === 'agent'); - let promptTokens = 0, completionTokens = 0; - for (const step of agentSteps) { - const m = step.metadata?.metrics; - if (m) { - promptTokens += (m.input_tokens || 0) + (m.cache_read_tokens || 0) + (m.cache_creation_tokens || 0); - completionTokens += (m.output_tokens || 0); + + // Devin exports ATIF v1.7 transcripts. Token metrics live in step.metrics + // (standard ATIF) with prompt_tokens/completion_tokens, but older exports + // put them in step.metadata.metrics as input_tokens/output_tokens. + const allSteps = (data.steps || []); + const candidateSteps = allSteps.filter(s => s.source === 'agent' || s.source === 'model' || s.source === 'assistant'); + const steps = candidateSteps.length > 0 ? candidateSteps : allSteps.filter(s => !s.metadata?.is_user_input); + + let promptTokens = 0, completionTokens = 0, cachedTokens = 0, cacheCreationTokens = 0; + for (const step of steps) { + const m = step.metrics || step.metadata?.metrics || {}; + if (m && Object.keys(m).length > 0) { + promptTokens += (m.prompt_tokens ?? m.input_tokens ?? 0) + (m.cache_read_input_tokens ?? m.cache_read_tokens ?? 0); + completionTokens += (m.completion_tokens ?? m.output_tokens ?? 0); + cachedTokens += (m.cached_tokens ?? m.cache_read_input_tokens ?? m.cache_read_tokens ?? 0); + cacheCreationTokens += (m.cache_creation_input_tokens ?? m.cache_creation_tokens ?? 0); } } - return { total: promptTokens + completionTokens, prompt: promptTokens, completion: completionTokens }; + + // Legacy fallback: include legacy metadata.metrics cache creation in prompt + // if no ATIF cache_creation_input_tokens was present. + promptTokens += cacheCreationTokens; + + return { + total: promptTokens + completionTokens, + prompt: promptTokens, + completion: completionTokens, + cached: cachedTokens, + numRequests: 1, + }; } catch { - return { total: 0, prompt: 0, completion: 0 }; + return { total: 0, prompt: 0, completion: 0, cached: 0, numRequests: 1 }; } }