From eb4f65d4d5585384e2c6f0745d1df05bfc38aef9 Mon Sep 17 00:00:00 2001 From: Matt Rubens <2600+mrubens@users.noreply.github.com> Date: Fri, 31 Jul 2026 05:52:10 +0000 Subject: [PATCH] fix: prevent large repository clone timeout loops --- apps/docs/environment-variables.mdx | 1 + apps/docs/providers/compute/docker.mdx | 4 + .../__tests__/command-executor.test.ts | 16 +++ .../src/command-executor/command-executor.ts | 20 +++- apps/worker/src/commands/setup.ts | 1 + .../commands/setup/workspace/repositories.ts | 2 + .../src/commands/setup/workspace/shared.ts | 2 + .../src/commands/setup/workspace/types.ts | 2 + .../src/env/__tests__/worker-env.test.ts | 21 ++++ apps/worker/src/env/worker-env.ts | 28 ++++- .../__tests__/workspace-manager-clone.test.ts | 96 ++++++++++++++++- .../worker/src/workspace/workspace-manager.ts | 100 +++++++++++++++--- .../src/worker-env/__tests__/base.test.ts | 10 ++ .../compute-providers/src/worker-env/base.ts | 4 + packages/env/src/__tests__/index.test.ts | 17 +++ packages/env/src/index.ts | 5 + 16 files changed, 307 insertions(+), 22 deletions(-) diff --git a/apps/docs/environment-variables.mdx b/apps/docs/environment-variables.mdx index 997acb80b..c46947e99 100644 --- a/apps/docs/environment-variables.mdx +++ b/apps/docs/environment-variables.mdx @@ -261,6 +261,7 @@ as per-task auth tokens or workspace paths. | `AZURE_SANDBOX_EGRESS_INSPECTION` | Optional | Egress proxy TLS inspection: `Partial` (default — only rule-matched traffic inspected), `Full`, `Legacy`, `None`. Use `Full` only with egress rules/transforms; it TLS-resigns all traffic and blocks non-HTTP. | | `WORKER_RELEASE_CHANNEL` | Optional | Worker release channel, `stable` or `preview`, for hosted worker release selection. | | `WORKER_RELEASE_VERSION` | Optional | Explicit worker release version for hosted worker bootstrap. | +| `WORKER_REPOSITORY_CLONE_TIMEOUT_SECONDS` | Optional | Maximum time allowed for an initial repository clone. Defaults to `600`; increase this for large repositories or slower networks. | | `LOCAL_SANDBOX_FILES_DIR` | Local development | Local override for sandbox bootstrap files. Used only by development workflows. | ### Source control providers diff --git a/apps/docs/providers/compute/docker.mdx b/apps/docs/providers/compute/docker.mdx index 4e4b382d3..ab6802a66 100644 --- a/apps/docs/providers/compute/docker.mdx +++ b/apps/docs/providers/compute/docker.mdx @@ -166,6 +166,10 @@ one to a fix. they include the underlying crash, missing dependency, or fetch failure. - **The task cannot reach Roomote services.** Check `DOCKER_WORKER_NETWORK` and the Compose network used by the API and controller. +- **A large repository times out while cloning.** Increase + `WORKER_REPOSITORY_CLONE_TIMEOUT_SECONDS` from its 600-second default and + restart the Roomote services. Timeout failures stop immediately instead of + restarting the same clone from zero. - **A worker reports that its disk limit is unsupported.** Configure a Docker storage driver with per-container writable-layer quota support. Only set `DOCKER_WORKER_ALLOW_UNBOUNDED_DISK=true` when an equivalent host-level quota diff --git a/apps/worker/src/command-executor/__tests__/command-executor.test.ts b/apps/worker/src/command-executor/__tests__/command-executor.test.ts index a9d0a9f0b..5ed70eeff 100644 --- a/apps/worker/src/command-executor/__tests__/command-executor.test.ts +++ b/apps/worker/src/command-executor/__tests__/command-executor.test.ts @@ -173,6 +173,22 @@ describe('CommandExecutor', () => { expect(result.stdout).toBe('stdin closed'); }); + it('classifies command timeouts in execution diagnostics', async () => { + const executor = new CommandExecutor(mockRepoPath, mockEnv); + const command: Command = { + name: 'Timeout Test', + run: 'sleep 1', + timeout: 0.05, + continue_on_error: false, + }; + + const error = await executor.execute(command).catch((caught) => caught); + + expect(error).toBeInstanceOf(ExecutionError); + expect(error.result.timedOut).toBe(true); + expect(error.formatDetails()).toContain('timeout -> 0.05 seconds'); + }); + it('should execute multi-line commands as separate commands', async () => { const executor = new CommandExecutor(mockRepoPath, mockEnv); diff --git a/apps/worker/src/command-executor/command-executor.ts b/apps/worker/src/command-executor/command-executor.ts index b5ae2cd7e..72cac5eb4 100644 --- a/apps/worker/src/command-executor/command-executor.ts +++ b/apps/worker/src/command-executor/command-executor.ts @@ -38,6 +38,7 @@ export interface ExecutionResult { stdout?: string; stderr?: string; error?: string; + timedOut?: boolean; } export class ExecutionError extends Error { @@ -56,15 +57,25 @@ export class ExecutionError extends Error { formatDetails(): string { const { result } = this; - const truncate = (s: string, max = 4000) => - s.length > max ? '... (truncated)\n' + s.slice(-max) : s; + const sanitize = (value: string) => + value.replace(/(https?:\/\/)[^\s/@]+(?::[^\s/@]*)?@/gi, '$1[redacted]@'); + const truncate = (s: string, max = 4000) => { + const sanitized = sanitize(s); + return sanitized.length > max + ? '... (truncated)\n' + sanitized.slice(-max) + : sanitized; + }; - const parts = [result.command.run]; + const parts = [sanitize(result.command.run)]; if (result.exitCode !== undefined) { parts.push(`\nexit code -> ${result.exitCode}`); } + if (result.timedOut) { + parts.push(`\ntimeout -> ${result.command.timeout} seconds`); + } + if (result.error) { parts.push(`\nerror -> ${truncate(result.error)}`); } @@ -104,6 +115,7 @@ export class CommandExecutor { let duration; let error; let exitCode: number | undefined; + let timedOut = false; let stderr = ''; let stdout = ''; @@ -178,6 +190,7 @@ export class CommandExecutor { if (e instanceof ExecaError) { error = e.shortMessage; exitCode = e.exitCode; + timedOut = e.timedOut; if (e.stdout) { if (verbose) { @@ -209,6 +222,7 @@ export class CommandExecutor { stdout, stderr, error, + timedOut, }; if (!success && !command.continue_on_error) { diff --git a/apps/worker/src/commands/setup.ts b/apps/worker/src/commands/setup.ts index f9766ae66..2b02e3dfe 100644 --- a/apps/worker/src/commands/setup.ts +++ b/apps/worker/src/commands/setup.ts @@ -137,6 +137,7 @@ export async function setup({ ...workspaceOpts, cleanupLegacyPaths: workspaceOpts.taskRunType === TaskPayloadKind.SnapshotEnvironment, + repositoryCloneTimeoutSeconds: workerEnv.repositoryCloneTimeoutSeconds, envVars: { ...workerEnv.buildUserFacingEnv(), ...workspaceOpts.envVars, diff --git a/apps/worker/src/commands/setup/workspace/repositories.ts b/apps/worker/src/commands/setup/workspace/repositories.ts index c25d05196..1af8d154a 100644 --- a/apps/worker/src/commands/setup/workspace/repositories.ts +++ b/apps/worker/src/commands/setup/workspace/repositories.ts @@ -70,6 +70,7 @@ export async function initializeRepositories( gitAuthorName, gitAuthorEmail, sourceControlProvider, + repositoryCloneTimeoutSeconds, }: PrepareWorkspaceOptions, ): Promise { const resolvedSourceControlProvider = @@ -81,6 +82,7 @@ export async function initializeRepositories( const { workspaceRoot, workspaceManager } = createWorkspaceManager( envVars, logger, + repositoryCloneTimeoutSeconds, ); await timedStep(logger, 'initializeRepositories: configure git', () => diff --git a/apps/worker/src/commands/setup/workspace/shared.ts b/apps/worker/src/commands/setup/workspace/shared.ts index 894f7695c..b8af910e1 100644 --- a/apps/worker/src/commands/setup/workspace/shared.ts +++ b/apps/worker/src/commands/setup/workspace/shared.ts @@ -39,6 +39,7 @@ export function resolveRuntimePathsForWorker() { export function createWorkspaceManager( envVars: Record, logger?: StartupLogger, + repositoryCloneTimeoutSeconds?: number, ): { workspaceRoot: string; workspaceManager: WorkspaceManager; @@ -53,6 +54,7 @@ export function createWorkspaceManager( logger ? (label, fn) => timedStep(logger, `initializeRepositories: ${label}`, fn) : undefined, + { repositoryCloneTimeoutSeconds }, ); return { workspaceRoot, workspaceManager }; diff --git a/apps/worker/src/commands/setup/workspace/types.ts b/apps/worker/src/commands/setup/workspace/types.ts index 64750c34e..173cd9131 100644 --- a/apps/worker/src/commands/setup/workspace/types.ts +++ b/apps/worker/src/commands/setup/workspace/types.ts @@ -69,6 +69,8 @@ export interface PrepareWorkspaceOptions { serviceContext?: ServiceContext; gitAuthorName?: string; gitAuthorEmail?: string; + /** Deployment-managed clone deadline; never exposed to project commands. */ + repositoryCloneTimeoutSeconds?: number; } function formatWorkspaceRepositoryPreparationMessage( diff --git a/apps/worker/src/env/__tests__/worker-env.test.ts b/apps/worker/src/env/__tests__/worker-env.test.ts index cf13dca1d..31f88657d 100644 --- a/apps/worker/src/env/__tests__/worker-env.test.ts +++ b/apps/worker/src/env/__tests__/worker-env.test.ts @@ -315,6 +315,27 @@ describe('WorkerEnv', () => { expect(env.buildUserFacingEnv().DOCKER_HOST).toBe('tcp://127.0.0.1:2375'); }); + it('captures clone timeout configuration without exposing it to task processes', () => { + const processEnv = { + HOME: '/home/worker', + PATH: '/usr/bin', + AUTH_TOKEN: 'my-auth-token', + TRPC_URL: 'https://trpc.example.com', + R_APP_URL: 'https://api.example.com', + WORKER_REPOSITORY_CLONE_TIMEOUT_SECONDS: '1200', + } as NodeJS.ProcessEnv; + + const env = WorkerEnv.fromProcessEnv(processEnv); + + expect(env.repositoryCloneTimeoutSeconds).toBe(1_200); + expect( + processEnv.WORKER_REPOSITORY_CLONE_TIMEOUT_SECONDS, + ).toBeUndefined(); + expect(env.buildUserFacingEnv()).not.toHaveProperty( + 'WORKER_REPOSITORY_CLONE_TIMEOUT_SECONDS', + ); + }); + it('should keep sandbox auth validation working after process.env cleanup', async () => { const { privateKey, publicKey } = generateKeyPairSync('ec', { namedCurve: 'P-256', diff --git a/apps/worker/src/env/worker-env.ts b/apps/worker/src/env/worker-env.ts index a32058c31..4fa1e9c5c 100644 --- a/apps/worker/src/env/worker-env.ts +++ b/apps/worker/src/env/worker-env.ts @@ -2,6 +2,7 @@ import * as os from 'node:os'; import { configureAuthClientEnv } from '@roomote/auth/client'; import { + COMMAND_DEFAULT_TIMEOUT, DEFAULT_MODEL_PROVIDER_ENV_KEYS, OPENCODE_AUTH_CONTENT_ENV_VAR_NAME, parseModelProviderEnvKeys, @@ -21,6 +22,7 @@ interface WorkerConfig { previewAuthPublicKey?: string; previewAuthCookieName?: string; appEnv?: string; + repositoryCloneTimeoutSeconds?: number; } const PRESET_SYSTEM_ENV: Record = { @@ -45,7 +47,12 @@ const SYSTEM_KEYS = [ // so nested application commands do not inherit it accidentally. This includes // the legacy ROOMOTE_APP_ENV alias the controller still injects for pre-rename // snapshot workers. -const WORKER_INTERNAL_CONFIG_KEYS = ['R_APP_ENV', 'APP_ENV', 'ROOMOTE_APP_ENV']; +const WORKER_INTERNAL_CONFIG_KEYS = [ + 'R_APP_ENV', + 'APP_ENV', + 'ROOMOTE_APP_ENV', + 'WORKER_REPOSITORY_CLONE_TIMEOUT_SECONDS', +]; const BLOCKED_USER_FACING_ENV_KEYS = new Set([ 'AUTH_TOKEN', 'TRPC_URL', @@ -111,6 +118,15 @@ function buildLauncherOpenCodeEnv( return env; } +function parsePositiveInteger( + value: string | undefined, + fallback: number, +): number { + const parsed = Number(value); + + return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback; +} + /** * Centralized environment manager for the worker. * @@ -224,6 +240,10 @@ export class WorkerEnv { previewAuthCookieName: processEnv.PREVIEW_AUTH_COOKIE_NAME, roomoteAppUrl: processEnv.R_APP_URL!, appEnv: processEnv.R_APP_ENV ?? processEnv.APP_ENV, + repositoryCloneTimeoutSeconds: parsePositiveInteger( + processEnv.WORKER_REPOSITORY_CLONE_TIMEOUT_SECONDS, + COMMAND_DEFAULT_TIMEOUT, + ), }; const env = new WorkerEnv({ @@ -405,4 +425,10 @@ export class WorkerEnv { get appEnv(): string | undefined { return this.workerConfig.appEnv; } + + get repositoryCloneTimeoutSeconds(): number { + return ( + this.workerConfig.repositoryCloneTimeoutSeconds ?? COMMAND_DEFAULT_TIMEOUT + ); + } } diff --git a/apps/worker/src/workspace/__tests__/workspace-manager-clone.test.ts b/apps/worker/src/workspace/__tests__/workspace-manager-clone.test.ts index cbdd45edd..6170d692f 100644 --- a/apps/worker/src/workspace/__tests__/workspace-manager-clone.test.ts +++ b/apps/worker/src/workspace/__tests__/workspace-manager-clone.test.ts @@ -5,6 +5,7 @@ import { mkdir, mkdtemp, rm, writeFile } from 'fs/promises'; import { tmpdir } from 'os'; import { join } from 'path'; +import { ExecutionError } from '../../command-executor'; import { WorkspaceManager } from '../workspace-manager'; vi.mock('@roomote/sdk/client', () => ({ @@ -80,8 +81,13 @@ describe('WorkspaceManager repository clone preparation', () => { await rm(workspaceRoot, { recursive: true, force: true }); }); - function createManager(env: NodeJS.ProcessEnv = { PATH: '/usr/bin' }) { - const manager = new WorkspaceManager(workspaceRoot, env); + function createManager( + env: NodeJS.ProcessEnv = { PATH: '/usr/bin' }, + repositoryCloneTimeoutSeconds?: number, + ) { + const manager = new WorkspaceManager(workspaceRoot, env, false, undefined, { + repositoryCloneTimeoutSeconds, + }); // Stub the post-clone steps that shell out or hit the network; these // tests only cover the credential fail-fast and clone/cleanup behavior. @@ -148,6 +154,92 @@ describe('WorkspaceManager repository clone preparation', () => { expect(clone!.run).toBe( `rm -rf -- 'acme/backend' && git clone 'https://github.com/acme/backend.git' 'acme/backend'`, ); + expect(clone).toMatchObject({ timeout: 600 }); + }); + + it('uses the deployment-configured clone timeout', async () => { + stubTokenFile({ nonEmpty: true }); + mockCloneCreatesRepo(); + const manager = createManager({ PATH: '/usr/bin' }, 1_200); + + await manager.prepareRepository(REPO_FULL_NAME, 'main', undefined); + + expect(getCloneCommand()).toMatchObject({ timeout: 1_200 }); + }); + + it('fails immediately with an actionable diagnostic when clone times out', async () => { + stubTokenFile({ nonEmpty: true }); + executeMock.mockImplementation(async (command) => { + if (command.name === 'Git clone') { + throw new ExecutionError('Command timed out', { + command, + success: false, + duration: 600_000, + stderr: 'Receiving objects: 42% (420/1000)', + error: 'Command timed out', + timedOut: true, + }); + } + + return { success: true, stdout: '', stderr: '' }; + }); + const manager = createManager({ PATH: '/usr/bin' }, 600); + + await expect( + manager.prepareRepository(REPO_FULL_NAME, 'main', undefined), + ).rejects.toThrow( + /timed out after 600 seconds.*WORKER_REPOSITORY_CLONE_TIMEOUT_SECONDS/s, + ); + + expect( + executeMock.mock.calls.filter( + ([command]) => command.name === 'Git clone', + ), + ).toHaveLength(1); + }); + + it('retries transient clone failures with sanitized diagnostics', async () => { + stubTokenFile({ nonEmpty: true }); + let cloneAttempts = 0; + executeMock.mockImplementation(async (command) => { + if (command.name === 'Git clone') { + cloneAttempts += 1; + if (cloneAttempts === 1) { + throw new ExecutionError('Clone failed', { + command, + success: false, + duration: 100, + exitCode: 128, + stderr: + "fatal: unable to access 'https://token:secret@example.com/acme/backend.git'", + error: 'Command failed with exit code 128', + }); + } + + await mkdir(join(workspaceRoot, REPO_FULL_NAME, '.git'), { + recursive: true, + }); + } + + return { success: true, stdout: '', stderr: '' }; + }); + const manager = createManager(); + const sleepSpy = vi + .spyOn( + manager as unknown as { sleep: (ms: number) => Promise }, + 'sleep', + ) + .mockResolvedValue(undefined); + const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}); + + await manager.prepareRepository(REPO_FULL_NAME, 'main', undefined); + + expect(cloneAttempts).toBe(2); + expect(sleepSpy).toHaveBeenCalledWith(2_000); + expect(warnSpy).toHaveBeenCalledWith( + expect.stringContaining('https://[redacted]@example.com'), + ); + expect(warnSpy).not.toHaveBeenCalledWith(expect.stringContaining('secret')); }); it('materializes an env GH_TOKEN into the token file before cloning', async () => { diff --git a/apps/worker/src/workspace/workspace-manager.ts b/apps/worker/src/workspace/workspace-manager.ts index c7df19615..e3b5ac97e 100644 --- a/apps/worker/src/workspace/workspace-manager.ts +++ b/apps/worker/src/workspace/workspace-manager.ts @@ -55,6 +55,8 @@ type WorkspaceTimingRecorder = ( const SKILLS_INSTALL_COMMAND = 'npx -y skills add'; const REPOSITORY_FETCH_MAX_ATTEMPTS = 5; const REPOSITORY_FETCH_INITIAL_RETRY_DELAY_MS = 2_000; +const REPOSITORY_CLONE_MAX_ATTEMPTS = 5; +const REPOSITORY_CLONE_INITIAL_RETRY_DELAY_MS = 2_000; const RESET_LOCAL_CHANGES_COMMAND = 'if git rev-parse --verify HEAD >/dev/null 2>&1; then git reset --hard HEAD; fi'; const REPOSITORY_WORKTREE_SYNC_TIMEOUT = COMMAND_DEFAULT_TIMEOUT; @@ -141,6 +143,9 @@ export class WorkspaceManager { private readonly env: NodeJS.ProcessEnv, private readonly verbose = false, private readonly recordTiming?: WorkspaceTimingRecorder, + private readonly options: { + repositoryCloneTimeoutSeconds?: number; + } = {}, ) {} private async timed(label: string, fn: () => Promise | T): Promise { @@ -669,22 +674,25 @@ export class WorkspaceManager { await mkdir(dirname(resolvedRepoPath), { recursive: true }); await this.timed(`prepare ${fullName}: clone`, () => - executor.execute({ - name: 'Git clone', - // Use git transport directly so clone works with the worker's - // file-backed credential helper and avoids gh's extra API lookup. - // Escape both args: cloneUrl is provider-synced and may include - // shell metacharacters from a hostile self-hosted origin. - // Retries re-run this same line, and a clone killed mid-transfer - // (e.g. by the timeout) leaves a partial target directory that - // would make every retry fail with "destination path already - // exists" — remove it first. Safe: needsClone guarantees the path - // did not exist before the first attempt. - run: `rm -rf -- '${shellEscape(fullName)}' && git clone '${shellEscape(cloneUrl)}' '${shellEscape(fullName)}'`, - // Allow brief auth and repository visibility propagation delays. - retries: 4, - timeout: 300, - continue_on_error: false, + this.cloneRepositoryWithRetry({ + executor, + repoFullName: fullName, + command: { + name: 'Git clone', + // Use git transport directly so clone works with the worker's + // file-backed credential helper and avoids gh's extra API lookup. + // Escape both args: cloneUrl is provider-synced and may include + // shell metacharacters from a hostile self-hosted origin. + // Retries re-run this same line, and a failed clone leaves a partial + // target directory that would make the next attempt fail with + // "destination path already exists". Safe: needsClone guarantees + // the path did not exist before the first attempt. + run: `rm -rf -- '${shellEscape(fullName)}' && git clone '${shellEscape(cloneUrl)}' '${shellEscape(fullName)}'`, + timeout: + this.options.repositoryCloneTimeoutSeconds ?? + COMMAND_DEFAULT_TIMEOUT, + continue_on_error: false, + }, }), ); } @@ -1249,6 +1257,66 @@ export class WorkspaceManager { } } + private async cloneRepositoryWithRetry({ + executor, + repoFullName, + command, + }: { + executor: CommandExecutor; + repoFullName: string; + command: Parameters[0]; + }): Promise { + for ( + let attempt = 1; + attempt <= REPOSITORY_CLONE_MAX_ATTEMPTS; + attempt += 1 + ) { + try { + await executor.execute(command); + return; + } catch (error) { + if (!(error instanceof ExecutionError)) { + throw error; + } + + if (error.result.timedOut) { + throw new ExecutionError( + `Git clone for ${repoFullName} timed out after ${command.timeout} seconds. Increase WORKER_REPOSITORY_CLONE_TIMEOUT_SECONDS for repositories that need more time.`, + error.result, + ); + } + + if (attempt === REPOSITORY_CLONE_MAX_ATTEMPTS) { + throw error; + } + + const delayMs = + REPOSITORY_CLONE_INITIAL_RETRY_DELAY_MS * 2 ** (attempt - 1); + const failureReason = this.formatRepositoryFailureReason(error); + + console.warn( + `[WorkspaceManager] Git clone attempt ${attempt}/${REPOSITORY_CLONE_MAX_ATTEMPTS} failed for ${repoFullName}: ${failureReason}. Retrying in ${delayMs}ms...`, + ); + + await this.sleep(delayMs); + } + } + } + + private formatRepositoryFailureReason(error: ExecutionError): string { + const reason = + error.result.stderr?.trim() || error.result.error || error.message; + const sanitized = reason.replace( + /(https?:\/\/)[^\s/@]+(?::[^\s/@]*)?@/gi, + '$1[redacted]@', + ); + const maxLength = 1_000; + + return sanitized.length > maxLength + ? `... (truncated) ${sanitized.slice(-maxLength)}` + : sanitized; + } + public async executeEnvironmentRepositoryCommands( repositories: EnvironmentRepositoryConfig[], repoPaths: Record, diff --git a/packages/compute-providers/src/worker-env/__tests__/base.test.ts b/packages/compute-providers/src/worker-env/__tests__/base.test.ts index 78b67666a..11ab6ff9f 100644 --- a/packages/compute-providers/src/worker-env/__tests__/base.test.ts +++ b/packages/compute-providers/src/worker-env/__tests__/base.test.ts @@ -2,6 +2,7 @@ vi.mock('@roomote/env', () => ({ Env: { R_APP_URL: 'https://web.roomote.example.com', TRPC_URL: 'https://api.roomote.example.com', + WORKER_REPOSITORY_CLONE_TIMEOUT_SECONDS: 600, }, })); @@ -56,6 +57,15 @@ describe('buildBaseWorkerEnv', () => { expect(env.ROOMOTE_APP_URL).toBe(env.R_APP_URL); }); + it('forwards deployment clone timeout configuration to workers', () => { + const env = buildBaseWorkerEnv({ + authToken: 'auth-token', + extraEnv: {}, + }); + + expect(env.WORKER_REPOSITORY_CLONE_TIMEOUT_SECONDS).toBe('600'); + }); + it('forwards an explicit preview proxy base URL', () => { process.env.PREVIEW_PROXY_BASE_URL = 'https://preview.example.com'; diff --git a/packages/compute-providers/src/worker-env/base.ts b/packages/compute-providers/src/worker-env/base.ts index cd996bb0c..82fc148c4 100644 --- a/packages/compute-providers/src/worker-env/base.ts +++ b/packages/compute-providers/src/worker-env/base.ts @@ -18,6 +18,7 @@ const BLOCKED_WORKER_ENV_KEYS = new Set([ 'DASHBOARD_PASSWORD', 'SETUP_TOKEN', 'MODAL_TOKEN_SECRET', + 'WORKER_REPOSITORY_CLONE_TIMEOUT_SECONDS', ...DISABLED_MODEL_PROVIDER_ENV_VAR_NAMES, ]); @@ -152,6 +153,9 @@ export function buildBaseWorkerEnv({ // Remove once pre-rename snapshots have aged out. ROOMOTE_APP_URL: Env.R_APP_URL, TRPC_URL: Env.TRPC_URL, + WORKER_REPOSITORY_CLONE_TIMEOUT_SECONDS: String( + Env.WORKER_REPOSITORY_CLONE_TIMEOUT_SECONDS, + ), SKIP_ENV_VALIDATION: '1', // These are launcher-to-worker transport values. Keep them tied to the // current process env instead of the shared Env snapshot because the worker diff --git a/packages/env/src/__tests__/index.test.ts b/packages/env/src/__tests__/index.test.ts index ce498505e..c53677d7e 100644 --- a/packages/env/src/__tests__/index.test.ts +++ b/packages/env/src/__tests__/index.test.ts @@ -126,6 +126,7 @@ describe('Env', () => { expect(env.DOCKER_STANDBY_MAX_AGE_HOURS).toBe(24); expect(env.BLAXEL_STANDBY_MAX_COUNT).toBe(25); expect(env.BLAXEL_STANDBY_MAX_AGE_HOURS).toBe(168); + expect(env.WORKER_REPOSITORY_CLONE_TIMEOUT_SECONDS).toBe(600); expect(env.R_MODEL).toBeUndefined(); expect(env.R_SMALL_MODEL).toBeUndefined(); expect(env.R_VISION_MODEL).toBeUndefined(); @@ -155,6 +156,22 @@ describe('Env', () => { } }); + it('validates repository clone timeout overrides', () => { + expect( + createRoomoteEnv({ + ...productionCoreEnv, + WORKER_REPOSITORY_CLONE_TIMEOUT_SECONDS: '1200', + }).WORKER_REPOSITORY_CLONE_TIMEOUT_SECONDS, + ).toBe(1_200); + + expect(() => + createRoomoteEnv({ + ...productionCoreEnv, + WORKER_REPOSITORY_CLONE_TIMEOUT_SECONDS: '0', + }), + ).toThrow(); + }); + it('requires an explicit opt-in for unbounded Docker task disks', () => { const runtimeEnv: NodeJS.ProcessEnv = { ...process.env, diff --git a/packages/env/src/index.ts b/packages/env/src/index.ts index 8e9519db8..ad44f3dae 100644 --- a/packages/env/src/index.ts +++ b/packages/env/src/index.ts @@ -165,6 +165,11 @@ const serverSchema = { GITLAB_WEBHOOK_SIGNING_TOKEN: emptyStringDefault(), WORKER_RELEASE_CHANNEL: z.enum(['stable', 'preview']).optional(), WORKER_RELEASE_VERSION: z.string().min(1).optional(), + WORKER_REPOSITORY_CLONE_TIMEOUT_SECONDS: z.coerce + .number() + .int() + .positive() + .default(600), SLACK_APP_ID: emptyStringDefault(), R_SLACK_CLIENT_ID: z.string().min(1).optional(), R_SLACK_CLIENT_SECRET: z.string().min(1).optional(),