diff --git a/lambdas/functions/control-plane/src/github/auth.ts b/lambdas/functions/control-plane/src/github/auth.ts index f64ac00b30..3e177ab253 100644 --- a/lambdas/functions/control-plane/src/github/auth.ts +++ b/lambdas/functions/control-plane/src/github/auth.ts @@ -1,11 +1,21 @@ import { createAppAuth, type AppAuthentication, type InstallationAccessTokenAuthentication } from '@octokit/auth-app'; import type { OctokitOptions, Octokit as CoreOctokit } from '@octokit/core'; import type { RequestInterface } from '@octokit/types'; +import { createSign, randomUUID } from 'node:crypto'; +import { request } from '@octokit/request'; +import { Octokit } from '@octokit/rest'; +import { retry } from '@octokit/plugin-retry'; +import { throttling } from '@octokit/plugin-throttling'; +import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; +import { + createCommonStorage, + type GitHubAppCredential, + type GitHubAppCredentialsStore, +} from '@aws-github-runner/storage-providers'; +import { EndpointDefaults } from '@octokit/types'; -// Define types that are not directly exported type AppAuthOptions = { type: 'app' }; type InstallationAuthOptions = { type: 'installation'; installationId?: number }; -// Use a more generalized AuthInterface to match what createAppAuth returns type AuthInterface = { (options: AppAuthOptions): Promise; (options: InstallationAuthOptions): Promise; @@ -16,33 +26,14 @@ type StrategyOptions = { installationId?: number; request?: RequestInterface; }; -import { createSign, randomUUID } from 'node:crypto'; -import { request } from '@octokit/request'; -import { Octokit } from '@octokit/rest'; -import { retry } from '@octokit/plugin-retry'; -import { throttling } from '@octokit/plugin-throttling'; -import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; -import { getParameters } from '@aws-github-runner/aws-ssm-util'; -import { EndpointDefaults } from '@octokit/types'; const logger = createChildLogger('gh-auth'); - -// Retry caps for the throttling plugin. Returning `true` from a limit handler tells -// the plugin to retry after the interval GitHub asked for; returning `false` gives up. -// Primary rate limits reset on a fixed schedule, so a couple of retries is worthwhile. -// Secondary rate limits are abuse-detection signals — retry once, then back off and -// let the message return to the queue rather than pushing harder. const MAX_RATE_LIMIT_RETRIES = 2; const MAX_SECONDARY_RATE_LIMIT_RETRIES = 1; -// Exported for tests: the plugin only surfaces these via the client constructor, -// so there is no other seam to assert the retry cap against. export function onRateLimit( retryAfter: number, options: Required, - // The throttling plugin types this as @octokit/core's Octokit, not the wider - // @octokit/rest one imported above; matching it keeps the handler assignable to - // the plugin's LimitHandler. Unused here regardless. _octokit: CoreOctokit, retryCount: number, ): boolean { @@ -56,9 +47,6 @@ export function onRateLimit( export function onSecondaryRateLimit( retryAfter: number, options: Required, - // The throttling plugin types this as @octokit/core's Octokit, not the wider - // @octokit/rest one imported above; matching it keeps the handler assignable to - // the plugin's LimitHandler. Unused here regardless. _octokit: CoreOctokit, retryCount: number, ): boolean { @@ -69,102 +57,69 @@ export function onSecondaryRateLimit( return retryCount < MAX_SECONDARY_RATE_LIMIT_RETRIES; } -interface GitHubAppCredential { - appId: number; - privateKey: string; - installationId?: number; -} - let appCredentialsPromise: Promise | null = null; async function loadAppCredentials(): Promise { - if (!process.env.PARAMETER_GITHUB_APP_ID_NAME) { - throw new Error('Environment variable PARAMETER_GITHUB_APP_ID_NAME is not set'); - } - if (!process.env.PARAMETER_GITHUB_APP_KEY_BASE64_NAME) { - throw new Error('Environment variable PARAMETER_GITHUB_APP_KEY_BASE64_NAME is not set'); - } - const idParams = process.env.PARAMETER_GITHUB_APP_ID_NAME.split(':').filter(Boolean); - const keyParams = process.env.PARAMETER_GITHUB_APP_KEY_BASE64_NAME.split(':').filter(Boolean); - const installationIdParams = (process.env.PARAMETER_GITHUB_APP_INSTALLATION_ID_NAME || '').split(':'); - if (idParams.length !== keyParams.length) { - throw new Error(`GitHub App parameter count mismatch: ${idParams.length} IDs vs ${keyParams.length} keys`); - } - // Batch fetch all SSM parameters in a single call to reduce API calls - const allParamNames = [...idParams, ...keyParams, ...installationIdParams.filter((p) => p.length > 0)]; - const params = await getParameters(allParamNames); - - const credentials: GitHubAppCredential[] = []; - for (let i = 0; i < idParams.length; i++) { - const appIdValue = params.get(idParams[i]); - if (!appIdValue) { - throw new Error(`Parameter ${idParams[i]} not found`); - } - const appId = parseInt(appIdValue, 10); - const privateKeyBase64 = params.get(keyParams[i]); - if (!privateKeyBase64) { - throw new Error(`Parameter ${keyParams[i]} not found`); - } - // replace literal \n characters with new lines to allow the key to be stored as a - // single line variable. This logic should match how the GitHub Terraform provider - // processes private keys to retain compatibility between the projects - const privateKey = Buffer.from(privateKeyBase64, 'base64').toString().replace(/\\n/g, '\n'); - const installationIdParam = installationIdParams[i]; - const installationIdValue = - installationIdParam && installationIdParam.length > 0 ? params.get(installationIdParam) : undefined; - const installationId = installationIdValue ? parseInt(installationIdValue, 10) : undefined; - credentials.push({ appId, privateKey, installationId }); - } + const credentials = await createCommonStorage().githubAppCredentials.get(); logger.info(`Loaded ${credentials.length} GitHub App credential(s)`); return credentials; } -function getAppCredentials(): Promise { +function getAppCredentials(credentialsStore?: GitHubAppCredentialsStore): Promise { + if (credentialsStore) { + return credentialsStore.get(); + } if (!appCredentialsPromise) appCredentialsPromise = loadAppCredentials(); return appCredentialsPromise; } -export async function getAppCount(): Promise { - return (await getAppCredentials()).length; +export async function getAppCount(credentialsStore?: GitHubAppCredentialsStore): Promise { + return (await getAppCredentials(credentialsStore)).length; } export function resetAppCredentialsCache(): void { appCredentialsPromise = null; } -export async function getStoredInstallationId(appIndex: number): Promise { - const credentials = await getAppCredentials(); +export async function getStoredInstallationId( + appIndex: number, + credentialsStore?: GitHubAppCredentialsStore, +): Promise { + const credentials = await getAppCredentials(credentialsStore); return credentials[appIndex]?.installationId; } +export async function getAppId(appIndex = 0, credentialsStore?: GitHubAppCredentialsStore): Promise { + const credential = (await getAppCredentials(credentialsStore))[appIndex]; + if (!credential) { + throw new Error(`GitHub App credential at index ${appIndex} not found`); + } + return credential.appId.toString(); +} + export async function createOctokitClient(token: string, ghesApiUrl = ''): Promise { const CustomOctokit = Octokit.plugin(retry, throttling); - const ocktokitOptions: OctokitOptions = { - auth: token, - }; + const octokitOptions: OctokitOptions = { auth: token }; if (ghesApiUrl) { - ocktokitOptions.baseUrl = ghesApiUrl; - ocktokitOptions.previews = ['antiope']; + octokitOptions.baseUrl = ghesApiUrl; + octokitOptions.previews = ['antiope']; } return new CustomOctokit({ - ...ocktokitOptions, + ...octokitOptions, userAgent: process.env.USER_AGENT || 'github-aws-runners', retry: { - onRetry: (retryCount: number, error: Error, request: { method: string; url: string }) => { + onRetry: (retryCount: number, error: Error, retryRequest: { method: string; url: string }) => { logger.warn('GitHub API request retry attempt', { retryCount, - method: request.method, - url: request.url, + method: retryRequest.method, + url: retryRequest.url, error: error.message, status: (error as Error & { status?: number }).status, }); }, }, - throttle: { - onRateLimit, - onSecondaryRateLimit, - }, + throttle: { onRateLimit, onSecondaryRateLimit }, }); } @@ -172,22 +127,23 @@ export async function createGithubAppAuth( installationId: number | undefined, ghesApiUrl = '', appIndex?: number, + credentialsStore?: GitHubAppCredentialsStore, ): Promise { - const credentials = await getAppCredentials(); + const credentials = await getAppCredentials(credentialsStore); const idx = appIndex ?? Math.floor(Math.random() * credentials.length); - const auth = await createAuth(installationId, ghesApiUrl, idx); - const result = await auth({ type: 'app' }); - return { ...result, appIndex: idx }; + const auth = await createAuth(installationId, ghesApiUrl, idx, credentialsStore); + return { ...(await auth({ type: 'app' })), appIndex: idx }; } export async function createGithubInstallationAuth( installationId: number | undefined, ghesApiUrl = '', appIndex?: number, + credentialsStore?: GitHubAppCredentialsStore, ): Promise { - const credentials = await getAppCredentials(); + const credentials = await getAppCredentials(credentialsStore); const idx = appIndex ?? Math.floor(Math.random() * credentials.length); - const auth = await createAuth(installationId, ghesApiUrl, idx); + const auth = await createAuth(installationId, ghesApiUrl, idx, credentialsStore); return auth({ type: 'installation', installationId }); } @@ -203,17 +159,16 @@ async function createAuth( installationId: number | undefined, ghesApiUrl: string, appIndex?: number, + credentialsStore?: GitHubAppCredentialsStore, ): Promise { - const credentials = await getAppCredentials(); + const credentials = await getAppCredentials(credentialsStore); const selected = appIndex !== undefined ? credentials[appIndex] : credentials[Math.floor(Math.random() * credentials.length)]; + if (!selected) { + throw new Error(`GitHub App credential at index ${appIndex ?? 0} not found`); + } logger.debug(`Selected GitHub App ${selected.appId} for authentication`); - - // Use a custom createJwt callback to include a jti (JWT ID) claim in every token. - // Without this, concurrent Lambda invocations generating JWTs within the same second - // produce byte-identical tokens (same iat, exp, iss), which GitHub rejects as duplicates. - // See: https://github.com/github-aws-runners/terraform-aws-github-runner/issues/5025 const createJwt = async (appId: string | number, timeDifference?: number) => { const now = Math.floor(Date.now() / 1000) + (timeDifference ?? 0); const iat = now - 30; @@ -222,14 +177,13 @@ async function createAuth( return { jwt, expiresAt: new Date(exp * 1000).toISOString() }; }; - let authOptions: StrategyOptions = { appId: selected.appId, createJwt }; - if (installationId) authOptions = { ...authOptions, installationId }; - - logger.debug(`GHES API URL: ${ghesApiUrl}`); + const authOptions: StrategyOptions = { + appId: selected.appId, + createJwt, + ...(installationId ? { installationId } : {}), + }; if (ghesApiUrl) { - authOptions.request = request.defaults({ - baseUrl: ghesApiUrl, - }); + authOptions.request = request.defaults({ baseUrl: ghesApiUrl }); } return createAppAuth(authOptions); } diff --git a/lambdas/functions/control-plane/src/github/rate-limit.test.ts b/lambdas/functions/control-plane/src/github/rate-limit.test.ts index d9d18c5921..40c83ff6a3 100644 --- a/lambdas/functions/control-plane/src/github/rate-limit.test.ts +++ b/lambdas/functions/control-plane/src/github/rate-limit.test.ts @@ -1,174 +1,64 @@ import { ResponseHeaders } from '@octokit/types'; import { createSingleMetric } from '@aws-github-runner/aws-powertools-util'; import { MetricUnit } from '@aws-lambda-powertools/metrics'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { getAppId } from './auth'; import { metricGitHubAppRateLimit } from './rate-limit'; -import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; -import { getParameter } from '@aws-github-runner/aws-ssm-util'; -process.env.PARAMETER_GITHUB_APP_ID_NAME = 'test'; -vi.mock('@aws-github-runner/aws-ssm-util', async () => { - // Return only what we need without spreading actual - return { - getParameter: vi.fn((name: string) => { - if (name === process.env.PARAMETER_GITHUB_APP_ID_NAME) { - return '1234'; - } else { - return ''; - } - }), - }; -}); +vi.mock('./auth', () => ({ + getAppId: vi.fn(), +})); +vi.mock('@aws-github-runner/aws-powertools-util', () => ({ + logger: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + createSingleMetric: vi.fn(() => ({ addMetadata: vi.fn() })), +})); -vi.mock('@aws-github-runner/aws-powertools-util', async () => { - // Provide only what's needed without spreading actual - return { - // Mock the logger - logger: { - debug: vi.fn(), - info: vi.fn(), - warn: vi.fn(), - error: vi.fn(), - }, - // eslint-disable-next-line @typescript-eslint/no-unused-vars - createSingleMetric: vi.fn((name: string, unit: string, value: number, dimensions?: Record) => { - return { - addMetadata: vi.fn(), - }; - }), - }; -}); +const mockedGetAppId = vi.mocked(getAppId); describe('metricGitHubAppRateLimit', () => { beforeEach(() => { vi.clearAllMocks(); + mockedGetAppId.mockResolvedValue('1234'); + process.env.ENABLE_METRIC_GITHUB_APP_RATE_LIMIT = 'true'; }); - it('should update rate limit metric', async () => { - // set process.env.ENABLE_METRIC_GITHUB_APP_RATE_LIMIT to true - process.env.ENABLE_METRIC_GITHUB_APP_RATE_LIMIT = 'true'; + it('updates the rate limit metric using the selected app credential', async () => { const headers: ResponseHeaders = { 'x-ratelimit-remaining': '10', 'x-ratelimit-limit': '60', }; - await metricGitHubAppRateLimit(headers); + await metricGitHubAppRateLimit(headers, 1); + expect(mockedGetAppId).toHaveBeenCalledWith(1); expect(createSingleMetric).toHaveBeenCalledWith('GitHubAppRateLimitRemaining', MetricUnit.Count, 10, { AppId: '1234', }); }); - it('should not update rate limit metric', async () => { - // set process.env.ENABLE_METRIC_GITHUB_APP_RATE_LIMIT to false + it('does not update the metric when disabled', async () => { process.env.ENABLE_METRIC_GITHUB_APP_RATE_LIMIT = 'false'; - const headers: ResponseHeaders = { - 'x-ratelimit-remaining': '10', - 'x-ratelimit-limit': '60', - }; - await metricGitHubAppRateLimit(headers); + await metricGitHubAppRateLimit({ 'x-ratelimit-remaining': '10', 'x-ratelimit-limit': '60' }); expect(createSingleMetric).not.toHaveBeenCalled(); + expect(mockedGetAppId).not.toHaveBeenCalled(); }); - it('should not update rate limit metric if headers are undefined', async () => { - // set process.env.ENABLE_METRIC_GITHUB_APP_RATE_LIMIT to true - process.env.ENABLE_METRIC_GITHUB_APP_RATE_LIMIT = 'true'; - - await metricGitHubAppRateLimit(undefined as unknown as ResponseHeaders); - + it('does not throw when headers are unavailable', async () => { + await expect(metricGitHubAppRateLimit(undefined as unknown as ResponseHeaders)).resolves.not.toThrow(); expect(createSingleMetric).not.toHaveBeenCalled(); }); - it('should cache GitHub App ID and only call getParameter once', async () => { - // Reset modules to clear the appIdPromises Map cache - vi.resetModules(); - const { metricGitHubAppRateLimit: freshMetricFunction } = await import('./rate-limit'); - - process.env.ENABLE_METRIC_GITHUB_APP_RATE_LIMIT = 'true'; - const headers: ResponseHeaders = { - 'x-ratelimit-remaining': '10', - 'x-ratelimit-limit': '60', - }; - - const mockGetParameter = vi.mocked(getParameter); - mockGetParameter.mockClear(); - - await freshMetricFunction(headers); - await freshMetricFunction(headers); - await freshMetricFunction(headers); - - // getParameter should only be called once due to caching (index 0 cached after first call) - expect(mockGetParameter).toHaveBeenCalledTimes(1); - // split(':')[0] of 'test' is still 'test' - expect(mockGetParameter).toHaveBeenCalledWith(process.env.PARAMETER_GITHUB_APP_ID_NAME); - }); -}); - -describe('metricGitHubAppRateLimit multi-app', () => { - let freshMetricFunction: typeof metricGitHubAppRateLimit; - let mockGetParam: ReturnType; - - beforeEach(async () => { - // Reset modules to get a clean appIdPromises Map for each test - vi.resetModules(); - - process.env.PARAMETER_GITHUB_APP_ID_NAME = 'app0:app1'; - process.env.ENABLE_METRIC_GITHUB_APP_RATE_LIMIT = 'true'; - - mockGetParam = vi.fn((name: string) => { - if (name === 'app0') return Promise.resolve('1234'); - if (name === 'app1') return Promise.resolve('5678'); - return Promise.resolve(''); - }); - - vi.doMock('@aws-github-runner/aws-ssm-util', () => ({ getParameter: mockGetParam })); - vi.doMock('@aws-github-runner/aws-powertools-util', () => ({ - logger: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() }, - createSingleMetric: vi.fn(() => ({ addMetadata: vi.fn() })), - })); - - const mod = await import('./rate-limit'); - freshMetricFunction = mod.metricGitHubAppRateLimit; - }); - - afterEach(() => { - vi.resetModules(); - process.env.PARAMETER_GITHUB_APP_ID_NAME = 'test'; - }); - - it('should label metric with correct appId for index 0 (primary app)', async () => { - const { createSingleMetric: mockMetric } = await import('@aws-github-runner/aws-powertools-util'); - const headers: ResponseHeaders = { 'x-ratelimit-remaining': '50', 'x-ratelimit-limit': '5000' }; - await freshMetricFunction(headers, 0); - expect(mockMetric).toHaveBeenCalledWith('GitHubAppRateLimitRemaining', MetricUnit.Count, 50, { AppId: '1234' }); - }); - - it('should label metric with correct appId for index 1 (additional app)', async () => { - const { createSingleMetric: mockMetric } = await import('@aws-github-runner/aws-powertools-util'); - const headers: ResponseHeaders = { 'x-ratelimit-remaining': '100', 'x-ratelimit-limit': '5000' }; - await freshMetricFunction(headers, 1); - expect(mockMetric).toHaveBeenCalledWith('GitHubAppRateLimitRemaining', MetricUnit.Count, 100, { AppId: '5678' }); - }); - - it('should default to index 0 when no appIndex is passed', async () => { - const { createSingleMetric: mockMetric } = await import('@aws-github-runner/aws-powertools-util'); - const headers: ResponseHeaders = { 'x-ratelimit-remaining': '75', 'x-ratelimit-limit': '5000' }; - await freshMetricFunction(headers); - expect(mockMetric).toHaveBeenCalledWith('GitHubAppRateLimitRemaining', MetricUnit.Count, 75, { AppId: '1234' }); - }); - - it('should cache per index and call getParameter separately for each index', async () => { - const headers: ResponseHeaders = { 'x-ratelimit-remaining': '10', 'x-ratelimit-limit': '5000' }; + it('passes each app index to the credential seam', async () => { + mockedGetAppId.mockImplementation(async (appIndex = 0) => String(1000 + appIndex)); + const headers: ResponseHeaders = { 'x-ratelimit-remaining': '10', 'x-ratelimit-limit': '60' }; - // Two calls with index 1, then one with index 0 - await freshMetricFunction(headers, 1); - await freshMetricFunction(headers, 1); - await freshMetricFunction(headers, 0); + await metricGitHubAppRateLimit(headers, 0); + await metricGitHubAppRateLimit(headers, 1); - // getParameter should be called exactly once per distinct index - expect(mockGetParam).toHaveBeenCalledTimes(2); - expect(mockGetParam).toHaveBeenCalledWith('app1'); - expect(mockGetParam).toHaveBeenCalledWith('app0'); + expect(mockedGetAppId).toHaveBeenNthCalledWith(1, 0); + expect(mockedGetAppId).toHaveBeenNthCalledWith(2, 1); }); }); diff --git a/lambdas/functions/control-plane/src/github/rate-limit.ts b/lambdas/functions/control-plane/src/github/rate-limit.ts index df2372a255..710d7cf80a 100644 --- a/lambdas/functions/control-plane/src/github/rate-limit.ts +++ b/lambdas/functions/control-plane/src/github/rate-limit.ts @@ -2,22 +2,7 @@ import { ResponseHeaders } from '@octokit/types'; import { createSingleMetric, logger } from '@aws-github-runner/aws-powertools-util'; import { MetricUnit } from '@aws-lambda-powertools/metrics'; import yn from 'yn'; -import { getParameter } from '@aws-github-runner/aws-ssm-util'; - -// Cache the app ID per app index to avoid repeated SSM calls across Lambda invocations. -// In multi-app mode PARAMETER_GITHUB_APP_ID_NAME is a ':'-joined list of SSM param names, -// one per app in app-index order; index 0 is the primary app. -const appIdPromises = new Map>(); - -async function getAppId(appIndex = 0): Promise { - let cached = appIdPromises.get(appIndex); - if (!cached) { - const paramName = process.env.PARAMETER_GITHUB_APP_ID_NAME.split(':')[appIndex]; - cached = getParameter(paramName); - appIdPromises.set(appIndex, cached); - } - return cached; -} +import { getAppId } from './auth'; export async function metricGitHubAppRateLimit(headers: ResponseHeaders, appIndex?: number): Promise { try { diff --git a/lambdas/functions/control-plane/src/lambda.test.ts b/lambdas/functions/control-plane/src/lambda.test.ts index 4d970f0f22..b4540a9934 100644 --- a/lambdas/functions/control-plane/src/lambda.test.ts +++ b/lambdas/functions/control-plane/src/lambda.test.ts @@ -1,4 +1,5 @@ import { captureLambdaHandler, logger } from '@aws-github-runner/aws-powertools-util'; +import { createRunnerConfigHousekeeper } from '@aws-github-runner/storage-providers'; import { Context, SQSEvent, SQSRecord } from 'aws-lambda'; import { addMiddleware, adjustPool, scaleDownHandler, scaleUpHandler, ssmHousekeeper, jobRetryCheck } from './lambda'; @@ -6,7 +7,6 @@ import { adjust } from './pool/pool'; import { scaleDown } from './scale-runners/scale-down'; import { scaleUp } from './scale-runners/scale-up'; import type { ActionRequestMessage } from './scale-runners/types'; -import { cleanSSMTokens } from './scale-runners/ssm-housekeeper'; import { checkAndRetryJob } from './scale-runners/job-retry'; import { describe, it, expect, vi, MockedFunction, beforeEach } from 'vitest'; @@ -64,10 +64,14 @@ const context: Context = { vi.mock('./pool/pool'); vi.mock('./scale-runners/scale-down'); vi.mock('./scale-runners/scale-up'); -vi.mock('./scale-runners/ssm-housekeeper'); vi.mock('./scale-runners/job-retry'); vi.mock('@aws-github-runner/aws-powertools-util'); vi.mock('@aws-github-runner/aws-ssm-util'); +vi.mock('@aws-github-runner/storage-providers', () => ({ + createRunnerConfigHousekeeper: vi.fn(), +})); + +const mockedCreateRunnerConfigHousekeeper = vi.mocked(createRunnerConfigHousekeeper); describe('Test scale up lambda wrapper.', () => { it('Do not handle empty record sets.', async () => { @@ -298,19 +302,15 @@ describe('Test middleware', () => { describe('Test ssm housekeeper lambda wrapper.', () => { it('Invoke without errors.', async () => { - vi.mocked(cleanSSMTokens).mockResolvedValue(); - - process.env.SSM_CLEANUP_CONFIG = JSON.stringify({ - dryRun: false, - minimumDaysOld: 1, - tokenPath: '/path/to/tokens/', - }); + const houseKeeper = vi.fn().mockResolvedValue(); + mockedCreateRunnerConfigHousekeeper.mockReturnValue({ houseKeeper }); await expect(ssmHousekeeper({}, context)).resolves.not.toThrow(); + expect(houseKeeper).toHaveBeenCalledOnce(); }); it('Errors not throws.', async () => { - vi.mocked(cleanSSMTokens).mockRejectedValue(new Error()); + mockedCreateRunnerConfigHousekeeper.mockReturnValue({ houseKeeper: vi.fn().mockRejectedValue(new Error()) }); await expect(ssmHousekeeper({}, context)).resolves.not.toThrow(); }); }); diff --git a/lambdas/functions/control-plane/src/lambda.ts b/lambdas/functions/control-plane/src/lambda.ts index d229a0350e..4594a1289e 100644 --- a/lambdas/functions/control-plane/src/lambda.ts +++ b/lambdas/functions/control-plane/src/lambda.ts @@ -1,13 +1,13 @@ import middy from '@middy/core'; import { logger, setContext } from '@aws-github-runner/aws-powertools-util'; import { captureLambdaHandler, tracer } from '@aws-github-runner/aws-powertools-util'; +import { createRunnerConfigHousekeeper } from '@aws-github-runner/storage-providers'; import { Context, type SQSBatchItemFailure, type SQSBatchResponse, SQSEvent } from 'aws-lambda'; import { PoolEvent, adjust } from './pool/pool'; import { scaleDown } from './scale-runners/scale-down'; import { scaleUp } from './scale-runners/scale-up'; import type { ActionRequestMessage, ActionRequestMessageSQS } from './scale-runners/types'; -import { SSMCleanupOptions, cleanSSMTokens } from './scale-runners/ssm-housekeeper'; import { checkAndRetryJob } from './scale-runners/job-retry'; export async function scaleUpHandler(event: SQSEvent, context: Context): Promise { @@ -114,22 +114,25 @@ export const addMiddleware = () => { middy(scaleUpHandler).use(handler); middy(scaleDownHandler).use(handler); middy(adjustPool).use(handler); - middy(ssmHousekeeper).use(handler); + middy(runnerConfigHousekeeper).use(handler); }; addMiddleware(); -export async function ssmHousekeeper(event: unknown, context: Context): Promise { +export async function runnerConfigHousekeeper(event: unknown, context: Context): Promise { setContext(context, 'lambda.ts'); logger.logEventIfEnabled(event); - const config = JSON.parse(process.env.SSM_CLEANUP_CONFIG) as SSMCleanupOptions; + const housekeeper = createRunnerConfigHousekeeper(); try { - await cleanSSMTokens(config); + await housekeeper.houseKeeper(); } catch (e) { logger.error(`${(e as Error).message}`, { error: e as Error }); } } +/** @deprecated Use runnerConfigHousekeeper. Kept for existing Terraform handler configuration. */ +export const ssmHousekeeper = runnerConfigHousekeeper; + export async function jobRetryCheck(event: SQSEvent, context: Context): Promise { setContext(context, 'lambda.ts'); logger.logEventIfEnabled(event); diff --git a/lambdas/functions/control-plane/src/local-ssm-housekeeper.ts b/lambdas/functions/control-plane/src/local-ssm-housekeeper.ts index ec635b13ad..81c4cbafd5 100644 --- a/lambdas/functions/control-plane/src/local-ssm-housekeeper.ts +++ b/lambdas/functions/control-plane/src/local-ssm-housekeeper.ts @@ -1,4 +1,4 @@ -import { cleanSSMTokens } from './scale-runners/ssm-housekeeper'; +import { cleanSSMTokens } from '@aws-github-runner/storage-providers/aws/ssm/runner-config-housekeeper'; export function run(): void { cleanSSMTokens({ diff --git a/lambdas/functions/control-plane/src/pool/pool-contract.test.ts b/lambdas/functions/control-plane/src/pool/pool-contract.test.ts index 1a1e88a7f6..1a208edde7 100644 --- a/lambdas/functions/control-plane/src/pool/pool-contract.test.ts +++ b/lambdas/functions/control-plane/src/pool/pool-contract.test.ts @@ -20,7 +20,6 @@ vi.mock('../github/auth', () => ({ vi.mock('../scale-runners/github-runner', () => ({ createStartRunnerConfig: vi.fn(), getGitHubEnterpriseApiUrl: vi.fn(), - validateSsmParameterStoreTags: vi.fn(), })); const mockedAppAuth = vi.mocked(ghAuth.createGithubAppAuth); @@ -49,6 +48,9 @@ beforeEach(() => { vi.clearAllMocks(); process.env = { ...cleanEnv }; process.env.SSM_TOKEN_PATH = '/github-action-runners/default/runners/tokens'; + process.env.SSM_CONFIG_PATH = '/github-action-runners/default/runners/config'; + process.env.PARAMETER_GITHUB_APP_ID_NAME = 'github-app-id'; + process.env.PARAMETER_GITHUB_APP_KEY_BASE64_NAME = 'github-app-key'; mockedAppAuth.mockResolvedValue({ type: 'app', token: 'app-token', appId: 1, expiresAt: 'some-date' }); mockedInstallationAuth.mockResolvedValue({ @@ -63,7 +65,6 @@ beforeEach(() => { }); mockedCreateClient.mockResolvedValue(githubClient); vi.mocked(githubRunner.getGitHubEnterpriseApiUrl).mockReturnValue({ ghesApiUrl: '', ghesBaseUrl: '' }); - vi.mocked(githubRunner.validateSsmParameterStoreTags).mockReturnValue([]); vi.mocked(githubClient.apps.getOrgInstallation).mockResolvedValue({ data: { id: 2 } } as never); vi.mocked(githubClient.paginate).mockResolvedValue([]); }); diff --git a/lambdas/functions/control-plane/src/pool/pool.test.ts b/lambdas/functions/control-plane/src/pool/pool.test.ts index 8372ab6403..5253fd5147 100644 --- a/lambdas/functions/control-plane/src/pool/pool.test.ts +++ b/lambdas/functions/control-plane/src/pool/pool.test.ts @@ -31,7 +31,6 @@ vi.mock('../scale-runners/github-runner', () => ({ ghesApiUrl: '', ghesBaseUrl: '', }), - validateSsmParameterStoreTags: vi.fn().mockReturnValue([]), })); const mockedAppAuth = vi.mocked(ghAuth.createGithubAppAuth); @@ -93,6 +92,9 @@ beforeEach(() => { process.env.RUNNERS_MAXIMUM_COUNT = '-1'; process.env.ENVIRONMENT = 'unit-test-environment'; process.env.SSM_TOKEN_PATH = '/github-action-runners/default/runners/tokens'; + process.env.SSM_CONFIG_PATH = '/github-action-runners/default/runners/config'; + process.env.PARAMETER_GITHUB_APP_ID_NAME = 'github-app-id'; + process.env.PARAMETER_GITHUB_APP_KEY_BASE64_NAME = 'github-app-key'; process.env.RUNNER_OWNER = ORG; githubClient.paginate.mockResolvedValue(githubRunnersRegistered); @@ -325,7 +327,7 @@ describe('pool adjustment', () => { await adjust({ poolSize: 3 }); - expect(mockedInstallationAuth).toHaveBeenCalledWith(expect.any(Number), expect.any(String), 1); + expect(mockedInstallationAuth).toHaveBeenCalledWith(expect.any(Number), expect.any(String), 1, expect.anything()); }); it('looks up installationId using the selected app JWT', async () => { diff --git a/lambdas/functions/control-plane/src/pool/pool.ts b/lambdas/functions/control-plane/src/pool/pool.ts index 029c494bac..c02787426f 100644 --- a/lambdas/functions/control-plane/src/pool/pool.ts +++ b/lambdas/functions/control-plane/src/pool/pool.ts @@ -1,6 +1,7 @@ import { Octokit } from '@octokit/rest'; import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; import { resolveComputeProviderType } from '@aws-github-runner/compute-providers/provider-types'; +import { createStorageProviders, type StorageProviders } from '@aws-github-runner/storage-providers'; import yn from 'yn'; import { @@ -10,7 +11,7 @@ import { getStoredInstallationId, } from '../github/auth'; import { controlPlaneProviderRegistry } from '../control-plane-providers'; -import { getGitHubEnterpriseApiUrl, validateSsmParameterStoreTags } from '../scale-runners/github-runner'; +import { getGitHubEnterpriseApiUrl } from '../scale-runners/github-runner'; import type { RunnerStatus } from './pool-provider'; const logger = createChildLogger('pool'); @@ -21,6 +22,7 @@ export interface PoolEvent { } export async function adjust(event: PoolEvent): Promise { + const storage = createStorageProviders(); const computeProviderType = resolveComputeProviderType(event.type); const computeProvider = { ...controlPlaneProviderRegistry.capability(computeProviderType, 'pool')(), @@ -31,15 +33,10 @@ export async function adjust(event: PoolEvent): Promise { const runnerGroup = process.env.RUNNER_GROUP_NAME || ''; const runnerNamePrefix = process.env.RUNNER_NAME_PREFIX || ''; const environment = process.env.ENVIRONMENT; - const ssmConfigPath = process.env.SSM_CONFIG_PATH || ''; const ephemeral = yn(process.env.ENABLE_EPHEMERAL_RUNNERS, { default: false }); const enableJitConfig = yn(process.env.ENABLE_JIT_CONFIG, { default: ephemeral }); const disableAutoUpdate = yn(process.env.DISABLE_RUNNER_AUTOUPDATE, { default: false }); const runnerOwner = process.env.RUNNER_OWNER; - const ssmParameterStoreTags: { Key: string; Value: string }[] = - process.env.SSM_PARAMETER_STORE_TAGS && process.env.SSM_PARAMETER_STORE_TAGS.trim() !== '' - ? validateSsmParameterStoreTags(process.env.SSM_PARAMETER_STORE_TAGS) - : []; // -1 disables the maximum check, matching the scale-up lambda's semantics. Defaults to unlimited // when unset so the pool keeps its previous behavior on stacks that do not provide the variable. const maximumRunners = parseInt(process.env.RUNNERS_MAXIMUM_COUNT || '-1'); @@ -49,11 +46,11 @@ export async function adjust(event: PoolEvent): Promise { // Select one GitHub App for this entire invocation so every API call draws // from the same rate-limit bucket. - const ghAppAuth = await createGithubAppAuth(undefined, ghesApiUrl); + const ghAppAuth = await createGithubAppAuth(undefined, ghesApiUrl, undefined, storage.githubAppCredentials); const appIdx = ghAppAuth.appIndex; - const installationId = await getInstallationId(ghAppAuth.token, ghesApiUrl, runnerOwner, appIdx); - const ghAuth = await createGithubInstallationAuth(installationId, ghesApiUrl, appIdx); + const installationId = await getInstallationId(ghAppAuth.token, ghesApiUrl, runnerOwner, appIdx, storage); + const ghAuth = await createGithubInstallationAuth(installationId, ghesApiUrl, appIdx, storage.githubAppCredentials); const githubInstallationClient = await createOctokitClient(ghAuth.token, ghesApiUrl); // Get statuses of runners registered in GitHub @@ -102,20 +99,25 @@ export async function adjust(event: PoolEvent): Promise { runnerNamePrefix, runnerType: 'Org', disableAutoUpdate: disableAutoUpdate, - ssmConfigPath, - ssmParameterStoreTags, }, numberOfRunners: topUp, githubInstallationClient, + storage, }); } else { logger.info(`Pool will not be topped up. Found ${numberOfRunnersInPool} managed idle runners.`); } } -async function getInstallationId(appToken: string, ghesApiUrl: string, org: string, appIndex: number): Promise { +async function getInstallationId( + appToken: string, + ghesApiUrl: string, + org: string, + appIndex: number, + storage?: StorageProviders, +): Promise { // Use the pre-configured installation ID when available (avoids an API call). - const storedId = await getStoredInstallationId(appIndex); + const storedId = await getStoredInstallationId(appIndex, storage?.githubAppCredentials); if (storedId !== undefined) return storedId; const githubClient = await createOctokitClient(appToken, ghesApiUrl); diff --git a/lambdas/functions/control-plane/src/scale-runners/github-runner.ts b/lambdas/functions/control-plane/src/scale-runners/github-runner.ts index 31534adb79..2a87739e83 100644 --- a/lambdas/functions/control-plane/src/scale-runners/github-runner.ts +++ b/lambdas/functions/control-plane/src/scale-runners/github-runner.ts @@ -1,9 +1,9 @@ import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; import { - createRunnerConfigStore, - createRunnerGroupCacheStore, + createStorageProviders, type RunnerConfigMetadata, type RunnerConfigStore, + type GitHubAppCredentialsStore, type RunnerGroupCacheStore, } from '@aws-github-runner/storage-providers'; import { Octokit } from '@octokit/rest'; @@ -20,6 +20,7 @@ export interface GitHubRunnerMetadata { } export interface StartRunnerConfigOptions { + runnerConfigStore?: RunnerConfigStore; runnerGroupCacheStore?: RunnerGroupCacheStore; getRunnerConfigMetadata?: (runnerId: string) => RunnerConfigMetadata[]; onJitConfigCreated?: (runnerId: string, metadata: GitHubRunnerMetadata) => Promise; @@ -58,37 +59,6 @@ function quoteShellArg(value: string): string { return `'${value.replace(/'/g, `'\\''`)}'`; } -export function validateSsmParameterStoreTags(tagsJson: string): { Key: string; Value: string }[] { - try { - const tags = JSON.parse(tagsJson); - - if (!Array.isArray(tags)) { - throw new Error('Tags must be an array'); - } - - if (tags.length === 0) { - return []; - } - - tags.forEach((tag, index) => { - if (typeof tag !== 'object' || tag === null) { - throw new Error(`Tag at index ${index} must be an object`); - } - if (!tag.Key || typeof tag.Key !== 'string' || tag.Key.trim() === '') { - throw new Error(`Tag at index ${index} has missing or invalid 'Key' property`); - } - if (!Object.prototype.hasOwnProperty.call(tag, 'Value') || typeof tag.Value !== 'string') { - throw new Error(`Tag at index ${index} has missing or invalid 'Value' property`); - } - }); - - return tags; - } catch (err) { - logger.error('Invalid SSM_PARAMETER_STORE_TAGS format', { error: err }); - throw new Error(`Failed to parse SSM_PARAMETER_STORE_TAGS: ${(err as Error).message}`); - } -} - async function getGithubRunnerRegistrationToken(githubRunnerConfig: CreateGitHubRunnerConfig, ghClient: Octokit) { const registrationToken = githubRunnerConfig.runnerType === 'Org' @@ -137,10 +107,11 @@ export async function getInstallationId( enableOrgLevel: boolean, payload: ActionRequestMessage, appIndex?: number, + credentialsStore?: GitHubAppCredentialsStore, ): Promise { // Use the pre-configured installation ID when available (avoids an API call). if (appIndex !== undefined) { - const storedId = await getStoredInstallationId(appIndex); + const storedId = await getStoredInstallationId(appIndex, credentialsStore); if (storedId !== undefined) return storedId; } @@ -194,7 +165,7 @@ export async function getRunnerGroupId( // if the runnerType is Repo, then runnerGroupId is default to 1 let runnerGroupId: number | undefined = 1; if (githubRunnerConfig.runnerType === 'Org' && githubRunnerConfig.runnerGroup !== undefined) { - const cacheStore = runnerGroupCacheStore ?? createRunnerGroupCacheStore(); + const cacheStore = runnerGroupCacheStore ?? createStorageProviders().runnerGroupCache; const runnerGroup = await cacheStore.get(githubRunnerConfig.runnerGroup); if (runnerGroup === undefined) { // get runner group id from GitHub @@ -236,7 +207,7 @@ export async function createStartRunnerConfig( ghClient: Octokit, options: StartRunnerConfigOptions = {}, ): Promise { - const runnerConfigStore = createRunnerConfigStore(); + const runnerConfigStore = options.runnerConfigStore ?? createStorageProviders().runnerConfig; if (githubRunnerConfig.enableJitConfig && githubRunnerConfig.ephemeral) { return await createJitConfig(githubRunnerConfig, runnerIds, ghClient, runnerConfigStore, options); } else { @@ -338,6 +309,7 @@ async function createJitConfig( runnerLabels, }); + // Store the JIT config through the selected storage provider. logger.debug('Runner JIT config for ephemeral runner generated.', { instance: runnerId, }); diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up-contract.test.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up-contract.test.ts index 7c381c1526..d7a025fd88 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-up-contract.test.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-up-contract.test.ts @@ -57,6 +57,9 @@ beforeEach(() => { vi.clearAllMocks(); process.env = { ...cleanEnv }; process.env.SSM_TOKEN_PATH = '/github-action-runners/default/runners/tokens'; + process.env.SSM_CONFIG_PATH = '/github-action-runners/default/runners/config'; + process.env.PARAMETER_GITHUB_APP_ID_NAME = 'github-app-id'; + process.env.PARAMETER_GITHUB_APP_KEY_BASE64_NAME = 'github-app-key'; mockedAppAuth.mockResolvedValue({ type: 'app', token: 'app-token', appId: 1, expiresAt: 'some-date' }); mockedInstallationAuth.mockResolvedValue({ diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts index 1ca7536b2a..d406c7c5f5 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts @@ -1033,8 +1033,18 @@ describe('scaleUp with GHES', () => { await scaleUpModule.scaleUp(messages); expect(mockCreateClient).toHaveBeenCalledTimes(3); // 1 app client, 2 repo installation clients - expect(mockedInstallationAuth).toHaveBeenCalledWith(100, 'https://github.enterprise.something/api/v3', 0); - expect(mockedInstallationAuth).toHaveBeenCalledWith(200, 'https://github.enterprise.something/api/v3', 0); + expect(mockedInstallationAuth).toHaveBeenCalledWith( + 100, + 'https://github.enterprise.something/api/v3', + 0, + expect.anything(), + ); + expect(mockedInstallationAuth).toHaveBeenCalledWith( + 200, + 'https://github.enterprise.something/api/v3', + 0, + expect.anything(), + ); }); it('Should reuse GitHub clients for same installation', async () => { @@ -1467,8 +1477,8 @@ describe('scaleUp with public GH', () => { await scaleUpModule.scaleUp(messages); expect(mockCreateClient).toHaveBeenCalledTimes(3); // 1 app client, 2 repo installation clients - expect(mockedInstallationAuth).toHaveBeenCalledWith(100, '', 0); - expect(mockedInstallationAuth).toHaveBeenCalledWith(200, '', 0); + expect(mockedInstallationAuth).toHaveBeenCalledWith(100, '', 0, expect.anything()); + expect(mockedInstallationAuth).toHaveBeenCalledWith(200, '', 0, expect.anything()); }); it('Should reuse GitHub clients for same installation', async () => { @@ -1945,8 +1955,8 @@ describe('scaleUp with Github Data Residency', () => { await scaleUpModule.scaleUp(messages); expect(mockCreateClient).toHaveBeenCalledTimes(3); // 1 app client, 2 repo installation clients - expect(mockedInstallationAuth).toHaveBeenCalledWith(100, '', 0); - expect(mockedInstallationAuth).toHaveBeenCalledWith(200, '', 0); + expect(mockedInstallationAuth).toHaveBeenCalledWith(100, '', 0, expect.anything()); + expect(mockedInstallationAuth).toHaveBeenCalledWith(200, '', 0, expect.anything()); }); it('Should reuse GitHub clients for same installation', async () => { @@ -2209,6 +2219,7 @@ describe('Multi-app round-robin', () => { expect.any(Number), expect.any(String), 1, // appIndex must match the one from createGithubAppAuth + expect.anything(), ); }); @@ -2234,6 +2245,7 @@ describe('Multi-app round-robin', () => { TEST_DATA_SINGLE.installationId, // from mockOctokit.apps.getOrgInstallation mock expect.any(String), 1, + expect.anything(), ); }); @@ -2251,7 +2263,7 @@ describe('Multi-app round-robin', () => { // Should use 999 from webhook directly — no API lookup expect(mockOctokit.apps.getOrgInstallation).not.toHaveBeenCalled(); - expect(mockedInstallationAuth).toHaveBeenCalledWith(999, expect.any(String), 0); + expect(mockedInstallationAuth).toHaveBeenCalledWith(999, expect.any(String), 0, expect.anything()); }); it('primary app (appIndex 0) reuses webhook installationId even in multi-app deployment', async () => { @@ -2270,7 +2282,7 @@ describe('Multi-app round-robin', () => { // Primary app must NOT do an API lookup — reuses webhook installationId expect(mockOctokit.apps.getOrgInstallation).not.toHaveBeenCalled(); - expect(mockedInstallationAuth).toHaveBeenCalledWith(999, expect.any(String), 0); + expect(mockedInstallationAuth).toHaveBeenCalledWith(999, expect.any(String), 0, expect.anything()); }); it('stored installationId takes precedence over webhook payload for additional app', async () => { @@ -2289,7 +2301,7 @@ describe('Multi-app round-robin', () => { // Stored id (77) wins — no API lookup needed expect(mockOctokit.apps.getOrgInstallation).not.toHaveBeenCalled(); - expect(mockedInstallationAuth).toHaveBeenCalledWith(77, expect.any(String), 1); + expect(mockedInstallationAuth).toHaveBeenCalledWith(77, expect.any(String), 1, expect.anything()); }); }); diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up.ts index 0b00d620b3..6633f5c6b3 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-up.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-up.ts @@ -1,5 +1,6 @@ import { addPersistentContextToChildLogger, createChildLogger } from '@aws-github-runner/aws-powertools-util'; import { resolveComputeProviderType } from '@aws-github-runner/compute-providers/provider-types'; +import { createStorageProviders, type StorageProviders } from '@aws-github-runner/storage-providers'; import { Octokit } from '@octokit/rest'; import yn from 'yn'; @@ -11,7 +12,6 @@ import { resolveInstallationId, isJobQueued, UnsupportedEventError, - validateSsmParameterStoreTags, } from './github-runner'; import { publishRetryMessage } from './job-retry'; import type { @@ -39,11 +39,23 @@ async function createGithubInstallationClient( payload: ActionRequestMessage, ghesApiUrl: string, appIndex?: number, + storage?: StorageProviders, ): Promise { - const installationId = await getInstallationId(githubAppClient, enableOrgLevel, payload, appIndex); + const installationId = await getInstallationId( + githubAppClient, + enableOrgLevel, + payload, + appIndex, + storage?.githubAppCredentials, + ); try { - const ghAuth = await createGithubInstallationAuth(installationId, ghesApiUrl, appIndex); + const ghAuth = await createGithubInstallationAuth( + installationId, + ghesApiUrl, + appIndex, + storage?.githubAppCredentials, + ); return await createOctokitClient(ghAuth.token, ghesApiUrl); } catch (error) { // The installation id can be stale when it was reused from the webhook payload or from the @@ -66,12 +78,18 @@ async function createGithubInstallationClient( repositoryName: payload.repositoryName, }); - const ghAuth = await createGithubInstallationAuth(resolvedInstallationId, ghesApiUrl, appIndex); + const ghAuth = await createGithubInstallationAuth( + resolvedInstallationId, + ghesApiUrl, + appIndex, + storage?.githubAppCredentials, + ); return await createOctokitClient(ghAuth.token, ghesApiUrl); } } export async function scaleUp(payloads: ActionRequestMessageSQS[]): Promise { + const storage = createStorageProviders(); logger.info('Received scale up requests', { n_requests: payloads.length, }); @@ -85,11 +103,6 @@ export async function scaleUp(payloads: ActionRequestMessageSQS[]): Promise 0) { - throw new Error(errorMessages.join(', ')); - } -} - -export async function cleanSSMTokens(options: SSMCleanupOptions): Promise { - logger.info(`Cleaning tokens / JIT config older then ${options.minimumDaysOld} days, dryRun: ${options.dryRun}`); - logger.debug('Cleaning with options', { options }); - validateOptions(options); - - const client = getTracedAWSV3Client(new SSMClient({ region: process.env.AWS_REGION })); - const parameters = await client.send(new GetParametersByPathCommand({ Path: options.tokenPath })); - while (parameters.NextToken) { - const nextParameters = await client.send( - new GetParametersByPathCommand({ Path: options.tokenPath, NextToken: parameters.NextToken }), - ); - parameters.Parameters?.push(...(nextParameters.Parameters ?? [])); - parameters.NextToken = nextParameters.NextToken; - } - logger.info(`Found #${parameters.Parameters?.length} parameters in path ${options.tokenPath}`); - logger.debug('Found parameters', { parameters }); - - // minimumDate = today - minimumDaysOld - const minimumDate = new Date(); - minimumDate.setDate(minimumDate.getDate() - options.minimumDaysOld); - - for (const parameter of parameters.Parameters ?? []) { - if (parameter.LastModifiedDate && new Date(parameter.LastModifiedDate) < minimumDate) { - logger.info(`Deleting parameter ${parameter.Name} with last modified date ${parameter.LastModifiedDate}`); - try { - if (!options.dryRun) { - // sleep 50ms to avoid rait limit - await new Promise((resolve) => setTimeout(resolve, 50)); - await client.send(new DeleteParameterCommand({ Name: parameter.Name })); - } - } catch (e) { - logger.warn(`Failed to delete parameter ${parameter.Name} with error ${(e as Error).message}`); - logger.debug('Failed to delete parameter', { e }); - } - } else { - logger.debug(`Skipping parameter ${parameter.Name} with last modified date ${parameter.LastModifiedDate}`); - } - } -} +/** @deprecated Import the SSM runner-config housekeeper from storage-providers. */ +export { + cleanSSMTokens, + createAwsSsmRunnerConfigHousekeeper, + type SSMCleanupOptions, +} from '@aws-github-runner/storage-providers/aws/ssm/runner-config-housekeeper'; diff --git a/lambdas/libs/aws-ssm-util/src/index.test.ts b/lambdas/libs/aws-ssm-util/src/index.test.ts index 8a1d8d3864..2f6080bb8a 100644 --- a/lambdas/libs/aws-ssm-util/src/index.test.ts +++ b/lambdas/libs/aws-ssm-util/src/index.test.ts @@ -1,6 +1,9 @@ import { + AddTagsToResourceCommand, + DeleteParameterCommand, GetParameterCommand, GetParameterCommandOutput, + GetParametersByPathCommand, GetParametersCommand, PutParameterCommand, PutParameterCommandOutput, @@ -10,7 +13,17 @@ import 'aws-sdk-client-mock-jest/vitest'; import { mockClient } from 'aws-sdk-client-mock'; import nock from 'nock'; -import { getParameter, getParameters, putParameter, resetSSMClient, ssmClient, SSM_ADVANCED_TIER_THRESHOLD } from '.'; +import { + addParameterTags, + deleteParameter, + getParameter, + getParameters, + getParametersByPath, + putParameter, + resetSSMClient, + ssmClient, + SSM_ADVANCED_TIER_THRESHOLD, +} from '.'; import { describe, it, expect, beforeEach, vi } from 'vitest'; const mockSSMClient = mockClient(SSMClient); @@ -104,6 +117,30 @@ describe('Test getParameter and putParameter', () => { }); }); + it('overwrites a parameter only when explicitly requested', async () => { + mockSSMClient.on(PutParameterCommand).resolves({}); + + await putParameter('testParam', 'updated', false, { overwrite: true }); + + expect(mockSSMClient).toHaveReceivedCommandWith(PutParameterCommand, { + Name: 'testParam', + Value: 'updated', + Type: 'String', + Overwrite: true, + }); + }); + + it('rejects tags when overwriting an existing parameter', async () => { + mockSSMClient.resetHistory(); + await expect( + putParameter('testParam', 'updated', false, { + overwrite: true, + tags: [{ Key: 'owner', Value: 'runner' }], + } as never), + ).rejects.toThrow('tags cannot be supplied when overwriting'); + expect(mockSSMClient).not.toHaveReceivedCommand(PutParameterCommand); + }); + it('Puts parameters as SecureString', async () => { // Arrange const parameterValue = 'test'; @@ -256,6 +293,70 @@ describe('Test getParameters (batch)', () => { }); }); +describe('Test direct parameter path operations', () => { + beforeEach(() => { + mockSSMClient.reset(); + }); + + it('paginates direct, non-secret children of a parameter path', async () => { + mockSSMClient + .on(GetParametersByPathCommand, { + Path: '/metadata', + Recursive: false, + WithDecryption: false, + NextToken: undefined, + }) + .resolves({ Parameters: [{ Name: '/metadata/one', Value: '1' }], NextToken: 'page-2' }) + .on(GetParametersByPathCommand, { + Path: '/metadata', + Recursive: false, + WithDecryption: false, + NextToken: 'page-2', + }) + .resolves({ Parameters: [{ Name: '/metadata/two', Value: '2' }] }); + + await expect(getParametersByPath('/metadata')).resolves.toEqual( + new Map([ + ['/metadata/one', '1'], + ['/metadata/two', '2'], + ]), + ); + expect(mockSSMClient).toHaveReceivedCommandTimes(GetParametersByPathCommand, 2); + }); + + it('deletes an exact parameter name', async () => { + mockSSMClient.on(DeleteParameterCommand).resolves({}); + + await deleteParameter('/metadata/one'); + + expect(mockSSMClient).toHaveReceivedCommandWith(DeleteParameterCommand, { Name: '/metadata/one' }); + }); + + it('adds tags to an exact parameter name', async () => { + mockSSMClient.on(AddTagsToResourceCommand).resolves({}); + + await addParameterTags('/metadata/one', [{ Key: 'ghr:environment', Value: 'unit-test' }]); + + expect(mockSSMClient).toHaveReceivedCommandWith(AddTagsToResourceCommand, { + ResourceType: 'Parameter', + ResourceId: '/metadata/one', + Tags: [{ Key: 'ghr:environment', Value: 'unit-test' }], + }); + }); + + it('does not call SSM when there are no parameter tags to add', async () => { + await addParameterTags('/metadata/one', []); + + expect(mockSSMClient).not.toHaveReceivedCommand(AddTagsToResourceCommand); + }); + + it('propagates failures when adding parameter tags', async () => { + mockSSMClient.on(AddTagsToResourceCommand).rejects(new Error('AccessDenied')); + + await expect(addParameterTags('/metadata/one', [{ Key: 'Name', Value: 'runner' }])).rejects.toThrow('AccessDenied'); + }); +}); + describe('SSM client configuration', () => { it('configures adaptive retry with a raised attempt cap', async () => { const config = ssmClient().config; diff --git a/lambdas/libs/aws-ssm-util/src/index.ts b/lambdas/libs/aws-ssm-util/src/index.ts index 71b33cbf41..ad448b57ac 100644 --- a/lambdas/libs/aws-ssm-util/src/index.ts +++ b/lambdas/libs/aws-ssm-util/src/index.ts @@ -1,4 +1,12 @@ -import { GetParametersCommand, PutParameterCommand, SSMClient, Tag } from '@aws-sdk/client-ssm'; +import { + AddTagsToResourceCommand, + DeleteParameterCommand, + GetParametersByPathCommand, + GetParametersCommand, + PutParameterCommand, + SSMClient, + Tag, +} from '@aws-sdk/client-ssm'; import { getTracedAWSV3Client } from '@aws-github-runner/aws-powertools-util'; import { SSMProvider } from '@aws-lambda-powertools/parameters/ssm'; @@ -103,14 +111,68 @@ export async function getParameters(parameter_names: string[]): Promise> { + const result = new Map(); + let nextToken: string | undefined; + + do { + const response = await ssmClient().send( + new GetParametersByPathCommand({ + Path: parameter_path, + Recursive: false, + WithDecryption: false, + NextToken: nextToken, + }), + ); + + for (const parameter of response.Parameters ?? []) { + if (parameter.Name && parameter.Value) { + result.set(parameter.Name, parameter.Value); + } + } + nextToken = response.NextToken; + } while (nextToken); + + return result; +} + +export async function deleteParameter(parameter_name: string): Promise { + await ssmClient().send(new DeleteParameterCommand({ Name: parameter_name })); +} + +export async function addParameterTags(parameter_name: string, tags: Tag[]): Promise { + if (tags.length === 0) return; + + await ssmClient().send( + new AddTagsToResourceCommand({ + ResourceType: 'Parameter', + ResourceId: parameter_name, + Tags: tags, + }), + ); +} + export const SSM_ADVANCED_TIER_THRESHOLD = 4000; +type PutParameterOptions = { overwrite: true; tags?: never } | { overwrite?: false | undefined; tags?: Tag[] }; + export async function putParameter( parameter_name: string, parameter_value: string, secure: boolean, - options: { tags?: Tag[] } = {}, + options: PutParameterOptions = {}, ): Promise { + if (options.overwrite && options.tags !== undefined) { + throw new Error('SSM parameter tags cannot be supplied when overwriting an existing parameter'); + } + const client = ssmClient(); // Determine tier based on parameter_value size @@ -121,6 +183,7 @@ export async function putParameter( Name: parameter_name, Value: parameter_value, Type: secure ? 'SecureString' : 'String', + Overwrite: options.overwrite, Tags: options.tags, Tier: valueSizeBytes >= SSM_ADVANCED_TIER_THRESHOLD ? 'Advanced' : 'Standard', }), diff --git a/lambdas/libs/compute-providers/aws/ec2/src/control-plane/pool.test.ts b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/pool.test.ts index 85b65ba7cf..3f8df82173 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/control-plane/pool.test.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/pool.test.ts @@ -114,9 +114,6 @@ describe('createEc2PoolCapability.createRunners', () => { runnerOwner: 'owner', runnerType: 'Org', disableAutoUpdate: false, - ssmTokenPath: '/runners/tokens', - ssmConfigPath: '/runners/config', - ssmParameterStoreTags: [], }; const providerConfig: Ec2ProviderConfig = { environment: 'test-environment', @@ -159,6 +156,7 @@ describe('createEc2PoolCapability.createRunners', () => { githubInstallationClient, createStartRunnerConfig, 'pool-lambda', + undefined, ); }); }); diff --git a/lambdas/libs/compute-providers/aws/ec2/src/control-plane/pool.ts b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/pool.ts index f032a78e20..43ec0aacf9 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/control-plane/pool.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/pool.ts @@ -44,7 +44,7 @@ export function createEc2PoolCapability( statuses: ['running'], }), countAvailableRunners: countAvailableEc2PoolRunners, - createRunners: async ({ githubRunnerConfig, numberOfRunners, githubInstallationClient }) => { + createRunners: async ({ githubRunnerConfig, numberOfRunners, githubInstallationClient, storage }) => { const config = loadEc2ProviderConfig(); const { instances } = await createRunners( @@ -64,6 +64,7 @@ export function createEc2PoolCapability( githubInstallationClient, createStartRunnerConfig, 'pool-lambda', + storage, ); return instances; }, diff --git a/lambdas/libs/compute-providers/aws/ec2/src/control-plane/runner-creation.ts b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/runner-creation.ts index 59d01733eb..124b676ae6 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/control-plane/runner-creation.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/runner-creation.ts @@ -7,6 +7,7 @@ import type { RunnerSource, StartRunnerConfigOptions, } from '../../../../core'; +import type { RunnerConfigStorage } from '@aws-github-runner/storage-providers'; import { Octokit } from '@octokit/rest'; import type { Tag } from '@aws-sdk/client-ec2'; import yn from 'yn'; @@ -69,6 +70,7 @@ export async function createRunners( ghClient: Octokit, createStartRunnerConfig: CreateStartRunnerConfig, source: RunnerSource, + storage?: RunnerConfigStorage, ): Promise { let result: CreateRunnerResult; try { @@ -97,7 +99,7 @@ export async function createRunners( githubRunnerConfig, result.instances, ghClient, - createEc2StartRunnerConfigOptions(ec2Operations), + createEc2StartRunnerConfigOptions(ec2Operations, storage), ); } catch (error) { logger.error('Unexpected error while registering GitHub runners.', { @@ -146,8 +148,13 @@ async function terminateFailedInstances( } } -function createEc2StartRunnerConfigOptions(ec2Operations: Ec2RunnerResourceOperations): StartRunnerConfigOptions { +function createEc2StartRunnerConfigOptions( + ec2Operations: Ec2RunnerResourceOperations, + storage?: RunnerConfigStorage, +): StartRunnerConfigOptions { return { + runnerConfigStore: storage?.runnerConfig, + runnerGroupCacheStore: storage?.runnerGroupCache, getRunnerConfigMetadata: (instanceId) => [{ key: 'InstanceId', value: instanceId }], onJitConfigCreated: async (instanceId, metadata) => await tagEc2RunnerMetadata(ec2Operations, instanceId, metadata), }; diff --git a/lambdas/libs/compute-providers/aws/ec2/src/control-plane/scale-up.test.ts b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/scale-up.test.ts index 98c376e028..1512955e05 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/control-plane/scale-up.test.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/scale-up.test.ts @@ -51,8 +51,6 @@ function runnerConfig(overrides: Partial = {}): Create runnerOwner, runnerType: 'Org', disableAutoUpdate: false, - ssmConfigPath: '/github-action-runners/default/runners/config', - ssmParameterStoreTags: [], ...overrides, }; } diff --git a/lambdas/libs/compute-providers/aws/ec2/src/control-plane/scale-up.ts b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/scale-up.ts index 0cbda11a8f..951193a4fb 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/control-plane/scale-up.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/scale-up.ts @@ -55,7 +55,7 @@ export function createEc2ScaleUpCapability( resolveLabelsForRunners: (labels) => resolveEc2ScaleUpRunnerLabels(ec2Operations, labels), getCurrentRunners: async (_state, { runnerType, runnerOwner }) => (await ec2Operations.list({ environment: process.env.ENVIRONMENT, runnerType, runnerOwner })).length, - createRunners: async ({ githubRunnerConfig, numberOfRunners, githubInstallationClient, state }) => { + createRunners: async ({ githubRunnerConfig, numberOfRunners, githubInstallationClient, state, storage }) => { const config = loadEc2ScaleUpProviderConfig(); return await createRunners( @@ -69,6 +69,7 @@ export function createEc2ScaleUpCapability( githubInstallationClient, createStartRunnerConfig, 'scale-up-lambda', + storage, ); }, }; diff --git a/lambdas/libs/compute-providers/aws/microvm/README.md b/lambdas/libs/compute-providers/aws/microvm/README.md new file mode 100644 index 0000000000..bc672dd2a2 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/README.md @@ -0,0 +1,149 @@ +# Lambda MicroVM compute provider + +This provider manages a compatible AWS Lambda MicroVM image through the control-plane Lambda. It currently supports ephemeral JIT runners only. + +The MicroVM image `/run` hook receives this `runHookPayload`: + +```json +{ + "version": 1, + "imageArn": "arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner", + "imageVersion": "12.0", + "runnerConfigSsmPath": "/github-action-runners/example/runners/config", + "runnerTokenSsmPath": "/github-action-runners/example/runners/tokens" +} +``` + +Lambda adds `microvmId` beside that payload. `imageArn` and `imageVersion` are the requested launch values and are included together when an explicit image version is selected. The image must poll the SecureString parameter at `/`, start the GitHub runner with its encoded JIT configuration, delete the parameter after reading it, and exit its lifecycle entrypoint after the job completes. The image separately polls its complete non-secret tag map at `/microvm-metadata/.tags`. The control plane stores the JIT parameter before the provider callback writes the tag map, preventing cleanup from deleting an absent JIT that could otherwise be recreated later. Neither metadata record contains the JIT configuration value. Trusted control-plane cleanup and the fixed lifetime remain termination backstops. + +Runner ownership and lifecycle state are stored separately as non-secret `String` +parameters under `/`. The immutable base +record and independent state parameters prevent concurrent GitHub ID, orphan, +and cleanup updates from overwriting one another. Deleting the JIT SecureString +does not delete this metadata. Use a dedicated metadata prefix that does not +overlap the JIT path, and grant the MicroVM execution role only the exact +value-read access described below, without path-listing permissions. The control +plane retries pending cleanup, removes metadata after termination, and reconciles +expired records during inventory. + +The immutable base metadata parameter carries the same AWS resource tags that +are serialized as a JSON object in the `.tags` parameter. The tag +set starts with `SSM_PARAMETER_STORE_TAGS`, omits `Name`, and derives +`ghr:environment`, `ghr:ssm_config_path`, and `ghr:runner_name_prefix` from +the existing `ENVIRONMENT`, `SSM_CONFIG_PATH`, and `RUNNER_NAME_PREFIX` +settings. The Lambda then adds authoritative runtime tags: +`ghr:Application`, `ghr:created_by`, `ghr:environment`, `ghr:Owner`, +`ghr:Type`, `ghr:microvm_id`, `ghr:microvm_image_arn`, and, when available, +`ghr:microvm_image_version`. After JIT registration, the control plane adds +`ghr:github_runner_id` and base64url-encoded runner-label groups under +`ghr:runner_labels` through `ghr:runner_labels:5`. Runtime-owned values override +configured collisions. The `aws:` tag prefix is reserved and cannot be used for +these SSM parameters. The `.tags` value may use the Parameter Store advanced +tier when its UTF-8 representation is at least 4,000 bytes and is rejected if +the complete value could exceed the 8 KiB Parameter Store limit. + +Final cleanup deletes `/`, the +`.github-runner-id`, `.orphan`, and `.tags` companions, the base ownership +record, and `.cleanup-requested-at` last. The tombstone keeps its original +timestamp through a five-minute grace window so cleanup can repeatedly revoke a +late JIT write before removing every record. Missing parameters are treated as +already cleaned. + +The runner configuration publishes `/enable_cloudwatch` +and, when enabled, `/cloudwatch_agent_config_runner`. +The generated agent configuration reads these image-owned files by default: + +- `/var/log/microvm/internal-services.log` +- `/var/log/microvm/run.log` +- `/opt/actions-runner/_diag/Runner_**.log` + +Their default log-group suffixes are `internal_service`, `run`, and `runner`, +and `{microvm_id}` is an image-expanded log-stream placeholder. The first two +files are part of the MicroVM image contract; the portable lifecycle hook does +not create CloudWatch-specific files. Native RunMicrovm stdout and stderr stay +enabled independently as the early-startup and failure backstop. + +The control-plane Lambda requires these provider environment variables: + +- `MICROVM_IMAGE_ARN` +- `MICROVM_EXECUTION_ROLE_ARN` +- `MICROVM_IMAGE_VERSION` (optional) +- `MICROVM_INGRESS_NETWORK_CONNECTORS` (optional JSON array or comma-separated list) +- `MICROVM_EGRESS_NETWORK_CONNECTORS` (optional JSON array or comma-separated list) +- `MICROVM_METADATA_SSM_PATH` (dedicated SSM path for control-plane metadata) +- `MICROVM_LOG_GROUP` (optional) +- `SSM_TOKEN_PATH` (lane-scoped JIT parameter path) + +Each runner is launched with a fixed lifetime of 28,800 seconds (8 hours). + +The control-plane role requires `ssm:GetParametersByPath`, `ssm:GetParameters`, +`ssm:PutParameter`, `ssm:AddTagsToResource`, and `ssm:DeleteParameter` on the +dedicated metadata prefix, plus a separate `ssm:DeleteParameter` grant on the +lane-scoped JIT prefix, and `lambda:ListMicrovms`, `lambda:RunMicrovm`, and +`lambda:TerminateMicrovm` for inventory and lifecycle reconciliation. Restrict +`lambda:RunMicrovm` and `lambda:TerminateMicrovm` to approved image resources; +`lambda:ListMicrovms` does not support resource-level permissions. + +The MicroVM execution role must trust `lambda.amazonaws.com` for both +`sts:AssumeRole` and `sts:TagSession`. Restrict `iam:PassRole` to that exact role +ARN. Network connectors also require `lambda:PassNetworkConnector`; because +that action does not currently support resource-level permissions, enforce the +connector boundary with the explicit dynamic-label allowlist described below. + +All MicroVMs using one execution role, JIT prefix, and metadata prefix share a +trust boundary. Grant that role only `ssm:GetParameter` on the Parameter Store +ARN corresponding to `/microvm-metadata/*` and the exact +CloudWatch configuration parameters, `ssm:GetParameter` and +`ssm:DeleteParameter` on the lane-scoped JIT prefix, and stream-write access to +the provider-managed log groups. The image must address its own metadata with +its AWS-provided `microvmId` and must not receive path-listing access. IAM cannot +bind that ID to the calling MicroVM session, so a MicroVM can read other +metadata records in the same lane if it learns their IDs. Only allow trusted +images and workloads within a shared role, or isolate trust domains with +separate roles, prefixes, and provider deployments. + +## Dynamic labels + +When a runner matcher enables dynamic labels, workflow jobs can override the +following `RunMicrovm` inputs: + +| Label | Override | +| --------------------------------------------- | -------------------------------- | +| `ghr-microvm-egress-network-connectors:` | One egress network connector ARN | +| `ghr-microvm-image-arn:` | MicroVM image ARN | +| `ghr-microvm-image-version:` | MicroVM image version | + +Repeat `ghr-microvm-egress-network-connectors:` to attach multiple +connectors. Specify one ARN per label; `RunMicrovm` accepts at most 10. These +labels replace the compute provider's configured +`MICROVM_EGRESS_NETWORK_CONNECTORS` value for that job. + +Lambda MicroVM does not expose CPU or memory as `RunMicrovm` inputs. Select an +image and version with the required resources instead. Labels such as +`ghr-microvm-memory` are rejected. + +Execution roles, ingress network connectors, logging, idle policy, run hook +payloads, and client tokens remain deployment-controlled. Image ARN, image +version, and egress connector overrides change executable code or the network +boundary, so they are rejected unless `awsDynamicLabelsPolicy` supplies an +explicit `allowed` list for the corresponding key. + +Use the matcher's `awsDynamicLabelsPolicy` to restrict values accepted from +workflow jobs. The MicroVM policy keys are `egress-network-connectors`, +`image-arn`, and `image-version`. For example: + +```json +{ + "restricted_keys": { + "egress-network-connectors": { + "allowed": ["arn:aws:lambda:eu-west-1:123456789012:network-connector:github-runner-*"] + }, + "image-arn": { + "allowed": ["arn:aws:lambda:eu-west-1:123456789012:microvm-image:github-runner-*"] + }, + "image-version": { + "allowed": ["3.*"] + } + } +} +``` diff --git a/lambdas/libs/compute-providers/aws/microvm/control-plane.ts b/lambdas/libs/compute-providers/aws/microvm/control-plane.ts new file mode 100644 index 0000000000..d6287ca1e1 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/control-plane.ts @@ -0,0 +1,25 @@ +import type { ComputeProviderPlugin, CreateStartRunnerConfig } from '../../core'; + +import type { ControlPlaneProviderCapabilities, ControlPlaneProviderModule } from '../../contracts'; +import type {} from './src/environment'; +import { createMicrovmPoolProvider } from './src/control-plane/pool'; +import { createMicrovmScaleDownProvider } from './src/control-plane/scale-down'; +import { createMicrovmScaleUpProvider } from './src/control-plane/scale-up'; + +export function createMicrovmControlPlanePlugin( + createStartRunnerConfig: CreateStartRunnerConfig, +): ComputeProviderPlugin { + return { + type: 'microvm', + capabilities: { + pool: () => createMicrovmPoolProvider(createStartRunnerConfig), + scaleUp: () => createMicrovmScaleUpProvider(createStartRunnerConfig), + scaleDown: createMicrovmScaleDownProvider, + }, + }; +} + +export const provider = { + type: 'microvm', + createPlugin: createMicrovmControlPlanePlugin, +} satisfies ControlPlaneProviderModule<'microvm'>; diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.test.ts new file mode 100644 index 0000000000..e58d73093c --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.test.ts @@ -0,0 +1,85 @@ +import { beforeEach, describe, expect, it } from 'vitest'; + +import { loadMicrovmProviderConfig } from './config'; + +const cleanEnv = process.env; + +beforeEach(() => { + process.env = { ...cleanEnv }; + process.env.MICROVM_IMAGE_ARN = 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner'; + process.env.MICROVM_EXECUTION_ROLE_ARN = 'arn:aws:iam::123456789012:role/microvm-runner'; + process.env.MICROVM_METADATA_SSM_PATH = '/github-action-runners/unit-test/microvm-metadata/'; + process.env.SSM_TOKEN_PATH = '/github-action-runners/unit-test/token/'; + delete process.env.MICROVM_IMAGE_VERSION; + delete process.env.MICROVM_INGRESS_NETWORK_CONNECTORS; + delete process.env.MICROVM_EGRESS_NETWORK_CONNECTORS; + delete process.env.MICROVM_LOG_GROUP; +}); + +describe('loadMicrovmProviderConfig', () => { + it('loads required values and applies optional defaults', () => { + expect(loadMicrovmProviderConfig()).toEqual({ + imageIdentifier: process.env.MICROVM_IMAGE_ARN, + imageVersion: undefined, + executionRoleArn: process.env.MICROVM_EXECUTION_ROLE_ARN, + ingressNetworkConnectors: undefined, + egressNetworkConnectors: undefined, + metadataSsmPath: '/github-action-runners/unit-test/microvm-metadata', + runnerTokenSsmPath: '/github-action-runners/unit-test/token', + logging: undefined, + }); + }); + + it('loads versions, logging, and either connector list format', () => { + process.env.MICROVM_IMAGE_VERSION = ' 3.0 '; + process.env.MICROVM_INGRESS_NETWORK_CONNECTORS = '["arn:ingress:one","arn:ingress:two"]'; + process.env.MICROVM_EGRESS_NETWORK_CONNECTORS = 'arn:egress:one, arn:egress:two'; + process.env.MICROVM_LOG_GROUP = ' /aws/lambda-microvms/runner '; + + expect(loadMicrovmProviderConfig()).toMatchObject({ + imageVersion: '3.0', + ingressNetworkConnectors: ['arn:ingress:one', 'arn:ingress:two'], + egressNetworkConnectors: ['arn:egress:one', 'arn:egress:two'], + logging: { cloudWatch: { logGroup: '/aws/lambda-microvms/runner' } }, + }); + }); + + it.each([ + ['MICROVM_IMAGE_ARN', 'MICROVM_IMAGE_ARN'], + ['MICROVM_EXECUTION_ROLE_ARN', 'MICROVM_EXECUTION_ROLE_ARN'], + ['MICROVM_METADATA_SSM_PATH', 'MICROVM_METADATA_SSM_PATH'], + ['SSM_TOKEN_PATH', 'SSM_TOKEN_PATH'], + ])('requires %s', (environmentVariable, expectedName) => { + delete process.env[environmentVariable]; + + expect(() => loadMicrovmProviderConfig()).toThrow( + `${expectedName} must be configured for the MicroVM compute provider`, + ); + }); + + it.each(['[not-json', '[]', '["valid", 2]', 'first,'])('rejects malformed connector lists %s', (connectors) => { + process.env.MICROVM_EGRESS_NETWORK_CONNECTORS = connectors; + + expect(() => loadMicrovmProviderConfig()).toThrow(/MICROVM_EGRESS_NETWORK_CONNECTORS must/); + }); + + it.each(['metadata', '/', '/metadata//nested', '/metadata/../nested', '/metadata/has space'])( + 'rejects malformed metadata SSM path %s', + (metadataPath) => { + process.env.MICROVM_METADATA_SSM_PATH = metadataPath; + + expect(() => loadMicrovmProviderConfig()).toThrow( + 'MICROVM_METADATA_SSM_PATH must be a valid absolute SSM parameter path', + ); + }, + ); + + it.each(['token', '/', '/token//nested', '/token/../nested', '/token/has space'])( + 'rejects malformed JIT SSM path %s', + (tokenPath) => { + process.env.SSM_TOKEN_PATH = tokenPath; + + expect(() => loadMicrovmProviderConfig()).toThrow('SSM_TOKEN_PATH must be a valid absolute SSM parameter path'); + }, + ); +}); diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.ts new file mode 100644 index 0000000000..b86331967b --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.ts @@ -0,0 +1,78 @@ +import type { Logging, RunMicrovmCommandInput } from '@aws-sdk/client-lambda-microvms'; + +export interface MicrovmProviderConfig { + egressNetworkConnectors?: string[]; + executionRoleArn: string; + imageIdentifier: string; + imageVersion?: string; + ingressNetworkConnectors?: string[]; + logging?: Logging; + metadataSsmPath: string; + runnerTokenSsmPath: string; +} + +function requiredEnvironmentValue(name: string, value: string | undefined): string { + const trimmed = value?.trim(); + if (!trimmed) { + throw new Error(`${name} must be configured for the MicroVM compute provider`); + } + return trimmed; +} + +function optionalEnvironmentValue(value: string | undefined): string | undefined { + const trimmed = value?.trim(); + return trimmed ? trimmed : undefined; +} + +function parseSsmPath(name: string, value: string | undefined): string { + const path = requiredEnvironmentValue(name, value).replace(/\/+$/, ''); + if (path === '' || !/^\/[A-Za-z0-9_.\-/]+$/.test(path) || path.includes('//') || path.split('/').includes('..')) { + throw new Error(`${name} must be a valid absolute SSM parameter path`); + } + return path; +} + +function parseNetworkConnectors(name: string, value: string | undefined): string[] | undefined { + const configuredValue = optionalEnvironmentValue(value); + if (!configuredValue) return undefined; + + let connectors: unknown; + try { + connectors = configuredValue.startsWith('[') + ? JSON.parse(configuredValue) + : configuredValue.split(',').map((connector) => connector.trim()); + } catch (error) { + throw new Error(`${name} must be a JSON array or comma-separated list`, { cause: error }); + } + + if ( + !Array.isArray(connectors) || + connectors.length === 0 || + connectors.some((connector) => typeof connector !== 'string' || connector.trim().length === 0) + ) { + throw new Error(`${name} must contain one or more non-empty connector ARNs`); + } + + return connectors.map((connector) => connector.trim()); +} + +export function loadMicrovmProviderConfig(): MicrovmProviderConfig { + const logGroup = optionalEnvironmentValue(process.env.MICROVM_LOG_GROUP); + + return { + imageIdentifier: requiredEnvironmentValue('MICROVM_IMAGE_ARN', process.env.MICROVM_IMAGE_ARN), + imageVersion: optionalEnvironmentValue(process.env.MICROVM_IMAGE_VERSION), + executionRoleArn: requiredEnvironmentValue('MICROVM_EXECUTION_ROLE_ARN', process.env.MICROVM_EXECUTION_ROLE_ARN), + ingressNetworkConnectors: parseNetworkConnectors( + 'MICROVM_INGRESS_NETWORK_CONNECTORS', + process.env.MICROVM_INGRESS_NETWORK_CONNECTORS, + ), + egressNetworkConnectors: parseNetworkConnectors( + 'MICROVM_EGRESS_NETWORK_CONNECTORS', + process.env.MICROVM_EGRESS_NETWORK_CONNECTORS, + ), + metadataSsmPath: parseSsmPath('MICROVM_METADATA_SSM_PATH', process.env.MICROVM_METADATA_SSM_PATH), + runnerTokenSsmPath: parseSsmPath('SSM_TOKEN_PATH', process.env.SSM_TOKEN_PATH), + logging: logGroup ? ({ cloudWatch: { logGroup } } satisfies RunMicrovmCommandInput['logging']) : undefined, + }; +} diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/lifetime.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/lifetime.ts new file mode 100644 index 0000000000..09b6a46f8d --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/lifetime.ts @@ -0,0 +1 @@ +export const MICROVM_LIFETIME_IN_SECONDS = 28_800; diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.test.ts new file mode 100644 index 0000000000..3271fd8b98 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.test.ts @@ -0,0 +1,418 @@ +import { + LambdaMicrovmsClient, + ListMicrovmsCommand, + RunMicrovmCommand, + TerminateMicrovmCommand, +} from '@aws-sdk/client-lambda-microvms'; +import { mockClient } from 'aws-sdk-client-mock'; +import 'aws-sdk-client-mock-jest/vitest'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import type { MicrovmProviderConfig } from './config'; +import { + isRetryableMicrovmError, + listMicrovmRunners, + microvmBootTimeExceeded, + runMicrovmRunner, + terminateMicrovm, +} from './microvms'; +import { + createMicrovmRunnerMetadata, + deleteMicrovmRunnerJitConfig, + listMicrovmRunnerMetadata, + markMicrovmCleanupPending, + type MicrovmRunnerMetadata, +} from './runner-metadata'; + +vi.mock('./runner-metadata', async (importOriginal) => ({ + ...(await importOriginal()), + createMicrovmRunnerMetadata: vi.fn(), + deleteMicrovmRunnerJitConfig: vi.fn(), + listMicrovmRunnerMetadata: vi.fn(), + markMicrovmCleanupPending: vi.fn(), +})); + +const mockMicrovmClient = mockClient(LambdaMicrovmsClient); +const imageArn = 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner'; +const metadataSsmPath = '/github-action-runners/unit-test/microvm-metadata'; +const runnerTokenSsmPath = '/github-action-runners/unit-test/token'; +const ssmPaths = { metadataSsmPath, runnerTokenSsmPath }; +const config: MicrovmProviderConfig = { + imageIdentifier: imageArn, + imageVersion: '3.0', + executionRoleArn: 'arn:aws:iam::123456789012:role/microvm-runner', + egressNetworkConnectors: ['arn:egress'], + metadataSsmPath, + runnerTokenSsmPath, + logging: { cloudWatch: { logGroup: '/aws/lambda-microvms/runner' } }, +}; +const ssmParameterStoreTags = [{ Key: 'CostCenter', Value: '1234' }]; + +function metadata(overrides: Partial = {}): MicrovmRunnerMetadata { + return { + version: 1, + microvmId: 'mvm-managed', + environment: 'unit-test', + runnerOwner: 'Codertocat', + runnerType: 'Org', + source: 'scale-up-lambda', + imageArn, + imageVersion: '3.0', + createdAt: '2026-08-06T10:00:00.000Z', + expiresAt: '2026-08-06T11:00:00.000Z', + ...overrides, + }; +} + +beforeEach(() => { + mockMicrovmClient.reset(); + vi.clearAllMocks(); + vi.useRealTimers(); + delete process.env.MICROVM_MAXIMUM_DURATION_IN_SECONDS; + process.env.AWS_REGION = 'eu-west-1'; + process.env.RUNNER_BOOT_TIME_IN_MINUTES = '5'; + vi.mocked(createMicrovmRunnerMetadata).mockResolvedValue(ssmParameterStoreTags); + vi.mocked(deleteMicrovmRunnerJitConfig).mockResolvedValue(); + vi.mocked(listMicrovmRunnerMetadata).mockResolvedValue({ cleanupMicrovmIds: [], metadataById: new Map() }); + vi.mocked(markMicrovmCleanupPending).mockResolvedValue(); +}); + +describe('runMicrovmRunner', () => { + it('launches a runner for the fixed lifetime and records durable ownership metadata', async () => { + process.env.MICROVM_MAXIMUM_DURATION_IN_SECONDS = '1200'; + mockMicrovmClient.on(RunMicrovmCommand).resolves({ microvmId: 'mvm-123', imageArn, imageVersion: '3.1' }); + + await expect( + runMicrovmRunner({ + config, + environment: 'unit-test', + runHookPayload: '{"version":1}', + runnerOwner: 'Codertocat', + runnerType: 'Org', + ssmParameterStoreTags, + source: 'scale-up-lambda', + }), + ).resolves.toEqual({ microvmId: 'mvm-123', metadataTags: ssmParameterStoreTags }); + + expect(mockMicrovmClient).toHaveReceivedCommandWith(RunMicrovmCommand, { + imageIdentifier: imageArn, + imageVersion: '3.0', + executionRoleArn: config.executionRoleArn, + egressNetworkConnectors: ['arn:egress'], + maximumDurationInSeconds: 28_800, + logging: config.logging, + runHookPayload: '{"version":1}', + clientToken: expect.any(String), + }); + expect(createMicrovmRunnerMetadata).toHaveBeenCalledWith(metadataSsmPath, { + microvmId: 'mvm-123', + environment: 'unit-test', + runnerOwner: 'Codertocat', + runnerType: 'Org', + source: 'scale-up-lambda', + imageArn, + imageVersion: '3.1', + ssmParameterStoreTags, + }); + }); + + it('rejects invalid metadata tags before launching a MicroVM', async () => { + await expect( + runMicrovmRunner({ + config, + environment: 'unit-test', + runHookPayload: '{}', + runnerOwner: 'Codertocat', + runnerType: 'Org', + ssmParameterStoreTags: [{ Key: 'aws:microvm:image-arn', Value: imageArn }], + source: 'scale-up-lambda', + }), + ).rejects.toThrow('AWS-reserved tag prefix'); + expect(mockMicrovmClient).not.toHaveReceivedCommand(RunMicrovmCommand); + }); + + it('rejects a launch response without an ID', async () => { + mockMicrovmClient.on(RunMicrovmCommand).resolves({}); + + await expect( + runMicrovmRunner({ + config, + environment: 'unit-test', + runHookPayload: '{}', + runnerOwner: 'Codertocat', + runnerType: 'Org', + ssmParameterStoreTags: [], + source: 'pool-lambda', + }), + ).rejects.toThrow('RunMicrovm returned no microvmId'); + }); + + it('terminates a new runner when required metadata cannot be recorded', async () => { + mockMicrovmClient.on(RunMicrovmCommand).resolves({ microvmId: 'mvm-untracked', imageArn }); + mockMicrovmClient.on(TerminateMicrovmCommand).resolves({}); + vi.mocked(createMicrovmRunnerMetadata).mockRejectedValue(new Error('metadata failed')); + + await expect( + runMicrovmRunner({ + config, + environment: 'unit-test', + runHookPayload: '{}', + runnerOwner: 'Codertocat', + runnerType: 'Org', + ssmParameterStoreTags: [], + source: 'scale-up-lambda', + }), + ).rejects.toThrow('metadata failed'); + + expect(mockMicrovmClient).toHaveReceivedCommandWith(TerminateMicrovmCommand, { + microvmIdentifier: 'mvm-untracked', + }); + }); + + it('preserves the metadata error when termination also fails', async () => { + mockMicrovmClient.on(RunMicrovmCommand).resolves({ microvmId: 'mvm-untracked', imageArn }); + mockMicrovmClient.on(TerminateMicrovmCommand).rejects(new Error('terminate failed')); + vi.mocked(createMicrovmRunnerMetadata).mockRejectedValue(new Error('metadata failed')); + + await expect( + runMicrovmRunner({ + config, + environment: 'unit-test', + runHookPayload: '{}', + runnerOwner: 'Codertocat', + runnerType: 'Org', + ssmParameterStoreTags: [], + source: 'scale-up-lambda', + }), + ).rejects.toThrow('metadata failed'); + expect(markMicrovmCleanupPending).toHaveBeenCalledWith(metadataSsmPath, 'mvm-untracked'); + }); +}); + +describe('listMicrovmRunners', () => { + it('paginates active MicroVMs and filters them by durable metadata', async () => { + const startedAt = new Date('2026-08-06T10:00:00.000Z'); + mockMicrovmClient + .on(ListMicrovmsCommand) + .resolvesOnce({ + nextToken: 'page-2', + items: [ + { microvmId: 'mvm-managed', imageArn, imageVersion: '3.0', startedAt, state: 'RUNNING' }, + { microvmId: 'mvm-terminated', imageArn, imageVersion: '3.0', startedAt, state: 'TERMINATED' }, + ], + }) + .resolvesOnce({ + items: [{ microvmId: 'mvm-other', imageArn, imageVersion: '3.0', startedAt, state: 'PENDING' }], + }); + vi.mocked(listMicrovmRunnerMetadata).mockResolvedValue({ + cleanupMicrovmIds: [], + metadataById: new Map([ + ['mvm-managed', metadata({ githubRunnerId: '42', bypassRemoval: true })], + ['mvm-other', metadata({ microvmId: 'mvm-other', runnerOwner: 'Other' })], + ]), + }); + + await expect( + listMicrovmRunners( + { + environment: 'unit-test', + runnerOwner: 'Codertocat', + runnerType: 'Org', + }, + ssmPaths, + ), + ).resolves.toEqual([ + { + id: 'mvm-managed', + imageArn, + launchTime: startedAt, + owner: 'Codertocat', + type: 'Org', + orphan: false, + githubRunnerId: '42', + bypassRemoval: true, + state: 'RUNNING', + }, + ]); + + expect(mockMicrovmClient).toHaveReceivedNthCommandWith(2, ListMicrovmsCommand, { + maxResults: 50, + nextToken: 'page-2', + }); + expect(listMicrovmRunnerMetadata).toHaveBeenCalledWith( + ssmPaths, + new Map([ + ['mvm-managed', 'RUNNING'], + ['mvm-terminated', 'TERMINATED'], + ['mvm-other', 'PENDING'], + ]), + ); + }); + + it('applies environment, owner, type, and orphan filters after loading metadata', async () => { + mockMicrovmClient.on(ListMicrovmsCommand).resolves({ + items: [ + { + microvmId: 'mvm-filtered', + imageArn, + imageVersion: '3.0', + startedAt: new Date(), + state: 'SUSPENDED', + }, + ], + }); + vi.mocked(listMicrovmRunnerMetadata).mockResolvedValue({ + cleanupMicrovmIds: [], + metadataById: new Map([ + [ + 'mvm-filtered', + metadata({ microvmId: 'mvm-filtered', environment: 'other', runnerOwner: 'Other', runnerType: 'Repo' }), + ], + ]), + }); + + await expect(listMicrovmRunners({ environment: 'unit-test' }, ssmPaths)).resolves.toEqual([]); + await expect(listMicrovmRunners({ runnerOwner: 'Codertocat' }, ssmPaths)).resolves.toEqual([]); + await expect(listMicrovmRunners({ runnerType: 'Org' }, ssmPaths)).resolves.toEqual([]); + await expect(listMicrovmRunners({ orphan: true }, ssmPaths)).resolves.toEqual([]); + }); + + it('fails closed for an image mismatch while ignoring unowned MicroVMs', async () => { + mockMicrovmClient.on(ListMicrovmsCommand).resolves({ + items: [ + { microvmId: 'mvm-missing', imageArn, imageVersion: '3.0', state: 'RUNNING' }, + { microvmId: 'mvm-mismatch', imageArn, imageVersion: '3.0', state: 'RUNNING' }, + ], + }); + vi.mocked(listMicrovmRunnerMetadata).mockResolvedValue({ + cleanupMicrovmIds: [], + metadataById: new Map([ + ['mvm-mismatch', metadata({ microvmId: 'mvm-mismatch', imageArn: imageArn.replace(':runner', ':other') })], + ]), + }); + + await expect(listMicrovmRunners({}, ssmPaths)).rejects.toThrow('does not match its metadata'); + }); + + it('attempts every pending cleanup and fails inventory closed when a retry fails', async () => { + const cleanupFailure = new Error('cleanup failed'); + mockMicrovmClient.on(ListMicrovmsCommand).resolves({ + items: [ + { microvmId: 'mvm-first', imageArn, imageVersion: '3.0', state: 'RUNNING' }, + { microvmId: 'mvm-second', imageArn, imageVersion: '3.0', state: 'PENDING' }, + ], + }); + mockMicrovmClient.on(TerminateMicrovmCommand, { microvmIdentifier: 'mvm-first' }).rejects(cleanupFailure); + mockMicrovmClient.on(TerminateMicrovmCommand, { microvmIdentifier: 'mvm-second' }).resolves({}); + vi.mocked(listMicrovmRunnerMetadata).mockResolvedValue({ + cleanupMicrovmIds: ['mvm-first', 'mvm-second'], + metadataById: new Map(), + }); + + await expect(listMicrovmRunners({}, ssmPaths)).rejects.toThrow('cleanup failed'); + expect(mockMicrovmClient).toHaveReceivedCommandWith(TerminateMicrovmCommand, { + microvmIdentifier: 'mvm-first', + }); + expect(mockMicrovmClient).toHaveReceivedCommandWith(TerminateMicrovmCommand, { + microvmIdentifier: 'mvm-second', + }); + expect(markMicrovmCleanupPending).toHaveBeenCalledTimes(2); + }); + + it('surfaces metadata lookup failures instead of reporting zero runners', async () => { + mockMicrovmClient.on(ListMicrovmsCommand).resolves({ + items: [{ microvmId: 'mvm-error', imageArn, imageVersion: '3.0', state: 'RUNNING' }], + }); + vi.mocked(listMicrovmRunnerMetadata).mockRejectedValue(new Error('AccessDenied')); + + await expect(listMicrovmRunners({}, ssmPaths)).rejects.toThrow('AccessDenied'); + }); +}); + +describe('MicroVM lifecycle helpers', () => { + it('retains metadata until inventory observes a terminated MicroVM', async () => { + mockMicrovmClient.on(TerminateMicrovmCommand).resolves({}); + + await terminateMicrovm('mvm-123', ssmPaths); + + expect(deleteMicrovmRunnerJitConfig).toHaveBeenCalledWith(runnerTokenSsmPath, 'mvm-123'); + expect(markMicrovmCleanupPending).toHaveBeenCalledWith(metadataSsmPath, 'mvm-123'); + }); + + it('retains the tombstone when the MicroVM is already terminated so a late JIT write can be revoked', async () => { + const notFound = Object.assign(new Error('gone'), { name: 'ResourceNotFoundException' }); + mockMicrovmClient.on(TerminateMicrovmCommand).rejects(notFound); + + await expect(terminateMicrovm('mvm-gone', ssmPaths)).resolves.toBeUndefined(); + expect(markMicrovmCleanupPending).toHaveBeenCalledWith(metadataSsmPath, 'mvm-gone'); + expect(deleteMicrovmRunnerJitConfig).toHaveBeenCalledWith(runnerTokenSsmPath, 'mvm-gone'); + }); + + it('retains metadata and marks cleanup pending when termination fails', async () => { + mockMicrovmClient.on(TerminateMicrovmCommand).rejects(new Error('terminate failed')); + + await expect(terminateMicrovm('mvm-123', ssmPaths)).rejects.toThrow('terminate failed'); + expect(markMicrovmCleanupPending).toHaveBeenCalledWith(metadataSsmPath, 'mvm-123'); + }); + + it('retains the cleanup marker and reports a JIT deletion failure after termination succeeds', async () => { + const error = new Error('JIT cleanup failed'); + vi.mocked(deleteMicrovmRunnerJitConfig).mockRejectedValue(error); + mockMicrovmClient.on(TerminateMicrovmCommand).resolves({}); + + await expect(terminateMicrovm('mvm-123', ssmPaths)).rejects.toBe(error); + expect(markMicrovmCleanupPending).toHaveBeenCalledWith(metadataSsmPath, 'mvm-123'); + }); + + it('still terminates and reports a cleanup-marker failure for retry', async () => { + const error = new Error('metadata cleanup marker failed'); + vi.mocked(markMicrovmCleanupPending).mockRejectedValue(error); + mockMicrovmClient.on(TerminateMicrovmCommand).resolves({}); + + await expect(terminateMicrovm('mvm-123', ssmPaths)).rejects.toBe(error); + expect(deleteMicrovmRunnerJitConfig).toHaveBeenCalledWith(runnerTokenSsmPath, 'mvm-123'); + expect(mockMicrovmClient).toHaveReceivedCommandWith(TerminateMicrovmCommand, { + microvmIdentifier: 'mvm-123', + }); + }); + + it('evaluates the configured boot window', () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-08-06T10:10:00.000Z')); + + expect(microvmBootTimeExceeded({})).toBe(false); + expect(microvmBootTimeExceeded({ launchTime: new Date('2026-08-06T10:06:00.000Z') })).toBe(false); + expect(microvmBootTimeExceeded({ launchTime: new Date('2026-08-06T10:04:00.000Z') })).toBe(true); + }); +}); + +describe('isRetryableMicrovmError', () => { + it.each([ + 'ConflictException', + 'InternalServerException', + 'ServiceQuotaExceededException', + 'ThrottlingException', + 'TooManyUpdates', + ])('classifies %s as retryable', (name) => { + expect(isRetryableMicrovmError(Object.assign(new Error(name), { name }))).toBe(true); + }); + + it('classifies server, throttling, network, and nested failures as retryable', () => { + expect(isRetryableMicrovmError(Object.assign(new Error('server'), { $fault: 'server' }))).toBe(true); + expect(isRetryableMicrovmError(Object.assign(new Error('throttle'), { $metadata: { httpStatusCode: 429 } }))).toBe( + true, + ); + expect(isRetryableMicrovmError(Object.assign(new Error('network'), { code: 'ECONNRESET' }))).toBe(true); + expect( + isRetryableMicrovmError( + Object.assign(new Error('outer'), { cause: Object.assign(new Error(), { code: 'ETIMEDOUT' }) }), + ), + ).toBe(true); + }); + + it('does not retry configuration, unknown, or non-error failures', () => { + expect(isRetryableMicrovmError(Object.assign(new Error('invalid'), { name: 'ValidationException' }))).toBe(false); + expect(isRetryableMicrovmError(new Error('unknown'))).toBe(false); + expect(isRetryableMicrovmError('failure')).toBe(false); + }); +}); diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.ts new file mode 100644 index 0000000000..3293db55ba --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.ts @@ -0,0 +1,270 @@ +import { randomUUID } from 'node:crypto'; + +import { createChildLogger, getTracedAWSV3Client } from '@aws-github-runner/aws-powertools-util'; +import { + LambdaMicrovmsClient, + ListMicrovmsCommand, + RunMicrovmCommand, + TerminateMicrovmCommand, +} from '@aws-sdk/client-lambda-microvms'; +import type { MicrovmItem, MicrovmState, RunMicrovmCommandInput } from '@aws-sdk/client-lambda-microvms'; + +import type { RunnerSource, ListRunnerFilters, RunnerInfo, RunnerType } from '../../../../core'; +import { loadMicrovmProviderConfig, type MicrovmProviderConfig } from './config'; +import { MICROVM_LIFETIME_IN_SECONDS } from './lifetime'; +import { + assertValidMicrovmMetadataTags, + createMicrovmRunnerMetadata, + deleteMicrovmRunnerJitConfig, + listMicrovmRunnerMetadata, + markMicrovmCleanupPending, + type MicrovmMetadataTag, + type MicrovmSsmPaths, +} from './runner-metadata'; + +const logger = createChildLogger('microvm-runners'); + +const ACTIVE_STATES = new Set(['PENDING', 'RUNNING', 'SUSPENDING', 'SUSPENDED']); + +export interface MicrovmRunnerInfo extends RunnerInfo { + imageArn?: string; + state?: MicrovmState; +} + +export interface RunMicrovmRunnerInput { + config: MicrovmProviderConfig; + environment: string; + runHookPayload: string; + runnerOwner: string; + runnerType: RunnerType; + ssmParameterStoreTags: MicrovmMetadataTag[]; + source: RunnerSource; +} + +export interface RunMicrovmRunnerResult { + metadataTags: MicrovmMetadataTag[]; + microvmId: string; +} + +interface AwsErrorLike extends Error { + cause?: unknown; + code?: string; + $fault?: 'client' | 'server'; + $metadata?: { httpStatusCode?: number }; +} + +const RETRYABLE_ERROR_NAMES = new Set([ + 'ConflictException', + 'InternalServerException', + 'RequestTimeout', + 'RequestTimeoutException', + 'ResourceConflictException', + 'ServiceException', + 'ServiceQuotaExceededException', + 'Throttling', + 'ThrottlingException', + 'TooManyUpdates', + 'TooManyRequestsException', +]); + +const RETRYABLE_NETWORK_ERROR_CODES = new Set([ + 'EAI_AGAIN', + 'ECONNREFUSED', + 'ECONNRESET', + 'ENETUNREACH', + 'ENOTFOUND', + 'ETIMEDOUT', +]); + +function microvmClient(): LambdaMicrovmsClient { + return getTracedAWSV3Client(new LambdaMicrovmsClient({ region: process.env.AWS_REGION })); +} + +export async function runMicrovmRunner(input: RunMicrovmRunnerInput): Promise { + assertValidMicrovmMetadataTags({ + microvmId: 'microvm-validation', + environment: input.environment, + runnerOwner: input.runnerOwner, + runnerType: input.runnerType, + source: input.source, + imageArn: input.config.imageIdentifier, + imageVersion: input.config.imageVersion ?? 'version-validation', + ssmParameterStoreTags: input.ssmParameterStoreTags, + }); + + const commandInput: RunMicrovmCommandInput = { + imageIdentifier: input.config.imageIdentifier, + imageVersion: input.config.imageVersion, + executionRoleArn: input.config.executionRoleArn, + ingressNetworkConnectors: input.config.ingressNetworkConnectors, + egressNetworkConnectors: input.config.egressNetworkConnectors, + maximumDurationInSeconds: MICROVM_LIFETIME_IN_SECONDS, + logging: input.config.logging, + runHookPayload: input.runHookPayload, + clientToken: randomUUID(), + }; + + logger.debug('Launching Lambda MicroVM runner', { + imageIdentifier: commandInput.imageIdentifier, + imageVersion: commandInput.imageVersion, + maximumDurationInSeconds: commandInput.maximumDurationInSeconds, + }); + + const response = await microvmClient().send(new RunMicrovmCommand(commandInput)); + if (!response.microvmId) { + throw new Error('RunMicrovm returned no microvmId'); + } + + const imageArn = response.imageArn ?? input.config.imageIdentifier; + const imageVersion = response.imageVersion ?? input.config.imageVersion; + + try { + const metadataTags = await createMicrovmRunnerMetadata(input.config.metadataSsmPath, { + microvmId: response.microvmId, + environment: input.environment, + runnerOwner: input.runnerOwner, + runnerType: input.runnerType, + source: input.source, + imageArn, + imageVersion, + ssmParameterStoreTags: input.ssmParameterStoreTags, + }); + return { microvmId: response.microvmId, metadataTags }; + } catch (error) { + logger.error(`Failed to record metadata for new MicroVM runner '${response.microvmId}', terminating it`, { + error, + }); + await terminateMicrovm(response.microvmId, input.config).catch((terminationError) => { + logger.error(`Failed to terminate untracked MicroVM runner '${response.microvmId}'`, { + error: terminationError, + }); + }); + throw error; + } +} + +export async function listMicrovmRunners( + filters: ListRunnerFilters = {}, + paths: MicrovmSsmPaths = loadMicrovmProviderConfig(), +): Promise { + const client = microvmClient(); + const items: MicrovmItem[] = []; + let nextToken: string | undefined; + + do { + const response = await client.send( + new ListMicrovmsCommand({ + maxResults: 50, + nextToken, + }), + ); + items.push(...(response.items ?? [])); + nextToken = response.nextToken; + } while (nextToken); + + const activeItems = items.filter( + (item): item is MicrovmItem & { imageArn: string; microvmId: string; state: MicrovmState } => + Boolean(item.microvmId && item.imageArn && item.state && ACTIVE_STATES.has(item.state)), + ); + const microvmStates = new Map( + items.flatMap((item) => (item.microvmId && item.state ? [[item.microvmId, item.state] as const] : [])), + ); + const { cleanupMicrovmIds, metadataById } = await listMicrovmRunnerMetadata(paths, microvmStates); + + let cleanupError: unknown; + for (const microvmId of cleanupMicrovmIds) { + logger.warn(`Retrying cleanup of MicroVM runner '${microvmId}'`); + try { + await terminateMicrovm(microvmId, paths); + } catch (error) { + cleanupError ??= error; + logger.error(`Failed to retry cleanup of MicroVM runner '${microvmId}'`, { error }); + } + } + if (cleanupError !== undefined) throw cleanupError; + + const runners: MicrovmRunnerInfo[] = []; + for (const item of activeItems) { + const metadata = metadataById.get(item.microvmId); + if (!metadata) continue; + if (metadata.imageArn !== item.imageArn) { + throw new Error(`Active MicroVM runner '${item.microvmId}' has an image that does not match its metadata`); + } + + const orphan = Boolean(metadata.orphan); + if (filters.environment !== undefined && metadata.environment !== filters.environment) continue; + if (filters.runnerType !== undefined && metadata.runnerType !== filters.runnerType) continue; + if (filters.runnerOwner !== undefined && metadata.runnerOwner !== filters.runnerOwner) continue; + if (filters.orphan && !orphan) continue; + + runners.push({ + id: item.microvmId, + imageArn: item.imageArn, + launchTime: item.startedAt, + owner: metadata.runnerOwner, + type: metadata.runnerType, + orphan, + githubRunnerId: metadata.githubRunnerId, + bypassRemoval: metadata.bypassRemoval ?? false, + state: item.state, + }); + } + + return runners; +} + +export async function terminateMicrovm(microvmId: string, paths: MicrovmSsmPaths): Promise { + let cleanupPreparationError: unknown; + try { + await markMicrovmCleanupPending(paths.metadataSsmPath, microvmId); + } catch (error) { + cleanupPreparationError = error; + logger.error(`Failed to mark MicroVM runner '${microvmId}' for cleanup`, { error }); + } + + try { + await deleteMicrovmRunnerJitConfig(paths.runnerTokenSsmPath, microvmId); + } catch (error) { + cleanupPreparationError ??= error; + logger.error(`Failed to delete JIT configuration for MicroVM runner '${microvmId}'`, { error }); + } + + try { + await microvmClient().send(new TerminateMicrovmCommand({ microvmIdentifier: microvmId })); + } catch (error) { + if (error instanceof Error && error.name === 'ResourceNotFoundException') { + if (cleanupPreparationError !== undefined) throw cleanupPreparationError; + return; + } + + throw error; + } + + if (cleanupPreparationError !== undefined) throw cleanupPreparationError; +} + +export function microvmBootTimeExceeded(runner: { launchTime?: Date }): boolean { + if (!runner.launchTime) return false; + + const bootTimeInMinutes = Number(process.env.RUNNER_BOOT_TIME_IN_MINUTES || '5'); + return runner.launchTime.getTime() + bootTimeInMinutes * 60_000 < Date.now(); +} + +export function isRetryableMicrovmError(error: unknown): boolean { + if (!(error instanceof Error)) return false; + + const awsError = error as AwsErrorLike; + if (RETRYABLE_ERROR_NAMES.has(awsError.name)) return true; + + const statusCode = awsError.$metadata?.httpStatusCode; + if ( + awsError.$fault === 'server' || + statusCode === 429 || + (statusCode !== undefined && statusCode >= 500) || + (awsError.code !== undefined && RETRYABLE_NETWORK_ERROR_CODES.has(awsError.code)) + ) { + return true; + } + + return awsError.cause !== undefined && awsError.cause !== error ? isRetryableMicrovmError(awsError.cause) : false; +} diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/pool.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/pool.test.ts new file mode 100644 index 0000000000..719a0058c1 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/pool.test.ts @@ -0,0 +1,109 @@ +import type { Octokit } from '@octokit/rest'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import type { CreateGitHubRunnerConfig, CreateStartRunnerConfig } from '../../../../core'; +import { listMicrovmRunners, microvmBootTimeExceeded } from './microvms'; +import type { MicrovmRunnerInfo } from './microvms'; +import { calculateMicrovmPoolSize, createMicrovmPoolProvider } from './pool'; +import { createMicrovmRunners } from './runner-config'; + +vi.mock('./microvms', () => ({ + listMicrovmRunners: vi.fn(), + microvmBootTimeExceeded: vi.fn(), +})); +vi.mock('./runner-config', () => ({ createMicrovmRunners: vi.fn() })); + +const createStartRunnerConfig = vi.fn(); +const githubClient = {} as Octokit; +function runner(id: string, state: MicrovmRunnerInfo['state']): MicrovmRunnerInfo { + return { id, state, owner: 'Codertocat', type: 'Org' }; +} + +function githubRunnerConfig(): CreateGitHubRunnerConfig { + return { + ephemeral: true, + enableJitConfig: true, + runnerLabels: 'self-hosted,microvm', + runnerGroup: 'Default', + runnerNamePrefix: '', + runnerOwner: 'Codertocat', + runnerType: 'Org', + disableAutoUpdate: true, + }; +} + +beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(listMicrovmRunners).mockResolvedValue([]); + vi.mocked(microvmBootTimeExceeded).mockReturnValue(false); + vi.mocked(createMicrovmRunners).mockResolvedValue({ + instances: ['mvm-1'], + retryableErrorCount: 0, + nonRetryableErrorCount: 0, + }); +}); + +describe('calculateMicrovmPoolSize', () => { + it('counts online idle running runners', () => { + expect( + calculateMicrovmPoolSize( + [runner('mvm-idle', 'RUNNING')], + new Map([['mvm-idle', { busy: false, status: 'online' }]]), + ), + ).toBe(1); + }); + + it('optionally counts online busy runners', () => { + const runners = [runner('mvm-busy', 'RUNNING')]; + const statuses = new Map([['mvm-busy', { busy: true, status: 'online' }]]); + + expect(calculateMicrovmPoolSize(runners, statuses)).toBe(0); + expect(calculateMicrovmPoolSize(runners, statuses, true)).toBe(1); + }); + + it('counts pending runners only during their boot window', () => { + const runners = [runner('mvm-pending', 'PENDING')]; + vi.mocked(microvmBootTimeExceeded).mockReturnValueOnce(false).mockReturnValueOnce(true); + + expect(calculateMicrovmPoolSize(runners, new Map())).toBe(1); + expect(calculateMicrovmPoolSize(runners, new Map())).toBe(0); + }); + + it('does not count suspended or offline runners', () => { + expect( + calculateMicrovmPoolSize( + [runner('mvm-suspended', 'SUSPENDED'), runner('mvm-offline', 'RUNNING')], + new Map([['mvm-offline', { busy: false, status: 'offline' }]]), + ), + ).toBe(0); + }); +}); + +describe('createMicrovmPoolProvider', () => { + it('lists managed MicroVMs and returns successfully created IDs', async () => { + const provider = createMicrovmPoolProvider(createStartRunnerConfig); + const input = { + environment: 'unit-test', + runnerOwner: 'Codertocat', + runnerType: 'Org' as const, + }; + + await expect(provider.listRunners(input)).resolves.toEqual([]); + expect(listMicrovmRunners).toHaveBeenCalledWith(input); + + await expect( + provider.createRunners({ + githubRunnerConfig: githubRunnerConfig(), + numberOfRunners: 1, + githubInstallationClient: githubClient, + }), + ).resolves.toEqual(['mvm-1']); + expect(createMicrovmRunners).toHaveBeenCalledWith( + expect.any(Object), + 1, + githubClient, + createStartRunnerConfig, + 'pool-lambda', + ); + }); +}); diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/pool.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/pool.ts new file mode 100644 index 0000000000..8deed5562d --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/pool.ts @@ -0,0 +1,65 @@ +import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; + +import type { + CreatePoolRunnersInput, + CreateStartRunnerConfig, + ListPoolRunnersInput, + PoolComputeProvider, + RunnerStatus, +} from '../../../../core'; +import type { MicrovmRunnerInfo } from './microvms'; +import { listMicrovmRunners, microvmBootTimeExceeded } from './microvms'; +import { createMicrovmRunners } from './runner-config'; + +const logger = createChildLogger('microvm-pool'); + +async function listMicrovmPoolRunners(input: ListPoolRunnersInput): Promise { + return await listMicrovmRunners(input); +} + +async function createMicrovmPoolRunners( + { githubRunnerConfig, numberOfRunners, githubInstallationClient }: CreatePoolRunnersInput, + createStartRunnerConfig: CreateStartRunnerConfig, +): Promise { + const result = await createMicrovmRunners( + githubRunnerConfig, + numberOfRunners, + githubInstallationClient, + createStartRunnerConfig, + 'pool-lambda', + ); + return result.instances; +} + +export function calculateMicrovmPoolSize( + runners: MicrovmRunnerInfo[], + runnerStatus: Map, + includeBusyRunners = false, +): number { + let availableRunners = 0; + + for (const runner of runners) { + const status = runnerStatus.get(runner.id); + if (runner.state === 'RUNNING' && status?.status === 'online' && (!status.busy || includeBusyRunners)) { + availableRunners++; + logger.debug(`MicroVM runner ${runner.id} is online and counted as part of the pool`); + } else if (runner.state === 'PENDING' && !microvmBootTimeExceeded(runner)) { + availableRunners++; + logger.info(`MicroVM runner ${runner.id} is still booting and counted as part of the pool`); + } else { + logger.debug(`MicroVM runner ${runner.id} is not available and is not counted as part of the pool`); + } + } + + return availableRunners; +} + +export function createMicrovmPoolProvider( + createStartRunnerConfig: CreateStartRunnerConfig, +): Omit, 'type'> { + return { + listRunners: listMicrovmPoolRunners, + countAvailableRunners: calculateMicrovmPoolSize, + createRunners: (input) => createMicrovmPoolRunners(input, createStartRunnerConfig), + }; +} diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.test.ts new file mode 100644 index 0000000000..6d3f92fc76 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.test.ts @@ -0,0 +1,322 @@ +import type { Octokit } from '@octokit/rest'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import type { CreateGitHubRunnerConfig, CreateStartRunnerConfig } from '../../../../core'; +import { loadMicrovmProviderConfig } from './config'; +import { isRetryableMicrovmError, runMicrovmRunner, terminateMicrovm } from './microvms'; +import { createMicrovmRunHookPayload, createMicrovmRunners } from './runner-config'; +import { setMicrovmGithubRunnerMetadata } from './runner-metadata'; + +vi.mock('./config', () => ({ loadMicrovmProviderConfig: vi.fn() })); +vi.mock('./microvms', () => ({ + isRetryableMicrovmError: vi.fn(), + runMicrovmRunner: vi.fn(), + terminateMicrovm: vi.fn(), +})); +vi.mock('./runner-metadata', async (importOriginal) => ({ + ...(await importOriginal()), + setMicrovmGithubRunnerMetadata: vi.fn(), +})); + +const imageArn = 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner'; +const metadataSsmPath = '/github-action-runners/unit-test/microvm-metadata'; +const runnerConfigSsmPath = '/github-action-runners/unit-test/config'; +const runnerTokenSsmPath = '/github-action-runners/unit-test/token'; +const githubClient = {} as Octokit; +const createStartRunnerConfig = vi.fn(); +const ssmParameterStoreTags = [ + { Key: 'CostCenter', Value: '1234' }, + { Key: 'Name', Value: 'not-used-for-microvm-metadata' }, + { Key: 'ghr:environment', Value: 'caller-cannot-override' }, + { Key: 'ghr:runner_name_prefix', Value: 'caller-cannot-override' }, + { Key: 'ghr:ssm_config_path', Value: 'caller-cannot-override' }, +]; +const microvmMetadataTags = [ + { Key: 'CostCenter', Value: '1234' }, + { Key: 'ghr:environment', Value: 'unit-test' }, + { Key: 'ghr:runner_name_prefix', Value: 'unit-test-' }, + { Key: 'ghr:ssm_config_path', Value: runnerConfigSsmPath }, +]; +const canonicalMetadataTags = [ + ...microvmMetadataTags, + { Key: 'ghr:Application', Value: 'github-action-runner' }, + { Key: 'ghr:microvm_id', Value: 'mvm-1' }, +]; +const providerConfig = { + imageIdentifier: imageArn, + imageVersion: '2.0', + executionRoleArn: 'arn:aws:iam::123456789012:role/microvm-runner', + metadataSsmPath, + runnerTokenSsmPath, +}; + +function runnerConfig(overrides: Partial = {}): CreateGitHubRunnerConfig { + return { + ephemeral: true, + enableJitConfig: true, + runnerLabels: 'self-hosted,linux,arm64,microvm', + runnerGroup: 'Default', + runnerNamePrefix: 'unit-test-', + runnerOwner: 'Codertocat', + runnerType: 'Org', + disableAutoUpdate: true, + ...overrides, + }; +} + +beforeEach(() => { + vi.clearAllMocks(); + process.env.ENVIRONMENT = 'unit-test'; + process.env.SSM_CONFIG_PATH = runnerConfigSsmPath; + process.env.SSM_PARAMETER_STORE_TAGS = JSON.stringify(ssmParameterStoreTags); + process.env.SSM_TOKEN_PATH = runnerTokenSsmPath; + vi.mocked(loadMicrovmProviderConfig).mockReturnValue(providerConfig); + vi.mocked(runMicrovmRunner).mockResolvedValue({ microvmId: 'mvm-1', metadataTags: canonicalMetadataTags }); + vi.mocked(setMicrovmGithubRunnerMetadata).mockResolvedValue(); + vi.mocked(terminateMicrovm).mockResolvedValue(); + vi.mocked(isRetryableMicrovmError).mockReturnValue(false); + createStartRunnerConfig.mockResolvedValue([]); +}); + +describe('createMicrovmRunHookPayload', () => { + it('contains the image and versioned runner paths', () => { + expect( + JSON.parse( + createMicrovmRunHookPayload({ + imageArn, + imageVersion: '2.0', + runnerConfigSsmPath, + runnerTokenSsmPath: '/runner/token', + }), + ), + ).toEqual({ + imageArn, + imageVersion: '2.0', + version: 1, + runnerConfigSsmPath, + runnerTokenSsmPath: '/runner/token', + }); + }); + + it('requires the image ARN and version to be provided together', () => { + expect(() => + createMicrovmRunHookPayload({ + imageArn, + runnerConfigSsmPath, + runnerTokenSsmPath, + }), + ).toThrow('MicroVM hook payload image ARN and version must be provided together'); + }); + + it('omits image metadata when no explicit image version is selected', () => { + expect(JSON.parse(createMicrovmRunHookPayload({ runnerConfigSsmPath, runnerTokenSsmPath }))).toEqual({ + version: 1, + runnerConfigSsmPath, + runnerTokenSsmPath, + }); + }); +}); + +describe('createMicrovmRunners', () => { + it.each([{ ephemeral: false }, { enableJitConfig: false }])( + 'rejects unsupported runner configuration %j', + async (overrides) => { + await expect( + createMicrovmRunners(runnerConfig(overrides), 2, githubClient, createStartRunnerConfig, 'scale-up-lambda'), + ).resolves.toEqual({ instances: [], retryableErrorCount: 0, nonRetryableErrorCount: 2 }); + + expect(runMicrovmRunner).not.toHaveBeenCalled(); + }, + ); + + it('requires an SSM token path', async () => { + process.env.SSM_TOKEN_PATH = ''; + await expect( + createMicrovmRunners(runnerConfig(), 1, githubClient, createStartRunnerConfig, 'scale-up-lambda'), + ).resolves.toEqual({ instances: [], retryableErrorCount: 0, nonRetryableErrorCount: 1 }); + + expect(runMicrovmRunner).not.toHaveBeenCalled(); + }); + + it('requires an SSM config path', async () => { + process.env.SSM_CONFIG_PATH = ''; + await expect( + createMicrovmRunners(runnerConfig(), 1, githubClient, createStartRunnerConfig, 'scale-up-lambda'), + ).resolves.toEqual({ instances: [], retryableErrorCount: 0, nonRetryableErrorCount: 1 }); + + expect(runMicrovmRunner).not.toHaveBeenCalled(); + }); + + it('rejects a metadata path that overlaps the JIT token path', async () => { + vi.mocked(loadMicrovmProviderConfig).mockReturnValue({ + imageIdentifier: imageArn, + executionRoleArn: 'arn:aws:iam::123456789012:role/microvm-runner', + metadataSsmPath: '/github-action-runners/unit-test/token/metadata', + runnerTokenSsmPath, + }); + + await expect( + createMicrovmRunners(runnerConfig(), 1, githubClient, createStartRunnerConfig, 'scale-up-lambda'), + ).resolves.toEqual({ instances: [], retryableErrorCount: 0, nonRetryableErrorCount: 1 }); + expect(runMicrovmRunner).not.toHaveBeenCalled(); + }); + + it('canonicalizes the configuration and token paths before launching or writing JIT configuration', async () => { + process.env.SSM_CONFIG_PATH = `${runnerConfigSsmPath}/`; + process.env.SSM_TOKEN_PATH = `${runnerTokenSsmPath}/`; + await expect( + createMicrovmRunners(runnerConfig(), 1, githubClient, createStartRunnerConfig, 'scale-up-lambda'), + ).resolves.toEqual({ instances: ['mvm-1'], retryableErrorCount: 0, nonRetryableErrorCount: 0 }); + + expect(runMicrovmRunner).toHaveBeenCalledWith( + expect.objectContaining({ + runHookPayload: createMicrovmRunHookPayload({ + imageArn, + imageVersion: '2.0', + runnerConfigSsmPath, + runnerTokenSsmPath, + }), + ssmParameterStoreTags: microvmMetadataTags, + }), + ); + expect(createStartRunnerConfig).toHaveBeenCalledWith(runnerConfig(), ['mvm-1'], githubClient, expect.any(Object)); + }); + + it('classifies invalid provider configuration as non-retryable', async () => { + vi.mocked(loadMicrovmProviderConfig).mockImplementation(() => { + throw new Error('missing image'); + }); + + await expect( + createMicrovmRunners(runnerConfig(), 3, githubClient, createStartRunnerConfig, 'scale-up-lambda'), + ).resolves.toEqual({ instances: [], retryableErrorCount: 0, nonRetryableErrorCount: 3 }); + }); + + it('launches each MicroVM and delivers its JIT configuration', async () => { + vi.mocked(runMicrovmRunner) + .mockResolvedValueOnce({ microvmId: 'mvm-1', metadataTags: canonicalMetadataTags }) + .mockResolvedValueOnce({ microvmId: 'mvm-2', metadataTags: canonicalMetadataTags }); + createStartRunnerConfig.mockImplementation(async (_config, runnerIds, _client, options) => { + await options?.onJitConfigCreated?.(runnerIds[0], { + githubRunnerId: `github-${runnerIds[0]}`, + runnerLabels: ['self-hosted', 'microvm'], + }); + return []; + }); + + await expect( + createMicrovmRunners(runnerConfig(), 2, githubClient, createStartRunnerConfig, 'pool-lambda'), + ).resolves.toEqual({ instances: ['mvm-1', 'mvm-2'], retryableErrorCount: 0, nonRetryableErrorCount: 0 }); + + expect(runMicrovmRunner).toHaveBeenNthCalledWith(1, { + config: expect.objectContaining({ imageIdentifier: imageArn }), + environment: 'unit-test', + runHookPayload: createMicrovmRunHookPayload({ + imageArn, + imageVersion: '2.0', + runnerConfigSsmPath, + runnerTokenSsmPath, + }), + runnerOwner: 'Codertocat', + runnerType: 'Org', + ssmParameterStoreTags: microvmMetadataTags, + source: 'pool-lambda', + }); + expect(createStartRunnerConfig).toHaveBeenCalledTimes(2); + const options = createStartRunnerConfig.mock.calls[0][3]; + expect(options?.getRunnerConfigMetadata?.('mvm-1')).toEqual([{ key: 'MicrovmId', value: 'mvm-1' }]); + expect(setMicrovmGithubRunnerMetadata).toHaveBeenNthCalledWith( + 1, + providerConfig, + 'mvm-1', + { + githubRunnerId: 'github-mvm-1', + runnerLabels: ['self-hosted', 'microvm'], + }, + canonicalMetadataTags, + ); + }); + + it('applies supported dynamic labels to the provider configuration', async () => { + const overrideImageArn = 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner-large'; + const overrideEgressConnectorArn = + 'arn:aws:lambda:eu-west-1:123456789012:network-connector:github-runner-private-egress'; + createStartRunnerConfig.mockImplementation(async (_config, runnerIds, _client, options) => { + await options?.onJitConfigCreated?.(runnerIds[0], { githubRunnerId: 'github-mvm-1', runnerLabels: [] }); + return []; + }); + + await createMicrovmRunners(runnerConfig(), 1, githubClient, createStartRunnerConfig, 'scale-up-lambda', { + egressNetworkConnectors: [overrideEgressConnectorArn], + imageIdentifier: overrideImageArn, + imageVersion: '3.0', + }); + + expect(runMicrovmRunner).toHaveBeenCalledWith({ + config: { + egressNetworkConnectors: [overrideEgressConnectorArn], + imageIdentifier: overrideImageArn, + imageVersion: '3.0', + executionRoleArn: 'arn:aws:iam::123456789012:role/microvm-runner', + metadataSsmPath, + runnerTokenSsmPath, + }, + environment: 'unit-test', + runHookPayload: createMicrovmRunHookPayload({ + imageArn: overrideImageArn, + imageVersion: '3.0', + runnerConfigSsmPath, + runnerTokenSsmPath, + }), + runnerOwner: 'Codertocat', + runnerType: 'Org', + ssmParameterStoreTags: microvmMetadataTags, + source: 'scale-up-lambda', + }); + expect(setMicrovmGithubRunnerMetadata).toHaveBeenCalledWith( + { + ...providerConfig, + egressNetworkConnectors: [overrideEgressConnectorArn], + imageIdentifier: overrideImageArn, + imageVersion: '3.0', + }, + 'mvm-1', + { githubRunnerId: 'github-mvm-1', runnerLabels: [] }, + canonicalMetadataTags, + ); + }); + + it('retries a JIT setup failure even when runner cleanup fails', async () => { + createStartRunnerConfig.mockResolvedValue(['mvm-1']); + vi.mocked(terminateMicrovm).mockRejectedValue(new Error('cleanup failed')); + + await expect( + createMicrovmRunners(runnerConfig(), 1, githubClient, createStartRunnerConfig, 'scale-up-lambda'), + ).resolves.toEqual({ instances: [], retryableErrorCount: 1, nonRetryableErrorCount: 0 }); + + expect(terminateMicrovm).toHaveBeenCalledWith('mvm-1', providerConfig); + }); + + it.each([ + [true, { instances: [], retryableErrorCount: 1, nonRetryableErrorCount: 0 }], + [false, { instances: [], retryableErrorCount: 0, nonRetryableErrorCount: 1 }], + ])('classifies launch failures with retryable=%s', async (retryable, expected) => { + vi.mocked(runMicrovmRunner).mockRejectedValue(new Error('launch failed')); + vi.mocked(isRetryableMicrovmError).mockReturnValue(retryable); + + await expect( + createMicrovmRunners(runnerConfig(), 1, githubClient, createStartRunnerConfig, 'scale-up-lambda'), + ).resolves.toEqual(expected); + }); + + it('attempts cleanup when setup throws after launch', async () => { + createStartRunnerConfig.mockRejectedValue(new Error('JIT setup failed')); + vi.mocked(terminateMicrovm).mockRejectedValue(new Error('cleanup failed')); + + await expect( + createMicrovmRunners(runnerConfig(), 1, githubClient, createStartRunnerConfig, 'scale-up-lambda'), + ).resolves.toEqual({ instances: [], retryableErrorCount: 0, nonRetryableErrorCount: 1 }); + + expect(terminateMicrovm).toHaveBeenCalledWith('mvm-1', providerConfig); + }); +}); diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.ts new file mode 100644 index 0000000000..9d7154707b --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.ts @@ -0,0 +1,208 @@ +import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; +import type { Octokit } from '@octokit/rest'; + +import type { + CreateGitHubRunnerConfig, + CreateRunnerResult, + CreateStartRunnerConfig, + RunnerSource, +} from '../../../../core'; +import type { MicrovmDynamicLabelOverrides } from '../dynamic-labels'; +import { loadMicrovmProviderConfig } from './config'; +import { isRetryableMicrovmError, runMicrovmRunner, terminateMicrovm } from './microvms'; +import { + assertMatchingMicrovmRunnerTokenPath, + assertSeparatedMicrovmMetadataPath, + normalizeMicrovmSsmPath, + type MicrovmMetadataTag, + setMicrovmGithubRunnerMetadata, +} from './runner-metadata'; + +const logger = createChildLogger('microvm-runner-config'); +const MICROVM_METADATA_CONTEXT_TAG_KEYS = new Set([ + 'Name', + 'ghr:environment', + 'ghr:runner_name_prefix', + 'ghr:ssm_config_path', +]); + +export interface MicrovmRunHookPayloadV1 { + imageArn?: string; + imageVersion?: string; + runnerConfigSsmPath: string; + runnerTokenSsmPath: string; + version: 1; +} + +export function createMicrovmRunHookPayload(payload: Omit): string { + const hasImageArn = payload.imageArn !== undefined; + const hasImageVersion = payload.imageVersion !== undefined; + if (hasImageArn !== hasImageVersion) { + throw new Error('MicroVM hook payload image ARN and version must be provided together'); + } + + return JSON.stringify({ + version: 1, + ...(hasImageArn + ? { + imageArn: payload.imageArn, + imageVersion: payload.imageVersion, + } + : {}), + runnerConfigSsmPath: payload.runnerConfigSsmPath, + runnerTokenSsmPath: payload.runnerTokenSsmPath, + } satisfies MicrovmRunHookPayloadV1); +} + +function createMicrovmMetadataTags( + config: CreateGitHubRunnerConfig, + environment: string, + ssmConfigPath: string, + ssmParameterStoreTags: MicrovmMetadataTag[], +): MicrovmMetadataTag[] { + return [ + ...ssmParameterStoreTags.filter((tag) => !MICROVM_METADATA_CONTEXT_TAG_KEYS.has(tag.Key)), + { Key: 'ghr:environment', Value: environment }, + { Key: 'ghr:runner_name_prefix', Value: config.runnerNamePrefix }, + { Key: 'ghr:ssm_config_path', Value: ssmConfigPath }, + ]; +} + +function loadSsmParameterStoreTags(): MicrovmMetadataTag[] { + const encodedTags = process.env.SSM_PARAMETER_STORE_TAGS; + if (encodedTags === undefined || encodedTags.trim() === '') { + return []; + } + + try { + const parsed: unknown = JSON.parse(encodedTags); + if (!Array.isArray(parsed)) { + throw new Error('tags must be an array'); + } + + return parsed.map((tag, index) => { + if ( + tag === null || + typeof tag !== 'object' || + typeof (tag as Record).Key !== 'string' || + typeof (tag as Record).Value !== 'string' + ) { + throw new Error(`tag at index ${index} is invalid`); + } + const candidate = tag as Record; + return { Key: candidate.Key as string, Value: candidate.Value as string }; + }); + } catch (error) { + throw new Error(`Failed to parse SSM_PARAMETER_STORE_TAGS: ${(error as Error).message}`); + } +} + +export async function createMicrovmRunners( + githubRunnerConfig: CreateGitHubRunnerConfig, + numberOfRunners: number, + githubInstallationClient: Octokit, + createStartRunnerConfig: CreateStartRunnerConfig, + source: RunnerSource, + overrides: MicrovmDynamicLabelOverrides = {}, +): Promise { + if (!githubRunnerConfig.ephemeral || !githubRunnerConfig.enableJitConfig) { + logger.error('Lambda MicroVM runners require ephemeral runners with JIT configuration enabled'); + return { instances: [], retryableErrorCount: 0, nonRetryableErrorCount: numberOfRunners }; + } + + if (!process.env.SSM_TOKEN_PATH?.trim()) { + logger.error('Lambda MicroVM runners require SSM_TOKEN_PATH to deliver JIT configuration'); + return { instances: [], retryableErrorCount: 0, nonRetryableErrorCount: numberOfRunners }; + } + if (!process.env.SSM_CONFIG_PATH?.trim()) { + logger.error('Lambda MicroVM runners require SSM_CONFIG_PATH to locate runner metadata'); + return { instances: [], retryableErrorCount: 0, nonRetryableErrorCount: numberOfRunners }; + } + let config; + let normalizedRunnerConfigPath: string; + let normalizedRunnerTokenPath: string; + let ssmParameterStoreTags: MicrovmMetadataTag[]; + try { + config = { ...loadMicrovmProviderConfig(), ...overrides }; + normalizedRunnerConfigPath = normalizeMicrovmSsmPath(process.env.SSM_CONFIG_PATH); + normalizedRunnerTokenPath = normalizeMicrovmSsmPath(process.env.SSM_TOKEN_PATH); + assertMatchingMicrovmRunnerTokenPath(config.runnerTokenSsmPath, normalizedRunnerTokenPath); + assertSeparatedMicrovmMetadataPath(config.metadataSsmPath, config.runnerTokenSsmPath); + ssmParameterStoreTags = loadSsmParameterStoreTags(); + } catch (error) { + logger.error('Invalid Lambda MicroVM provider configuration', { error }); + return { instances: [], retryableErrorCount: 0, nonRetryableErrorCount: numberOfRunners }; + } + + const result: CreateRunnerResult = { + instances: [], + retryableErrorCount: 0, + nonRetryableErrorCount: 0, + }; + const runHookPayload = createMicrovmRunHookPayload({ + ...(config.imageVersion !== undefined + ? { + imageArn: config.imageIdentifier, + imageVersion: config.imageVersion, + } + : {}), + runnerConfigSsmPath: normalizedRunnerConfigPath, + runnerTokenSsmPath: normalizedRunnerTokenPath, + }); + const environment = process.env.ENVIRONMENT; + const metadataTags = createMicrovmMetadataTags( + githubRunnerConfig, + environment, + normalizedRunnerConfigPath, + ssmParameterStoreTags, + ); + + for (let runnerIndex = 0; runnerIndex < numberOfRunners; runnerIndex++) { + let microvmId: string | undefined; + try { + const runner = await runMicrovmRunner({ + config, + environment, + runHookPayload, + runnerOwner: githubRunnerConfig.runnerOwner, + runnerType: githubRunnerConfig.runnerType, + ssmParameterStoreTags: metadataTags, + source, + }); + microvmId = runner.microvmId; + + const failedRunnerIds = await createStartRunnerConfig(githubRunnerConfig, [microvmId], githubInstallationClient, { + getRunnerConfigMetadata: (runnerId) => [{ key: 'MicrovmId', value: runnerId }], + onJitConfigCreated: async (runnerId, metadata) => { + await setMicrovmGithubRunnerMetadata(config, runnerId, metadata, runner.metadataTags); + }, + }); + + if (failedRunnerIds.includes(microvmId)) { + await terminateMicrovm(microvmId, config).catch((terminationError) => { + logger.error(`Failed to terminate MicroVM runner '${microvmId}' after JIT configuration failed`, { + error: terminationError, + }); + }); + result.retryableErrorCount++; + } else { + result.instances.push(microvmId); + } + } catch (error) { + if (microvmId) { + await terminateMicrovm(microvmId, config).catch((terminationError) => { + logger.error(`Failed to terminate MicroVM runner '${microvmId}' after setup failed`, { + error: terminationError, + }); + }); + } + + const retryable = isRetryableMicrovmError(error); + logger.error('Failed to create Lambda MicroVM runner', { error, retryable }); + if (retryable) result.retryableErrorCount++; + else result.nonRetryableErrorCount++; + } + } + + return result; +} diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.test.ts new file mode 100644 index 0000000000..502fc77d05 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.test.ts @@ -0,0 +1,631 @@ +import { + addParameterTags, + deleteParameter, + getParameters, + getParametersByPath, + putParameter, +} from '@aws-github-runner/aws-ssm-util'; +import type { MicrovmState } from '@aws-sdk/client-lambda-microvms'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { + assertMatchingMicrovmRunnerTokenPath, + assertSeparatedMicrovmMetadataPath, + createMicrovmRunnerMetadata, + deleteMicrovmRunnerJitConfig, + deleteMicrovmRunnerSsmState, + listMicrovmRunnerMetadata, + markMicrovmCleanupPending, + microvmMetadataParameterName, + microvmRunnerJitParameterName, + setMicrovmGithubRunnerMetadata, + setMicrovmOrphan, + type MicrovmRunnerMetadata, +} from './runner-metadata'; + +vi.mock('@aws-github-runner/aws-ssm-util', () => ({ + addParameterTags: vi.fn(), + deleteParameter: vi.fn(), + getParameters: vi.fn(), + getParametersByPath: vi.fn(), + putParameter: vi.fn(), +})); + +const metadataSsmPath = '/github-action-runners/unit-test/microvm-metadata'; +const runnerTokenSsmPath = '/github-action-runners/unit-test/token'; +const ssmPaths = { metadataSsmPath, runnerTokenSsmPath }; +const launchTags = [ + { Key: 'CostCenter', Value: '1234' }, + { Key: 'ghr:Application', Value: 'github-action-runner' }, + { Key: 'ghr:microvm_id', Value: 'mvm-1' }, +]; + +function metadata(overrides: Partial = {}): MicrovmRunnerMetadata { + return { + version: 1, + microvmId: 'mvm-1', + environment: 'unit-test', + runnerOwner: 'Codertocat', + runnerType: 'Org', + source: 'scale-up-lambda', + imageArn: 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner', + imageVersion: '3.0', + createdAt: '2026-08-19T10:00:00.000Z', + expiresAt: '2026-08-19T11:00:00.000Z', + ...overrides, + }; +} + +function states(entries: [string, MicrovmState][]): Map { + return new Map(entries); +} + +beforeEach(() => { + vi.clearAllMocks(); + vi.useRealTimers(); + vi.mocked(deleteParameter).mockResolvedValue(); + vi.mocked(addParameterTags).mockResolvedValue(); + vi.mocked(getParameters).mockImplementation(async (names) => new Map([[names[0], '{}']])); + vi.mocked(getParametersByPath).mockResolvedValue(new Map()); + vi.mocked(putParameter).mockResolvedValue(); +}); + +describe('MicroVM metadata paths', () => { + it('uses one base parameter per validated MicroVM ID', () => { + expect(microvmMetadataParameterName(`${metadataSsmPath}/`, 'microvm-123')).toBe(`${metadataSsmPath}/microvm-123`); + expect(() => microvmMetadataParameterName(metadataSsmPath, '../other')).toThrow('Invalid MicroVM identifier'); + expect(microvmRunnerJitParameterName(`${runnerTokenSsmPath}/`, 'microvm-123')).toBe( + `${runnerTokenSsmPath}/microvm-123`, + ); + expect(() => microvmRunnerJitParameterName(runnerTokenSsmPath, '../other')).toThrow('Invalid MicroVM identifier'); + }); + + it('requires metadata to use a prefix separate from JIT configuration', () => { + expect(() => + assertSeparatedMicrovmMetadataPath(metadataSsmPath, '/github-action-runners/unit-test/token'), + ).not.toThrow(); + expect(() => assertSeparatedMicrovmMetadataPath('/runner/token/metadata', '/runner/token')).toThrow( + 'must be separate', + ); + expect(() => assertSeparatedMicrovmMetadataPath('/runner', '/runner/token')).toThrow('must be separate'); + expect(() => assertMatchingMicrovmRunnerTokenPath(`${runnerTokenSsmPath}/`, runnerTokenSsmPath)).not.toThrow(); + expect(() => assertMatchingMicrovmRunnerTokenPath('/runner/other-token', runnerTokenSsmPath)).toThrow( + 'must match the runner JIT token path', + ); + }); +}); + +describe('MicroVM metadata lifecycle', () => { + it('creates non-secret, expiring ownership metadata without overwrite', async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-08-19T10:00:00.000Z')); + + const createdTags = await createMicrovmRunnerMetadata(metadataSsmPath, { + microvmId: 'mvm-1', + environment: 'unit-test', + runnerOwner: 'Codertocat', + runnerType: 'Org', + source: 'scale-up-lambda', + imageArn: 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner', + imageVersion: '3.0', + ssmParameterStoreTags: [ + { Key: 'CostCenter', Value: '1234' }, + { Key: 'Name', Value: 'not-used-for-microvm-metadata' }, + { Key: 'ghr:Owner', Value: 'configured-owner-cannot-win' }, + { Key: 'ghr:created_by', Value: 'configured-source-cannot-win' }, + { Key: 'ghr:environment', Value: 'unit-test' }, + { Key: 'ghr:runner_name_prefix', Value: 'unit-test-' }, + { Key: 'ghr:ssm_config_path', Value: '/github-action-runners/unit-test/config' }, + { Key: 'ghr:github_runner_id', Value: 'configured-id-is-not-launch-metadata' }, + { Key: 'ghr:runner_labels', Value: 'configured-labels-are-not-launch-metadata' }, + ], + }); + + expect(putParameter).toHaveBeenCalledWith( + `${metadataSsmPath}/mvm-1`, + JSON.stringify(metadata({ expiresAt: '2026-08-19T18:05:00.000Z' })), + false, + { + tags: [ + { Key: 'CostCenter', Value: '1234' }, + { Key: 'ghr:Owner', Value: 'Codertocat' }, + { Key: 'ghr:created_by', Value: 'scale-up-lambda' }, + { Key: 'ghr:environment', Value: 'unit-test' }, + { Key: 'ghr:runner_name_prefix', Value: 'unit-test-' }, + { Key: 'ghr:ssm_config_path', Value: '/github-action-runners/unit-test/config' }, + { Key: 'ghr:Application', Value: 'github-action-runner' }, + { Key: 'ghr:Type', Value: 'Org' }, + { Key: 'ghr:microvm_id', Value: 'mvm-1' }, + { + Key: 'ghr:microvm_image_arn', + Value: 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner', + }, + { Key: 'ghr:microvm_image_version', Value: '3.0' }, + ], + }, + ); + expect(createdTags).toEqual(vi.mocked(putParameter).mock.calls[0][3]?.tags); + }); + + it('rejects reserved tag keys and preserves room for late GitHub metadata', async () => { + const input = { + microvmId: 'mvm-1', + environment: 'unit-test', + runnerOwner: 'Codertocat', + runnerType: 'Org' as const, + source: 'scale-up-lambda' as const, + imageArn: 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner', + imageVersion: '3.0', + ssmParameterStoreTags: [], + }; + + await expect( + createMicrovmRunnerMetadata(metadataSsmPath, { + ...input, + ssmParameterStoreTags: [{ Key: 'aws:microvm:image-arn', Value: input.imageArn }], + }), + ).rejects.toThrow('AWS-reserved tag prefix'); + + await expect( + createMicrovmRunnerMetadata(metadataSsmPath, { + ...input, + ssmParameterStoreTags: Array.from({ length: 37 }, (_, index) => ({ + Key: `Custom${index}`, + Value: 'value', + })), + }), + ).rejects.toThrow('cannot have more than 44 launch tags'); + expect(putParameter).not.toHaveBeenCalled(); + }); + + it('rejects launch tags whose complete serialized metadata could exceed the Parameter Store value limit', async () => { + await expect( + createMicrovmRunnerMetadata(metadataSsmPath, { + microvmId: 'mvm-1', + environment: 'unit-test', + runnerOwner: 'Codertocat', + runnerType: 'Org', + source: 'scale-up-lambda', + imageArn: 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner', + imageVersion: '3.0', + ssmParameterStoreTags: Array.from({ length: 20 }, (_, index) => ({ + Key: `Custom${index}${'k'.repeat(100)}`, + Value: 'v'.repeat(256), + })), + }), + ).rejects.toThrow('cannot exceed 8192 bytes when serialized'); + expect(putParameter).not.toHaveBeenCalled(); + }); + + it('loads active metadata and schedules expired or invalid inactive records for two-phase cleanup', async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-08-19T12:00:00.000Z')); + const active = metadata({ expiresAt: '2026-08-19T12:30:00.000Z' }); + const expiredInactive = metadata({ microvmId: 'mvm-old', expiresAt: '2026-08-19T11:00:00.000Z' }); + const unexpiredInactive = metadata({ microvmId: 'mvm-new', expiresAt: '2026-08-19T12:30:00.000Z' }); + vi.mocked(getParametersByPath).mockResolvedValue( + new Map([ + [`${metadataSsmPath}/mvm-1`, JSON.stringify(active)], + [`${metadataSsmPath}/mvm-1.github-runner-id`, 'github-42'], + [`${metadataSsmPath}/mvm-1.orphan`, 'true'], + [`${metadataSsmPath}/mvm-old`, JSON.stringify(expiredInactive)], + [`${metadataSsmPath}/mvm-new`, JSON.stringify(unexpiredInactive)], + [`${metadataSsmPath}/mvm-invalid`, '{not-json'], + ]), + ); + + await expect(listMicrovmRunnerMetadata(ssmPaths, states([['mvm-1', 'RUNNING']]))).resolves.toEqual({ + cleanupMicrovmIds: ['mvm-old', 'mvm-invalid'], + metadataById: new Map([['mvm-1', { ...active, githubRunnerId: 'github-42', orphan: true }]]), + }); + expect(getParametersByPath).toHaveBeenCalledWith(metadataSsmPath); + expect(deleteParameter).not.toHaveBeenCalled(); + expect(deleteParameter).not.toHaveBeenCalledWith(`${metadataSsmPath}/mvm-new`); + }); + + it('fails closed for invalid ownership metadata belonging to an active MicroVM', async () => { + vi.mocked(getParametersByPath).mockResolvedValue(new Map([[`${metadataSsmPath}/mvm-1`, '{not-json']])); + await expect(listMicrovmRunnerMetadata(ssmPaths, states([['mvm-1', 'RUNNING']]))).rejects.toThrow( + 'invalid ownership metadata', + ); + }); + + it('schedules provider-owned metadata with invalid orphan state for two-phase cleanup', async () => { + vi.mocked(getParametersByPath).mockResolvedValue( + new Map([ + [`${metadataSsmPath}/mvm-1`, JSON.stringify(metadata())], + [`${metadataSsmPath}/mvm-1.orphan`, 'invalid'], + ]), + ); + await expect(listMicrovmRunnerMetadata(ssmPaths, states([['mvm-1', 'RUNNING']]))).resolves.toEqual({ + cleanupMicrovmIds: ['mvm-1'], + metadataById: new Map(), + }); + }); + + it('propagates metadata path lookup errors so inventory fails closed', async () => { + vi.mocked(getParametersByPath).mockRejectedValue(new Error('AccessDenied')); + + await expect(listMicrovmRunnerMetadata(ssmPaths, states([['mvm-1', 'RUNNING']]))).rejects.toThrow('AccessDenied'); + }); + + it('updates GitHub state and adds late GitHub metadata tags to the base parameter', async () => { + const runnerLabels = ['self-hosted', 'linux', 'env:unit-test']; + await setMicrovmGithubRunnerMetadata( + ssmPaths, + 'mvm-1', + { + githubRunnerId: 'github-42', + runnerLabels, + }, + launchTags, + ); + expect(putParameter).toHaveBeenCalledWith(`${metadataSsmPath}/mvm-1.github-runner-id`, 'github-42', false, { + overwrite: true, + }); + expect(putParameter).toHaveBeenCalledWith(`${metadataSsmPath}/mvm-1.tags`, expect.any(String), false, { + overwrite: true, + }); + const tagsValue = vi.mocked(putParameter).mock.calls.find(([name]) => name.endsWith('.tags'))?.[1]; + expect(JSON.parse(tagsValue ?? '{}')).toEqual({ + CostCenter: '1234', + 'ghr:Application': 'github-action-runner', + 'ghr:github_runner_id': 'github-42', + 'ghr:microvm_id': 'mvm-1', + 'ghr:runner_labels': `base64url:${Buffer.from(JSON.stringify(runnerLabels), 'utf8').toString('base64url')}`, + }); + expect(addParameterTags).toHaveBeenCalledWith(`${metadataSsmPath}/mvm-1`, [ + { Key: 'ghr:github_runner_id', Value: 'github-42' }, + { + Key: 'ghr:runner_labels', + Value: `base64url:${Buffer.from(JSON.stringify(runnerLabels), 'utf8').toString('base64url')}`, + }, + ]); + }); + + it('revokes JIT configuration when cleanup starts before late metadata is recorded', async () => { + vi.mocked(getParameters).mockResolvedValue( + new Map([ + [`${metadataSsmPath}/mvm-1`, '{}'], + [`${metadataSsmPath}/mvm-1.cleanup-requested-at`, '2026-08-19T12:00:00.000Z'], + ]), + ); + + await expect( + setMicrovmGithubRunnerMetadata(ssmPaths, 'mvm-1', { githubRunnerId: 'github-42', runnerLabels: [] }, launchTags), + ).rejects.toThrow('no longer accepting JIT configuration'); + expect(deleteParameter).toHaveBeenCalledWith(`${runnerTokenSsmPath}/mvm-1`); + expect(putParameter).not.toHaveBeenCalled(); + }); + + it('revokes JIT configuration when ownership metadata is already absent', async () => { + vi.mocked(getParameters).mockResolvedValue(new Map()); + + await expect( + setMicrovmGithubRunnerMetadata(ssmPaths, 'mvm-1', { githubRunnerId: 'github-42', runnerLabels: [] }, launchTags), + ).rejects.toThrow('no longer accepting JIT configuration'); + expect(deleteParameter).toHaveBeenCalledWith(`${runnerTokenSsmPath}/mvm-1`); + expect(putParameter).not.toHaveBeenCalled(); + }); + + it('revokes JIT configuration when the post-write ownership fence cannot be read', async () => { + vi.mocked(getParameters).mockRejectedValue(new Error('AccessDenied')); + + await expect( + setMicrovmGithubRunnerMetadata(ssmPaths, 'mvm-1', { githubRunnerId: 'github-42', runnerLabels: [] }, launchTags), + ).rejects.toThrow('AccessDenied'); + expect(deleteParameter).toHaveBeenCalledWith(`${runnerTokenSsmPath}/mvm-1`); + expect(putParameter).not.toHaveBeenCalled(); + }); + + it('splits encoded runner labels into SSM-safe tag values', async () => { + const runnerLabels = [`label-${'a'.repeat(140)}`, `label-${'b'.repeat(140)}`]; + + await setMicrovmGithubRunnerMetadata( + ssmPaths, + 'mvm-1', + { + githubRunnerId: 'github-42', + runnerLabels, + }, + launchTags, + ); + + expect(addParameterTags).toHaveBeenCalledWith(`${metadataSsmPath}/mvm-1`, [ + { Key: 'ghr:github_runner_id', Value: 'github-42' }, + { + Key: 'ghr:runner_labels', + Value: `base64url:${Buffer.from(JSON.stringify([runnerLabels[0]]), 'utf8').toString('base64url')}`, + }, + { + Key: 'ghr:runner_labels:2', + Value: `base64url:${Buffer.from(JSON.stringify([runnerLabels[1]]), 'utf8').toString('base64url')}`, + }, + ]); + }); + + it('keeps the GitHub runner ID tag when a runner label is too large', async () => { + await setMicrovmGithubRunnerMetadata( + ssmPaths, + 'mvm-1', + { + githubRunnerId: 'github-42', + runnerLabels: ['x'.repeat(300)], + }, + launchTags, + ); + + expect(addParameterTags).toHaveBeenCalledWith(`${metadataSsmPath}/mvm-1`, [ + { Key: 'ghr:github_runner_id', Value: 'github-42' }, + ]); + }); + + it('keeps the durable GitHub runner ID when late metadata tagging fails', async () => { + vi.mocked(addParameterTags).mockRejectedValue(new Error('AccessDenied')); + + await expect( + setMicrovmGithubRunnerMetadata( + ssmPaths, + 'mvm-1', + { + githubRunnerId: 'github-42', + runnerLabels: [], + }, + launchTags, + ), + ).resolves.toBeUndefined(); + expect(putParameter).toHaveBeenCalledWith(`${metadataSsmPath}/mvm-1.github-runner-id`, 'github-42', false, { + overwrite: true, + }); + }); + + it('fails JIT setup when the canonical tag-value parameter cannot be written', async () => { + vi.mocked(putParameter).mockImplementation(async (name) => { + if (name.endsWith('.tags')) throw new Error('AccessDenied'); + }); + + await expect( + setMicrovmGithubRunnerMetadata(ssmPaths, 'mvm-1', { githubRunnerId: 'github-42', runnerLabels: [] }, launchTags), + ).rejects.toThrow('AccessDenied'); + expect(addParameterTags).not.toHaveBeenCalled(); + }); + + it('updates orphan state without a shared read-modify-write record', async () => { + await setMicrovmOrphan(metadataSsmPath, 'mvm-1', true); + expect(putParameter).toHaveBeenLastCalledWith(`${metadataSsmPath}/mvm-1.orphan`, 'true', false, { + overwrite: true, + }); + }); + + it('marks cleanup independently and deletes JIT plus metadata while retaining the tombstone until last', async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-08-19T12:00:00.000Z')); + + await markMicrovmCleanupPending(metadataSsmPath, 'mvm-1'); + expect(putParameter).toHaveBeenCalledWith( + `${metadataSsmPath}/mvm-1.cleanup-requested-at`, + '2026-08-19T12:00:00.000Z', + false, + ); + + await deleteMicrovmRunnerSsmState(ssmPaths, 'mvm-1'); + expect(vi.mocked(deleteParameter).mock.calls.map(([name]) => name)).toEqual([ + `${runnerTokenSsmPath}/mvm-1`, + `${metadataSsmPath}/mvm-1.github-runner-id`, + `${metadataSsmPath}/mvm-1.orphan`, + `${metadataSsmPath}/mvm-1.tags`, + `${metadataSsmPath}/mvm-1`, + `${metadataSsmPath}/mvm-1.cleanup-requested-at`, + ]); + }); + + it('does not reset the cleanup grace window when its tombstone already exists', async () => { + vi.mocked(putParameter).mockRejectedValueOnce( + Object.assign(new Error('ParameterAlreadyExists'), { __type: 'ParameterAlreadyExists' }), + ); + + await expect(markMicrovmCleanupPending(metadataSsmPath, 'mvm-1')).resolves.toBeUndefined(); + expect(putParameter).toHaveBeenCalledOnce(); + }); + + it('continues deleting metadata when optional parameters are already absent', async () => { + vi.mocked(deleteParameter) + .mockRejectedValueOnce( + Object.assign(new Error('ParameterNotFound'), { + __type: 'ParameterNotFound', + $fault: 'client', + $metadata: { httpStatusCode: 400 }, + }), + ) + .mockRejectedValueOnce(Object.assign(new Error('missing parameter'), { name: 'ParameterNotFound' })); + + await expect(deleteMicrovmRunnerSsmState(ssmPaths, 'mvm-1')).resolves.toBeUndefined(); + expect(vi.mocked(deleteParameter).mock.calls.map(([name]) => name)).toEqual([ + `${runnerTokenSsmPath}/mvm-1`, + `${metadataSsmPath}/mvm-1.github-runner-id`, + `${metadataSsmPath}/mvm-1.orphan`, + `${metadataSsmPath}/mvm-1.tags`, + `${metadataSsmPath}/mvm-1`, + `${metadataSsmPath}/mvm-1.cleanup-requested-at`, + ]); + }); + + it('propagates metadata deletion failures other than missing parameters', async () => { + const error = Object.assign(new Error('AccessDeniedException'), { + __type: 'AccessDeniedException', + $fault: 'client', + $metadata: { httpStatusCode: 400 }, + }); + vi.mocked(deleteParameter).mockRejectedValueOnce(error); + + await expect(deleteMicrovmRunnerSsmState(ssmPaths, 'mvm-1')).rejects.toBe(error); + expect(deleteParameter).toHaveBeenCalledTimes(1); + }); + + it('deletes only the lane JIT parameter when revoking pending runner configuration', async () => { + await deleteMicrovmRunnerJitConfig(runnerTokenSsmPath, 'mvm-1'); + + expect(deleteParameter).toHaveBeenCalledOnce(); + expect(deleteParameter).toHaveBeenCalledWith(`${runnerTokenSsmPath}/mvm-1`); + }); + + it('returns tracked and state-only active cleanup requests for termination retry', async () => { + vi.mocked(getParametersByPath).mockResolvedValue( + new Map([ + [`${metadataSsmPath}/mvm-1`, JSON.stringify(metadata())], + [`${metadataSsmPath}/mvm-1.github-runner-id`, 'github-42'], + [`${metadataSsmPath}/mvm-1.cleanup-requested-at`, '2026-08-19T10:15:00.000Z'], + [`${metadataSsmPath}/mvm-untracked.cleanup-requested-at`, '2026-08-19T10:15:00.000Z'], + [`${metadataSsmPath}/mvm-terminating.cleanup-requested-at`, '2026-08-19T10:15:00.000Z'], + ]), + ); + + await expect( + listMicrovmRunnerMetadata( + ssmPaths, + states([ + ['mvm-1', 'RUNNING'], + ['mvm-untracked', 'PENDING'], + ['mvm-terminating', 'TERMINATING'], + ]), + ), + ).resolves.toEqual({ + cleanupMicrovmIds: ['mvm-1', 'mvm-untracked'], + metadataById: new Map(), + }); + expect(deleteParameter).not.toHaveBeenCalled(); + }); + + it('does not starve cleanup requests when more than one reconciliation batch is pending', async () => { + const cleanupIds = Array.from({ length: 11 }, (_, index) => `mvm-cleanup-${index}`); + vi.mocked(getParametersByPath).mockResolvedValue( + new Map( + cleanupIds.map((microvmId) => [ + `${metadataSsmPath}/${microvmId}.cleanup-requested-at`, + '2026-08-19T10:15:00.000Z', + ]), + ), + ); + + await expect( + listMicrovmRunnerMetadata( + ssmPaths, + states(cleanupIds.map((microvmId): [string, MicrovmState] => [microvmId, 'RUNNING'])), + ), + ).resolves.toEqual({ cleanupMicrovmIds: cleanupIds, metadataById: new Map() }); + }); + + it('keeps cleanup discoverable through the grace window before deleting JIT and every metadata record', async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-08-19T12:00:00.000Z')); + vi.mocked(getParametersByPath).mockResolvedValue( + new Map([ + [`${metadataSsmPath}/mvm-terminal.github-runner-id`, 'github-42'], + [`${metadataSsmPath}/mvm-missing.cleanup-requested-at`, '2026-08-19T11:54:59.000Z'], + [`${metadataSsmPath}/mvm-missing.tags`, '{"ghr:microvm_id":"mvm-missing"}'], + [`${metadataSsmPath}/mvm-recent.cleanup-requested-at`, '2026-08-19T11:59:00.000Z'], + [`${metadataSsmPath}/mvm-recent.tags`, '{"ghr:microvm_id":"mvm-recent"}'], + ]), + ); + + await expect(listMicrovmRunnerMetadata(ssmPaths, states([['mvm-terminal', 'TERMINATED']]))).resolves.toEqual({ + cleanupMicrovmIds: ['mvm-terminal', 'mvm-recent'], + metadataById: new Map(), + }); + expect(deleteParameter).toHaveBeenCalledTimes(6); + expect(deleteParameter).toHaveBeenCalledWith(`${runnerTokenSsmPath}/mvm-missing`); + expect(deleteParameter).toHaveBeenCalledWith(`${metadataSsmPath}/mvm-missing`); + expect(deleteParameter).toHaveBeenCalledWith(`${metadataSsmPath}/mvm-missing.tags`); + expect(deleteParameter).toHaveBeenLastCalledWith(`${metadataSsmPath}/mvm-missing.cleanup-requested-at`); + expect(deleteParameter).not.toHaveBeenCalledWith(`${runnerTokenSsmPath}/mvm-terminal`); + expect(deleteParameter).not.toHaveBeenCalledWith(`${runnerTokenSsmPath}/mvm-recent`); + expect(deleteParameter).not.toHaveBeenCalledWith(`${metadataSsmPath}/mvm-recent`); + }); + + it('deletes invalid ownership metadata after its valid cleanup tombstone ages', async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-08-19T12:00:00.000Z')); + vi.mocked(getParametersByPath).mockResolvedValue( + new Map([ + [`${metadataSsmPath}/mvm-invalid`, '{not-json'], + [`${metadataSsmPath}/mvm-invalid.cleanup-requested-at`, '2026-08-19T11:54:59.000Z'], + ]), + ); + + await expect(listMicrovmRunnerMetadata(ssmPaths, new Map())).resolves.toEqual({ + cleanupMicrovmIds: [], + metadataById: new Map(), + }); + expect(vi.mocked(deleteParameter).mock.calls.map(([name]) => name)).toEqual([ + `${runnerTokenSsmPath}/mvm-invalid`, + `${metadataSsmPath}/mvm-invalid.github-runner-id`, + `${metadataSsmPath}/mvm-invalid.orphan`, + `${metadataSsmPath}/mvm-invalid.tags`, + `${metadataSsmPath}/mvm-invalid`, + `${metadataSsmPath}/mvm-invalid.cleanup-requested-at`, + ]); + }); + + it('repairs an invalid cleanup timestamp before recreating the two-phase cleanup marker', async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-08-19T12:00:00.000Z')); + vi.mocked(getParametersByPath).mockResolvedValue( + new Map([ + [`${metadataSsmPath}/mvm-1`, JSON.stringify(metadata())], + [`${metadataSsmPath}/mvm-1.cleanup-requested-at`, 'not-a-timestamp'], + ]), + ); + + await expect(listMicrovmRunnerMetadata(ssmPaths, states([['mvm-1', 'TERMINATED']]))).resolves.toEqual({ + cleanupMicrovmIds: ['mvm-1'], + metadataById: new Map(), + }); + expect(deleteParameter).toHaveBeenCalledOnce(); + expect(deleteParameter).toHaveBeenCalledWith(`${metadataSsmPath}/mvm-1.cleanup-requested-at`); + + await markMicrovmCleanupPending(metadataSsmPath, 'mvm-1'); + expect(putParameter).toHaveBeenCalledWith( + `${metadataSsmPath}/mvm-1.cleanup-requested-at`, + '2026-08-19T12:00:00.000Z', + false, + ); + + vi.clearAllMocks(); + vi.setSystemTime(new Date('2026-08-19T12:06:00.000Z')); + vi.mocked(deleteParameter).mockResolvedValue(); + vi.mocked(getParametersByPath).mockResolvedValue( + new Map([ + [`${metadataSsmPath}/mvm-1`, JSON.stringify(metadata())], + [`${metadataSsmPath}/mvm-1.cleanup-requested-at`, '2026-08-19T12:00:00.000Z'], + ]), + ); + + await expect(listMicrovmRunnerMetadata(ssmPaths, states([['mvm-1', 'TERMINATED']]))).resolves.toEqual({ + cleanupMicrovmIds: [], + metadataById: new Map(), + }); + expect(deleteParameter).toHaveBeenCalledTimes(6); + }); + + it('marks a terminal tags-only companion for two-phase cleanup instead of deleting it immediately', async () => { + vi.mocked(getParametersByPath).mockResolvedValue( + new Map([[`${metadataSsmPath}/mvm-tags-only.tags`, '{"ghr:microvm_id":"mvm-tags-only"}']]), + ); + + await expect(listMicrovmRunnerMetadata(ssmPaths, states([['mvm-tags-only', 'TERMINATED']]))).resolves.toEqual({ + cleanupMicrovmIds: ['mvm-tags-only'], + metadataById: new Map(), + }); + expect(deleteParameter).not.toHaveBeenCalled(); + }); + + it('fails closed for active state metadata without ownership or a cleanup request', async () => { + vi.mocked(getParametersByPath).mockResolvedValue( + new Map([[`${metadataSsmPath}/mvm-1.github-runner-id`, 'github-42']]), + ); + + await expect(listMicrovmRunnerMetadata(ssmPaths, states([['mvm-1', 'RUNNING']]))).rejects.toThrow( + 'state metadata but no ownership metadata', + ); + }); +}); diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.ts new file mode 100644 index 0000000000..0bae7a8d80 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.ts @@ -0,0 +1,594 @@ +import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; +import { + addParameterTags, + deleteParameter, + getParameters, + getParametersByPath, + putParameter, +} from '@aws-github-runner/aws-ssm-util'; +import type { MicrovmState } from '@aws-sdk/client-lambda-microvms'; + +import type { GitHubRunnerMetadata, RunnerSource, RunnerType } from '../../../../core'; +import { MICROVM_LIFETIME_IN_SECONDS } from './lifetime'; + +const logger = createChildLogger('microvm-runner-metadata'); + +const METADATA_VERSION = 1; +const EXPIRATION_GRACE_IN_SECONDS = 300; +const MAX_RECONCILED_RUNNERS = 10; +const MAX_PARAMETER_TAGS = 50; +const MAX_RUNNER_LABEL_TAGS = 5; +const MAX_BASE_PARAMETER_TAGS = MAX_PARAMETER_TAGS - MAX_RUNNER_LABEL_TAGS - 1; +const MAX_TAG_KEY_LENGTH = 128; +const MAX_TAG_VALUE_LENGTH = 256; +const MAX_PARAMETER_VALUE_SIZE_IN_BYTES = 8 * 1024; +const SSM_TAG_VALUE_PATTERN = /^[\p{L}\p{Z}\p{N}_.:/=+\-@]*$/u; +const MICROVM_ID_PATTERN = /^[A-Za-z0-9_-]+$/; +const GITHUB_RUNNER_ID_SUFFIX = '.github-runner-id'; +const ORPHAN_SUFFIX = '.orphan'; +const CLEANUP_REQUESTED_AT_SUFFIX = '.cleanup-requested-at'; +const TAGS_SUFFIX = '.tags'; +const METADATA_COMPANION_SUFFIXES = [ + GITHUB_RUNNER_ID_SUFFIX, + ORPHAN_SUFFIX, + CLEANUP_REQUESTED_AT_SUFFIX, + TAGS_SUFFIX, +] as const; +const ACTIVE_STATES = new Set(['PENDING', 'RUNNING', 'SUSPENDING', 'SUSPENDED']); +export interface MicrovmMetadataTag { + Key: string; + Value: string; +} + +export interface MicrovmSsmPaths { + metadataSsmPath: string; + runnerTokenSsmPath: string; +} + +export interface MicrovmRunnerMetadata { + bypassRemoval?: boolean; + createdAt: string; + environment: string; + expiresAt: string; + githubRunnerId?: string; + imageArn: string; + imageVersion?: string; + microvmId: string; + orphan?: boolean; + runnerOwner: string; + runnerType: RunnerType; + source: RunnerSource; + version: 1; +} + +export interface MicrovmRunnerMetadataInventory { + cleanupMicrovmIds: string[]; + metadataById: Map; +} + +export interface CreateMicrovmRunnerMetadataInput { + environment: string; + imageArn: string; + imageVersion?: string; + microvmId: string; + runnerOwner: string; + runnerType: RunnerType; + ssmParameterStoreTags: MicrovmMetadataTag[]; + source: RunnerSource; +} + +function isProviderOwnedLateTag(key: string): boolean { + return key === 'ghr:github_runner_id' || key === 'ghr:runner_labels' || key.startsWith('ghr:runner_labels:'); +} + +function assertValidParameterTags(tags: MicrovmMetadataTag[]): void { + if (tags.length > MAX_PARAMETER_TAGS) { + throw new Error(`MicroVM metadata cannot have more than ${MAX_PARAMETER_TAGS} tags`); + } + + for (const tag of tags) { + if ( + Array.from(tag.Key).length === 0 || + Array.from(tag.Key).length > MAX_TAG_KEY_LENGTH || + Array.from(tag.Value).length > MAX_TAG_VALUE_LENGTH || + !SSM_TAG_VALUE_PATTERN.test(tag.Key) || + !SSM_TAG_VALUE_PATTERN.test(tag.Value) + ) { + throw new Error(`MicroVM metadata tag '${tag.Key}' does not satisfy SSM tag constraints`); + } + if (tag.Key.toLowerCase().startsWith('aws:')) { + throw new Error(`MicroVM metadata tag '${tag.Key}' uses the AWS-reserved tag prefix`); + } + } +} + +function mergeParameterTags(...tagSets: MicrovmMetadataTag[][]): MicrovmMetadataTag[] { + const tagsByKey = new Map(); + for (const tags of tagSets) { + for (const tag of tags) tagsByKey.set(tag.Key, tag.Value); + } + + return [...tagsByKey].map(([Key, Value]) => ({ Key, Value })); +} + +function serializeParameterTags(tags: MicrovmMetadataTag[]): string { + assertValidParameterTags(tags); + const tagValues: Record = Object.create(null) as Record; + for (const { Key, Value } of [...tags].sort((left, right) => + left.Key < right.Key ? -1 : left.Key > right.Key ? 1 : 0, + )) { + tagValues[Key] = Value; + } + + const value = JSON.stringify(tagValues); + if (Buffer.byteLength(value, 'utf8') > MAX_PARAMETER_VALUE_SIZE_IN_BYTES) { + throw new Error(`MicroVM metadata tags cannot exceed ${MAX_PARAMETER_VALUE_SIZE_IN_BYTES} bytes when serialized`); + } + return value; +} + +function maximumGitHubRunnerMetadataTags(): MicrovmMetadataTag[] { + return [ + { Key: 'ghr:github_runner_id', Value: '0'.repeat(MAX_TAG_VALUE_LENGTH) }, + ...Array.from({ length: MAX_RUNNER_LABEL_TAGS }, (_, index) => ({ + Key: index === 0 ? 'ghr:runner_labels' : `ghr:runner_labels:${index + 1}`, + Value: '0'.repeat(MAX_TAG_VALUE_LENGTH), + })), + ]; +} + +function createMetadataParameterTags(input: CreateMicrovmRunnerMetadataInput): MicrovmMetadataTag[] { + const configuredTags = mergeParameterTags(input.ssmParameterStoreTags).filter( + (tag) => !isProviderOwnedLateTag(tag.Key) && tag.Key !== 'ghr:microvm_image_version' && tag.Key !== 'Name', + ); + const providerTags: MicrovmMetadataTag[] = [ + { Key: 'ghr:Application', Value: 'github-action-runner' }, + { Key: 'ghr:created_by', Value: input.source }, + { Key: 'ghr:environment', Value: input.environment }, + { Key: 'ghr:Owner', Value: input.runnerOwner }, + { Key: 'ghr:Type', Value: input.runnerType }, + { Key: 'ghr:microvm_id', Value: input.microvmId }, + { Key: 'ghr:microvm_image_arn', Value: input.imageArn }, + ]; + if (input.imageVersion !== undefined) { + providerTags.push({ Key: 'ghr:microvm_image_version', Value: input.imageVersion }); + } + + const tags = mergeParameterTags(configuredTags, providerTags); + assertValidParameterTags(tags); + if (tags.length > MAX_BASE_PARAMETER_TAGS) { + throw new Error( + `MicroVM metadata cannot have more than ${MAX_BASE_PARAMETER_TAGS} launch tags because ${MAX_RUNNER_LABEL_TAGS + 1} tags are reserved for GitHub runner metadata`, + ); + } + serializeParameterTags(mergeParameterTags(tags, maximumGitHubRunnerMetadataTags())); + return tags; +} + +export function assertValidMicrovmMetadataTags(input: CreateMicrovmRunnerMetadataInput): void { + createMetadataParameterTags(input); +} + +function encodeRunnerLabelGroups(labels: string[]): string[] { + const encodedGroups: string[] = []; + let group: string[] = []; + const encode = (values: string[]) => `base64url:${Buffer.from(JSON.stringify(values), 'utf8').toString('base64url')}`; + + for (const label of labels) { + const candidate = [...group, label]; + if (Array.from(encode(candidate)).length <= MAX_TAG_VALUE_LENGTH) { + group = candidate; + continue; + } + if (group.length === 0) { + logger.warn('A GitHub runner label was omitted because its encoded value exceeds the SSM tag limit', { + labelLength: Array.from(label).length, + }); + continue; + } + encodedGroups.push(encode(group)); + group = [label]; + if (Array.from(encode(group)).length > MAX_TAG_VALUE_LENGTH) { + logger.warn('A GitHub runner label was omitted because its encoded value exceeds the SSM tag limit', { + labelLength: Array.from(label).length, + }); + group = []; + } + } + if (group.length > 0) encodedGroups.push(encode(group)); + + if (encodedGroups.length > MAX_RUNNER_LABEL_TAGS) { + logger.warn('GitHub runner label SSM tags were truncated to avoid exceeding the metadata tag budget', { + maxRunnerLabelsTagCount: MAX_RUNNER_LABEL_TAGS, + }); + } + return encodedGroups.slice(0, MAX_RUNNER_LABEL_TAGS); +} + +function createGitHubRunnerMetadataTags(metadata: GitHubRunnerMetadata): MicrovmMetadataTag[] { + const tags: MicrovmMetadataTag[] = [{ Key: 'ghr:github_runner_id', Value: metadata.githubRunnerId }]; + tags.push( + ...encodeRunnerLabelGroups(metadata.runnerLabels).map((Value, index) => ({ + Key: index === 0 ? 'ghr:runner_labels' : `ghr:runner_labels:${index + 1}`, + Value, + })), + ); + assertValidParameterTags(tags); + return tags; +} + +export function normalizeMicrovmSsmPath(path: string): string { + const normalized = path.trim().replace(/\/+$/, ''); + if (!/^\/[A-Za-z0-9_.\-/]+$/.test(normalized) || normalized.includes('//') || normalized.split('/').includes('..')) { + throw new Error(`Invalid SSM parameter path '${path}'`); + } + return normalized; +} + +export function microvmMetadataParameterName(metadataSsmPath: string, microvmId: string): string { + if (!MICROVM_ID_PATTERN.test(microvmId)) { + throw new Error(`Invalid MicroVM identifier '${microvmId}'`); + } + return `${normalizeMicrovmSsmPath(metadataSsmPath)}/${microvmId}`; +} + +export function microvmRunnerJitParameterName(runnerTokenSsmPath: string, microvmId: string): string { + if (!MICROVM_ID_PATTERN.test(microvmId)) { + throw new Error(`Invalid MicroVM identifier '${microvmId}'`); + } + return `${normalizeMicrovmSsmPath(runnerTokenSsmPath)}/${microvmId}`; +} + +function stateParameterName(metadataSsmPath: string, microvmId: string, suffix: string): string { + return `${microvmMetadataParameterName(metadataSsmPath, microvmId)}${suffix}`; +} + +function metadataParameterNames(metadataSsmPath: string, microvmId: string): string[] { + const baseName = microvmMetadataParameterName(metadataSsmPath, microvmId); + return [ + `${baseName}${GITHUB_RUNNER_ID_SUFFIX}`, + `${baseName}${ORPHAN_SUFFIX}`, + `${baseName}${TAGS_SUFFIX}`, + baseName, + `${baseName}${CLEANUP_REQUESTED_AT_SUFFIX}`, + ]; +} + +export function assertSeparatedMicrovmMetadataPath(metadataSsmPath: string, runnerTokenSsmPath: string): void { + const metadataPath = normalizeMicrovmSsmPath(metadataSsmPath); + const runnerTokenPath = normalizeMicrovmSsmPath(runnerTokenSsmPath); + if ( + metadataPath === runnerTokenPath || + metadataPath.startsWith(`${runnerTokenPath}/`) || + runnerTokenPath.startsWith(`${metadataPath}/`) + ) { + throw new Error('MICROVM_METADATA_SSM_PATH must be separate from the runner JIT token path'); + } +} + +export function assertMatchingMicrovmRunnerTokenPath( + configuredRunnerTokenSsmPath: string, + runnerTokenSsmPath: string, +): void { + if (normalizeMicrovmSsmPath(configuredRunnerTokenSsmPath) !== normalizeMicrovmSsmPath(runnerTokenSsmPath)) { + throw new Error('MicroVM provider SSM_TOKEN_PATH must match the runner JIT token path'); + } +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function isParameterError(error: unknown, type: string): boolean { + return error instanceof Error && (error.name === type || ('__type' in error && error.__type === type)); +} + +function isParameterNotFound(error: unknown): boolean { + return isParameterError(error, 'ParameterNotFound'); +} + +function optionalString(value: unknown): value is string | undefined { + return value === undefined || (typeof value === 'string' && value.length > 0); +} + +function optionalBoolean(value: unknown): value is boolean | undefined { + return value === undefined || typeof value === 'boolean'; +} + +function parseMetadata(value: string, expectedMicrovmId: string): MicrovmRunnerMetadata | undefined { + let parsed: unknown; + try { + parsed = JSON.parse(value); + } catch { + return undefined; + } + + if (!isRecord(parsed)) return undefined; + + const createdAt = typeof parsed.createdAt === 'string' ? Date.parse(parsed.createdAt) : Number.NaN; + const expiresAt = typeof parsed.expiresAt === 'string' ? Date.parse(parsed.expiresAt) : Number.NaN; + if ( + parsed.version !== METADATA_VERSION || + parsed.microvmId !== expectedMicrovmId || + typeof parsed.environment !== 'string' || + parsed.environment.length === 0 || + typeof parsed.runnerOwner !== 'string' || + parsed.runnerOwner.length === 0 || + (parsed.runnerType !== 'Org' && parsed.runnerType !== 'Repo') || + (parsed.source !== 'scale-up-lambda' && parsed.source !== 'pool-lambda') || + typeof parsed.imageArn !== 'string' || + parsed.imageArn.length === 0 || + !optionalString(parsed.imageVersion) || + !optionalBoolean(parsed.bypassRemoval) || + !Number.isFinite(createdAt) || + !Number.isFinite(expiresAt) || + expiresAt <= createdAt + ) { + return undefined; + } + + return { + version: METADATA_VERSION, + microvmId: expectedMicrovmId, + environment: parsed.environment, + runnerOwner: parsed.runnerOwner, + runnerType: parsed.runnerType, + source: parsed.source, + imageArn: parsed.imageArn, + imageVersion: parsed.imageVersion, + bypassRemoval: parsed.bypassRemoval, + createdAt: parsed.createdAt as string, + expiresAt: parsed.expiresAt as string, + }; +} + +export async function createMicrovmRunnerMetadata( + metadataSsmPath: string, + input: CreateMicrovmRunnerMetadataInput, +): Promise { + const createdAt = new Date(); + const metadata: MicrovmRunnerMetadata = { + version: METADATA_VERSION, + microvmId: input.microvmId, + environment: input.environment, + runnerOwner: input.runnerOwner, + runnerType: input.runnerType, + source: input.source, + imageArn: input.imageArn, + imageVersion: input.imageVersion, + createdAt: createdAt.toISOString(), + expiresAt: new Date( + createdAt.getTime() + (MICROVM_LIFETIME_IN_SECONDS + EXPIRATION_GRACE_IN_SECONDS) * 1000, + ).toISOString(), + }; + + const metadataTags = createMetadataParameterTags(input); + await putParameter(microvmMetadataParameterName(metadataSsmPath, input.microvmId), JSON.stringify(metadata), false, { + tags: metadataTags, + }); + return metadataTags; +} + +function invalidOrphanState(parameters: Map, baseName: string): boolean { + const orphan = parameters.get(`${baseName}${ORPHAN_SUFFIX}`); + return orphan !== undefined && orphan !== 'true' && orphan !== 'false'; +} + +type CleanupRequestStatus = 'absent' | 'elapsed' | 'invalid' | 'pending'; + +function cleanupRequestStatus(parameters: Map, baseName: string, now: number): CleanupRequestStatus { + const cleanupRequestedAt = parameters.get(`${baseName}${CLEANUP_REQUESTED_AT_SUFFIX}`); + if (cleanupRequestedAt === undefined) return 'absent'; + const requestedAt = Date.parse(cleanupRequestedAt); + if (!Number.isFinite(requestedAt)) return 'invalid'; + return requestedAt + EXPIRATION_GRACE_IN_SECONDS * 1000 <= now ? 'elapsed' : 'pending'; +} + +export async function listMicrovmRunnerMetadata( + paths: MicrovmSsmPaths, + microvmStates: ReadonlyMap, +): Promise { + const { metadataSsmPath } = paths; + const metadataById = new Map(); + const cleanupMicrovmIds = new Set(); + const parameters = await getParametersByPath(normalizeMicrovmSsmPath(metadataSsmPath)); + const parameterPrefix = `${normalizeMicrovmSsmPath(metadataSsmPath)}/`; + const now = Date.now(); + const metadataBaseIds = new Set(); + const stateParameterIds = new Set(); + const runnersToDelete = new Set(); + + for (const parameterName of parameters.keys()) { + if (!parameterName.startsWith(parameterPrefix)) continue; + for (const suffix of METADATA_COMPANION_SUFFIXES) { + if (!parameterName.endsWith(suffix)) continue; + const microvmId = parameterName.slice(parameterPrefix.length, -suffix.length); + if (MICROVM_ID_PATTERN.test(microvmId)) stateParameterIds.add(microvmId); + break; + } + } + + for (const [parameterName, value] of parameters) { + if (!parameterName.startsWith(parameterPrefix)) continue; + const microvmId = parameterName.slice(parameterPrefix.length); + if (!MICROVM_ID_PATTERN.test(microvmId)) continue; + metadataBaseIds.add(microvmId); + + const state = microvmStates.get(microvmId); + const baseName = microvmMetadataParameterName(metadataSsmPath, microvmId); + const cleanupStatus = cleanupRequestStatus(parameters, baseName, now); + if (cleanupStatus === 'pending' || cleanupStatus === 'elapsed') { + if (cleanupStatus === 'elapsed' && (state === undefined || state === 'TERMINATED')) { + runnersToDelete.add(microvmId); + } else { + cleanupMicrovmIds.add(microvmId); + } + continue; + } + + const metadata = parseMetadata(value, microvmId); + if (!metadata) { + if (state !== undefined && ACTIVE_STATES.has(state)) { + throw new Error(`Active MicroVM runner '${microvmId}' has invalid ownership metadata`); + } + if (cleanupStatus === 'invalid') { + await deleteParameterIfPresent(`${baseName}${CLEANUP_REQUESTED_AT_SUFFIX}`); + } + cleanupMicrovmIds.add(microvmId); + logger.warn(`Scheduling invalid MicroVM runner metadata for '${microvmId}' for cleanup`); + continue; + } + + if (cleanupStatus === 'invalid') { + await deleteParameterIfPresent(`${baseName}${CLEANUP_REQUESTED_AT_SUFFIX}`); + cleanupMicrovmIds.add(microvmId); + logger.warn(`Repairing invalid cleanup request metadata for '${microvmId}'`); + continue; + } + + if (invalidOrphanState(parameters, baseName)) { + cleanupMicrovmIds.add(microvmId); + logger.warn(`Scheduling MicroVM runner metadata for '${microvmId}' with invalid orphan state for cleanup`); + continue; + } + + if (state === 'TERMINATED') { + cleanupMicrovmIds.add(microvmId); + continue; + } + if (state === undefined) { + if (Date.parse(metadata.expiresAt) <= now) cleanupMicrovmIds.add(microvmId); + continue; + } + if (!ACTIVE_STATES.has(state)) continue; + + metadataById.set(microvmId, { + ...metadata, + githubRunnerId: parameters.get(`${baseName}${GITHUB_RUNNER_ID_SUFFIX}`), + orphan: parameters.get(`${baseName}${ORPHAN_SUFFIX}`) === 'true', + }); + } + + for (const microvmId of stateParameterIds) { + if (metadataBaseIds.has(microvmId)) continue; + + const baseName = microvmMetadataParameterName(metadataSsmPath, microvmId); + const state = microvmStates.get(microvmId); + const cleanupStatus = cleanupRequestStatus(parameters, baseName, now); + + if (cleanupStatus === 'pending' || cleanupStatus === 'elapsed') { + if (cleanupStatus === 'elapsed' && (state === undefined || state === 'TERMINATED')) { + runnersToDelete.add(microvmId); + } else if (state === undefined || state === 'TERMINATED' || ACTIVE_STATES.has(state)) { + cleanupMicrovmIds.add(microvmId); + } + continue; + } + + if (cleanupStatus === 'invalid') { + if (state !== undefined && ACTIVE_STATES.has(state)) { + throw new Error(`Active MicroVM runner '${microvmId}' has an invalid cleanup request timestamp`); + } + await deleteParameterIfPresent(`${baseName}${CLEANUP_REQUESTED_AT_SUFFIX}`); + cleanupMicrovmIds.add(microvmId); + continue; + } + + if (state !== undefined && ACTIVE_STATES.has(state)) { + throw new Error(`Active MicroVM runner '${microvmId}' has state metadata but no ownership metadata`); + } + if (state === 'TERMINATED' || state === undefined) { + cleanupMicrovmIds.add(microvmId); + } + } + + for (const microvmId of [...runnersToDelete].slice(0, MAX_RECONCILED_RUNNERS)) { + try { + await deleteMicrovmRunnerSsmState(paths, microvmId); + } catch (error) { + logger.warn(`Failed to delete reconciled MicroVM runner metadata '${microvmId}'`, { error }); + } + } + + return { + cleanupMicrovmIds: [...cleanupMicrovmIds], + metadataById, + }; +} + +export async function setMicrovmGithubRunnerMetadata( + paths: MicrovmSsmPaths, + microvmId: string, + metadata: GitHubRunnerMetadata, + launchTags: MicrovmMetadataTag[], +): Promise { + if (!metadata.githubRunnerId) throw new Error('GitHub runner ID must not be empty'); + const baseName = microvmMetadataParameterName(paths.metadataSsmPath, microvmId); + const cleanupMarkerName = `${baseName}${CLEANUP_REQUESTED_AT_SUFFIX}`; + try { + const parameters = await getParameters([baseName, cleanupMarkerName]); + if (!parameters.has(baseName) || parameters.has(cleanupMarkerName)) { + throw new Error(`MicroVM runner '${microvmId}' is no longer accepting JIT configuration`); + } + } catch (error) { + await deleteMicrovmRunnerJitConfig(paths.runnerTokenSsmPath, microvmId); + throw error; + } + + const githubRunnerTags = createGitHubRunnerMetadataTags(metadata); + const tags = mergeParameterTags(launchTags, githubRunnerTags); + const serializedTags = serializeParameterTags(tags); + await putParameter( + stateParameterName(paths.metadataSsmPath, microvmId, GITHUB_RUNNER_ID_SUFFIX), + metadata.githubRunnerId, + false, + { + overwrite: true, + }, + ); + await putParameter(stateParameterName(paths.metadataSsmPath, microvmId, TAGS_SUFFIX), serializedTags, false, { + overwrite: true, + }); + try { + await addParameterTags(baseName, githubRunnerTags); + } catch (error) { + logger.error(`Failed to tag MicroVM runner '${microvmId}' with GitHub runner metadata`, { error }); + } +} + +export async function setMicrovmOrphan(metadataSsmPath: string, microvmId: string, orphan: boolean): Promise { + await putParameter(stateParameterName(metadataSsmPath, microvmId, ORPHAN_SUFFIX), String(orphan), false, { + overwrite: true, + }); +} + +export async function markMicrovmCleanupPending(metadataSsmPath: string, microvmId: string): Promise { + try { + await putParameter( + stateParameterName(metadataSsmPath, microvmId, CLEANUP_REQUESTED_AT_SUFFIX), + new Date().toISOString(), + false, + ); + } catch (error) { + if (!isParameterError(error, 'ParameterAlreadyExists')) throw error; + } +} + +async function deleteParameterIfPresent(parameterName: string): Promise { + try { + await deleteParameter(parameterName); + } catch (error) { + if (!isParameterNotFound(error)) throw error; + } +} + +export async function deleteMicrovmRunnerJitConfig(runnerTokenSsmPath: string, microvmId: string): Promise { + await deleteParameterIfPresent(microvmRunnerJitParameterName(runnerTokenSsmPath, microvmId)); +} + +export async function deleteMicrovmRunnerSsmState(paths: MicrovmSsmPaths, microvmId: string): Promise { + await deleteMicrovmRunnerJitConfig(paths.runnerTokenSsmPath, microvmId); + for (const parameterName of metadataParameterNames(paths.metadataSsmPath, microvmId)) { + await deleteParameterIfPresent(parameterName); + } +} diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.test.ts new file mode 100644 index 0000000000..f98eb88628 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.test.ts @@ -0,0 +1,79 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { loadMicrovmProviderConfig } from './config'; +import { listMicrovmRunners, microvmBootTimeExceeded, terminateMicrovm } from './microvms'; +import { createMicrovmScaleDownProvider } from './scale-down'; +import { setMicrovmOrphan } from './runner-metadata'; + +vi.mock('./config', () => ({ loadMicrovmProviderConfig: vi.fn() })); +vi.mock('./microvms', () => ({ + listMicrovmRunners: vi.fn(), + microvmBootTimeExceeded: vi.fn(), + terminateMicrovm: vi.fn(), +})); +vi.mock('./runner-metadata', () => ({ setMicrovmOrphan: vi.fn() })); + +const imageArn = 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner'; +const metadataSsmPath = '/github-action-runners/unit-test/microvm-metadata'; +const runnerTokenSsmPath = '/github-action-runners/unit-test/token'; +const providerConfig = { + imageIdentifier: imageArn, + executionRoleArn: 'arn:aws:iam::123456789012:role/microvm-runner', + metadataSsmPath, + runnerTokenSsmPath, +}; + +beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(loadMicrovmProviderConfig).mockReturnValue(providerConfig); + vi.mocked(listMicrovmRunners).mockResolvedValue([]); + vi.mocked(microvmBootTimeExceeded).mockReturnValue(false); + vi.mocked(setMicrovmOrphan).mockResolvedValue(); + vi.mocked(terminateMicrovm).mockResolvedValue(); +}); + +describe('createMicrovmScaleDownProvider', () => { + it('lists active and orphan runners through provider filters', async () => { + const provider = createMicrovmScaleDownProvider(); + + await provider.list('unit-test'); + await provider.list('unit-test', true); + + expect(listMicrovmRunners).toHaveBeenNthCalledWith( + 1, + { + environment: 'unit-test', + orphan: undefined, + }, + providerConfig, + ); + expect(listMicrovmRunners).toHaveBeenNthCalledWith( + 2, + { + environment: 'unit-test', + orphan: true, + }, + providerConfig, + ); + }); + + it('uses durable metadata when marking, unmarking, and terminating runners', async () => { + const provider = createMicrovmScaleDownProvider(); + + await provider.markOrphan('mvm-1'); + await provider.unmarkOrphan('mvm-1'); + await provider.terminate('mvm-1'); + + expect(setMicrovmOrphan).toHaveBeenNthCalledWith(1, metadataSsmPath, 'mvm-1', true); + expect(setMicrovmOrphan).toHaveBeenNthCalledWith(2, metadataSsmPath, 'mvm-1', false); + expect(terminateMicrovm).toHaveBeenCalledWith('mvm-1', providerConfig); + }); + + it('uses the MicroVM boot-time policy', () => { + const provider = createMicrovmScaleDownProvider(); + const runner = { id: 'mvm-1', owner: 'Codertocat', type: 'Org' as const }; + + expect(provider.bootTimeExceeded(runner)).toBe(false); + expect(microvmBootTimeExceeded).toHaveBeenCalledWith(runner); + }); +}); diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.ts new file mode 100644 index 0000000000..82038711df --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.ts @@ -0,0 +1,21 @@ +import type { ScaleDownComputeProvider } from '../../../../core'; +import { loadMicrovmProviderConfig } from './config'; +import type { MicrovmRunnerInfo } from './microvms'; +import { listMicrovmRunners, microvmBootTimeExceeded, terminateMicrovm } from './microvms'; +import { setMicrovmOrphan } from './runner-metadata'; + +export function createMicrovmScaleDownProvider(): Omit { + const ssmPaths = () => loadMicrovmProviderConfig(); + + async function list(environment: string, orphan?: boolean): Promise { + return await listMicrovmRunners({ environment, orphan }, ssmPaths()); + } + + return { + list, + bootTimeExceeded: microvmBootTimeExceeded, + markOrphan: async (id) => await setMicrovmOrphan(ssmPaths().metadataSsmPath, id, true), + unmarkOrphan: async (id) => await setMicrovmOrphan(ssmPaths().metadataSsmPath, id, false), + terminate: async (id) => await terminateMicrovm(id, ssmPaths()), + }; +} diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-up.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-up.test.ts new file mode 100644 index 0000000000..ab3d850b03 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-up.test.ts @@ -0,0 +1,112 @@ +import type { Octokit } from '@octokit/rest'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import type { CreateGitHubRunnerConfig, CreateStartRunnerConfig } from '../../../../core'; +import { listMicrovmRunners } from './microvms'; +import { createMicrovmRunners } from './runner-config'; +import { createMicrovmScaleUpProvider } from './scale-up'; + +vi.mock('./microvms', () => ({ listMicrovmRunners: vi.fn() })); +vi.mock('./runner-config', () => ({ createMicrovmRunners: vi.fn() })); + +const createStartRunnerConfig = vi.fn(); +const githubClient = {} as Octokit; +const overrideImageArn = 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner-large'; +const overrideEgressConnectorArn = + 'arn:aws:lambda:eu-west-1:123456789012:network-connector:github-runner-private-egress'; +const githubRunnerConfig: CreateGitHubRunnerConfig = { + ephemeral: true, + enableJitConfig: true, + runnerLabels: 'self-hosted,linux,arm64,microvm', + runnerGroup: 'Default', + runnerNamePrefix: '', + runnerOwner: 'Codertocat', + runnerType: 'Org', + disableAutoUpdate: true, +}; + +beforeEach(() => { + vi.clearAllMocks(); + process.env.ENVIRONMENT = 'unit-test'; + vi.mocked(listMicrovmRunners).mockResolvedValue([ + { id: 'mvm-current', owner: 'Codertocat', type: 'Org', state: 'RUNNING' }, + ]); + vi.mocked(createMicrovmRunners).mockResolvedValue({ + instances: ['mvm-new'], + retryableErrorCount: 0, + nonRetryableErrorCount: 0, + }); +}); + +describe('createMicrovmScaleUpProvider', () => { + it('resolves supported resource override labels and registers them on the runner', async () => { + const provider = createMicrovmScaleUpProvider(createStartRunnerConfig); + + await expect( + provider.resolveLabelsForRunners([ + `ghr-microvm-egress-network-connectors:${overrideEgressConnectorArn}`, + `ghr-microvm-image-arn:${overrideImageArn}`, + 'ghr-microvm-image-version:3.0', + ]), + ).resolves.toEqual({ + runnerLabels: [ + `ghr-microvm-egress-network-connectors:${overrideEgressConnectorArn}`, + `ghr-microvm-image-arn:${overrideImageArn}`, + 'ghr-microvm-image-version:3.0', + ], + state: { + overrides: { + egressNetworkConnectors: [overrideEgressConnectorArn], + imageIdentifier: overrideImageArn, + imageVersion: '3.0', + }, + }, + }); + }); + + it.each([ + ['ghr-microvm-memory:8192', "key 'memory' is not a supported MicroVM override"], + [ + 'ghr-microvm-maximum-duration-in-seconds:7200', + "key 'maximum-duration-in-seconds' is not a supported MicroVM override", + ], + ])('rejects unsupported MicroVM override label %s at the control-plane boundary', async (label, reason) => { + const provider = createMicrovmScaleUpProvider(createStartRunnerConfig); + + await expect(provider.resolveLabelsForRunners([label])).rejects.toThrow(reason); + }); + + it('counts managed MicroVMs for the runner owner', async () => { + const provider = createMicrovmScaleUpProvider(createStartRunnerConfig); + + await expect( + provider.getCurrentRunners({ overrides: {} }, { runnerOwner: 'Codertocat', runnerType: 'Org' }), + ).resolves.toBe(1); + expect(listMicrovmRunners).toHaveBeenCalledWith({ + environment: 'unit-test', + runnerOwner: 'Codertocat', + runnerType: 'Org', + }); + }); + + it('delegates runner creation to the shared MicroVM lifecycle', async () => { + const provider = createMicrovmScaleUpProvider(createStartRunnerConfig); + + await expect( + provider.createRunners({ + githubRunnerConfig, + numberOfRunners: 1, + githubInstallationClient: githubClient, + state: { overrides: { imageVersion: '3.0' } }, + }), + ).resolves.toEqual({ instances: ['mvm-new'], retryableErrorCount: 0, nonRetryableErrorCount: 0 }); + expect(createMicrovmRunners).toHaveBeenCalledWith( + githubRunnerConfig, + 1, + githubClient, + createStartRunnerConfig, + 'scale-up-lambda', + { imageVersion: '3.0' }, + ); + }); +}); diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-up.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-up.ts new file mode 100644 index 0000000000..a3dcf1219e --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-up.ts @@ -0,0 +1,77 @@ +import type { + CreateRunnerResult, + CreateScaleUpRunnersInput, + CreateStartRunnerConfig, + CurrentRunnersInput, + RunnerLabelResolution, + ScaleUpComputeProvider, +} from '../../../../core'; +import type { MicrovmDynamicLabelOverrides } from '../dynamic-labels'; +import { parseMicrovmDynamicLabels } from '../dynamic-labels'; +import { listMicrovmRunners } from './microvms'; +import { createMicrovmRunners } from './runner-config'; + +interface MicrovmScaleUpState { + overrides: MicrovmDynamicLabelOverrides; +} + +async function resolveMicrovmLabelsForRunners( + messageLabels: string[], +): Promise> { + const trimmedLabels = messageLabels.map((label) => label.trim()); + const parsed = parseMicrovmDynamicLabels(trimmedLabels); + if (parsed.violations.length > 0) { + throw new Error( + `Invalid MicroVM dynamic labels: ${parsed.violations + .map((violation) => `${violation.label} (${violation.reason})`) + .join(', ')}`, + ); + } + + return { + runnerLabels: trimmedLabels.filter((label) => label.startsWith('ghr-')), + state: { overrides: parsed.overrides }, + }; +} + +async function getCurrentMicrovmRunners( + _state: MicrovmScaleUpState, + { runnerType, runnerOwner }: CurrentRunnersInput, +): Promise { + return ( + await listMicrovmRunners({ + environment: process.env.ENVIRONMENT, + runnerType, + runnerOwner, + }) + ).length; +} + +async function createMicrovmScaleUpRunners( + { + githubRunnerConfig, + numberOfRunners, + githubInstallationClient, + state, + }: CreateScaleUpRunnersInput, + createStartRunnerConfig: CreateStartRunnerConfig, +): Promise { + return await createMicrovmRunners( + githubRunnerConfig, + numberOfRunners, + githubInstallationClient, + createStartRunnerConfig, + 'scale-up-lambda', + state.overrides, + ); +} + +export function createMicrovmScaleUpProvider( + createStartRunnerConfig: CreateStartRunnerConfig, +): Omit, 'type'> { + return { + resolveLabelsForRunners: resolveMicrovmLabelsForRunners, + getCurrentRunners: getCurrentMicrovmRunners, + createRunners: (input) => createMicrovmScaleUpRunners(input, createStartRunnerConfig), + }; +} diff --git a/lambdas/libs/compute-providers/aws/microvm/src/dynamic-labels.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/dynamic-labels.test.ts new file mode 100644 index 0000000000..442986a0f8 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/dynamic-labels.test.ts @@ -0,0 +1,73 @@ +import { describe, expect, it } from 'vitest'; + +import { parseMicrovmDynamicLabels } from './dynamic-labels'; + +const imageArn = 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner-large'; +const egressConnectorArn = 'arn:aws:lambda:eu-west-1:123456789012:network-connector:github-runner-private-egress'; +const internetEgressConnectorArn = + 'arn:aws:lambda:eu-west-1:aws:network-connector:aws-network-connector:INTERNET_EGRESS'; + +describe('parseMicrovmDynamicLabels', () => { + it('parses every supported RunMicrovm override', () => { + expect( + parseMicrovmDynamicLabels([ + `ghr-microvm-egress-network-connectors:${egressConnectorArn}`, + `ghr-microvm-egress-network-connectors:${internetEgressConnectorArn}`, + `ghr-microvm-image-arn:${imageArn}`, + 'ghr-microvm-image-version:3.0', + ]), + ).toEqual({ + overrides: { + egressNetworkConnectors: [egressConnectorArn, internetEgressConnectorArn], + imageIdentifier: imageArn, + imageVersion: '3.0', + }, + violations: [], + }); + }); + + it.each([ + ['ghr-microvm-memory:8192', "key 'memory' is not a supported MicroVM override"], + [ + 'ghr-microvm-egress-network-connectors:not-an-arn', + 'is not a valid Lambda network connector ARN; specify one ARN per label', + ], + [ + `ghr-microvm-egress-network-connectors:${egressConnectorArn};${internetEgressConnectorArn}`, + 'is not a valid Lambda network connector ARN; specify one ARN per label', + ], + ['ghr-microvm-image-arn:not-an-arn', 'is not a valid customer MicroVM image ARN'], + ['ghr-microvm-image-version:', "key 'image-version' requires a value"], + [ + 'ghr-microvm-maximum-duration-in-seconds:7200', + "key 'maximum-duration-in-seconds' is not a supported MicroVM override", + ], + ])('rejects invalid override %s', (label, reason) => { + const result = parseMicrovmDynamicLabels([label]); + + expect(result.overrides).toEqual({}); + expect(result.violations).toEqual([{ label, reason: expect.stringContaining(reason) }]); + }); + + it('ignores generic dynamic labels', () => { + expect(parseMicrovmDynamicLabels(['ghr-team:platform'])).toEqual({ overrides: {}, violations: [] }); + }); + + it('rejects more than ten egress network connectors', () => { + const labels = Array.from( + { length: 11 }, + (_, index) => + `ghr-microvm-egress-network-connectors:arn:aws:lambda:eu-west-1:123456789012:network-connector:connector-${index}`, + ); + + const result = parseMicrovmDynamicLabels(labels); + + expect(result.overrides.egressNetworkConnectors).toHaveLength(10); + expect(result.violations).toEqual([ + { + label: labels[10], + reason: 'at most 10 egress network connector labels are supported', + }, + ]); + }); +}); diff --git a/lambdas/libs/compute-providers/aws/microvm/src/dynamic-labels.ts b/lambdas/libs/compute-providers/aws/microvm/src/dynamic-labels.ts new file mode 100644 index 0000000000..2851716149 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/dynamic-labels.ts @@ -0,0 +1,76 @@ +export const MICROVM_DYNAMIC_LABEL_PREFIX = 'ghr-microvm-'; + +const MAXIMUM_EGRESS_NETWORK_CONNECTORS = 10; +const MICROVM_IMAGE_ARN_PATTERN = /^arn:[^:]+:lambda:[^:]+:[0-9]{12}:microvm-image:.+$/; +const MICROVM_NETWORK_CONNECTOR_ARN_PATTERN = + /^arn:aws[a-zA-Z-]*:lambda:[a-z0-9-]+:(?:[0-9]{12}|aws):network-connector:[a-zA-Z0-9_-]+(?::[a-zA-Z0-9_-]+)?$/; + +export interface MicrovmDynamicLabelOverrides { + egressNetworkConnectors?: string[]; + imageIdentifier?: string; + imageVersion?: string; +} + +export interface MicrovmDynamicLabelViolation { + label: string; + reason: string; +} + +export function parseMicrovmDynamicLabels(labels: string[]): { + overrides: MicrovmDynamicLabelOverrides; + violations: MicrovmDynamicLabelViolation[]; +} { + const overrides: MicrovmDynamicLabelOverrides = {}; + const violations: MicrovmDynamicLabelViolation[] = []; + + for (const label of labels) { + if (!label.startsWith(MICROVM_DYNAMIC_LABEL_PREFIX)) continue; + + const stripped = label.slice(MICROVM_DYNAMIC_LABEL_PREFIX.length); + const colonIndex = stripped.indexOf(':'); + const key = colonIndex === -1 ? stripped : stripped.slice(0, colonIndex); + const value = colonIndex === -1 ? '' : stripped.slice(colonIndex + 1).trim(); + + if (!value) { + violations.push({ label, reason: `key '${key}' requires a value` }); + continue; + } + + switch (key) { + case 'egress-network-connectors': { + if (!MICROVM_NETWORK_CONNECTOR_ARN_PATTERN.test(value)) { + violations.push({ + label, + reason: `'${value}' is not a valid Lambda network connector ARN; specify one ARN per label`, + }); + break; + } + + const connectors = overrides.egressNetworkConnectors ?? []; + if (connectors.length >= MAXIMUM_EGRESS_NETWORK_CONNECTORS) { + violations.push({ + label, + reason: `at most ${MAXIMUM_EGRESS_NETWORK_CONNECTORS} egress network connector labels are supported`, + }); + } else { + overrides.egressNetworkConnectors = [...connectors, value]; + } + break; + } + case 'image-arn': + if (!MICROVM_IMAGE_ARN_PATTERN.test(value)) { + violations.push({ label, reason: `'${value}' is not a valid customer MicroVM image ARN` }); + } else { + overrides.imageIdentifier = value; + } + break; + case 'image-version': + overrides.imageVersion = value; + break; + default: + violations.push({ label, reason: `key '${key}' is not a supported MicroVM override` }); + } + } + + return { overrides, violations }; +} diff --git a/lambdas/libs/compute-providers/aws/microvm/src/environment.d.ts b/lambdas/libs/compute-providers/aws/microvm/src/environment.d.ts new file mode 100644 index 0000000000..16a38ec54c --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/environment.d.ts @@ -0,0 +1,18 @@ +export {}; + +declare global { + namespace NodeJS { + interface ProcessEnv { + MICROVM_EGRESS_NETWORK_CONNECTORS: string | undefined; + MICROVM_EXECUTION_ROLE_ARN: string; + MICROVM_IMAGE_ARN: string; + MICROVM_IMAGE_VERSION: string | undefined; + MICROVM_INGRESS_NETWORK_CONNECTORS: string | undefined; + MICROVM_LOG_GROUP: string | undefined; + MICROVM_METADATA_SSM_PATH: string; + SSM_CONFIG_PATH: string; + SSM_PARAMETER_STORE_TAGS: string | undefined; + SSM_TOKEN_PATH: string; + } + } +} diff --git a/lambdas/libs/compute-providers/aws/microvm/src/webhook/dynamic-labels.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/webhook/dynamic-labels.test.ts new file mode 100644 index 0000000000..2b7b85d74e --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/webhook/dynamic-labels.test.ts @@ -0,0 +1,141 @@ +import { describe, expect, it } from 'vitest'; + +import type { RunnerMatcherConfig } from '../../../../contracts'; +import { microvmDynamicLabelProvider } from './dynamic-labels'; + +const imageArn = 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner-large'; +const egressConnectorArn = 'arn:aws:lambda:eu-west-1:123456789012:network-connector:github-runner-private-egress'; + +describe('microvmDynamicLabelProvider', () => { + it('accepts supported MicroVM overrides', () => { + const queue = microvmQueue(); + queue.matcherConfig.awsDynamicLabelsPolicy = { + restricted_keys: { + 'egress-network-connectors': { allowed: [egressConnectorArn] }, + 'image-arn': { allowed: [imageArn] }, + 'image-version': { allowed: ['3.0'] }, + }, + }; + const dynamicLabels = [ + `ghr-microvm-egress-network-connectors:${egressConnectorArn}`, + `ghr-microvm-image-arn:${imageArn}`, + 'ghr-microvm-image-version:3.0', + ]; + + expect(getViolations(queue, dynamicLabels)).toEqual([]); + }); + + it('requires explicit allowlists for image code and network-boundary overrides', () => { + expect( + getViolations(microvmQueue(), [ + `ghr-microvm-egress-network-connectors:${egressConnectorArn}`, + `ghr-microvm-image-arn:${imageArn}`, + 'ghr-microvm-image-version:3.0', + ]), + ).toEqual([ + { + label: `ghr-microvm-egress-network-connectors:${egressConnectorArn}`, + reason: "key 'egress-network-connectors' requires an explicit allowed list", + }, + { + label: `ghr-microvm-image-arn:${imageArn}`, + reason: "key 'image-arn' requires an explicit allowed list", + }, + { + label: 'ghr-microvm-image-version:3.0', + reason: "key 'image-version' requires an explicit allowed list", + }, + ]); + }); + + it.each([ + ['ghr-microvm-memory:8192', "key 'memory' is not a supported MicroVM override"], + [ + 'ghr-microvm-maximum-duration-in-seconds:7200', + "key 'maximum-duration-in-seconds' is not a supported MicroVM override", + ], + ])('preserves the parser violation for %s', (label, reason) => { + expect(getViolations(microvmQueue(), [label])).toEqual([ + { + label, + reason, + }, + ]); + }); + + it('enforces the AWS dynamic-label policy', () => { + const queue = microvmQueue(); + queue.matcherConfig.awsDynamicLabelsPolicy = { + restricted_keys: { 'image-version': { allowed: ['2.*'] } }, + }; + + expect(getViolations(queue, ['ghr-microvm-image-version:3.0'])).toEqual([ + { + label: 'ghr-microvm-image-version:3.0', + reason: "value '3.0' not in allowed list", + }, + ]); + }); + + it('applies allowed patterns to the complete image ARN', () => { + const queue = microvmQueue(); + queue.matcherConfig.awsDynamicLabelsPolicy = { + restricted_keys: { + 'image-arn': { + allowed: ['arn:aws:lambda:eu-west-1:123456789012:microvm-image:approved-*'], + }, + }, + }; + + expect( + getViolations(queue, [ + 'ghr-microvm-image-arn:arn:aws:lambda:eu-west-1:123456789012:microvm-image:approved-large', + ]), + ).toEqual([]); + expect( + getViolations(queue, ['ghr-microvm-image-arn:arn:aws:lambda:eu-west-1:123456789012:microvm-image:unapproved']), + ).toHaveLength(1); + }); + + it('applies the policy to each egress connector label', () => { + const queue = microvmQueue(); + queue.matcherConfig.awsDynamicLabelsPolicy = { + restricted_keys: { + 'egress-network-connectors': { + allowed: ['arn:aws:lambda:eu-west-1:123456789012:network-connector:approved-*'], + }, + }, + }; + + expect( + getViolations(queue, [ + 'ghr-microvm-egress-network-connectors:arn:aws:lambda:eu-west-1:123456789012:network-connector:approved-private', + ]), + ).toEqual([]); + expect( + getViolations(queue, [ + 'ghr-microvm-egress-network-connectors:arn:aws:lambda:eu-west-1:123456789012:network-connector:unapproved', + ]), + ).toHaveLength(1); + }); +}); + +function getViolations(queue: RunnerMatcherConfig, labels: string[]) { + return microvmDynamicLabelProvider.getViolations({ + queue, + labels, + }); +} + +function microvmQueue(): RunnerMatcherConfig { + return { + id: 'microvm', + arn: 'arn:aws:sqs:eu-west-1:123456789012:microvm', + computeProvider: 'microvm', + matcherConfig: { + labelMatchers: [['self-hosted', 'linux', 'arm64', 'microvm']], + exactMatch: false, + enableDynamicLabels: true, + }, + }; +} diff --git a/lambdas/libs/compute-providers/aws/microvm/src/webhook/dynamic-labels.ts b/lambdas/libs/compute-providers/aws/microvm/src/webhook/dynamic-labels.ts new file mode 100644 index 0000000000..e7c5485617 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/webhook/dynamic-labels.ts @@ -0,0 +1,32 @@ +import type { DynamicLabelProvider } from '../../../../contracts'; +import { violationsAgainstAwsDynamicLabelsPolicy } from '../../../dynamic-labels-policy'; +import { MICROVM_DYNAMIC_LABEL_PREFIX, parseMicrovmDynamicLabels } from '../dynamic-labels'; + +const RESOURCE_BOUNDARY_KEYS = new Set(['egress-network-connectors', 'image-arn', 'image-version']); + +function resourceBoundaryViolations( + labels: string[], + policy: Parameters[1], +) { + return labels.flatMap((label) => { + if (!label.startsWith(MICROVM_DYNAMIC_LABEL_PREFIX)) return []; + + const key = label.slice(MICROVM_DYNAMIC_LABEL_PREFIX.length).split(':', 1)[0]; + if (!RESOURCE_BOUNDARY_KEYS.has(key) || policy?.blocked_keys?.includes(key)) return []; + + const allowed = policy?.restricted_keys?.[key]?.allowed; + return allowed && allowed.length > 0 ? [] : [{ label, reason: `key '${key}' requires an explicit allowed list` }]; + }); +} + +export const microvmDynamicLabelProvider: DynamicLabelProvider = { + getViolations: ({ queue, labels }) => [ + ...parseMicrovmDynamicLabels(labels).violations, + ...resourceBoundaryViolations(labels, queue.matcherConfig.awsDynamicLabelsPolicy), + ...violationsAgainstAwsDynamicLabelsPolicy( + labels, + queue.matcherConfig.awsDynamicLabelsPolicy, + MICROVM_DYNAMIC_LABEL_PREFIX, + ), + ], +}; diff --git a/lambdas/libs/compute-providers/aws/microvm/webhook.test.ts b/lambdas/libs/compute-providers/aws/microvm/webhook.test.ts new file mode 100644 index 0000000000..ad3baed78b --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/webhook.test.ts @@ -0,0 +1,34 @@ +import { defineWebhookProviderContractTests } from '../../test/webhook-provider-contract'; +import { provider } from './webhook'; + +defineWebhookProviderContractTests({ + provider, + acceptedDynamicLabels: ['ghr-microvm-image-version:3.0'], + configureQueue: (queue) => { + queue.matcherConfig.awsDynamicLabelsPolicy = { + restricted_keys: { + 'image-version': { allowed: ['3.0'] }, + }, + }; + }, + rejectingPolicies: [ + { + name: 'blocked keys', + apply: (queue) => { + queue.matcherConfig.awsDynamicLabelsPolicy = { + blocked_keys: ['image-version'], + }; + }, + }, + { + name: 'restricted keys', + apply: (queue) => { + queue.matcherConfig.awsDynamicLabelsPolicy = { + restricted_keys: { + 'image-version': { allowed: ['2.*'] }, + }, + }; + }, + }, + ], +}); diff --git a/lambdas/libs/compute-providers/aws/microvm/webhook.ts b/lambdas/libs/compute-providers/aws/microvm/webhook.ts new file mode 100644 index 0000000000..48d603e476 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/webhook.ts @@ -0,0 +1,16 @@ +import type { ComputeProviderPlugin } from '../../core'; + +import type { WebhookProviderCapabilities, WebhookProviderModule } from '../../contracts'; +import { microvmDynamicLabelProvider } from './src/webhook/dynamic-labels'; + +export function createMicrovmWebhookPlugin(): ComputeProviderPlugin { + return { + type: 'microvm', + capabilities: { dynamicLabels: microvmDynamicLabelProvider }, + }; +} + +export const provider = { + type: 'microvm', + createPlugin: createMicrovmWebhookPlugin, +} satisfies WebhookProviderModule<'microvm'>; diff --git a/lambdas/libs/compute-providers/core/index.ts b/lambdas/libs/compute-providers/core/index.ts index 772455c84c..8579932f00 100644 --- a/lambdas/libs/compute-providers/core/index.ts +++ b/lambdas/libs/compute-providers/core/index.ts @@ -21,8 +21,6 @@ export interface CreateGitHubRunnerConfig { runnerOwner: string; runnerType: RunnerType; disableAutoUpdate: boolean; - ssmConfigPath: string; - ssmParameterStoreTags: { Key: string; Value: string }[]; } export interface GitHubRunnerMetadata { @@ -31,6 +29,7 @@ export interface GitHubRunnerMetadata { } export interface StartRunnerConfigOptions { + runnerConfigStore?: import('@aws-github-runner/storage-providers').RunnerConfigStore; runnerGroupCacheStore?: import('@aws-github-runner/storage-providers').RunnerGroupCacheStore; getRunnerConfigMetadata?: (runnerId: string) => { key: string; value: string }[]; onJitConfigCreated?: (runnerId: string, metadata: GitHubRunnerMetadata) => Promise; @@ -53,6 +52,7 @@ export interface CreateScaleUpRunnersInput { numberOfRunners: number; githubInstallationClient: Octokit; state: TState; + storage?: import('@aws-github-runner/storage-providers').RunnerConfigStorage; } export interface RunnerLabelResolution { @@ -114,6 +114,7 @@ export interface CreatePoolRunnersInput { githubRunnerConfig: CreateGitHubRunnerConfig; numberOfRunners: number; githubInstallationClient: Octokit; + storage?: import('@aws-github-runner/storage-providers').RunnerConfigStorage; } export interface PoolComputeProvider extends ComputeProvider { diff --git a/lambdas/libs/compute-providers/package.json b/lambdas/libs/compute-providers/package.json index a6fecab50c..a1d0e293cf 100644 --- a/lambdas/libs/compute-providers/package.json +++ b/lambdas/libs/compute-providers/package.json @@ -11,7 +11,11 @@ "./aws/ec2/webhook": "./aws/ec2/webhook.ts", "./aws/ec2/control-plane": "./aws/ec2/control-plane.ts", "./aws/ec2/runners": "./aws/ec2/src/runners.ts", - "./aws/ec2/control-plane/runner-creation": "./aws/ec2/src/control-plane/runner-creation.ts" + "./aws/ec2/control-plane/runner-creation": "./aws/ec2/src/control-plane/runner-creation.ts", + "./aws/ec2/control-plane/runners": "./aws/ec2/src/control-plane/runners.ts", + "./aws/ec2/control-plane/runner-config": "./aws/ec2/src/control-plane/runner-config.ts", + "./aws/microvm/webhook": "./aws/microvm/webhook.ts", + "./aws/microvm/control-plane": "./aws/microvm/control-plane.ts" }, "type": "module", "license": "MIT", @@ -28,6 +32,7 @@ "@aws-github-runner/aws-ssm-util": "*", "@aws-github-runner/storage-providers": "*", "@aws-sdk/client-ec2": "^3.1009.0", + "@aws-sdk/client-lambda-microvms": "^3.1074.0", "@octokit/rest": "22.0.1", "moment": "2.29.4", "yn": "3.1.1" diff --git a/lambdas/libs/compute-providers/provider-types.ts b/lambdas/libs/compute-providers/provider-types.ts index 64d7be8e5f..087f61de71 100644 --- a/lambdas/libs/compute-providers/provider-types.ts +++ b/lambdas/libs/compute-providers/provider-types.ts @@ -1,4 +1,4 @@ -export const computeProviderTypes = ['ec2'] as const; +export const computeProviderTypes = ['ec2', 'microvm'] as const; export type ComputeProviderType = (typeof computeProviderTypes)[number]; diff --git a/lambdas/libs/compute-providers/providers.config.control-plane.ts b/lambdas/libs/compute-providers/providers.config.control-plane.ts index 55ebaca95e..45a584bc06 100644 --- a/lambdas/libs/compute-providers/providers.config.control-plane.ts +++ b/lambdas/libs/compute-providers/providers.config.control-plane.ts @@ -1,5 +1,6 @@ import { provider as ec2 } from './aws/ec2/control-plane'; +import { provider as microvm } from './aws/microvm/control-plane'; import type { ControlPlaneProviderModule } from './contracts'; /** Provider plugins included in the control-plane bundle. */ -export const enabledControlPlaneProviders = [ec2] as const satisfies readonly ControlPlaneProviderModule[]; +export const enabledControlPlaneProviders = [ec2, microvm] as const satisfies readonly ControlPlaneProviderModule[]; diff --git a/lambdas/libs/compute-providers/providers.config.webhook.ts b/lambdas/libs/compute-providers/providers.config.webhook.ts index 19c92734da..a4aec0853a 100644 --- a/lambdas/libs/compute-providers/providers.config.webhook.ts +++ b/lambdas/libs/compute-providers/providers.config.webhook.ts @@ -1,5 +1,6 @@ import { provider as ec2 } from './aws/ec2/webhook'; +import { provider as microvm } from './aws/microvm/webhook'; import type { WebhookProviderModule } from './contracts'; /** Provider plugins included in the webhook bundle. */ -export const enabledWebhookProviders = [ec2] as const satisfies readonly WebhookProviderModule[]; +export const enabledWebhookProviders = [ec2, microvm] as const satisfies readonly WebhookProviderModule[]; diff --git a/lambdas/libs/compute-providers/test/webhook-provider-contract.ts b/lambdas/libs/compute-providers/test/webhook-provider-contract.ts index dd4e3097b0..a6d01b7e6e 100644 --- a/lambdas/libs/compute-providers/test/webhook-provider-contract.ts +++ b/lambdas/libs/compute-providers/test/webhook-provider-contract.ts @@ -13,17 +13,25 @@ interface RejectingPolicyCase { interface WebhookProviderContractOptions { provider: WebhookProviderModule; acceptedDynamicLabels: readonly [string, ...string[]]; + configureQueue?(queue: RunnerMatcherConfig): void; rejectingPolicies: readonly [RejectingPolicyCase, ...RejectingPolicyCase[]]; } export function defineWebhookProviderContractTests({ provider, acceptedDynamicLabels, + configureQueue, rejectingPolicies, }: WebhookProviderContractOptions): void { const nonGhrLabels = ['self-hosted', 'linux']; const dynamicLabels = [...acceptedDynamicLabels]; + function configuredRunnerQueue(id: string, computeProvider?: ComputeProviderType): RunnerMatcherConfig { + const queue = runnerQueue(id, computeProvider); + configureQueue?.(queue); + return queue; + } + function expectProviderSelected(queue: RunnerMatcherConfig) { expect(selectDynamicLabelQueue([queue], nonGhrLabels, dynamicLabels)).toEqual({ queue, @@ -33,11 +41,11 @@ export function defineWebhookProviderContractTests { it('selects an explicitly configured provider through the production registry', () => { - expectProviderSelected(runnerQueue(`${provider.type}-configured`, provider.type)); + expectProviderSelected(configuredRunnerQueue(`${provider.type}-configured`, provider.type)); }); it('skips the provider when dynamic labels are disabled', () => { - const queue = runnerQueue(`${provider.type}-disabled`, provider.type); + const queue = configuredRunnerQueue(`${provider.type}-disabled`, provider.type); queue.matcherConfig.enableDynamicLabels = false; expect(selectDynamicLabelQueue([queue], nonGhrLabels, dynamicLabels)).toBeUndefined(); @@ -45,7 +53,7 @@ export function defineWebhookProviderContractTests { - const queue = runnerQueue(`${provider.type}-policy-rejected`, provider.type); + const queue = configuredRunnerQueue(`${provider.type}-policy-rejected`, provider.type); policy.apply(queue); expect(selectDynamicLabelQueue([queue], nonGhrLabels, dynamicLabels)).toBeUndefined(); @@ -53,7 +61,7 @@ export function defineWebhookProviderContractTests { - const queue = runnerQueue(`${provider.type}-normalized`); + const queue = configuredRunnerQueue(`${provider.type}-normalized`); (queue as unknown as { computeProvider: string }).computeProvider = ` ${provider.type.toUpperCase()} `; expectProviderSelected(queue); @@ -61,7 +69,7 @@ export function defineWebhookProviderContractTests { - expectProviderSelected(runnerQueue(`${provider.type}-default`)); + expectProviderSelected(configuredRunnerQueue(`${provider.type}-default`)); }); } }); diff --git a/lambdas/libs/storage-providers/aws/ssm/environment.d.ts b/lambdas/libs/storage-providers/aws/ssm/environment.d.ts index c6dd725742..fe29bfc244 100644 --- a/lambdas/libs/storage-providers/aws/ssm/environment.d.ts +++ b/lambdas/libs/storage-providers/aws/ssm/environment.d.ts @@ -4,7 +4,11 @@ declare global { namespace NodeJS { interface ProcessEnv { SSM_PARAMETER_STORE_TAGS?: string; + SSM_CONFIG_PATH?: string; SSM_TOKEN_PATH?: string; + PARAMETER_GITHUB_APP_ID_NAME?: string; + PARAMETER_GITHUB_APP_KEY_BASE64_NAME?: string; + PARAMETER_GITHUB_APP_INSTALLATION_ID_NAME?: string; } } } diff --git a/lambdas/libs/storage-providers/aws/ssm/github-app-credentials-store.test.ts b/lambdas/libs/storage-providers/aws/ssm/github-app-credentials-store.test.ts new file mode 100644 index 0000000000..b028a4c098 --- /dev/null +++ b/lambdas/libs/storage-providers/aws/ssm/github-app-credentials-store.test.ts @@ -0,0 +1,65 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { getParameters } from '@aws-github-runner/aws-ssm-util'; + +import { createAwsSsmGitHubAppCredentialsStore } from './github-app-credentials-store'; + +vi.mock('@aws-github-runner/aws-ssm-util', () => ({ + getParameters: vi.fn(), +})); + +const getParametersMock = vi.mocked(getParameters); + +describe('aws_ssm GitHub App credentials store', () => { + beforeEach(() => { + vi.clearAllMocks(); + process.env.PARAMETER_GITHUB_APP_ID_NAME = 'app-id'; + process.env.PARAMETER_GITHUB_APP_KEY_BASE64_NAME = 'app-key'; + delete process.env.PARAMETER_GITHUB_APP_INSTALLATION_ID_NAME; + }); + + it('loads batched credentials and decodes escaped newlines', async () => { + const privateKey = Buffer.from('private-key\\nline-2').toString('base64'); + getParametersMock.mockResolvedValue( + new Map([ + ['app-id', '123'], + ['app-key', privateKey], + ]), + ); + + await expect(createAwsSsmGitHubAppCredentialsStore().get()).resolves.toEqual([ + { appId: 123, privateKey: 'private-key\nline-2', installationId: undefined }, + ]); + expect(getParametersMock).toHaveBeenCalledWith(['app-id', 'app-key']); + }); + + it('loads per-app installation IDs in the same order as app IDs', async () => { + process.env.PARAMETER_GITHUB_APP_ID_NAME = 'id-0:id-1'; + process.env.PARAMETER_GITHUB_APP_KEY_BASE64_NAME = 'key-0:key-1'; + process.env.PARAMETER_GITHUB_APP_INSTALLATION_ID_NAME = ':installation-1'; + getParametersMock.mockResolvedValue( + new Map([ + ['id-0', '123'], + ['id-1', '456'], + ['key-0', Buffer.from('key-0').toString('base64')], + ['key-1', Buffer.from('key-1').toString('base64')], + ['installation-1', '789'], + ]), + ); + + await expect(createAwsSsmGitHubAppCredentialsStore().get()).resolves.toMatchObject([ + { appId: 123, installationId: undefined }, + { appId: 456, installationId: 789 }, + ]); + }); + + it.each(['PARAMETER_GITHUB_APP_ID_NAME', 'PARAMETER_GITHUB_APP_KEY_BASE64_NAME'])('requires %s', (name) => { + delete process.env[name]; + expect(() => createAwsSsmGitHubAppCredentialsStore()).toThrow(`Environment variable ${name} is not set`); + }); + + it('rejects mismatched app and key parameter lists', () => { + process.env.PARAMETER_GITHUB_APP_ID_NAME = 'id-0:id-1'; + expect(() => createAwsSsmGitHubAppCredentialsStore()).toThrow('parameter count mismatch'); + }); +}); diff --git a/lambdas/libs/storage-providers/aws/ssm/github-app-credentials-store.ts b/lambdas/libs/storage-providers/aws/ssm/github-app-credentials-store.ts new file mode 100644 index 0000000000..0b9ce7bc5a --- /dev/null +++ b/lambdas/libs/storage-providers/aws/ssm/github-app-credentials-store.ts @@ -0,0 +1,66 @@ +import { getParameters } from '@aws-github-runner/aws-ssm-util'; + +import type { GitHubAppCredential, GitHubAppCredentialsStore } from '../../core'; + +interface AwsSsmGitHubAppCredentialsEnvironment { + PARAMETER_GITHUB_APP_ID_NAME?: string; + PARAMETER_GITHUB_APP_KEY_BASE64_NAME?: string; + PARAMETER_GITHUB_APP_INSTALLATION_ID_NAME?: string; +} + +export function createAwsSsmGitHubAppCredentialsStore( + environment: Readonly = process.env, +): GitHubAppCredentialsStore { + const idParameters = splitParameterNames(environment.PARAMETER_GITHUB_APP_ID_NAME, 'PARAMETER_GITHUB_APP_ID_NAME'); + const keyParameters = splitParameterNames( + environment.PARAMETER_GITHUB_APP_KEY_BASE64_NAME, + 'PARAMETER_GITHUB_APP_KEY_BASE64_NAME', + ); + const installationIdParameters = environment.PARAMETER_GITHUB_APP_INSTALLATION_ID_NAME?.split(':') ?? []; + + if (idParameters.length !== keyParameters.length) { + throw new Error(`GitHub App parameter count mismatch: ${idParameters.length} IDs vs ${keyParameters.length} keys`); + } + + return new AwsSsmGitHubAppCredentialsStore(idParameters, keyParameters, installationIdParameters); +} + +class AwsSsmGitHubAppCredentialsStore implements GitHubAppCredentialsStore { + constructor( + private readonly idParameters: string[], + private readonly keyParameters: string[], + private readonly installationIdParameters: string[], + ) {} + + async get(): Promise { + const parameters = await getParameters([ + ...this.idParameters, + ...this.keyParameters, + ...this.installationIdParameters.filter(Boolean), + ]); + return this.idParameters.map((idParameter, index) => { + const appIdValue = parameters.get(idParameter); + if (!appIdValue) { + throw new Error(`Parameter ${idParameter} not found`); + } + const privateKeyBase64 = parameters.get(this.keyParameters[index]); + if (!privateKeyBase64) { + throw new Error(`Parameter ${this.keyParameters[index]} not found`); + } + const installationIdParameter = this.installationIdParameters[index]; + const installationIdValue = installationIdParameter ? parameters.get(installationIdParameter) : undefined; + return { + appId: Number.parseInt(appIdValue, 10), + privateKey: Buffer.from(privateKeyBase64, 'base64').toString().replace(/\\n/g, '\n'), + installationId: installationIdValue ? Number.parseInt(installationIdValue, 10) : undefined, + }; + }); + } +} + +function splitParameterNames(value: string | undefined, name: string): string[] { + if (!value || value.trim() === '') { + throw new Error(`Environment variable ${name} is not set`); + } + return value.split(':').filter(Boolean); +} diff --git a/lambdas/libs/storage-providers/aws/ssm/parameter-store-tags.ts b/lambdas/libs/storage-providers/aws/ssm/parameter-store-tags.ts index d35150e10a..07c329c4dc 100644 --- a/lambdas/libs/storage-providers/aws/ssm/parameter-store-tags.ts +++ b/lambdas/libs/storage-providers/aws/ssm/parameter-store-tags.ts @@ -3,9 +3,11 @@ interface SsmParameterStoreTag { Value: string; } -export function loadSsmParameterStoreTagsFromEnvironment(): SsmParameterStoreTag[] { - return process.env.SSM_PARAMETER_STORE_TAGS && process.env.SSM_PARAMETER_STORE_TAGS.trim() !== '' - ? validateSsmParameterStoreTags(process.env.SSM_PARAMETER_STORE_TAGS) +export function loadSsmParameterStoreTagsFromEnvironment( + environment: Readonly> = process.env, +): SsmParameterStoreTag[] { + return environment.SSM_PARAMETER_STORE_TAGS && environment.SSM_PARAMETER_STORE_TAGS.trim() !== '' + ? validateSsmParameterStoreTags(environment.SSM_PARAMETER_STORE_TAGS) : []; } diff --git a/lambdas/libs/storage-providers/aws/ssm/runner-config-consumer-common.ts b/lambdas/libs/storage-providers/aws/ssm/runner-config-consumer-common.ts new file mode 100644 index 0000000000..9b69ad3a5d --- /dev/null +++ b/lambdas/libs/storage-providers/aws/ssm/runner-config-consumer-common.ts @@ -0,0 +1,193 @@ +import type { RunnerConfigConsumeOptions } from '../../core'; + +export const DEFAULT_CALL_TIMEOUT_MS = 5_000; +export const DEFAULT_CONFIG_TIMEOUT_MS = 40_000; +export const DEFAULT_POLL_INTERVAL_MS = 2_000; + +const RUNNER_ID_PATTERN = /^[A-Za-z0-9_.-]{1,256}$/; +const SSM_PARAMETER_PATH_PATTERN = /^\/[A-Za-z0-9_.\-/]+$/; +const MAX_SSM_PARAMETER_NAME_LENGTH = 900; +const RETRYABLE_ERROR_NAMES = new Set([ + 'AbortError', + 'ConnectionError', + 'InternalServerException', + 'ProvisionedThroughputExceededException', + 'RequestLimitExceeded', + 'RequestTimeout', + 'ServiceUnavailable', + 'ThrottlingException', + 'TimeoutError', +]); + +export interface RunnerConfigPollingOptions { + callTimeoutMs?: number; + configTimeoutMs?: number; + pollIntervalMs?: number; +} + +export interface ResolvedRunnerConfigPollingOptions { + callTimeoutMs: number; + configTimeoutMs: number; + pollIntervalMs: number; +} + +class RunnerConfigCallDeadlineError extends Error { + public constructor() { + super('runner configuration provider call exceeded its deadline'); + this.name = 'RunnerConfigCallDeadlineError'; + } +} + +export function resolvePollingOptions(options: RunnerConfigPollingOptions): ResolvedRunnerConfigPollingOptions { + return { + callTimeoutMs: positiveIntegerOption('callTimeoutMs', options.callTimeoutMs, DEFAULT_CALL_TIMEOUT_MS), + configTimeoutMs: positiveIntegerOption('configTimeoutMs', options.configTimeoutMs, DEFAULT_CONFIG_TIMEOUT_MS), + pollIntervalMs: positiveIntegerOption('pollIntervalMs', options.pollIntervalMs, DEFAULT_POLL_INTERVAL_MS), + }; +} + +export function positiveIntegerOption(name: string, value: number | undefined, fallback: number): number { + if (value === undefined) return fallback; + if (!Number.isSafeInteger(value) || value <= 0) throw new Error(`${name} must be a positive integer`); + return value; +} + +export function validateRunnerId(runnerId: string): void { + if (!RUNNER_ID_PATTERN.test(runnerId)) throw new Error('runnerId is invalid'); +} + +export function canonicalSsmTokenPath(tokenPath: string): string { + if (tokenPath.includes('//')) throw new Error('aws_ssm tokenPath is invalid'); + const canonical = tokenPath.endsWith('/') ? tokenPath.slice(0, -1) : tokenPath; + const segments = canonical.split('/').slice(1); + if ( + canonical.length === 0 || + canonical.length > MAX_SSM_PARAMETER_NAME_LENGTH || + !SSM_PARAMETER_PATH_PATTERN.test(canonical) || + segments.length > 14 || + segments.some((segment) => segment === '' || segment === '.' || segment === '..') || + /^(aws|ssm)/i.test(segments[0] ?? '') + ) { + throw new Error('aws_ssm tokenPath is invalid'); + } + return canonical; +} + +export function composeSsmParameterName(tokenPath: string, runnerId: string): string { + validateRunnerId(runnerId); + const parameterName = `${canonicalSsmTokenPath(tokenPath)}/${runnerId}`; + const segments = parameterName.split('/').slice(1); + if (parameterName.length > MAX_SSM_PARAMETER_NAME_LENGTH || segments.length > 15) { + throw new Error('aws_ssm runner configuration key is invalid'); + } + return parameterName; +} + +export function validateConsumeOptions(options: RunnerConfigConsumeOptions): void { + if (!Number.isSafeInteger(options.deadlineMs) || options.deadlineMs <= 0) { + throw new Error('deadlineMs must be a positive integer'); + } + if ( + options.signal === null || + typeof options.signal !== 'object' || + typeof options.signal.aborted !== 'boolean' || + typeof options.signal.addEventListener !== 'function' || + typeof options.signal.removeEventListener !== 'function' + ) { + throw new Error('signal must be an AbortSignal'); + } +} + +export function errorName(error: unknown): string { + if (error !== null && typeof error === 'object' && 'name' in error && typeof error.name === 'string') { + return error.name; + } + return 'UnknownError'; +} + +function httpStatus(error: unknown): number | undefined { + if ( + error !== null && + typeof error === 'object' && + '$metadata' in error && + error.$metadata !== null && + typeof error.$metadata === 'object' && + 'httpStatusCode' in error.$metadata && + typeof error.$metadata.httpStatusCode === 'number' + ) { + return error.$metadata.httpStatusCode; + } + return undefined; +} + +export function isRetryableProviderError(error: unknown): boolean { + const status = httpStatus(error); + return ( + error instanceof RunnerConfigCallDeadlineError || + RETRYABLE_ERROR_NAMES.has(errorName(error)) || + (status !== undefined && status >= 500) + ); +} + +export function delay(ms: number, signal: AbortSignal): Promise { + if (signal.aborted) return Promise.reject(new Error('runner configuration consumption was cancelled')); + return new Promise((resolve, reject) => { + let settled = false; + const cleanup = (): void => signal.removeEventListener('abort', cancel); + const finish = (): void => { + if (settled) return; + settled = true; + cleanup(); + resolve(); + }; + const timer = setTimeout(finish, ms); + const cancel = (): void => { + if (settled) return; + settled = true; + clearTimeout(timer); + cleanup(); + reject(new Error('runner configuration consumption was cancelled')); + }; + signal.addEventListener('abort', cancel, { once: true }); + }); +} + +export async function withCallDeadline( + parentSignal: AbortSignal, + deadlineMs: number, + callTimeoutMs: number, + operation: (signal: AbortSignal) => Promise, +): Promise { + if (parentSignal.aborted) throw new Error('runner configuration consumption was cancelled'); + const remaining = deadlineMs - Date.now(); + if (remaining <= 0) throw new RunnerConfigCallDeadlineError(); + + const controller = new AbortController(); + let cancel!: () => void; + let timeout: ReturnType | undefined; + const deadline = new Promise((_resolve, reject) => { + cancel = (): void => { + reject(new Error('runner configuration consumption was cancelled')); + controller.abort(); + }; + parentSignal.addEventListener('abort', cancel, { once: true }); + timeout = setTimeout( + () => { + reject(new RunnerConfigCallDeadlineError()); + controller.abort(); + }, + Math.max(1, Math.min(remaining, callTimeoutMs)), + ); + }); + + try { + return await Promise.race([operation(controller.signal), deadline]); + } finally { + if (timeout !== undefined) clearTimeout(timeout); + parentSignal.removeEventListener('abort', cancel); + } +} + +export function throwIfCancelled(signal: AbortSignal): void { + if (signal.aborted) throw new Error('runner configuration consumption was cancelled'); +} diff --git a/lambdas/libs/storage-providers/aws/ssm/runner-config-consumer.test.ts b/lambdas/libs/storage-providers/aws/ssm/runner-config-consumer.test.ts new file mode 100644 index 0000000000..2a5d2217ea --- /dev/null +++ b/lambdas/libs/storage-providers/aws/ssm/runner-config-consumer.test.ts @@ -0,0 +1,220 @@ +import { DeleteParameterCommand, GetParameterCommand, type SSMClient } from '@aws-sdk/client-ssm'; +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import { + AwsSdkSsmRunnerConfigApi, + createAwsSsmRunnerConfigConsumer, + type AwsSsmRunnerConfigApi, +} from './runner-config-consumer'; + +function namedError(name: string, message = 'provider detail'): Error { + const error = new Error(message); + error.name = name; + return error; +} + +describe('AWS SDK SSM runner config API', () => { + it('decrypts the parameter and deletes it with the caller abort signal', async () => { + const send = vi + .fn() + .mockResolvedValueOnce({ Parameter: { Value: 'encoded-jit' } }) + .mockResolvedValueOnce({}); + const api = new AwsSdkSsmRunnerConfigApi({ send } as unknown as SSMClient); + const signal = new AbortController().signal; + + await expect(api.getParameter('/runner/tokens/runner-123', signal)).resolves.toBe('encoded-jit'); + await expect(api.deleteParameter('/runner/tokens/runner-123', signal)).resolves.toBeUndefined(); + + expect(send.mock.calls[0][0]).toBeInstanceOf(GetParameterCommand); + expect(send.mock.calls[0][0].input).toEqual({ + Name: '/runner/tokens/runner-123', + WithDecryption: true, + }); + expect(send.mock.calls[0][1]).toEqual({ abortSignal: signal }); + expect(send.mock.calls[1][0]).toBeInstanceOf(DeleteParameterCommand); + expect(send.mock.calls[1][0].input).toEqual({ Name: '/runner/tokens/runner-123' }); + expect(send.mock.calls[1][1]).toEqual({ abortSignal: signal }); + }); +}); + +describe('SSM runner config consumer', () => { + afterEach(() => { + vi.useRealTimers(); + }); + + it('polls a missing parameter, reads it, and deletes it before returning', async () => { + const getParameter = vi + .fn() + .mockRejectedValueOnce(namedError('ParameterNotFound')) + .mockResolvedValueOnce('encoded-jit'); + const deleteParameter = vi.fn().mockResolvedValue(undefined); + const consumer = createAwsSsmRunnerConfigConsumer( + { RUNNER_CONFIG_STORAGE_PROVIDER: 'aws_ssm', SSM_TOKEN_PATH: '/runner/tokens' }, + { + api: { getParameter, deleteParameter }, + callTimeoutMs: 100, + configTimeoutMs: 500, + pollIntervalMs: 1, + }, + ); + + await expect( + consumer.consume('runner-123', { + deadlineMs: Date.now() + 1_000, + signal: new AbortController().signal, + }), + ).resolves.toBe('encoded-jit'); + expect(getParameter).toHaveBeenCalledTimes(2); + expect(deleteParameter).toHaveBeenCalledOnce(); + expect(deleteParameter).toHaveBeenCalledWith('/runner/tokens/runner-123', expect.any(AbortSignal)); + }); + + it('retries a transient delete failure without returning the value early', async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-01-01T00:00:00.000Z')); + const api: AwsSsmRunnerConfigApi = { + getParameter: vi.fn().mockResolvedValue('encoded-jit'), + deleteParameter: vi + .fn() + .mockRejectedValueOnce(namedError('ThrottlingException')) + .mockResolvedValueOnce(undefined), + }; + const consumer = createAwsSsmRunnerConfigConsumer( + { RUNNER_CONFIG_STORAGE_PROVIDER: 'aws_ssm', SSM_TOKEN_PATH: '/runner/tokens' }, + { api, callTimeoutMs: 100, configTimeoutMs: 2_000, deleteAttempts: 2, pollIntervalMs: 1 }, + ); + + const pending = consumer.consume('runner-123', { + deadlineMs: Date.now() + 3_000, + signal: new AbortController().signal, + }); + await vi.runAllTimersAsync(); + + await expect(pending).resolves.toBe('encoded-jit'); + expect(api.deleteParameter).toHaveBeenCalledTimes(2); + }); + + it('fails closed when another reader deletes the SSM parameter first', async () => { + const api: AwsSsmRunnerConfigApi = { + getParameter: vi.fn().mockResolvedValue('encoded-jit'), + deleteParameter: vi.fn().mockRejectedValue(namedError('ParameterNotFound')), + }; + const consumer = createAwsSsmRunnerConfigConsumer( + { RUNNER_CONFIG_STORAGE_PROVIDER: 'aws_ssm', SSM_TOKEN_PATH: '/runner/tokens' }, + { api, callTimeoutMs: 100, configTimeoutMs: 100, deleteAttempts: 3, pollIntervalMs: 1 }, + ); + + await expect( + consumer.consume('runner-123', { + deadlineMs: Date.now() + 1_000, + signal: new AbortController().signal, + }), + ).rejects.toThrow('runner configuration could not be deleted from SSM'); + expect(api.deleteParameter).toHaveBeenCalledOnce(); + }); + + it('sanitizes non-retryable provider failures', async () => { + const api: AwsSsmRunnerConfigApi = { + getParameter: vi.fn().mockRejectedValue(namedError('AccessDeniedException', 'encoded-jit-secret')), + deleteParameter: vi.fn(), + }; + const consumer = createAwsSsmRunnerConfigConsumer( + { RUNNER_CONFIG_STORAGE_PROVIDER: 'aws_ssm', SSM_TOKEN_PATH: '/runner/tokens' }, + { api, callTimeoutMs: 100, configTimeoutMs: 100, pollIntervalMs: 1 }, + ); + + const pending = consumer.consume('runner-123', { + deadlineMs: Date.now() + 1_000, + signal: new AbortController().signal, + }); + await expect(pending).rejects.toThrow('failed to read runner configuration from SSM'); + await expect(pending).rejects.not.toThrow('encoded-jit-secret'); + expect(api.deleteParameter).not.toHaveBeenCalled(); + }); + + it('rejects an empty SSM parameter value without attempting deletion', async () => { + const api: AwsSsmRunnerConfigApi = { + getParameter: vi.fn().mockResolvedValue(''), + deleteParameter: vi.fn(), + }; + const consumer = createAwsSsmRunnerConfigConsumer( + { RUNNER_CONFIG_STORAGE_PROVIDER: 'aws_ssm', SSM_TOKEN_PATH: '/runner/tokens' }, + { api, callTimeoutMs: 100, configTimeoutMs: 100, pollIntervalMs: 1 }, + ); + + await expect( + consumer.consume('runner-123', { + deadlineMs: Date.now() + 1_000, + signal: new AbortController().signal, + }), + ).rejects.toThrow('failed to read runner configuration from SSM'); + expect(api.deleteParameter).not.toHaveBeenCalled(); + }); + + it('validates the full parameter name before calling SSM', async () => { + const api: AwsSsmRunnerConfigApi = { + getParameter: vi.fn(), + deleteParameter: vi.fn(), + }; + const consumer = createAwsSsmRunnerConfigConsumer( + { RUNNER_CONFIG_STORAGE_PROVIDER: 'aws_ssm', SSM_TOKEN_PATH: `/${'x'.repeat(890)}` }, + { api, callTimeoutMs: 100, configTimeoutMs: 100, pollIntervalMs: 1 }, + ); + + await expect( + consumer.consume('runner-1234567890', { + deadlineMs: Date.now() + 1_000, + signal: new AbortController().signal, + }), + ).rejects.toThrow('aws_ssm runner configuration key is invalid'); + expect(api.getParameter).not.toHaveBeenCalled(); + }); + + it('stops a provider call immediately when the caller aborts', async () => { + const api: AwsSsmRunnerConfigApi = { + getParameter: vi.fn().mockReturnValue(new Promise(() => undefined)), + deleteParameter: vi.fn(), + }; + const controller = new AbortController(); + const consumer = createAwsSsmRunnerConfigConsumer( + { RUNNER_CONFIG_STORAGE_PROVIDER: 'aws_ssm', SSM_TOKEN_PATH: '/runner/tokens' }, + { api, callTimeoutMs: 10_000, configTimeoutMs: 10_000, pollIntervalMs: 1 }, + ); + const pending = consumer.consume('runner-123', { + deadlineMs: Date.now() + 10_000, + signal: controller.signal, + }); + + controller.abort(); + + await expect(pending).rejects.toThrow('runner configuration consumption was cancelled'); + }); + + it('reserves a bounded delete attempt when the value appears near the polling deadline', async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-01-01T00:00:00.000Z')); + const startedAt = Date.now(); + let deleteStartedAt: number | undefined; + const api: AwsSsmRunnerConfigApi = { + getParameter: vi.fn().mockResolvedValueOnce(undefined).mockResolvedValueOnce('encoded-jit'), + deleteParameter: vi.fn().mockImplementation(async () => { + deleteStartedAt = Date.now(); + }), + }; + const consumer = createAwsSsmRunnerConfigConsumer( + { RUNNER_CONFIG_STORAGE_PROVIDER: 'aws_ssm', SSM_TOKEN_PATH: '/runner/tokens' }, + { api, callTimeoutMs: 100, configTimeoutMs: 1_000, pollIntervalMs: 99 }, + ); + + const pending = consumer.consume('runner-123', { + deadlineMs: startedAt + 200, + signal: new AbortController().signal, + }); + await vi.runAllTimersAsync(); + + await expect(pending).resolves.toBe('encoded-jit'); + expect(api.getParameter).toHaveBeenCalledTimes(2); + expect(deleteStartedAt).toBe(startedAt + 99); + expect(deleteStartedAt).toBeLessThanOrEqual(startedAt + 100); + }); +}); diff --git a/lambdas/libs/storage-providers/aws/ssm/runner-config-consumer.ts b/lambdas/libs/storage-providers/aws/ssm/runner-config-consumer.ts new file mode 100644 index 0000000000..da26a25402 --- /dev/null +++ b/lambdas/libs/storage-providers/aws/ssm/runner-config-consumer.ts @@ -0,0 +1,163 @@ +import { DeleteParameterCommand, GetParameterCommand, SSMClient } from '@aws-sdk/client-ssm'; + +import type { RunnerConfigConsumer, RunnerConfigConsumeOptions } from '../../core'; +import { + composeSsmParameterName, + delay, + errorName, + isRetryableProviderError, + positiveIntegerOption, + resolvePollingOptions, + throwIfCancelled, + validateConsumeOptions, + withCallDeadline, + type RunnerConfigPollingOptions, +} from './runner-config-consumer-common'; + +const DEFAULT_DELETE_ATTEMPTS = 3; + +export interface AwsSsmRunnerConfigApi { + getParameter(name: string, signal: AbortSignal): Promise; + deleteParameter(name: string, signal: AbortSignal): Promise; +} + +export class AwsSdkSsmRunnerConfigApi implements AwsSsmRunnerConfigApi { + private client?: SSMClient; + + public constructor(client?: SSMClient) { + this.client = client; + } + + private getClient(): SSMClient { + // Lifecycle hooks can be snapshotted before their first request. Constructing + // the untraced client here avoids persisting connection state in that snapshot. + this.client ??= new SSMClient({ maxAttempts: 1 }); + return this.client; + } + + public async getParameter(name: string, signal: AbortSignal): Promise { + const response = await this.getClient().send(new GetParameterCommand({ Name: name, WithDecryption: true }), { + abortSignal: signal, + }); + return response.Parameter?.Value; + } + + public async deleteParameter(name: string, signal: AbortSignal): Promise { + await this.getClient().send(new DeleteParameterCommand({ Name: name }), { abortSignal: signal }); + } +} + +export interface AwsSsmRunnerConfigConsumerOptions extends RunnerConfigPollingOptions { + api?: AwsSsmRunnerConfigApi; + deleteAttempts?: number; +} + +export function createAwsSsmRunnerConfigConsumer( + environment: { SSM_TOKEN_PATH: string }, + options: AwsSsmRunnerConfigConsumerOptions = {}, +): RunnerConfigConsumer { + return new AwsSsmRunnerConfigConsumer( + environment.SSM_TOKEN_PATH, + options.api ?? new AwsSdkSsmRunnerConfigApi(), + options, + ); +} + +class AwsSsmRunnerConfigConsumer implements RunnerConfigConsumer { + private readonly callTimeoutMs: number; + private readonly configTimeoutMs: number; + private readonly deleteAttempts: number; + private readonly pollIntervalMs: number; + + public constructor( + private readonly tokenPath: string, + private readonly api: AwsSsmRunnerConfigApi, + options: AwsSsmRunnerConfigConsumerOptions, + ) { + const polling = resolvePollingOptions(options); + this.callTimeoutMs = polling.callTimeoutMs; + this.configTimeoutMs = polling.configTimeoutMs; + this.pollIntervalMs = polling.pollIntervalMs; + this.deleteAttempts = positiveIntegerOption('deleteAttempts', options.deleteAttempts, DEFAULT_DELETE_ATTEMPTS); + } + + public async consume(runnerId: string, options: RunnerConfigConsumeOptions): Promise { + validateConsumeOptions(options); + const parameterName = composeSsmParameterName(this.tokenPath, runnerId); + const startedAt = Date.now(); + const remainingMs = Math.max(0, options.deadlineMs - startedAt); + // Preserve enough of short hook budgets for at least one bounded delete + // attempt without reviving the old fixed reserve that could consume the + // entire polling window. + const deleteReserveMs = Math.min(this.callTimeoutMs, Math.max(1, Math.floor(remainingMs / 2))); + const pollDeadline = Math.min(startedAt + this.configTimeoutMs, options.deadlineMs - deleteReserveMs); + let runnerConfig: string | undefined; + + while (Date.now() < pollDeadline) { + throwIfCancelled(options.signal); + try { + runnerConfig = await this.read(parameterName, pollDeadline, options.signal); + if (runnerConfig !== undefined) { + if (runnerConfig.length === 0) { + throw new Error('runner configuration record has an invalid value'); + } + break; + } + } catch (error) { + if (options.signal.aborted) { + throw new Error('runner configuration consumption was cancelled'); + } + if (!isSsmNotFound(error) && !isRetryableProviderError(error)) { + throw new Error('failed to read runner configuration from SSM'); + } + } + + const remaining = pollDeadline - Date.now(); + if (remaining > 0) { + await delay(Math.min(this.pollIntervalMs, remaining), options.signal); + } + } + + if (runnerConfig === undefined) { + throw new Error('runner configuration did not become available before the deadline'); + } + + await this.delete(parameterName, options); + return runnerConfig; + } + + private async read(name: string, deadlineMs: number, signal: AbortSignal): Promise { + return withCallDeadline(signal, deadlineMs, this.callTimeoutMs, (callSignal) => + this.api.getParameter(name, callSignal), + ); + } + + private async delete(name: string, options: RunnerConfigConsumeOptions): Promise { + for (let attempt = 1; attempt <= this.deleteAttempts; attempt += 1) { + try { + await withCallDeadline(options.signal, options.deadlineMs, this.callTimeoutMs, (callSignal) => + this.api.deleteParameter(name, callSignal), + ); + return; + } catch (error) { + if (options.signal.aborted) { + throw new Error('runner configuration consumption was cancelled'); + } + if (!isRetryableProviderError(error) || attempt === this.deleteAttempts) { + break; + } + + const remaining = options.deadlineMs - Date.now(); + if (remaining <= 0) { + break; + } + await delay(Math.min(2 ** (attempt - 1) * 1_000, 5_000, remaining), options.signal); + } + } + throw new Error('runner configuration could not be deleted from SSM'); + } +} + +function isSsmNotFound(error: unknown): boolean { + return errorName(error) === 'ParameterNotFound'; +} diff --git a/lambdas/libs/storage-providers/aws/ssm/runner-config-housekeeper.ts b/lambdas/libs/storage-providers/aws/ssm/runner-config-housekeeper.ts new file mode 100644 index 0000000000..a7bfdccbb0 --- /dev/null +++ b/lambdas/libs/storage-providers/aws/ssm/runner-config-housekeeper.ts @@ -0,0 +1,80 @@ +import { DeleteParameterCommand, GetParametersByPathCommand, SSMClient } from '@aws-sdk/client-ssm'; +import { getTracedAWSV3Client, logger } from '@aws-github-runner/aws-powertools-util'; + +import type { RunnerConfigHousekeeper } from '../../core'; + +export interface SSMCleanupOptions { + dryRun: boolean; + minimumDaysOld: number; + tokenPath: string; +} + +export function createAwsSsmRunnerConfigHousekeeper(options?: SSMCleanupOptions): RunnerConfigHousekeeper { + return new AwsSsmRunnerConfigHousekeeper(options ?? loadCleanupOptions()); +} + +export async function cleanSSMTokens(options: SSMCleanupOptions): Promise { + logger.info(`Cleaning tokens / JIT config older then ${options.minimumDaysOld} days, dryRun: ${options.dryRun}`); + logger.debug('Cleaning with options', { options }); + validateOptions(options); + + const client = getTracedAWSV3Client(new SSMClient({ region: process.env.AWS_REGION })); + const parameters = await client.send(new GetParametersByPathCommand({ Path: options.tokenPath })); + while (parameters.NextToken) { + const nextParameters = await client.send( + new GetParametersByPathCommand({ Path: options.tokenPath, NextToken: parameters.NextToken }), + ); + parameters.Parameters?.push(...(nextParameters.Parameters ?? [])); + parameters.NextToken = nextParameters.NextToken; + } + logger.info(`Found #${parameters.Parameters?.length} parameters in path ${options.tokenPath}`); + + const minimumDate = new Date(); + minimumDate.setDate(minimumDate.getDate() - options.minimumDaysOld); + + for (const parameter of parameters.Parameters ?? []) { + if (parameter.LastModifiedDate && new Date(parameter.LastModifiedDate) < minimumDate) { + logger.info(`Deleting parameter ${parameter.Name} with last modified date ${parameter.LastModifiedDate}`); + try { + if (!options.dryRun) { + await new Promise((resolve) => setTimeout(resolve, 50)); + await client.send(new DeleteParameterCommand({ Name: parameter.Name })); + } + } catch (error) { + logger.warn(`Failed to delete parameter ${parameter.Name} with error ${(error as Error).message}`); + logger.debug('Failed to delete parameter', { error }); + } + } else { + logger.debug(`Skipping parameter ${parameter.Name} with last modified date ${parameter.LastModifiedDate}`); + } + } +} + +class AwsSsmRunnerConfigHousekeeper implements RunnerConfigHousekeeper { + constructor(private readonly options: SSMCleanupOptions) {} + + houseKeeper(): Promise { + return cleanSSMTokens(this.options); + } +} + +function loadCleanupOptions(): SSMCleanupOptions { + const value = process.env.SSM_CLEANUP_CONFIG; + if (!value || value.trim() === '') { + throw new Error('Environment variable SSM_CLEANUP_CONFIG is not set'); + } + return JSON.parse(value) as SSMCleanupOptions; +} + +function validateOptions(options: SSMCleanupOptions): void { + const errorMessages: string[] = []; + if (!options.minimumDaysOld || options.minimumDaysOld < 1) { + errorMessages.push(`minimumDaysOld must be greater then 0, value is set to "${options.minimumDaysOld}"`); + } + if (!options.tokenPath) { + errorMessages.push('tokenPath must be defined'); + } + if (errorMessages.length > 0) { + throw new Error(errorMessages.join(', ')); + } +} diff --git a/lambdas/libs/storage-providers/aws/ssm/runner-config-store.ts b/lambdas/libs/storage-providers/aws/ssm/runner-config-store.ts index 928e5f8f3d..80941822a5 100644 --- a/lambdas/libs/storage-providers/aws/ssm/runner-config-store.ts +++ b/lambdas/libs/storage-providers/aws/ssm/runner-config-store.ts @@ -4,12 +4,20 @@ import type { RunnerConfigMetadata, RunnerConfigRecord, RunnerConfigStore } from import type {} from './environment'; import { loadSsmParameterStoreTagsFromEnvironment } from './parameter-store-tags'; -interface AwsSsmRunnerConfigStoreConfig { +export interface AwsSsmRunnerConfigStoreConfig { tokenPath: string; - parameterStoreTags: { Key: string; Value: string }[]; + parameterStoreTags: ReadonlyArray>; } -export function createAwsSsmRunnerConfigStore(): RunnerConfigStore { +export function createAwsSsmRunnerConfigStore(config?: AwsSsmRunnerConfigStoreConfig): RunnerConfigStore { + if (config) { + return new AwsSsmRunnerConfigStore( + Object.freeze({ + ...config, + parameterStoreTags: Object.freeze(config.parameterStoreTags.map((tag) => Object.freeze({ ...tag }))), + }), + ); + } const tokenPath = process.env.SSM_TOKEN_PATH; if (!tokenPath || tokenPath.trim() === '') { throw new Error('Environment variable SSM_TOKEN_PATH is not set'); diff --git a/lambdas/libs/storage-providers/aws/ssm/runner-group-cache-store.ts b/lambdas/libs/storage-providers/aws/ssm/runner-group-cache-store.ts index d451851e09..6e16a84456 100644 --- a/lambdas/libs/storage-providers/aws/ssm/runner-group-cache-store.ts +++ b/lambdas/libs/storage-providers/aws/ssm/runner-group-cache-store.ts @@ -4,12 +4,20 @@ import type { RunnerGroupCacheRecord, RunnerGroupCacheStore } from '../../core'; import type {} from './environment'; import { loadSsmParameterStoreTagsFromEnvironment } from './parameter-store-tags'; -interface AwsSsmRunnerGroupCacheStoreConfig { +export interface AwsSsmRunnerGroupCacheStoreConfig { configPath: string; - parameterStoreTags: { Key: string; Value: string }[]; + parameterStoreTags: ReadonlyArray>; } -export function createAwsSsmRunnerGroupCacheStore(): RunnerGroupCacheStore { +export function createAwsSsmRunnerGroupCacheStore(config?: AwsSsmRunnerGroupCacheStoreConfig): RunnerGroupCacheStore { + if (config) { + return new AwsSsmRunnerGroupCacheStore( + Object.freeze({ + ...config, + parameterStoreTags: Object.freeze(config.parameterStoreTags.map((tag) => Object.freeze({ ...tag }))), + }), + ); + } const configPath = process.env.SSM_CONFIG_PATH; if (!configPath || configPath.trim() === '') { throw new Error('Environment variable SSM_CONFIG_PATH is not set'); @@ -42,7 +50,7 @@ class AwsSsmRunnerGroupCacheStore implements RunnerGroupCacheStore { async create(record: RunnerGroupCacheRecord): Promise { await putParameter(this.parameterName(record.runnerGroupName), record.runnerGroupId.toString(), false, { - tags: this.config.parameterStoreTags, + tags: [...this.config.parameterStoreTags], }); } diff --git a/lambdas/libs/storage-providers/core/index.ts b/lambdas/libs/storage-providers/core/index.ts index 25d9f23f83..27339a2159 100644 --- a/lambdas/libs/storage-providers/core/index.ts +++ b/lambdas/libs/storage-providers/core/index.ts @@ -13,6 +13,42 @@ export interface RunnerConfigStore { create(record: RunnerConfigRecord, options?: { metadata?: RunnerConfigMetadata[] }): Promise; } +export interface RunnerConfigHousekeeper { + houseKeeper(): Promise; +} + +export interface GitHubAppCredential { + appId: number; + privateKey: string; + installationId?: number; +} + +export interface GitHubAppCredentialsStore { + get(): Promise; +} + +export interface RunnerConfigConsumeOptions { + /** Absolute Unix time in milliseconds after which the operation must stop. */ + deadlineMs: number; + signal: AbortSignal; +} + +export interface RunnerConfigConsumer { + consume(runnerId: string, options: RunnerConfigConsumeOptions): Promise; +} + +export interface RunnerConfigStorage { + runnerConfig: RunnerConfigStore; + runnerGroupCache: RunnerGroupCacheStore; + consumer: RunnerConfigConsumer; +} + +export interface CommonStorage { + githubAppCredentials: GitHubAppCredentialsStore; +} + +export type StorageProviders = RunnerConfigStorage & CommonStorage; + export interface RunnerGroupCacheRecord { runnerGroupName: string; runnerGroupId: number; diff --git a/lambdas/libs/storage-providers/index.ts b/lambdas/libs/storage-providers/index.ts index ed76bccc68..91aea6910d 100644 --- a/lambdas/libs/storage-providers/index.ts +++ b/lambdas/libs/storage-providers/index.ts @@ -1,9 +1,18 @@ export type { + GitHubAppCredential, + GitHubAppCredentialsStore, + RunnerConfigConsumer, + RunnerConfigConsumeOptions, + RunnerConfigHousekeeper, RunnerConfigMetadata, RunnerConfigRecord, RunnerConfigStore, RunnerGroupCacheRecord, RunnerGroupCacheStore, } from './core'; -export { createRunnerConfigStore } from './runner-config'; -export { createRunnerGroupCacheStore } from './runner-group-cache'; +export { createRunnerConfigHousekeeper } from './runner-config-housekeeper'; +export { createRunnerConfigConsumer, type RunnerConfigConsumerConfig } from './runner-config-consumer'; +export { resolveRunnerConfigStorageProvider, runnerConfigStorageProviders } from './provider'; +export type { RunnerConfigStorageProvider } from './provider'; +export { createCommonStorage, createStorageProviders } from './storage-providers'; +export type { StorageProviders, RunnerConfigStorage, CommonStorage } from './core'; diff --git a/lambdas/libs/storage-providers/package.json b/lambdas/libs/storage-providers/package.json index 65e93c2c08..330d60a3f6 100644 --- a/lambdas/libs/storage-providers/package.json +++ b/lambdas/libs/storage-providers/package.json @@ -3,7 +3,9 @@ "version": "1.0.0", "main": "index.ts", "exports": { - ".": "./index.ts" + ".": "./index.ts", + "./aws/ssm/runner-config-housekeeper": "./aws/ssm/runner-config-housekeeper.ts", + "./runner-config-consumer": "./runner-config-consumer.ts" }, "type": "module", "license": "MIT", diff --git a/lambdas/libs/storage-providers/provider.test.ts b/lambdas/libs/storage-providers/provider.test.ts new file mode 100644 index 0000000000..173b0199a5 --- /dev/null +++ b/lambdas/libs/storage-providers/provider.test.ts @@ -0,0 +1,15 @@ +import { describe, expect, it } from 'vitest'; + +import { resolveRunnerConfigStorageProvider } from './provider'; + +describe('resolveRunnerConfigStorageProvider', () => { + it.each([undefined, '', ' ', 'aws_ssm', 'AWS_SSM'])('resolves %j to aws_ssm', (value) => { + expect(resolveRunnerConfigStorageProvider(value)).toBe('aws_ssm'); + }); + + it.each([null, 'dynamodb', 'aws-ssm'])('rejects unsupported providers: %j', (value) => { + expect(() => resolveRunnerConfigStorageProvider(value)).toThrow( + `Unsupported runner config storage provider '${value}'`, + ); + }); +}); diff --git a/lambdas/libs/storage-providers/provider.ts b/lambdas/libs/storage-providers/provider.ts new file mode 100644 index 0000000000..1ade3e414c --- /dev/null +++ b/lambdas/libs/storage-providers/provider.ts @@ -0,0 +1,16 @@ +export const runnerConfigStorageProviders = ['aws_ssm'] as const; + +export type RunnerConfigStorageProvider = (typeof runnerConfigStorageProviders)[number]; + +export function resolveRunnerConfigStorageProvider(value: unknown): RunnerConfigStorageProvider { + if (value === undefined || (typeof value === 'string' && value.trim() === '')) { + return 'aws_ssm'; + } + if ( + typeof value !== 'string' || + !runnerConfigStorageProviders.includes(value.trim().toLowerCase() as RunnerConfigStorageProvider) + ) { + throw new Error(`Unsupported runner config storage provider '${String(value)}'`); + } + return value.trim().toLowerCase() as RunnerConfigStorageProvider; +} diff --git a/lambdas/libs/storage-providers/runner-config-consumer-subpath.test.ts b/lambdas/libs/storage-providers/runner-config-consumer-subpath.test.ts new file mode 100644 index 0000000000..885893fa79 --- /dev/null +++ b/lambdas/libs/storage-providers/runner-config-consumer-subpath.test.ts @@ -0,0 +1,31 @@ +import { + createRunnerConfigConsumerFromEnvironment, + exportRunnerConfigStorageEnvironment, + loadRunnerConfigStorageContextFromEnvironment, + parseRunnerConfigStorageContext, + type RunnerConfigConsumeOptions, + type RunnerConfigConsumer, + type RunnerConfigStorageContext, + type RunnerConfigStorageEnvironment, +} from '@aws-github-runner/storage-providers/runner-config-consumer'; +import { describe, expect, it } from 'vitest'; + +describe('runner config consumer package subpath', () => { + it('exposes the portable environment round-trip and consumer contract', () => { + const context: RunnerConfigStorageContext = { + RUNNER_CONFIG_STORAGE_PROVIDER: 'aws_ssm', + SSM_TOKEN_PATH: '/runner/tokens', + }; + const options: RunnerConfigConsumeOptions = { + deadlineMs: Date.now() + 1_000, + signal: new AbortController().signal, + }; + const exported: RunnerConfigStorageEnvironment = exportRunnerConfigStorageEnvironment(context); + const consumerFactory: () => RunnerConfigConsumer = createRunnerConfigConsumerFromEnvironment; + + expect(parseRunnerConfigStorageContext(context)).toEqual(context); + expect(loadRunnerConfigStorageContextFromEnvironment(exported)).toEqual(context); + expect(consumerFactory).toBeTypeOf('function'); + expect(options.signal.aborted).toBe(false); + }); +}); diff --git a/lambdas/libs/storage-providers/runner-config-consumer.test.ts b/lambdas/libs/storage-providers/runner-config-consumer.test.ts new file mode 100644 index 0000000000..f51f067589 --- /dev/null +++ b/lambdas/libs/storage-providers/runner-config-consumer.test.ts @@ -0,0 +1,174 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import { AwsSdkSsmRunnerConfigApi, type AwsSsmRunnerConfigApi } from './aws/ssm/runner-config-consumer'; +import { + createRunnerConfigConsumerFromEnvironment, + exportRunnerConfigStorageEnvironment, + loadRunnerConfigConsumerConfigFromEnvironment, + loadRunnerConfigStorageContextFromEnvironment, + parseRunnerConfigStorageContext, + runnerConfigStorageEnvironment, +} from './runner-config-consumer'; + +const ssmContext = { + RUNNER_CONFIG_STORAGE_PROVIDER: 'aws_ssm', + SSM_TOKEN_PATH: '/runner/tokens', +} as const; +describe('runner config storage context', () => { + it('parses and freezes an exact SSM environment map while canonicalizing one trailing slash', () => { + const context = parseRunnerConfigStorageContext({ ...ssmContext, SSM_TOKEN_PATH: '/runner/tokens/' }); + + expect(context).toEqual(ssmContext); + expect(Object.isFrozen(context)).toBe(true); + expect(runnerConfigStorageEnvironment(context)).toEqual(ssmContext); + }); + + it.each([ + null, + [], + 'aws_ssm', + { RUNNER_CONFIG_STORAGE_PROVIDER: 'AWS_SSM', SSM_TOKEN_PATH: '/runner/tokens' }, + { RUNNER_CONFIG_STORAGE_PROVIDER: 'aws_ssm' }, + { ...ssmContext, unexpected: true }, + { ...ssmContext, AWS_ACCESS_KEY_ID: 'payload-must-not-export-credentials' }, + { ...ssmContext, RUNNER_CONFIG_TIMEOUT_SECONDS: '60' }, + { ...ssmContext, RUNNER_CONFIG_DYNAMODB_RUNNER_STATE_TABLE_NAME: 'runner-state' }, + { ...ssmContext, SSM_TOKEN_PATH: '/runner//tokens' }, + { ...ssmContext, SSM_TOKEN_PATH: '/runner/tokens//' }, + { ...ssmContext, SSM_TOKEN_PATH: '/awsParameters/tokens' }, + { ...ssmContext, SSM_TOKEN_PATH: '/ssm-private/tokens' }, + { ...ssmContext, SSM_TOKEN_PATH: '/runner/../tokens' }, + { RUNNER_CONFIG_STORAGE_PROVIDER: 'aws_dynamodb', SSM_TOKEN_PATH: '/runner/tokens' }, + { provider: 'aws_dynamodb', tableName: 'runner-state' }, + ])('rejects a non-allowlisted or incomplete storage context %#', (value) => { + expect(() => parseRunnerConfigStorageContext(value)).toThrow(); + }); + + it('rejects symbol fields that would be hidden by JSON-style key enumeration', () => { + const context = { ...ssmContext }; + Object.defineProperty(context, Symbol('unexpected'), { value: true }); + + expect(() => parseRunnerConfigStorageContext(context)).toThrow('storage context is invalid'); + }); + + it.each([ + [ + { + ...ssmContext, + AWS_ACCESS_KEY_ID: 'producer-only', + RUNNER_CONFIG_TIMEOUT_SECONDS: '30', + UNRELATED: 'kept-out', + }, + ssmContext, + ], + ])('selects only the chosen provider locator from a broader producer environment %#', (environment, expected) => { + expect(loadRunnerConfigStorageContextFromEnvironment(environment)).toEqual(expected); + }); + + it('defaults a legacy producer environment with only SSM_TOKEN_PATH to aws_ssm', () => { + expect(loadRunnerConfigStorageContextFromEnvironment({ SSM_TOKEN_PATH: '/runner/tokens' })).toEqual(ssmContext); + }); + + it('round-trips each producer environment through payload context and hook environment export', () => { + for (const producerEnvironment of [ssmContext]) { + const payloadContext = loadRunnerConfigStorageContextFromEnvironment(producerEnvironment); + const exported = exportRunnerConfigStorageEnvironment(payloadContext); + expect(exported).toEqual(payloadContext); + expect(loadRunnerConfigStorageContextFromEnvironment(exported)).toEqual(payloadContext); + } + }); + + it('rejects unexpected context keys', () => { + expect(() => exportRunnerConfigStorageEnvironment({ ...ssmContext, unexpected: 'forbidden' } as never)).toThrow(); + }); +}); + +describe('runner config consumer environment factory', () => { + afterEach(() => { + vi.useRealTimers(); + vi.restoreAllMocks(); + }); + + it('creates an injected SSM consumer from exported environment', async () => { + const api: AwsSsmRunnerConfigApi = { + getParameter: vi.fn().mockResolvedValue('encoded-jit'), + deleteParameter: vi.fn().mockResolvedValue(undefined), + }; + const consumer = createRunnerConfigConsumerFromEnvironment(ssmContext, { + awsSsmApi: api, + callTimeoutMs: 100, + configTimeoutMs: 100, + pollIntervalMs: 1, + }); + + await expect( + consumer.consume('microvm-123', { + deadlineMs: Date.now() + 1_000, + signal: new AbortController().signal, + }), + ).resolves.toBe('encoded-jit'); + expect(api.getParameter).toHaveBeenCalledWith('/runner/tokens/microvm-123', expect.any(AbortSignal)); + expect(api.deleteParameter).toHaveBeenCalledWith('/runner/tokens/microvm-123', expect.any(AbortSignal)); + }); + + it('loads timing defaults and overrides from the supplied factory environment', async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-01-01T00:00:00.000Z')); + vi.spyOn(AwsSdkSsmRunnerConfigApi.prototype, 'getParameter').mockResolvedValue(undefined); + const consumer = createRunnerConfigConsumerFromEnvironment( + { + ...ssmContext, + RUNNER_CONFIG_TIMEOUT_SECONDS: '1', + RUNNER_CONFIG_POLL_SECONDS: '1', + }, + undefined, + ); + const startedAt = Date.now(); + const pending = consumer.consume('microvm-123', { + deadlineMs: startedAt + 10_000, + signal: new AbortController().signal, + }); + const rejection = expect(pending).rejects.toThrow( + 'runner configuration did not become available before the deadline', + ); + + await vi.runAllTimersAsync(); + + await rejection; + expect(Date.now() - startedAt).toBe(1_000); + }); + + it('loads source-compatible timing defaults', () => { + expect(loadRunnerConfigConsumerConfigFromEnvironment({})).toEqual({ + callTimeoutMs: 5_000, + configTimeoutMs: 20_000, + deleteAttempts: 3, + pollIntervalMs: 2_000, + }); + }); + + it('loads bounded timing overrides from an environment', () => { + expect( + loadRunnerConfigConsumerConfigFromEnvironment({ + AWS_SDK_CALL_TIMEOUT_SECONDS: '7', + RUNNER_CONFIG_TIMEOUT_SECONDS: '31', + RUNNER_CONFIG_DELETE_ATTEMPTS: '4', + RUNNER_CONFIG_POLL_SECONDS: '3', + }), + ).toEqual({ + callTimeoutMs: 7_000, + configTimeoutMs: 31_000, + deleteAttempts: 4, + pollIntervalMs: 3_000, + }); + }); + + it.each([ + ['AWS_SDK_CALL_TIMEOUT_SECONDS', '0', 'callTimeoutMs', 5_000], + ['RUNNER_CONFIG_TIMEOUT_SECONDS', '61', 'configTimeoutMs', 20_000], + ['RUNNER_CONFIG_DELETE_ATTEMPTS', '11', 'deleteAttempts', 3], + ['RUNNER_CONFIG_POLL_SECONDS', 'not-a-number', 'pollIntervalMs', 2_000], + ])('falls back for invalid %s=%j', (name, value, property, expected) => { + expect(loadRunnerConfigConsumerConfigFromEnvironment({ [name]: value })).toHaveProperty(property, expected); + }); +}); diff --git a/lambdas/libs/storage-providers/runner-config-consumer.ts b/lambdas/libs/storage-providers/runner-config-consumer.ts new file mode 100644 index 0000000000..fa7732fc4b --- /dev/null +++ b/lambdas/libs/storage-providers/runner-config-consumer.ts @@ -0,0 +1,128 @@ +import { createAwsSsmRunnerConfigConsumer, type AwsSsmRunnerConfigApi } from './aws/ssm/runner-config-consumer'; +import type { RunnerConfigConsumer } from './core'; +import { canonicalSsmTokenPath, type RunnerConfigPollingOptions } from './aws/ssm/runner-config-consumer-common'; + +export type { RunnerConfigConsumeOptions, RunnerConfigConsumer } from './core'; +export type { AwsSsmRunnerConfigApi } from './aws/ssm/runner-config-consumer'; + +type Environment = Readonly>; + +export interface RunnerConfigConsumerConfig extends RunnerConfigPollingOptions { + awsSsmApi?: AwsSsmRunnerConfigApi; + deleteAttempts?: number; +} + +export interface RunnerConfigStorageContext { + RUNNER_CONFIG_STORAGE_PROVIDER: 'aws_ssm'; + SSM_TOKEN_PATH: string; +} + +export type RunnerConfigStorageEnvironment = RunnerConfigStorageContext; + +/** Creates the consumer from one immutable snapshot of the producer environment. */ +export function createRunnerConfigConsumer( + environment: Environment = process.env, + config?: RunnerConfigConsumerConfig, +): RunnerConfigConsumer { + const tokenPath = canonicalSsmTokenPath(requireTokenPath(environment)); + const resolvedConfig = config ?? loadRunnerConfigConsumerConfigFromEnvironment(environment); + return createAwsSsmRunnerConfigConsumer( + { SSM_TOKEN_PATH: tokenPath }, + { + api: resolvedConfig.awsSsmApi, + callTimeoutMs: resolvedConfig.callTimeoutMs, + configTimeoutMs: resolvedConfig.configTimeoutMs, + deleteAttempts: resolvedConfig.deleteAttempts, + pollIntervalMs: resolvedConfig.pollIntervalMs, + }, + ); +} + +/** @deprecated Use createRunnerConfigConsumer. */ +export const createRunnerConfigConsumerFromEnvironment = createRunnerConfigConsumer; + +export function parseRunnerConfigStorageContext(value: unknown): RunnerConfigStorageContext { + if (!isPlainObject(value) || value.RUNNER_CONFIG_STORAGE_PROVIDER !== 'aws_ssm') { + throw new Error('runner configuration storage context is invalid'); + } + if ( + !hasExactKeys(value, ['RUNNER_CONFIG_STORAGE_PROVIDER', 'SSM_TOKEN_PATH']) || + typeof value.SSM_TOKEN_PATH !== 'string' + ) { + throw new Error('aws_ssm runner configuration storage context is invalid'); + } + return Object.freeze({ + RUNNER_CONFIG_STORAGE_PROVIDER: 'aws_ssm', + SSM_TOKEN_PATH: canonicalSsmTokenPath(value.SSM_TOKEN_PATH), + }); +} + +export function loadRunnerConfigStorageContextFromEnvironment( + environment: Environment = process.env, +): RunnerConfigStorageContext { + return parseRunnerConfigStorageContext({ + RUNNER_CONFIG_STORAGE_PROVIDER: 'aws_ssm', + SSM_TOKEN_PATH: environment.SSM_TOKEN_PATH, + }); +} + +export function runnerConfigStorageEnvironment(context: RunnerConfigStorageContext): RunnerConfigStorageEnvironment { + return parseRunnerConfigStorageContext(context); +} + +/** Returns an allowlisted environment object without mutating process.env or a caller-owned target. */ +export function exportRunnerConfigStorageEnvironment( + context: RunnerConfigStorageContext, +): RunnerConfigStorageEnvironment { + return runnerConfigStorageEnvironment(context); +} + +export function loadRunnerConfigConsumerConfigFromEnvironment( + environment: Environment = process.env, +): RunnerConfigConsumerConfig { + return { + callTimeoutMs: secondsEnvironmentValue(environment, 'AWS_SDK_CALL_TIMEOUT_SECONDS', 5) * 1_000, + configTimeoutMs: secondsEnvironmentValue(environment, 'RUNNER_CONFIG_TIMEOUT_SECONDS', 20) * 1_000, + deleteAttempts: positiveIntegerEnvironmentValue(environment, 'RUNNER_CONFIG_DELETE_ATTEMPTS', 3, 10), + pollIntervalMs: secondsEnvironmentValue(environment, 'RUNNER_CONFIG_POLL_SECONDS', 2) * 1_000, + }; +} + +function requireTokenPath(environment: Environment): string { + const tokenPath = environment.SSM_TOKEN_PATH; + if (!tokenPath || tokenPath.trim() === '') { + throw new Error('Environment variable SSM_TOKEN_PATH is not set'); + } + return tokenPath; +} + +function secondsEnvironmentValue(environment: Environment, name: string, fallback: number): number { + return positiveIntegerEnvironmentValue(environment, name, fallback, 60); +} + +function positiveIntegerEnvironmentValue( + environment: Environment, + name: string, + fallback: number, + maximum: number, +): number { + const value = environment[name]; + if (value === undefined || !/^\d+$/.test(value)) { + return fallback; + } + const parsed = Number(value); + return Number.isSafeInteger(parsed) && parsed > 0 && parsed <= maximum ? parsed : fallback; +} + +function isPlainObject(value: unknown): value is Record { + if (value === null || typeof value !== 'object' || Array.isArray(value)) { + return false; + } + const prototype = Object.getPrototypeOf(value) as unknown; + return prototype === Object.prototype || prototype === null; +} + +function hasExactKeys(value: object, expected: readonly string[]): boolean { + const keys = Reflect.ownKeys(value); + return keys.length === expected.length && keys.every((key) => typeof key === 'string' && expected.includes(key)); +} diff --git a/lambdas/libs/storage-providers/runner-config-housekeeper.ts b/lambdas/libs/storage-providers/runner-config-housekeeper.ts new file mode 100644 index 0000000000..040cd1b801 --- /dev/null +++ b/lambdas/libs/storage-providers/runner-config-housekeeper.ts @@ -0,0 +1,6 @@ +import { createAwsSsmRunnerConfigHousekeeper } from './aws/ssm/runner-config-housekeeper'; +import type { RunnerConfigHousekeeper } from './core'; + +export function createRunnerConfigHousekeeper(): RunnerConfigHousekeeper { + return createAwsSsmRunnerConfigHousekeeper(); +} diff --git a/lambdas/libs/storage-providers/runner-config.test.ts b/lambdas/libs/storage-providers/runner-config.test.ts deleted file mode 100644 index 4ae4bd6351..0000000000 --- a/lambdas/libs/storage-providers/runner-config.test.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { describe, expect, it, vi } from 'vitest'; - -import { createAwsSsmRunnerConfigStore } from './aws/ssm/runner-config-store'; -import { createRunnerConfigStore } from './runner-config'; - -vi.mock('./aws/ssm/runner-config-store', () => ({ - createAwsSsmRunnerConfigStore: vi.fn(), -})); - -describe('runner config store factory', () => { - it('creates the SSM implementation while the provider seam is being introduced', () => { - const store = { create: vi.fn() }; - vi.mocked(createAwsSsmRunnerConfigStore).mockReturnValue(store); - - expect(createRunnerConfigStore()).toBe(store); - expect(createAwsSsmRunnerConfigStore).toHaveBeenCalledOnce(); - }); -}); diff --git a/lambdas/libs/storage-providers/runner-config.ts b/lambdas/libs/storage-providers/runner-config.ts deleted file mode 100644 index 65fe0e1ee3..0000000000 --- a/lambdas/libs/storage-providers/runner-config.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { createAwsSsmRunnerConfigStore } from './aws/ssm/runner-config-store'; -import type { RunnerConfigStore } from './core'; -import type {} from './environment'; - -export function createRunnerConfigStore(): RunnerConfigStore { - return createAwsSsmRunnerConfigStore(); -} diff --git a/lambdas/libs/storage-providers/runner-group-cache.ts b/lambdas/libs/storage-providers/runner-group-cache.ts deleted file mode 100644 index 40721df5b6..0000000000 --- a/lambdas/libs/storage-providers/runner-group-cache.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { createAwsSsmRunnerGroupCacheStore } from './aws/ssm/runner-group-cache-store'; -import type { RunnerGroupCacheStore } from './core'; - -export function createRunnerGroupCacheStore(): RunnerGroupCacheStore { - return createAwsSsmRunnerGroupCacheStore(); -} diff --git a/lambdas/libs/storage-providers/storage-providers.test.ts b/lambdas/libs/storage-providers/storage-providers.test.ts new file mode 100644 index 0000000000..f1b01ee66b --- /dev/null +++ b/lambdas/libs/storage-providers/storage-providers.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it, vi } from 'vitest'; + +import { createStorageProviders } from './storage-providers'; + +vi.mock('./aws/ssm/runner-config-store', () => ({ + createAwsSsmRunnerConfigStore: vi.fn(() => ({ create: vi.fn() })), +})); +vi.mock('./aws/ssm/runner-group-cache-store', () => ({ + createAwsSsmRunnerGroupCacheStore: vi.fn(() => ({ get: vi.fn(), create: vi.fn() })), +})); +vi.mock('./aws/ssm/runner-config-consumer', () => ({ + createAwsSsmRunnerConfigConsumer: vi.fn(() => ({ consume: vi.fn() })), +})); +vi.mock('./aws/ssm/github-app-credentials-store', () => ({ + createAwsSsmGitHubAppCredentialsStore: vi.fn(() => ({ get: vi.fn() })), +})); + +describe('createStorageProviders', () => { + it('parses environment once and composes independent runner and common capabilities', () => { + const environment = Object.freeze({ + RUNNER_CONFIG_STORAGE_PROVIDER: 'AWS_SSM', + SSM_TOKEN_PATH: '/runners/tokens', + SSM_CONFIG_PATH: '/runners/config', + SSM_PARAMETER_STORE_TAGS: JSON.stringify([{ Key: 'Environment', Value: 'test' }]), + PARAMETER_GITHUB_APP_ID_NAME: 'app-id', + PARAMETER_GITHUB_APP_KEY_BASE64_NAME: 'app-key', + AWS_SDK_CALL_TIMEOUT_SECONDS: '7', + RUNNER_CONFIG_TIMEOUT_SECONDS: '11', + RUNNER_CONFIG_POLL_SECONDS: '2', + RUNNER_CONFIG_DELETE_ATTEMPTS: '4', + }); + + const storage = createStorageProviders(environment); + + expect(storage).toEqual({ + runnerConfig: expect.any(Object), + runnerGroupCache: expect.any(Object), + consumer: expect.any(Object), + githubAppCredentials: expect.any(Object), + }); + }); +}); diff --git a/lambdas/libs/storage-providers/storage-providers.ts b/lambdas/libs/storage-providers/storage-providers.ts new file mode 100644 index 0000000000..2cbe89ed57 --- /dev/null +++ b/lambdas/libs/storage-providers/storage-providers.ts @@ -0,0 +1,42 @@ +import { createAwsSsmGitHubAppCredentialsStore } from './aws/ssm/github-app-credentials-store'; +import { createAwsSsmRunnerConfigConsumer } from './aws/ssm/runner-config-consumer'; +import { createAwsSsmRunnerConfigStore } from './aws/ssm/runner-config-store'; +import { createAwsSsmRunnerGroupCacheStore } from './aws/ssm/runner-group-cache-store'; +import type { CommonStorage, StorageProviders } from './core'; +import { loadRunnerConfigConsumerConfigFromEnvironment } from './runner-config-consumer'; +import { loadSsmParameterStoreTagsFromEnvironment } from './aws/ssm/parameter-store-tags'; +import { resolveRunnerConfigStorageProvider } from './provider'; + +type Environment = Readonly>; + +export function createStorageProviders(environment: Environment = process.env): StorageProviders { + const provider = resolveRunnerConfigStorageProvider(environment.RUNNER_CONFIG_STORAGE_PROVIDER); + if (provider !== 'aws_ssm') { + throw new Error(`Unsupported runner config storage provider '${provider}'`); + } + + const tokenPath = required(environment.SSM_TOKEN_PATH, 'SSM_TOKEN_PATH'); + const configPath = required(environment.SSM_CONFIG_PATH, 'SSM_CONFIG_PATH'); + const parameterStoreTags = loadSsmParameterStoreTagsFromEnvironment(environment); + const consumerConfig = loadRunnerConfigConsumerConfigFromEnvironment(environment); + + return { + runnerConfig: createAwsSsmRunnerConfigStore({ tokenPath, parameterStoreTags }), + runnerGroupCache: createAwsSsmRunnerGroupCacheStore({ configPath, parameterStoreTags }), + consumer: createAwsSsmRunnerConfigConsumer({ SSM_TOKEN_PATH: tokenPath }, consumerConfig), + ...createCommonStorage(environment), + }; +} + +export function createCommonStorage(environment: Environment = process.env): CommonStorage { + return { + githubAppCredentials: createAwsSsmGitHubAppCredentialsStore(environment), + }; +} + +function required(value: string | undefined, name: string): string { + if (!value || value.trim() === '') { + throw new Error(`Environment variable ${name} is not set`); + } + return value; +} diff --git a/lambdas/libs/storage-providers/vitest.config.ts b/lambdas/libs/storage-providers/vitest.config.ts index a5812ad13e..20c739f253 100644 --- a/lambdas/libs/storage-providers/vitest.config.ts +++ b/lambdas/libs/storage-providers/vitest.config.ts @@ -7,7 +7,15 @@ export default mergeConfig(defaultConfig, { test: { setupFiles: [resolve(__dirname, '../../aws-vitest-setup.ts')], coverage: { - include: ['index.ts', 'runner-config.ts', 'core/**/*.ts', 'aws/**/*.ts'], + include: [ + 'index.ts', + 'runner-config-housekeeper.ts', + 'runner-config-consumer.ts', + 'storage-providers.ts', + 'provider.ts', + 'core/**/*.ts', + 'aws/**/*.ts', + ], exclude: ['**/*.test.ts', '**/*.d.ts'], }, }, diff --git a/lambdas/yarn.lock b/lambdas/yarn.lock index 0e7daca0c7..4f313f5413 100644 --- a/lambdas/yarn.lock +++ b/lambdas/yarn.lock @@ -139,6 +139,7 @@ __metadata: "@aws-github-runner/aws-ssm-util": "npm:*" "@aws-github-runner/storage-providers": "npm:*" "@aws-sdk/client-ec2": "npm:^3.1009.0" + "@aws-sdk/client-lambda-microvms": "npm:^3.1074.0" "@octokit/rest": "npm:22.0.1" aws-sdk-client-mock: "npm:^4.1.0" aws-sdk-client-mock-jest: "npm:^4.1.0" @@ -443,6 +444,22 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/client-lambda-microvms@npm:^3.1074.0": + version: 3.1104.0 + resolution: "@aws-sdk/client-lambda-microvms@npm:3.1104.0" + dependencies: + "@aws-sdk/core": "npm:^3.977.6" + "@aws-sdk/credential-provider-node": "npm:^3.972.78" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/fetch-http-handler": "npm:^5.6.13" + "@smithy/node-http-handler": "npm:^4.9.13" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/219ad52f822def4caa4a20d8d91d46a1b78e6726363a145be86c37cdeef4e4c13653e8a59ada67154146c6c2554e2c12944efad35c689850bf6f72f2d55246f4 + languageName: node + linkType: hard + "@aws-sdk/client-s3@npm:^3.1009.0": version: 3.1014.0 resolution: "@aws-sdk/client-s3@npm:3.1014.0" @@ -624,6 +641,22 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/core@npm:^3.977.6": + version: 3.977.6 + resolution: "@aws-sdk/core@npm:3.977.6" + dependencies: + "@aws-sdk/types": "npm:^3.974.2" + "@aws-sdk/xml-builder": "npm:^3.972.37" + "@aws/lambda-invoke-store": "npm:^0.3.0" + "@smithy/core": "npm:^3.31.1" + "@smithy/signature-v4": "npm:^5.6.12" + "@smithy/types": "npm:^4.16.1" + bowser: "npm:^2.11.0" + tslib: "npm:^2.6.2" + checksum: 10c0/4d743603bb41aeed426e2928be0947202191c341f9fbefe9ea347b0b4b7154b1ea94189d01c8abf3b03b9635449e2e7c268bd67379ea2294b9f49a61b909b9af + languageName: node + linkType: hard + "@aws-sdk/crc64-nvme@npm:^3.972.5": version: 3.972.5 resolution: "@aws-sdk/crc64-nvme@npm:3.972.5" @@ -647,6 +680,19 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-env@npm:^3.972.67": + version: 3.972.67 + resolution: "@aws-sdk/credential-provider-env@npm:3.972.67" + dependencies: + "@aws-sdk/core": "npm:^3.977.6" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/547bcac01ac0912d0e42bb11f7d51bafcf2eaab1db35a098bea2be322211a86457ea60455a5294e58081c32376c240b07e66f81946a9be30e9722f723c6eaac2 + languageName: node + linkType: hard + "@aws-sdk/credential-provider-http@npm:^3.972.23": version: 3.972.23 resolution: "@aws-sdk/credential-provider-http@npm:3.972.23" @@ -665,6 +711,21 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-http@npm:^3.972.69": + version: 3.972.69 + resolution: "@aws-sdk/credential-provider-http@npm:3.972.69" + dependencies: + "@aws-sdk/core": "npm:^3.977.6" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/fetch-http-handler": "npm:^5.6.13" + "@smithy/node-http-handler": "npm:^4.9.13" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/6e4cf9628919163a2a9784bf8618bc85a8c0ba7056813bedb9758c04eb3b36663f5099cfad329f89ac86c4e408bb3d0698ee7cff7e4a61c8a0335ab98078d678 + languageName: node + linkType: hard + "@aws-sdk/credential-provider-ini@npm:^3.972.23": version: 3.972.23 resolution: "@aws-sdk/credential-provider-ini@npm:3.972.23" @@ -687,6 +748,27 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-ini@npm:^3.973.12": + version: 3.973.12 + resolution: "@aws-sdk/credential-provider-ini@npm:3.973.12" + dependencies: + "@aws-sdk/core": "npm:^3.977.6" + "@aws-sdk/credential-provider-env": "npm:^3.972.67" + "@aws-sdk/credential-provider-http": "npm:^3.972.69" + "@aws-sdk/credential-provider-login": "npm:^3.972.74" + "@aws-sdk/credential-provider-process": "npm:^3.972.67" + "@aws-sdk/credential-provider-sso": "npm:^3.973.11" + "@aws-sdk/credential-provider-web-identity": "npm:^3.972.73" + "@aws-sdk/nested-clients": "npm:^3.997.41" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/credential-provider-imds": "npm:^4.4.16" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/84646fee1c61e31b2052d902559ecf163c1d00558ecdc21d77b396250527348b9ebba324d0bf8ffee4b3e45476c691de502e6faad3d39d1f7420eee5d326c7c5 + languageName: node + linkType: hard + "@aws-sdk/credential-provider-login@npm:^3.972.23": version: 3.972.23 resolution: "@aws-sdk/credential-provider-login@npm:3.972.23" @@ -703,6 +785,20 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-login@npm:^3.972.74": + version: 3.972.74 + resolution: "@aws-sdk/credential-provider-login@npm:3.972.74" + dependencies: + "@aws-sdk/core": "npm:^3.977.6" + "@aws-sdk/nested-clients": "npm:^3.997.41" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/1ab9996accb61bccbdaefae023e9befab9e5062435370a37f672089457dc13c485d9d2fee6926381672bdb32143c00266b1aa5913b18ef4873584907835b3a92 + languageName: node + linkType: hard + "@aws-sdk/credential-provider-node@npm:^3.972.24": version: 3.972.24 resolution: "@aws-sdk/credential-provider-node@npm:3.972.24" @@ -723,6 +819,25 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-node@npm:^3.972.78": + version: 3.972.78 + resolution: "@aws-sdk/credential-provider-node@npm:3.972.78" + dependencies: + "@aws-sdk/credential-provider-env": "npm:^3.972.67" + "@aws-sdk/credential-provider-http": "npm:^3.972.69" + "@aws-sdk/credential-provider-ini": "npm:^3.973.12" + "@aws-sdk/credential-provider-process": "npm:^3.972.67" + "@aws-sdk/credential-provider-sso": "npm:^3.973.11" + "@aws-sdk/credential-provider-web-identity": "npm:^3.972.73" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/credential-provider-imds": "npm:^4.4.16" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/2b6e5bd455a3c2b530a884a0c5919bb7d2d91941b655a56351b957de038d318c1d42b86674e20893ee7dab6db6ea32c4653bce9b1d3ca98ac803d6b49a948343 + languageName: node + linkType: hard + "@aws-sdk/credential-provider-process@npm:^3.972.21": version: 3.972.21 resolution: "@aws-sdk/credential-provider-process@npm:3.972.21" @@ -737,6 +852,19 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-process@npm:^3.972.67": + version: 3.972.67 + resolution: "@aws-sdk/credential-provider-process@npm:3.972.67" + dependencies: + "@aws-sdk/core": "npm:^3.977.6" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/0381c39f171df2119791b03647545ab5084f6a8d2c227c5d3c5bfa9db027d0566102b6322bc09075c0779b0f0aa88ae1ca7bbdc773d8415d8dd8f163e69e45ea + languageName: node + linkType: hard + "@aws-sdk/credential-provider-sso@npm:^3.972.23": version: 3.972.23 resolution: "@aws-sdk/credential-provider-sso@npm:3.972.23" @@ -753,6 +881,21 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-sso@npm:^3.973.11": + version: 3.973.11 + resolution: "@aws-sdk/credential-provider-sso@npm:3.973.11" + dependencies: + "@aws-sdk/core": "npm:^3.977.6" + "@aws-sdk/nested-clients": "npm:^3.997.41" + "@aws-sdk/token-providers": "npm:3.1103.0" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/d6df0ae72009c2f74f1c7f12e41c0a7b395ba1860d4f9f1554fd8fbc3b5f0c1be64c83aacd2b329561e27844520ae0b57bb268265f5e7850b1d0fb769455d7a1 + languageName: node + linkType: hard + "@aws-sdk/credential-provider-web-identity@npm:^3.972.23": version: 3.972.23 resolution: "@aws-sdk/credential-provider-web-identity@npm:3.972.23" @@ -768,6 +911,20 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-web-identity@npm:^3.972.73": + version: 3.972.73 + resolution: "@aws-sdk/credential-provider-web-identity@npm:3.972.73" + dependencies: + "@aws-sdk/core": "npm:^3.977.6" + "@aws-sdk/nested-clients": "npm:^3.997.41" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/a7bee06b4200ff04141d4ce07d49d69f24b55b57f3aab929b445a9d9ea70f043d0b09fca8b95872d2b92df6837b771aeb14b03ca784d17ce1ee871b14173558c + languageName: node + linkType: hard + "@aws-sdk/lib-storage@npm:^3.1009.0": version: 3.1014.0 resolution: "@aws-sdk/lib-storage@npm:3.1014.0" @@ -1006,6 +1163,22 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/nested-clients@npm:^3.997.41": + version: 3.997.41 + resolution: "@aws-sdk/nested-clients@npm:3.997.41" + dependencies: + "@aws-sdk/core": "npm:^3.977.6" + "@aws-sdk/signature-v4-multi-region": "npm:^3.996.43" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/fetch-http-handler": "npm:^5.6.13" + "@smithy/node-http-handler": "npm:^4.9.13" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/fe1a84bb58675a24ecd0ce3b7bcaf1a456494f10c1a9dd5b55bd268be6713f83bd3c3d3dadee6bb51a53bc24ee18b2b0882d6741bcbabc224feeae97454fdb4a + languageName: node + linkType: hard + "@aws-sdk/region-config-resolver@npm:^3.972.9": version: 3.972.9 resolution: "@aws-sdk/region-config-resolver@npm:3.972.9" @@ -1033,6 +1206,18 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/signature-v4-multi-region@npm:^3.996.43": + version: 3.996.43 + resolution: "@aws-sdk/signature-v4-multi-region@npm:3.996.43" + dependencies: + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/signature-v4": "npm:^5.6.12" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/268608dd5624c6377243903d588b9c13b8de3f3f3e6bea68fc684d125bc92a991fd15a67cb178d1a7a599d0415ce5283f44ba6b96d14909b185d7ff26a9d979b + languageName: node + linkType: hard + "@aws-sdk/token-providers@npm:3.1014.0": version: 3.1014.0 resolution: "@aws-sdk/token-providers@npm:3.1014.0" @@ -1048,6 +1233,20 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/token-providers@npm:3.1103.0": + version: 3.1103.0 + resolution: "@aws-sdk/token-providers@npm:3.1103.0" + dependencies: + "@aws-sdk/core": "npm:^3.977.6" + "@aws-sdk/nested-clients": "npm:^3.997.41" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/5f86aa221e537b8a3fd11ed76ac025935f8859cc62b3af293abd759b8ca3aa390c17f6716723c08056b3373997a17bbdee2ff568eab5049bf0a372d35893b48d + languageName: node + linkType: hard + "@aws-sdk/types@npm:^3.222.0, @aws-sdk/types@npm:^3.4.1, @aws-sdk/types@npm:^3.973.6": version: 3.973.6 resolution: "@aws-sdk/types@npm:3.973.6" @@ -1058,6 +1257,16 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/types@npm:^3.974.2": + version: 3.974.2 + resolution: "@aws-sdk/types@npm:3.974.2" + dependencies: + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/b5ce05e8a4160c545edce1e8527e8ac490be7a6651c736f6811190b5d31d5682699889d51186ab0600df756679bebd2df9d650a17f577523441df803c4fb5777 + languageName: node + linkType: hard + "@aws-sdk/util-arn-parser@npm:^3.972.3": version: 3.972.3 resolution: "@aws-sdk/util-arn-parser@npm:3.972.3" @@ -1143,6 +1352,16 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/xml-builder@npm:^3.972.37": + version: 3.972.37 + resolution: "@aws-sdk/xml-builder@npm:3.972.37" + dependencies: + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/738f9302f495b3b95602641166a4182244add6e9e079201dba7e8994657dd442df0e4cea3355aa8c7d7f08efb385decaaf0b543f03efdb291c118536f36ac1a1 + languageName: node + linkType: hard + "@aws/lambda-invoke-store@npm:0.2.3, @aws/lambda-invoke-store@npm:^0.2.2": version: 0.2.3 resolution: "@aws/lambda-invoke-store@npm:0.2.3" @@ -1150,6 +1369,13 @@ __metadata: languageName: node linkType: hard +"@aws/lambda-invoke-store@npm:^0.3.0": + version: 0.3.0 + resolution: "@aws/lambda-invoke-store@npm:0.3.0" + checksum: 10c0/b4a2e6b3b5397bc606053e64270d26dc5c886336f88a98cad587b1592eec17058f8fb172f1827a9f0e591f3595cf8f01575c8c9b36cde38c06456f8a65204046 + languageName: node + linkType: hard + "@babel/code-frame@npm:^7.0.0, @babel/code-frame@npm:^7.12.13, @babel/code-frame@npm:^7.28.6, @babel/code-frame@npm:^7.29.0": version: 7.29.0 resolution: "@babel/code-frame@npm:7.29.0" @@ -4570,6 +4796,16 @@ __metadata: languageName: node linkType: hard +"@smithy/core@npm:^3.31.1": + version: 3.31.1 + resolution: "@smithy/core@npm:3.31.1" + dependencies: + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/b953c792dea2c13249b58c1799e4d6aaf21eb1a61e203b83e8e3a9156bebe14ca0585f0ca1ffdf65a193294dddff92a06fbe5c3fbd63ff0c174c88130b47a128 + languageName: node + linkType: hard + "@smithy/credential-provider-imds@npm:^4.2.12": version: 4.2.12 resolution: "@smithy/credential-provider-imds@npm:4.2.12" @@ -4583,6 +4819,17 @@ __metadata: languageName: node linkType: hard +"@smithy/credential-provider-imds@npm:^4.4.16": + version: 4.4.16 + resolution: "@smithy/credential-provider-imds@npm:4.4.16" + dependencies: + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/d03687efbbd1f95e77b7dcb639f24f1600671929627cd743f7acf9640238746664e91f955026f22e235603e10537d46e31fa60f231adbdf37457e53720bc80f9 + languageName: node + linkType: hard + "@smithy/eventstream-codec@npm:^4.2.12": version: 4.2.12 resolution: "@smithy/eventstream-codec@npm:4.2.12" @@ -4651,6 +4898,17 @@ __metadata: languageName: node linkType: hard +"@smithy/fetch-http-handler@npm:^5.6.13": + version: 5.6.13 + resolution: "@smithy/fetch-http-handler@npm:5.6.13" + dependencies: + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/028ba8794a6c487ebefae7f40d0124f70e51a1f4e0e465457845c1a44fd607320cd3c64d4a961f159aef59470f0fd43f0d2011b44ee5ef753b7e1dccbdf32ca3 + languageName: node + linkType: hard + "@smithy/hash-blob-browser@npm:^4.2.13": version: 4.2.13 resolution: "@smithy/hash-blob-browser@npm:4.2.13" @@ -4816,6 +5074,17 @@ __metadata: languageName: node linkType: hard +"@smithy/node-http-handler@npm:^4.9.13": + version: 4.9.13 + resolution: "@smithy/node-http-handler@npm:4.9.13" + dependencies: + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/2f1cdef7a300ad49c3bb698c2ca4773af5e9202d291cfcd855c1b21ab08b3c4ddf56f3722d3251db4e9b7ac39ec1ebc551b156abf3fa70f74c5491bec421f6b5 + languageName: node + linkType: hard + "@smithy/property-provider@npm:^4.2.12": version: 4.2.12 resolution: "@smithy/property-provider@npm:4.2.12" @@ -4901,6 +5170,17 @@ __metadata: languageName: node linkType: hard +"@smithy/signature-v4@npm:^5.6.12": + version: 5.6.12 + resolution: "@smithy/signature-v4@npm:5.6.12" + dependencies: + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/33656a41ad61dee16209703cb96b46b29014b3c4fad23bfbb90cdb5415ac06c6577b2bfff958ef9e6c19091364945135a0370b12ddc2daed557c903846e81fe7 + languageName: node + linkType: hard + "@smithy/smithy-client@npm:^4.12.7": version: 4.12.7 resolution: "@smithy/smithy-client@npm:4.12.7" @@ -4934,6 +5214,15 @@ __metadata: languageName: node linkType: hard +"@smithy/types@npm:^4.16.1": + version: 4.16.1 + resolution: "@smithy/types@npm:4.16.1" + dependencies: + tslib: "npm:^2.6.2" + checksum: 10c0/e024d9d148deca7bd21d032a9316db109bbe7cf256ffbb8d3981655b9f4f7695c08ec9b87f5a8cf1442e783ba26cb27e4f09603c5bfa3ba1e526c41b1b3e94d2 + languageName: node + linkType: hard + "@smithy/url-parser@npm:^4.2.12": version: 4.2.12 resolution: "@smithy/url-parser@npm:4.2.12"