Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions lambdas/functions/control-plane/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
"@aws-github-runner/aws-powertools-util": "*",
"@aws-github-runner/aws-ssm-util": "*",
"@aws-github-runner/compute-providers": "*",
"@aws-github-runner/storage-providers": "*",
"@aws-lambda-powertools/parameters": "^2.31.0",
"@aws-sdk/client-ec2": "^3.1009.0",
"@aws-sdk/client-sqs": "^3.1009.0",
Expand Down
1 change: 0 additions & 1 deletion lambdas/functions/control-plane/src/modules.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ declare namespace NodeJS {
RUNNER_OWNER: string;
COMPUTE_PROVIDER_TYPE?: string;
SCALE_DOWN_CONFIG: string;
SSM_TOKEN_PATH: string;
SSM_CLEANUP_CONFIG: string;
SUBNET_IDS: string;
INSTANCE_TYPES: string;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ const computeProviders = providerTypes.map((type) => ({
beforeEach(() => {
vi.clearAllMocks();
process.env = { ...cleanEnv };
process.env.SSM_TOKEN_PATH = '/github-action-runners/default/runners/tokens';

mockedAppAuth.mockResolvedValue({ type: 'app', token: 'app-token', appId: 1, expiresAt: 'some-date' });
mockedInstallationAuth.mockResolvedValue({
Expand Down
2 changes: 0 additions & 2 deletions lambdas/functions/control-plane/src/pool/pool.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@ export async function adjust(event: PoolEvent): Promise<void> {
const runnerGroup = process.env.RUNNER_GROUP_NAME || '';
const runnerNamePrefix = process.env.RUNNER_NAME_PREFIX || '';
const environment = process.env.ENVIRONMENT;
const ssmTokenPath = process.env.SSM_TOKEN_PATH;
const ssmConfigPath = process.env.SSM_CONFIG_PATH || '';
const ephemeral = yn(process.env.ENABLE_EPHEMERAL_RUNNERS, { default: false });
const enableJitConfig = yn(process.env.ENABLE_JIT_CONFIG, { default: ephemeral });
Expand Down Expand Up @@ -103,7 +102,6 @@ export async function adjust(event: PoolEvent): Promise<void> {
runnerNamePrefix,
runnerType: 'Org',
disableAutoUpdate: disableAutoUpdate,
ssmTokenPath,
ssmConfigPath,
ssmParameterStoreTags,
},
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,10 @@
import { createChildLogger } from '@aws-github-runner/aws-powertools-util';
import { getParameter, putParameter } from '@aws-github-runner/aws-ssm-util';
import {
createRunnerConfigStore,
type RunnerConfigMetadata,
type RunnerConfigStore,
} from '@aws-github-runner/storage-providers';
import { Octokit } from '@octokit/rest';

import { getStoredInstallationId } from '../github/auth';
Expand All @@ -14,7 +19,7 @@ export interface GitHubRunnerMetadata {
}

export interface StartRunnerConfigOptions {
getSsmParameterTags?: (runnerId: string) => { Key: string; Value: string }[];
getRunnerConfigMetadata?: (runnerId: string) => RunnerConfigMetadata[];
onJitConfigCreated?: (runnerId: string, metadata: GitHubRunnerMetadata) => Promise<void>;
}

Expand Down Expand Up @@ -250,18 +255,20 @@ export async function createStartRunnerConfig(
ghClient: Octokit,
options: StartRunnerConfigOptions = {},
): Promise<string[]> {
const runnerConfigStore = createRunnerConfigStore();
if (githubRunnerConfig.enableJitConfig && githubRunnerConfig.ephemeral) {
return await createJitConfig(githubRunnerConfig, runnerIds, ghClient, options);
return await createJitConfig(githubRunnerConfig, runnerIds, ghClient, runnerConfigStore, options);
} else {
return await createRegistrationTokenConfig(githubRunnerConfig, runnerIds, ghClient, options);
return await createRegistrationTokenConfig(githubRunnerConfig, runnerIds, ghClient, runnerConfigStore, options);
}
}

function addDelay(runnerIds: string[]) {
function addDelay(runnerIds: string[], runnerConfigStore: RunnerConfigStore) {
const delay = async (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
const ssmParameterStoreMaxThroughput = 40;
const isDelay = runnerIds.length >= ssmParameterStoreMaxThroughput;
return { isDelay, delay };
const maxWritesPerSecond = runnerConfigStore.maxWritesPerSecond;
const isDelay = maxWritesPerSecond !== undefined && runnerIds.length >= maxWritesPerSecond;
const delayMilliseconds = maxWritesPerSecond === undefined ? 0 : 1000 / maxWritesPerSecond;
return { isDelay, delay, delayMilliseconds };
}

/**
Expand All @@ -273,9 +280,10 @@ async function createRegistrationTokenConfig(
githubRunnerConfig: CreateGitHubRunnerConfig,
runnerIds: string[],
ghClient: Octokit,
runnerConfigStore: RunnerConfigStore,
options: StartRunnerConfigOptions,
): Promise<string[]> {
const { isDelay, delay } = addDelay(runnerIds);
const { isDelay, delay, delayMilliseconds } = addDelay(runnerIds, runnerConfigStore);
const token = await getGithubRunnerRegistrationToken(githubRunnerConfig, ghClient);
const runnerServiceConfig = generateRunnerServiceConfig(githubRunnerConfig, token);

Expand All @@ -284,12 +292,13 @@ async function createRegistrationTokenConfig(
});

for (const runnerId of runnerIds) {
await putParameter(`${githubRunnerConfig.ssmTokenPath}/${runnerId}`, runnerServiceConfig.join(' '), true, {
tags: [...(options.getSsmParameterTags?.(runnerId) ?? []), ...githubRunnerConfig.ssmParameterStoreTags],
});
await runnerConfigStore.create(
{ runnerId, value: runnerServiceConfig.join(' ') },
{ metadata: options.getRunnerConfigMetadata?.(runnerId) },
);
if (isDelay) {
// Delay to prevent AWS ssm rate limits by being within the max throughput limit
await delay(25);
// Delay to stay within the selected store's maximum write throughput.
await delay(delayMilliseconds);
}
}

Expand All @@ -306,10 +315,11 @@ async function createJitConfig(
githubRunnerConfig: CreateGitHubRunnerConfig,
runnerIds: string[],
ghClient: Octokit,
runnerConfigStore: RunnerConfigStore,
options: StartRunnerConfigOptions,
): Promise<string[]> {
const runnerGroupId = await getRunnerGroupId(githubRunnerConfig, ghClient);
const { isDelay, delay } = addDelay(runnerIds);
const { isDelay, delay, delayMilliseconds } = addDelay(runnerIds, runnerConfigStore);
const runnerLabels = githubRunnerConfig.runnerLabels.split(',');
const failedRunnerIds: string[] = [];

Expand Down Expand Up @@ -347,16 +357,16 @@ async function createJitConfig(
runnerLabels,
});

// store jit config in ssm parameter store
logger.debug('Runner JIT config for ephemeral runner generated.', {
instance: runnerId,
});
await putParameter(`${githubRunnerConfig.ssmTokenPath}/${runnerId}`, runnerConfig.data.encoded_jit_config, true, {
tags: [...(options.getSsmParameterTags?.(runnerId) ?? []), ...githubRunnerConfig.ssmParameterStoreTags],
});
await runnerConfigStore.create(
{ runnerId, value: runnerConfig.data.encoded_jit_config },
{ metadata: options.getRunnerConfigMetadata?.(runnerId) },
);
if (isDelay) {
// Delay to prevent AWS ssm rate limits by being within the max throughput limit
await delay(25);
// Delay to stay within the selected store's maximum write throughput.
await delay(delayMilliseconds);
}
} catch (error) {
failedRunnerIds.push(runnerId);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ const computeProviders = providerTypes.map((type) => ({
beforeEach(() => {
vi.clearAllMocks();
process.env = { ...cleanEnv };
process.env.SSM_TOKEN_PATH = '/github-action-runners/default/runners/tokens';

mockedAppAuth.mockResolvedValue({ type: 'app', token: 'app-token', appId: 1, expiresAt: 'some-date' });
mockedInstallationAuth.mockResolvedValue({
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,7 @@ function setDefaults() {
process.env.GITHUB_APP_CLIENT_SECRET = 'TEST_CLIENT_SECRET';
process.env.RUNNERS_MAXIMUM_COUNT = '3';
process.env.ENVIRONMENT = EXPECTED_RUNNER_PARAMS.environment;
process.env.SSM_TOKEN_PATH = '/github-action-runners/default/runners/config';
}

async function createTestProviderRunners(input: CreateScaleUpRunnersInput<unknown>): Promise<CreateRunnerResult> {
Expand All @@ -168,7 +169,7 @@ async function createTestProviderRunners(input: CreateScaleUpRunnersInput<unknow
result.instances,
input.githubInstallationClient,
{
getSsmParameterTags: (runnerId) => [{ Key: 'RunnerId', Value: runnerId }],
getRunnerConfigMetadata: (runnerId) => [{ key: 'RunnerId', value: runnerId }],
},
);
} catch {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,6 @@ export async function scaleUp(payloads: ActionRequestMessageSQS[]): Promise<stri
const maximumRunners = parseInt(process.env.RUNNERS_MAXIMUM_COUNT || '3');
const runnerLabels = process.env.RUNNER_LABELS || '';
const runnerGroup = process.env.RUNNER_GROUP_NAME || 'Default';
const ssmTokenPath = process.env.SSM_TOKEN_PATH;
const ephemeralEnabled = yn(process.env.ENABLE_EPHEMERAL_RUNNERS, { default: false });
const enableJitConfig = yn(process.env.ENABLE_JIT_CONFIG, { default: ephemeralEnabled });
const disableAutoUpdate = yn(process.env.DISABLE_RUNNER_AUTOUPDATE, { default: false });
Expand Down Expand Up @@ -318,7 +317,6 @@ export async function scaleUp(payloads: ActionRequestMessageSQS[]): Promise<stri
runnerOwner: runnerOwner,
runnerType,
disableAutoUpdate,
ssmTokenPath,
ssmConfigPath,
ssmParameterStoreTags,
};
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ async function terminateFailedInstances(

function createEc2StartRunnerConfigOptions(ec2Operations: Ec2RunnerResourceOperations): StartRunnerConfigOptions {
return {
getSsmParameterTags: (instanceId) => [{ Key: 'InstanceId', Value: instanceId }],
getRunnerConfigMetadata: (instanceId) => [{ key: 'InstanceId', value: instanceId }],
onJitConfigCreated: async (instanceId, metadata) => await tagEc2RunnerMetadata(ec2Operations, instanceId, metadata),
};
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,6 @@ function runnerConfig(overrides: Partial<CreateGitHubRunnerConfig> = {}): Create
runnerOwner,
runnerType: 'Org',
disableAutoUpdate: false,
ssmTokenPath: '/github-action-runners/default/runners/config',
ssmConfigPath: '/github-action-runners/default/runners/config',
ssmParameterStoreTags: [],
...overrides,
Expand Down Expand Up @@ -175,7 +174,7 @@ describe('scaleUp with GHES', () => {
{ Key: 'ghr:runner_labels', Value: 'label1,label2' },
]);
const [, , , options] = mockCreateStartRunnerConfig.mock.calls[0];
expect(options?.getSsmParameterTags?.('i-12345')).toEqual([{ Key: 'InstanceId', Value: 'i-12345' }]);
expect(options?.getRunnerConfigMetadata?.('i-12345')).toEqual([{ key: 'InstanceId', value: 'i-12345' }]);
});

it('chunks comma-joined GitHub runner labels by the EC2 tag value max length', async () => {
Expand Down
3 changes: 1 addition & 2 deletions lambdas/libs/compute-providers/core/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ export interface CreateGitHubRunnerConfig {
runnerOwner: string;
runnerType: RunnerType;
disableAutoUpdate: boolean;
ssmTokenPath: string;
ssmConfigPath: string;
ssmParameterStoreTags: { Key: string; Value: string }[];
}
Expand All @@ -32,7 +31,7 @@ export interface GitHubRunnerMetadata {
}

export interface StartRunnerConfigOptions {
getSsmParameterTags?: (runnerId: string) => { Key: string; Value: string }[];
getRunnerConfigMetadata?: (runnerId: string) => { key: string; value: string }[];
onJitConfigCreated?: (runnerId: string, metadata: GitHubRunnerMetadata) => Promise<void>;
}

Expand Down
1 change: 1 addition & 0 deletions lambdas/libs/compute-providers/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
"dependencies": {
"@aws-github-runner/aws-powertools-util": "*",
"@aws-github-runner/aws-ssm-util": "*",
"@aws-github-runner/storage-providers": "*",
Comment thread
edersonbrilhante marked this conversation as resolved.
"@aws-sdk/client-ec2": "^3.1009.0",
"@octokit/rest": "22.0.1",
"moment": "2.29.4",
Expand Down
10 changes: 10 additions & 0 deletions lambdas/libs/storage-providers/aws/ssm/environment.d.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
export {};

declare global {
namespace NodeJS {
interface ProcessEnv {
SSM_PARAMETER_STORE_TAGS?: string;
SSM_TOKEN_PATH?: string;
}
}
}
42 changes: 42 additions & 0 deletions lambdas/libs/storage-providers/aws/ssm/parameter-store-tags.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
interface SsmParameterStoreTag {
Key: string;
Value: string;
}

export function loadSsmParameterStoreTagsFromEnvironment(): SsmParameterStoreTag[] {
return process.env.SSM_PARAMETER_STORE_TAGS && process.env.SSM_PARAMETER_STORE_TAGS.trim() !== ''
? validateSsmParameterStoreTags(process.env.SSM_PARAMETER_STORE_TAGS)
: [];
}

function validateSsmParameterStoreTags(tagsJson: string): SsmParameterStoreTag[] {
try {
const tags: unknown = JSON.parse(tagsJson);

if (!Array.isArray(tags)) {
throw new Error('Tags must be an array');
}

if (tags.length === 0) {
return [];
}

tags.forEach((tag: unknown, index: number) => {
if (typeof tag !== 'object' || tag === null) {
throw new Error(`Tag at index ${index} must be an object`);
}

const candidate = tag as Record<string, unknown>;
if (!candidate.Key || typeof candidate.Key !== 'string' || candidate.Key.trim() === '') {
throw new Error(`Tag at index ${index} has missing or invalid 'Key' property`);
}
if (!Object.prototype.hasOwnProperty.call(candidate, 'Value') || typeof candidate.Value !== 'string') {
throw new Error(`Tag at index ${index} has missing or invalid 'Value' property`);
}
});

return tags as SsmParameterStoreTag[];
} catch (error) {
throw new Error(`Failed to parse SSM_PARAMETER_STORE_TAGS: ${(error as Error).message}`);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
import { putParameter } from '@aws-github-runner/aws-ssm-util';
import { beforeEach, describe, expect, it, vi } from 'vitest';

import { createAwsSsmRunnerConfigStore } from './runner-config-store';

vi.mock('@aws-github-runner/aws-ssm-util', () => ({
putParameter: vi.fn(),
}));

const putParameterMock = vi.mocked(putParameter);
const cleanEnv = process.env;

describe('aws_ssm runner config store', () => {
beforeEach(() => {
vi.clearAllMocks();
process.env = { ...cleanEnv };
delete process.env.SSM_PARAMETER_STORE_TAGS;
process.env.SSM_TOKEN_PATH = '/runner/tokens';
});

it('creates a secure parameter at the legacy path with metadata tags before configured tags', async () => {
process.env.SSM_PARAMETER_STORE_TAGS = JSON.stringify([
{ Key: 'Environment', Value: 'test' },
{ Key: 'Team', Value: 'actions' },
]);
const store = createAwsSsmRunnerConfigStore();

await store.create(
{ runnerId: 'i-123', value: 'encoded-jit-config' },
{ metadata: [{ key: 'InstanceId', value: 'i-123' }] },
);

expect(store.maxWritesPerSecond).toBe(40);
expect(putParameterMock).toHaveBeenCalledWith('/runner/tokens/i-123', 'encoded-jit-config', true, {
tags: [
{ Key: 'InstanceId', Value: 'i-123' },
{ Key: 'Environment', Value: 'test' },
{ Key: 'Team', Value: 'actions' },
],
});
});

it('uses an empty tag list when no tags are configured', async () => {
const store = createAwsSsmRunnerConfigStore();

await store.create({ runnerId: 'runner-1', value: 'registration-config' });

expect(putParameterMock).toHaveBeenCalledWith('/runner/tokens/runner-1', 'registration-config', true, {
tags: [],
});
});

it.each([undefined, '', ' '])('rejects missing or blank SSM_TOKEN_PATH %j before writing', (tokenPath) => {
setTokenPath(tokenPath);

expect(() => createAwsSsmRunnerConfigStore()).toThrow('Environment variable SSM_TOKEN_PATH is not set');
expect(putParameterMock).not.toHaveBeenCalled();
});

it.each([
['{}', 'Tags must be an array'],
['[null]', 'Tag at index 0 must be an object'],
[JSON.stringify([{ Key: '', Value: 'test' }]), "Tag at index 0 has missing or invalid 'Key' property"],
[JSON.stringify([{ Key: 'Environment' }]), "Tag at index 0 has missing or invalid 'Value' property"],
])('rejects invalid legacy SSM parameter tags', (tags, reason) => {
process.env.SSM_PARAMETER_STORE_TAGS = tags;

expect(() => createAwsSsmRunnerConfigStore()).toThrow(`Failed to parse SSM_PARAMETER_STORE_TAGS: ${reason}`);
expect(putParameterMock).not.toHaveBeenCalled();
});

it('treats a blank legacy tag value as no configured tags', async () => {
process.env.SSM_PARAMETER_STORE_TAGS = ' ';
const store = createAwsSsmRunnerConfigStore();

await store.create({ runnerId: 'runner-1', value: 'jit-config' });

expect(putParameterMock).toHaveBeenCalledWith('/runner/tokens/runner-1', 'jit-config', true, { tags: [] });
});
});

function setTokenPath(tokenPath: string | undefined): void {
if (tokenPath === undefined) {
delete process.env.SSM_TOKEN_PATH;
} else {
process.env.SSM_TOKEN_PATH = tokenPath;
}
}
Loading