diff --git a/apps/server-nestjs/src/modules/gitlab/gitlab-client.service.spec.ts b/apps/server-nestjs/src/modules/gitlab/gitlab-client.service.spec.ts index 4a4c908120..41889dc66f 100644 --- a/apps/server-nestjs/src/modules/gitlab/gitlab-client.service.spec.ts +++ b/apps/server-nestjs/src/modules/gitlab/gitlab-client.service.spec.ts @@ -287,7 +287,7 @@ describe('gitlab-client', () => { gitlabApi.Projects.show.mockResolvedValue(makeProjectSchema({ id: repoId })) gitlabApi.Projects.edit.mockResolvedValue(makeProjectSchema({ id: repoId, name: repoName })) - const result = await service.upsertProjectGroupRepo(projectSlug, repoName, 'desc') + const result = await service.upsertProjectGroupRepo(projectSlug, repoName, { description: 'desc' }) expect(result).toEqual(expect.objectContaining({ id: repoId, name: repoName })) expect(gitlabApi.ProjectCustomAttributes.set).toHaveBeenCalledWith(repoId, MANAGED_BY_CONSOLE_CUSTOM_ATTRIBUTE_KEY, 'true') diff --git a/apps/server-nestjs/src/modules/gitlab/gitlab-client.service.ts b/apps/server-nestjs/src/modules/gitlab/gitlab-client.service.ts index 5876cad00a..ed6f71a606 100644 --- a/apps/server-nestjs/src/modules/gitlab/gitlab-client.service.ts +++ b/apps/server-nestjs/src/modules/gitlab/gitlab-client.service.ts @@ -33,6 +33,7 @@ import { SPECIAL_REPO_NAMES, TOKEN_DESCRIPTION, TOPIC_PLUGIN_MANAGED, + TOPIC_SYSTEM_MANAGED, USER_ID_CUSTOM_ATTRIBUTE_KEY, } from './gitlab.constants' import { generateGitlabCIConfigContent, generateMirrorScriptContent, hasFileContentChanged, hasGitbeakerCause, isGitbeakerNotFound } from './gitlab.utils' @@ -42,6 +43,12 @@ export const GITLAB_REST_CLIENT = Symbol('GITLAB_REST_CLIENT') type With = T & Required> export type CondensedGroupSchemaWith = With export type CondensedProjectSchemaWith = With + +export interface UpsertProjectGroupRepoOptions { + description?: string + ciConfigPath?: string + extraTopics?: string[] +} export type EditUserOptionsWith = With type UserSchema = SimpleUserSchema | ExpandedUserSchema @@ -460,19 +467,28 @@ export class GitlabClientService { } } - async upsertProjectGroupRepo(projectSlug: string, repoName: string, description?: string, ciConfigPath?: string) { + async upsertProjectGroupRepo(projectSlug: string, repoName: string, options: UpsertProjectGroupRepoOptions = {}) { + const { description, ciConfigPath, extraTopics = [] } = options const fullPath = `${projectSlug}/${repoName}` const repo = await this.getOrCreateProjectGroupRepo(projectSlug, fullPath, ciConfigPath) const updated = await this.client.Projects.edit(repo.id, { name: repoName, path: repoName, - topics: [TOPIC_PLUGIN_MANAGED], + topics: [TOPIC_PLUGIN_MANAGED, ...extraTopics], description, ciConfigPath: ciConfigPath ?? '', }) return updated } + // System repos (mirror, infra-apps, observability values, ...) are console-owned plumbing: + // created in the project subgroup but never listed in project.repositories, so the orphan + // purge must never delete them. They carry the dedicated system-managed topic; any plugin + // can opt its system repo in by upserting it through this wrapper. + async upsertProjectGroupSystemRepo(projectSlug: string, repoName: string, options: Omit = {}) { + return this.upsertProjectGroupRepo(projectSlug, repoName, { ...options, extraTopics: [TOPIC_SYSTEM_MANAGED] }) + } + async deleteProjectGroupRepo(projectSlug: string, repoName: string) { const fullPath = `${projectSlug}/${repoName}` const repo = await this.getOrCreateProjectGroupRepo(projectSlug, fullPath) @@ -585,7 +601,7 @@ export class GitlabClientService { } async upsertProjectMirrorRepo(projectSlug: string) { - return this.upsertProjectGroupRepo(projectSlug, MIRROR_REPO_NAME) + return this.upsertProjectGroupSystemRepo(projectSlug, MIRROR_REPO_NAME) } async getProjectToken(group: CondensedGroupSchemaWith<'id'>, projectSlug: string) { diff --git a/apps/server-nestjs/src/modules/gitlab/gitlab.constants.ts b/apps/server-nestjs/src/modules/gitlab/gitlab.constants.ts index ec9f282a49..25c6f95e0d 100644 --- a/apps/server-nestjs/src/modules/gitlab/gitlab.constants.ts +++ b/apps/server-nestjs/src/modules/gitlab/gitlab.constants.ts @@ -15,6 +15,9 @@ export const GITLAB_CI_CONFIG_PATH = '.gitlab-ci-dso.yml' // Managed resources sentinel export const TOPIC_PLUGIN_MANAGED = 'plugin-managed' + +// Console-owned plumbing repos (mirror, infra-apps, observability...), protected from orphan purge +export const TOPIC_SYSTEM_MANAGED = 'system-managed' export const TOKEN_DESCRIPTION = 'mirroring-from-external-repo' // Default group paths for console roles diff --git a/apps/server-nestjs/src/modules/gitlab/gitlab.service.spec.ts b/apps/server-nestjs/src/modules/gitlab/gitlab.service.spec.ts index 51c3f9949a..7769d0832a 100644 --- a/apps/server-nestjs/src/modules/gitlab/gitlab.service.spec.ts +++ b/apps/server-nestjs/src/modules/gitlab/gitlab.service.spec.ts @@ -7,11 +7,12 @@ import { Test } from '@nestjs/testing' import { beforeEach, describe, expect, it, vi } from 'vitest' import { mockDeep } from 'vitest-mock-extended' import { gitlabConfigFactory } from '../../config/gitlab.config' +import { OBSERVABILITY_REPOSITORY } from '../observability/observability.constants' import { VaultClientService } from '../vault/vault-client.service' import { GitlabClientService } from './gitlab-client.service' import { GitlabDatastoreService } from './gitlab-datastore.service' import { makeAccessTokenExposedSchema, makeExpandedUserSchema, makeGroupSchema, makeMemberSchema, makePipeline, makePipelineTriggerToken, makeProjectSchema, makeProjectWithDetails } from './gitlab-testing.utils' -import { PLUGIN_NAME, TOPIC_PLUGIN_MANAGED } from './gitlab.constants' +import { INFRA_APPS_REPO_NAME, MIRROR_REPO_NAME, PLUGIN_NAME, TOPIC_PLUGIN_MANAGED, TOPIC_SYSTEM_MANAGED } from './gitlab.constants' import { GitlabService } from './gitlab.service' describe('gitlabService', () => { @@ -181,6 +182,55 @@ describe('gitlabService', () => { expect(gitlab.deleteProjectGroupRepo).toHaveBeenCalledTimes(1) }) + it('should never delete system repositories (infra-apps, mirror) even though they are not in project.repositories', async () => { + const project = makeProjectWithDetails({ repositories: [] }) + const group = makeGroupSchema({ id: 123, name: 'project-1', path: 'project-1', full_path: 'forge/console/project-1', full_name: 'forge/console/project-1', parent_id: 1 }) + const infraApps = makeProjectSchema({ name: INFRA_APPS_REPO_NAME, topics: [TOPIC_PLUGIN_MANAGED, TOPIC_SYSTEM_MANAGED] }) + const mirror = makeProjectSchema({ name: MIRROR_REPO_NAME, topics: [TOPIC_PLUGIN_MANAGED, TOPIC_SYSTEM_MANAGED] }) + const userRepo = makeProjectSchema({ name: 'user-repo', topics: [TOPIC_PLUGIN_MANAGED] }) + + gitlab.getOrCreateProjectSubGroup.mockResolvedValue(group) + gitlab.getGroupMembers.mockResolvedValue([]) + gitlab.getRepos.mockImplementation(() => (async function* () { + yield infraApps + yield mirror + yield userRepo + })()) + gitlab.deleteProjectGroupRepo.mockResolvedValue(undefined) + gitlab.upsertProjectMirrorRepo.mockResolvedValue(makeProjectSchema({ id: 1, name: 'mirror', path: 'mirror', path_with_namespace: 'forge/console/project-1/mirror', empty_repo: false })) + gitlab.getOrCreateMirrorPipelineTriggerToken.mockResolvedValue(makePipelineTriggerToken()) + + await service.handleUpsert(project) + + expect(gitlab.deleteProjectGroupRepo).toHaveBeenCalledTimes(1) + expect(gitlab.deleteProjectGroupRepo).toHaveBeenCalledWith(project.slug, 'user-repo') + expect(gitlab.deleteProjectGroupRepo).not.toHaveBeenCalledWith(project.slug, INFRA_APPS_REPO_NAME) + expect(gitlab.deleteProjectGroupRepo).not.toHaveBeenCalledWith(project.slug, MIRROR_REPO_NAME) + }) + + it('should never delete the observability system repository (infra-observability) even though it is not in project.repositories', async () => { + const project = makeProjectWithDetails({ repositories: [] }) + const group = makeGroupSchema({ id: 123, name: 'project-1', path: 'project-1', full_path: 'forge/console/project-1', full_name: 'forge/console/project-1', parent_id: 1 }) + const observabilityRepo = makeProjectSchema({ name: OBSERVABILITY_REPOSITORY, topics: [TOPIC_PLUGIN_MANAGED, TOPIC_SYSTEM_MANAGED] }) + const userRepo = makeProjectSchema({ name: 'user-repo', topics: [TOPIC_PLUGIN_MANAGED] }) + + gitlab.getOrCreateProjectSubGroup.mockResolvedValue(group) + gitlab.getGroupMembers.mockResolvedValue([]) + gitlab.getRepos.mockImplementation(() => (async function* () { + yield observabilityRepo + yield userRepo + })()) + gitlab.deleteProjectGroupRepo.mockResolvedValue(undefined) + gitlab.upsertProjectMirrorRepo.mockResolvedValue(makeProjectSchema({ id: 1, name: 'mirror', path: 'mirror', path_with_namespace: 'forge/console/project-1/mirror', empty_repo: false })) + gitlab.getOrCreateMirrorPipelineTriggerToken.mockResolvedValue(makePipelineTriggerToken()) + + await service.handleUpsert(project) + + expect(gitlab.deleteProjectGroupRepo).toHaveBeenCalledTimes(1) + expect(gitlab.deleteProjectGroupRepo).toHaveBeenCalledWith(project.slug, 'user-repo') + expect(gitlab.deleteProjectGroupRepo).not.toHaveBeenCalledWith(project.slug, OBSERVABILITY_REPOSITORY) + }) + it('should not delete orphan repositories without the correct topic even if purge enabled', async () => { const project = makeProjectWithDetails({ plugins: [{ pluginName: PLUGIN_NAME, key: 'purge', value: ENABLED }], diff --git a/apps/server-nestjs/src/modules/gitlab/gitlab.service.ts b/apps/server-nestjs/src/modules/gitlab/gitlab.service.ts index a1c089dde1..eb729dd1b9 100644 --- a/apps/server-nestjs/src/modules/gitlab/gitlab.service.ts +++ b/apps/server-nestjs/src/modules/gitlab/gitlab.service.ts @@ -41,6 +41,8 @@ import { generateUsername, generateUsernameCandidates, getProjectPluginConfig, + hasGitbeakerCause, + isDeclaredRepo, isOwnedRepo, isOwnedUser, isSystemRepo, @@ -146,9 +148,9 @@ export class GitlabService { const members = await this.gitlab.getGroupMembers(group) this.logger.verbose(`Loaded GitLab project group state (${project.slug}): groupId=${group.id} members=${members.length}`) await this.ensureProjectGroupMembers(project, group, members) + await this.ensureSystemRepos(project) await this.ensureProjectRepos(project) await this.purgeOrphanRepos(project) - await this.ensureSystemRepos(project) this.logger.verbose(`GitLab project group reconciled (${project.slug})`) } @@ -370,12 +372,9 @@ export class GitlabService { ...(externalHost ? { 'repository.external.host': externalHost } : {}), 'repository.external': !!repo.externalRepoUrl, }) - await this.gitlab.upsertProjectGroupRepo( - project.slug, - repo.internalRepoName, - undefined, - repo.externalRepoUrl ? GITLAB_CI_CONFIG_PATH : undefined, - ) + await this.gitlab.upsertProjectGroupRepo(project.slug, repo.internalRepoName, { + ciConfigPath: repo.externalRepoUrl ? GITLAB_CI_CONFIG_PATH : undefined, + }) if (repo.externalRepoUrl) { span?.setAttribute('repository.mirroring', true) @@ -399,12 +398,22 @@ export class GitlabService { span?.setAttribute('gitlab.repositories.count', gitlabRepositories.length) // Delete console-owned repos no longer tracked (e.g. console repository deletion). - const orphanRepos = gitlabRepositories.filter(r => isOwnedRepo(r) && !isSystemRepo(project, r)) + const orphanRepos = gitlabRepositories.filter(r => isOwnedRepo(r) && !isSystemRepo(r) && !isDeclaredRepo(project, r)) span?.setAttribute('orphan.repositories.count', orphanRepos.length) let removedCount = 0 await Promise.all(orphanRepos.map(async (orphan) => { - await this.gitlab.deleteProjectGroupRepo(project.slug, orphan.name) + try { + await this.gitlab.deleteProjectGroupRepo(project.slug, orphan.name) + } catch (err) { + // GitLab deletion is asynchronous: a repo already marked for deletion by a + // prior run surfaces a transient 400. Ignore only that; let real errors propagate. + if (hasGitbeakerCause(err, /already marked for deletion/)) { + this.logger.warn(`Repository already marked for deletion, skipping (project=${project.slug}, repoName=${orphan.name})`) + return + } + throw err + } removedCount++ this.logger.log(`Removed a repository from the GitLab project (project=${project.slug}, repoName=${orphan.name})`) })) @@ -464,7 +473,7 @@ export class GitlabService { } private async ensureInfraAppsRepo(project: ProjectWithDetails) { - await this.gitlab.upsertProjectGroupRepo(project.slug, INFRA_APPS_REPO_NAME) + await this.gitlab.upsertProjectGroupSystemRepo(project.slug, INFRA_APPS_REPO_NAME) } private async ensureMirrorRepo(project: ProjectWithDetails) { diff --git a/apps/server-nestjs/src/modules/gitlab/gitlab.utils.ts b/apps/server-nestjs/src/modules/gitlab/gitlab.utils.ts index 74f3620aaa..d5699f039c 100644 --- a/apps/server-nestjs/src/modules/gitlab/gitlab.utils.ts +++ b/apps/server-nestjs/src/modules/gitlab/gitlab.utils.ts @@ -4,7 +4,7 @@ import { createHash } from 'node:crypto' import { AccessLevel } from '@gitbeaker/core' import { GitbeakerRequestError } from '@gitbeaker/requester-utils' import { stringify } from 'yaml' -import { TOPIC_PLUGIN_MANAGED } from './gitlab.constants' +import { TOPIC_PLUGIN_MANAGED, TOPIC_SYSTEM_MANAGED } from './gitlab.constants' export type ProjectAccessLevel = Exclude @@ -218,7 +218,19 @@ export function isOwnedRepo(repo: ProjectSchema) { return repo.topics?.includes(TOPIC_PLUGIN_MANAGED) ?? false } -export function isSystemRepo(project: ProjectWithDetails, repo: ProjectSchema) { +export function isSystemRepo(repo: ProjectSchema) { + // Console-owned plumbing repos (infra-apps, mirror, observability values, ...) carry the + // `system-managed` topic and are never listed in project.repositories; protect them from purge. + // Topic-based instead of a hardcoded name list so any plugin can opt its system repo in. + return repo.topics?.includes(TOPIC_SYSTEM_MANAGED) ?? false +} + +export function isDeclaredRepo(project: ProjectWithDetails, repo: ProjectSchema) { + // A repo declared in project.repositories is managed by the console and must never be purged, + // even though it may not carry the `system-managed` topic yet (e.g. repos created before the + // topic existed). The orphan purge only targets repos that are neither system- nor declared + // in the project; keeping this check makes the purge safe for pre-existing repos until the + // next reconciliation tags them. return project.repositories.some(r => r.internalRepoName === repo.name) } diff --git a/apps/server-nestjs/src/modules/observability/observability.service.ts b/apps/server-nestjs/src/modules/observability/observability.service.ts index b66016c5aa..3a391c4355 100644 --- a/apps/server-nestjs/src/modules/observability/observability.service.ts +++ b/apps/server-nestjs/src/modules/observability/observability.service.ts @@ -100,7 +100,7 @@ export class ObservabilityService { const span = trace.getActiveSpan() span?.setAttribute('project.slug', project.slug) this.logger.verbose(`Ensuring observability project repository for ${project.slug}`) - await this.gitlab.upsertProjectGroupRepo(project.slug, OBSERVABILITY_REPOSITORY) + await this.gitlab.upsertProjectGroupSystemRepo(project.slug, OBSERVABILITY_REPOSITORY) } @StartActiveSpan() @@ -114,7 +114,7 @@ export class ObservabilityService { } private async syncChartFiles(project: ProjectWithDetails) { - const projectRepo = await this.gitlab.upsertProjectGroupRepo(project.slug, OBSERVABILITY_REPOSITORY) + const projectRepo = await this.gitlab.upsertProjectGroupSystemRepo(project.slug, OBSERVABILITY_REPOSITORY) const actions = await this.buildChartActions(projectRepo) await this.gitlab.maybeCreateCommit(projectRepo, 'ci: :robot_face: Sync observability chart', actions) } diff --git a/apps/server-nestjs/test/argocd.e2e-spec.ts b/apps/server-nestjs/test/argocd.e2e-spec.ts index 4d4c7aa3f8..107d665d2b 100644 --- a/apps/server-nestjs/test/argocd.e2e-spec.ts +++ b/apps/server-nestjs/test/argocd.e2e-spec.ts @@ -20,7 +20,7 @@ import { LoggerModule } from '../src/modules/infrastructure/logger/logger.module import { PermissionModule } from '../src/modules/infrastructure/permission/permission.module' import { VaultClientService } from '../src/modules/vault/vault-client.service' import { getDotenvPaths } from '../src/utils/dotenv.utils' -import { ARGOCD_RECONCILE_TIMEOUT, EXTERNAL_SYNC_TIMEOUT } from './e2e-timeout' +import { ARGOCD_RECONCILE_TIMEOUT, GITLAB_SYNC_TIMEOUT } from './constants' const canRunArgoCDE2E = Boolean(process.env.E2E) @@ -306,5 +306,5 @@ describeWithArgoCD('ArgoCDService (e2e)', () => { const prodFile = await gitlab.getFile(infraProject, prodFilePath, 'main') expect(prodFile).toBeUndefined() - }, EXTERNAL_SYNC_TIMEOUT) + }, GITLAB_SYNC_TIMEOUT) }) diff --git a/apps/server-nestjs/test/e2e-timeout.ts b/apps/server-nestjs/test/constants.ts similarity index 63% rename from apps/server-nestjs/test/e2e-timeout.ts rename to apps/server-nestjs/test/constants.ts index d4787dac14..73e9ef1a8d 100644 --- a/apps/server-nestjs/test/e2e-timeout.ts +++ b/apps/server-nestjs/test/constants.ts @@ -2,6 +2,8 @@ // Each timeout names the task it bounds so a reader knows which system and operation it covers. export const SONARQUBE_PROJECT_TIMEOUT = 30_000 // provision + delete a SonarQube project/user export const KEYCLOAK_GROUP_SYNC_TIMEOUT = 60_000 // reconcile Keycloak groups/roles -export const EXTERNAL_SYNC_TIMEOUT = 72_000 // sync GitLab groups/members, teardown Nexus/registry +export const GITLAB_SYNC_TIMEOUT = 72_000 // sync GitLab groups/members/repos, mirror pipeline trigger +export const GITLAB_PURGE_SYNC_TIMEOUT = 150_000 // GitLab reconcile twice (create + orphan purge) on shared env +export const NEXUS_SYNC_TIMEOUT = 72_000 // reconcile Nexus repos/roles/users, teardown on delete export const ARGOCD_RECONCILE_TIMEOUT = 144_000 // ArgoCD commit + sync to git export const VAULT_PROVISION_TIMEOUT = 180_000 // provision Vault mounts/policies/approles, zone secrets diff --git a/apps/server-nestjs/test/gitlab.e2e-spec.ts b/apps/server-nestjs/test/gitlab.e2e-spec.ts index e254b5ee95..47904e76e5 100644 --- a/apps/server-nestjs/test/gitlab.e2e-spec.ts +++ b/apps/server-nestjs/test/gitlab.e2e-spec.ts @@ -10,7 +10,7 @@ import z from 'zod' import { baseConfigFactory } from '../src/config/base.config' import { GITLAB_REST_CLIENT, GitlabClientService } from '../src/modules/gitlab/gitlab-client.service' import { projectSelect } from '../src/modules/gitlab/gitlab-datastore.service' -import { GITLAB_CI_CONFIG_PATH } from '../src/modules/gitlab/gitlab.constants' +import { GITLAB_CI_CONFIG_PATH, INFRA_APPS_REPO_NAME, MIRROR_REPO_NAME, TOPIC_PLUGIN_MANAGED, TOPIC_SYSTEM_MANAGED } from '../src/modules/gitlab/gitlab.constants' import { GitlabModule } from '../src/modules/gitlab/gitlab.module' import { AuthModule } from '../src/modules/infrastructure/auth/auth.module' import { DatabaseModule } from '../src/modules/infrastructure/database/database.module' @@ -21,7 +21,7 @@ import { PermissionModule } from '../src/modules/infrastructure/permission/permi import { VaultClientService } from '../src/modules/vault/vault-client.service' import { getDotenvPaths } from '../src/utils/dotenv.utils' import { getAll } from '../src/utils/iterable.utils' -import { EXTERNAL_SYNC_TIMEOUT } from './e2e-timeout' +import { GITLAB_PURGE_SYNC_TIMEOUT, GITLAB_SYNC_TIMEOUT } from './constants' const canRunGitlabE2E = Boolean(process.env.E2E) @@ -170,7 +170,7 @@ describeWithGitLab('GitlabService (e2e)', () => { const repoSecret = await vaultService.read(repoVaultPath) expect(repoSecret?.data?.GIT_OUTPUT_USER).toBeTruthy() expect(repoSecret?.data?.GIT_OUTPUT_PASSWORD).toBeTruthy() - }, EXTERNAL_SYNC_TIMEOUT) + }, GITLAB_SYNC_TIMEOUT) it('system repos use the default CI file, user repos mirroring an external URL use the custom one', async () => { const project = await prisma.project.findUniqueOrThrow({ @@ -200,7 +200,7 @@ describeWithGitLab('GitlabService (e2e)', () => { // commitMirror wrote the default CI file into the mirror repo const ciFile = await gitlabClientService.getFile(mirror, '.gitlab-ci.yml', 'main') expect(ciFile).toBeTruthy() - }, EXTERNAL_SYNC_TIMEOUT) + }, GITLAB_SYNC_TIMEOUT) it('should trigger the mirror pipeline using the mirror repo default CI config', async () => { const project = await prisma.project.findUniqueOrThrow({ @@ -226,7 +226,7 @@ describeWithGitLab('GitlabService (e2e)', () => { const pipeline = await gitlabClientService.triggerMirror(testProjectSlug, 'app', false, 'main') expect(pipeline.id).toBeTruthy() - }, EXTERNAL_SYNC_TIMEOUT) + }, GITLAB_SYNC_TIMEOUT) describe('project members', () => { let newUserId: string | undefined @@ -290,7 +290,111 @@ describeWithGitLab('GitlabService (e2e)', () => { const members = await gitlabClientService.getGroupMembers(group) const isNewMemberPresent = members.some(m => m.id === newUserGitlabId) expect(isNewMemberPresent).toBe(true) - }, EXTERNAL_SYNC_TIMEOUT) + }, GITLAB_SYNC_TIMEOUT) + }) + + describe('system repo purge protection', () => { + it('system repos (mirror, infra-apps) survive a reprovisioning that purges an orphan', async () => { + const project = await prisma.project.findUniqueOrThrow({ + where: { id: testProjectId }, + select: projectSelect, + }) + await eventEmitter.emitAsync('project.upsert', project) + + const groupPath = `${config.projectsRootDir}/${testProjectSlug}` + const group = z.object({ id: z.number() }).parse(await gitlabClientService.getGroupByPath(groupPath)) + const reposBefore = await getAll(gitlabClientService.getRepos(testProjectSlug)) + const mirror = reposBefore.find(repo => repo.name === MIRROR_REPO_NAME) + const infraApps = reposBefore.find(repo => repo.name === INFRA_APPS_REPO_NAME) + if (!mirror) throw new Error('mirror repo not found') + if (!infraApps) throw new Error('infra-apps repo not found') + + // System repos must carry the protection topic + expect(mirror.topics).toContain(TOPIC_SYSTEM_MANAGED) + expect(infraApps.topics).toContain(TOPIC_SYSTEM_MANAGED) + + // Create an orphan plugin-managed repo directly in GitLab (simulates a repo the + // console no longer tracks: DB row removed, GitLab project left behind) + const orphanName = `orphan-${faker.string.uuid().slice(0, 8)}` + const orphan = await gitlabClient.Projects.create({ + name: orphanName, + path: orphanName, + namespaceId: group.id, + }) + await gitlabClient.Projects.edit(orphan.id, { topics: [TOPIC_PLUGIN_MANAGED] }) + + // Second reconciliation: the orphan must be purged, system repos must survive + const project2 = await prisma.project.findUniqueOrThrow({ + where: { id: testProjectId }, + select: projectSelect, + }) + await eventEmitter.emitAsync('project.upsert', project2) + + const reposAfter = await getAll(gitlabClientService.getRepos(testProjectSlug)) + const namesAfter = reposAfter.map(repo => repo.name) + + expect(namesAfter).not.toContain(orphanName) + expect(namesAfter).toContain(MIRROR_REPO_NAME) + expect(namesAfter).toContain(INFRA_APPS_REPO_NAME) + expect(namesAfter).toContain('app') + }, GITLAB_PURGE_SYNC_TIMEOUT) + + it('declared user repos are never purged even without the system-managed topic', async () => { + const project = await prisma.project.findUniqueOrThrow({ + where: { id: testProjectId }, + select: projectSelect, + }) + await eventEmitter.emitAsync('project.upsert', project) + + const repos = await getAll(gitlabClientService.getRepos(testProjectSlug)) + const app = repos.find(repo => repo.name === 'app') + if (!app) throw new Error('app repo not found') + + // The declared user repo carries plugin-managed but NOT system-managed; the + // declared-in-project guard must keep it alive across reconciliations. + expect(app.topics).toContain(TOPIC_PLUGIN_MANAGED) + expect(app.topics).not.toContain(TOPIC_SYSTEM_MANAGED) + + const project2 = await prisma.project.findUniqueOrThrow({ + where: { id: testProjectId }, + select: projectSelect, + }) + await eventEmitter.emitAsync('project.upsert', project2) + + const reposAfter = await getAll(gitlabClientService.getRepos(testProjectSlug)) + expect(reposAfter.some(repo => repo.name === 'app')).toBe(true) + }, GITLAB_PURGE_SYNC_TIMEOUT) + + it('purge does not fail when a repo is already marked for deletion', async () => { + // Create a NEW orphan repo in GitLab, then delete it directly (async). The + // subsequent reconcile's purge observes it as still present and DELETEs again + // → GitLab answers 400 "Already Marked for Deletion" which must be swallowed + // so the whole reconciliation succeeds and the system repos survive. + const groupPath = `${config.projectsRootDir}/${testProjectSlug}` + const group = z.object({ id: z.number() }).parse(await gitlabClientService.getGroupByPath(groupPath)) + const orphanName = `orphan-race-${faker.string.uuid().slice(0, 8)}` + const orphan = await gitlabClient.Projects.create({ + name: orphanName, + path: orphanName, + namespaceId: group.id, + }) + await gitlabClient.Projects.edit(orphan.id, { topics: [TOPIC_PLUGIN_MANAGED] }) + + // Simulate the async-deletion race: DELETE the orphan directly (not through + // the reconcile) so it is marked for deletion, then reconcile immediately. + await gitlabClientService.deleteProjectGroupRepo(testProjectSlug, orphanName).catch(() => {}) + + const project = await prisma.project.findUniqueOrThrow({ + where: { id: testProjectId }, + select: projectSelect, + }) + await expect(eventEmitter.emitAsync('project.upsert', project)).resolves.not.toThrow() + + const namesAfter = (await getAll(gitlabClientService.getRepos(testProjectSlug))).map(repo => repo.name) + expect(namesAfter).toContain(MIRROR_REPO_NAME) + expect(namesAfter).toContain(INFRA_APPS_REPO_NAME) + expect(namesAfter).toContain('app') + }, GITLAB_PURGE_SYNC_TIMEOUT) }) it('should remove project group from GitLab on delete', async () => { @@ -306,5 +410,5 @@ describeWithGitLab('GitlabService (e2e)', () => { const group = await gitlabClientService.getGroupByPath(groupPath) expect(group).toBeUndefined() - }, EXTERNAL_SYNC_TIMEOUT) + }, GITLAB_SYNC_TIMEOUT) }) diff --git a/apps/server-nestjs/test/keycloak.e2e-spec.ts b/apps/server-nestjs/test/keycloak.e2e-spec.ts index 673b696594..5abcb02576 100644 --- a/apps/server-nestjs/test/keycloak.e2e-spec.ts +++ b/apps/server-nestjs/test/keycloak.e2e-spec.ts @@ -18,7 +18,7 @@ import { KEYCLOAK_ADMIN_CLIENT, KeycloakClientService } from '../src/modules/key import { projectSelect } from '../src/modules/keycloak/keycloak-datastore.service' import { KeycloakModule } from '../src/modules/keycloak/keycloak.module' import { getDotenvPaths } from '../src/utils/dotenv.utils' -import { KEYCLOAK_GROUP_SYNC_TIMEOUT } from './e2e-timeout' +import { KEYCLOAK_GROUP_SYNC_TIMEOUT } from './constants' const canRunKeycloakE2E = Boolean(process.env.E2E) diff --git a/apps/server-nestjs/test/nexus.e2e-spec.ts b/apps/server-nestjs/test/nexus.e2e-spec.ts index 5a680a404c..3c0d31f7ef 100644 --- a/apps/server-nestjs/test/nexus.e2e-spec.ts +++ b/apps/server-nestjs/test/nexus.e2e-spec.ts @@ -22,6 +22,7 @@ import { generateNexusCredPath } from '../src/modules/nexus/nexus.utils' import { VaultClientService } from '../src/modules/vault/vault-client.service' import { VaultModule } from '../src/modules/vault/vault.module' import { getDotenvPaths } from '../src/utils/dotenv.utils' +import { NEXUS_SYNC_TIMEOUT } from './constants' const canRunNexusE2E = Boolean(process.env.E2E) @@ -146,7 +147,7 @@ describeWithNexus('NexusService (e2e)', () => { const secret = await vaultService.read(vaultPath) expect(secret.data?.NEXUS_USERNAME).toBe(testProjectSlug) expect(secret.data?.NEXUS_PASSWORD).toBeTruthy() - }) + }, NEXUS_SYNC_TIMEOUT) it('should remove project from Nexus on delete', async () => { const project = await prisma.project.findUniqueOrThrow({ @@ -168,5 +169,5 @@ describeWithNexus('NexusService (e2e)', () => { const users = await nexusClient.getSecurityUsers(testProjectSlug) expect(users.some(u => u.userId === testProjectSlug)).toBe(false) - }) + }, NEXUS_SYNC_TIMEOUT) }) diff --git a/apps/server-nestjs/test/sonarqube.e2e-spec.ts b/apps/server-nestjs/test/sonarqube.e2e-spec.ts index 0830696c7a..ac301a8a53 100644 --- a/apps/server-nestjs/test/sonarqube.e2e-spec.ts +++ b/apps/server-nestjs/test/sonarqube.e2e-spec.ts @@ -25,7 +25,7 @@ import { VaultClientService } from '../src/modules/vault/vault-client.service' import { VaultModule } from '../src/modules/vault/vault.module' import { getDotenvPaths } from '../src/utils/dotenv.utils' import { getAll } from '../src/utils/iterable.utils' -import { SONARQUBE_PROJECT_TIMEOUT } from './e2e-timeout' +import { SONARQUBE_PROJECT_TIMEOUT } from './constants' const canRunSonarqubeE2E = Boolean(process.env.E2E) diff --git a/apps/server-nestjs/test/vault.e2e-spec.ts b/apps/server-nestjs/test/vault.e2e-spec.ts index 73c484cafb..8404cbcdcf 100644 --- a/apps/server-nestjs/test/vault.e2e-spec.ts +++ b/apps/server-nestjs/test/vault.e2e-spec.ts @@ -16,7 +16,7 @@ import { projectSelect } from '../src/modules/vault/vault-datastore.service' import { makeProjectWithDetails } from '../src/modules/vault/vault-testing.utils' import { VaultModule } from '../src/modules/vault/vault.module' import { getDotenvPaths } from '../src/utils/dotenv.utils' -import { VAULT_PROVISION_TIMEOUT } from './e2e-timeout' +import { VAULT_PROVISION_TIMEOUT } from './constants' const canRunVaultE2E = Boolean(process.env.E2E) diff --git a/apps/server-nestjs/test/zone.e2e-spec.ts b/apps/server-nestjs/test/zone.e2e-spec.ts index 462359a575..76123f904a 100644 --- a/apps/server-nestjs/test/zone.e2e-spec.ts +++ b/apps/server-nestjs/test/zone.e2e-spec.ts @@ -16,7 +16,7 @@ import { makeZoneWithDetails } from '../src/modules/vault/vault-testing.utils' import { VaultModule } from '../src/modules/vault/vault.module' import { VaultService } from '../src/modules/vault/vault.service' import { getDotenvPaths } from '../src/utils/dotenv.utils' -import { VAULT_PROVISION_TIMEOUT } from './e2e-timeout' +import { VAULT_PROVISION_TIMEOUT } from './constants' const canRunZoneE2E = Boolean(process.env.E2E)