fix(server-nestjs): protect system GitLab repos by topic, cover observability

apps/server-nestjs/src/modules/gitlab/gitlab-client.service.spec.ts

@@ -287,7 +287,7 @@ describe('gitlab-client', () => {
       gitlabApi.Projects.show.mockResolvedValue(makeProjectSchema({ id: repoId }))
       gitlabApi.Projects.edit.mockResolvedValue(makeProjectSchema({ id: repoId, name: repoName }))
 
-      const result = await service.upsertProjectGroupRepo(projectSlug, repoName, 'desc')
+      const result = await service.upsertProjectGroupRepo(projectSlug, repoName, { description: 'desc' })
 
       expect(result).toEqual(expect.objectContaining({ id: repoId, name: repoName }))
       expect(gitlabApi.ProjectCustomAttributes.set).toHaveBeenCalledWith(repoId, MANAGED_BY_CONSOLE_CUSTOM_ATTRIBUTE_KEY, 'true')

apps/server-nestjs/src/modules/gitlab/gitlab-client.service.ts

@@ -33,6 +33,7 @@ import {
   SPECIAL_REPO_NAMES,
   TOKEN_DESCRIPTION,
   TOPIC_PLUGIN_MANAGED,
+  TOPIC_SYSTEM_MANAGED,
   USER_ID_CUSTOM_ATTRIBUTE_KEY,
 } from './gitlab.constants'
 import { generateGitlabCIConfigContent, generateMirrorScriptContent, hasFileContentChanged, hasGitbeakerCause, isGitbeakerNotFound } from './gitlab.utils'
@@ -42,6 +43,12 @@ export const GITLAB_REST_CLIENT = Symbol('GITLAB_REST_CLIENT')
 type With<T, K extends keyof T> = T & Required<Pick<T, K>>
 export type CondensedGroupSchemaWith<T extends keyof CondensedGroupSchema> = With<CondensedGroupSchema, T>
 export type CondensedProjectSchemaWith<T extends keyof CondensedProjectSchema> = With<CondensedProjectSchema, T>
+
+export interface UpsertProjectGroupRepoOptions {
+  description?: string
+  ciConfigPath?: string
+  extraTopics?: string[]
+}
 export type EditUserOptionsWith<T extends keyof EditUserOptions> = With<EditUserOptions, T>
 type UserSchema = SimpleUserSchema | ExpandedUserSchema
 
@@ -460,19 +467,28 @@ export class GitlabClientService {
     }
   }
 
-  async upsertProjectGroupRepo(projectSlug: string, repoName: string, description?: string, ciConfigPath?: string) {
+  async upsertProjectGroupRepo(projectSlug: string, repoName: string, options: UpsertProjectGroupRepoOptions = {}) {
+    const { description, ciConfigPath, extraTopics = [] } = options
     const fullPath = `${projectSlug}/${repoName}`
     const repo = await this.getOrCreateProjectGroupRepo(projectSlug, fullPath, ciConfigPath)
     const updated = await this.client.Projects.edit(repo.id, {
       name: repoName,
       path: repoName,
-      topics: [TOPIC_PLUGIN_MANAGED],
+      topics: [TOPIC_PLUGIN_MANAGED, ...extraTopics],
       description,
       ciConfigPath: ciConfigPath ?? '',
     })
     return updated
   }
 
+  // System repos (mirror, infra-apps, observability values, ...) are console-owned plumbing:
+  // created in the project subgroup but never listed in project.repositories, so the orphan
+  // purge must never delete them. They carry the dedicated system-managed topic; any plugin
+  // can opt its system repo in by upserting it through this wrapper.
+  async upsertProjectGroupSystemRepo(projectSlug: string, repoName: string, options: Omit<UpsertProjectGroupRepoOptions, 'extraTopics'> = {}) {
+    return this.upsertProjectGroupRepo(projectSlug, repoName, { ...options, extraTopics: [TOPIC_SYSTEM_MANAGED] })
+  }
+
   async deleteProjectGroupRepo(projectSlug: string, repoName: string) {
     const fullPath = `${projectSlug}/${repoName}`
     const repo = await this.getOrCreateProjectGroupRepo(projectSlug, fullPath)
@@ -585,7 +601,7 @@ export class GitlabClientService {
   }
 
   async upsertProjectMirrorRepo(projectSlug: string) {
-    return this.upsertProjectGroupRepo(projectSlug, MIRROR_REPO_NAME)
+    return this.upsertProjectGroupSystemRepo(projectSlug, MIRROR_REPO_NAME)
   }
 
   async getProjectToken(group: CondensedGroupSchemaWith<'id'>, projectSlug: string) {

apps/server-nestjs/src/modules/gitlab/gitlab.constants.ts

@@ -15,6 +15,9 @@ export const GITLAB_CI_CONFIG_PATH = '.gitlab-ci-dso.yml'
 
 // Managed resources sentinel
 export const TOPIC_PLUGIN_MANAGED = 'plugin-managed'
+
+// Console-owned plumbing repos (mirror, infra-apps, observability...), protected from orphan purge
+export const TOPIC_SYSTEM_MANAGED = 'system-managed'
 export const TOKEN_DESCRIPTION = 'mirroring-from-external-repo'
 
 // Default group paths for console roles

apps/server-nestjs/src/modules/gitlab/gitlab.service.spec.ts

@@ -7,11 +7,12 @@ import { Test } from '@nestjs/testing'
 import { beforeEach, describe, expect, it, vi } from 'vitest'
 import { mockDeep } from 'vitest-mock-extended'
 import { gitlabConfigFactory } from '../../config/gitlab.config'
+import { OBSERVABILITY_REPOSITORY } from '../observability/observability.constants'
 import { VaultClientService } from '../vault/vault-client.service'
 import { GitlabClientService } from './gitlab-client.service'
 import { GitlabDatastoreService } from './gitlab-datastore.service'
 import { makeAccessTokenExposedSchema, makeExpandedUserSchema, makeGroupSchema, makeMemberSchema, makePipeline, makePipelineTriggerToken, makeProjectSchema, makeProjectWithDetails } from './gitlab-testing.utils'
-import { PLUGIN_NAME, TOPIC_PLUGIN_MANAGED } from './gitlab.constants'
+import { INFRA_APPS_REPO_NAME, MIRROR_REPO_NAME, PLUGIN_NAME, TOPIC_PLUGIN_MANAGED, TOPIC_SYSTEM_MANAGED } from './gitlab.constants'
 import { GitlabService } from './gitlab.service'
 
 describe('gitlabService', () => {
@@ -181,6 +182,55 @@ describe('gitlabService', () => {
       expect(gitlab.deleteProjectGroupRepo).toHaveBeenCalledTimes(1)
     })
 
+    it('should never delete system repositories (infra-apps, mirror) even though they are not in project.repositories', async () => {
+      const project = makeProjectWithDetails({ repositories: [] })
+      const group = makeGroupSchema({ id: 123, name: 'project-1', path: 'project-1', full_path: 'forge/console/project-1', full_name: 'forge/console/project-1', parent_id: 1 })
+      const infraApps = makeProjectSchema({ name: INFRA_APPS_REPO_NAME, topics: [TOPIC_PLUGIN_MANAGED, TOPIC_SYSTEM_MANAGED] })
+      const mirror = makeProjectSchema({ name: MIRROR_REPO_NAME, topics: [TOPIC_PLUGIN_MANAGED, TOPIC_SYSTEM_MANAGED] })
+      const userRepo = makeProjectSchema({ name: 'user-repo', topics: [TOPIC_PLUGIN_MANAGED] })
+
+      gitlab.getOrCreateProjectSubGroup.mockResolvedValue(group)
+      gitlab.getGroupMembers.mockResolvedValue([])
+      gitlab.getRepos.mockImplementation(() => (async function* () {
+        yield infraApps
+        yield mirror
+        yield userRepo
+      })())
+      gitlab.deleteProjectGroupRepo.mockResolvedValue(undefined)
+      gitlab.upsertProjectMirrorRepo.mockResolvedValue(makeProjectSchema({ id: 1, name: 'mirror', path: 'mirror', path_with_namespace: 'forge/console/project-1/mirror', empty_repo: false }))
+      gitlab.getOrCreateMirrorPipelineTriggerToken.mockResolvedValue(makePipelineTriggerToken())
+
+      await service.handleUpsert(project)
+
+      expect(gitlab.deleteProjectGroupRepo).toHaveBeenCalledTimes(1)
+      expect(gitlab.deleteProjectGroupRepo).toHaveBeenCalledWith(project.slug, 'user-repo')
+      expect(gitlab.deleteProjectGroupRepo).not.toHaveBeenCalledWith(project.slug, INFRA_APPS_REPO_NAME)
+      expect(gitlab.deleteProjectGroupRepo).not.toHaveBeenCalledWith(project.slug, MIRROR_REPO_NAME)
+    })
+
+    it('should never delete the observability system repository (infra-observability) even though it is not in project.repositories', async () => {
+      const project = makeProjectWithDetails({ repositories: [] })
+      const group = makeGroupSchema({ id: 123, name: 'project-1', path: 'project-1', full_path: 'forge/console/project-1', full_name: 'forge/console/project-1', parent_id: 1 })
+      const observabilityRepo = makeProjectSchema({ name: OBSERVABILITY_REPOSITORY, topics: [TOPIC_PLUGIN_MANAGED, TOPIC_SYSTEM_MANAGED] })
+      const userRepo = makeProjectSchema({ name: 'user-repo', topics: [TOPIC_PLUGIN_MANAGED] })
+
+      gitlab.getOrCreateProjectSubGroup.mockResolvedValue(group)
+      gitlab.getGroupMembers.mockResolvedValue([])
+      gitlab.getRepos.mockImplementation(() => (async function* () {
+        yield observabilityRepo
+        yield userRepo
+      })())
+      gitlab.deleteProjectGroupRepo.mockResolvedValue(undefined)
+      gitlab.upsertProjectMirrorRepo.mockResolvedValue(makeProjectSchema({ id: 1, name: 'mirror', path: 'mirror', path_with_namespace: 'forge/console/project-1/mirror', empty_repo: false }))
+      gitlab.getOrCreateMirrorPipelineTriggerToken.mockResolvedValue(makePipelineTriggerToken())
+
+      await service.handleUpsert(project)
+
+      expect(gitlab.deleteProjectGroupRepo).toHaveBeenCalledTimes(1)
+      expect(gitlab.deleteProjectGroupRepo).toHaveBeenCalledWith(project.slug, 'user-repo')
+      expect(gitlab.deleteProjectGroupRepo).not.toHaveBeenCalledWith(project.slug, OBSERVABILITY_REPOSITORY)
+    })
+
     it('should not delete orphan repositories without the correct topic even if purge enabled', async () => {
       const project = makeProjectWithDetails({
         plugins: [{ pluginName: PLUGIN_NAME, key: 'purge', value: ENABLED }],

apps/server-nestjs/src/modules/gitlab/gitlab.service.ts

@@ -41,6 +41,8 @@ import {
   generateUsername,
   generateUsernameCandidates,
   getProjectPluginConfig,
+  hasGitbeakerCause,
+  isDeclaredRepo,
   isOwnedRepo,
   isOwnedUser,
   isSystemRepo,
@@ -146,9 +148,9 @@ export class GitlabService {
     const members = await this.gitlab.getGroupMembers(group)
     this.logger.verbose(`Loaded GitLab project group state (${project.slug}): groupId=${group.id} members=${members.length}`)
     await this.ensureProjectGroupMembers(project, group, members)
+    await this.ensureSystemRepos(project)
     await this.ensureProjectRepos(project)
     await this.purgeOrphanRepos(project)
-    await this.ensureSystemRepos(project)
     this.logger.verbose(`GitLab project group reconciled (${project.slug})`)
   }
 
@@ -370,12 +372,9 @@ export class GitlabService {
         ...(externalHost ? { 'repository.external.host': externalHost } : {}),
         'repository.external': !!repo.externalRepoUrl,
       })
-      await this.gitlab.upsertProjectGroupRepo(
-        project.slug,
-        repo.internalRepoName,
-        undefined,
-        repo.externalRepoUrl ? GITLAB_CI_CONFIG_PATH : undefined,
-      )
+      await this.gitlab.upsertProjectGroupRepo(project.slug, repo.internalRepoName, {
+        ciConfigPath: repo.externalRepoUrl ? GITLAB_CI_CONFIG_PATH : undefined,
+      })
 
       if (repo.externalRepoUrl) {
         span?.setAttribute('repository.mirroring', true)
@@ -399,12 +398,22 @@ export class GitlabService {
     span?.setAttribute('gitlab.repositories.count', gitlabRepositories.length)
 
     // Delete console-owned repos no longer tracked (e.g. console repository deletion).
-    const orphanRepos = gitlabRepositories.filter(r => isOwnedRepo(r) && !isSystemRepo(project, r))
+    const orphanRepos = gitlabRepositories.filter(r => isOwnedRepo(r) && !isSystemRepo(r) && !isDeclaredRepo(project, r))
     span?.setAttribute('orphan.repositories.count', orphanRepos.length)
 
     let removedCount = 0
     await Promise.all(orphanRepos.map(async (orphan) => {
-      await this.gitlab.deleteProjectGroupRepo(project.slug, orphan.name)
+      try {
+        await this.gitlab.deleteProjectGroupRepo(project.slug, orphan.name)
+      } catch (err) {
+        // GitLab deletion is asynchronous: a repo already marked for deletion by a
+        // prior run surfaces a transient 400. Ignore only that; let real errors propagate.
+        if (hasGitbeakerCause(err, /already marked for deletion/)) {
+          this.logger.warn(`Repository already marked for deletion, skipping (project=${project.slug}, repoName=${orphan.name})`)
+          return
+        }
+        throw err
+      }
       removedCount++
       this.logger.log(`Removed a repository from the GitLab project (project=${project.slug}, repoName=${orphan.name})`)
     }))
@@ -464,7 +473,7 @@ export class GitlabService {
   }
 
   private async ensureInfraAppsRepo(project: ProjectWithDetails) {
-    await this.gitlab.upsertProjectGroupRepo(project.slug, INFRA_APPS_REPO_NAME)
+    await this.gitlab.upsertProjectGroupSystemRepo(project.slug, INFRA_APPS_REPO_NAME)
   }
 
   private async ensureMirrorRepo(project: ProjectWithDetails) {

apps/server-nestjs/src/modules/gitlab/gitlab.utils.ts

@@ -4,7 +4,7 @@ import { createHash } from 'node:crypto'
 import { AccessLevel } from '@gitbeaker/core'
 import { GitbeakerRequestError } from '@gitbeaker/requester-utils'
 import { stringify } from 'yaml'
-import { TOPIC_PLUGIN_MANAGED } from './gitlab.constants'
+import { TOPIC_PLUGIN_MANAGED, TOPIC_SYSTEM_MANAGED } from './gitlab.constants'
 
 export type ProjectAccessLevel = Exclude<AccessLevel, (typeof AccessLevel)['ADMIN']>
 
@@ -218,7 +218,19 @@ export function isOwnedRepo(repo: ProjectSchema) {
   return repo.topics?.includes(TOPIC_PLUGIN_MANAGED) ?? false
 }
 
-export function isSystemRepo(project: ProjectWithDetails, repo: ProjectSchema) {
+export function isSystemRepo(repo: ProjectSchema) {
+  // Console-owned plumbing repos (infra-apps, mirror, observability values, ...) carry the
+  // `system-managed` topic and are never listed in project.repositories; protect them from purge.
+  // Topic-based instead of a hardcoded name list so any plugin can opt its system repo in.
+  return repo.topics?.includes(TOPIC_SYSTEM_MANAGED) ?? false
+}
+
+export function isDeclaredRepo(project: ProjectWithDetails, repo: ProjectSchema) {
+  // A repo declared in project.repositories is managed by the console and must never be purged,
+  // even though it may not carry the `system-managed` topic yet (e.g. repos created before the
+  // topic existed). The orphan purge only targets repos that are neither system- nor declared
+  // in the project; keeping this check makes the purge safe for pre-existing repos until the
+  // next reconciliation tags them.
   return project.repositories.some(r => r.internalRepoName === repo.name)
 }
 

apps/server-nestjs/src/modules/observability/observability.service.ts

@@ -100,7 +100,7 @@ export class ObservabilityService {
     const span = trace.getActiveSpan()
     span?.setAttribute('project.slug', project.slug)
     this.logger.verbose(`Ensuring observability project repository for ${project.slug}`)
-    await this.gitlab.upsertProjectGroupRepo(project.slug, OBSERVABILITY_REPOSITORY)
+    await this.gitlab.upsertProjectGroupSystemRepo(project.slug, OBSERVABILITY_REPOSITORY)
   }
 
   @StartActiveSpan()
@@ -114,7 +114,7 @@ export class ObservabilityService {
   }
 
   private async syncChartFiles(project: ProjectWithDetails) {
-    const projectRepo = await this.gitlab.upsertProjectGroupRepo(project.slug, OBSERVABILITY_REPOSITORY)
+    const projectRepo = await this.gitlab.upsertProjectGroupSystemRepo(project.slug, OBSERVABILITY_REPOSITORY)
     const actions = await this.buildChartActions(projectRepo)
     await this.gitlab.maybeCreateCommit(projectRepo, 'ci: :robot_face: Sync observability chart', actions)
   }

apps/server-nestjs/test/argocd.e2e-spec.ts

@@ -20,7 +20,7 @@ import { LoggerModule } from '../src/modules/infrastructure/logger/logger.module
 import { PermissionModule } from '../src/modules/infrastructure/permission/permission.module'
 import { VaultClientService } from '../src/modules/vault/vault-client.service'
 import { getDotenvPaths } from '../src/utils/dotenv.utils'
-import { ARGOCD_RECONCILE_TIMEOUT, EXTERNAL_SYNC_TIMEOUT } from './e2e-timeout'
+import { ARGOCD_RECONCILE_TIMEOUT, GITLAB_SYNC_TIMEOUT } from './constants'
 
 const canRunArgoCDE2E
   = Boolean(process.env.E2E)
@@ -306,5 +306,5 @@ describeWithArgoCD('ArgoCDService (e2e)', () => {
 
     const prodFile = await gitlab.getFile(infraProject, prodFilePath, 'main')
     expect(prodFile).toBeUndefined()
-  }, EXTERNAL_SYNC_TIMEOUT)
+  }, GITLAB_SYNC_TIMEOUT)
 })

apps/server-nestjs/test/e2e-timeout.ts → apps/server-nestjs/test/constants.ts

@@ -2,6 +2,8 @@
 // Each timeout names the task it bounds so a reader knows which system and operation it covers.
 export const SONARQUBE_PROJECT_TIMEOUT = 30_000 // provision + delete a SonarQube project/user
 export const KEYCLOAK_GROUP_SYNC_TIMEOUT = 60_000 // reconcile Keycloak groups/roles
-export const EXTERNAL_SYNC_TIMEOUT = 72_000 // sync GitLab groups/members, teardown Nexus/registry
+export const GITLAB_SYNC_TIMEOUT = 72_000 // sync GitLab groups/members/repos, mirror pipeline trigger
+export const GITLAB_PURGE_SYNC_TIMEOUT = 150_000 // GitLab reconcile twice (create + orphan purge) on shared env
+export const NEXUS_SYNC_TIMEOUT = 72_000 // reconcile Nexus repos/roles/users, teardown on delete
 export const ARGOCD_RECONCILE_TIMEOUT = 144_000 // ArgoCD commit + sync to git
 export const VAULT_PROVISION_TIMEOUT = 180_000 // provision Vault mounts/policies/approles, zone secrets