fix(server-nestjs): make GitLab repo create and commit idempotent

apps/server-nestjs/src/modules/gitlab/gitlab-client.service.spec.ts

@@ -14,6 +14,7 @@ import { GITLAB_REST_CLIENT, GitlabClientService } from './gitlab-client.service
 import {
   makeAccessTokenExposedSchema,
   makeAccessTokenSchema,
+  makeCommitAction,
   makeExpandedGroupSchema,
   makeExpandedUserSchema,
   makeGitbeakerRequestError,
@@ -183,6 +184,54 @@ describe('gitlab-client', () => {
 
       expect(gitlabApi.Commits.create).not.toHaveBeenCalled()
     })
+
+    it('should tolerate an already-applied commit (race) when the file now exists', async () => {
+      const repoId = 1
+      const repo = makeProjectSchema({ id: repoId })
+      const message = 'ci: :robot_face: Sync file'
+      const alreadyExistsError = makeGitbeakerRequestError({ description: 'has already been taken', status: 400, statusText: 'Bad Request' })
+      gitlabApi.Commits.create.mockRejectedValue(alreadyExistsError)
+      gitlabApi.RepositoryFiles.show.mockResolvedValue(makeRepositoryFileExpandedSchema())
+
+      await expect(service.maybeCreateCommit(repo, message, [makeCommitAction({ action: 'create' })]))
+        .resolves.toBeUndefined()
+      expect(gitlabApi.Commits.create).toHaveBeenCalledOnce()
+    })
+
+    it('should rethrow when the file is still absent after an already-exists commit error', async () => {
+      const repoId = 1
+      const repo = makeProjectSchema({ id: repoId })
+      const message = 'ci: :robot_face: Sync file'
+      const alreadyExistsError = makeGitbeakerRequestError({ description: 'has already been taken', status: 400, statusText: 'Bad Request' })
+      gitlabApi.Commits.create.mockRejectedValue(alreadyExistsError)
+      gitlabApi.RepositoryFiles.show.mockRejectedValue(makeGitbeakerRequestError({ description: '404 File Not Found' }))
+
+      await expect(service.maybeCreateCommit(repo, message, [makeCommitAction({ action: 'create' })]))
+        .rejects.toThrow()
+      expect(gitlabApi.Commits.create).toHaveBeenCalledOnce()
+    })
+  })
+
+  describe('ensureGroupRepo', () => {
+    it('should reload the existing repo on a create collision (race)', async () => {
+      const groupId = 99
+      const repoName = 'observability-values'
+      const existingRepo = makeProjectSchema({ id: 42, name: repoName, path_with_namespace: `forge/${repoName}` })
+      const collisionError = makeGitbeakerRequestError({ description: 'has already been taken', status: 400, statusText: 'Bad Request' })
+      gitlabApi.Projects.create.mockRejectedValue(collisionError)
+      const gitlabProjectsAllMock = gitlabApi.Projects.all as MockedFunction<typeof gitlabApi.Projects.all>
+      gitlabProjectsAllMock.mockResolvedValueOnce({ data: [existingRepo], paginationInfo: { next: null } })
+
+      const result = await service.ensureGroupRepo(groupId, repoName)
+
+      expect(result).toEqual(existingRepo)
+      expect(gitlabApi.Projects.create).toHaveBeenCalledWith(expect.objectContaining({
+        name: repoName,
+        path: repoName,
+        namespaceId: groupId,
+      }))
+      expect(gitlabProjectsAllMock).toHaveBeenCalledOnce()
+    })
   })
 
   describe('getOrCreateProjectGroup', () => {

apps/server-nestjs/src/modules/gitlab/gitlab-client.service.ts

@@ -18,7 +18,6 @@ import type {
 import type { ConfigType } from '@nestjs/config'
 import { join } from 'node:path'
 import { defaultBranchName } from '@cpn-console/shared'
-import { GitbeakerRequestError } from '@gitbeaker/requester-utils'
 import { Gitlab as GitlabRest } from '@gitbeaker/rest'
 import { Inject, Injectable, Logger } from '@nestjs/common'
 import { gitlabConfigFactory } from '../../config/gitlab.config'
@@ -36,7 +35,7 @@ import {
   TOPIC_SYSTEM_MANAGED,
   USER_ID_CUSTOM_ATTRIBUTE_KEY,
 } from './gitlab.constants'
-import { generateGitlabCIConfigContent, generateMirrorScriptContent, hasFileContentChanged, hasGitbeakerCause, isGitbeakerNotFound } from './gitlab.utils'
+import { generateGitlabCIConfigContent, generateMirrorScriptContent, hasFileContentChanged, hasGitbeakerCause, isGitbeakerNotFound, isGitbeakerUnauthorized } from './gitlab.utils'
 
 export const GITLAB_REST_CLIENT = Symbol('GITLAB_REST_CLIENT')
 
@@ -136,7 +135,7 @@ export class GitlabClientService {
     )
   }
 
-  async createGroup(path: string) {
+  async ensureGroup(path: string) {
     this.logger.log(`Creating a GitLab group at path ${path}`)
     try {
       const created = await this.client.Groups.create(path, path)
@@ -157,7 +156,7 @@ export class GitlabClientService {
     }
   }
 
-  async createSubGroup(parentGroup: CondensedGroupSchemaWith<'id' | 'full_path'>, name: string, fullPath: string) {
+  async ensureSubGroup(parentGroup: CondensedGroupSchemaWith<'id' | 'full_path'>, name: string, fullPath: string) {
     this.logger.log(`Creating a GitLab subgroup ${fullPath} (parentId=${parentGroup.id})`)
     try {
       const created = await this.client.Groups.create(name, name, { parentId: parentGroup.id })
@@ -185,15 +184,15 @@ export class GitlabClientService {
     if (!rootGroupPath) throw new Error('Invalid projects root dir')
 
     this.logger.verbose(`Resolving GitLab group path ${path} (depth=${1 + parts.length})`)
-    let parentGroup = await this.getGroupByPath(rootGroupPath) ?? await this.createGroup(rootGroupPath)
+    let parentGroup = await this.getGroupByPath(rootGroupPath) ?? await this.ensureGroup(rootGroupPath)
     if (this.config.projectRootDir && parentGroup.full_path === this.config.projectRootDir) {
       await this.setManagedRootGroupAttributes(parentGroup.id)
     }
 
     let currentFullPath: string
     for (const part of parts) {
       currentFullPath = `${parentGroup.full_path}/${part}`
-      parentGroup = await this.getGroupByPath(currentFullPath) ?? await this.createSubGroup(parentGroup, part, currentFullPath)
+      parentGroup = await this.getGroupByPath(currentFullPath) ?? await this.ensureSubGroup(parentGroup, part, currentFullPath)
     }
 
     this.logger.verbose(`GitLab group path resolved (path=${path}, groupId=${parentGroup.id})`)
@@ -294,16 +293,32 @@ export class GitlabClientService {
     return repo
   }
 
-  async createGroupRepo(groupId: number, repoName: string, description?: string) {
+  async ensureGroupRepo(groupId: number, repoName: string, description?: string) {
     this.logger.log(`Creating a GitLab repository in a standalone group (groupId=${groupId}, repoName=${repoName})`)
-    const created = await this.client.Projects.create({
-      name: repoName,
-      path: repoName,
-      namespaceId: groupId,
-      description,
-      defaultBranch: defaultBranchName,
-    })
-    return created
+    try {
+      const created = await this.client.Projects.create({
+        name: repoName,
+        path: repoName,
+        namespaceId: groupId,
+        description,
+        defaultBranch: defaultBranchName,
+      })
+      return created
+    } catch (error) {
+      if (hasGitbeakerCause(error, 'has already been taken')) {
+        this.logger.warn(`GitLab repository already exists (race); reloading (groupId=${groupId}, repoName=${repoName})`)
+        const existing = await find(
+          this.offsetPaginate(opts => this.client.Projects.all({
+            search: repoName,
+            orderBy: 'path',
+            ...opts,
+          })),
+          p => p.name === repoName,
+        )
+        if (existing) return existing
+      }
+      throw error
+    }
   }
 
   async getFile(repo: CondensedProjectSchemaWith<'id'>, filePath: string, ref: string = 'main') {
@@ -329,7 +344,24 @@ export class GitlabClientService {
       return
     }
     this.logger.log(`Creating a GitLab commit (repoId=${repo.id}, ref=${ref}, actions=${actions.length})`)
-    await this.client.Commits.create(repo.id, ref, message, actions)
+    try {
+      await this.client.Commits.create(repo.id, ref, message, actions)
+    } catch (error) {
+      // Two overlapping syncs can both see a file absent and both emit a `create`
+      // action; the loser's Commits.create collides with the winner's commit (400).
+      // Treat that as already-committed and continue when the file is now present.
+      const alreadyCommitted = hasGitbeakerCause(error, 'has already been taken')
+        || hasGitbeakerCause(error, /already exists/i)
+        || (error instanceof GitbeakerRequestError && error.cause?.response?.status === 400)
+      if (alreadyCommitted) {
+        const createAction = actions.find(action => action.action === 'create')
+        if (!createAction || await this.getFile(repo, createAction.filePath, ref)) {
+          this.logger.warn(`GitLab commit already applied (race); continuing (repoId=${repo.id}, ref=${ref})`)
+          return
+        }
+      }
+      throw error
+    }
     this.logger.verbose(`GitLab commit created (repoId=${repo.id}, ref=${ref}, actions=${actions.length})`)
   }
 
@@ -638,7 +670,7 @@ export class GitlabClientService {
       const self = await client.PersonalAccessTokens.show()
       return self.active && !self.revoked
     } catch (error) {
-      if (error instanceof GitbeakerRequestError && error.cause?.response.status === 401) return false
+      if (isGitbeakerUnauthorized(error)) return false
       throw error
     }
   }

apps/server-nestjs/src/modules/gitlab/gitlab.utils.ts

@@ -259,3 +259,7 @@ export function hasGitbeakerCause(error: unknown, pattern: string | RegExp): err
     : JSON.stringify(error.cause?.description ?? '')
   return typeof pattern === 'string' ? description.includes(pattern) : pattern.test(description)
 }
+
+export function isGitbeakerUnauthorized(error: unknown): error is GitbeakerRequestError {
+  return error instanceof GitbeakerRequestError && error.cause?.response?.status === 401
+}

apps/server-nestjs/src/modules/nexus/nexus-client.service.ts

@@ -1,6 +1,7 @@
 import { Inject, Injectable } from '@nestjs/common'
 import { StartActiveSpan } from '../infrastructure/telemetry/telemetry.decorator'
-import { NexusError, NexusHttpClientService } from './nexus-http-client.service'
+import { NexusHttpClientService } from './nexus-http-client.service'
+import { isNexusNotFound } from './nexus.utils'
 
 interface NexusRepositoryStorage {
   blobStoreName: string
@@ -114,7 +115,7 @@ export class NexusClientService {
       const res = await this.http.fetch<NexusMavenHostedRepository>(`repositories/maven/hosted/${name}`)
       return res.data
     } catch (error) {
-      if (error instanceof NexusError && error.status === 404) return null
+      if (isNexusNotFound(error)) return null
       throw error
     }
   }
@@ -145,7 +146,7 @@ export class NexusClientService {
       const res = await this.http.fetch<NexusMavenGroupRepository>(`repositories/maven/group/${name}`)
       return res.data
     } catch (error) {
-      if (error instanceof NexusError && error.status === 404) return null
+      if (isNexusNotFound(error)) return null
       throw error
     }
   }
@@ -156,7 +157,7 @@ export class NexusClientService {
       const res = await this.http.fetch<NexusNpmHostedRepository>(`repositories/npm/hosted/${name}`)
       return res.data
     } catch (error) {
-      if (error instanceof NexusError && error.status === 404) return null
+      if (isNexusNotFound(error)) return null
       throw error
     }
   }
@@ -177,7 +178,7 @@ export class NexusClientService {
       const res = await this.http.fetch<NexusNpmGroupRepository>(`repositories/npm/group/${name}`)
       return res.data
     } catch (error) {
-      if (error instanceof NexusError && error.status === 404) return null
+      if (isNexusNotFound(error)) return null
       throw error
     }
   }
@@ -198,7 +199,7 @@ export class NexusClientService {
       const res = await this.http.fetch<NexusPrivilege>(`security/privileges/${name}`)
       return res.data
     } catch (error) {
-      if (error instanceof NexusError && error.status === 404) return null
+      if (isNexusNotFound(error)) return null
       throw error
     }
   }
@@ -218,7 +219,7 @@ export class NexusClientService {
     try {
       await this.http.fetch(`security/privileges/${name}`, { method: 'DELETE' })
     } catch (error) {
-      if (error instanceof NexusError && error.status === 404) return
+      if (isNexusNotFound(error)) return
       throw error
     }
   }
@@ -229,7 +230,7 @@ export class NexusClientService {
       const res = await this.http.fetch<NexusRole>(`security/roles/${id}`)
       return res.data
     } catch (error) {
-      if (error instanceof NexusError && error.status === 404) return null
+      if (isNexusNotFound(error)) return null
       throw error
     }
   }
@@ -249,7 +250,7 @@ export class NexusClientService {
     try {
       await this.http.fetch(`security/roles/${id}`, { method: 'DELETE' })
     } catch (error) {
-      if (error instanceof NexusError && error.status === 404) return
+      if (isNexusNotFound(error)) return
       throw error
     }
   }
@@ -280,7 +281,7 @@ export class NexusClientService {
     try {
       await this.http.fetch(`security/users/${userId}`, { method: 'DELETE' })
     } catch (error) {
-      if (error instanceof NexusError && error.status === 404) return
+      if (isNexusNotFound(error)) return
       throw error
     }
   }
@@ -290,7 +291,7 @@ export class NexusClientService {
     try {
       await this.http.fetch(`repositories/${name}`, { method: 'DELETE' })
     } catch (error) {
-      if (error instanceof NexusError && error.status === 404) return
+      if (isNexusNotFound(error)) return
       throw error
     }
   }

apps/server-nestjs/src/modules/nexus/nexus.service.ts

@@ -14,7 +14,7 @@ import { nexusConfigFactory } from '../../config/nexus.config'
 import { StartActiveSpan } from '../infrastructure/telemetry/telemetry.decorator'
 import { capturePluginResult } from '../plugin/plugin.utils'
 import { VaultClientService } from '../vault/vault-client.service'
-import { VaultError } from '../vault/vault-http-client.service'
+import { isVaultNotFound } from '../vault/vault.utils'
 import { NexusClientService } from './nexus-client.service'
 import { NexusDatastoreService } from './nexus-datastore.service'
 import {
@@ -437,7 +437,7 @@ export class NexusService {
     try {
       existingPassword = await this.vault.read(vaultPath).then(res => res.data?.NEXUS_PASSWORD)
     } catch (error) {
-      if (error instanceof VaultError && error.kind === 'NotFound') {
+      if (isVaultNotFound(error)) {
         existingPassword = undefined
       } else {
         throw error
@@ -581,7 +581,7 @@ export class NexusService {
     try {
       await this.vault.delete(vaultPath)
     } catch (error) {
-      if (error instanceof VaultError && error.kind === 'NotFound') return
+      if (isVaultNotFound(error)) return
       throw error
     }
   }

apps/server-nestjs/src/modules/nexus/nexus.utils.spec.ts

@@ -1,8 +1,21 @@
 import { describe, expect, it } from 'vitest'
-import { generateNexusCredPath } from './nexus.utils'
+import { NexusError } from './nexus-http-client.service'
+import { generateNexusCredPath, isNexusNotFound } from './nexus.utils'
 
 describe('nexus path helpers', () => {
   it('scopes the NEXUS credentials to the project', () => {
     expect(generateNexusCredPath('forge', 'my-project')).toBe('forge/my-project/NEXUS')
   })
 })
+
+describe('isNexusNotFound', () => {
+  it('matches a 404 NexusError', () => {
+    expect(isNexusNotFound(new NexusError('HttpError', 'not found', { status: 404 }))).toBe(true)
+  })
+
+  it('rejects a non-404 NexusError and non-Nexus errors', () => {
+    expect(isNexusNotFound(new NexusError('HttpError', 'conflict', { status: 409 }))).toBe(false)
+    expect(isNexusNotFound(new Error('boom'))).toBe(false)
+    expect(isNexusNotFound(null)).toBe(false)
+  })
+})

apps/server-nestjs/src/modules/nexus/nexus.utils.ts

@@ -1,5 +1,6 @@
 import type { ProjectWithDetails } from './nexus-datastore.service'
 import { randomBytes } from 'node:crypto'
+import { NexusError } from './nexus-http-client.service'
 
 export function getPluginConfig(project: ProjectWithDetails, key: string) {
   return project.plugins?.find(p => p.key === key)?.value
@@ -23,3 +24,7 @@ export function generateMavenHostedRepoName(project: ProjectWithDetails, kind: M
 export function generateNpmHostedRepoName(project: ProjectWithDetails) {
   return `${project.slug}-npm`
 }
+
+export function isNexusNotFound(error: unknown): error is NexusError {
+  return error instanceof NexusError && error.status === 404
+}

apps/server-nestjs/src/modules/observability/observability-client.service.ts

@@ -35,7 +35,7 @@ export class ObservabilityClientService {
     }
 
     this.logger.log(`Creating GitLab observability values repository ${OBSERVABILITY_REPO_NAME}`)
-    return this.gitlab.createGroupRepo(group.id, OBSERVABILITY_REPO_NAME)
+    return this.gitlab.ensureGroupRepo(group.id, OBSERVABILITY_REPO_NAME)
   }
 
   async getValuesFile(repo: CondensedProjectSchemaWith<'id'>): Promise<ObservabilityData> {

apps/server-nestjs/src/modules/registry/registry.service.ts

@@ -22,7 +22,7 @@ import { find } from '../../utils/iterable.utils'
 import { StartActiveSpan } from '../infrastructure/telemetry/telemetry.decorator'
 import { capturePluginResult } from '../plugin/plugin.utils'
 import { VaultClientService } from '../vault/vault-client.service'
-import { VaultError } from '../vault/vault-http-client.service'
+import { isVaultNotFound } from '../vault/vault.utils'
 import { RegistryClientService, roAccess, rwAccess } from './registry-client.service'
 import { RegistryDatastoreService } from './registry-datastore.service'
 import {
@@ -106,7 +106,7 @@ export class RegistryService {
     const relativeVaultPath = `REGISTRY/${robotName}`
     const vaultPath = getProjectVaultPath(project, this.baseConfig.projectsRootDir, relativeVaultPath)
     const vaultRobotSecret = await this.vault.read<VaultRobotSecret>(vaultPath).catch((error) => {
-      if (error instanceof VaultError && error.kind === 'NotFound') return null
+      if (isVaultNotFound(error)) return null
       throw error
     })