diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..37ca40d --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,67 @@ +name: Release gates + +on: + push: + pull_request: + +permissions: + contents: read + +jobs: + quality: + name: Python and template checks + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + - name: Set up Python 3.12 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 + with: + python-version: "3.12" + cache: pip + - name: Install project and verification tools + run: | + python -m pip install --upgrade pip + python -m pip install -e ".[dev]" + - name: Verify dependencies + run: python -m pip check + - name: Run unit tests + run: python -m unittest discover -s tests -v + - name: Run pytest + run: python -m pytest -q + - name: Compile Python sources + run: python -m compileall -q ome_memoryguard tests + - name: Lint Python sources + run: python -m ruff check . + - name: Lint CloudFormation template + run: cfn-lint template.yaml + + secrets: + name: Secret scan + runs-on: ubuntu-latest + steps: + - name: Check out full history + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + fetch-depth: 0 + - name: Scan repository + uses: gitleaks/gitleaks-action@ff98106e4c7b2bc287b24eaf42907196329070c7 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + sam: + name: SAM validate and build + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + - name: Set up Python 3.12 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 + with: + python-version: "3.12" + - name: Set up AWS SAM CLI + uses: aws-actions/setup-sam@f84ec7d548307efafe33230528756de3c5841a17 + - name: Validate template + run: sam validate --lint --template-file template.yaml + - name: Build application + run: sam build --no-cached --template-file template.yaml diff --git a/.gitleaks.toml b/.gitleaks.toml new file mode 100644 index 0000000..35558fe --- /dev/null +++ b/.gitleaks.toml @@ -0,0 +1,17 @@ +[extend] +useDefault = true + +[[allowlists]] +description = "Synthetic credential used by the fail-closed demo and test" +targetRules = ["generic-api-key"] +regexTarget = "secret" +regexes = ['''^1234567890abcdef$'''] + +[[allowlists]] +description = "Local dependencies, generated output, and secret-bearing local files" +paths = [ + '''(^|[\\/])\.venv([\\/]|$)''', + '''(^|[\\/])\.aws-sam([\\/]|$)''', + '''(^|[\\/])__pycache__([\\/]|$)''', + '''(^|[\\/])\.env(?:\..+)?$''', +] diff --git a/README.md b/README.md index d405f6f..cac96ea 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,17 @@ risk score, retrieval result, and audit timeline. Local mode does not read `.env .\scripts\verify.ps1 ``` +Run the full local release gate after installing gitleaks and AWS SAM CLI: + +```powershell +.\scripts\verify.ps1 -WithGitleaks -WithSam +``` + +The gitleaks configuration excludes local dependencies, generated artifacts, and `.env`, while its +only source allowlist is the deliberate synthetic credential used by the fail-closed demo/test. +The SAM helper builds from an isolated ASCII-only temporary copy so Windows paths containing Unicode +characters do not break dependency resolution. The temporary copy is removed after the check. + Or run the checks individually: ```powershell @@ -69,7 +80,7 @@ python -m unittest discover -s tests -v python -m compileall -q ome_memoryguard tests ``` -If AWS SAM CLI is installed and read-only validation credentials are configured: +To run only the SAM validation/build gate: ```powershell .\scripts\verify.ps1 -WithSam diff --git a/docs/AWS_DEPLOYMENT.md b/docs/AWS_DEPLOYMENT.md index fd369b1..e844c68 100644 --- a/docs/AWS_DEPLOYMENT.md +++ b/docs/AWS_DEPLOYMENT.md @@ -24,12 +24,16 @@ parameters. Pass only the secret ARN to SAM. ```powershell python -m unittest discover -s tests -v python -m compileall -q ome_memoryguard tests -sam validate --lint --template-file template.yaml -sam build --template-file template.yaml +.\scripts\sam-verify.ps1 ``` -`sam validate` may require configured AWS credentials even though it does not deploy. `sam build` -creates only local `.aws-sam` artifacts. +The helper runs `sam validate --lint` and `sam build --no-cached` without deployment. It stages an +ASCII-only temporary copy because SAM's Windows dependency builder can fail when the repository path +contains Unicode characters. The staging copy excludes `.env`, dependencies, Git metadata, and +generated files, and is removed after the check. + +`sam validate` may require configured AWS credentials in some environments even though it does not +deploy. `sam build` creates only local artifacts. Optional read-only account checks: diff --git a/docs/VERIFICATION.md b/docs/VERIFICATION.md index 4f367be..fae3708 100644 --- a/docs/VERIFICATION.md +++ b/docs/VERIFICATION.md @@ -17,14 +17,17 @@ Date: 2026-08-04 (Asia/Qyzylorda) | Initial `python -m unittest discover -s tests -v` | PASS, 7 tests | | Embedding provider fail-closed RED/GREEN test | PASS; unknown provider rejected without cloud calls | | API request-boundary RED/GREEN test | PASS; six invalid search/audit cases rejected with 422 | -| Final `python -m unittest discover -s tests -v` | PASS, 33 tests | +| Final `python -m unittest discover -s tests -v` | PASS, 34 tests | | `python -m compileall -q ome_memoryguard tests` | PASS | | `.venv\Scripts\python.exe -m pip check` | PASS, no broken requirements | -| `.venv\Scripts\python.exe -m pytest -q` | PASS, 33 tests; one upstream Starlette deprecation warning | +| `.venv\Scripts\python.exe -m pytest -q` | PASS, 34 tests; one upstream Starlette deprecation warning | | `.venv\Scripts\ruff.exe check .` | PASS, Ruff 0.16.1 | | `.venv\Scripts\cfn-lint.exe template.yaml` | PASS, cfn-lint 1.53.3 | -| `scripts\verify.ps1` | PASS for unittest, compileall, pytest, Ruff, cfn-lint | -| High-confidence token/private-key scan | PASS, no matching files | +| `scripts\verify.ps1 -WithGitleaks -WithSam` | PASS for all local release gates | +| gitleaks 8.30.1 Git history scan | PASS with a narrow allowlist for one synthetic denial fixture | +| gitleaks 8.30.1 working-tree scan | PASS; `.env`, dependencies, caches, and build output excluded | +| AWS SAM CLI 1.165.0 `validate --lint` | PASS | +| AWS SAM CLI 1.165.0 `build --no-cached` | PASS from isolated ASCII-only staging copy | | `.env` existence check | PASS, absent | | UI smoke on `127.0.0.1:8765` | PASS, in-memory/deterministic | | Browser console warnings/errors | PASS, none | @@ -44,9 +47,9 @@ The temporary local server was stopped and port `8765` was verified free after t | Check | Reason | |---|---| -| `sam validate --lint` / `sam build` | AWS SAM CLI not installed | | Live CockroachDB integration | live writes/schema changes forbidden for this task | | AWS Lambda/Bedrock/S3 integration | deployment/cloud writes and costs forbidden for this task | `tests/test_architecture.py` provides local static assertions for the SAM route/IAM invariants and -CockroachDB vector-index/query alignment; these do not replace vendor CLI or live validation. +CockroachDB vector-index/query alignment. SAM validation/build now covers local packaging, but neither +local check replaces live authorization, regional service availability, or integration testing. diff --git a/scripts/sam-verify.ps1 b/scripts/sam-verify.ps1 new file mode 100644 index 0000000..5c963c0 --- /dev/null +++ b/scripts/sam-verify.ps1 @@ -0,0 +1,73 @@ +[CmdletBinding()] +param() + +$ErrorActionPreference = 'Stop' +$ProjectRoot = (Resolve-Path -LiteralPath (Join-Path $PSScriptRoot '..')).Path +$TempRoot = [System.IO.Path]::GetFullPath([System.IO.Path]::GetTempPath()) +$StageRoot = Join-Path $TempRoot ("ome-memoryguard-sam-" + [guid]::NewGuid().ToString('N')) + +function Resolve-SamCli { + $Command = Get-Command sam -ErrorAction SilentlyContinue + if ($null -ne $Command) { + return $Command.Source + } + + $PerUserSam = Join-Path $env:LOCALAPPDATA 'Programs\Amazon\AWSSAMCLI\bin\sam.cmd' + if (Test-Path -LiteralPath $PerUserSam) { + return $PerUserSam + } + + throw 'AWS SAM CLI is not installed or could not be located' +} + +$Sam = Resolve-SamCli +$OldTelemetry = $env:SAM_CLI_TELEMETRY + +try { + New-Item -ItemType Directory -Path $StageRoot | Out-Null + + # SAM CLI on Windows can fail while resolving dependencies from a Unicode path. + # Build from an isolated ASCII-only copy and never copy local secrets or generated files. + & robocopy $ProjectRoot $StageRoot /E /XD .git .venv .aws-sam __pycache__ /XF .env '*.pyc' '*.pyo' '*.log' /NFL /NDL /NJH /NJS /NC /NS /NP | Out-Null + $RobocopyExit = $LASTEXITCODE + if ($RobocopyExit -gt 7) { + throw "SAM staging copy failed with robocopy exit code $RobocopyExit" + } + + $env:SAM_CLI_TELEMETRY = '0' + Push-Location -LiteralPath $StageRoot + try { + & $Sam validate --lint --template-file template.yaml + if ($LASTEXITCODE -ne 0) { + throw 'SAM validation failed' + } + Write-Output 'CHECK_SAM_VALIDATE=PASS' + + & $Sam build --no-cached --template-file template.yaml + if ($LASTEXITCODE -ne 0) { + throw 'SAM build failed' + } + if (-not (Test-Path -LiteralPath (Join-Path $StageRoot '.aws-sam\build\template.yaml'))) { + throw 'SAM build did not create the expected template artifact' + } + Write-Output 'CHECK_SAM_BUILD=PASS' + } + finally { + Pop-Location + } +} +finally { + $env:SAM_CLI_TELEMETRY = $OldTelemetry + + if (Test-Path -LiteralPath $StageRoot) { + $ResolvedStage = [System.IO.Path]::GetFullPath((Resolve-Path -LiteralPath $StageRoot).Path) + $ExpectedPrefix = $TempRoot.TrimEnd([System.IO.Path]::DirectorySeparatorChar) + [System.IO.Path]::DirectorySeparatorChar + if (-not $ResolvedStage.StartsWith($ExpectedPrefix, [System.StringComparison]::OrdinalIgnoreCase)) { + throw "Refusing to remove a staging directory outside the temporary root: $ResolvedStage" + } + if (-not ([System.IO.Path]::GetFileName($ResolvedStage)).StartsWith('ome-memoryguard-sam-')) { + throw "Refusing to remove an unexpected staging directory: $ResolvedStage" + } + Remove-Item -LiteralPath $ResolvedStage -Recurse -Force + } +} diff --git a/scripts/secret-scan.ps1 b/scripts/secret-scan.ps1 new file mode 100644 index 0000000..7e17aa7 --- /dev/null +++ b/scripts/secret-scan.ps1 @@ -0,0 +1,59 @@ +[CmdletBinding()] +param() + +$ErrorActionPreference = 'Stop' +$ProjectRoot = (Resolve-Path -LiteralPath (Join-Path $PSScriptRoot '..')).Path +$ConfigPath = Join-Path $ProjectRoot '.gitleaks.toml' + +function Resolve-Gitleaks { + $Command = Get-Command gitleaks -ErrorAction SilentlyContinue + if ($null -ne $Command) { + return $Command.Source + } + + $WinGetRoot = Join-Path $env:LOCALAPPDATA 'Microsoft\WinGet\Packages' + if (Test-Path -LiteralPath $WinGetRoot) { + $Candidate = Get-ChildItem -LiteralPath $WinGetRoot -Filter 'gitleaks.exe' -Recurse -File | + Where-Object { $_.FullName -match 'Gitleaks\.Gitleaks_' } | + Select-Object -First 1 + if ($null -ne $Candidate) { + return $Candidate.FullName + } + } + + throw 'gitleaks is not installed or could not be located' +} + +if (-not (Test-Path -LiteralPath $ConfigPath)) { + throw 'Missing .gitleaks.toml' +} + +$Gitleaks = Resolve-Gitleaks +$OldGitConfigCount = $env:GIT_CONFIG_COUNT +$OldGitConfigKey = $env:GIT_CONFIG_KEY_0 +$OldGitConfigValue = $env:GIT_CONFIG_VALUE_0 + +try { + # Keep the ownership exception local to this process. Do not change global Git configuration. + $env:GIT_CONFIG_COUNT = '1' + $env:GIT_CONFIG_KEY_0 = 'safe.directory' + $env:GIT_CONFIG_VALUE_0 = $ProjectRoot + + & $Gitleaks git $ProjectRoot --config $ConfigPath --redact=100 --no-banner + if ($LASTEXITCODE -ne 0) { + throw 'gitleaks Git history scan failed' + } + Write-Output 'CHECK_GITLEAKS_GIT=PASS' + + # The config excludes .env, local dependencies, caches, and SAM build output. + & $Gitleaks dir $ProjectRoot --config $ConfigPath --redact=100 --no-banner + if ($LASTEXITCODE -ne 0) { + throw 'gitleaks working-tree scan failed' + } + Write-Output 'CHECK_GITLEAKS_DIR=PASS' +} +finally { + $env:GIT_CONFIG_COUNT = $OldGitConfigCount + $env:GIT_CONFIG_KEY_0 = $OldGitConfigKey + $env:GIT_CONFIG_VALUE_0 = $OldGitConfigValue +} diff --git a/scripts/verify.ps1 b/scripts/verify.ps1 index eb7fa23..d592c51 100644 --- a/scripts/verify.ps1 +++ b/scripts/verify.ps1 @@ -1,6 +1,7 @@ [CmdletBinding()] param( - [switch]$WithSam + [switch]$WithSam, + [switch]$WithGitleaks ) $ErrorActionPreference = 'Stop' @@ -8,18 +9,32 @@ $ProjectRoot = (Resolve-Path -LiteralPath (Join-Path $PSScriptRoot '..')).Path Push-Location -LiteralPath $ProjectRoot try { - python -m unittest discover -s tests -v + $LocalPython = Join-Path $ProjectRoot '.venv\Scripts\python.exe' + $Python = if (Test-Path -LiteralPath $LocalPython) { + $LocalPython + } + else { + (Get-Command python -ErrorAction Stop).Source + } + + & $Python -m unittest discover -s tests -v if ($LASTEXITCODE -ne 0) { throw 'Unit tests failed' } Write-Output 'CHECK_UNITTEST=PASS' - python -m compileall -q ome_memoryguard tests + & $Python -m compileall -q ome_memoryguard tests if ($LASTEXITCODE -ne 0) { throw 'compileall failed' } Write-Output 'CHECK_COMPILEALL=PASS' + & $Python -m pip check + if ($LASTEXITCODE -ne 0) { + throw 'pip check failed' + } + Write-Output 'CHECK_PIP=PASS' + $LocalPytest = Join-Path $ProjectRoot '.venv\Scripts\pytest.exe' $Pytest = if (Test-Path -LiteralPath $LocalPytest) { $LocalPytest @@ -75,18 +90,19 @@ try { } if ($WithSam) { - $Sam = Get-Command sam -ErrorAction SilentlyContinue - if ($null -eq $Sam) { - throw 'AWS SAM CLI is not installed' - } - sam validate --lint --template-file template.yaml - if ($LASTEXITCODE -ne 0) { - throw 'SAM validation failed' - } - Write-Output 'CHECK_SAM_VALIDATE=PASS' + & (Join-Path $PSScriptRoot 'sam-verify.ps1') } else { Write-Output 'CHECK_SAM_VALIDATE=SKIP_NOT_REQUESTED' + Write-Output 'CHECK_SAM_BUILD=SKIP_NOT_REQUESTED' + } + + if ($WithGitleaks) { + & (Join-Path $PSScriptRoot 'secret-scan.ps1') + } + else { + Write-Output 'CHECK_GITLEAKS_GIT=SKIP_NOT_REQUESTED' + Write-Output 'CHECK_GITLEAKS_DIR=SKIP_NOT_REQUESTED' } } finally { diff --git a/tests/test_architecture.py b/tests/test_architecture.py index 6662021..eaa6204 100644 --- a/tests/test_architecture.py +++ b/tests/test_architecture.py @@ -48,10 +48,27 @@ def test_local_artifacts_are_ignored(self) -> None: def test_verification_script_covers_local_quality_tools(self) -> None: script = (ROOT / "scripts" / "verify.ps1").read_text(encoding="utf-8") + sam_script = (ROOT / "scripts" / "sam-verify.ps1").read_text(encoding="utf-8") + secret_script = (ROOT / "scripts" / "secret-scan.ps1").read_text(encoding="utf-8") self.assertIn("CHECK_PYTEST=PASS", script) self.assertIn("CHECK_RUFF=PASS", script) self.assertIn("CHECK_CFN_LINT=PASS", script) - self.assertIn("CHECK_SAM_VALIDATE=PASS", script) + self.assertIn("sam-verify.ps1", script) + self.assertIn("secret-scan.ps1", script) + self.assertIn("CHECK_SAM_VALIDATE=PASS", sam_script) + self.assertIn("CHECK_SAM_BUILD=PASS", sam_script) + self.assertIn("CHECK_GITLEAKS_GIT=PASS", secret_script) + self.assertIn("CHECK_GITLEAKS_DIR=PASS", secret_script) + + def test_release_gate_workflow_pins_third_party_actions(self) -> None: + workflow = (ROOT / ".github" / "workflows" / "ci.yml").read_text(encoding="utf-8") + uses_lines = [line.strip() for line in workflow.splitlines() if "uses:" in line] + self.assertGreaterEqual(len(uses_lines), 6) + for line in uses_lines: + _, reference = line.split("uses:", 1) + _, sha = reference.strip().rsplit("@", 1) + self.assertEqual(len(sha), 40) + self.assertTrue(all(character in "0123456789abcdef" for character in sha)) if __name__ == "__main__":