From 40fc824d95c171d10ac9246830571f71381f0eb2 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 19 May 2025 17:23:13 +0000 Subject: [PATCH 1/8] Initial plan for issue From f616f730ef80c4f2d97227874c23cf75394e4403 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 19 May 2025 17:26:53 +0000 Subject: [PATCH 2/8] Create GitHub Actions workflow for ARM template validation Co-authored-by: MSBrett <24294904+MSBrett@users.noreply.github.com> --- .github/workflows/arm-template-validation.yml | 155 ++++++++++++++++++ 1 file changed, 155 insertions(+) create mode 100644 .github/workflows/arm-template-validation.yml diff --git a/.github/workflows/arm-template-validation.yml b/.github/workflows/arm-template-validation.yml new file mode 100644 index 000000000..3823be8a8 --- /dev/null +++ b/.github/workflows/arm-template-validation.yml @@ -0,0 +1,155 @@ +name: 'ARM Template Validation' + +on: + pull_request: + paths: + - 'src/templates/**' + - 'src/bicep-registry/**' + - '.github/workflows/arm-template-validation.yml' + +jobs: + validate_templates: + name: Validate ARM Templates + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Setup Az PowerShell module + shell: pwsh + run: | + Install-Module -Name Az -Force -AllowClobber -Scope CurrentUser + Install-Module -Name PSRule.Rules.Azure -Force -Scope CurrentUser + + - name: Setup Azure CLI + uses: azure/setup-azure-cli@v1 + + - name: Setup Bicep + uses: anthony-c-martin/setup-bicep@v0.5 + + - name: Build templates + shell: pwsh + run: | + cd ${{ github.workspace }} + ./src/scripts/Build-Toolkit + + - name: Download ARM-TTK + shell: pwsh + run: | + cd ${{ github.workspace }} + $armTtkPath = "./arm-ttk" + New-Item -Path $armTtkPath -ItemType Directory -Force + Invoke-WebRequest -Uri "https://github.com/Azure/arm-ttk/archive/refs/heads/master.zip" -OutFile "./arm-ttk.zip" + Expand-Archive -Path "./arm-ttk.zip" -DestinationPath $armTtkPath -Force + Import-Module "$armTtkPath/arm-ttk-master/arm-ttk/arm-ttk.psd1" -Force + + - name: Validate templates with PSRule + shell: pwsh + run: | + cd ${{ github.workspace }} + + # Get all ARM JSON templates + $templates = Get-ChildItem -Path "release" -Filter "*.json" -Recurse + + foreach ($template in $templates) { + Write-Host "Validating template: $($template.FullName)" + + # Run PSRule validation + $results = $template.FullName | Invoke-PSRule -Module PSRule.Rules.Azure -WarningAction SilentlyContinue + + # Check for failures + $failures = $results | Where-Object { $_.Outcome -eq 'Fail' } + if ($failures) { + Write-Host "::error::PSRule validation failed for $($template.Name):" + $failures | Format-Table -Property RuleName, TargetName, Message -AutoSize | Out-String | Write-Host + exit 1 + } + } + + Write-Host "All templates validated successfully with PSRule!" + + - name: Validate templates with ARM-TTK + shell: pwsh + run: | + cd ${{ github.workspace }} + + # Get all ARM JSON templates + $templates = Get-ChildItem -Path "release" -Filter "*.json" -Recurse + + $hasErrors = $false + + foreach ($template in $templates) { + Write-Host "Validating template with ARM-TTK: $($template.FullName)" + + # Run ARM-TTK validation + $testResults = Test-AzTemplate -TemplatePath $template.FullName + + # Check for failures + $failures = $testResults | Where-Object { -not $_.Passed } + if ($failures) { + $hasErrors = $true + Write-Host "::error::ARM-TTK validation failed for $($template.Name):" + $failures | Format-Table -Property Name, Group, Errors -AutoSize | Out-String | Write-Host + } + } + + if ($hasErrors) { + exit 1 + } + + Write-Host "All templates validated successfully with ARM-TTK!" + + - name: Validate templates with az CLI + shell: pwsh + run: | + cd ${{ github.workspace }} + + # Get all ARM JSON templates + $templates = Get-ChildItem -Path "release" -Filter "*.json" -Recurse + + $hasErrors = $false + + foreach ($template in $templates) { + Write-Host "Validating template with az CLI: $($template.FullName)" + + # Skip files that are not ARM templates (like UI definitions) + if ($template.Name -like "*.ui.json") { + Write-Host "Skipping UI definition file: $($template.Name)" + continue + } + + # Determine deployment scope based on template content + $templateContent = Get-Content -Path $template.FullName -Raw | ConvertFrom-Json + $deploymentScope = if ($templateContent.resources -and $templateContent.resources[0].type -eq "Microsoft.Resources/deployments") { + # This is likely a subscription level template + "subscription" + } else { + # Default to resource group level + "resourcegroup" + } + + # Run appropriate az validate command based on scope + try { + if ($deploymentScope -eq "subscription") { + Write-Host "Running subscription-level validation" + az deployment sub validate --location eastus --template-file $template.FullName --no-prompt + } else { + Write-Host "Running resource-group level validation" + az deployment group validate --resource-group "validation-rg" --template-file $template.FullName --no-prompt + } + + if ($LASTEXITCODE -ne 0) { + $hasErrors = $true + Write-Host "::error::Azure CLI validation failed for $($template.Name)" + } + } catch { + $hasErrors = $true + Write-Host "::error::Exception during Azure CLI validation for $($template.Name): $_" + } + } + + if ($hasErrors) { + exit 1 + } + + Write-Host "All templates validated successfully with az CLI!" \ No newline at end of file From d172e1bccb74460ec2056c2370b36f6e1b83345b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 19 May 2025 17:27:38 +0000 Subject: [PATCH 3/8] Update documentation with ARM template validation information Co-authored-by: MSBrett <24294904+MSBrett@users.noreply.github.com> --- docs-wiki/Build-and-test.md | 57 +++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/docs-wiki/Build-and-test.md b/docs-wiki/Build-and-test.md index 1548bee3e..a8dfd18ed 100644 --- a/docs-wiki/Build-and-test.md +++ b/docs-wiki/Build-and-test.md @@ -7,6 +7,7 @@ On this page: - [⚙️ Building tools](#️-building-tools) - [🤏 Lint tests](#-lint-tests) - [🤞 PS -WhatIf / az validate](#-ps--whatif--az-validate) +- [🔍 Automated ARM template validation](#-automated-arm-template-validation) - [👍 Manually deployed + verified](#-manually-deployed--verified) - [💪 Unit tests](#-unit-tests) - [🙌 Integration tests](#-integration-tests) @@ -189,6 +190,62 @@ src/scripts/Deploy-Toolkit "" -Build -WhatIf
+## 🔍 Automated ARM template validation + +ARM templates in the repository are automatically validated using GitHub Actions as part of the PR process. This validation includes multiple checks to ensure templates meet best practices and will deploy successfully. + +### GitHub Actions workflow + +The validation workflow is triggered automatically when a PR includes changes to ARM templates or Bicep files. The following validations are performed: + +1. **Bicep Linting**: The Bicep linter checks for syntax errors and best practices. +2. **PSRule.Rules.Azure**: [PSRule.Rules.Azure](https://github.com/Azure/PSRule.Rules.Azure) runs comprehensive validation against Azure best practices and security standards. +3. **ARM Template Test Toolkit (ARM-TTK)**: [ARM-TTK](https://learn.microsoft.com/azure/azure-resource-manager/templates/test-toolkit) provides additional checks for common deployment issues. +4. **Azure CLI validation**: Templates are validated using `az deployment validate` to check for syntax errors without actual deployment. + +### Running validation locally + +To run ARM template validation locally before submitting a PR: + +1. **Build the templates**: + + ```powershell + cd "" + src/scripts/Build-Toolkit "" + ``` + +2. **Run PSRule validation** (requires [PSRule.Rules.Azure](https://github.com/Azure/PSRule.Rules.Azure) module): + + ```powershell + cd "" + Install-Module -Name PSRule.Rules.Azure -Force -Scope CurrentUser + Get-ChildItem -Path "release" -Filter "*.json" -Recurse | Invoke-PSRule -Module PSRule.Rules.Azure + ``` + +3. **Run ARM-TTK** (requires [ARM-TTK](https://github.com/Azure/arm-ttk)): + + ```powershell + cd "" + # Install ARM-TTK if not already installed + $armTtkPath = "" + Import-Module "$armTtkPath/arm-ttk.psd1" + + # Run validation + Get-ChildItem -Path "release" -Filter "*.json" -Recurse | ForEach-Object { + Test-AzTemplate -TemplatePath $_.FullName + } + ``` + +4. **Validate with Azure CLI**: + + ```powershell + cd "" + $template = "" + az deployment group validate --resource-group "validation-rg" --template-file $template + ``` + +
+ ## 👍 Manually deployed + verified Manual verification is always expected; however, we do prefer automated tests. Unit test are preferred with integration tests next. Refer to the details above for how to build and deploy each type of tool. From b9e1d0e8645e8ead6c7880e08e12bd51d3c9ac52 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 19 May 2025 17:29:16 +0000 Subject: [PATCH 4/8] Add Test-ArmTemplate script and update documentation Co-authored-by: MSBrett <24294904+MSBrett@users.noreply.github.com> --- docs-wiki/Build-and-test.md | 22 +++- src/scripts/README.md | 38 ++++++ src/scripts/Test-ArmTemplate.ps1 | 208 +++++++++++++++++++++++++++++++ 3 files changed, 267 insertions(+), 1 deletion(-) create mode 100644 src/scripts/Test-ArmTemplate.ps1 diff --git a/docs-wiki/Build-and-test.md b/docs-wiki/Build-and-test.md index a8dfd18ed..c808e718b 100644 --- a/docs-wiki/Build-and-test.md +++ b/docs-wiki/Build-and-test.md @@ -205,7 +205,27 @@ The validation workflow is triggered automatically when a PR includes changes to ### Running validation locally -To run ARM template validation locally before submitting a PR: +To run ARM template validation locally before submitting a PR, use the `Test-ArmTemplate` script: + +```powershell +cd "" +src/scripts/Test-ArmTemplate +``` + +This script will: +1. Validate all ARM templates in the release directory +2. Run checks with PSRule.Rules.Azure +3. Run validation with ARM-TTK +4. Validate templates with Azure CLI + +You can also validate a specific template: + +```powershell +cd "" +src/scripts/Test-ArmTemplate -TemplatePath "release/finops-hub/azuredeploy.json" +``` + +Alternatively, you can run individual validation steps manually: 1. **Build the templates**: diff --git a/src/scripts/README.md b/src/scripts/README.md index 600aec74e..651a0138a 100644 --- a/src/scripts/README.md +++ b/src/scripts/README.md @@ -9,6 +9,7 @@ On this page: - [📦 Build-Toolkit](#-build-toolkit) - [🚀 Deploy-Toolkit](#-deploy-toolkit) - [🧪 Test-PowerShell](#-test-powershell) +- [🔍 Test-ArmTemplate](#-test-armtemplate) - [🏷️ Get-Version](#️-get-version) - [🏷️ Update-Version](#️-update-version) - [🚚 Publish-Toolkit](#-publish-toolkit) @@ -247,6 +248,43 @@ Examples:
+## 🔍 Test-ArmTemplate + +[Test-ArmTemplate.ps1](./Test-ArmTemplate.ps1) validates ARM templates for deployment issues and best practices, using multiple validation tools: + +- PSRule.Rules.Azure for best practices validation +- ARM-TTK (ARM Template Test Toolkit) for template quality checks +- Azure CLI for deployment validation without actually deploying + +| Parameter | Description | +| ----------------- | -------------------------------------------------------------------------------- | +| `‑TemplatePath` | Optional. Path to the ARM template to validate. If not specified, all templates in the release directory will be validated. | +| `‑SkipPSRule` | Optional. Skip PSRule.Rules.Azure validation. Default = false. | +| `‑SkipArmTtk` | Optional. Skip ARM-TTK validation. Default = false. | +| `‑SkipAzValidate` | Optional. Skip Azure CLI validation. Default = false. | + +Examples: + +- Validate all ARM templates: + + ```powershell + ./Test-ArmTemplate + ``` + +- Validate a specific template: + + ```powershell + ./Test-ArmTemplate -TemplatePath "release/finops-hub/azuredeploy.json" + ``` + +- Skip specific validation methods: + + ```powershell + ./Test-ArmTemplate -SkipArmTtk -SkipAzValidate + ``` + +
+ ## 🏷️ Get-Version [Get-Version.ps1](./Get-Version.ps1) gets the latest version of the toolkit. diff --git a/src/scripts/Test-ArmTemplate.ps1 b/src/scripts/Test-ArmTemplate.ps1 new file mode 100644 index 000000000..b3cdcd1f0 --- /dev/null +++ b/src/scripts/Test-ArmTemplate.ps1 @@ -0,0 +1,208 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +<# + .SYNOPSIS + Validates ARM templates for deployment issues and best practices. + + .EXAMPLE + Test-ArmTemplate + + Validates all ARM templates in the release directory. + + .EXAMPLE + Test-ArmTemplate -TemplatePath "release/finops-hub/azuredeploy.json" + + Validates a specific ARM template. + + .PARAMETER TemplatePath + Optional. Path to the ARM template to validate. If not specified, all templates in the release directory will be validated. + + .PARAMETER SkipPSRule + Optional. Skip PSRule.Rules.Azure validation. Default = false. + + .PARAMETER SkipArmTtk + Optional. Skip ARM-TTK validation. Default = false. + + .PARAMETER SkipAzValidate + Optional. Skip Azure CLI validation. Default = false. +#> + +[CmdletBinding()] +param( + [Parameter(Mandatory = $false)] + [string] $TemplatePath, + + [Parameter(Mandatory = $false)] + [switch] $SkipPSRule, + + [Parameter(Mandatory = $false)] + [switch] $SkipArmTtk, + + [Parameter(Mandatory = $false)] + [switch] $SkipAzValidate +) + +# Get the root directory of the repo +$repoRoot = Split-Path -Parent (Split-Path -Parent $PSScriptRoot) + +# Function to check if a module is installed +function Test-ModuleInstalled($moduleName) { + return (Get-Module -ListAvailable -Name $moduleName) -ne $null +} + +# Function to ensure a module is installed +function Ensure-ModuleInstalled($moduleName) { + if (-not (Test-ModuleInstalled $moduleName)) { + Write-Host "Installing $moduleName module..." -ForegroundColor Yellow + Install-Module -Name $moduleName -Force -Scope CurrentUser + } +} + +# Function to ensure Azure CLI is installed +function Ensure-AzureCliInstalled { + try { + $azVersion = az --version + return $true + } + catch { + Write-Host "Azure CLI is not installed or not in PATH. Please install it from https://docs.microsoft.com/cli/azure/install-azure-cli" -ForegroundColor Red + return $false + } +} + +# Get templates to validate +$templates = @() +if ($TemplatePath) { + if (Test-Path $TemplatePath) { + $templates = @(Get-Item $TemplatePath) + } + else { + Write-Error "Template path not found: $TemplatePath" + exit 1 + } +} +else { + # Find all JSON templates (excluding UI definitions) + $templates = @(Get-ChildItem -Path "$repoRoot/release" -Filter "*.json" -Recurse | Where-Object { $_.Name -notlike "*.ui.json" }) +} + +# Check if any templates were found +if ($templates.Count -eq 0) { + Write-Host "No ARM templates found to validate. Run Build-Toolkit first to generate templates." -ForegroundColor Yellow + exit 0 +} + +$hasErrors = $false + +# Validate with PSRule.Rules.Azure +if (-not $SkipPSRule) { + Write-Host "Running PSRule.Rules.Azure validation..." -ForegroundColor Cyan + + Ensure-ModuleInstalled "PSRule.Rules.Azure" + + foreach ($template in $templates) { + Write-Host "Validating $($template.FullName)..." -ForegroundColor Green + + $results = $template.FullName | Invoke-PSRule -Module PSRule.Rules.Azure -WarningAction SilentlyContinue + + # Check for failures + $failures = $results | Where-Object { $_.Outcome -eq 'Fail' } + if ($failures) { + $hasErrors = $true + Write-Host "PSRule validation failed for $($template.Name):" -ForegroundColor Red + $failures | Format-Table -Property RuleName, TargetName, Message -AutoSize + } + } +} + +# Validate with ARM-TTK +if (-not $SkipArmTtk) { + Write-Host "Running ARM-TTK validation..." -ForegroundColor Cyan + + # Check if ARM-TTK is installed + if (-not (Test-ModuleInstalled "arm-ttk")) { + Write-Host "ARM-TTK not found. Installing..." -ForegroundColor Yellow + + $armTtkPath = "$env:TEMP/arm-ttk" + if (-not (Test-Path $armTtkPath)) { + New-Item -Path $armTtkPath -ItemType Directory -Force | Out-Null + } + + $armTtkZip = "$env:TEMP/arm-ttk.zip" + Invoke-WebRequest -Uri "https://github.com/Azure/arm-ttk/archive/refs/heads/master.zip" -OutFile $armTtkZip + Expand-Archive -Path $armTtkZip -DestinationPath $armTtkPath -Force + + Import-Module "$armTtkPath/arm-ttk-master/arm-ttk/arm-ttk.psd1" -Force + } + + foreach ($template in $templates) { + Write-Host "Validating $($template.FullName) with ARM-TTK..." -ForegroundColor Green + + try { + $testResults = Test-AzTemplate -TemplatePath $template.FullName + + # Check for failures + $failures = $testResults | Where-Object { -not $_.Passed } + if ($failures) { + $hasErrors = $true + Write-Host "ARM-TTK validation failed for $($template.Name):" -ForegroundColor Red + $failures | Format-Table -Property Name, Group, Errors -AutoSize + } + } + catch { + $hasErrors = $true + Write-Host "Error running ARM-TTK on $($template.Name): $_" -ForegroundColor Red + } + } +} + +# Validate with Azure CLI +if (-not $SkipAzValidate) { + Write-Host "Running Azure CLI validation..." -ForegroundColor Cyan + + if (Ensure-AzureCliInstalled) { + foreach ($template in $templates) { + Write-Host "Validating $($template.FullName) with Azure CLI..." -ForegroundColor Green + + # Determine deployment scope based on template content + $templateContent = Get-Content -Path $template.FullName -Raw | ConvertFrom-Json + $deploymentScope = if ($templateContent.resources -and $templateContent.resources[0].type -eq "Microsoft.Resources/deployments") { + "subscription" + } + else { + "resourcegroup" + } + + # Run appropriate az validate command based on scope + try { + if ($deploymentScope -eq "subscription") { + Write-Host "Running subscription-level validation" -ForegroundColor Gray + az deployment sub validate --location eastus --template-file $template.FullName --no-prompt + } + else { + Write-Host "Running resource-group level validation" -ForegroundColor Gray + az deployment group validate --resource-group "validation-rg" --template-file $template.FullName --no-prompt + } + + if ($LASTEXITCODE -ne 0) { + $hasErrors = $true + Write-Host "Azure CLI validation failed for $($template.Name)" -ForegroundColor Red + } + } + catch { + $hasErrors = $true + Write-Host "Exception during Azure CLI validation for $($template.Name): $_" -ForegroundColor Red + } + } + } +} + +# Report validation results +if ($hasErrors) { + Write-Host "`nValidation failed! Please fix the issues before committing." -ForegroundColor Red + exit 1 +} +else { + Write-Host "`nAll ARM templates validated successfully!" -ForegroundColor Green +} \ No newline at end of file From 93ec5eefe3b97d03c74013e7bbb21a7fa85aa631 Mon Sep 17 00:00:00 2001 From: Brett Wilson Date: Mon, 19 May 2025 10:57:39 -0700 Subject: [PATCH 5/8] Potential fix for code scanning alert no. 4: Workflow does not contain permissions Test and adjust as needed. Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- .github/workflows/arm-template-validation.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/arm-template-validation.yml b/.github/workflows/arm-template-validation.yml index 3823be8a8..10ec347ae 100644 --- a/.github/workflows/arm-template-validation.yml +++ b/.github/workflows/arm-template-validation.yml @@ -1,5 +1,8 @@ name: 'ARM Template Validation' +permissions: + contents: read + on: pull_request: paths: From 3356e4b0bbf29b5961005d5b4a7f844cecc56f4d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 19 May 2025 18:05:12 +0000 Subject: [PATCH 6/8] Update Azure CLI setup action to v3 Co-authored-by: MSBrett <24294904+MSBrett@users.noreply.github.com> --- .github/workflows/arm-template-validation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/arm-template-validation.yml b/.github/workflows/arm-template-validation.yml index 10ec347ae..d1f0bc24f 100644 --- a/.github/workflows/arm-template-validation.yml +++ b/.github/workflows/arm-template-validation.yml @@ -25,7 +25,7 @@ jobs: Install-Module -Name PSRule.Rules.Azure -Force -Scope CurrentUser - name: Setup Azure CLI - uses: azure/setup-azure-cli@v1 + uses: azure/setup-azure-cli@v3 - name: Setup Bicep uses: anthony-c-martin/setup-bicep@v0.5 From 58f321b5b76f5b5020e5533ed7cb9daeca935811 Mon Sep 17 00:00:00 2001 From: msbrett Date: Sun, 15 Jun 2025 00:20:36 -0700 Subject: [PATCH 7/8] Implement phased rollout for ARM template validation Phase 1 implementation: - Move ARM-TTK download location from .temp to release/.tools - Disable CI/CD validation (manual dispatch only) - Update documentation with phased rollout plan - Create issue #1696 for Phase 2 template fixes - Add validation level support (Strict/Lenient modes) This allows developers to use validation locally while we fix existing template errors before enabling CI/CD validation. --- .github/workflows/arm-template-validation.yml | 37 ++++++-- .gitignore | 12 +++ docs-wiki/Build-and-test.md | 75 +++++++++++++++- src/scripts/Test-ArmTemplate.ps1 | 86 ++++++++++++++++--- 4 files changed, 189 insertions(+), 21 deletions(-) diff --git a/.github/workflows/arm-template-validation.yml b/.github/workflows/arm-template-validation.yml index d1f0bc24f..de008b929 100644 --- a/.github/workflows/arm-template-validation.yml +++ b/.github/workflows/arm-template-validation.yml @@ -3,12 +3,18 @@ name: 'ARM Template Validation' permissions: contents: read +# Phase 1 of ARM template validation rollout - workflow is disabled for CI/CD +# To enable in Phase 2, uncomment the 'on' section below +# on: +# pull_request: +# paths: +# - 'src/templates/**' +# - 'src/bicep-registry/**' +# - '.github/workflows/arm-template-validation.yml' + +# Workflow can still be run manually during Phase 1 on: - pull_request: - paths: - - 'src/templates/**' - - 'src/bicep-registry/**' - - '.github/workflows/arm-template-validation.yml' + workflow_dispatch: jobs: validate_templates: @@ -40,11 +46,24 @@ jobs: shell: pwsh run: | cd ${{ github.workspace }} - $armTtkPath = "./arm-ttk" + # ARM-TTK version pinning - using stable release 0.26 (20250401) + # Update this version when newer stable releases are available + $armTtkVersion = "20250401" + $armTtkPath = "./release/.tools/arm-ttk" + New-Item -Path $armTtkPath -ItemType Directory -Force - Invoke-WebRequest -Uri "https://github.com/Azure/arm-ttk/archive/refs/heads/master.zip" -OutFile "./arm-ttk.zip" - Expand-Archive -Path "./arm-ttk.zip" -DestinationPath $armTtkPath -Force - Import-Module "$armTtkPath/arm-ttk-master/arm-ttk/arm-ttk.psd1" -Force + Write-Host "Downloading ARM-TTK version $armTtkVersion..." + $armTtkZip = "$armTtkPath/arm-ttk-$armTtkVersion.zip" + Invoke-WebRequest -Uri "https://github.com/Azure/arm-ttk/archive/refs/tags/$armTtkVersion.zip" -OutFile $armTtkZip + + # Extract to a versioned subfolder + $extractPath = "$armTtkPath/arm-ttk-$armTtkVersion" + Expand-Archive -Path $armTtkZip -DestinationPath $extractPath -Force + + # Clean up the zip file + Remove-Item -Path $armTtkZip -Force + + Import-Module "$armTtkPath/arm-ttk-$armTtkVersion/arm-ttk-$armTtkVersion/arm-ttk/arm-ttk.psd1" -Force - name: Validate templates with PSRule shell: pwsh diff --git a/.gitignore b/.gitignore index 03b4087cc..2db114891 100644 --- a/.gitignore +++ b/.gitignore @@ -360,3 +360,15 @@ coverage.xml last-deployment-state.json database-connection-settings.json deployment-settings-*.json + +# Python virtual environments +venv/ +.env/ +.venv/ +ENV/ +env/ + +# ARM-TTK download location +release/.tools/ + +CLAUDE.md diff --git a/docs-wiki/Build-and-test.md b/docs-wiki/Build-and-test.md index c808e718b..e74e82a71 100644 --- a/docs-wiki/Build-and-test.md +++ b/docs-wiki/Build-and-test.md @@ -192,11 +192,28 @@ src/scripts/Deploy-Toolkit "" -Build -WhatIf ## 🔍 Automated ARM template validation -ARM templates in the repository are automatically validated using GitHub Actions as part of the PR process. This validation includes multiple checks to ensure templates meet best practices and will deploy successfully. +> **Note**: ARM template validation is currently in Phase 1 of rollout and is available for local use only. Automated CI/CD validation is temporarily disabled while we fix existing template validation errors. See issue #1696 for Phase 2 rollout plans. + +ARM templates in the repository can be validated using multiple tools to ensure templates meet best practices and will deploy successfully. During Phase 1, validation must be run locally before submitting PRs. + +### Phased rollout plan + +The ARM template validation is being rolled out in phases to ensure smooth integration: + +**Phase 1 (Current)**: +- Validation tools are available for local use only +- CI/CD workflow is disabled to prevent PR failures +- Contributors should run validation locally before submitting PRs +- ARM-TTK is downloaded to `release/.tools/arm-ttk` instead of `.temp` + +**Phase 2 (Planned)**: +- Fix all existing template validation errors +- Re-enable CI/CD workflow for automatic PR validation +- All PRs will be required to pass validation checks ### GitHub Actions workflow -The validation workflow is triggered automatically when a PR includes changes to ARM templates or Bicep files. The following validations are performed: +The validation workflow will be triggered automatically when a PR includes changes to ARM templates or Bicep files (**currently disabled in Phase 1**). The following validations are performed: 1. **Bicep Linting**: The Bicep linter checks for syntax errors and best practices. 2. **PSRule.Rules.Azure**: [PSRule.Rules.Azure](https://github.com/Azure/PSRule.Rules.Azure) runs comprehensive validation against Azure best practices and security standards. @@ -264,6 +281,60 @@ Alternatively, you can run individual validation steps manually: az deployment group validate --resource-group "validation-rg" --template-file $template ``` +### What's Being Validated + +The ARM template validation process helps prevent common deployment failures and ensures templates follow Azure best practices. Here's what each validation tool checks: + +#### PSRule.Rules.Azure + +PSRule.Rules.Azure validates templates against Azure best practices, including: + +- **Security standards**: Ensures resources follow security best practices (e.g., HTTPS enforcement, encryption at rest) +- **Resource configuration**: Validates proper resource naming, tagging, and configuration +- **Parameter usage**: Checks that parameters are properly defined and used +- **API versions**: Ensures recent and stable API versions are used +- **Network security**: Validates network security rules and configurations +- **Diagnostics**: Checks that diagnostic settings are properly configured + +#### ARM Template Test Toolkit (ARM-TTK) + +ARM-TTK performs additional validation checks including: + +- **Template structure**: Validates JSON syntax and schema compliance +- **Parameter files**: Ensures parameter files match template parameters +- **Security**: Checks for hardcoded passwords, secure parameter usage +- **Resource dependencies**: Validates proper use of dependsOn +- **Output usage**: Ensures outputs are properly defined +- **Location handling**: Validates proper use of location parameters +- **Resource naming**: Checks for proper resource naming conventions + +#### Azure CLI Validation + +Azure CLI validation (`az deployment validate`) performs: + +- **Syntax validation**: Checks JSON syntax and ARM template schema +- **Resource provider registration**: Validates required providers are available +- **Quota checks**: Ensures deployment won't exceed subscription quotas +- **Permission validation**: Checks if the deployment has required permissions +- **Parameter validation**: Ensures all required parameters are provided +- **Deployment scope**: Validates resources match the deployment scope + +### Validation Modes + +The validation script supports two modes: + +- **Strict mode** (default): All validation rules are enforced. Use this for production-ready templates. +- **Lenient mode**: Skips certain validation rules that might fail for experimental features or prototypes. Use `-ValidationLevel Lenient` when running `Test-ArmTemplate`. + +Rules skipped in lenient mode include: +- Hardcoded values in templates (for quick prototypes) +- Missing parameter definitions (for experimental features) +- Debug deployment settings +- Larger parameter files +- Flexible location handling + +This multi-layered validation approach helps catch issues early in the development process, reducing failed deployments and improving template quality. +
## 👍 Manually deployed + verified diff --git a/src/scripts/Test-ArmTemplate.ps1 b/src/scripts/Test-ArmTemplate.ps1 index b3cdcd1f0..fcc28f76c 100644 --- a/src/scripts/Test-ArmTemplate.ps1 +++ b/src/scripts/Test-ArmTemplate.ps1 @@ -26,6 +26,11 @@ .PARAMETER SkipAzValidate Optional. Skip Azure CLI validation. Default = false. + + .PARAMETER ValidationLevel + Optional. Validation level (Strict or Lenient). Default = Strict. + - Strict: All validation rules are enforced (default) + - Lenient: Skip certain validation rules that might fail for experimental features #> [CmdletBinding()] @@ -40,7 +45,11 @@ param( [switch] $SkipArmTtk, [Parameter(Mandatory = $false)] - [switch] $SkipAzValidate + [switch] $SkipAzValidate, + + [Parameter(Mandatory = $false)] + [ValidateSet('Strict', 'Lenient')] + [string] $ValidationLevel = 'Strict' ) # Get the root directory of the repo @@ -95,9 +104,18 @@ if ($templates.Count -eq 0) { $hasErrors = $false +# Define rules to skip in lenient mode +$lenientSkipRules = @( + 'Azure.Template.UseParameters', # Allow hardcoded values for experimental features + 'Azure.Template.DefineParameters', # Allow missing parameters for prototypes + 'Azure.Template.DebugDeployment', # Allow debug settings in experimental templates + 'Azure.ARM.MaxParameterFile', # Allow larger parameter files for complex scenarios + 'Azure.Template.LocationType' # Allow flexible location handling +) + # Validate with PSRule.Rules.Azure if (-not $SkipPSRule) { - Write-Host "Running PSRule.Rules.Azure validation..." -ForegroundColor Cyan + Write-Host "Running PSRule.Rules.Azure validation (Mode: $ValidationLevel)..." -ForegroundColor Cyan Ensure-ModuleInstalled "PSRule.Rules.Azure" @@ -108,6 +126,18 @@ if (-not $SkipPSRule) { # Check for failures $failures = $results | Where-Object { $_.Outcome -eq 'Fail' } + + # In lenient mode, filter out rules that should be skipped + if ($ValidationLevel -eq 'Lenient' -and $failures) { + $originalFailureCount = $failures.Count + $failures = $failures | Where-Object { $_.RuleName -notin $lenientSkipRules } + + if ($originalFailureCount -gt $failures.Count) { + $skippedCount = $originalFailureCount - $failures.Count + Write-Host "Skipped $skippedCount validation rule(s) in lenient mode" -ForegroundColor Yellow + } + } + if ($failures) { $hasErrors = $true Write-Host "PSRule validation failed for $($template.Name):" -ForegroundColor Red @@ -116,24 +146,44 @@ if (-not $SkipPSRule) { } } +# Define ARM-TTK tests to skip in lenient mode +$lenientSkipArmTtkTests = @( + 'Parameters Should Be Derived From DeploymentTemplate', + 'Parameters Must Be Referenced', + 'Secure String Parameters Cannot Have Default', + 'Min And Max Value Are Numbers', + 'DeploymentTemplate Must Not Contain Hardcoded Uri' +) + # Validate with ARM-TTK if (-not $SkipArmTtk) { - Write-Host "Running ARM-TTK validation..." -ForegroundColor Cyan + Write-Host "Running ARM-TTK validation (Mode: $ValidationLevel)..." -ForegroundColor Cyan # Check if ARM-TTK is installed if (-not (Test-ModuleInstalled "arm-ttk")) { Write-Host "ARM-TTK not found. Installing..." -ForegroundColor Yellow - $armTtkPath = "$env:TEMP/arm-ttk" + # ARM-TTK version pinning - using stable release 0.26 (20250401) + # Update this version when newer stable releases are available + $armTtkVersion = "20250401" + $armTtkPath = "$repoRoot/release/.tools/arm-ttk" + if (-not (Test-Path $armTtkPath)) { New-Item -Path $armTtkPath -ItemType Directory -Force | Out-Null + + $armTtkZip = "$armTtkPath/arm-ttk-$armTtkVersion.zip" + Write-Host "Downloading ARM-TTK version $armTtkVersion..." -ForegroundColor Yellow + Invoke-WebRequest -Uri "https://github.com/Azure/arm-ttk/archive/refs/tags/$armTtkVersion.zip" -OutFile $armTtkZip + + # Extract to a versioned subfolder + $extractPath = "$armTtkPath/arm-ttk-$armTtkVersion" + Expand-Archive -Path $armTtkZip -DestinationPath $extractPath -Force + + # Clean up the zip file + Remove-Item -Path $armTtkZip -Force } - $armTtkZip = "$env:TEMP/arm-ttk.zip" - Invoke-WebRequest -Uri "https://github.com/Azure/arm-ttk/archive/refs/heads/master.zip" -OutFile $armTtkZip - Expand-Archive -Path $armTtkZip -DestinationPath $armTtkPath -Force - - Import-Module "$armTtkPath/arm-ttk-master/arm-ttk/arm-ttk.psd1" -Force + Import-Module "$armTtkPath/arm-ttk-$armTtkVersion/arm-ttk-$armTtkVersion/arm-ttk/arm-ttk.psd1" -Force } foreach ($template in $templates) { @@ -144,6 +194,18 @@ if (-not $SkipArmTtk) { # Check for failures $failures = $testResults | Where-Object { -not $_.Passed } + + # In lenient mode, filter out tests that should be skipped + if ($ValidationLevel -eq 'Lenient' -and $failures) { + $originalFailureCount = $failures.Count + $failures = $failures | Where-Object { $_.Name -notin $lenientSkipArmTtkTests } + + if ($originalFailureCount -gt $failures.Count) { + $skippedCount = $originalFailureCount - $failures.Count + Write-Host "Skipped $skippedCount ARM-TTK test(s) in lenient mode" -ForegroundColor Yellow + } + } + if ($failures) { $hasErrors = $true Write-Host "ARM-TTK validation failed for $($template.Name):" -ForegroundColor Red @@ -159,7 +221,11 @@ if (-not $SkipArmTtk) { # Validate with Azure CLI if (-not $SkipAzValidate) { - Write-Host "Running Azure CLI validation..." -ForegroundColor Cyan + Write-Host "Running Azure CLI validation (Mode: $ValidationLevel)..." -ForegroundColor Cyan + + if ($ValidationLevel -eq 'Lenient') { + Write-Host "Note: Azure CLI validation warnings will be ignored in lenient mode" -ForegroundColor Yellow + } if (Ensure-AzureCliInstalled) { foreach ($template in $templates) { From aab720f03aa0b0ab8c87d05fa61ba61ddd2688d8 Mon Sep 17 00:00:00 2001 From: msbrett Date: Fri, 10 Oct 2025 20:49:42 -0700 Subject: [PATCH 8/8] docs: fix version number (v13 not v0.13) --- docs-mslearn/toolkit/changelog.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs-mslearn/toolkit/changelog.md b/docs-mslearn/toolkit/changelog.md index c297b99ee..979a4471a 100644 --- a/docs-mslearn/toolkit/changelog.md +++ b/docs-mslearn/toolkit/changelog.md @@ -24,6 +24,14 @@ This article summarizes the features and enhancements in each release of the Fin The following section lists features and enhancements that are currently in development. +### Development tools v13 + +- **Added** + - Added ARM template validation infrastructure with GitHub Actions workflow and PowerShell script ([#1606](https://github.com/microsoft/finops-toolkit/issues/1606)). + - New `Test-ArmTemplate.ps1` script for local ARM template validation with PSRule, ARM-TTK, and Azure CLI. + - New GitHub Actions workflow for automated ARM template validation in CI/CD (currently manual trigger only, phased rollout planned). + - Supports strict and lenient validation modes for different development scenarios. + ### Bicep Registry module pending updates - Cost Management export modules for subscriptions and resource groups.