From 596a96abb715ecfaf1fccf95dfcb9860bef2adfb Mon Sep 17 00:00:00 2001 From: ramsessanchez <63934382+ramsessanchez@users.noreply.github.com> Date: Fri, 12 Dec 2025 12:09:12 -0800 Subject: [PATCH 01/11] Separate weekly open api -doc refresh and command metadata refresh --- .../common-templates/create-pr.yml | 2 +- .../download-openapi-docs.yml | 22 +-- .../individual-workload-module.yml | 42 +++++ .../module-metadata-generation.yml | 150 ++++++++++++++++++ .azure-pipelines/weekly-generation.yml | 120 ++++++++++---- .github/workflows/metadatachanges.yml | 2 +- 6 files changed, 294 insertions(+), 44 deletions(-) create mode 100644 .azure-pipelines/generation-templates/individual-workload-module.yml create mode 100644 .azure-pipelines/module-metadata-generation.yml diff --git a/.azure-pipelines/common-templates/create-pr.yml b/.azure-pipelines/common-templates/create-pr.yml index 337a46b5b5e..7c572b93c23 100644 --- a/.azure-pipelines/common-templates/create-pr.yml +++ b/.azure-pipelines/common-templates/create-pr.yml @@ -4,7 +4,7 @@ parameters: - name: BaseBranch type: string - default: dev + default: main - name: TargetBranch type: string default: "" diff --git a/.azure-pipelines/common-templates/download-openapi-docs.yml b/.azure-pipelines/common-templates/download-openapi-docs.yml index 9a0a1c58053..ba03e11cab4 100644 --- a/.azure-pipelines/common-templates/download-openapi-docs.yml +++ b/.azure-pipelines/common-templates/download-openapi-docs.yml @@ -73,25 +73,19 @@ steps: $diff = git diff --name-only $ModulesWithChanges = @{} $diff | %{ - if (($_ -match 'openApiDocs\/(v1.0|beta)\/(.*).yml') -and !$ModulesWithChanges.ContainsKey($matches.2)) - { - $ModulesWithChanges.Add($matches.2, $matches.1) + if ($_ -match 'openApiDocs\/(v1.0|beta)\/(.*)\.yml') { + $version = if ($matches[1] -eq 'v1.0') { 'v1.0' } else { 'beta' } + $moduleName = "$($matches[2])_$version" + if (!$ModulesWithChanges.ContainsKey($moduleName)) { + $ModulesWithChanges.Add($moduleName, $matches[1]) + } } } $ModuleNames = $ModulesWithChanges.Keys + Write-Host "Modules with changes: $ModuleNames" } Write-Host "##vso[task.setvariable variable=ModulesWithChanges;isOutput=true]$ModuleNames" - - task: PowerShell@2 - name: CalculateAndBumpModuleVersion - displayName: Calculate and bump module version - condition: and(succeeded(), ne(variables['OpenAPIDocDiff.ModulesWithChanges'], '')) - inputs: - pwsh: true - targetType: inline - script: | - . "$(System.DefaultWorkingDirectory)\tools\Versions\BumpModuleVersion.ps1" -BumpV1Module -BumpBetaModule -BumpAuthModule -Debug - - task: PowerShell@2 name: BuildOpenApiMetadataDetectionTool displayName: Build tool for detecting metadata changes @@ -128,4 +122,4 @@ steps: # References # [0] https://help.github.com/en/actions/configuring-and-managing-workflows/using-environment-variables # [1] https://hub.github.com/hub-pull-request.1.html -# https://help.github.com/en/actions/configuring-and-managing-workflows/authenticating-with-the-github_token +# https://help.github.com/en/actions/configuring-and-managing-workflows/authenticating-with-the-github_token \ No newline at end of file diff --git a/.azure-pipelines/generation-templates/individual-workload-module.yml b/.azure-pipelines/generation-templates/individual-workload-module.yml new file mode 100644 index 00000000000..2d95ba9d605 --- /dev/null +++ b/.azure-pipelines/generation-templates/individual-workload-module.yml @@ -0,0 +1,42 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + + +parameters: + - name: Test + type: boolean + default: true + - name: ModuleName + type: string + default: "" + - name: ModuleVersion + type: string + default: "" + +steps: + - task: PowerShell@2 + displayName: Generate Workload Modules + inputs: + targetType: inline + pwsh: true + script: | + . $(System.DefaultWorkingDirectory)/tools/GenerateModules.ps1 -Build -ExcludeExampleTemplates -ExcludeNotesSection -ModuleToGenerate ${{ parameters.ModuleName }} -ApiVersion ${{ parameters.ModuleVersion }} + - template: ../common-templates/guardian-analyzer.yml + + - task: PowerShell@2 + displayName: Test Workload Modules + enabled: false + inputs: + targetType: inline + pwsh: true + script: | + . $(System.DefaultWorkingDirectory)/tools/GenerateModules.ps1 -SkipGeneration -Test -ModuleToGenerate ${{ parameters.ModuleName }} -ApiVersion ${{ parameters.ModuleVersion }} + + - task: PowerShell@2 + displayName: Find Duplicate Commands + inputs: + targetType: inline + pwsh: true + script: | + . $(System.DefaultWorkingDirectory)/tools/PostGeneration/FindDuplicateCommand.ps1 -SourcePath "$(System.DefaultWorkingDirectory)/src/" + \ No newline at end of file diff --git a/.azure-pipelines/module-metadata-generation.yml b/.azure-pipelines/module-metadata-generation.yml new file mode 100644 index 00000000000..f513da1d940 --- /dev/null +++ b/.azure-pipelines/module-metadata-generation.yml @@ -0,0 +1,150 @@ +# This Yaml Document has been converted by ESAI Yaml Pipeline Conversion Tool. +# Please make sure to check all the converted content, it is your team's responsibility to make sure that the pipeline is still valid and functions as expected. +# This pipeline will be extended to the OneESPT template +# If you are not using the E+D shared hosted pool with windows-2022, replace the pool section with your hosted pool, os, and image name. If you are using a Linux image, you must specify an additional windows image for SDL: https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/1es-pipeline-templates/features/sdlanalysis/overview#how-to-specify-a-windows-pool-for-the-sdl-source-analysis-stage +# The Task 'PublishBuildArtifacts@1' has been converted to an output named 'Publish Module Artifacts' in the templateContext section. +# The Task 'NuGetCommand@2' has been converted to an output named 'Publish NuGet to feed' in the templateContext section. + +name: $(BuildDefinitionName)_$(SourceBranchName)_$(Date:yyyyMMdd)$(Rev:.r) +parameters: +- name: BuildAgent + displayName: Build Agent + default: 1es-windows-ps-compute-m +- name: BaseBranch + displayName: Base Branch + default: main +- name: Test + type: boolean + default: true +- name: Pack + type: boolean + default: false +- name: Sign + type: boolean + default: false +- name: BumpModuleVersion + type: boolean + default: false +- name: CreatePullRequest + type: boolean + default: true + +variables: + BuildAgent: ${{ parameters.BuildAgent }} + Branch: "ModuleMetadataRefresh" + BaseBranch: ${{ parameters.BaseBranch }} + +trigger: + branches: + include: + - main + +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + pool: $(BuildAgent) + sdl: + binskim: + enabled: false + justificationForDisabling: "Binskim keeps on crashing and failing the weekly build pipeline. Disabling it for now because we are unable to publish the artifacts to internal feeds." + credscan: + suppressionsFile: $(Build.SourcesDirectory)/.azure-pipelines/config/credscan/credscan-suppressions.json + policheck: + exclusionFile: $(Build.SourcesDirectory)/.azure-pipelines/config/policheck/policheck-exclusions.xml + customBuildTags: + - ES365AIMigrationTooling + stages: + - stage: stage + jobs: + - job: CreateMetadataRefreshBranch + displayName: Create Metadata Refresh Branch + timeoutInMinutes: 1200 + steps: + - template: .azure-pipelines/common-templates/checkout.yml@self + parameters: + TargetBranch: ${{ parameters.BaseBranch }} + - task: PowerShell@2 + name: "ComputeBranch" + displayName: "Compute Module Metadata Refresh Branch Name" + inputs: + targetType: inline + script: | + $branch = "{0}/{1}" -f "$(Branch)", (Get-Date -Format yyyyMMddHHmm) + Write-Host "##vso[task.setvariable variable=RefreshBranch;isOutput=true]$branch" + - task: Bash@3 + displayName: "Create Module Metadata Refresh Branch" + inputs: + targetType: inline + script: | + git status + git branch $(ComputeBranch.RefreshBranch) + git checkout $(ComputeBranch.RefreshBranch) + git status + + - job: MsGraphPsSdkModuleMetadataGeneration + dependsOn: CreateMetadataRefreshBranch + displayName: Microsoft Graph PowerShell SDK Generation + condition: succeeded() + timeoutInMinutes: 840 + variables: + RefreshBranch: $[ dependencies.CreateMetadataRefreshBranch.outputs['ComputeBranch.RefreshBranch'] ] + steps: + - template: .azure-pipelines/common-templates/checkout.yml@self + parameters: + TargetBranch: $(RefreshBranch) + - template: .azure-pipelines/common-templates/install-tools.yml@self + - template: .azure-pipelines/common-templates/security-pre-checks.yml@self + - template: .azure-pipelines/generation-templates/authentication-module.yml@self + parameters: + Test: ${{ parameters.Test }} + Pack: ${{ parameters.Pack }} + Sign: ${{ parameters.Sign }} + - template: .azure-pipelines/generation-templates/workload-modules.yml@self + parameters: + Test: ${{ parameters.Test }} + Pack: ${{ parameters.Pack }} + Sign: ${{ parameters.Sign }} + - template: .azure-pipelines/generation-templates/meta-module.yml@self + parameters: + Test: ${{ parameters.Test }} + Pack: ${{ parameters.Pack }} + Sign: ${{ parameters.Sign }} + - template: .azure-pipelines/generation-templates/generate-command-metadata.yml@self + - template: .azure-pipelines/common-templates/security-post-checks.yml@self + + - ${{ if eq(parameters.BumpModuleVersion, true) }}: + - task: PowerShell@2 + name: CalculateAndBumpModuleVersion + displayName: Calculate and bump module version + condition: succeeded() + inputs: + pwsh: true + targetType: inline + script: | + . "$(System.DefaultWorkingDirectory)\tools\Versions\BumpModuleVersion.ps1" -BumpV1Module -BumpBetaModule -BumpAuthModule -Debug + - task: Bash@3 + displayName: Push version bump changes + env: + GITHUB_TOKEN: $(GITHUB_TOKEN) + inputs: + targetType: inline + script: | + git status + git add "$(System.DefaultWorkingDirectory)/config/ModuleMetadata.json" + git commit -m 'Bump module versions after metadata generation.' + git push "https://$(GITHUB_TOKEN)@github.com/microsoftgraph/msgraph-sdk-powershell.git" + git status + + - ${{ if eq(parameters.CreatePullRequest, true) }}: + - template: .azure-pipelines/common-templates/create-pr.yml@self + parameters: + BaseBranch: $(BaseBranch) + TargetBranch: $(RefreshBranch) + Title: "[v2] Module Metadata Refresh" + Body: "This pull request was automatically created by Azure Pipelines. **Important** Check for unexpected deletions or changes in this PR." \ No newline at end of file diff --git a/.azure-pipelines/weekly-generation.yml b/.azure-pipelines/weekly-generation.yml index 60e36a5188b..5e9cadc5813 100644 --- a/.azure-pipelines/weekly-generation.yml +++ b/.azure-pipelines/weekly-generation.yml @@ -34,6 +34,15 @@ parameters: displayName: Skip OpenAPI Docs Download default: false type: boolean +- name: SpecifyModules + displayName: Specify Modules to Generate + default: false + type: boolean +- name: ModulesToGenerate + displayName: Modules To Generate + default: Insert Module Names Here separated by space, e.g., Users_v1.0 Users_beta + type: string + variables: Branch: "WeeklyApiRefresh" BaseBranch: ${{ parameters.BaseBranch }} @@ -82,28 +91,80 @@ extends: BuildAgent: $(BuildAgent) SkipForceRefresh: $(SkipForceRefresh) SkipOpenAPIDocsDownload: ${{ parameters.SkipOpenAPIDocsDownload }} - - job: MsGraphPsSdkWeeklyGeneration + + - job: SetUpModuleMatrix + displayName: Setup Module JSON for Matrix dependsOn: RefreshOpenAPIDocuments + condition: and(succeeded(), ne(dependencies.RefreshOpenAPIDocuments.outputs['OpenAPIDocDiff.ModulesWithChanges'], '')) + timeoutInMinutes: 840 + variables: + WeeklyBranch: $[ dependencies.RefreshOpenAPIDocuments.outputs['ComputeBranch.WeeklyBranch'] ] + ModulesToUpdate: $[ dependencies.RefreshOpenAPIDocuments.outputs['OpenAPIDocDiff.ModulesWithChanges'] ] + steps: + - template: .azure-pipelines/common-templates/checkout.yml@self + parameters: + TargetBranch: $(WeeklyBranch) + + - task: PowerShell@2 + name: CreateMatrixJson + displayName: Create Matrix Json + inputs: + targetType: inline + pwsh: true + script: | + if ($${{ parameters.SpecifyModules }}) { + $modules = "${{ parameters.ModulesToGenerate }}" + } else { + $modules = $env:ModulesToUpdate + } + + Write-Host "ModulesToUpdate: $modules" + $testDataArray = $modules -split ' ' + $jsonOutput = @{} + + foreach ($item in $testDataArray) { + if ($item -notmatch '_') { + # If '_' is not present, create two versions: v1.0 and beta + $jsonOutput["${item}_v1.0"] = @{ + moduleVersion = 'v1.0' + moduleName = $item + } + $jsonOutput["${item}_beta"] = @{ + moduleVersion = 'beta' + moduleName = $item + } + } else { + $name, $version = $item -split '_' + $jsonOutput[$item] = @{ + moduleVersion = $version + moduleName = $name + } + } + } + + $result = $jsonOutput | ConvertTo-Json -Compress + Write-Host "##vso[task.setvariable variable=matrixJson;isOutput=true]$result" + + - job: MsGraphPsSdkWeeklyGeneration + dependsOn: + - SetUpModuleMatrix + - RefreshOpenAPIDocuments displayName: Microsoft Graph PowerShell SDK Generation condition: and(succeeded(), ne(dependencies.RefreshOpenAPIDocuments.outputs['OpenAPIDocDiff.ModulesWithChanges'], '')) timeoutInMinutes: 840 variables: WeeklyBranch: $[ dependencies.RefreshOpenAPIDocuments.outputs['ComputeBranch.WeeklyBranch'] ] + ModulesMatrix: $[ dependencies.SetUpModuleMatrix.outputs['CreateMatrixJson.matrixJson'] ] templateContext: outputs: - ${{ if and(eq(parameters.Pack, true), eq(parameters.Sign, true)) }}: - output: pipelineArtifact displayName: 'Publish Module Artifacts' targetPath: "$(Build.ArtifactStagingDirectory)" - artifactName: "drop" + artifactName: "$(moduleName)_$(moduleVersion)_drop" publishLocation: "Container" - - ${{ if and(eq(parameters.Pack, true), eq(parameters.Sign, true)) }}: - - output: nuget - displayName: 'Publish NuGet to feed' - packageParentPath: '$(Build.ArtifactStagingDirectory)' - packagesToPush: $(Build.ArtifactStagingDirectory)/**/Microsoft.Graph.*.nupkg - publishVstsFeed: $(PROJECT_NAME)/$(FEED_NAME) - allowPackageConflicts: true + strategy: + matrix: $[ dependencies.SetUpModuleMatrix.outputs['CreateMatrixJson.matrixJson'] ] steps: - template: .azure-pipelines/common-templates/checkout.yml@self parameters: @@ -115,27 +176,30 @@ extends: Test: ${{ parameters.Test }} Pack: ${{ parameters.Pack }} Sign: ${{ parameters.Sign }} - - template: .azure-pipelines/generation-templates/workload-modules.yml@self + - template: .azure-pipelines/generation-templates/individualized-workload-modules.yml@self parameters: Test: ${{ parameters.Test }} Pack: ${{ parameters.Pack }} Sign: ${{ parameters.Sign }} - - template: .azure-pipelines/generation-templates/meta-module.yml@self - parameters: - Test: ${{ parameters.Test }} - Pack: ${{ parameters.Pack }} - Sign: ${{ parameters.Sign }} - - ${{ if and(eq(parameters.Pack, true), eq(parameters.Sign, true)) }}: - - template: .azure-pipelines/common-templates/esrp/codesign-nuget.yml@self - parameters: - FolderPath: "$(Build.ArtifactStagingDirectory)" - Pattern: "Microsoft.Graph*.nupkg" - - template: .azure-pipelines/generation-templates/generate-command-metadata.yml@self + ModuleName: $(moduleName) + ModuleVersion: $(moduleVersion) - template: .azure-pipelines/common-templates/security-post-checks.yml@self - - ${{ if eq(parameters.CreatePullRequest, true) }}: - - template: .azure-pipelines/common-templates/create-pr.yml@self - parameters: - BaseBranch: $(BaseBranch) - TargetBranch: $(WeeklyBranch) - Title: "[v2] Weekly OpenApiDocs Refresh" - Body: "This pull request was automatically created by Azure Pipelines. **Important** Check for unexpected deletions or changes in this PR." \ No newline at end of file + + - job: CreatePullRequest + displayName: Create Pull Request for all changes + dependsOn: + - MsGraphPsSdkWeeklyGeneration + - RefreshOpenAPIDocuments + condition: and(succeeded(), eq(dependencies.MsGraphPsSdkWeeklyGeneration.result, 'Succeeded'), eq(${{ parameters.CreatePullRequest }}, true)) + variables: + WeeklyBranch: $[ dependencies.RefreshOpenAPIDocuments.outputs['ComputeBranch.WeeklyBranch'] ] + steps: + - template: .azure-pipelines/common-templates/checkout.yml@self + parameters: + TargetBranch: $(WeeklyBranch) + - template: .azure-pipelines/common-templates/create-pr.yml@self + parameters: + BaseBranch: $(BaseBranch) + TargetBranch: $(WeeklyBranch) + Title: "[v2] Weekly OpenApiDocs Refresh" + Body: "This pull request was automatically created by Azure Pipelines. **Important** Check for unexpected deletions or changes in this PR." \ No newline at end of file diff --git a/.github/workflows/metadatachanges.yml b/.github/workflows/metadatachanges.yml index d91394e2ecf..687f9db9bf6 100644 --- a/.github/workflows/metadatachanges.yml +++ b/.github/workflows/metadatachanges.yml @@ -7,7 +7,7 @@ on: workflow_dispatch: pull_request: branches: - - dev + - main paths: - openApiDocs/** - src/Authentication/Authentication/custom/common/MgCommandMetadata.json From f737713dbd7d8731e1bf2d22399c6e47dc808a66 Mon Sep 17 00:00:00 2001 From: ramsessanchez <63934382+ramsessanchez@users.noreply.github.com> Date: Fri, 12 Dec 2025 12:25:18 -0800 Subject: [PATCH 02/11] rename file --- ...ata-generation.yml => command-metadata-refresh.yml} | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) rename .azure-pipelines/{module-metadata-generation.yml => command-metadata-refresh.yml} (95%) diff --git a/.azure-pipelines/module-metadata-generation.yml b/.azure-pipelines/command-metadata-refresh.yml similarity index 95% rename from .azure-pipelines/module-metadata-generation.yml rename to .azure-pipelines/command-metadata-refresh.yml index f513da1d940..205439a46a1 100644 --- a/.azure-pipelines/module-metadata-generation.yml +++ b/.azure-pipelines/command-metadata-refresh.yml @@ -31,7 +31,7 @@ parameters: variables: BuildAgent: ${{ parameters.BuildAgent }} - Branch: "ModuleMetadataRefresh" + Branch: "ModuleCommandMetadataRefresh" BaseBranch: ${{ parameters.BaseBranch }} trigger: @@ -71,14 +71,14 @@ extends: TargetBranch: ${{ parameters.BaseBranch }} - task: PowerShell@2 name: "ComputeBranch" - displayName: "Compute Module Metadata Refresh Branch Name" + displayName: "Compute Module Command Metadata Refresh Branch Name" inputs: targetType: inline script: | $branch = "{0}/{1}" -f "$(Branch)", (Get-Date -Format yyyyMMddHHmm) Write-Host "##vso[task.setvariable variable=RefreshBranch;isOutput=true]$branch" - task: Bash@3 - displayName: "Create Module Metadata Refresh Branch" + displayName: "Create Module Command Metadata Refresh Branch" inputs: targetType: inline script: | @@ -87,7 +87,7 @@ extends: git checkout $(ComputeBranch.RefreshBranch) git status - - job: MsGraphPsSdkModuleMetadataGeneration + - job: MsGraphPsSdkModuleCommandMetadataGeneration dependsOn: CreateMetadataRefreshBranch displayName: Microsoft Graph PowerShell SDK Generation condition: succeeded() @@ -146,5 +146,5 @@ extends: parameters: BaseBranch: $(BaseBranch) TargetBranch: $(RefreshBranch) - Title: "[v2] Module Metadata Refresh" + Title: "[v2] Module Command Metadata Refresh" Body: "This pull request was automatically created by Azure Pipelines. **Important** Check for unexpected deletions or changes in this PR." \ No newline at end of file From 1af8aa364a88ce5a1cf09a6c9479c23015b7a0d8 Mon Sep 17 00:00:00 2001 From: ramsessanchez <63934382+ramsessanchez@users.noreply.github.com> Date: Thu, 22 Jan 2026 12:36:49 -0800 Subject: [PATCH 03/11] skip openApi download steps entirley on check --- .azure-pipelines/weekly-generation.yml | 7 ++++--- tools/OpenApiDocDiff.ps1 | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) create mode 100644 tools/OpenApiDocDiff.ps1 diff --git a/.azure-pipelines/weekly-generation.yml b/.azure-pipelines/weekly-generation.yml index 5e9cadc5813..f0862cfb8bf 100644 --- a/.azure-pipelines/weekly-generation.yml +++ b/.azure-pipelines/weekly-generation.yml @@ -85,6 +85,7 @@ extends: timeoutInMinutes: 1200 steps: - template: .azure-pipelines/common-templates/download-openapi-docs.yml@self + condition: eq('${{ parameters.SkipOpenAPIDocsDownload }}', false) parameters: Branch: $(Branch) BaseBranch: $(BaseBranch) @@ -101,9 +102,9 @@ extends: WeeklyBranch: $[ dependencies.RefreshOpenAPIDocuments.outputs['ComputeBranch.WeeklyBranch'] ] ModulesToUpdate: $[ dependencies.RefreshOpenAPIDocuments.outputs['OpenAPIDocDiff.ModulesWithChanges'] ] steps: - - template: .azure-pipelines/common-templates/checkout.yml@self - parameters: - TargetBranch: $(WeeklyBranch) + # - template: .azure-pipelines/common-templates/checkout.yml@self + # parameters: + # TargetBranch: $(WeeklyBranch) - task: PowerShell@2 name: CreateMatrixJson diff --git a/tools/OpenApiDocDiff.ps1 b/tools/OpenApiDocDiff.ps1 new file mode 100644 index 00000000000..60d42da9941 --- /dev/null +++ b/tools/OpenApiDocDiff.ps1 @@ -0,0 +1,20 @@ + Write-Host "SkipOpenAPIDocsDownload: ${{ parameters.SkipOpenAPIDocsDownload }}" + if ('${{ parameters.SkipOpenAPIDocsDownload }}' -eq 'True') { + $ModuleNames = "Skipped" + Write-Warning "Skipped OpenAPI Docs Download." + } else { + $diff = git diff --name-only + $ModulesWithChanges = @{} + $diff | %{ + if ($_ -match 'openApiDocs\/(v1.0|beta)\/(.*)\.yml') { + $version = if ($matches[1] -eq 'v1.0') { 'v1.0' } else { 'beta' } + $moduleName = "$($matches[2])_$version" + if (!$ModulesWithChanges.ContainsKey($moduleName)) { + $ModulesWithChanges.Add($moduleName, $matches[1]) + } + } + } + $ModuleNames = $ModulesWithChanges.Keys + Write-Host "Modules with changes: $ModuleNames" + } + Write-Host "##vso[task.setvariable variable=ModulesWithChanges;isOutput=true]$ModuleNames" \ No newline at end of file From 3482604c2a55c1015687a70a871353630e920daa Mon Sep 17 00:00:00 2001 From: ramsessanchez <63934382+ramsessanchez@users.noreply.github.com> Date: Wed, 28 Jan 2026 11:00:13 -0800 Subject: [PATCH 04/11] temp disable condition param --- .azure-pipelines/weekly-generation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/weekly-generation.yml b/.azure-pipelines/weekly-generation.yml index f0862cfb8bf..e1cc8e00dbe 100644 --- a/.azure-pipelines/weekly-generation.yml +++ b/.azure-pipelines/weekly-generation.yml @@ -85,7 +85,7 @@ extends: timeoutInMinutes: 1200 steps: - template: .azure-pipelines/common-templates/download-openapi-docs.yml@self - condition: eq('${{ parameters.SkipOpenAPIDocsDownload }}', false) + #condition: eq('${{ parameters.SkipOpenAPIDocsDownload }}', false) parameters: Branch: $(Branch) BaseBranch: $(BaseBranch) From afe92422ba0e431c05e2f4c3f927d7839bbf2fda Mon Sep 17 00:00:00 2001 From: ramsessanchez <63934382+ramsessanchez@users.noreply.github.com> Date: Wed, 28 Jan 2026 11:03:41 -0800 Subject: [PATCH 05/11] file rename ref --- .azure-pipelines/weekly-generation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/weekly-generation.yml b/.azure-pipelines/weekly-generation.yml index e1cc8e00dbe..65565597b52 100644 --- a/.azure-pipelines/weekly-generation.yml +++ b/.azure-pipelines/weekly-generation.yml @@ -177,7 +177,7 @@ extends: Test: ${{ parameters.Test }} Pack: ${{ parameters.Pack }} Sign: ${{ parameters.Sign }} - - template: .azure-pipelines/generation-templates/individualized-workload-modules.yml@self + - template: .azure-pipelines/generation-templates/individual-workload-module.yml@self parameters: Test: ${{ parameters.Test }} Pack: ${{ parameters.Pack }} From 9ab4199f4f18c414ff12ddb9eb7db148fe3bcfb6 Mon Sep 17 00:00:00 2001 From: ramsessanchez <63934382+ramsessanchez@users.noreply.github.com> Date: Wed, 28 Jan 2026 11:08:29 -0800 Subject: [PATCH 06/11] re introduce sign param --- .../generation-templates/individual-workload-module.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines/generation-templates/individual-workload-module.yml b/.azure-pipelines/generation-templates/individual-workload-module.yml index 2d95ba9d605..3f461e6ec2c 100644 --- a/.azure-pipelines/generation-templates/individual-workload-module.yml +++ b/.azure-pipelines/generation-templates/individual-workload-module.yml @@ -12,6 +12,12 @@ parameters: - name: ModuleVersion type: string default: "" + - name: Pack + type: boolean + default: true + - name: Sign + type: boolean + default: true steps: - task: PowerShell@2 @@ -20,7 +26,7 @@ steps: targetType: inline pwsh: true script: | - . $(System.DefaultWorkingDirectory)/tools/GenerateModules.ps1 -Build -ExcludeExampleTemplates -ExcludeNotesSection -ModuleToGenerate ${{ parameters.ModuleName }} -ApiVersion ${{ parameters.ModuleVersion }} + . $(System.DefaultWorkingDirectory)/tools/GenerateModules.ps1 -EnableSigning:$${{ parameters.Sign }} -Build -ExcludeExampleTemplates -ExcludeNotesSection -ModuleToGenerate ${{ parameters.ModuleName }} -ApiVersion ${{ parameters.ModuleVersion }} - template: ../common-templates/guardian-analyzer.yml - task: PowerShell@2 From dfaa4de7de21aaeb19a768efd649b6df947cf628 Mon Sep 17 00:00:00 2001 From: ramsessanchez <63934382+ramsessanchez@users.noreply.github.com> Date: Tue, 3 Feb 2026 13:28:10 -0800 Subject: [PATCH 07/11] consolidate skipDownload parameters --- .../common-templates/download-openapi-docs.yml | 16 ++++++++++------ .azure-pipelines/weekly-generation.yml | 17 +++-------------- 2 files changed, 13 insertions(+), 20 deletions(-) diff --git a/.azure-pipelines/common-templates/download-openapi-docs.yml b/.azure-pipelines/common-templates/download-openapi-docs.yml index ba03e11cab4..729e6066b9f 100644 --- a/.azure-pipelines/common-templates/download-openapi-docs.yml +++ b/.azure-pipelines/common-templates/download-openapi-docs.yml @@ -8,8 +8,6 @@ parameters: type: string - name: BuildAgent displayName: Build Agent - - name: SkipForceRefresh - type: string - name: SkipOpenAPIDocsDownload type: boolean default: false @@ -27,11 +25,18 @@ steps: inputs: targetType: inline script: | - $branch = "{0}/{1}" -f "$(Branch)", (Get-Date -Format yyyyMMddHHmm) + if ('${{ parameters.SkipOpenAPIDocsDownload }}' -eq 'True') { + $branch = "" + Write-Host "Skipping branch creation - using current branch" + } else { + $branch = "{0}/{1}" -f "$(Branch)", (Get-Date -Format yyyyMMddHHmm) + Write-Host "Computed weekly branch: $branch" + } Write-Host "##vso[task.setvariable variable=WeeklyBranch;isOutput=true]$branch" - task: Bash@3 displayName: "Create weekly branch" + condition: eq('${{ parameters.SkipOpenAPIDocsDownload }}', false) inputs: targetType: inline script: | @@ -46,7 +51,6 @@ steps: continueOnError: false inputs: filePath: "$(System.DefaultWorkingDirectory)/tools/UpdateOpenApi.ps1" - arguments: -SkipForceRefresh:$$(SkipForceRefresh) pwsh: true - task: PowerShell@2 @@ -55,7 +59,7 @@ steps: continueOnError: false inputs: filePath: "$(System.DefaultWorkingDirectory)/tools/UpdateOpenApi.ps1" - arguments: -BetaGraphVersion -SkipForceRefresh:$$(SkipForceRefresh) + arguments: -BetaGraphVersion pwsh: true - task: PowerShell@2 @@ -106,7 +110,7 @@ steps: - task: Bash@3 displayName: Commit downloaded files - condition: and(succeeded(), ne(variables['OpenAPIDocDiff.ModulesWithChanges'], '')) + condition: and(succeeded(), ne(variables['OpenAPIDocDiff.ModulesWithChanges'], ''), eq('${{ parameters.SkipOpenAPIDocsDownload }}', false)) env: GITHUB_TOKEN: $(GITHUB_TOKEN) inputs: diff --git a/.azure-pipelines/weekly-generation.yml b/.azure-pipelines/weekly-generation.yml index 65565597b52..2e0f0ca4aec 100644 --- a/.azure-pipelines/weekly-generation.yml +++ b/.azure-pipelines/weekly-generation.yml @@ -14,8 +14,8 @@ parameters: - name: BaseBranch displayName: Base Branch default: main -- name: SkipForceRefresh - displayName: Skip Force Refresh +- name: SkipOpenAPIDocsDownload + displayName: Skip OpenAPI Docs Download (also skips branch creation and PR) default: false type: boolean - name: Test @@ -30,10 +30,6 @@ parameters: - name: CreatePullRequest type: boolean default: true -- name: SkipOpenAPIDocsDownload - displayName: Skip OpenAPI Docs Download - default: false - type: boolean - name: SpecifyModules displayName: Specify Modules to Generate default: false @@ -47,7 +43,6 @@ variables: Branch: "WeeklyApiRefresh" BaseBranch: ${{ parameters.BaseBranch }} BuildAgent: ${{ parameters.BuildAgent }} - SkipForceRefresh: ${{ parameters.SkipForceRefresh }} trigger: none pr: none schedules: @@ -85,12 +80,10 @@ extends: timeoutInMinutes: 1200 steps: - template: .azure-pipelines/common-templates/download-openapi-docs.yml@self - #condition: eq('${{ parameters.SkipOpenAPIDocsDownload }}', false) parameters: Branch: $(Branch) BaseBranch: $(BaseBranch) BuildAgent: $(BuildAgent) - SkipForceRefresh: $(SkipForceRefresh) SkipOpenAPIDocsDownload: ${{ parameters.SkipOpenAPIDocsDownload }} - job: SetUpModuleMatrix @@ -102,10 +95,6 @@ extends: WeeklyBranch: $[ dependencies.RefreshOpenAPIDocuments.outputs['ComputeBranch.WeeklyBranch'] ] ModulesToUpdate: $[ dependencies.RefreshOpenAPIDocuments.outputs['OpenAPIDocDiff.ModulesWithChanges'] ] steps: - # - template: .azure-pipelines/common-templates/checkout.yml@self - # parameters: - # TargetBranch: $(WeeklyBranch) - - task: PowerShell@2 name: CreateMatrixJson displayName: Create Matrix Json @@ -191,7 +180,7 @@ extends: dependsOn: - MsGraphPsSdkWeeklyGeneration - RefreshOpenAPIDocuments - condition: and(succeeded(), eq(dependencies.MsGraphPsSdkWeeklyGeneration.result, 'Succeeded'), eq(${{ parameters.CreatePullRequest }}, true)) + condition: and(succeeded(), eq(dependencies.MsGraphPsSdkWeeklyGeneration.result, 'Succeeded'), eq(${{ parameters.CreatePullRequest }}, true), eq(${{ parameters.SkipOpenAPIDocsDownload }}, false)) variables: WeeklyBranch: $[ dependencies.RefreshOpenAPIDocuments.outputs['ComputeBranch.WeeklyBranch'] ] steps: From 5e5e1211d9a5704bb24e3e131767a1d1a6c8df87 Mon Sep 17 00:00:00 2001 From: ramsessanchez <63934382+ramsessanchez@users.noreply.github.com> Date: Tue, 3 Feb 2026 15:54:27 -0800 Subject: [PATCH 08/11] move module matrix to independent script --- .azure-pipelines/weekly-generation.yml | 69 +++++++------------ tools/Utilities/CreateGenerationMatrix.ps1 | 79 ++++++++++++++++++++++ 2 files changed, 105 insertions(+), 43 deletions(-) create mode 100644 tools/Utilities/CreateGenerationMatrix.ps1 diff --git a/.azure-pipelines/weekly-generation.yml b/.azure-pipelines/weekly-generation.yml index 2e0f0ca4aec..3df1a0a9d77 100644 --- a/.azure-pipelines/weekly-generation.yml +++ b/.azure-pipelines/weekly-generation.yml @@ -77,6 +77,7 @@ extends: jobs: - job: RefreshOpenAPIDocuments displayName: Refresh OpenApi documents + condition: eq(${{ parameters.SkipOpenAPIDocsDownload }}, false) timeoutInMinutes: 1200 steps: - template: .azure-pipelines/common-templates/download-openapi-docs.yml@self @@ -85,65 +86,47 @@ extends: BaseBranch: $(BaseBranch) BuildAgent: $(BuildAgent) SkipOpenAPIDocsDownload: ${{ parameters.SkipOpenAPIDocsDownload }} - + - job: SetUpModuleMatrix displayName: Setup Module JSON for Matrix - dependsOn: RefreshOpenAPIDocuments - condition: and(succeeded(), ne(dependencies.RefreshOpenAPIDocuments.outputs['OpenAPIDocDiff.ModulesWithChanges'], '')) + ${{ if eq(parameters.SkipOpenAPIDocsDownload, false) }}: + dependsOn: RefreshOpenAPIDocuments + condition: and(succeeded(), ne(dependencies.RefreshOpenAPIDocuments.outputs['OpenAPIDocDiff.ModulesWithChanges'], '')) + ${{ else }}: + condition: succeeded() timeoutInMinutes: 840 variables: - WeeklyBranch: $[ dependencies.RefreshOpenAPIDocuments.outputs['ComputeBranch.WeeklyBranch'] ] - ModulesToUpdate: $[ dependencies.RefreshOpenAPIDocuments.outputs['OpenAPIDocDiff.ModulesWithChanges'] ] + ${{ if eq(parameters.SkipOpenAPIDocsDownload, false) }}: + WeeklyBranch: $[ dependencies.RefreshOpenAPIDocuments.outputs['ComputeBranch.WeeklyBranch'] ] + ModulesToGenerate: $[ dependencies.RefreshOpenAPIDocuments.outputs['OpenAPIDocDiff.ModulesWithChanges'] ] + ${{ else }}: + WeeklyBranch: '' + ModulesToGenerate: ${{ parameters.ModulesToGenerate }} steps: - task: PowerShell@2 name: CreateMatrixJson - displayName: Create Matrix Json + displayName: Create Matrix Json inputs: targetType: inline pwsh: true script: | - if ($${{ parameters.SpecifyModules }}) { - $modules = "${{ parameters.ModulesToGenerate }}" - } else { - $modules = $env:ModulesToUpdate - } - - Write-Host "ModulesToUpdate: $modules" - $testDataArray = $modules -split ' ' - $jsonOutput = @{} - - foreach ($item in $testDataArray) { - if ($item -notmatch '_') { - # If '_' is not present, create two versions: v1.0 and beta - $jsonOutput["${item}_v1.0"] = @{ - moduleVersion = 'v1.0' - moduleName = $item - } - $jsonOutput["${item}_beta"] = @{ - moduleVersion = 'beta' - moduleName = $item - } - } else { - $name, $version = $item -split '_' - $jsonOutput[$item] = @{ - moduleVersion = $version - moduleName = $name - } - } - } - - $result = $jsonOutput | ConvertTo-Json -Compress - Write-Host "##vso[task.setvariable variable=matrixJson;isOutput=true]$result" + & "$(System.DefaultWorkingDirectory)/tools/utilities/CreateGenerationMatrix.ps1" -ModuleNames "$(ModulesToGenerate)" -OutputVariable "matrixJson" - job: MsGraphPsSdkWeeklyGeneration - dependsOn: - - SetUpModuleMatrix - - RefreshOpenAPIDocuments + ${{ if eq(parameters.SkipOpenAPIDocsDownload, false) }}: + dependsOn: + - SetUpModuleMatrix + - RefreshOpenAPIDocuments + ${{ else }}: + dependsOn: SetUpModuleMatrix displayName: Microsoft Graph PowerShell SDK Generation - condition: and(succeeded(), ne(dependencies.RefreshOpenAPIDocuments.outputs['OpenAPIDocDiff.ModulesWithChanges'], '')) + condition: succeeded() timeoutInMinutes: 840 variables: - WeeklyBranch: $[ dependencies.RefreshOpenAPIDocuments.outputs['ComputeBranch.WeeklyBranch'] ] + ${{ if eq(parameters.SkipOpenAPIDocsDownload, false) }}: + WeeklyBranch: $[ dependencies.RefreshOpenAPIDocuments.outputs['ComputeBranch.WeeklyBranch'] ] + ${{ else }}: + WeeklyBranch: '' ModulesMatrix: $[ dependencies.SetUpModuleMatrix.outputs['CreateMatrixJson.matrixJson'] ] templateContext: outputs: diff --git a/tools/Utilities/CreateGenerationMatrix.ps1 b/tools/Utilities/CreateGenerationMatrix.ps1 new file mode 100644 index 00000000000..86110e68ebe --- /dev/null +++ b/tools/Utilities/CreateGenerationMatrix.ps1 @@ -0,0 +1,79 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +<# +.SYNOPSIS + Creates a JSON matrix for Azure DevOps parallel module generation. + +.DESCRIPTION + This script takes a space-separated list of module names and converts them into a JSON matrix + for use with Azure DevOps matrix strategy. If a module name doesn't include a version suffix, + it creates entries for both v1.0 and beta versions. + +.PARAMETER ModuleNames + Space-separated string of module names. Can include version suffixes (e.g., "Users_v1.0") or not (e.g., "Users"). + If no version suffix is provided, both v1.0 and beta entries will be created. + +.PARAMETER OutputVariable + The name of the Azure DevOps output variable to set. If not provided, the JSON is written to the pipeline. + +.EXAMPLE + .\CreateGenerationMatrix.ps1 -ModuleNames "Users Groups" + Creates matrix with Users_v1.0, Users_beta, Groups_v1.0, Groups_beta + +.EXAMPLE + .\CreateGenerationMatrix.ps1 -ModuleNames "Users_v1.0 Groups_beta" + Creates matrix with only Users_v1.0 and Groups_beta +#> + +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [ValidateNotNullOrEmpty()] + [string] $ModuleNames, + + [Parameter(Mandatory = $false)] + [string] $OutputVariable = "matrixJson" +) + +Write-Host "ModulesToUpdate: $ModuleNames" + +# Split the input string into an array +$testDataArray = $ModuleNames -split ' ' +$jsonOutput = @{} + +foreach ($item in $testDataArray) { + # Skip empty entries + if ([string]::IsNullOrWhiteSpace($item)) { + continue + } + + if ($item -notmatch '_') { + # If '_' is not present, create two versions: v1.0 and beta + $jsonOutput["${item}_v1.0"] = @{ + moduleVersion = 'v1.0' + moduleName = $item + } + $jsonOutput["${item}_beta"] = @{ + moduleVersion = 'beta' + moduleName = $item + } + } + else { + # Split on '_' to get name and version + $name, $version = $item -split '_' + $jsonOutput[$item] = @{ + moduleVersion = $version + moduleName = $name + } + } +} + +# Convert to compressed JSON +$result = $jsonOutput | ConvertTo-Json -Compress + +Write-Host "Generated matrix JSON:" +Write-Host $result + +# Set Azure DevOps output variable +Write-Host "##vso[task.setvariable variable=$OutputVariable;isOutput=true]$result" From 9e3064056d11e5b9261dd906aa618218f9c7aa9f Mon Sep 17 00:00:00 2001 From: ramsessanchez <63934382+ramsessanchez@users.noreply.github.com> Date: Tue, 3 Feb 2026 16:44:30 -0800 Subject: [PATCH 09/11] consolidate conditionals and variables --- .../download-openapi-docs.yml | 60 ++++++++----------- .azure-pipelines/weekly-generation.yml | 27 +++------ 2 files changed, 32 insertions(+), 55 deletions(-) diff --git a/.azure-pipelines/common-templates/download-openapi-docs.yml b/.azure-pipelines/common-templates/download-openapi-docs.yml index 729e6066b9f..633a6b75359 100644 --- a/.azure-pipelines/common-templates/download-openapi-docs.yml +++ b/.azure-pipelines/common-templates/download-openapi-docs.yml @@ -4,13 +4,11 @@ parameters: - name: Branch type: string + default: "WeeklyApiRefresh" - name: BaseBranch type: string - name: BuildAgent displayName: Build Agent - - name: SkipOpenAPIDocsDownload - type: boolean - default: false steps: - template: ./checkout.yml @@ -21,33 +19,27 @@ steps: - task: PowerShell@2 name: "ComputeBranch" - displayName: "Compute weekly branch name" + displayName: "Compute weekly branch name and set Branch variable" inputs: targetType: inline script: | - if ('${{ parameters.SkipOpenAPIDocsDownload }}' -eq 'True') { - $branch = "" - Write-Host "Skipping branch creation - using current branch" - } else { - $branch = "{0}/{1}" -f "$(Branch)", (Get-Date -Format yyyyMMddHHmm) - Write-Host "Computed weekly branch: $branch" - } - Write-Host "##vso[task.setvariable variable=WeeklyBranch;isOutput=true]$branch" + $branch = "{0}/{1}" -f "${{ parameters.Branch }}", (Get-Date -Format yyyyMMddHHmm) + Write-Host "Computed weekly branch: $branch" + # Set the Branch pipeline variable + Write-Host "##vso[task.setvariable variable=Branch;isOutput=false]$branch" - task: Bash@3 displayName: "Create weekly branch" - condition: eq('${{ parameters.SkipOpenAPIDocsDownload }}', false) inputs: targetType: inline script: | git status - git branch $(ComputeBranch.WeeklyBranch) - git checkout $(ComputeBranch.WeeklyBranch) + git branch $(Branch) + git checkout $(Branch) git status - task: PowerShell@2 displayName: Download v1.0 OpenApi docs - condition: and(succeeded(), eq('${{ parameters.SkipOpenAPIDocsDownload }}', false)) continueOnError: false inputs: filePath: "$(System.DefaultWorkingDirectory)/tools/UpdateOpenApi.ps1" @@ -55,7 +47,6 @@ steps: - task: PowerShell@2 displayName: Download beta OpenApi docs - condition: and(succeeded(), eq('${{ parameters.SkipOpenAPIDocsDownload }}', false)) continueOnError: false inputs: filePath: "$(System.DefaultWorkingDirectory)/tools/UpdateOpenApi.ps1" @@ -64,30 +55,27 @@ steps: - task: PowerShell@2 name: OpenAPIDocDiff - displayName: Get OpenAPI docs diff + displayName: Get OpenAPI docs diff and set ModuleGenerationList variable inputs: pwsh: true targetType: "inline" script: | - Write-Host "SkipOpenAPIDocsDownload: ${{ parameters.SkipOpenAPIDocsDownload }}" - if ('${{ parameters.SkipOpenAPIDocsDownload }}' -eq 'True') { - $ModuleNames = "Skipped" - Write-Warning "Skipped OpenAPI Docs Download." - } else { - $diff = git diff --name-only - $ModulesWithChanges = @{} - $diff | %{ - if ($_ -match 'openApiDocs\/(v1.0|beta)\/(.*)\.yml') { - $version = if ($matches[1] -eq 'v1.0') { 'v1.0' } else { 'beta' } - $moduleName = "$($matches[2])_$version" - if (!$ModulesWithChanges.ContainsKey($moduleName)) { - $ModulesWithChanges.Add($moduleName, $matches[1]) - } + $diff = git diff --name-only + $ModulesWithChanges = @{} + $diff | %{ + if ($_ -match 'openApiDocs\/(v1.0|beta)\/(.*)\.yml') { + $version = if ($matches[1] -eq 'v1.0') { 'v1.0' } else { 'beta' } + $moduleName = "$($matches[2])_$version" + if (!$ModulesWithChanges.ContainsKey($moduleName)) { + $ModulesWithChanges.Add($moduleName, $matches[1]) } } - $ModuleNames = $ModulesWithChanges.Keys - Write-Host "Modules with changes: $ModuleNames" } + $ModuleNames = $ModulesWithChanges.Keys + Write-Host "Modules with changes: $ModuleNames" + # Set the ModuleGenerationList pipeline variable + Write-Host "##vso[task.setvariable variable=ModuleGenerationList;isOutput=false]$ModuleNames" + # Set output variable for job condition checks Write-Host "##vso[task.setvariable variable=ModulesWithChanges;isOutput=true]$ModuleNames" - task: PowerShell@2 @@ -110,7 +98,7 @@ steps: - task: Bash@3 displayName: Commit downloaded files - condition: and(succeeded(), ne(variables['OpenAPIDocDiff.ModulesWithChanges'], ''), eq('${{ parameters.SkipOpenAPIDocsDownload }}', false)) + condition: and(succeeded(), ne(variables['OpenAPIDocDiff.ModulesWithChanges'], '')) env: GITHUB_TOKEN: $(GITHUB_TOKEN) inputs: @@ -120,7 +108,7 @@ steps: git add . git commit -m 'Weekly OpenApiDocs Download.' git status - git push --set-upstream "https://$(GITHUB_TOKEN)@github.com/microsoftgraph/msgraph-sdk-powershell.git" $(ComputeBranch.WeeklyBranch) + git push --set-upstream "https://$(GITHUB_TOKEN)@github.com/microsoftgraph/msgraph-sdk-powershell.git" $(Branch) git status # References diff --git a/.azure-pipelines/weekly-generation.yml b/.azure-pipelines/weekly-generation.yml index 3df1a0a9d77..1c98219d79f 100644 --- a/.azure-pipelines/weekly-generation.yml +++ b/.azure-pipelines/weekly-generation.yml @@ -40,9 +40,10 @@ parameters: type: string variables: - Branch: "WeeklyApiRefresh" + Branch: "" BaseBranch: ${{ parameters.BaseBranch }} BuildAgent: ${{ parameters.BuildAgent }} + ModuleGenerationList: "${{ parameters.ModulesToGenerate }}" trigger: none pr: none schedules: @@ -85,7 +86,6 @@ extends: Branch: $(Branch) BaseBranch: $(BaseBranch) BuildAgent: $(BuildAgent) - SkipOpenAPIDocsDownload: ${{ parameters.SkipOpenAPIDocsDownload }} - job: SetUpModuleMatrix displayName: Setup Module JSON for Matrix @@ -95,13 +95,6 @@ extends: ${{ else }}: condition: succeeded() timeoutInMinutes: 840 - variables: - ${{ if eq(parameters.SkipOpenAPIDocsDownload, false) }}: - WeeklyBranch: $[ dependencies.RefreshOpenAPIDocuments.outputs['ComputeBranch.WeeklyBranch'] ] - ModulesToGenerate: $[ dependencies.RefreshOpenAPIDocuments.outputs['OpenAPIDocDiff.ModulesWithChanges'] ] - ${{ else }}: - WeeklyBranch: '' - ModulesToGenerate: ${{ parameters.ModulesToGenerate }} steps: - task: PowerShell@2 name: CreateMatrixJson @@ -110,7 +103,9 @@ extends: targetType: inline pwsh: true script: | - & "$(System.DefaultWorkingDirectory)/tools/utilities/CreateGenerationMatrix.ps1" -ModuleNames "$(ModulesToGenerate)" -OutputVariable "matrixJson" + $modules = "$(ModuleGenerationList)" + Write-Host "Using ModuleGenerationList: $modules" + & "$(System.DefaultWorkingDirectory)/tools/utilities/CreateGenerationMatrix.ps1" -ModuleNames $modules -OutputVariable "matrixJson" - job: MsGraphPsSdkWeeklyGeneration ${{ if eq(parameters.SkipOpenAPIDocsDownload, false) }}: @@ -123,10 +118,6 @@ extends: condition: succeeded() timeoutInMinutes: 840 variables: - ${{ if eq(parameters.SkipOpenAPIDocsDownload, false) }}: - WeeklyBranch: $[ dependencies.RefreshOpenAPIDocuments.outputs['ComputeBranch.WeeklyBranch'] ] - ${{ else }}: - WeeklyBranch: '' ModulesMatrix: $[ dependencies.SetUpModuleMatrix.outputs['CreateMatrixJson.matrixJson'] ] templateContext: outputs: @@ -141,7 +132,7 @@ extends: steps: - template: .azure-pipelines/common-templates/checkout.yml@self parameters: - TargetBranch: $(WeeklyBranch) + TargetBranch: $(Branch) - template: .azure-pipelines/common-templates/install-tools.yml@self - template: .azure-pipelines/common-templates/security-pre-checks.yml@self - template: .azure-pipelines/generation-templates/authentication-module.yml@self @@ -164,15 +155,13 @@ extends: - MsGraphPsSdkWeeklyGeneration - RefreshOpenAPIDocuments condition: and(succeeded(), eq(dependencies.MsGraphPsSdkWeeklyGeneration.result, 'Succeeded'), eq(${{ parameters.CreatePullRequest }}, true), eq(${{ parameters.SkipOpenAPIDocsDownload }}, false)) - variables: - WeeklyBranch: $[ dependencies.RefreshOpenAPIDocuments.outputs['ComputeBranch.WeeklyBranch'] ] steps: - template: .azure-pipelines/common-templates/checkout.yml@self parameters: - TargetBranch: $(WeeklyBranch) + TargetBranch: $(Branch) - template: .azure-pipelines/common-templates/create-pr.yml@self parameters: BaseBranch: $(BaseBranch) - TargetBranch: $(WeeklyBranch) + TargetBranch: $(Branch) Title: "[v2] Weekly OpenApiDocs Refresh" Body: "This pull request was automatically created by Azure Pipelines. **Important** Check for unexpected deletions or changes in this PR." \ No newline at end of file From cbd40d3d618bb8956ab94798ea7a5714e2b85eb0 Mon Sep 17 00:00:00 2001 From: ramsessanchez <63934382+ramsessanchez@users.noreply.github.com> Date: Tue, 3 Feb 2026 18:06:57 -0800 Subject: [PATCH 10/11] move openapidocdiff to dedicated script file --- .../download-openapi-docs.yml | 22 +------- .azure-pipelines/weekly-generation.yml | 6 +- tools/Utilities/OpenApiDocDiff.ps1 | 56 +++++++++++++++++++ 3 files changed, 60 insertions(+), 24 deletions(-) create mode 100644 tools/Utilities/OpenApiDocDiff.ps1 diff --git a/.azure-pipelines/common-templates/download-openapi-docs.yml b/.azure-pipelines/common-templates/download-openapi-docs.yml index 633a6b75359..729f5d2a015 100644 --- a/.azure-pipelines/common-templates/download-openapi-docs.yml +++ b/.azure-pipelines/common-templates/download-openapi-docs.yml @@ -57,26 +57,8 @@ steps: name: OpenAPIDocDiff displayName: Get OpenAPI docs diff and set ModuleGenerationList variable inputs: + filePath: "$(System.DefaultWorkingDirectory)/tools/Utilities/OpenApiDocDiff.ps1" pwsh: true - targetType: "inline" - script: | - $diff = git diff --name-only - $ModulesWithChanges = @{} - $diff | %{ - if ($_ -match 'openApiDocs\/(v1.0|beta)\/(.*)\.yml') { - $version = if ($matches[1] -eq 'v1.0') { 'v1.0' } else { 'beta' } - $moduleName = "$($matches[2])_$version" - if (!$ModulesWithChanges.ContainsKey($moduleName)) { - $ModulesWithChanges.Add($moduleName, $matches[1]) - } - } - } - $ModuleNames = $ModulesWithChanges.Keys - Write-Host "Modules with changes: $ModuleNames" - # Set the ModuleGenerationList pipeline variable - Write-Host "##vso[task.setvariable variable=ModuleGenerationList;isOutput=false]$ModuleNames" - # Set output variable for job condition checks - Write-Host "##vso[task.setvariable variable=ModulesWithChanges;isOutput=true]$ModuleNames" - task: PowerShell@2 name: BuildOpenApiMetadataDetectionTool @@ -98,7 +80,7 @@ steps: - task: Bash@3 displayName: Commit downloaded files - condition: and(succeeded(), ne(variables['OpenAPIDocDiff.ModulesWithChanges'], '')) + condition: and(succeeded(), ne(variables['ModuleGenerationList'], '')) env: GITHUB_TOKEN: $(GITHUB_TOKEN) inputs: diff --git a/.azure-pipelines/weekly-generation.yml b/.azure-pipelines/weekly-generation.yml index 1c98219d79f..055eaa22fa4 100644 --- a/.azure-pipelines/weekly-generation.yml +++ b/.azure-pipelines/weekly-generation.yml @@ -91,9 +91,7 @@ extends: displayName: Setup Module JSON for Matrix ${{ if eq(parameters.SkipOpenAPIDocsDownload, false) }}: dependsOn: RefreshOpenAPIDocuments - condition: and(succeeded(), ne(dependencies.RefreshOpenAPIDocuments.outputs['OpenAPIDocDiff.ModulesWithChanges'], '')) - ${{ else }}: - condition: succeeded() + condition: and(succeeded(), ne(variables['ModuleGenerationList'], '')) timeoutInMinutes: 840 steps: - task: PowerShell@2 @@ -105,7 +103,7 @@ extends: script: | $modules = "$(ModuleGenerationList)" Write-Host "Using ModuleGenerationList: $modules" - & "$(System.DefaultWorkingDirectory)/tools/utilities/CreateGenerationMatrix.ps1" -ModuleNames $modules -OutputVariable "matrixJson" + & "$(System.DefaultWorkingDirectory)/tools/utilities/CreateGenerationMatrix.ps1" -ModuleNames $modules - job: MsGraphPsSdkWeeklyGeneration ${{ if eq(parameters.SkipOpenAPIDocsDownload, false) }}: diff --git a/tools/Utilities/OpenApiDocDiff.ps1 b/tools/Utilities/OpenApiDocDiff.ps1 new file mode 100644 index 00000000000..90488eb6fd3 --- /dev/null +++ b/tools/Utilities/OpenApiDocDiff.ps1 @@ -0,0 +1,56 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +<# +.SYNOPSIS + Detects changes in OpenAPI documentation files and outputs the list of affected modules. + +.DESCRIPTION + This script performs a git diff to identify which OpenAPI YAML files have changed, + extracts the module names and versions from the file paths, and outputs a space-separated + list of modules that need to be regenerated. + +.PARAMETER OutputVariable + The name of the Azure DevOps pipeline variable to set with the results. + Default is "ModuleGenerationList". + +.EXAMPLE + .\OpenApiDocDiff.ps1 + Detects changes and sets the ModuleGenerationList pipeline variable. + +.EXAMPLE + .\OpenApiDocDiff.ps1 -OutputVariable "ChangedModules" + Detects changes and sets the ChangedModules pipeline variable. +#> + +[CmdletBinding()] +param( + [Parameter(Mandatory = $false)] + [string] $OutputVariable = "ModuleGenerationList" +) + +# Get the list of changed files from git +$diff = git diff --name-only +$ModulesWithChanges = @{} + +# Parse each changed file path +$diff | ForEach-Object { + # Match OpenAPI docs paths: openApiDocs/(v1.0|beta)/{ModuleName}.yml + if ($_ -match 'openApiDocs\/(v1.0|beta)\/(.*)\.yml') { + $version = if ($matches[1] -eq 'v1.0') { 'v1.0' } else { 'beta' } + $moduleName = "$($matches[2])_$version" + + # Add to hashtable if not already present + if (!$ModulesWithChanges.ContainsKey($moduleName)) { + $ModulesWithChanges.Add($moduleName, $matches[1]) + } + } +} + +# Convert hashtable keys to space-separated string +$ModuleNames = $ModulesWithChanges.Keys + +Write-Host "Modules with changes: $ModuleNames" + +# Set the pipeline variable +Write-Host "##vso[task.setvariable variable=$OutputVariable;isOutput=false]$ModuleNames" From c73d5e910d0080b02c6f747c0e603b084a9a4bb5 Mon Sep 17 00:00:00 2001 From: ramsessanchez <63934382+ramsessanchez@users.noreply.github.com> Date: Tue, 3 Feb 2026 23:55:02 -0800 Subject: [PATCH 11/11] remove duplicate diff script --- tools/OpenApiDocDiff.ps1 | 20 -------------------- 1 file changed, 20 deletions(-) delete mode 100644 tools/OpenApiDocDiff.ps1 diff --git a/tools/OpenApiDocDiff.ps1 b/tools/OpenApiDocDiff.ps1 deleted file mode 100644 index 60d42da9941..00000000000 --- a/tools/OpenApiDocDiff.ps1 +++ /dev/null @@ -1,20 +0,0 @@ - Write-Host "SkipOpenAPIDocsDownload: ${{ parameters.SkipOpenAPIDocsDownload }}" - if ('${{ parameters.SkipOpenAPIDocsDownload }}' -eq 'True') { - $ModuleNames = "Skipped" - Write-Warning "Skipped OpenAPI Docs Download." - } else { - $diff = git diff --name-only - $ModulesWithChanges = @{} - $diff | %{ - if ($_ -match 'openApiDocs\/(v1.0|beta)\/(.*)\.yml') { - $version = if ($matches[1] -eq 'v1.0') { 'v1.0' } else { 'beta' } - $moduleName = "$($matches[2])_$version" - if (!$ModulesWithChanges.ContainsKey($moduleName)) { - $ModulesWithChanges.Add($moduleName, $matches[1]) - } - } - } - $ModuleNames = $ModulesWithChanges.Keys - Write-Host "Modules with changes: $ModuleNames" - } - Write-Host "##vso[task.setvariable variable=ModulesWithChanges;isOutput=true]$ModuleNames" \ No newline at end of file