Skip to content

Commit fb83966

Browse files
ramsessanchezramsessanchez
committed
Split Weekly Generation Pipeline
Split Weekly generation pipeline into two separate pipelines, one to retrieve openApi updates and individually build the modules. A second pipeline to take the current api docs and generate the commandMetadata when we approach a release date.
1 parent 5f7bf07 commit fb83966

File tree

4 files changed

+312
-31
lines changed

4 files changed

+312
-31
lines changed

‎.azure-pipelines/common-templates/download-openapi-docs.yml‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -73,12 +73,17 @@ steps:
7373
$diff = git diff --name-only
7474
$ModulesWithChanges = @{}
7575
$diff | %{
76-
if (($_ -match 'openApiDocs\/(v1.0|beta)\/(.*).yml') -and !$ModulesWithChanges.ContainsKey($matches.2))
77-
{
78-
$ModulesWithChanges.Add($matches.2, $matches.1)
76+
if ($_ -match 'openApiDocs\/(v1.0|beta)\/(.*)\.yml') {
77+
$version = if ($matches[1] -eq 'v1.0') { 'v1.0' } else { 'beta' }
78+
$moduleName = "$($matches[2])_$version"
79+
if (!$ModulesWithChanges.ContainsKey($moduleName)) {
80+
$ModulesWithChanges.Add($moduleName, $matches[1])
81+
}
7982
}
8083
}
8184
$ModuleNames = $ModulesWithChanges.Keys
85+
Write-Host "Modules with changes: $ModuleNames"
86+
Write-Host "Testing Line"
8287
}
8388
Write-Host "##vso[task.setvariable variable=ModulesWithChanges;isOutput=true]$ModuleNames"
8489

‎.azure-pipelines/generation-templates/individualized-workload-modules.yml‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
# Copyright (c) Microsoft Corporation. All rights reserved.
2+
# Licensed under the MIT License.
3+
4+
# Copyright (c) Microsoft Corporation. All rights reserved.
5+
# Licensed under the MIT License.
6+
7+
parameters:
8+
- name: Test
9+
type: boolean
10+
default: true
11+
- name: Pack
12+
type: boolean
13+
default: true
14+
- name: Sign
15+
type: boolean
16+
default: true
17+
- name: ModuleName
18+
type: string
19+
default: ""
20+
- name: ModuleVersion
21+
type: string
22+
default: ""
23+
24+
steps:
25+
- task: PowerShell@2
26+
displayName: Generate Workload Modules
27+
inputs:
28+
targetType: inline
29+
pwsh: true
30+
script: |
31+
. $(System.DefaultWorkingDirectory)/tools/GenerateModules.ps1 -EnableSigning:$${{ parameters.Sign }} -Build -ExcludeExampleTemplates -ExcludeNotesSection -ModuleToGenerate ${{ parameters.ModuleName }} -ApiVersion ${{ parameters.ModuleVersion }}
32+
- template: ../common-templates/guardian-analyzer.yml
33+
34+
- task: PowerShell@2
35+
displayName: Test Workload Modules
36+
enabled: false
37+
inputs:
38+
targetType: inline
39+
pwsh: true
40+
script: |
41+
. $(System.DefaultWorkingDirectory)/tools/GenerateModules.ps1 -SkipGeneration -Test -ModuleToGenerate ${{ parameters.ModuleName }} -ApiVersion ${{ parameters.ModuleVersion }}
42+
43+
- task: PowerShell@2
44+
displayName: Find Duplicate Commands
45+
inputs:
46+
targetType: inline
47+
pwsh: true
48+
script: |
49+
. $(System.DefaultWorkingDirectory)/tools/PostGeneration/FindDuplicateCommand.ps1 -SourcePath "$(System.DefaultWorkingDirectory)/src/"
50+
51+
- ${{ if eq(parameters.Pack, true) }}:
52+
- task: PowerShell@2
53+
displayName: Pack Workload Modules
54+
inputs:
55+
targetType: inline
56+
pwsh: true
57+
script: |
58+
. $(System.DefaultWorkingDirectory)/tools/GenerateModules.ps1 -SkipGeneration -Pack -ArtifactsLocation $(Build.ArtifactStagingDirectory)

‎.azure-pipelines/module-metadata-generation.yml‎

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,154 @@
1+
# This Yaml Document has been converted by ESAI Yaml Pipeline Conversion Tool.
2+
# Please make sure to check all the converted content, it is your team's responsibility to make sure that the pipeline is still valid and functions as expected.
3+
# This pipeline will be extended to the OneESPT template
4+
# If you are not using the E+D shared hosted pool with windows-2022, replace the pool section with your hosted pool, os, and image name. If you are using a Linux image, you must specify an additional windows image for SDL: https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/1es-pipeline-templates/features/sdlanalysis/overview#how-to-specify-a-windows-pool-for-the-sdl-source-analysis-stage
5+
# The Task 'PublishBuildArtifacts@1' has been converted to an output named 'Publish Module Artifacts' in the templateContext section.
6+
# The Task 'NuGetCommand@2' has been converted to an output named 'Publish NuGet to feed' in the templateContext section.
7+
# Output added to job "MsGraphPsSdkWeeklyGeneration" with YAML conditionals extracted using AI. Review this expression against the originating file for correctness.
8+
# Output added to job "MsGraphPsSdkWeeklyGeneration" with YAML conditionals extracted using AI. Review this expression against the originating file for correctness.
9+
name: $(BuildDefinitionName)_$(SourceBranchName)_$(Date:yyyyMMdd)$(Rev:.r)
10+
parameters:
11+
- name: BuildAgent
12+
displayName: Build Agent
13+
default: 1es-windows-ps-compute-m
14+
- name: BaseBranch
15+
displayName: Base Branch
16+
default: dev
17+
- name: Test
18+
type: boolean
19+
default: true
20+
- name: Pack
21+
type: boolean
22+
default: true
23+
- name: Sign
24+
type: boolean
25+
default: true
26+
- name: CreatePullRequest
27+
type: boolean
28+
default: true
29+
30+
variables:
31+
BuildAgent: ${{ parameters.BuildAgent }}
32+
Branch: "ModuleMetadataRefresh"
33+
BaseBranch: ${{ parameters.BaseBranch }}
34+
35+
trigger:
36+
branches:
37+
include:
38+
- dev
39+
40+
resources:
41+
repositories:
42+
- repository: 1ESPipelineTemplates
43+
type: git
44+
name: 1ESPipelineTemplates/1ESPipelineTemplates
45+
ref: refs/tags/release
46+
extends:
47+
template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates
48+
parameters:
49+
pool: $(BuildAgent)
50+
sdl:
51+
binskim:
52+
enabled: false
53+
justificationForDisabling: "Binskim keeps on crushing and failing the weekly build pipeline. Disabling it for now because we are unable to publish the artifacts to internal feeds."
54+
credscan:
55+
suppressionsFile: $(Build.SourcesDirectory)/.azure-pipelines/config/credscan/credscan-suppressions.json
56+
policheck:
57+
exclusionFile: $(Build.SourcesDirectory)/.azure-pipelines/config/policheck/policheck-exclusions.xml
58+
customBuildTags:
59+
- ES365AIMigrationTooling
60+
stages:
61+
- stage: stage
62+
jobs:
63+
- job: CreateMetadataRefreshBranch
64+
displayName: Create Metadata Refresh Branch
65+
timeoutInMinutes: 1200
66+
steps:
67+
- template: ./checkout.yml
68+
parameters:
69+
TargetBranch: ${{ parameters.BaseBranch }}
70+
- task: PowerShell@2
71+
name: "ComputeBranch"
72+
displayName: "Compute weekly branch name"
73+
inputs:
74+
targetType: inline
75+
script: |
76+
$branch = "{0}/{1}" -f "$(Branch)", (Get-Date -Format yyyyMMddHHmm)
77+
Write-Host "##vso[task.setvariable variable=WeeklyBranch;isOutput=true]$branch"
78+
- task: Bash@3
79+
displayName: "Create weekly branch"
80+
inputs:
81+
targetType: inline
82+
script: |
83+
git status
84+
git branch $(ComputeBranch.WeeklyBranch)
85+
git checkout $(ComputeBranch.WeeklyBranch)
86+
git status
87+
- task: PowerShell@2
88+
name: BuildOpenApiMetadataDetectionTool
89+
displayName: Build tool for detecting metadata changes
90+
inputs:
91+
pwsh: true
92+
targetType: inline
93+
script: dotnet build --configuration Release
94+
workingDirectory: "$(System.DefaultWorkingDirectory)/tools/OpenApiInfoGenerator/OpenApiInfoGenerator"
95+
96+
97+
98+
- job: MsGraphPsSdkWeeklyGeneration
99+
dependsOn: CreateMetadataRefreshBranch
100+
displayName: Microsoft Graph PowerShell SDK Generation
101+
condition: succeeded()
102+
timeoutInMinutes: 840
103+
variables:
104+
WeeklyBranch: $[ dependencies.CreateMetadataRefreshBranch.outputs['ComputeBranch.WeeklyBranch'] ]
105+
templateContext:
106+
outputs:
107+
- ${{ if and(eq(parameters.Pack, true), eq(parameters.Sign, true)) }}:
108+
- output: pipelineArtifact
109+
displayName: 'Publish Module Artifacts'
110+
targetPath: "$(Build.ArtifactStagingDirectory)"
111+
artifactName: "drop"
112+
publishLocation: "Container"
113+
- ${{ if and(eq(parameters.Pack, true), eq(parameters.Sign, true)) }}:
114+
- output: nuget
115+
displayName: 'Publish NuGet to feed'
116+
packageParentPath: '$(Build.ArtifactStagingDirectory)'
117+
packagesToPush: $(Build.ArtifactStagingDirectory)/**/Microsoft.Graph.*.nupkg
118+
publishVstsFeed: $(PROJECT_NAME)/$(FEED_NAME)
119+
allowPackageConflicts: true
120+
steps:
121+
- template: .azure-pipelines/common-templates/checkout.yml@self
122+
parameters:
123+
TargetBranch: $(WeeklyBranch)
124+
- template: .azure-pipelines/common-templates/install-tools.yml@self
125+
- template: .azure-pipelines/common-templates/security-pre-checks.yml@self
126+
- template: .azure-pipelines/generation-templates/authentication-module.yml@self
127+
parameters:
128+
Test: ${{ parameters.Test }}
129+
Pack: ${{ parameters.Pack }}
130+
Sign: ${{ parameters.Sign }}
131+
- template: .azure-pipelines/generation-templates/workload-modules.yml@self
132+
parameters:
133+
Test: ${{ parameters.Test }}
134+
Pack: ${{ parameters.Pack }}
135+
Sign: ${{ parameters.Sign }}
136+
- template: .azure-pipelines/generation-templates/meta-module.yml@self
137+
parameters:
138+
Test: ${{ parameters.Test }}
139+
Pack: ${{ parameters.Pack }}
140+
Sign: ${{ parameters.Sign }}
141+
- ${{ if and(eq(parameters.Pack, true), eq(parameters.Sign, true)) }}:
142+
- template: .azure-pipelines/common-templates/esrp/codesign-nuget.yml@self
143+
parameters:
144+
FolderPath: "$(Build.ArtifactStagingDirectory)"
145+
Pattern: "Microsoft.Graph*.nupkg"
146+
- template: .azure-pipelines/generation-templates/generate-command-metadata.yml@self
147+
- template: .azure-pipelines/common-templates/security-post-checks.yml@self
148+
- ${{ if eq(parameters.CreatePullRequest, true) }}:
149+
- template: .azure-pipelines/common-templates/create-pr.yml@self
150+
parameters:
151+
BaseBranch: $(BaseBranch)
152+
TargetBranch: $(WeeklyBranch)
153+
Title: "[v2] Weekly OpenApiDocs Refresh"
154+
Body: "This pull request was automatically created by Azure Pipelines. **Important** Check for unexpected deletions or changes in this PR."

‎.azure-pipelines/weekly-generation.yml‎

Lines changed: 92 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,15 @@ parameters:
3434
displayName: Skip OpenAPI Docs Download
3535
default: false
3636
type: boolean
37+
- name: SpecifyModules
38+
displayName: Specify Modules to Generate
39+
default: false
40+
type: boolean
41+
- name: ModulesToGenerate
42+
displayName: Modules To Generate
43+
default: Insert Module Names Here separated by space, e.g., Users_v1.0 Users_beta
44+
type: string
45+
3746
variables:
3847
Branch: "WeeklyApiRefresh"
3948
BaseBranch: ${{ parameters.BaseBranch }}
@@ -82,28 +91,80 @@ extends:
8291
BuildAgent: $(BuildAgent)
8392
SkipForceRefresh: $(SkipForceRefresh)
8493
SkipOpenAPIDocsDownload: ${{ parameters.SkipOpenAPIDocsDownload }}
85-
- job: MsGraphPsSdkWeeklyGeneration
94+
95+
- job: SetUpModuleMatrix
96+
displayName: Setup Module JSON for Matrix
8697
dependsOn: RefreshOpenAPIDocuments
98+
condition: and(succeeded(), ne(dependencies.RefreshOpenAPIDocuments.outputs['OpenAPIDocDiff.ModulesWithChanges'], ''))
99+
timeoutInMinutes: 840
100+
variables:
101+
WeeklyBranch: $[ dependencies.RefreshOpenAPIDocuments.outputs['ComputeBranch.WeeklyBranch'] ]
102+
ModulesToUpdate: $[ dependencies.RefreshOpenAPIDocuments.outputs['OpenAPIDocDiff.ModulesWithChanges'] ]
103+
steps:
104+
- template: .azure-pipelines/common-templates/checkout.yml@self
105+
parameters:
106+
TargetBranch: $(WeeklyBranch)
107+
108+
- task: PowerShell@2
109+
name: CreateMatrixJson
110+
displayName: Create Matrix Json
111+
inputs:
112+
targetType: inline
113+
pwsh: true
114+
script: |
115+
if ($${{ parameters.SpecifyModules }}) {
116+
$modules = "${{ parameters.ModulesToGenerate }}"
117+
} else {
118+
$modules = $env:ModulesToUpdate
119+
}
120+
121+
Write-Host "ModulesToUpdate: $modules"
122+
$testDataArray = $modules -split ' '
123+
$jsonOutput = @{}
124+
125+
foreach ($item in $testDataArray) {
126+
if ($item -notmatch '_') {
127+
# If '_' is not present, create two versions: v1.0 and beta
128+
$jsonOutput["${item}_v1.0"] = @{
129+
moduleVersion = 'v1.0'
130+
moduleName = $item
131+
}
132+
$jsonOutput["${item}_beta"] = @{
133+
moduleVersion = 'beta'
134+
moduleName = $item
135+
}
136+
} else {
137+
$name, $version = $item -split '_'
138+
$jsonOutput[$item] = @{
139+
moduleVersion = $version
140+
moduleName = $name
141+
}
142+
}
143+
}
144+
145+
$result = $jsonOutput | ConvertTo-Json -Compress
146+
Write-Host "##vso[task.setvariable variable=matrixJson;isOutput=true]$result"
147+
148+
- job: MsGraphPsSdkWeeklyGeneration
149+
dependsOn:
150+
- SetUpModuleMatrix
151+
- RefreshOpenAPIDocuments
87152
displayName: Microsoft Graph PowerShell SDK Generation
88153
condition: and(succeeded(), ne(dependencies.RefreshOpenAPIDocuments.outputs['OpenAPIDocDiff.ModulesWithChanges'], ''))
89154
timeoutInMinutes: 840
90155
variables:
91156
WeeklyBranch: $[ dependencies.RefreshOpenAPIDocuments.outputs['ComputeBranch.WeeklyBranch'] ]
157+
ModulesMatrix: $[ dependencies.SetUpModuleMatrix.outputs['CreateMatrixJson.matrixJson'] ]
92158
templateContext:
93159
outputs:
94160
- ${{ if and(eq(parameters.Pack, true), eq(parameters.Sign, true)) }}:
95161
- output: pipelineArtifact
96162
displayName: 'Publish Module Artifacts'
97163
targetPath: "$(Build.ArtifactStagingDirectory)"
98-
artifactName: "drop"
164+
artifactName: "$(moduleName)_$(moduleVersion)_drop"
99165
publishLocation: "Container"
100-
- ${{ if and(eq(parameters.Pack, true), eq(parameters.Sign, true)) }}:
101-
- output: nuget
102-
displayName: 'Publish NuGet to feed'
103-
packageParentPath: '$(Build.ArtifactStagingDirectory)'
104-
packagesToPush: $(Build.ArtifactStagingDirectory)/**/Microsoft.Graph.*.nupkg
105-
publishVstsFeed: $(PROJECT_NAME)/$(FEED_NAME)
106-
allowPackageConflicts: true
166+
strategy:
167+
matrix: $[ dependencies.SetUpModuleMatrix.outputs['CreateMatrixJson.matrixJson'] ]
107168
steps:
108169
- template: .azure-pipelines/common-templates/checkout.yml@self
109170
parameters:
@@ -115,27 +176,30 @@ extends:
115176
Test: ${{ parameters.Test }}
116177
Pack: ${{ parameters.Pack }}
117178
Sign: ${{ parameters.Sign }}
118-
- template: .azure-pipelines/generation-templates/workload-modules.yml@self
179+
- template: .azure-pipelines/generation-templates/individualized-workload-modules.yml@self
119180
parameters:
120181
Test: ${{ parameters.Test }}
121182
Pack: ${{ parameters.Pack }}
122183
Sign: ${{ parameters.Sign }}
123-
- template: .azure-pipelines/generation-templates/meta-module.yml@self
124-
parameters:
125-
Test: ${{ parameters.Test }}
126-
Pack: ${{ parameters.Pack }}
127-
Sign: ${{ parameters.Sign }}
128-
- ${{ if and(eq(parameters.Pack, true), eq(parameters.Sign, true)) }}:
129-
- template: .azure-pipelines/common-templates/esrp/codesign-nuget.yml@self
130-
parameters:
131-
FolderPath: "$(Build.ArtifactStagingDirectory)"
132-
Pattern: "Microsoft.Graph*.nupkg"
133-
- template: .azure-pipelines/generation-templates/generate-command-metadata.yml@self
184+
ModuleName: $(moduleName)
185+
ModuleVersion: $(moduleVersion)
134186
- template: .azure-pipelines/common-templates/security-post-checks.yml@self
135-
- ${{ if eq(parameters.CreatePullRequest, true) }}:
136-
- template: .azure-pipelines/common-templates/create-pr.yml@self
137-
parameters:
138-
BaseBranch: $(BaseBranch)
139-
TargetBranch: $(WeeklyBranch)
140-
Title: "[v2] Weekly OpenApiDocs Refresh"
141-
Body: "This pull request was automatically created by Azure Pipelines. **Important** Check for unexpected deletions or changes in this PR."
187+
188+
- job: CreatePullRequest
189+
displayName: Create Pull Request for all changes
190+
dependsOn:
191+
- MsGraphPsSdkWeeklyGeneration
192+
- RefreshOpenAPIDocuments
193+
condition: and(succeeded(), eq(dependencies.MsGraphPsSdkWeeklyGeneration.result, 'Succeeded'), eq(${{ parameters.CreatePullRequest }}, true))
194+
variables:
195+
WeeklyBranch: $[ dependencies.RefreshOpenAPIDocuments.outputs['ComputeBranch.WeeklyBranch'] ]
196+
steps:
197+
- template: .azure-pipelines/common-templates/checkout.yml@self
198+
parameters:
199+
TargetBranch: $(WeeklyBranch)
200+
- template: .azure-pipelines/common-templates/create-pr.yml@self
201+
parameters:
202+
BaseBranch: $(BaseBranch)
203+
TargetBranch: $(WeeklyBranch)
204+
Title: "[v2] Weekly OpenApiDocs Refresh"
205+
Body: "This pull request was automatically created by Azure Pipelines. **Important** Check for unexpected deletions or changes in this PR."

0 commit comments

Comments
 (0)