Fix ci timeout (#820)

tgauth · web-flow · commit 803cf83d8d9f · 2025-12-01T12:10:07.000-05:00
* split ci builds into separate jobs

* use template for jobs

* Fix template path in CI configuration
diff --git a/.azdo/ci.yml b/.azdo/ci.yml
@@ -14,109 +14,32 @@ stages:
 - stage: Build
   displayName: Build Win32-OpenSSH
   jobs:
-  - job: BuildPkg
-    displayName: Build Package
+  - job: BuildPkg_x64
+    displayName: Build Package (x64)
     pool:
       name: PS-PowerShell-x64
       demands:
       - ImageOverride -equals PSMMS2022-OpenSSH-Secure
-
     steps:
-    - pwsh: |
-        # Compare LibreSSL versions in vcpkg.json and add-resource-file.patch
-        $vcpkgObj = Get-Content "$(Build.SourcesDirectory)/contrib/win32/openssh/vcpkg.json" | ConvertFrom-Json
-        $libresslVersionJson = $vcpkgObj | Select-Object -ExpandProperty overrides | Where-Object { $_.name -eq 'libressl' } | Select-Object -ExpandProperty version
-
-        # resource file version needs to be trimmed (e.g. 4.0.0.0 to 4.0.0)
-        $patchContent = Get-Content "$(Build.SourcesDirectory)/contrib/win32/openssh/vcpkg_overlay_ports/libressl/add-version-file.patch"
-        $libresslVersionPatch = ($patchContent -join "`n" | Select-String -Pattern '"FileVersion",\s*"(\d+\.\d+\.\d+\.\d+)"' -AllMatches).Matches | ForEach-Object { $_.Groups[1].Value }
-        $libresslVersionPatchParts = $libresslVersionPatch -split '\.'
-        $libresslVersionPatchShort = ($libresslVersionPatchParts[0..2] -join '.')
-
-        if ($libresslVersionJson -ne $libresslVersionPatchShort) {
-          Write-Error "LibreSSL version mismatch: vcpkg.json has $libresslVersionJson, patch file has $libresslVersionPatch"
-          exit 1
-        } else {
-          Write-Verbose -Verbose "LibreSSL versions match: $libresslVersionJson"
-        }
-      displayName: 'Verify version info'
-
-    - pwsh: |-
-        git clone https://github.com/microsoft/vcpkg
-        cd vcpkg
-        & ./bootstrap-vcpkg.bat
-        & ./vcpkg.exe integrate install
-      displayName: Install vcpkg
-
-    - pwsh: |
-        Import-Module -Name "$(Build.SourcesDirectory)/contrib/win32/openssh/AzDOBuildTools" -Force
-        Invoke-AzDOBuild
-      displayName: Build Win32-OpenSSH
-
-    - pwsh: |
-        $BuildOutPath = "$(Build.SourcesDirectory)/bin"
-        $BuildOutx86Path = Join-Path -Path $BuildOutPath -ChildPath 'Win32/Release'
-        Get-ChildItem -Path $BuildOutx86Path
-        $BuildOutx64Path = Join-Path -Path $BuildOutPath -ChildPath 'x64/Release'
-        Get-ChildItem -Path $BuildOutx64Path
-      displayName: Capture build results
-
-    - pwsh: |
-        Import-Module -Name "$(Build.SourcesDirectory)/contrib/win32/openssh/AzDOBuildTools" -Force
-        #
-        # Copy build artifacts
-        $BuildDestPath = "$(Build.SourcesDirectory)/Win32-OpenSSH"
-        if (Test-Path -Path $BuildDestPath) {
-          Remove-Item -Path $BuildDestPath -Recurse -Force -ErrorAction SilentlyContinue
-        }
-        $null = New-Item -ItemType Directory -Path $BuildDestPath -Force
-        $BuildDestx86Path = Join-Path -Path $BuildDestPath -ChildPath 'x86/Release'
-        Copy-BuildResults -BuildResultsPath $BuildDestx86Path -NativeHostArch x86 -Configuration Release
-        $BuildDestX64Path = Join-Path -Path $BuildDestPath -ChildPath 'x64/Release'
-        Copy-BuildResults -BuildResultsPath $BuildDestx64Path -NativeHostArch x64 -Configuration Release
-        #
-        # Upload build artifacts
-        Write-Verbose -Verbose -Message "Uploading build artifacts"
-        $artifactName = 'Win32-OpenSSH'
-        Write-Host "##vso[artifact.upload containerfolder=$artifactName;artifactname=$artifactName;]$BuildDestPath"
-        #
-        # Copy unit tests
-        $BuildOutPath = "$(Build.SourcesDirectory)/bin"
-        $UnitTestDestPath = "$(Build.SourcesDirectory)/UnitTests"
-        Copy-UnitTests -UnitTestsSrcDir $BuildOutPath -UnitTestsDestDir $UnitTestDestPath -NativeHostArch x86 -Configuration Release
-        Copy-UnitTests -UnitTestsSrcDir $BuildOutPath -UnitTestsDestDir $UnitTestDestPath -NativeHostArch x64 -Configuration Release
-        #
-        # Upload unit test artifacts
-        Write-Verbose -Verbose -Message "Uploading unit test artifacts"
-        $artifactName = 'UnitTests'
-        Write-Host "##vso[artifact.upload containerfolder=$artifactName;artifactname=$artifactName;]$UnitTestDestPath"
-        #
-        # Upload bash tests config.h file
-        Write-Verbose -Verbose -Message "Uploading config.h file for bash tests"
-        $artifactName = 'ConfigFile'
-        $configFilePath = "$(Build.SourcesDirectory)/config.h"
-        Write-Host "##vso[artifact.upload containerfolder=$artifactName;artifactname=$artifactName;]$configFilePath"
-      displayName: Upload Win32-OpenSSH build artifacts
-
-    - pwsh: |
-        $logFileNames = @("OpenSSHReleasex64.log", "OpenSSHReleasex86.log")
-        $uniqueLines = [System.Collections.Generic.HashSet[string]]::new()
-        forEach ($logFile in $logFileNames) {
-          $logFilePath = "$(Build.SourcesDirectory)/contrib/win32/openssh/$logFile"
-          $logLines = Get-Content $logFilePath
-          $buildSucceededIndex = $logLines.IndexOf("Build succeeded.")
-          $linesToCheck = $logLines[($buildSucceededIndex + 1)..$logLines.Length]
-          forEach ($line in $linesToCheck) {
-            if ($line -match "warning C") {
-              $uniqueLines.Add($line) | Out-Null
-            }
-          }
-        }
-        forEach ($uniqueLine in $uniqueLines) {
-          Write-Output $uniqueLine
-          Write-Host "##vso[task.logissue type=warning]$uniqueLine"
-        }
-      displayName: Surface Warnings from Build Logs
+    - template: ./templates/build-win32-openssh-job.yml
+      parameters:
+        nativeHostArch: x64
+        buildOutputDir: x64
+        artifactSuffix: x64
+        includeConfig: true
+
+  - job: BuildPkg_x86
+    displayName: Build Package (x86)
+    pool:
+      name: PS-PowerShell-x64
+      demands:
+      - ImageOverride -equals PSMMS2022-OpenSSH-Secure
+    steps:
+    - template: ./templates/build-win32-openssh-job.yml
+      parameters:
+        nativeHostArch: x86
+        buildOutputDir: Win32
+        artifactSuffix: x86
 
 - stage: Test
   displayName: Test Win32-OpenSSH
@@ -134,23 +57,23 @@ stages:
       inputs:
         buildType: current
         downloadType: single
-        artifactName: Win32-OpenSSH
+        artifactName: Win32-OpenSSH-x64
         downloadPath: '$(System.ArtifactsDirectory)'
 
     - task: DownloadBuildArtifacts@0
       displayName: 'Download unit test artifacts'
       inputs:
         buildType: current
         downloadType: single
-        artifactName: UnitTests
+        artifactName: UnitTests-x64
         downloadPath: '$(System.ArtifactsDirectory)'
 
     - task: DownloadBuildArtifacts@0
       displayName: 'Download bash test config file artifact'
       inputs:
         buildType: current
         downloadType: single
-        artifactName: ConfigFile
+        artifactName: ConfigFile-x64
         downloadPath: '$(System.ArtifactsDirectory)'
 
     - pwsh: |
@@ -161,16 +84,16 @@ stages:
 
     - pwsh: |
         Import-Module -Name "$(Build.SourcesDirectory)/contrib/win32/openssh/AzDOBuildTools" -Force
-        Install-OpenSSH -SourceDir "$(System.ArtifactsDirectory)/Win32-OpenSSH/x64/Release" -OpenSSHDir "$env:SystemDrive/OpenSSH" -Verbose
+        Install-OpenSSH -SourceDir "$(System.ArtifactsDirectory)/Win32-OpenSSH-x64/x64/Release" -OpenSSHDir "$env:SystemDrive/OpenSSH" -Verbose
       displayName: Install Win32-OpenSSH
 
     - pwsh: |
         Import-Module -Name "$(Build.SourcesDirectory)/contrib/win32/openssh/AzDOBuildTools" -Force
-        Install-UnitTests -SourceDir "$(System.ArtifactsDirectory)/UnitTests/x64/Release" -OpenSSHDir "$env:SystemDrive/OpenSSH" -Verbose
+        Install-UnitTests -SourceDir "$(System.ArtifactsDirectory)/UnitTests-x64/x64/Release" -OpenSSHDir "$env:SystemDrive/OpenSSH" -Verbose
       displayName: Install Unit Tests
 
     - pwsh: |
-        $configFileSrc = "$(System.ArtifactsDirectory)/ConfigFile/config.h"
+        $configFileSrc = "$(System.ArtifactsDirectory)/ConfigFile-x64/config.h"
         $configFileDest = "$(Build.SourcesDirectory)"
         Write-Verbose -Verbose -Message "Copying config file from: ${configFileSrc} to: ${configFileDest}"
         Copy-Item -Path $configFileSrc -Dest $configFileDest -Force
diff --git a/.azdo/templates/build-win32-openssh-job.yml b/.azdo/templates/build-win32-openssh-job.yml
@@ -0,0 +1,118 @@
+parameters:
+- name: nativeHostArch
+  type: string
+- name: buildOutputDir
+  type: string
+- name: artifactSuffix
+  type: string
+- name: includeConfig
+  type: boolean
+  default: false
+
+steps:
+- pwsh: |
+    # Compare LibreSSL versions in vcpkg.json and add-resource-file.patch
+    $vcpkgObj = Get-Content "$(Build.SourcesDirectory)/contrib/win32/openssh/vcpkg.json" | ConvertFrom-Json
+    $libresslVersionJson = $vcpkgObj | Select-Object -ExpandProperty overrides | Where-Object { $_.name -eq 'libressl' } | Select-Object -ExpandProperty version
+
+    # resource file version needs to be trimmed (e.g. 4.0.0.0 to 4.0.0)
+    $patchContent = Get-Content "$(Build.SourcesDirectory)/contrib/win32/openssh/vcpkg_overlay_ports/libressl/add-version-file.patch"
+    $libresslVersionPatch = ($patchContent -join "`n" | Select-String -Pattern '"FileVersion",\s*"(\d+\.\d+\.\d+\.\d+)"' -AllMatches).Matches | ForEach-Object { $_.Groups[1].Value }
+    $libresslVersionPatchParts = $libresslVersionPatch -split '\.'
+    $libresslVersionPatchShort = ($libresslVersionPatchParts[0..2] -join '.')
+
+    if ($libresslVersionJson -ne $libresslVersionPatchShort) {
+      Write-Error "LibreSSL version mismatch: vcpkg.json has $libresslVersionJson, patch file has $libresslVersionPatch"
+      exit 1
+    } else {
+      Write-Verbose -Verbose "LibreSSL versions match: $libresslVersionJson"
+    }
+  displayName: 'Verify version info'
+
+- pwsh: |-
+    git clone https://github.com/microsoft/vcpkg
+    cd vcpkg
+    & ./bootstrap-vcpkg.bat
+    & ./vcpkg.exe integrate install
+  displayName: Install vcpkg
+
+- pwsh: |
+    $nativeArch = '${{ parameters.nativeHostArch }}'
+    Import-Module -Name "$(Build.SourcesDirectory)/contrib/win32/openssh/AzDOBuildTools" -Force
+    Invoke-AzDOBuild -NativeHostArch $nativeArch
+  displayName: Build Win32-OpenSSH (${{ parameters.nativeHostArch }})
+
+- pwsh: |
+    $buildOutPath = "$(Build.SourcesDirectory)/bin"
+    $buildOutputDir = '${{ parameters.buildOutputDir }}'
+    $fullPath = Join-Path -Path $buildOutPath -ChildPath $buildOutputDir
+    $releasePath = Join-Path -Path $fullPath -ChildPath 'Release'
+    Get-ChildItem -Path $releasePath
+  displayName: Capture build results (${{ parameters.artifactSuffix }})
+
+- pwsh: |
+    $nativeArch = '${{ parameters.nativeHostArch }}'
+    $artifactSuffix = '${{ parameters.artifactSuffix }}'
+    $includeConfig = [System.Convert]::ToBoolean('${{ parameters.includeConfig }}')
+
+    Import-Module -Name "$(Build.SourcesDirectory)/contrib/win32/openssh/AzDOBuildTools" -Force
+    #
+    # Copy build artifacts
+    $buildDestPath = "$(Build.SourcesDirectory)/Win32-OpenSSH-$artifactSuffix"
+    if (Test-Path -Path $buildDestPath) {
+      Remove-Item -Path $buildDestPath -Recurse -Force -ErrorAction SilentlyContinue
+    }
+    $null = New-Item -ItemType Directory -Path $buildDestPath -Force
+    $buildDestArchPath = Join-Path -Path $buildDestPath -ChildPath "$nativeArch/Release"
+    Copy-BuildResults -BuildResultsPath $buildDestArchPath -NativeHostArch $nativeArch -Configuration Release
+    #
+    # Upload build artifacts
+    Write-Verbose -Verbose -Message "Uploading build artifacts"
+    $artifactName = "Win32-OpenSSH-$artifactSuffix"
+    Write-Host "##vso[artifact.upload containerfolder=$artifactName;artifactname=$artifactName;]$buildDestPath"
+    #
+    # Copy unit tests
+    $buildOutPath = "$(Build.SourcesDirectory)/bin"
+    $unitTestDestPath = "$(Build.SourcesDirectory)/UnitTests-$artifactSuffix"
+    if (Test-Path -Path $unitTestDestPath) {
+      Remove-Item -Path $unitTestDestPath -Recurse -Force -ErrorAction SilentlyContinue
+    }
+    $null = New-Item -ItemType Directory -Path $unitTestDestPath -Force
+    Copy-UnitTests -UnitTestsSrcDir $buildOutPath -UnitTestsDestDir $unitTestDestPath -NativeHostArch $nativeArch -Configuration Release
+    #
+    # Upload unit test artifacts
+    Write-Verbose -Verbose -Message "Uploading unit test artifacts"
+    $unitTestArtifactName = "UnitTests-$artifactSuffix"
+    Write-Host "##vso[artifact.upload containerfolder=$unitTestArtifactName;artifactname=$unitTestArtifactName;]$unitTestDestPath"
+    #
+    # Optionally upload bash tests config.h file
+    if ($includeConfig) {
+      Write-Verbose -Verbose -Message "Uploading config.h file for bash tests"
+      $configArtifactName = "ConfigFile-$artifactSuffix"
+      $configFilePath = "$(Build.SourcesDirectory)/config.h"
+      Write-Host "##vso[artifact.upload containerfolder=$configArtifactName;artifactname=$configArtifactName;]$configFilePath"
+    }
+  displayName: Upload Win32-OpenSSH build artifacts (${{ parameters.artifactSuffix }})
+
+- pwsh: |
+    $artifactSuffix = '${{ parameters.artifactSuffix }}'
+    $logFileName = "OpenSSHRelease$artifactSuffix.log"
+    $uniqueLines = [System.Collections.Generic.HashSet[string]]::new()
+    $logFilePath = "$(Build.SourcesDirectory)/contrib/win32/openssh/$logFileName"
+    if (Test-Path -Path $logFilePath) {
+      $logLines = Get-Content $logFilePath
+      $buildSucceededIndex = $logLines.IndexOf("Build succeeded.")
+      if ($buildSucceededIndex -ge 0) {
+        $linesToCheck = $logLines[($buildSucceededIndex + 1)..$logLines.Length]
+        foreach ($line in $linesToCheck) {
+          if ($line -match "warning C") {
+            $uniqueLines.Add($line) | Out-Null
+          }
+        }
+      }
+    }
+    foreach ($uniqueLine in $uniqueLines) {
+      Write-Output $uniqueLine
+      Write-Host "##vso[task.logissue type=warning]$uniqueLine"
+    }
+  displayName: Surface warnings from build logs (${{ parameters.artifactSuffix }})
diff --git a/contrib/win32/openssh/AzDOBuildTools/AzDOBuildTools.psm1 b/contrib/win32/openssh/AzDOBuildTools/AzDOBuildTools.psm1
@@ -35,9 +35,19 @@ function Write-BuildMessage
 #>
 function Invoke-AzDOBuild
 {
-      Start-OpenSSHBuild -Configuration Release -NativeHostArch x64 -Verbose
-      Start-OpenSSHBuild -Configuration Release -NativeHostArch x86 -Verbose
-      Write-BuildMessage -Message "OpenSSH binaries build success!" -Category Information
+    [CmdletBinding()]
+    param (
+        [Parameter()]
+        [ValidateSet('x64', 'x86')]
+        [string[]] $NativeHostArch = @('x64', 'x86')
+    )
+
+    foreach ($arch in $NativeHostArch)
+    {
+        Start-OpenSSHBuild -Configuration Release -NativeHostArch $arch -Verbose
+    }
+
+    Write-BuildMessage -Message "OpenSSH binaries build success!" -Category Information
 }
 
 <#
