From cd3f8c52b1c18dccbd142d47dec343783ccb8b05 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Mon, 22 Sep 2025 19:09:09 +0000 Subject: [PATCH 01/87] Remove yank job; ESRP doesn't have direct programmatic support for that --- .../stages/archetype-rust-release.yml | 38 ------------------- 1 file changed, 38 deletions(-) diff --git a/eng/pipelines/templates/stages/archetype-rust-release.yml b/eng/pipelines/templates/stages/archetype-rust-release.yml index 5ac10abb01..82531d4f2f 100644 --- a/eng/pipelines/templates/stages/archetype-rust-release.yml +++ b/eng/pipelines/templates/stages/archetype-rust-release.yml @@ -158,41 +158,3 @@ stages: CloseAfterOpenForTesting: '${{parameters.TestPipeline}}' ${{ if startsWith(variables['Build.SourceBranch'], 'refs/pull/') }}: BaseBranchName: main - - - ${{ if eq(parameters.TestPipeline, true) }}: - - job: ManualApproval - displayName: "Manual approval" - dependsOn: PublishPackage - condition: ne(variables['Skip.PublishPackage'], 'true') - pool: server - timeoutInMinutes: 120 # 2 hours - steps: - - task: ManualValidation@1 - timeoutInMinutes: 60 # 1 hour - inputs: - notifyUsers: '' # Required, but empty string allowed - allowApproversToApproveTheirOwnRuns: true - instructions: "Approve yank of ${{ artifact.name }}" - onTimeout: 'resume' - - - job: YankCrates - displayName: "Yank Crates" - dependsOn: ManualApproval - condition: and(succeeded(), ne(variables['Skip.PublishPackage'], 'true')) - steps: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - - - download: current - displayName: Download ${{parameters.PipelineArtifactName}} artifact - artifact: ${{parameters.PipelineArtifactName}} - - - task: PowerShell@2 - displayName: Yank Crates - env: - CARGO_REGISTRY_TOKEN: $(azure-sdk-cratesio-token) - inputs: - targetType: filePath - filePath: $(Build.SourcesDirectory)/eng/scripts/Yank-Crates.ps1 - arguments: - -CrateNames '${{artifact.name}}' - -PackageInfoDirectory '$(Pipeline.Workspace)/${{parameters.PipelineArtifactName}}/PackageInfo' From b7d1bc4888c44e5098bab802209c908e098860c3 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Tue, 23 Sep 2025 22:10:37 +0000 Subject: [PATCH 02/87] Rough changes to Pack-Crates.ps1 to produce .crate files --- eng/scripts/Pack-Crates.ps1 | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/eng/scripts/Pack-Crates.ps1 b/eng/scripts/Pack-Crates.ps1 index b266667345..0ef7e0be80 100755 --- a/eng/scripts/Pack-Crates.ps1 +++ b/eng/scripts/Pack-Crates.ps1 @@ -183,7 +183,6 @@ try { Invoke-LoggedCommand -Command $command -GroupOutput - # copy the package to the local registry Add-CrateToLocalRegistry ` -LocalRegistryPath $localRegistryPath ` @@ -191,17 +190,11 @@ try { if ($OutputPath -and $package.OutputPackage) { $sourcePath = "$RepoRoot/target/package/$packageName-$packageVersion" - $targetPath = "$OutputPath/$packageName" - $targetContentsPath = "$targetPath/contents" - $targetApiReviewFile = "$targetPath/$packageName.rust.json" - - if (Test-Path -Path $targetContentsPath) { - Remove-Item -Path $targetContentsPath -Recurse -Force - } + $targetApiReviewFile = "$OutputPath/$packageName.rust.json" - Write-Host "Copying package '$packageName' to '$targetContentsPath'" - New-Item -ItemType Directory -Path $targetContentsPath -Force | Out-Null - Copy-Item -Path $sourcePath/* -Destination $targetContentsPath -Recurse -Exclude "Cargo.toml.orig" + Write-Host "Copying package '$packageName' to '$OutputPath'" + New-Item -ItemType Directory -Path $OutputPath -Force | Out-Null + Copy-Item -Path "$sourcePath.crate" -Destination $OutputPath Write-Host "Creating API review file" $apiReviewFile = Create-ApiViewFile $package From bb1962919a0b8364c6c4f8afa4ccb5001b9aac65 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Wed, 24 Sep 2025 04:00:08 +0000 Subject: [PATCH 03/87] First cut at ESRP publishing --- .../stages/archetype-rust-release.yml | 99 ++++++++++++++----- 1 file changed, 72 insertions(+), 27 deletions(-) diff --git a/eng/pipelines/templates/stages/archetype-rust-release.yml b/eng/pipelines/templates/stages/archetype-rust-release.yml index 82531d4f2f..469240ff9e 100644 --- a/eng/pipelines/templates/stages/archetype-rust-release.yml +++ b/eng/pipelines/templates/stages/archetype-rust-release.yml @@ -84,33 +84,78 @@ stages: runOnce: deploy: steps: - - template: /eng/pipelines/templates/steps/use-rust.yml@self - parameters: - Toolchain: stable - - - pwsh: | - $additionalOwners = @('heaths', 'hallipr') - $token = $env:CARGO_REGISTRY_TOKEN - $crateName = '${{artifact.name}}' - - $manifestPath = "$(Pipeline.Workspace)/drop/$crateName/contents/Cargo.toml" - Write-Host "> cargo publish --manifest-path `"$manifestPath`"" - cargo publish --manifest-path $manifestPath - if (!$?) { - Write-Error "Failed to publish package: '$crateName'" - exit 1 - } - - $existingOwners = (cargo owner --list $crateName) -replace " \(.*", "" - $missingOwners = $additionalOwners | Where-Object { $existingOwners -notcontains $_ } - - foreach ($owner in $missingOwners) { - Write-Host "> cargo owner --add $owner $crateName" - cargo owner --add $owner $crateName - } - displayName: Publish Crate - env: - CARGO_REGISTRY_TOKEN: $(azure-sdk-cratesio-token) + - template: /eng/pipelines/templates/steps/use-rust.yml@self + parameters: + Toolchain: stable + + - pwsh: | + $artifactName = '${{artifact.name}}' + Write-Host "Artifact name: $artifactName" + + $packageMetadataPath = "$(Pipeline.Workspace)/drop/PackageInfo/$artifactName.json" + if (!(Test-Path $packageMetadataPath)) { + Write-Error "Package metadata file not found: $packageMetadataPath" + exit 1 + } + + $packageMetadata = Get-Content -Raw $packageMetadataPath | ConvertFrom-Json -AsHashTable + $packageVersion = $packageMetadata.version + Write-Host "Package version: $packageVersion" + + New-Item -ItemType Directory -Path '$(Pipeline.Workspace)/release' -Force | Out-Null + Compress-Archive ` + -Path "$(Pipeline.Workspace)/drop/$artifactName-$packageVersion.crate" ` + -DestinationPath "$(Pipeline.Workspace)/release/release.zip" + + Get-ChildItem -Path "$(Pipeline.Workspace)/release" | ForEach-Object { Write-Host "Release artifact: $($_.FullName)" } + + displayName: Zip Crate for ESRP + + - task: EsrpRelease@10 + displayName: 'ESRP Release' + inputs: + usemanagedidentity: false + connectedservicename: 'Azure SDK PME Managed Identity' + KeyVaultName: $(kv-azuresdk-codesign) + SignCertName: $(azure-sdk-esrp-release-certificate) + # keyvaultname: $(2460c5ef-KeyVaultName) + # authcertname: $(2460c5ef-Cert-Auth) + # signcertname: $(2460c5ef-Cert-Signing) + clientid: '5f81938c-2544-4f1f-9251-dd9de5b8a81b' + intent: 'packagedistribution' + contenttype: 'Rust' + contentsource: 'Folder' + folderlocation: '$(Pipeline.Workspace)/release/release.zip' + waitforreleasecompletion: true + owners: ${{ coalesce(variables['Build.RequestedForEmail'], 'azuresdk@microsoft.com') }} + approvers: ${{ coalesce(variables['Build.RequestedForEmail'], 'azuresdk@microsoft.com') }} + serviceendpointurl: 'https://api.esrp.microsoft.com/' + mainpublisher: 'ESRPRELPACMAN' + domaintenantid: '72f988bf-86f1-41af-91ab-2d7cd011db47' + + # - pwsh: | + # $additionalOwners = @('heaths', 'hallipr') + # $token = $env:CARGO_REGISTRY_TOKEN + # $crateName = '${{artifact.name}}' + + # $manifestPath = "$(Pipeline.Workspace)/drop/$crateName/contents/Cargo.toml" + # Write-Host "> cargo publish --manifest-path `"$manifestPath`"" + # cargo publish --manifest-path $manifestPath + # if (!$?) { + # Write-Error "Failed to publish package: '$crateName'" + # exit 1 + # } + + # $existingOwners = (cargo owner --list $crateName) -replace " \(.*", "" + # $missingOwners = $additionalOwners | Where-Object { $existingOwners -notcontains $_ } + + # foreach ($owner in $missingOwners) { + # Write-Host "> cargo owner --add $owner $crateName" + # cargo owner --add $owner $crateName + # } + # displayName: Publish Crate + # env: + # CARGO_REGISTRY_TOKEN: $(azure-sdk-cratesio-token) - job: UpdatePackageVersion displayName: "API Review and Package Version Update" From ce9052c83330d512d89d72ba0b3e27ef3065bcc8 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Wed, 24 Sep 2025 05:17:33 +0000 Subject: [PATCH 04/87] Use *.crate package pattern, adjust Pack-Crates.ps1 to create packages in output folders --- eng/scripts/Language-Settings.ps1 | 3 ++- eng/scripts/Pack-Crates.ps1 | 9 +++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index a973cc38f3..965148d30a 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -1,7 +1,7 @@ $Language = "rust" $LanguageDisplayName = "Rust" $PackageRepository = "crates.io" -$packagePattern = "Cargo.toml" +$packagePattern = "*.crate" #$MetadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/main/_data/releases/latest/rust-packages.csv" $GithubUri = "https://github.com/Azure/azure-sdk-for-rust" $PackageRepositoryUri = "https://crates.io/crates" @@ -139,6 +139,7 @@ function Get-rust-AdditionalValidationPackagesFromPackageSet ($packagesWithChang return $additionalPackages ?? @() } +# $GetPackageInfoFromPackageFileFn = "Get-${Language}-PackageInfoFromPackageFile" function Get-rust-PackageInfoFromPackageFile([IO.FileInfo]$pkg, [string]$workingDirectory) { #$pkg will be a FileInfo object for the Cargo.toml file in a package artifact directory $package = cargo read-manifest --manifest-path $pkg.FullName | ConvertFrom-Json diff --git a/eng/scripts/Pack-Crates.ps1 b/eng/scripts/Pack-Crates.ps1 index 0ef7e0be80..38f4013ee2 100755 --- a/eng/scripts/Pack-Crates.ps1 +++ b/eng/scripts/Pack-Crates.ps1 @@ -190,11 +190,12 @@ try { if ($OutputPath -and $package.OutputPackage) { $sourcePath = "$RepoRoot/target/package/$packageName-$packageVersion" - $targetApiReviewFile = "$OutputPath/$packageName.rust.json" + $targetPath = "$OutputPath/$packageName" + $targetApiReviewFile = "$targetPath/$packageName.rust.json" - Write-Host "Copying package '$packageName' to '$OutputPath'" - New-Item -ItemType Directory -Path $OutputPath -Force | Out-Null - Copy-Item -Path "$sourcePath.crate" -Destination $OutputPath + Write-Host "Copying package '$packageName' to '$targetPath'" + New-Item -ItemType Directory -Path $targetPath -Force | Out-Null + Copy-Item -Path "$sourcePath.crate" -Destination $targetPath Write-Host "Creating API review file" $apiReviewFile = Create-ApiViewFile $package From dea5220019924ac490df9e8a7b3573516f26bfa6 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Wed, 24 Sep 2025 05:34:27 +0000 Subject: [PATCH 05/87] Extract package information from .crate file --- eng/scripts/Language-Settings.ps1 | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index 965148d30a..38c0cacc47 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -142,7 +142,29 @@ function Get-rust-AdditionalValidationPackagesFromPackageSet ($packagesWithChang # $GetPackageInfoFromPackageFileFn = "Get-${Language}-PackageInfoFromPackageFile" function Get-rust-PackageInfoFromPackageFile([IO.FileInfo]$pkg, [string]$workingDirectory) { #$pkg will be a FileInfo object for the Cargo.toml file in a package artifact directory - $package = cargo read-manifest --manifest-path $pkg.FullName | ConvertFrom-Json + + # Create a temporary folder for extraction + $extractionPath = Join-Path [System.IO.Path]::GetTempPath(), ([System.IO.Path]::GetRandomFileName()) + New-Item -ItemType Directory -Path $extractionPath | Out-Null + + $originalLocation = Get-Location + try { + Set-Location $extractionPath + tar -xvf $pkg.FullName + } + finally { + Set-Location $originalLocation + } + + $cargoTomlPath = Join-Path $extractionPath, $pkg.BaseName, 'Cargo.toml' + + Write-Host "Reading package info from $cargoTomlPath" + if (!(Test-Path $cargoTomlPath)) { + LogError "The Cargo.toml file was not found in the package artifact at $cargoTomlPath" + throw "error" + } + + $package = cargo read-manifest --manifest-path $cargoTomlPath | ConvertFrom-Json $packageName = $package.name $packageVersion = $package.version From 86fdb07689cbc4c393449849c9747cf98b2fc0df Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Wed, 24 Sep 2025 05:42:32 +0000 Subject: [PATCH 06/87] Syntax --- eng/scripts/Language-Settings.ps1 | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index 38c0cacc47..3e2e132031 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -144,18 +144,11 @@ function Get-rust-PackageInfoFromPackageFile([IO.FileInfo]$pkg, [string]$working #$pkg will be a FileInfo object for the Cargo.toml file in a package artifact directory # Create a temporary folder for extraction - $extractionPath = Join-Path [System.IO.Path]::GetTempPath(), ([System.IO.Path]::GetRandomFileName()) + $extractionPath = Join-Path ([System.IO.Path]::GetTempPath()), ([System.IO.Path]::GetRandomFileName()) New-Item -ItemType Directory -Path $extractionPath | Out-Null - $originalLocation = Get-Location - try { - Set-Location $extractionPath - tar -xvf $pkg.FullName - } - finally { - Set-Location $originalLocation - } - + # Extract the .crate file (which is a tarball) to the temporary folder + tar -xvf $pkg.FullName -C $extractionPath $cargoTomlPath = Join-Path $extractionPath, $pkg.BaseName, 'Cargo.toml' Write-Host "Reading package info from $cargoTomlPath" From da1123e24bbcdd6d919b003ced5ef632ea74073e Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Wed, 24 Sep 2025 16:38:57 +0000 Subject: [PATCH 07/87] Syntax --- eng/scripts/Language-Settings.ps1 | 2 -- 1 file changed, 2 deletions(-) diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index 3e2e132031..6b86f70ad3 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -141,8 +141,6 @@ function Get-rust-AdditionalValidationPackagesFromPackageSet ($packagesWithChang # $GetPackageInfoFromPackageFileFn = "Get-${Language}-PackageInfoFromPackageFile" function Get-rust-PackageInfoFromPackageFile([IO.FileInfo]$pkg, [string]$workingDirectory) { - #$pkg will be a FileInfo object for the Cargo.toml file in a package artifact directory - # Create a temporary folder for extraction $extractionPath = Join-Path ([System.IO.Path]::GetTempPath()), ([System.IO.Path]::GetRandomFileName()) New-Item -ItemType Directory -Path $extractionPath | Out-Null From 6b8b7ef82f60c08ba4cd69b8d5951fdc5253ec71 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Wed, 24 Sep 2025 16:39:07 +0000 Subject: [PATCH 08/87] Syntax --- eng/scripts/Language-Settings.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index 6b86f70ad3..17b018212a 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -142,7 +142,7 @@ function Get-rust-AdditionalValidationPackagesFromPackageSet ($packagesWithChang # $GetPackageInfoFromPackageFileFn = "Get-${Language}-PackageInfoFromPackageFile" function Get-rust-PackageInfoFromPackageFile([IO.FileInfo]$pkg, [string]$workingDirectory) { # Create a temporary folder for extraction - $extractionPath = Join-Path ([System.IO.Path]::GetTempPath()), ([System.IO.Path]::GetRandomFileName()) + $extractionPath = Join-Path ([System.IO.Path]::GetTempPath()) ([System.IO.Path]::GetRandomFileName()) New-Item -ItemType Directory -Path $extractionPath | Out-Null # Extract the .crate file (which is a tarball) to the temporary folder From 0eeed2febc0cd9e0a3eeb16eeface4ed61672dcb Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Wed, 24 Sep 2025 16:49:08 +0000 Subject: [PATCH 09/87] Syntax --- eng/scripts/Language-Settings.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index 17b018212a..6b1d5586be 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -147,7 +147,7 @@ function Get-rust-PackageInfoFromPackageFile([IO.FileInfo]$pkg, [string]$working # Extract the .crate file (which is a tarball) to the temporary folder tar -xvf $pkg.FullName -C $extractionPath - $cargoTomlPath = Join-Path $extractionPath, $pkg.BaseName, 'Cargo.toml' + $cargoTomlPath = Join-Path $extractionPath $pkg.BaseName 'Cargo.toml' Write-Host "Reading package info from $cargoTomlPath" if (!(Test-Path $cargoTomlPath)) { From 94800742d5ec7c78f888e1a714b83342bacdfa0f Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Wed, 24 Sep 2025 17:16:15 +0000 Subject: [PATCH 10/87] Compress-ReleaseCrate.ps1, wire into archetype-rust-release.yml --- .../stages/archetype-rust-release.yml | 34 ++++++------------- eng/scripts/Compress-ReleaseCrate.ps1 | 28 +++++++++++++++ 2 files changed, 39 insertions(+), 23 deletions(-) create mode 100644 eng/scripts/Compress-ReleaseCrate.ps1 diff --git a/eng/pipelines/templates/stages/archetype-rust-release.yml b/eng/pipelines/templates/stages/archetype-rust-release.yml index 469240ff9e..1702fa7059 100644 --- a/eng/pipelines/templates/stages/archetype-rust-release.yml +++ b/eng/pipelines/templates/stages/archetype-rust-release.yml @@ -88,28 +88,15 @@ stages: parameters: Toolchain: stable - - pwsh: | - $artifactName = '${{artifact.name}}' - Write-Host "Artifact name: $artifactName" - - $packageMetadataPath = "$(Pipeline.Workspace)/drop/PackageInfo/$artifactName.json" - if (!(Test-Path $packageMetadataPath)) { - Write-Error "Package metadata file not found: $packageMetadataPath" - exit 1 - } - - $packageMetadata = Get-Content -Raw $packageMetadataPath | ConvertFrom-Json -AsHashTable - $packageVersion = $packageMetadata.version - Write-Host "Package version: $packageVersion" - - New-Item -ItemType Directory -Path '$(Pipeline.Workspace)/release' -Force | Out-Null - Compress-Archive ` - -Path "$(Pipeline.Workspace)/drop/$artifactName-$packageVersion.crate" ` - -DestinationPath "$(Pipeline.Workspace)/release/release.zip" - - Get-ChildItem -Path "$(Pipeline.Workspace)/release" | ForEach-Object { Write-Host "Release artifact: $($_.FullName)" } - - displayName: Zip Crate for ESRP + - task: PowerShell@2 + displayName: Compress Crate for ESRP + inputs: + targetType: filePath + filePath: $(Build.SourcesDirectory)/eng/scripts/Compress-ReleaseCrate.ps1 + arguments: > + -ArtifactName '${{artifact.name}}' + -ArtifactRootPath '$(Pipeline.Workspace)/drop' + -OutFile '$(Pipeline.Workspace)/esrp-release/release.zip' - task: EsrpRelease@10 displayName: 'ESRP Release' @@ -122,10 +109,11 @@ stages: # authcertname: $(2460c5ef-Cert-Auth) # signcertname: $(2460c5ef-Cert-Signing) clientid: '5f81938c-2544-4f1f-9251-dd9de5b8a81b' + Usemanagedidentity: true intent: 'packagedistribution' contenttype: 'Rust' contentsource: 'Folder' - folderlocation: '$(Pipeline.Workspace)/release/release.zip' + folderlocation: '$(Pipeline.Workspace)/esrp-release' waitforreleasecompletion: true owners: ${{ coalesce(variables['Build.RequestedForEmail'], 'azuresdk@microsoft.com') }} approvers: ${{ coalesce(variables['Build.RequestedForEmail'], 'azuresdk@microsoft.com') }} diff --git a/eng/scripts/Compress-ReleaseCrate.ps1 b/eng/scripts/Compress-ReleaseCrate.ps1 new file mode 100644 index 0000000000..d718d14556 --- /dev/null +++ b/eng/scripts/Compress-ReleaseCrate.ps1 @@ -0,0 +1,28 @@ +#!/usr/bin/env pwsh + +#Requires -Version 7.0 +[CmdletBinding(DefaultParameterSetName = "none")] +param( + [string]$ArtifactName, + [string]$ArtifactRootPath, + [string]$OutFile +) + +Write-Host "Artifact name: $ArtifactName" + +$packageMetadataPath = "$ArtifactRootPath/PackageInfo/$ArtifactName.json" +if (!(Test-Path $packageMetadataPath)) { + Write-Error "Package metadata file not found: $packageMetadataPath" + exit 1 +} + +$packageMetadata = Get-Content -Raw $packageMetadataPath | ConvertFrom-Json +$packageVersion = $packageMetadata.version +Write-Host "Package version: $packageVersion" + +New-Item -ItemType Directory -Path '$(Pipeline.Workspace)/release' -Force | Out-Null +Compress-Archive ` + -Path "$ArtifactRootPath/$ArtifactName/$ArtifactName-$packageVersion.crate" ` + -DestinationPath $OutFile + +Write-Host "Created archive: $OutFile" From 1f23b8d8a769820695433ebeeea00389a3e18546 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Wed, 24 Sep 2025 17:17:19 +0000 Subject: [PATCH 11/87] Use managed identity --- eng/pipelines/templates/stages/archetype-rust-release.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/eng/pipelines/templates/stages/archetype-rust-release.yml b/eng/pipelines/templates/stages/archetype-rust-release.yml index 1702fa7059..8660bb5956 100644 --- a/eng/pipelines/templates/stages/archetype-rust-release.yml +++ b/eng/pipelines/templates/stages/archetype-rust-release.yml @@ -101,7 +101,6 @@ stages: - task: EsrpRelease@10 displayName: 'ESRP Release' inputs: - usemanagedidentity: false connectedservicename: 'Azure SDK PME Managed Identity' KeyVaultName: $(kv-azuresdk-codesign) SignCertName: $(azure-sdk-esrp-release-certificate) From 78da13059c6816c5a21237828dadca7d9e7998a6 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Wed, 24 Sep 2025 17:18:39 +0000 Subject: [PATCH 12/87] Redundant line --- eng/scripts/Compress-ReleaseCrate.ps1 | 1 - 1 file changed, 1 deletion(-) diff --git a/eng/scripts/Compress-ReleaseCrate.ps1 b/eng/scripts/Compress-ReleaseCrate.ps1 index d718d14556..a10b55f251 100644 --- a/eng/scripts/Compress-ReleaseCrate.ps1 +++ b/eng/scripts/Compress-ReleaseCrate.ps1 @@ -20,7 +20,6 @@ $packageMetadata = Get-Content -Raw $packageMetadataPath | ConvertFrom-Json $packageVersion = $packageMetadata.version Write-Host "Package version: $packageVersion" -New-Item -ItemType Directory -Path '$(Pipeline.Workspace)/release' -Force | Out-Null Compress-Archive ` -Path "$ArtifactRootPath/$ArtifactName/$ArtifactName-$packageVersion.crate" ` -DestinationPath $OutFile From 4ff0c808932ba99a6dbc2d3741256c4c922f792d Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Wed, 24 Sep 2025 17:33:56 +0000 Subject: [PATCH 13/87] Deployment scripts must be inline (no checkout) --- .../stages/archetype-rust-release.yml | 2 +- eng/scripts/Compress-ReleaseCrate.ps1 | 27 ------------------- 2 files changed, 1 insertion(+), 28 deletions(-) delete mode 100644 eng/scripts/Compress-ReleaseCrate.ps1 diff --git a/eng/pipelines/templates/stages/archetype-rust-release.yml b/eng/pipelines/templates/stages/archetype-rust-release.yml index 8660bb5956..423c72e025 100644 --- a/eng/pipelines/templates/stages/archetype-rust-release.yml +++ b/eng/pipelines/templates/stages/archetype-rust-release.yml @@ -92,7 +92,7 @@ stages: displayName: Compress Crate for ESRP inputs: targetType: filePath - filePath: $(Build.SourcesDirectory)/eng/scripts/Compress-ReleaseCrate.ps1 + filePath: eng/scripts/Compress-ReleaseCrate.ps1 arguments: > -ArtifactName '${{artifact.name}}' -ArtifactRootPath '$(Pipeline.Workspace)/drop' diff --git a/eng/scripts/Compress-ReleaseCrate.ps1 b/eng/scripts/Compress-ReleaseCrate.ps1 deleted file mode 100644 index a10b55f251..0000000000 --- a/eng/scripts/Compress-ReleaseCrate.ps1 +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env pwsh - -#Requires -Version 7.0 -[CmdletBinding(DefaultParameterSetName = "none")] -param( - [string]$ArtifactName, - [string]$ArtifactRootPath, - [string]$OutFile -) - -Write-Host "Artifact name: $ArtifactName" - -$packageMetadataPath = "$ArtifactRootPath/PackageInfo/$ArtifactName.json" -if (!(Test-Path $packageMetadataPath)) { - Write-Error "Package metadata file not found: $packageMetadataPath" - exit 1 -} - -$packageMetadata = Get-Content -Raw $packageMetadataPath | ConvertFrom-Json -$packageVersion = $packageMetadata.version -Write-Host "Package version: $packageVersion" - -Compress-Archive ` - -Path "$ArtifactRootPath/$ArtifactName/$ArtifactName-$packageVersion.crate" ` - -DestinationPath $OutFile - -Write-Host "Created archive: $OutFile" From 2a62b915bb8e021d61d48e5f382e414960464a1f Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Wed, 24 Sep 2025 17:35:49 +0000 Subject: [PATCH 14/87] Disable repo tagging to iterate on release, inline compression --- .../stages/archetype-rust-release.yml | 69 +++++++++++-------- 1 file changed, 42 insertions(+), 27 deletions(-) diff --git a/eng/pipelines/templates/stages/archetype-rust-release.yml b/eng/pipelines/templates/stages/archetype-rust-release.yml index 423c72e025..590b4de311 100644 --- a/eng/pipelines/templates/stages/archetype-rust-release.yml +++ b/eng/pipelines/templates/stages/archetype-rust-release.yml @@ -37,29 +37,30 @@ stages: os: linux jobs: - - job: TagRepository - displayName: "Create release tag" - condition: and(succeeded(), ne(variables['Skip.TagRepository'], 'true')) + # TODO: Ensure that CHANGELOG is properly processed + # - job: TagRepository + # displayName: "Create release tag" + # condition: and(succeeded(), ne(variables['Skip.TagRepository'], 'true')) - steps: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + # steps: + # - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - - download: current - displayName: Download ${{parameters.PipelineArtifactName}} artifact - artifact: ${{parameters.PipelineArtifactName}} + # - download: current + # displayName: Download ${{parameters.PipelineArtifactName}} artifact + # artifact: ${{parameters.PipelineArtifactName}} - - template: /eng/common/pipelines/templates/steps/retain-run.yml + # - template: /eng/common/pipelines/templates/steps/retain-run.yml - - script: | - echo "##vso[build.addbuildtag]${{artifact.name}}" - displayName: Add build tag '${{artifact.name}}' + # - script: | + # echo "##vso[build.addbuildtag]${{artifact.name}}" + # displayName: Add build tag '${{artifact.name}}' - - template: /eng/common/pipelines/templates/steps/create-tags-and-git-release.yml - parameters: - ArtifactLocation: $(Pipeline.Workspace)/${{parameters.PipelineArtifactName}}/${{artifact.name}} - PackageRepository: Crates.io - ReleaseSha: $(Build.SourceVersion) - WorkingDirectory: $(Pipeline.Workspace)/_work + # - template: /eng/common/pipelines/templates/steps/create-tags-and-git-release.yml + # parameters: + # ArtifactLocation: $(Pipeline.Workspace)/${{parameters.PipelineArtifactName}}/${{artifact.name}} + # PackageRepository: Crates.io + # ReleaseSha: $(Build.SourceVersion) + # WorkingDirectory: $(Pipeline.Workspace)/_work - deployment: PublishPackage displayName: "Publish to Crates.io" @@ -75,7 +76,7 @@ stages: # This timeout shouldn't be necessary once we're able to parallelize better. Right now, # this is here to ensure larger areas (30+) libraries don't time out. timeoutInMinutes: 120 - dependsOn: TagRepository + # dependsOn: TagRepository pool: name: azsdk-pool image: ubuntu-24.04 @@ -88,15 +89,29 @@ stages: parameters: Toolchain: stable - - task: PowerShell@2 + - pwsh: | + $ArtifactName = '${{artifact.name}}' + $ArtifactRootPath = '$(Pipeline.Workspace)/drop' + $OutFile = '$(Pipeline.Workspace)/esrp-release/release.zip' + + Write-Host "Artifact name: $ArtifactName" + + $packageMetadataPath = "$ArtifactRootPath/PackageInfo/$ArtifactName.json" + if (!(Test-Path $packageMetadataPath)) { + Write-Error "Package metadata file not found: $packageMetadataPath" + exit 1 + } + + $packageMetadata = Get-Content -Raw $packageMetadataPath | ConvertFrom-Json + $packageVersion = $packageMetadata.version + Write-Host "Package version: $packageVersion" + + Compress-Archive ` + -Path "$ArtifactRootPath/$ArtifactName/$ArtifactName-$packageVersion.crate" ` + -DestinationPath $OutFile + + Write-Host "Created archive: $OutFile" displayName: Compress Crate for ESRP - inputs: - targetType: filePath - filePath: eng/scripts/Compress-ReleaseCrate.ps1 - arguments: > - -ArtifactName '${{artifact.name}}' - -ArtifactRootPath '$(Pipeline.Workspace)/drop' - -OutFile '$(Pipeline.Workspace)/esrp-release/release.zip' - task: EsrpRelease@10 displayName: 'ESRP Release' From 8064fa0f583c163e9603297cebe15238f681d43d Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Wed, 24 Sep 2025 17:57:24 +0000 Subject: [PATCH 15/87] Create output directory --- .../templates/stages/archetype-rust-release.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/eng/pipelines/templates/stages/archetype-rust-release.yml b/eng/pipelines/templates/stages/archetype-rust-release.yml index 590b4de311..9fe125b5e1 100644 --- a/eng/pipelines/templates/stages/archetype-rust-release.yml +++ b/eng/pipelines/templates/stages/archetype-rust-release.yml @@ -92,7 +92,7 @@ stages: - pwsh: | $ArtifactName = '${{artifact.name}}' $ArtifactRootPath = '$(Pipeline.Workspace)/drop' - $OutFile = '$(Pipeline.Workspace)/esrp-release/release.zip' + $OutDir = '$(Pipeline.Workspace)/esrp-release' Write-Host "Artifact name: $ArtifactName" @@ -106,11 +106,14 @@ stages: $packageVersion = $packageMetadata.version Write-Host "Package version: $packageVersion" + New-Item -ItemType Directory -Path $OutDir -Force | Out-Null Compress-Archive ` -Path "$ArtifactRootPath/$ArtifactName/$ArtifactName-$packageVersion.crate" ` - -DestinationPath $OutFile + -DestinationPath "$OutDir/release.zip" + + Write-Host "Contents of $OutDir:" + Get-ChildItem -Path $OutDir | ForEach-Object { Write-Host $_.FullName } - Write-Host "Created archive: $OutFile" displayName: Compress Crate for ESRP - task: EsrpRelease@10 From 0adddb22a7b696fe658845a8e471fd2297d9eb12 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Wed, 24 Sep 2025 18:13:19 +0000 Subject: [PATCH 16/87] : --- eng/pipelines/templates/stages/archetype-rust-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/stages/archetype-rust-release.yml b/eng/pipelines/templates/stages/archetype-rust-release.yml index 9fe125b5e1..049f32e165 100644 --- a/eng/pipelines/templates/stages/archetype-rust-release.yml +++ b/eng/pipelines/templates/stages/archetype-rust-release.yml @@ -111,7 +111,7 @@ stages: -Path "$ArtifactRootPath/$ArtifactName/$ArtifactName-$packageVersion.crate" ` -DestinationPath "$OutDir/release.zip" - Write-Host "Contents of $OutDir:" + Write-Host "Contents of $OutDir" Get-ChildItem -Path $OutDir | ForEach-Object { Write-Host $_.FullName } displayName: Compress Crate for ESRP From 048d022b9f765737e9c3ee1bd1eea442d56f75b5 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Wed, 24 Sep 2025 18:28:46 +0000 Subject: [PATCH 17/87] EsrpRelease@10 --- .../templates/stages/archetype-rust-release.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/eng/pipelines/templates/stages/archetype-rust-release.yml b/eng/pipelines/templates/stages/archetype-rust-release.yml index 049f32e165..a6c610620b 100644 --- a/eng/pipelines/templates/stages/archetype-rust-release.yml +++ b/eng/pipelines/templates/stages/archetype-rust-release.yml @@ -120,12 +120,10 @@ stages: displayName: 'ESRP Release' inputs: connectedservicename: 'Azure SDK PME Managed Identity' - KeyVaultName: $(kv-azuresdk-codesign) - SignCertName: $(azure-sdk-esrp-release-certificate) - # keyvaultname: $(2460c5ef-KeyVaultName) - # authcertname: $(2460c5ef-Cert-Auth) - # signcertname: $(2460c5ef-Cert-Signing) - clientid: '5f81938c-2544-4f1f-9251-dd9de5b8a81b' + KeyVaultName: 'kv-azuresdk-codesign' + SignCertName: 'azure-sdk-esrp-release-certificate' + ClientId: '5f81938c-2544-4f1f-9251-dd9de5b8a81b' + DomainTenantId: '975f013f-7f24-47e8-a7d3-abc4752bf346' Usemanagedidentity: true intent: 'packagedistribution' contenttype: 'Rust' From 194f4a228e14b441629ca4a4943c5f2bb0fe0b5c Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Wed, 24 Sep 2025 18:44:31 +0000 Subject: [PATCH 18/87] Remove redundant keys --- eng/pipelines/templates/stages/archetype-rust-release.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/eng/pipelines/templates/stages/archetype-rust-release.yml b/eng/pipelines/templates/stages/archetype-rust-release.yml index a6c610620b..e360ab8052 100644 --- a/eng/pipelines/templates/stages/archetype-rust-release.yml +++ b/eng/pipelines/templates/stages/archetype-rust-release.yml @@ -120,8 +120,6 @@ stages: displayName: 'ESRP Release' inputs: connectedservicename: 'Azure SDK PME Managed Identity' - KeyVaultName: 'kv-azuresdk-codesign' - SignCertName: 'azure-sdk-esrp-release-certificate' ClientId: '5f81938c-2544-4f1f-9251-dd9de5b8a81b' DomainTenantId: '975f013f-7f24-47e8-a7d3-abc4752bf346' Usemanagedidentity: true From 155142d2070207828453e94efec1756030791fc9 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Wed, 24 Sep 2025 18:45:20 +0000 Subject: [PATCH 19/87] DomainTenantId --- eng/pipelines/templates/stages/archetype-rust-release.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/eng/pipelines/templates/stages/archetype-rust-release.yml b/eng/pipelines/templates/stages/archetype-rust-release.yml index e360ab8052..4949fde511 100644 --- a/eng/pipelines/templates/stages/archetype-rust-release.yml +++ b/eng/pipelines/templates/stages/archetype-rust-release.yml @@ -132,7 +132,6 @@ stages: approvers: ${{ coalesce(variables['Build.RequestedForEmail'], 'azuresdk@microsoft.com') }} serviceendpointurl: 'https://api.esrp.microsoft.com/' mainpublisher: 'ESRPRELPACMAN' - domaintenantid: '72f988bf-86f1-41af-91ab-2d7cd011db47' # - pwsh: | # $additionalOwners = @('heaths', 'hallipr') From d7c351f8ceb2550f93daf4a727cca27eee04c75b Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Wed, 24 Sep 2025 19:03:31 +0000 Subject: [PATCH 20/87] Don't nest in zip? --- eng/pipelines/templates/stages/archetype-rust-release.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/eng/pipelines/templates/stages/archetype-rust-release.yml b/eng/pipelines/templates/stages/archetype-rust-release.yml index 4949fde511..e346488d2c 100644 --- a/eng/pipelines/templates/stages/archetype-rust-release.yml +++ b/eng/pipelines/templates/stages/archetype-rust-release.yml @@ -107,10 +107,12 @@ stages: Write-Host "Package version: $packageVersion" New-Item -ItemType Directory -Path $OutDir -Force | Out-Null - Compress-Archive ` + # Compress-Archive ` + # -Path "$ArtifactRootPath/$ArtifactName/$ArtifactName-$packageVersion.crate" ` + # -DestinationPath "$OutDir/release.zip" + Copy-Item ` -Path "$ArtifactRootPath/$ArtifactName/$ArtifactName-$packageVersion.crate" ` - -DestinationPath "$OutDir/release.zip" - + -Destination $OutDir Write-Host "Contents of $OutDir" Get-ChildItem -Path $OutDir | ForEach-Object { Write-Host $_.FullName } From 8d8b35711938dbf97af7b02f448818d5e4226677 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Wed, 24 Sep 2025 19:19:49 +0000 Subject: [PATCH 21/87] Remove toolchain config --- eng/pipelines/templates/stages/archetype-rust-release.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/eng/pipelines/templates/stages/archetype-rust-release.yml b/eng/pipelines/templates/stages/archetype-rust-release.yml index e346488d2c..866b2adb3d 100644 --- a/eng/pipelines/templates/stages/archetype-rust-release.yml +++ b/eng/pipelines/templates/stages/archetype-rust-release.yml @@ -85,10 +85,6 @@ stages: runOnce: deploy: steps: - - template: /eng/pipelines/templates/steps/use-rust.yml@self - parameters: - Toolchain: stable - - pwsh: | $ArtifactName = '${{artifact.name}}' $ArtifactRootPath = '$(Pipeline.Workspace)/drop' From 2cb1634093d34b7152cec685ca87a726d40fb222 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Wed, 24 Sep 2025 19:21:09 +0000 Subject: [PATCH 22/87] Add KV and signing info --- eng/pipelines/templates/stages/archetype-rust-release.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eng/pipelines/templates/stages/archetype-rust-release.yml b/eng/pipelines/templates/stages/archetype-rust-release.yml index 866b2adb3d..da84a491de 100644 --- a/eng/pipelines/templates/stages/archetype-rust-release.yml +++ b/eng/pipelines/templates/stages/archetype-rust-release.yml @@ -121,6 +121,8 @@ stages: ClientId: '5f81938c-2544-4f1f-9251-dd9de5b8a81b' DomainTenantId: '975f013f-7f24-47e8-a7d3-abc4752bf346' Usemanagedidentity: true + KeyVaultName: 'kv-azuresdk-codesign' + SignCertName: 'azure-sdk-esrp-release-certificate' intent: 'packagedistribution' contenttype: 'Rust' contentsource: 'Folder' From 606863cb5248a1289d81135002e219190108f1fd Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Thu, 25 Sep 2025 16:32:10 +0000 Subject: [PATCH 23/87] Use ESRPRELPACMANTEST --- eng/pipelines/templates/stages/archetype-rust-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/stages/archetype-rust-release.yml b/eng/pipelines/templates/stages/archetype-rust-release.yml index da84a491de..7f583abf90 100644 --- a/eng/pipelines/templates/stages/archetype-rust-release.yml +++ b/eng/pipelines/templates/stages/archetype-rust-release.yml @@ -131,7 +131,7 @@ stages: owners: ${{ coalesce(variables['Build.RequestedForEmail'], 'azuresdk@microsoft.com') }} approvers: ${{ coalesce(variables['Build.RequestedForEmail'], 'azuresdk@microsoft.com') }} serviceendpointurl: 'https://api.esrp.microsoft.com/' - mainpublisher: 'ESRPRELPACMAN' + mainpublisher: 'ESRPRELPACMANTEST' # - pwsh: | # $additionalOwners = @('heaths', 'hallipr') From c66857d8c2549386114e34928a6a048565846974 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Thu, 25 Sep 2025 19:02:46 +0000 Subject: [PATCH 24/87] Check that crate is publishable --- .../stages/archetype-rust-release.yml | 108 ++++++++++-------- 1 file changed, 60 insertions(+), 48 deletions(-) diff --git a/eng/pipelines/templates/stages/archetype-rust-release.yml b/eng/pipelines/templates/stages/archetype-rust-release.yml index 7f583abf90..861b0408d4 100644 --- a/eng/pipelines/templates/stages/archetype-rust-release.yml +++ b/eng/pipelines/templates/stages/archetype-rust-release.yml @@ -38,29 +38,29 @@ stages: jobs: # TODO: Ensure that CHANGELOG is properly processed - # - job: TagRepository - # displayName: "Create release tag" - # condition: and(succeeded(), ne(variables['Skip.TagRepository'], 'true')) + - job: TagRepository + displayName: "Create release tag" + condition: and(succeeded(), ne(variables['Skip.TagRepository'], 'true')) - # steps: - # - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + steps: + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - # - download: current - # displayName: Download ${{parameters.PipelineArtifactName}} artifact - # artifact: ${{parameters.PipelineArtifactName}} + - download: current + displayName: Download ${{parameters.PipelineArtifactName}} artifact + artifact: ${{parameters.PipelineArtifactName}} - # - template: /eng/common/pipelines/templates/steps/retain-run.yml + - template: /eng/common/pipelines/templates/steps/retain-run.yml - # - script: | - # echo "##vso[build.addbuildtag]${{artifact.name}}" - # displayName: Add build tag '${{artifact.name}}' + - script: | + echo "##vso[build.addbuildtag]${{artifact.name}}" + displayName: Add build tag '${{artifact.name}}' - # - template: /eng/common/pipelines/templates/steps/create-tags-and-git-release.yml - # parameters: - # ArtifactLocation: $(Pipeline.Workspace)/${{parameters.PipelineArtifactName}}/${{artifact.name}} - # PackageRepository: Crates.io - # ReleaseSha: $(Build.SourceVersion) - # WorkingDirectory: $(Pipeline.Workspace)/_work + - template: /eng/common/pipelines/templates/steps/create-tags-and-git-release.yml + parameters: + ArtifactLocation: $(Pipeline.Workspace)/${{parameters.PipelineArtifactName}}/${{artifact.name}} + PackageRepository: Crates.io + ReleaseSha: $(Build.SourceVersion) + WorkingDirectory: $(Pipeline.Workspace)/_work - deployment: PublishPackage displayName: "Publish to Crates.io" @@ -76,7 +76,7 @@ stages: # This timeout shouldn't be necessary once we're able to parallelize better. Right now, # this is here to ensure larger areas (30+) libraries don't time out. timeoutInMinutes: 120 - # dependsOn: TagRepository + dependsOn: TagRepository pool: name: azsdk-pool image: ubuntu-24.04 @@ -102,17 +102,53 @@ stages: $packageVersion = $packageMetadata.version Write-Host "Package version: $packageVersion" + $cratePath = "$ArtifactRootPath/$ArtifactName/$ArtifactName-$packageVersion.crate" + Write-Host "##vso[task.setvariable variable=CratePath]$cratePath" + New-Item -ItemType Directory -Path $OutDir -Force | Out-Null - # Compress-Archive ` - # -Path "$ArtifactRootPath/$ArtifactName/$ArtifactName-$packageVersion.crate" ` - # -DestinationPath "$OutDir/release.zip" Copy-Item ` - -Path "$ArtifactRootPath/$ArtifactName/$ArtifactName-$packageVersion.crate" ` + -Path $cratePath` -Destination $OutDir Write-Host "Contents of $OutDir" Get-ChildItem -Path $OutDir | ForEach-Object { Write-Host $_.FullName } + displayName: Copy crate for ESRP - displayName: Compress Crate for ESRP + - pwsh: | + $CratePath = '$(CratePath)' + if (!(Test-Path $CratePath)) { + Write-Error "Crate file not found: $CratePath" + exit 1 + } + + $evaluationDir = (Join-Path ([System.IO.Path]::GetTempPath()) ([System.IO.Path]::GetRandomFileName())) + + New-Item -ItemType Directory -Path $evaluationDir -Force | Out-Null + Invoke-Expression "tar --exclude `"Cargo.toml.orig`" -xzvf $CratePath -C $evaluationDir" + + if ($LASTEXITCODE) { + Write-Host "Failed to extract crate $CratePath" + exit 1 + } + + Set-Location $evaluationDir + Set-Location (Join-Path $evaluationDir $crateName) + + $command = @( + "cargo publish", + "--locked", + "--dry-run", + "--package $packageName", + "--allow-dirty" + ) -join ' ' + Invoke-Expression $command + + if ($LASTEXITCODE) { + Write-Host "Crate $crateName is NOT publishable" + exit 1 + } else { + Write-Host "Crate $crateName is publishable" + } + displayName: Verify crate is publishable - task: EsrpRelease@10 displayName: 'ESRP Release' @@ -133,30 +169,6 @@ stages: serviceendpointurl: 'https://api.esrp.microsoft.com/' mainpublisher: 'ESRPRELPACMANTEST' - # - pwsh: | - # $additionalOwners = @('heaths', 'hallipr') - # $token = $env:CARGO_REGISTRY_TOKEN - # $crateName = '${{artifact.name}}' - - # $manifestPath = "$(Pipeline.Workspace)/drop/$crateName/contents/Cargo.toml" - # Write-Host "> cargo publish --manifest-path `"$manifestPath`"" - # cargo publish --manifest-path $manifestPath - # if (!$?) { - # Write-Error "Failed to publish package: '$crateName'" - # exit 1 - # } - - # $existingOwners = (cargo owner --list $crateName) -replace " \(.*", "" - # $missingOwners = $additionalOwners | Where-Object { $existingOwners -notcontains $_ } - - # foreach ($owner in $missingOwners) { - # Write-Host "> cargo owner --add $owner $crateName" - # cargo owner --add $owner $crateName - # } - # displayName: Publish Crate - # env: - # CARGO_REGISTRY_TOKEN: $(azure-sdk-cratesio-token) - - job: UpdatePackageVersion displayName: "API Review and Package Version Update" condition: and(succeeded(), ne(variables['Skip.UpdatePackageVersion'], 'true')) From c5f84bf5110f6c83cab3c7280aeed5e7a095d471 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Fri, 26 Sep 2025 19:29:10 +0000 Subject: [PATCH 25/87] Pack-Crates.ps1: Remove -PackageInfoDirectory, add -RequireDependencies to validate listed packages can be released --- eng/scripts/Pack-Crates.ps1 | 54 +++++++++++-------------------------- 1 file changed, 15 insertions(+), 39 deletions(-) diff --git a/eng/scripts/Pack-Crates.ps1 b/eng/scripts/Pack-Crates.ps1 index 38f4013ee2..45e176c1f4 100755 --- a/eng/scripts/Pack-Crates.ps1 +++ b/eng/scripts/Pack-Crates.ps1 @@ -4,10 +4,8 @@ [CmdletBinding(DefaultParameterSetName = "none")] param( [string]$OutputPath, - [Parameter(ParameterSetName = 'Named')] [string[]]$PackageNames, - [Parameter(ParameterSetName = 'PackageInfo')] - [string]$PackageInfoDirectory, + [switch]$RequireDependencies, [switch]$NoVerify ) @@ -24,39 +22,6 @@ if ($OutputPath) { $OutputPath = New-Item -ItemType Directory -Path $OutputPath -Force | Select-Object -ExpandProperty FullName } -function Get-OutputPackageNames($workspacePackages) { - $packablePackages = $workspacePackages | Where-Object -Property publish -NE -Value @() - $packablePackageNames = $packablePackages.name - - $names = @() - switch ($PsCmdlet.ParameterSetName) { - 'Named' { - $names = $PackageNames - } - - 'PackageInfo' { - $packageInfoFiles = Get-ChildItem -Path $PackageInfoDirectory -Filter '*.json' -File - foreach ($packageInfoFile in $packageInfoFiles) { - $packageInfo = Get-Content -Path $packageInfoFile.FullName | ConvertFrom-Json - $names += $packageInfo.name - } - } - - default { - return $packablePackageNames - } - } - - foreach ($name in $names) { - if (-not $packablePackageNames.Contains($name)) { - Write-Error "Package '$name' is not in the workspace or does not publish" - exit 1 - } - } - - return $names -} - function Get-CargoMetadata() { cargo metadata --no-deps --format-version 1 --manifest-path "$RepoRoot/Cargo.toml" | ConvertFrom-Json -Depth 100 -AsHashtable } @@ -81,10 +46,9 @@ function Get-CargoPackages() { function Get-PackagesToBuild() { $packages = Get-CargoPackages - $outputPackageNames = Get-OutputPackageNames $packages # We start with output packages, then recursively add unreleased dependencies to the list of packages that need to be built - [array]$packagesToBuild = $packages | Where-Object { $outputPackageNames.Contains($_.name) } + [array]$packagesToBuild = $packages | Where-Object { $PackageNames.Contains($_.name) } $toProcess = $packagesToBuild while ($toProcess.Length -gt 0) { @@ -93,6 +57,10 @@ function Get-PackagesToBuild() { foreach ($dependency in $package.UnreleasedDependencies) { if (!$packagesToBuild.Contains($dependency) -and !$toProcess.Contains($dependency)) { + if ($RequireDependencies -and $dependency.name -notin $PackageNames) { + Write-Warning "Package $($package.name) depends on unreleased or unspecified dependency: $($dependency.name)" + } + $packagesToBuild += $dependency $toProcess += $dependency } @@ -114,7 +82,7 @@ function Get-PackagesToBuild() { exit 1 } - $package.OutputPackage = $outputPackageNames.Contains($package.name) + $package.OutputPackage = $PackageNames.Contains($package.name) $buildOrder += $package $packagesToBuild = @($packagesToBuild -ne $package) @@ -162,6 +130,14 @@ try { [array]$packages = Get-PackagesToBuild + if ($RequireDependencies) { + $unspecifiedPackages = $packages.name | Where-Object { $_ -notin $PackageNames } + if ($unspecifiedPackages.Count -gt 0) { + Write-Error "Packages in -PackageNames require dependencies that are either not released or not listed for packing: $($unspecifiedPackages -join ', ')" + exit 1 + } + } + Write-Host "Building packages in the following order:" foreach ($package in $packages) { $packageName = $package.name From 465f1dfbfb9a51b78453b91d61a0ff6aaf903994 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Fri, 26 Sep 2025 19:34:25 +0000 Subject: [PATCH 26/87] Wire up release intent --- eng/pipelines/templates/jobs/ci.yml | 4 + eng/pipelines/templates/jobs/pack.yml | 4 +- .../stages/archetype-rust-release.yml | 274 ++++++++---------- .../templates/stages/archetype-sdk-client.yml | 11 +- sdk/canary/ci.yml | 16 +- 5 files changed, 149 insertions(+), 160 deletions(-) diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index 0cb6c2a9d5..c0320a94c4 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -3,6 +3,8 @@ parameters: type: string - name: Artifacts type: object +- name: RequireDependencies + type: boolean - name: PipelineArtifactName type: string - name: TestPipeline @@ -18,11 +20,13 @@ parameters: - name: MatrixReplace type: object + jobs: - template: /eng/pipelines/templates/jobs/pack.yml parameters: ServiceDirectory: ${{ parameters.ServiceDirectory }} Artifacts: ${{ parameters.Artifacts }} + RequireDependencies: ${{ parameters.RequireDependencies }} TestPipeline: ${{ parameters.TestPipeline }} PipelineArtifactName: ${{ parameters.PipelineArtifactName }} diff --git a/eng/pipelines/templates/jobs/pack.yml b/eng/pipelines/templates/jobs/pack.yml index 1d0f51a814..0c23878e25 100644 --- a/eng/pipelines/templates/jobs/pack.yml +++ b/eng/pipelines/templates/jobs/pack.yml @@ -63,8 +63,10 @@ jobs: filePath: $(Build.SourcesDirectory)/eng/scripts/Pack-Crates.ps1 arguments: > -OutputPath '$(Build.ArtifactStagingDirectory)' - -PackageInfoDirectory '$(Build.ArtifactStagingDirectory)/PackageInfo' + -PackageNames ${{ join(',', parameters.Artifacts.*.name) }} + -RequireDependencies:$${{ parameters.RequireDependencies }} + # TODO: Ensure APIView works given a change to the crates output folder structure - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml parameters: ArtifactPath: $(Build.ArtifactStagingDirectory) diff --git a/eng/pipelines/templates/stages/archetype-rust-release.yml b/eng/pipelines/templates/stages/archetype-rust-release.yml index 861b0408d4..55b9d36d98 100644 --- a/eng/pipelines/templates/stages/archetype-rust-release.yml +++ b/eng/pipelines/templates/stages/archetype-rust-release.yml @@ -21,36 +21,36 @@ parameters: stages: - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - ${{ if in(variables['Build.Reason'], 'Manual', '') }}: - - ${{ each artifact in parameters.Artifacts }}: - - stage: Release_${{artifact.safeName}} - displayName: "Release: ${{artifact.name}}" - dependsOn: ${{parameters.DependsOn}} - condition: and(succeeded(), ne(variables['SetDevVersion'], 'true'), ne(variables['Skip.Release'], 'true'), ne(variables['Build.Repository.Name'], 'Azure/azure-sdk-for-rust-pr')) - variables: - - template: /eng/pipelines/templates/variables/globals.yml - - template: /eng/pipelines/templates/variables/image.yml - - template: /eng/pipelines/templates/variables/rust.yml - - pool: - name: $(LINUXPOOL) - image: $(LINUXVMIMAGE) - os: linux - - jobs: - # TODO: Ensure that CHANGELOG is properly processed - - job: TagRepository - displayName: "Create release tag" - condition: and(succeeded(), ne(variables['Skip.TagRepository'], 'true')) - - steps: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - - - download: current - displayName: Download ${{parameters.PipelineArtifactName}} artifact - artifact: ${{parameters.PipelineArtifactName}} - - - template: /eng/common/pipelines/templates/steps/retain-run.yml - + - stage: Release_Batch + displayName: "Releasing: ${{length(parameters.Artifacts)}} crates" + dependsOn: ${{parameters.DependsOn}} + condition: and(succeeded(), ne(variables['SetDevVersion'], 'true'), ne(variables['Skip.Release'], 'true'), ne(variables['Build.Repository.Name'], 'Azure/azure-sdk-for-rust-pr')) + variables: + - template: /eng/pipelines/templates/variables/globals.yml + - template: /eng/pipelines/templates/variables/image.yml + - template: /eng/pipelines/templates/variables/rust.yml + + pool: + name: $(LINUXPOOL) + image: $(LINUXVMIMAGE) + os: linux + + jobs: + # TODO: Ensure that CHANGELOG is properly processed + - job: TagRepository + displayName: "Create release tag" + condition: and(succeeded(), ne(variables['Skip.TagRepository'], 'true')) + + steps: + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + + - download: current + displayName: Download ${{parameters.PipelineArtifactName}} artifact + artifact: ${{parameters.PipelineArtifactName}} + + - template: /eng/common/pipelines/templates/steps/retain-run.yml + + - ${{ each artifact in parameters.Artifacts }}: - script: | echo "##vso[build.addbuildtag]${{artifact.name}}" displayName: Add build tag '${{artifact.name}}' @@ -62,131 +62,95 @@ stages: ReleaseSha: $(Build.SourceVersion) WorkingDirectory: $(Pipeline.Workspace)/_work - - deployment: PublishPackage - displayName: "Publish to Crates.io" - condition: and(succeeded(), ne(variables['Skip.PublishPackage'], 'true')) - templateContext: - type: releaseJob # Required, this indicates this deployment job is a release job - isProduction: true # Required, must be 'true' or 'false' - inputs: # All input build artifacts must be declared here - - input: pipelineArtifact # Required, type of the input artifact - artifactName: ${{parameters.PipelineArtifactName}} # Required, name of the pipeline artifact - targetPath: $(Pipeline.Workspace)/drop # Optional, specifies where the artifact is downloaded to - environment: ${{parameters.Environment}} - # This timeout shouldn't be necessary once we're able to parallelize better. Right now, - # this is here to ensure larger areas (30+) libraries don't time out. - timeoutInMinutes: 120 - dependsOn: TagRepository - pool: - name: azsdk-pool - image: ubuntu-24.04 - os: linux - strategy: - runOnce: - deploy: - steps: - - pwsh: | - $ArtifactName = '${{artifact.name}}' - $ArtifactRootPath = '$(Pipeline.Workspace)/drop' - $OutDir = '$(Pipeline.Workspace)/esrp-release' - - Write-Host "Artifact name: $ArtifactName" - - $packageMetadataPath = "$ArtifactRootPath/PackageInfo/$ArtifactName.json" - if (!(Test-Path $packageMetadataPath)) { - Write-Error "Package metadata file not found: $packageMetadataPath" - exit 1 - } - - $packageMetadata = Get-Content -Raw $packageMetadataPath | ConvertFrom-Json - $packageVersion = $packageMetadata.version - Write-Host "Package version: $packageVersion" - - $cratePath = "$ArtifactRootPath/$ArtifactName/$ArtifactName-$packageVersion.crate" - Write-Host "##vso[task.setvariable variable=CratePath]$cratePath" - - New-Item -ItemType Directory -Path $OutDir -Force | Out-Null - Copy-Item ` - -Path $cratePath` - -Destination $OutDir - Write-Host "Contents of $OutDir" - Get-ChildItem -Path $OutDir | ForEach-Object { Write-Host $_.FullName } - displayName: Copy crate for ESRP - - - pwsh: | - $CratePath = '$(CratePath)' - if (!(Test-Path $CratePath)) { - Write-Error "Crate file not found: $CratePath" - exit 1 - } - - $evaluationDir = (Join-Path ([System.IO.Path]::GetTempPath()) ([System.IO.Path]::GetRandomFileName())) - - New-Item -ItemType Directory -Path $evaluationDir -Force | Out-Null - Invoke-Expression "tar --exclude `"Cargo.toml.orig`" -xzvf $CratePath -C $evaluationDir" - - if ($LASTEXITCODE) { - Write-Host "Failed to extract crate $CratePath" - exit 1 - } - - Set-Location $evaluationDir - Set-Location (Join-Path $evaluationDir $crateName) - - $command = @( - "cargo publish", - "--locked", - "--dry-run", - "--package $packageName", - "--allow-dirty" - ) -join ' ' - Invoke-Expression $command - - if ($LASTEXITCODE) { - Write-Host "Crate $crateName is NOT publishable" - exit 1 - } else { - Write-Host "Crate $crateName is publishable" - } - displayName: Verify crate is publishable - - - task: EsrpRelease@10 - displayName: 'ESRP Release' - inputs: - connectedservicename: 'Azure SDK PME Managed Identity' - ClientId: '5f81938c-2544-4f1f-9251-dd9de5b8a81b' - DomainTenantId: '975f013f-7f24-47e8-a7d3-abc4752bf346' - Usemanagedidentity: true - KeyVaultName: 'kv-azuresdk-codesign' - SignCertName: 'azure-sdk-esrp-release-certificate' - intent: 'packagedistribution' - contenttype: 'Rust' - contentsource: 'Folder' - folderlocation: '$(Pipeline.Workspace)/esrp-release' - waitforreleasecompletion: true - owners: ${{ coalesce(variables['Build.RequestedForEmail'], 'azuresdk@microsoft.com') }} - approvers: ${{ coalesce(variables['Build.RequestedForEmail'], 'azuresdk@microsoft.com') }} - serviceendpointurl: 'https://api.esrp.microsoft.com/' - mainpublisher: 'ESRPRELPACMANTEST' - - - job: UpdatePackageVersion - displayName: "API Review and Package Version Update" - condition: and(succeeded(), ne(variables['Skip.UpdatePackageVersion'], 'true')) - dependsOn: PublishPackage - steps: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - paths: - - "/*" - - - template: /eng/pipelines/templates/steps/use-rust.yml@self - parameters: - Toolchain: nightly - - - download: current - displayName: Download ${{parameters.PipelineArtifactName}} artifact - artifact: ${{parameters.PipelineArtifactName}} - + - deployment: PublishPackage + displayName: "Publish to Crates.io" + condition: and(succeeded(), ne(variables['Skip.PublishPackage'], 'true')) + templateContext: + type: releaseJob # Required, this indicates this deployment job is a release job + isProduction: true # Required, must be 'true' or 'false' + inputs: # All input build artifacts must be declared here + - input: pipelineArtifact # Required, type of the input artifact + artifactName: ${{parameters.PipelineArtifactName}} # Required, name of the pipeline artifact + targetPath: $(Pipeline.Workspace)/drop # Optional, specifies where the artifact is downloaded to + environment: ${{parameters.Environment}} + # This timeout shouldn't be necessary once we're able to parallelize better. Right now, + # this is here to ensure larger areas (30+) libraries don't time out. + timeoutInMinutes: 120 + dependsOn: TagRepository + pool: + name: azsdk-pool + image: ubuntu-24.04 + os: linux + strategy: + runOnce: + deploy: + steps: + - pwsh: | + $ArtifactName = '${{artifact.name}}' + $ArtifactRootPath = '$(Pipeline.Workspace)/drop' + $OutDir = '$(Pipeline.Workspace)/esrp-release' + + Write-Host "Artifact name: $ArtifactName" + + $packageMetadataPath = "$ArtifactRootPath/PackageInfo/$ArtifactName.json" + if (!(Test-Path $packageMetadataPath)) { + Write-Error "Package metadata file not found: $packageMetadataPath" + exit 1 + } + + $packageMetadata = Get-Content -Raw $packageMetadataPath | ConvertFrom-Json + $packageVersion = $packageMetadata.version + Write-Host "Package version: $packageVersion" + + $cratePath = "$ArtifactRootPath/$ArtifactName/$ArtifactName-$packageVersion.crate" + Write-Host "##vso[task.setvariable variable=CratePath]$cratePath" + + New-Item -ItemType Directory -Path $OutDir -Force | Out-Null + Copy-Item ` + -Path $cratePath` + -Destination $OutDir + Write-Host "Contents of $OutDir" + Get-ChildItem -Path $OutDir | ForEach-Object { Write-Host $_.FullName } + displayName: Copy crate for ESRP + + - task: EsrpRelease@10 + displayName: 'ESRP Release' + inputs: + connectedservicename: 'Azure SDK PME Managed Identity' + ClientId: '5f81938c-2544-4f1f-9251-dd9de5b8a81b' + DomainTenantId: '975f013f-7f24-47e8-a7d3-abc4752bf346' + Usemanagedidentity: true + KeyVaultName: 'kv-azuresdk-codesign' + SignCertName: 'azure-sdk-esrp-release-certificate' + intent: 'packagedistribution' + contenttype: 'Rust' + contentsource: 'Folder' + folderlocation: '$(Pipeline.Workspace)/esrp-release' + waitforreleasecompletion: true + owners: ${{ coalesce(variables['Build.RequestedForEmail'], 'azuresdk@microsoft.com') }} + approvers: ${{ coalesce(variables['Build.RequestedForEmail'], 'azuresdk@microsoft.com') }} + serviceendpointurl: 'https://api.esrp.microsoft.com/' + mainpublisher: 'ESRPRELPACMANTEST' + + - job: UpdatePackageVersion + displayName: "API Review and Package Version Update" + condition: and(succeeded(), ne(variables['Skip.UpdatePackageVersion'], 'true')) + dependsOn: PublishPackage + steps: + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + parameters: + paths: + - "/*" + + - template: /eng/pipelines/templates/steps/use-rust.yml@self + parameters: + Toolchain: nightly + + - download: current + displayName: Download ${{parameters.PipelineArtifactName}} artifact + artifact: ${{parameters.PipelineArtifactName}} + + - ${{each artifact in parameters.Artifacts }}: - template: /eng/common/pipelines/templates/steps/create-apireview.yml parameters: ArtifactPath: $(Pipeline.Workspace)/${{parameters.PipelineArtifactName}} diff --git a/eng/pipelines/templates/stages/archetype-sdk-client.yml b/eng/pipelines/templates/stages/archetype-sdk-client.yml index 141ec4de3a..a0b861dfd4 100644 --- a/eng/pipelines/templates/stages/archetype-sdk-client.yml +++ b/eng/pipelines/templates/stages/archetype-sdk-client.yml @@ -83,7 +83,11 @@ extends: parameters: ServiceDirectory: ${{ parameters.ServiceDirectory }} PipelineArtifactName: packages - Artifacts: ${{ parameters.Artifacts }} + Artifacts: + - ${{ each artifact in parameters.Artifacts }}: + - ${{ if ne('false', artifact.releaseInBatch) }}: + - ${{ artifact }} + RequireDependencies: ${{ in(variables['Build.Reason'], 'Manual', '') }} TestPipeline: ${{ eq(parameters.ServiceDirectory, 'canary') }} TestTimeoutInMinutes: ${{ parameters.TestTimeoutInMinutes }} TestProxy: ${{ parameters.TestProxy }} @@ -148,6 +152,9 @@ extends: parameters: DependsOn: "Build" ServiceDirectory: ${{ parameters.ServiceDirectory }} - Artifacts: ${{ parameters.Artifacts }} + Artifacts: + - ${{ each artifact in parameters.Artifacts }}: + - ${{ if ne(artifact.releaseInBatch, 'false')}}: + - ${{ artifact }} TestPipeline: ${{ eq(parameters.ServiceDirectory, 'canary') }} PipelineArtifactName: packages diff --git a/sdk/canary/ci.yml b/sdk/canary/ci.yml index 0ffe09337e..9a7eb64742 100644 --- a/sdk/canary/ci.yml +++ b/sdk/canary/ci.yml @@ -10,12 +10,24 @@ trigger: include: - sdk/canary/ +parameters: + - name: release_azurecanarycore + displayName: 'azure_canary_core' + type: boolean + default: true + - name: release_azurecanary + displayName: 'azure_canary' + type: boolean + default: true + extends: template: /eng/pipelines/templates/stages/archetype-sdk-client.yml parameters: ServiceDirectory: canary Artifacts: - name: azure_canary_core - safeName: AzureTemplateCore + safeName: AzureCanaryCore + releaseInBatch: ${{ parameters.release_azurecanarycore }} - name: azure_canary - safeName: AzureTemplate + safeName: AzureCanary + releaseInBatch: ${{ parameters.release_azurecanary }} From 3525680cf0d9d383586c0efa5d06146694a5c92e Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Fri, 26 Sep 2025 19:39:21 +0000 Subject: [PATCH 27/87] each artifact --- .../stages/archetype-rust-release.yml | 94 ++++++++++--------- 1 file changed, 48 insertions(+), 46 deletions(-) diff --git a/eng/pipelines/templates/stages/archetype-rust-release.yml b/eng/pipelines/templates/stages/archetype-rust-release.yml index 55b9d36d98..2868ca6390 100644 --- a/eng/pipelines/templates/stages/archetype-rust-release.yml +++ b/eng/pipelines/templates/stages/archetype-rust-release.yml @@ -85,52 +85,54 @@ stages: runOnce: deploy: steps: - - pwsh: | - $ArtifactName = '${{artifact.name}}' - $ArtifactRootPath = '$(Pipeline.Workspace)/drop' - $OutDir = '$(Pipeline.Workspace)/esrp-release' - - Write-Host "Artifact name: $ArtifactName" - - $packageMetadataPath = "$ArtifactRootPath/PackageInfo/$ArtifactName.json" - if (!(Test-Path $packageMetadataPath)) { - Write-Error "Package metadata file not found: $packageMetadataPath" - exit 1 - } - - $packageMetadata = Get-Content -Raw $packageMetadataPath | ConvertFrom-Json - $packageVersion = $packageMetadata.version - Write-Host "Package version: $packageVersion" - - $cratePath = "$ArtifactRootPath/$ArtifactName/$ArtifactName-$packageVersion.crate" - Write-Host "##vso[task.setvariable variable=CratePath]$cratePath" - - New-Item -ItemType Directory -Path $OutDir -Force | Out-Null - Copy-Item ` - -Path $cratePath` - -Destination $OutDir - Write-Host "Contents of $OutDir" - Get-ChildItem -Path $OutDir | ForEach-Object { Write-Host $_.FullName } - displayName: Copy crate for ESRP - - - task: EsrpRelease@10 - displayName: 'ESRP Release' - inputs: - connectedservicename: 'Azure SDK PME Managed Identity' - ClientId: '5f81938c-2544-4f1f-9251-dd9de5b8a81b' - DomainTenantId: '975f013f-7f24-47e8-a7d3-abc4752bf346' - Usemanagedidentity: true - KeyVaultName: 'kv-azuresdk-codesign' - SignCertName: 'azure-sdk-esrp-release-certificate' - intent: 'packagedistribution' - contenttype: 'Rust' - contentsource: 'Folder' - folderlocation: '$(Pipeline.Workspace)/esrp-release' - waitforreleasecompletion: true - owners: ${{ coalesce(variables['Build.RequestedForEmail'], 'azuresdk@microsoft.com') }} - approvers: ${{ coalesce(variables['Build.RequestedForEmail'], 'azuresdk@microsoft.com') }} - serviceendpointurl: 'https://api.esrp.microsoft.com/' - mainpublisher: 'ESRPRELPACMANTEST' + - ${{ each artifact in parameters.Artifacts }}: + - pwsh: | + $ArtifactName = '${{artifact.name}}' + $ArtifactRootPath = '$(Pipeline.Workspace)/drop' + $OutDir = '$(Pipeline.Workspace)/esrp-release' + + Write-Host "Cleaning output directory: $OutDir" + Remove-Item -Path $OutDir -Recurse -Force + New-Item -ItemType Directory -Path $OutDir -Force | Out-Null + + Write-Host "Artifact name: $ArtifactName" + + $packageMetadataPath = "$ArtifactRootPath/PackageInfo/$ArtifactName.json" + if (!(Test-Path $packageMetadataPath)) { + Write-Error "Package metadata file not found: $packageMetadataPath" + exit 1 + } + + $packageMetadata = Get-Content -Raw $packageMetadataPath | ConvertFrom-Json + $packageVersion = $packageMetadata.version + Write-Host "Package version: $packageVersion" + + $cratePath = "$ArtifactRootPath/$ArtifactName/$ArtifactName-$packageVersion.crate" + Copy-Item ` + -Path $cratePath` + -Destination $OutDir + Write-Host "Contents of $OutDir" + Get-ChildItem -Path $OutDir | ForEach-Object { Write-Host $_.FullName } + displayName: Copy crate for ESRP + + - task: EsrpRelease@10 + displayName: 'ESRP Release' + inputs: + connectedservicename: 'Azure SDK PME Managed Identity' + ClientId: '5f81938c-2544-4f1f-9251-dd9de5b8a81b' + DomainTenantId: '975f013f-7f24-47e8-a7d3-abc4752bf346' + Usemanagedidentity: true + KeyVaultName: 'kv-azuresdk-codesign' + SignCertName: 'azure-sdk-esrp-release-certificate' + intent: 'packagedistribution' + contenttype: 'Rust' + contentsource: 'Folder' + folderlocation: '$(Pipeline.Workspace)/esrp-release' + waitforreleasecompletion: true + owners: ${{ coalesce(variables['Build.RequestedForEmail'], 'azuresdk@microsoft.com') }} + approvers: ${{ coalesce(variables['Build.RequestedForEmail'], 'azuresdk@microsoft.com') }} + serviceendpointurl: 'https://api.esrp.microsoft.com/' + mainpublisher: 'ESRPRELPACMANTEST' - job: UpdatePackageVersion displayName: "API Review and Package Version Update" From 7c399f9998be106524a2d5071f2dec0839b71618 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Fri, 26 Sep 2025 19:40:40 +0000 Subject: [PATCH 28/87] Parameters --- eng/pipelines/templates/jobs/pack.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eng/pipelines/templates/jobs/pack.yml b/eng/pipelines/templates/jobs/pack.yml index 0c23878e25..88bff70938 100644 --- a/eng/pipelines/templates/jobs/pack.yml +++ b/eng/pipelines/templates/jobs/pack.yml @@ -3,6 +3,8 @@ parameters: type: string - name: Artifacts type: object +- name: RequireDependencies + type: boolean - name: TestPipeline type: boolean - name: PipelineArtifactName From a0832ce971f81d582e8abd77965cd91f1abf0e9a Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Fri, 26 Sep 2025 19:44:44 +0000 Subject: [PATCH 29/87] Only specify an environment if not in a test pipeline --- eng/pipelines/templates/stages/archetype-rust-release.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eng/pipelines/templates/stages/archetype-rust-release.yml b/eng/pipelines/templates/stages/archetype-rust-release.yml index 2868ca6390..68defd2cad 100644 --- a/eng/pipelines/templates/stages/archetype-rust-release.yml +++ b/eng/pipelines/templates/stages/archetype-rust-release.yml @@ -72,7 +72,8 @@ stages: - input: pipelineArtifact # Required, type of the input artifact artifactName: ${{parameters.PipelineArtifactName}} # Required, name of the pipeline artifact targetPath: $(Pipeline.Workspace)/drop # Optional, specifies where the artifact is downloaded to - environment: ${{parameters.Environment}} + ${{ if not(parameters.TestPipeline) }}: + environment: ${{parameters.Environment}} # This timeout shouldn't be necessary once we're able to parallelize better. Right now, # this is here to ensure larger areas (30+) libraries don't time out. timeoutInMinutes: 120 From 8f030bed22fc8765431fcf356ca7a0bd4ad3e40b Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Fri, 26 Sep 2025 19:50:11 +0000 Subject: [PATCH 30/87] Use environment: none if in TestPipeline --- eng/pipelines/templates/stages/archetype-rust-release.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/templates/stages/archetype-rust-release.yml b/eng/pipelines/templates/stages/archetype-rust-release.yml index 68defd2cad..ea6e62ecdd 100644 --- a/eng/pipelines/templates/stages/archetype-rust-release.yml +++ b/eng/pipelines/templates/stages/archetype-rust-release.yml @@ -72,8 +72,10 @@ stages: - input: pipelineArtifact # Required, type of the input artifact artifactName: ${{parameters.PipelineArtifactName}} # Required, name of the pipeline artifact targetPath: $(Pipeline.Workspace)/drop # Optional, specifies where the artifact is downloaded to - ${{ if not(parameters.TestPipeline) }}: - environment: ${{parameters.Environment}} + ${{if parameters.TestPipeline}}: + environment: none + ${{else}}: + environment: ${{parameters.Environment}} # This timeout shouldn't be necessary once we're able to parallelize better. Right now, # this is here to ensure larger areas (30+) libraries don't time out. timeoutInMinutes: 120 From 89b48dfd1b447de52258d04bfb6d47fd760f8013 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Fri, 26 Sep 2025 19:51:07 +0000 Subject: [PATCH 31/87] Remove Environment param --- eng/pipelines/templates/stages/archetype-rust-release.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/eng/pipelines/templates/stages/archetype-rust-release.yml b/eng/pipelines/templates/stages/archetype-rust-release.yml index ea6e62ecdd..db8cf89550 100644 --- a/eng/pipelines/templates/stages/archetype-rust-release.yml +++ b/eng/pipelines/templates/stages/archetype-rust-release.yml @@ -14,9 +14,6 @@ parameters: - name: DevFeedName type: string default: 'public/azure-sdk-for-rust' -- name: Environment - type: string - default: 'cratesio' stages: - ${{ if eq(variables['System.TeamProject'], 'internal') }}: @@ -75,7 +72,7 @@ stages: ${{if parameters.TestPipeline}}: environment: none ${{else}}: - environment: ${{parameters.Environment}} + environment: cratesio # This timeout shouldn't be necessary once we're able to parallelize better. Right now, # this is here to ensure larger areas (30+) libraries don't time out. timeoutInMinutes: 120 From d9e4c26627e187193fc2c6774dfba6697d6d6d7c Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Fri, 26 Sep 2025 19:56:55 +0000 Subject: [PATCH 32/87] (test) Depend on unreleased core --- sdk/canary/azure_canary_core/Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/canary/azure_canary_core/Cargo.toml b/sdk/canary/azure_canary_core/Cargo.toml index 18c2d595a9..2086b8fb06 100644 --- a/sdk/canary/azure_canary_core/Cargo.toml +++ b/sdk/canary/azure_canary_core/Cargo.toml @@ -9,6 +9,7 @@ repository.workspace = true rust-version.workspace = true [dependencies] +azure_core = { path = "../../core/azure_core", version = "0.29.0" } [dev-dependencies] azure_canary.path = "../azure_canary" From 3bc22612e6b2762cf3a38e3158d17154d559209c Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Fri, 26 Sep 2025 20:01:46 +0000 Subject: [PATCH 33/87] Remove azure_canary_core's dependency on azure_core --- sdk/canary/azure_canary_core/Cargo.toml | 3 --- 1 file changed, 3 deletions(-) diff --git a/sdk/canary/azure_canary_core/Cargo.toml b/sdk/canary/azure_canary_core/Cargo.toml index 2086b8fb06..5c6c490964 100644 --- a/sdk/canary/azure_canary_core/Cargo.toml +++ b/sdk/canary/azure_canary_core/Cargo.toml @@ -8,9 +8,6 @@ license.workspace = true repository.workspace = true rust-version.workspace = true -[dependencies] -azure_core = { path = "../../core/azure_core", version = "0.29.0" } - [dev-dependencies] azure_canary.path = "../azure_canary" From ab2b2ab77feef49e49ea42c430976df5803f6fc6 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Fri, 26 Sep 2025 20:12:31 +0000 Subject: [PATCH 34/87] Check existence --- eng/pipelines/templates/stages/archetype-rust-release.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/eng/pipelines/templates/stages/archetype-rust-release.yml b/eng/pipelines/templates/stages/archetype-rust-release.yml index db8cf89550..39c1169151 100644 --- a/eng/pipelines/templates/stages/archetype-rust-release.yml +++ b/eng/pipelines/templates/stages/archetype-rust-release.yml @@ -92,7 +92,9 @@ stages: $OutDir = '$(Pipeline.Workspace)/esrp-release' Write-Host "Cleaning output directory: $OutDir" - Remove-Item -Path $OutDir -Recurse -Force + if (Test-Path $OutDir) { + Remove-Item -Path $OutDir -Recurse -Force + } New-Item -ItemType Directory -Path $OutDir -Force | Out-Null Write-Host "Artifact name: $ArtifactName" From 3eccd36b79afe9b66f5b229ca483373357c8f2cd Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Fri, 26 Sep 2025 20:13:54 +0000 Subject: [PATCH 35/87] Naming --- eng/pipelines/templates/stages/archetype-rust-release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/templates/stages/archetype-rust-release.yml b/eng/pipelines/templates/stages/archetype-rust-release.yml index 39c1169151..31f155a39b 100644 --- a/eng/pipelines/templates/stages/archetype-rust-release.yml +++ b/eng/pipelines/templates/stages/archetype-rust-release.yml @@ -115,10 +115,10 @@ stages: -Destination $OutDir Write-Host "Contents of $OutDir" Get-ChildItem -Path $OutDir | ForEach-Object { Write-Host $_.FullName } - displayName: Copy crate for ESRP + displayName: 'Copy crate for ESRP: ${{artifact.name}}' - task: EsrpRelease@10 - displayName: 'ESRP Release' + displayName: 'ESRP Release: ${{artifact.name}}' inputs: connectedservicename: 'Azure SDK PME Managed Identity' ClientId: '5f81938c-2544-4f1f-9251-dd9de5b8a81b' From 7ae6ad9709803698f33e5f58b8cf3b2f45304519 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Fri, 26 Sep 2025 20:34:46 +0000 Subject: [PATCH 36/87] Log file paths --- eng/pipelines/templates/stages/archetype-rust-release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eng/pipelines/templates/stages/archetype-rust-release.yml b/eng/pipelines/templates/stages/archetype-rust-release.yml index 31f155a39b..a8a18f412e 100644 --- a/eng/pipelines/templates/stages/archetype-rust-release.yml +++ b/eng/pipelines/templates/stages/archetype-rust-release.yml @@ -110,6 +110,7 @@ stages: Write-Host "Package version: $packageVersion" $cratePath = "$ArtifactRootPath/$ArtifactName/$ArtifactName-$packageVersion.crate" + Get-ChildItem "$ArtifactRootPath" -Recurse | ForEach-Object { Write-Host $_.FullName } Copy-Item ` -Path $cratePath` -Destination $OutDir From d9e378633959607d048db3ad0e3e347547346d31 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Fri, 26 Sep 2025 20:51:58 +0000 Subject: [PATCH 37/87] backtick --- eng/pipelines/templates/stages/archetype-rust-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/stages/archetype-rust-release.yml b/eng/pipelines/templates/stages/archetype-rust-release.yml index a8a18f412e..354b5368d8 100644 --- a/eng/pipelines/templates/stages/archetype-rust-release.yml +++ b/eng/pipelines/templates/stages/archetype-rust-release.yml @@ -112,7 +112,7 @@ stages: $cratePath = "$ArtifactRootPath/$ArtifactName/$ArtifactName-$packageVersion.crate" Get-ChildItem "$ArtifactRootPath" -Recurse | ForEach-Object { Write-Host $_.FullName } Copy-Item ` - -Path $cratePath` + -Path $cratePath ` -Destination $OutDir Write-Host "Contents of $OutDir" Get-ChildItem -Path $OutDir | ForEach-Object { Write-Host $_.FullName } From f302f9992b8d3a8b974452593c905ba2763b2ca4 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Fri, 26 Sep 2025 22:02:54 +0000 Subject: [PATCH 38/87] Remove extra logging --- eng/pipelines/templates/stages/archetype-rust-release.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/eng/pipelines/templates/stages/archetype-rust-release.yml b/eng/pipelines/templates/stages/archetype-rust-release.yml index 354b5368d8..799cf8654d 100644 --- a/eng/pipelines/templates/stages/archetype-rust-release.yml +++ b/eng/pipelines/templates/stages/archetype-rust-release.yml @@ -110,7 +110,6 @@ stages: Write-Host "Package version: $packageVersion" $cratePath = "$ArtifactRootPath/$ArtifactName/$ArtifactName-$packageVersion.crate" - Get-ChildItem "$ArtifactRootPath" -Recurse | ForEach-Object { Write-Host $_.FullName } Copy-Item ` -Path $cratePath ` -Destination $OutDir From 40437b40985c5ec5d93a94859fb0108ef6d2d19f Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Fri, 26 Sep 2025 22:21:45 +0000 Subject: [PATCH 39/87] Test ESRP idempotency --- .../templates/stages/archetype-rust-release.yml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/templates/stages/archetype-rust-release.yml b/eng/pipelines/templates/stages/archetype-rust-release.yml index 799cf8654d..a5328e5f7c 100644 --- a/eng/pipelines/templates/stages/archetype-rust-release.yml +++ b/eng/pipelines/templates/stages/archetype-rust-release.yml @@ -91,8 +91,8 @@ stages: $ArtifactRootPath = '$(Pipeline.Workspace)/drop' $OutDir = '$(Pipeline.Workspace)/esrp-release' - Write-Host "Cleaning output directory: $OutDir" if (Test-Path $OutDir) { + Write-Host "Cleaning output directory: $OutDir" Remove-Item -Path $OutDir -Recurse -Force } New-Item -ItemType Directory -Path $OutDir -Force | Out-Null @@ -116,7 +116,7 @@ stages: Write-Host "Contents of $OutDir" Get-ChildItem -Path $OutDir | ForEach-Object { Write-Host $_.FullName } displayName: 'Copy crate for ESRP: ${{artifact.name}}' - + - task: EsrpRelease@10 displayName: 'ESRP Release: ${{artifact.name}}' inputs: @@ -135,6 +135,15 @@ stages: approvers: ${{ coalesce(variables['Build.RequestedForEmail'], 'azuresdk@microsoft.com') }} serviceendpointurl: 'https://api.esrp.microsoft.com/' mainpublisher: 'ESRPRELPACMANTEST' + + - pwsh: | + Write-Host "Fail if it's the first attempt..." + if ('$(System.JobAttempt)' -eq '1') { + Write-Host "Failing job on the first attempt" + exit 1 + } + Write-Host "Not the first attempt, continuing..." + displayName: 'Fail first attempt to test retry behavior' - job: UpdatePackageVersion displayName: "API Review and Package Version Update" From 892f1b6a998ae4b73a587f87b18a1204a0a2de34 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Fri, 26 Sep 2025 23:04:25 +0000 Subject: [PATCH 40/87] Remove test step --- .../templates/stages/archetype-rust-release.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/eng/pipelines/templates/stages/archetype-rust-release.yml b/eng/pipelines/templates/stages/archetype-rust-release.yml index a5328e5f7c..8b816dbd50 100644 --- a/eng/pipelines/templates/stages/archetype-rust-release.yml +++ b/eng/pipelines/templates/stages/archetype-rust-release.yml @@ -135,15 +135,6 @@ stages: approvers: ${{ coalesce(variables['Build.RequestedForEmail'], 'azuresdk@microsoft.com') }} serviceendpointurl: 'https://api.esrp.microsoft.com/' mainpublisher: 'ESRPRELPACMANTEST' - - - pwsh: | - Write-Host "Fail if it's the first attempt..." - if ('$(System.JobAttempt)' -eq '1') { - Write-Host "Failing job on the first attempt" - exit 1 - } - Write-Host "Not the first attempt, continuing..." - displayName: 'Fail first attempt to test retry behavior' - job: UpdatePackageVersion displayName: "API Review and Package Version Update" From 93818a7ae9206d3872760387358217a311bd433d Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Mon, 29 Sep 2025 02:56:30 +0000 Subject: [PATCH 41/87] Update ci.yml files with release parameters --- sdk/core/ci.yml | 22 ++++++++++++++++++++++ sdk/cosmos/ci.yml | 7 +++++++ sdk/eventhubs/ci.yml | 12 ++++++++++++ sdk/identity/ci.yml | 15 +++++++++++---- sdk/keyvault/ci.yml | 27 +++++++++++++++++++++------ sdk/servicebus/ci.yml | 21 ++++++++++++++------- sdk/storage/ci.yml | 27 +++++++++++++++++++++------ 7 files changed, 108 insertions(+), 23 deletions(-) diff --git a/sdk/core/ci.yml b/sdk/core/ci.yml index fcbcbd265a..a20be500bf 100644 --- a/sdk/core/ci.yml +++ b/sdk/core/ci.yml @@ -10,6 +10,24 @@ trigger: include: - sdk/core/ +parameters: +- name: release_azurecore + displayName: 'azure_core' + type: boolean + default: true +- name: release_azurecoremacros + displayName: 'azure_core_macros' + type: boolean + default: true +- name: release_azurecoreamqp + displayName: 'azure_core_amqp' + type: boolean + default: true +- name: release_azurecoreopentelemetry + displayName: 'azure_core_opentelemetry' + type: boolean + default: true + extends: template: /eng/pipelines/templates/stages/archetype-sdk-client.yml parameters: @@ -23,9 +41,13 @@ extends: safeName: TypespecClientCore - name: azure_core safeName: AzureCore + releaseInBatch: ${{ parameters.release_azurecore }} - name: azure_core_macros safeName: AzureCoreMacros + releaseInBatch: ${{ parameters.release_azurecoremacros }} - name: azure_core_amqp safeName: AzureCoreAmqp + releaseInBatch: ${{ parameters.release_azurecoreamqp }} - name: azure_core_opentelemetry safeName: AzureCoreOpentelemetry + releaseInBatch: ${{ parameters.release_azurecoreopentelemetry }} diff --git a/sdk/cosmos/ci.yml b/sdk/cosmos/ci.yml index d5509e143a..929405cdc2 100644 --- a/sdk/cosmos/ci.yml +++ b/sdk/cosmos/ci.yml @@ -10,6 +10,12 @@ trigger: include: - sdk/cosmos/ +parameters: +- name: release_azuredatacosmos + displayName: 'azure_data_cosmos' + type: boolean + default: true + extends: template: /eng/pipelines/templates/stages/archetype-sdk-client.yml parameters: @@ -17,3 +23,4 @@ extends: Artifacts: - name: azure_data_cosmos safeName: AzureDataCosmos + releaseInBatch: ${{ parameters.release_azuredatacosmos }} diff --git a/sdk/eventhubs/ci.yml b/sdk/eventhubs/ci.yml index 917e2a89f1..893ba8d208 100644 --- a/sdk/eventhubs/ci.yml +++ b/sdk/eventhubs/ci.yml @@ -10,6 +10,16 @@ trigger: include: - sdk/eventhubs/ +parameters: +- name: release_azuremessagingeventhubs + displayName: 'azure_messaging_eventhubs' + type: boolean + default: true +- name: release_azuremessagingeventhubscheckpointstoreblob + displayName: 'azure_messaging_eventhubs_checkpointstore_blob' + type: boolean + default: true + extends: template: /eng/pipelines/templates/stages/archetype-sdk-client.yml parameters: @@ -17,5 +27,7 @@ extends: Artifacts: - name: azure_messaging_eventhubs safeName: AzureMessagingEventHubs + releaseInBatch: ${{ parameters.release_azuremessagingeventhubs }} - name: azure_messaging_eventhubs_checkpointstore_blob safeName: AzureMessagingEventHubsBlobCheckpointStore + releaseInBatch: ${{ parameters.release_azuremessagingeventhubscheckpointstoreblob }} diff --git a/sdk/identity/ci.yml b/sdk/identity/ci.yml index 0e7a26102e..e641b876ed 100644 --- a/sdk/identity/ci.yml +++ b/sdk/identity/ci.yml @@ -3,12 +3,18 @@ trigger: branches: include: - - main - - hotfix/* - - release/* + - main + - hotfix/* + - release/* paths: include: - - sdk/identity/ + - sdk/identity/ + +parameters: +- name: release_azureidentity + displayName: 'azure_identity' + type: boolean + default: true extends: template: /eng/pipelines/templates/stages/archetype-sdk-client.yml @@ -17,6 +23,7 @@ extends: Artifacts: - name: azure_identity safeName: AzureIdentity + releaseInBatch: ${{ parameters.release_azureidentity }} ${{ if endsWith(variables['Build.DefinitionName'], 'weekly') }}: Location: uksouth diff --git a/sdk/keyvault/ci.yml b/sdk/keyvault/ci.yml index 44779bf03c..6894d4e5b6 100644 --- a/sdk/keyvault/ci.yml +++ b/sdk/keyvault/ci.yml @@ -1,10 +1,4 @@ # NOTE: Please refer to https://aka.ms/azsdk/engsys/ci-yaml before editing this file. -parameters: -- name: RunLiveTests - displayName: 'Run live tests' - type: boolean - default: false - trigger: branches: include: @@ -15,6 +9,24 @@ trigger: include: - sdk/keyvault/ +parameters: +- name: RunLiveTests + displayName: 'Run live tests' + type: boolean + default: false +- name: release_azuresecuritykeyvaultsecrets + displayName: 'azure_security_keyvault_secrets' + type: boolean + default: true +- name: release_azuresecuritykeyvaultkeys + displayName: 'azure_security_keyvault_keys' + type: boolean + default: true +- name: release_azuresecuritykeyvaultcertificates + displayName: 'azure_security_keyvault_certificates' + type: boolean + default: true + extends: template: /eng/pipelines/templates/stages/archetype-sdk-client.yml parameters: @@ -23,7 +35,10 @@ extends: Artifacts: - name: azure_security_keyvault_secrets safeName: AzureSecurityKeyvaultSecrets + releaseInBatch: ${{ parameters.release_azuresecuritykeyvaultsecrets }} - name: azure_security_keyvault_keys safeName: AzureSecurityKeyvaultKeys + releaseInBatch: ${{ parameters.release_azuresecuritykeyvaultkeys }} - name: azure_security_keyvault_certificates safeName: AzureSecurityKeyvaultCertificates + releaseInBatch: ${{ parameters.release_azuresecuritykeyvaultcertificates }} diff --git a/sdk/servicebus/ci.yml b/sdk/servicebus/ci.yml index 36c7d1c97a..9d0c667d60 100644 --- a/sdk/servicebus/ci.yml +++ b/sdk/servicebus/ci.yml @@ -3,18 +3,25 @@ trigger: branches: include: - - main - - feature/* - - release/* - - hotfix/* + - main + - feature/* + - release/* + - hotfix/* paths: include: - - sdk/servicebus/ + - sdk/servicebus/ + +parameters: +- name: release_azuremessagingservicebus + displayName: 'azure_messaging_servicebus' + type: boolean + default: true extends: template: /eng/pipelines/templates/stages/archetype-sdk-client.yml parameters: ServiceDirectory: servicebus Artifacts: - - name: azure_messaging_servicebus - safeName: AzureMessagingServiceBus + - name: azure_messaging_servicebus + safeName: AzureMessagingServiceBus + releaseInBatch: ${{ parameters.release_azuremessagingservicebus }} diff --git a/sdk/storage/ci.yml b/sdk/storage/ci.yml index bd92c09b5e..e7775989d2 100644 --- a/sdk/storage/ci.yml +++ b/sdk/storage/ci.yml @@ -1,10 +1,4 @@ # NOTE: Please refer to https://aka.ms/azsdk/engsys/ci-yaml before editing this file. -parameters: -- name: RunLiveTests - displayName: 'Run live tests' - type: boolean - default: false - trigger: branches: include: @@ -15,6 +9,24 @@ trigger: include: - sdk/storage/ +parameters: +- name: RunLiveTests + displayName: 'Run live tests' + type: boolean + default: false +- name: release_azurestoragecommon + displayName: 'azure_storage_common' + type: boolean + default: true +- name: release_azurestorageblob + displayName: 'azure_storage_blob' + type: boolean + default: true +- name: release_azurestoragequeue + displayName: 'azure_storage_queue' + type: boolean + default: true + extends: template: /eng/pipelines/templates/stages/archetype-sdk-client.yml parameters: @@ -24,7 +36,10 @@ extends: Artifacts: - name: azure_storage_common safeName: AzureStorageCommon + releaseInBatch: ${{ parameters.release_azurestoragecommon }} - name: azure_storage_blob safeName: AzureStorageBlob + releaseInBatch: ${{ parameters.release_azurestorageblob }} - name: azure_storage_queue safeName: AzureStorageQueue + releaseInBatch: ${{ parameters.release_azurestoragequeue }} From e0692f34d737926ec858c0f3f8eced575448c301 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Mon, 29 Sep 2025 03:36:36 +0000 Subject: [PATCH 42/87] Also require dependency order --- eng/scripts/Pack-Crates.ps1 | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/eng/scripts/Pack-Crates.ps1 b/eng/scripts/Pack-Crates.ps1 index 45e176c1f4..171c25cc43 100755 --- a/eng/scripts/Pack-Crates.ps1 +++ b/eng/scripts/Pack-Crates.ps1 @@ -130,12 +130,27 @@ try { [array]$packages = Get-PackagesToBuild - if ($RequireDependencies) { + if ($RequireDependencies) { $unspecifiedPackages = $packages.name | Where-Object { $_ -notin $PackageNames } if ($unspecifiedPackages.Count -gt 0) { Write-Error "Packages in -PackageNames require dependencies that are either not released or not listed for packing: $($unspecifiedPackages -join ', ')" exit 1 } + + $orderMatches = $true + for ($i = 0; $i -lt $PackageNames.Count; $i++) { + if ($packages[$i].name -ne $PackageNames[$i]) { + $orderMatches = $false + break + } + } + + if (!$orderMatches) { + Write-Host "Expected order: $($packages.name -join ', ')" + Write-Host "Provided order: $($PackageNames -join ', ')" + Write-Error "The order of packages in -PackageNames does not match the required build order." + exit 1 + } } Write-Host "Building packages in the following order:" From e1a3992d8d3fb499ea0a634406af5713f49a06df Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Mon, 29 Sep 2025 17:25:23 +0000 Subject: [PATCH 43/87] Wire up CHANGELOG.md and README.md --- eng/scripts/Language-Settings.ps1 | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index 6b1d5586be..63ad13c5c7 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -160,8 +160,10 @@ function Get-rust-PackageInfoFromPackageFile([IO.FileInfo]$pkg, [string]$working $packageName = $package.name $packageVersion = $package.version - $changeLogLoc = Get-ChildItem -Path $pkg.DirectoryName -Filter "CHANGELOG.md" | Select-Object -First 1 - $readmeContentLoc = Get-ChildItem -Path $pkg.DirectoryName -Filter "README.md" | Select-Object -First 1 + $packageAssetPath = Join-Path $extractionPath "$packageName-$packageVersion" + + $changeLogLoc = Get-ChildItem -Path $packageAssetPath -Filter "CHANGELOG.md" | Select-Object -First 1 + $readmeContentLoc = Get-ChildItem -Path $packageAssetPath -Filter "README.md" | Select-Object -First 1 if ($changeLogLoc) { $releaseNotes = Get-ChangeLogEntryAsString -ChangeLogLocation $changeLogLoc -VersionString $packageVersion From 592a67262f7bb2d1701ad94abf122bbbb614f525 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Mon, 29 Sep 2025 18:05:59 +0000 Subject: [PATCH 44/87] Test: remove package verison updating from release. Should fail Packing. --- eng/pipelines/templates/jobs/pack.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/eng/pipelines/templates/jobs/pack.yml b/eng/pipelines/templates/jobs/pack.yml index 88bff70938..95325a4cf2 100644 --- a/eng/pipelines/templates/jobs/pack.yml +++ b/eng/pipelines/templates/jobs/pack.yml @@ -41,14 +41,14 @@ jobs: parameters: Toolchain: stable - - ${{ if eq(parameters.TestPipeline, 'true') }}: - - template: /eng/common/pipelines/templates/steps/set-test-pipeline-version.yml - parameters: - PackageNames: ${{ join(',', parameters.Artifacts.*.name) }} - ServiceDirectory: ${{parameters.ServiceDirectory}} - # Crate names contain '_' and prerelease versions contain '-' e.g., `azure_core@1.0.1-beta.1`. - TagSeparator: '@' - TestPipeline: true + # - ${{ if eq(parameters.TestPipeline, 'true') }}: + # - template: /eng/common/pipelines/templates/steps/set-test-pipeline-version.yml + # parameters: + # PackageNames: ${{ join(',', parameters.Artifacts.*.name) }} + # ServiceDirectory: ${{parameters.ServiceDirectory}} + # # Crate names contain '_' and prerelease versions contain '-' e.g., `azure_core@1.0.1-beta.1`. + # TagSeparator: '@' + # TestPipeline: true - template: /eng/common/pipelines/templates/steps/set-default-branch.yml@self From 98414088eb087d6d43e2cf90591adec2634b12d3 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Mon, 29 Sep 2025 18:10:40 +0000 Subject: [PATCH 45/87] Better testing: turn off TestPipeline --- eng/pipelines/templates/jobs/pack.yml | 16 ++++++++-------- .../templates/stages/archetype-sdk-client.yml | 6 ++++-- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/eng/pipelines/templates/jobs/pack.yml b/eng/pipelines/templates/jobs/pack.yml index 95325a4cf2..88bff70938 100644 --- a/eng/pipelines/templates/jobs/pack.yml +++ b/eng/pipelines/templates/jobs/pack.yml @@ -41,14 +41,14 @@ jobs: parameters: Toolchain: stable - # - ${{ if eq(parameters.TestPipeline, 'true') }}: - # - template: /eng/common/pipelines/templates/steps/set-test-pipeline-version.yml - # parameters: - # PackageNames: ${{ join(',', parameters.Artifacts.*.name) }} - # ServiceDirectory: ${{parameters.ServiceDirectory}} - # # Crate names contain '_' and prerelease versions contain '-' e.g., `azure_core@1.0.1-beta.1`. - # TagSeparator: '@' - # TestPipeline: true + - ${{ if eq(parameters.TestPipeline, 'true') }}: + - template: /eng/common/pipelines/templates/steps/set-test-pipeline-version.yml + parameters: + PackageNames: ${{ join(',', parameters.Artifacts.*.name) }} + ServiceDirectory: ${{parameters.ServiceDirectory}} + # Crate names contain '_' and prerelease versions contain '-' e.g., `azure_core@1.0.1-beta.1`. + TagSeparator: '@' + TestPipeline: true - template: /eng/common/pipelines/templates/steps/set-default-branch.yml@self diff --git a/eng/pipelines/templates/stages/archetype-sdk-client.yml b/eng/pipelines/templates/stages/archetype-sdk-client.yml index a0b861dfd4..78c19ebcc4 100644 --- a/eng/pipelines/templates/stages/archetype-sdk-client.yml +++ b/eng/pipelines/templates/stages/archetype-sdk-client.yml @@ -88,7 +88,8 @@ extends: - ${{ if ne('false', artifact.releaseInBatch) }}: - ${{ artifact }} RequireDependencies: ${{ in(variables['Build.Reason'], 'Manual', '') }} - TestPipeline: ${{ eq(parameters.ServiceDirectory, 'canary') }} + # TODO: replace before merging: ${{ eq(parameters.ServiceDirectory, 'canary') }} + TestPipeline: false TestTimeoutInMinutes: ${{ parameters.TestTimeoutInMinutes }} TestProxy: ${{ parameters.TestProxy }} MatrixConfigs: @@ -156,5 +157,6 @@ extends: - ${{ each artifact in parameters.Artifacts }}: - ${{ if ne(artifact.releaseInBatch, 'false')}}: - ${{ artifact }} - TestPipeline: ${{ eq(parameters.ServiceDirectory, 'canary') }} + # TODO: replace before merging: ${{ eq(parameters.ServiceDirectory, 'canary') }} + TestPipeline: false PipelineArtifactName: packages From 07bfe404e6d24a11dd96abee68b426b5d4a49832 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Mon, 29 Sep 2025 19:04:58 +0000 Subject: [PATCH 46/87] Undo test changes --- eng/pipelines/templates/stages/archetype-sdk-client.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/eng/pipelines/templates/stages/archetype-sdk-client.yml b/eng/pipelines/templates/stages/archetype-sdk-client.yml index 78c19ebcc4..fdb05ee4d9 100644 --- a/eng/pipelines/templates/stages/archetype-sdk-client.yml +++ b/eng/pipelines/templates/stages/archetype-sdk-client.yml @@ -88,8 +88,7 @@ extends: - ${{ if ne('false', artifact.releaseInBatch) }}: - ${{ artifact }} RequireDependencies: ${{ in(variables['Build.Reason'], 'Manual', '') }} - # TODO: replace before merging: ${{ eq(parameters.ServiceDirectory, 'canary') }} - TestPipeline: false + TestPipeline: ${{ eq(parameters.ServiceDirectory, 'canary') }} TestTimeoutInMinutes: ${{ parameters.TestTimeoutInMinutes }} TestProxy: ${{ parameters.TestProxy }} MatrixConfigs: @@ -157,6 +156,5 @@ extends: - ${{ each artifact in parameters.Artifacts }}: - ${{ if ne(artifact.releaseInBatch, 'false')}}: - ${{ artifact }} - # TODO: replace before merging: ${{ eq(parameters.ServiceDirectory, 'canary') }} - TestPipeline: false + TestPipeline: ${{ eq(parameters.ServiceDirectory, 'canary') }} PipelineArtifactName: packages From 7a76bfbd8a8232b3fc8d6390e499d7c3ddf25ee1 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Mon, 29 Sep 2025 19:38:34 +0000 Subject: [PATCH 47/87] Move RequireDependencies into pack.yml --- eng/pipelines/templates/stages/archetype-sdk-client.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/eng/pipelines/templates/stages/archetype-sdk-client.yml b/eng/pipelines/templates/stages/archetype-sdk-client.yml index fdb05ee4d9..11dea7dcdb 100644 --- a/eng/pipelines/templates/stages/archetype-sdk-client.yml +++ b/eng/pipelines/templates/stages/archetype-sdk-client.yml @@ -87,7 +87,6 @@ extends: - ${{ each artifact in parameters.Artifacts }}: - ${{ if ne('false', artifact.releaseInBatch) }}: - ${{ artifact }} - RequireDependencies: ${{ in(variables['Build.Reason'], 'Manual', '') }} TestPipeline: ${{ eq(parameters.ServiceDirectory, 'canary') }} TestTimeoutInMinutes: ${{ parameters.TestTimeoutInMinutes }} TestProxy: ${{ parameters.TestProxy }} From b47a18dc7674523b20cf8618e515ea05f869f282 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Mon, 29 Sep 2025 19:39:02 +0000 Subject: [PATCH 48/87] Move RequireDependencies into pack.yml --- eng/pipelines/templates/jobs/ci.yml | 3 --- eng/pipelines/templates/jobs/pack.yml | 36 +++++++++++++++++++-------- 2 files changed, 26 insertions(+), 13 deletions(-) diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index c0320a94c4..9bdaef0601 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -3,8 +3,6 @@ parameters: type: string - name: Artifacts type: object -- name: RequireDependencies - type: boolean - name: PipelineArtifactName type: string - name: TestPipeline @@ -26,7 +24,6 @@ jobs: parameters: ServiceDirectory: ${{ parameters.ServiceDirectory }} Artifacts: ${{ parameters.Artifacts }} - RequireDependencies: ${{ parameters.RequireDependencies }} TestPipeline: ${{ parameters.TestPipeline }} PipelineArtifactName: ${{ parameters.PipelineArtifactName }} diff --git a/eng/pipelines/templates/jobs/pack.yml b/eng/pipelines/templates/jobs/pack.yml index 88bff70938..c3467e6011 100644 --- a/eng/pipelines/templates/jobs/pack.yml +++ b/eng/pipelines/templates/jobs/pack.yml @@ -20,6 +20,10 @@ jobs: image: $(LINUXVMIMAGE) os: linux + variables: + # Only require dependency correctness in manual builds + - RequireDependencies: ${{ in(variables['Build.Reason'], 'Manual', '') }} + steps: - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml parameters: @@ -57,16 +61,28 @@ jobs: ServiceDirectory: ${{ parameters.ServiceDirectory }} PackageInfoDirectory: $(Build.ArtifactStagingDirectory)/PackageInfo - - task: Powershell@2 - displayName: "Pack Crates" - condition: and(succeeded(), ne(variables['NoPackagesChanged'],'true')) - inputs: - pwsh: true - filePath: $(Build.SourcesDirectory)/eng/scripts/Pack-Crates.ps1 - arguments: > - -OutputPath '$(Build.ArtifactStagingDirectory)' - -PackageNames ${{ join(',', parameters.Artifacts.*.name) }} - -RequireDependencies:$${{ parameters.RequireDependencies }} + - ${{ if eq('auto', parameters.ServiceDirectory) }}: + - task: Powershell@2 + displayName: "Pack Crates (PR build)" + condition: and(succeeded(), ne(variables['NoPackagesChanged'],'true')) + inputs: + pwsh: true + filePath: $(Build.SourcesDirectory)/eng/scripts/Pack-Crates.ps1 + arguments: > + -OutputPath '$(Build.ArtifactStagingDirectory)' + -PackageInfoDirectory '$(Build.ArtifactStagingDirectory)/PackageInfo' + + - ${{ else }}: + - task: Powershell@2 + displayName: "Pack Crates" + condition: and(succeeded(), ne(variables['NoPackagesChanged'],'true')) + inputs: + pwsh: true + filePath: $(Build.SourcesDirectory)/eng/scripts/Pack-Crates.ps1 + arguments: > + -OutputPath '$(Build.ArtifactStagingDirectory)' + -PackageNames ${{ join(',', parameters.Artifacts.*.name) }} + -RequireDependencies:$$(RequireDependencies) # TODO: Ensure APIView works given a change to the crates output folder structure - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml From 76e13906f07377ac3704fc0e7131fcda7afb1202 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Mon, 29 Sep 2025 19:41:27 +0000 Subject: [PATCH 49/87] Pack-Crates.ps1: Use PackageInfoDirectory --- eng/scripts/Pack-Crates.ps1 | 42 +++++++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/eng/scripts/Pack-Crates.ps1 b/eng/scripts/Pack-Crates.ps1 index 171c25cc43..0e1fb04d77 100755 --- a/eng/scripts/Pack-Crates.ps1 +++ b/eng/scripts/Pack-Crates.ps1 @@ -4,8 +4,12 @@ [CmdletBinding(DefaultParameterSetName = "none")] param( [string]$OutputPath, + [Parameter(ParameterSetName = 'Named')] [string[]]$PackageNames, + [Parameter(ParameterSetName = 'Named')] [switch]$RequireDependencies, + [Parameter(ParameterSetName = 'PackageInfo')] + [string]$PackageInfoDirectory, [switch]$NoVerify ) @@ -22,6 +26,39 @@ if ($OutputPath) { $OutputPath = New-Item -ItemType Directory -Path $OutputPath -Force | Select-Object -ExpandProperty FullName } +function Get-OutputPackageNames($workspacePackages) { + $packablePackages = $workspacePackages | Where-Object -Property publish -NE -Value @() + $packablePackageNames = $packablePackages.name + + $names = @() + switch ($PsCmdlet.ParameterSetName) { + 'Named' { + $names = $PackageNames + } + + 'PackageInfo' { + $packageInfoFiles = Get-ChildItem -Path $PackageInfoDirectory -Filter '*.json' -File + foreach ($packageInfoFile in $packageInfoFiles) { + $packageInfo = Get-Content -Path $packageInfoFile.FullName | ConvertFrom-Json + $names += $packageInfo.name + } + } + + default { + return $packablePackageNames + } + } + + foreach ($name in $names) { + if (-not $packablePackageNames.Contains($name)) { + Write-Error "Package '$name' is not in the workspace or does not publish" + exit 1 + } + } + + return $names +} + function Get-CargoMetadata() { cargo metadata --no-deps --format-version 1 --manifest-path "$RepoRoot/Cargo.toml" | ConvertFrom-Json -Depth 100 -AsHashtable } @@ -46,9 +83,10 @@ function Get-CargoPackages() { function Get-PackagesToBuild() { $packages = Get-CargoPackages + $outputPackageNames = Get-OutputPackageNames $packages # We start with output packages, then recursively add unreleased dependencies to the list of packages that need to be built - [array]$packagesToBuild = $packages | Where-Object { $PackageNames.Contains($_.name) } + [array]$packagesToBuild = $packages | Where-Object { $outputPackageNames.Contains($_.name) } $toProcess = $packagesToBuild while ($toProcess.Length -gt 0) { @@ -82,7 +120,7 @@ function Get-PackagesToBuild() { exit 1 } - $package.OutputPackage = $PackageNames.Contains($package.name) + $package.OutputPackage = $outputPackageNames.Contains($package.name) $buildOrder += $package $packagesToBuild = @($packagesToBuild -ne $package) From ac8e1a05d8a4940007c0f8062c0dce0b140ff136 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Mon, 29 Sep 2025 19:46:53 +0000 Subject: [PATCH 50/87] -RequireDependencies --- eng/pipelines/templates/jobs/pack.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/eng/pipelines/templates/jobs/pack.yml b/eng/pipelines/templates/jobs/pack.yml index c3467e6011..e9f04963b3 100644 --- a/eng/pipelines/templates/jobs/pack.yml +++ b/eng/pipelines/templates/jobs/pack.yml @@ -3,8 +3,6 @@ parameters: type: string - name: Artifacts type: object -- name: RequireDependencies - type: boolean - name: TestPipeline type: boolean - name: PipelineArtifactName From 60893c18f70f5e22b7ba84e632e39b1e8cdb065f Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Mon, 29 Sep 2025 19:48:35 +0000 Subject: [PATCH 51/87] Syntax --- eng/pipelines/templates/jobs/pack.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/jobs/pack.yml b/eng/pipelines/templates/jobs/pack.yml index e9f04963b3..aadabba5e0 100644 --- a/eng/pipelines/templates/jobs/pack.yml +++ b/eng/pipelines/templates/jobs/pack.yml @@ -20,7 +20,7 @@ jobs: variables: # Only require dependency correctness in manual builds - - RequireDependencies: ${{ in(variables['Build.Reason'], 'Manual', '') }} + RequireDependencies: ${{ in(variables['Build.Reason'], 'Manual', '') }} steps: - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml From 35260275777dfa2ecb9d6eb421f44ee8cd1e3a5a Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Mon, 29 Sep 2025 19:58:38 +0000 Subject: [PATCH 52/87] Formatting/cleanup --- eng/pipelines/templates/jobs/ci.yml | 1 - .../templates/stages/archetype-rust-release.yml | 1 - sdk/canary/azure_canary_core/Cargo.toml | 2 ++ sdk/canary/ci.yml | 16 ++++++++-------- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index 9bdaef0601..0cb6c2a9d5 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -18,7 +18,6 @@ parameters: - name: MatrixReplace type: object - jobs: - template: /eng/pipelines/templates/jobs/pack.yml parameters: diff --git a/eng/pipelines/templates/stages/archetype-rust-release.yml b/eng/pipelines/templates/stages/archetype-rust-release.yml index 8b816dbd50..355a432217 100644 --- a/eng/pipelines/templates/stages/archetype-rust-release.yml +++ b/eng/pipelines/templates/stages/archetype-rust-release.yml @@ -33,7 +33,6 @@ stages: os: linux jobs: - # TODO: Ensure that CHANGELOG is properly processed - job: TagRepository displayName: "Create release tag" condition: and(succeeded(), ne(variables['Skip.TagRepository'], 'true')) diff --git a/sdk/canary/azure_canary_core/Cargo.toml b/sdk/canary/azure_canary_core/Cargo.toml index 5c6c490964..18c2d595a9 100644 --- a/sdk/canary/azure_canary_core/Cargo.toml +++ b/sdk/canary/azure_canary_core/Cargo.toml @@ -8,6 +8,8 @@ license.workspace = true repository.workspace = true rust-version.workspace = true +[dependencies] + [dev-dependencies] azure_canary.path = "../azure_canary" diff --git a/sdk/canary/ci.yml b/sdk/canary/ci.yml index 9a7eb64742..7e65a7366f 100644 --- a/sdk/canary/ci.yml +++ b/sdk/canary/ci.yml @@ -11,14 +11,14 @@ trigger: - sdk/canary/ parameters: - - name: release_azurecanarycore - displayName: 'azure_canary_core' - type: boolean - default: true - - name: release_azurecanary - displayName: 'azure_canary' - type: boolean - default: true +- name: release_azurecanarycore + displayName: 'azure_canary_core' + type: boolean + default: true +- name: release_azurecanary + displayName: 'azure_canary' + type: boolean + default: true extends: template: /eng/pipelines/templates/stages/archetype-sdk-client.yml From 5d61c2fde53f40d61d31e444c4b8d842f001d855 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Mon, 29 Sep 2025 20:47:39 +0000 Subject: [PATCH 53/87] cspell: allowCompoundWords: true --- .vscode/cspell.json | 70 +++++++++------------------------------------ 1 file changed, 14 insertions(+), 56 deletions(-) diff --git a/.vscode/cspell.json b/.vscode/cspell.json index 29996f7f5f..89a15d4ffb 100644 --- a/.vscode/cspell.json +++ b/.vscode/cspell.json @@ -2,6 +2,7 @@ "version": "0.2", "language": "en", "useGitignore": true, + "allowCompoundWords": true, "ignorePaths": [ "**/test-resources.bicep", "**/test-resources.json", @@ -146,91 +147,48 @@ "overrides": [ { "filename": "**/Cargo.toml", - "dictionaries": [ - "crates", - "rust-custom" - ] + "dictionaries": ["crates", "rust-custom"] }, { "filename": "**/*.md", - "dictionaries": [ - "crates", - "rust", - "rust-custom" - ], - "ignoreRegExpList": [ - "@\\w+" - ] + "dictionaries": ["crates", "rust", "rust-custom"], + "ignoreRegExpList": ["@\\w+"] }, { "filename": "**/*.rs", - "dictionaries": [ - "crates", - "rust-custom" - ] + "dictionaries": ["crates", "rust-custom"] }, { "filename": "sdk/core/azure_core_amqp/**", - "dictionaries": [ - "crates", - "rust-custom", - "azure_core_amqp" - ] + "dictionaries": ["crates", "rust-custom", "azure_core_amqp"] }, { "filename": "sdk/eventhubs/**", - "dictionaries": [ - "crates", - "rust-custom", - "eventhubs" - ] + "dictionaries": ["crates", "rust-custom", "eventhubs"] }, { "filename": "sdk/identity/**", - "dictionaries": [ - "crates", - "rust-custom", - "identity" - ] + "dictionaries": ["crates", "rust-custom", "identity"] }, { "filename": "sdk/keyvault/**", - "dictionaries": [ - "crates", - "rust-custom", - "keyvault" - ] + "dictionaries": ["crates", "rust-custom", "keyvault"] }, { "filename": "sdk/core/typespec*/**", - "flagWords": [ - "azure", - "azurite" - ] + "flagWords": ["azure", "azurite"] }, { "filename": "sdk/cosmos/**", - "dictionaries": [ - "crates", - "rust-custom", - "cosmos" - ] + "dictionaries": ["crates", "rust-custom", "cosmos"] }, { "filename": "sdk/servicebus/**", - "dictionaries": [ - "crates", - "rust-custom", - "servicebus" - ] + "dictionaries": ["crates", "rust-custom", "servicebus"] }, { "filename": "sdk/storage/**", - "dictionaries": [ - "crates", - "rust-custom", - "storage" - ] + "dictionaries": ["crates", "rust-custom", "storage"] } ] -} \ No newline at end of file +} From b9ca6a81327d16edeea7851cee929532d11dad3d Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Mon, 29 Sep 2025 22:59:54 +0000 Subject: [PATCH 54/87] Spelling --- .vscode/cspell.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.vscode/cspell.json b/.vscode/cspell.json index 89a15d4ffb..62aea9d8b9 100644 --- a/.vscode/cspell.json +++ b/.vscode/cspell.json @@ -189,6 +189,10 @@ { "filename": "sdk/storage/**", "dictionaries": ["crates", "rust-custom", "storage"] + }, + { + "filename": "sdk/*/ci.yml", + "ignoreWords": ["azurecoreamqp"] } ] } From c6a900cda9bd92be1443471fb04c3584a08987eb Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Wed, 1 Oct 2025 03:03:45 +0000 Subject: [PATCH 55/87] Review feedback: No artifacts selected means "just build all artifacts in the service and run live tests" --- eng/pipelines/templates/jobs/pack.yml | 24 +++++++++++++++---- .../templates/stages/archetype-sdk-client.yml | 5 +--- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/eng/pipelines/templates/jobs/pack.yml b/eng/pipelines/templates/jobs/pack.yml index aadabba5e0..b78ff332de 100644 --- a/eng/pipelines/templates/jobs/pack.yml +++ b/eng/pipelines/templates/jobs/pack.yml @@ -18,10 +18,6 @@ jobs: image: $(LINUXVMIMAGE) os: linux - variables: - # Only require dependency correctness in manual builds - RequireDependencies: ${{ in(variables['Build.Reason'], 'Manual', '') }} - steps: - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml parameters: @@ -71,6 +67,24 @@ jobs: -PackageInfoDirectory '$(Build.ArtifactStagingDirectory)/PackageInfo' - ${{ else }}: + - pwsh: | + $artifacts = '${{ toJson(parameters.Artifacts) }}' | ConvertFrom-Json + $requireDependencies = $true + $artifactsToBuild = $artifacts | Where-Object { $_.releaseInBatch } + + if (!$artifactsToBuild) { + Write-Host "No packages to release. Building all packages in the service directory with no dependency validation." + $artifactsToBuild = $artifacts + $requireDependencies = $false + } + + $packageNames = $artifactsToBuild.name -join ',' + + Write-Host "##vso[task.setvariable variable=PackageNames]$packageNames" + Write-Host "##vso[task.setvariable variable=RequireDependencies]$requireDependencies" + + displayName: Create package list + - task: Powershell@2 displayName: "Pack Crates" condition: and(succeeded(), ne(variables['NoPackagesChanged'],'true')) @@ -79,7 +93,7 @@ jobs: filePath: $(Build.SourcesDirectory)/eng/scripts/Pack-Crates.ps1 arguments: > -OutputPath '$(Build.ArtifactStagingDirectory)' - -PackageNames ${{ join(',', parameters.Artifacts.*.name) }} + -PackageNames $(PackageNames) -RequireDependencies:$$(RequireDependencies) # TODO: Ensure APIView works given a change to the crates output folder structure diff --git a/eng/pipelines/templates/stages/archetype-sdk-client.yml b/eng/pipelines/templates/stages/archetype-sdk-client.yml index 11dea7dcdb..4f1d8ee413 100644 --- a/eng/pipelines/templates/stages/archetype-sdk-client.yml +++ b/eng/pipelines/templates/stages/archetype-sdk-client.yml @@ -83,10 +83,7 @@ extends: parameters: ServiceDirectory: ${{ parameters.ServiceDirectory }} PipelineArtifactName: packages - Artifacts: - - ${{ each artifact in parameters.Artifacts }}: - - ${{ if ne('false', artifact.releaseInBatch) }}: - - ${{ artifact }} + Artifacts: ${{ parameters.Artifacts }} TestPipeline: ${{ eq(parameters.ServiceDirectory, 'canary') }} TestTimeoutInMinutes: ${{ parameters.TestTimeoutInMinutes }} TestProxy: ${{ parameters.TestProxy }} From 5ae05c60bb69483b6e7ccf7fc10f7a6bd143145a Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Wed, 1 Oct 2025 03:06:47 +0000 Subject: [PATCH 56/87] convertToJson --- eng/pipelines/templates/jobs/pack.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/jobs/pack.yml b/eng/pipelines/templates/jobs/pack.yml index b78ff332de..4ebeaf6620 100644 --- a/eng/pipelines/templates/jobs/pack.yml +++ b/eng/pipelines/templates/jobs/pack.yml @@ -68,7 +68,7 @@ jobs: - ${{ else }}: - pwsh: | - $artifacts = '${{ toJson(parameters.Artifacts) }}' | ConvertFrom-Json + $artifacts = '${{ convertToJson(parameters.Artifacts) }}' | ConvertFrom-Json $requireDependencies = $true $artifactsToBuild = $artifacts | Where-Object { $_.releaseInBatch } From 2b51b99dc80ed204c111058f2a1fc12083c01e7f Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Wed, 1 Oct 2025 03:10:43 +0000 Subject: [PATCH 57/87] Only create the stage if there are artifacts to release --- .../stages/archetype-rust-release.yml | 321 +++++++++--------- 1 file changed, 161 insertions(+), 160 deletions(-) diff --git a/eng/pipelines/templates/stages/archetype-rust-release.yml b/eng/pipelines/templates/stages/archetype-rust-release.yml index 355a432217..1de7ec0860 100644 --- a/eng/pipelines/templates/stages/archetype-rust-release.yml +++ b/eng/pipelines/templates/stages/archetype-rust-release.yml @@ -18,167 +18,168 @@ parameters: stages: - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - ${{ if in(variables['Build.Reason'], 'Manual', '') }}: - - stage: Release_Batch - displayName: "Releasing: ${{length(parameters.Artifacts)}} crates" - dependsOn: ${{parameters.DependsOn}} - condition: and(succeeded(), ne(variables['SetDevVersion'], 'true'), ne(variables['Skip.Release'], 'true'), ne(variables['Build.Repository.Name'], 'Azure/azure-sdk-for-rust-pr')) - variables: - - template: /eng/pipelines/templates/variables/globals.yml - - template: /eng/pipelines/templates/variables/image.yml - - template: /eng/pipelines/templates/variables/rust.yml - - pool: - name: $(LINUXPOOL) - image: $(LINUXVMIMAGE) - os: linux - - jobs: - - job: TagRepository - displayName: "Create release tag" - condition: and(succeeded(), ne(variables['Skip.TagRepository'], 'true')) - - steps: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - - - download: current - displayName: Download ${{parameters.PipelineArtifactName}} artifact - artifact: ${{parameters.PipelineArtifactName}} - - - template: /eng/common/pipelines/templates/steps/retain-run.yml - - - ${{ each artifact in parameters.Artifacts }}: - - script: | - echo "##vso[build.addbuildtag]${{artifact.name}}" - displayName: Add build tag '${{artifact.name}}' - - - template: /eng/common/pipelines/templates/steps/create-tags-and-git-release.yml - parameters: - ArtifactLocation: $(Pipeline.Workspace)/${{parameters.PipelineArtifactName}}/${{artifact.name}} - PackageRepository: Crates.io - ReleaseSha: $(Build.SourceVersion) - WorkingDirectory: $(Pipeline.Workspace)/_work - - - deployment: PublishPackage - displayName: "Publish to Crates.io" - condition: and(succeeded(), ne(variables['Skip.PublishPackage'], 'true')) - templateContext: - type: releaseJob # Required, this indicates this deployment job is a release job - isProduction: true # Required, must be 'true' or 'false' - inputs: # All input build artifacts must be declared here - - input: pipelineArtifact # Required, type of the input artifact - artifactName: ${{parameters.PipelineArtifactName}} # Required, name of the pipeline artifact - targetPath: $(Pipeline.Workspace)/drop # Optional, specifies where the artifact is downloaded to - ${{if parameters.TestPipeline}}: - environment: none - ${{else}}: - environment: cratesio - # This timeout shouldn't be necessary once we're able to parallelize better. Right now, - # this is here to ensure larger areas (30+) libraries don't time out. - timeoutInMinutes: 120 - dependsOn: TagRepository + - ${{ if gt(length(parameters.Artifacts), 0) }}: + - stage: Release_Batch + displayName: "Releasing: ${{length(parameters.Artifacts)}} crates" + dependsOn: ${{parameters.DependsOn}} + condition: and(succeeded(), ne(variables['SetDevVersion'], 'true'), ne(variables['Skip.Release'], 'true'), ne(variables['Build.Repository.Name'], 'Azure/azure-sdk-for-rust-pr')) + variables: + - template: /eng/pipelines/templates/variables/globals.yml + - template: /eng/pipelines/templates/variables/image.yml + - template: /eng/pipelines/templates/variables/rust.yml + pool: - name: azsdk-pool - image: ubuntu-24.04 + name: $(LINUXPOOL) + image: $(LINUXVMIMAGE) os: linux - strategy: - runOnce: - deploy: - steps: - - ${{ each artifact in parameters.Artifacts }}: - - pwsh: | - $ArtifactName = '${{artifact.name}}' - $ArtifactRootPath = '$(Pipeline.Workspace)/drop' - $OutDir = '$(Pipeline.Workspace)/esrp-release' - - if (Test-Path $OutDir) { - Write-Host "Cleaning output directory: $OutDir" - Remove-Item -Path $OutDir -Recurse -Force - } - New-Item -ItemType Directory -Path $OutDir -Force | Out-Null - - Write-Host "Artifact name: $ArtifactName" - - $packageMetadataPath = "$ArtifactRootPath/PackageInfo/$ArtifactName.json" - if (!(Test-Path $packageMetadataPath)) { - Write-Error "Package metadata file not found: $packageMetadataPath" - exit 1 - } - - $packageMetadata = Get-Content -Raw $packageMetadataPath | ConvertFrom-Json - $packageVersion = $packageMetadata.version - Write-Host "Package version: $packageVersion" - - $cratePath = "$ArtifactRootPath/$ArtifactName/$ArtifactName-$packageVersion.crate" - Copy-Item ` - -Path $cratePath ` - -Destination $OutDir - Write-Host "Contents of $OutDir" - Get-ChildItem -Path $OutDir | ForEach-Object { Write-Host $_.FullName } - displayName: 'Copy crate for ESRP: ${{artifact.name}}' - - - task: EsrpRelease@10 - displayName: 'ESRP Release: ${{artifact.name}}' - inputs: - connectedservicename: 'Azure SDK PME Managed Identity' - ClientId: '5f81938c-2544-4f1f-9251-dd9de5b8a81b' - DomainTenantId: '975f013f-7f24-47e8-a7d3-abc4752bf346' - Usemanagedidentity: true - KeyVaultName: 'kv-azuresdk-codesign' - SignCertName: 'azure-sdk-esrp-release-certificate' - intent: 'packagedistribution' - contenttype: 'Rust' - contentsource: 'Folder' - folderlocation: '$(Pipeline.Workspace)/esrp-release' - waitforreleasecompletion: true - owners: ${{ coalesce(variables['Build.RequestedForEmail'], 'azuresdk@microsoft.com') }} - approvers: ${{ coalesce(variables['Build.RequestedForEmail'], 'azuresdk@microsoft.com') }} - serviceendpointurl: 'https://api.esrp.microsoft.com/' - mainpublisher: 'ESRPRELPACMANTEST' - - - job: UpdatePackageVersion - displayName: "API Review and Package Version Update" - condition: and(succeeded(), ne(variables['Skip.UpdatePackageVersion'], 'true')) - dependsOn: PublishPackage - steps: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - paths: - - "/*" - - - template: /eng/pipelines/templates/steps/use-rust.yml@self - parameters: - Toolchain: nightly - - - download: current - displayName: Download ${{parameters.PipelineArtifactName}} artifact - artifact: ${{parameters.PipelineArtifactName}} - - - ${{each artifact in parameters.Artifacts }}: - - template: /eng/common/pipelines/templates/steps/create-apireview.yml + + jobs: + - job: TagRepository + displayName: "Create release tag" + condition: and(succeeded(), ne(variables['Skip.TagRepository'], 'true')) + + steps: + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + + - download: current + displayName: Download ${{parameters.PipelineArtifactName}} artifact + artifact: ${{parameters.PipelineArtifactName}} + + - template: /eng/common/pipelines/templates/steps/retain-run.yml + + - ${{ each artifact in parameters.Artifacts }}: + - script: | + echo "##vso[build.addbuildtag]${{artifact.name}}" + displayName: Add build tag '${{artifact.name}}' + + - template: /eng/common/pipelines/templates/steps/create-tags-and-git-release.yml + parameters: + ArtifactLocation: $(Pipeline.Workspace)/${{parameters.PipelineArtifactName}}/${{artifact.name}} + PackageRepository: Crates.io + ReleaseSha: $(Build.SourceVersion) + WorkingDirectory: $(Pipeline.Workspace)/_work + + - deployment: PublishPackage + displayName: "Publish to Crates.io" + condition: and(succeeded(), ne(variables['Skip.PublishPackage'], 'true')) + templateContext: + type: releaseJob # Required, this indicates this deployment job is a release job + isProduction: true # Required, must be 'true' or 'false' + inputs: # All input build artifacts must be declared here + - input: pipelineArtifact # Required, type of the input artifact + artifactName: ${{parameters.PipelineArtifactName}} # Required, name of the pipeline artifact + targetPath: $(Pipeline.Workspace)/drop # Optional, specifies where the artifact is downloaded to + ${{if parameters.TestPipeline}}: + environment: none + ${{else}}: + environment: cratesio + # This timeout shouldn't be necessary once we're able to parallelize better. Right now, + # this is here to ensure larger areas (30+) libraries don't time out. + timeoutInMinutes: 120 + dependsOn: TagRepository + pool: + name: azsdk-pool + image: ubuntu-24.04 + os: linux + strategy: + runOnce: + deploy: + steps: + - ${{ each artifact in parameters.Artifacts }}: + - pwsh: | + $ArtifactName = '${{artifact.name}}' + $ArtifactRootPath = '$(Pipeline.Workspace)/drop' + $OutDir = '$(Pipeline.Workspace)/esrp-release' + + if (Test-Path $OutDir) { + Write-Host "Cleaning output directory: $OutDir" + Remove-Item -Path $OutDir -Recurse -Force + } + New-Item -ItemType Directory -Path $OutDir -Force | Out-Null + + Write-Host "Artifact name: $ArtifactName" + + $packageMetadataPath = "$ArtifactRootPath/PackageInfo/$ArtifactName.json" + if (!(Test-Path $packageMetadataPath)) { + Write-Error "Package metadata file not found: $packageMetadataPath" + exit 1 + } + + $packageMetadata = Get-Content -Raw $packageMetadataPath | ConvertFrom-Json + $packageVersion = $packageMetadata.version + Write-Host "Package version: $packageVersion" + + $cratePath = "$ArtifactRootPath/$ArtifactName/$ArtifactName-$packageVersion.crate" + Copy-Item ` + -Path $cratePath ` + -Destination $OutDir + Write-Host "Contents of $OutDir" + Get-ChildItem -Path $OutDir | ForEach-Object { Write-Host $_.FullName } + displayName: 'Copy crate for ESRP: ${{artifact.name}}' + + - task: EsrpRelease@10 + displayName: 'ESRP Release: ${{artifact.name}}' + inputs: + connectedservicename: 'Azure SDK PME Managed Identity' + ClientId: '5f81938c-2544-4f1f-9251-dd9de5b8a81b' + DomainTenantId: '975f013f-7f24-47e8-a7d3-abc4752bf346' + Usemanagedidentity: true + KeyVaultName: 'kv-azuresdk-codesign' + SignCertName: 'azure-sdk-esrp-release-certificate' + intent: 'packagedistribution' + contenttype: 'Rust' + contentsource: 'Folder' + folderlocation: '$(Pipeline.Workspace)/esrp-release' + waitforreleasecompletion: true + owners: ${{ coalesce(variables['Build.RequestedForEmail'], 'azuresdk@microsoft.com') }} + approvers: ${{ coalesce(variables['Build.RequestedForEmail'], 'azuresdk@microsoft.com') }} + serviceendpointurl: 'https://api.esrp.microsoft.com/' + mainpublisher: 'ESRPRELPACMANTEST' + + - job: UpdatePackageVersion + displayName: "API Review and Package Version Update" + condition: and(succeeded(), ne(variables['Skip.UpdatePackageVersion'], 'true')) + dependsOn: PublishPackage + steps: + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml parameters: - ArtifactPath: $(Pipeline.Workspace)/${{parameters.PipelineArtifactName}} - Artifacts: ${{parameters.Artifacts}} - ConfigFileDir: $(Pipeline.Workspace)/${{parameters.PipelineArtifactName}}/PackageInfo - MarkPackageAsShipped: true - ArtifactName: ${{parameters.PipelineArtifactName}} - SourceRootPath: $(System.DefaultWorkingDirectory) - PackageName: ${{artifact.name}} - - # Apply the version increment to each library, which updates the Cargo.toml and changelog files. - - task: PowerShell@2 - displayName: Increment ${{artifact.name}} version - inputs: - targetType: filePath - filePath: $(Build.SourcesDirectory)/eng/scripts/Update-PackageVersion.ps1 - arguments: > - -ServiceDirectory '${{parameters.ServiceDirectory}}' - -PackageName '${{artifact.name}}' - - - template: /eng/common/pipelines/templates/steps/create-pull-request.yml + paths: + - "/*" + + - template: /eng/pipelines/templates/steps/use-rust.yml@self parameters: - PRBranchName: increment-package-version-${{parameters.ServiceDirectory}}-$(Build.BuildId) - CommitMsg: "Increment package version after release of ${{ artifact.name }}" - PRTitle: "Increment versions for ${{parameters.ServiceDirectory}} releases" - CloseAfterOpenForTesting: '${{parameters.TestPipeline}}' - ${{ if startsWith(variables['Build.SourceBranch'], 'refs/pull/') }}: - BaseBranchName: main + Toolchain: nightly + + - download: current + displayName: Download ${{parameters.PipelineArtifactName}} artifact + artifact: ${{parameters.PipelineArtifactName}} + + - ${{each artifact in parameters.Artifacts }}: + - template: /eng/common/pipelines/templates/steps/create-apireview.yml + parameters: + ArtifactPath: $(Pipeline.Workspace)/${{parameters.PipelineArtifactName}} + Artifacts: ${{parameters.Artifacts}} + ConfigFileDir: $(Pipeline.Workspace)/${{parameters.PipelineArtifactName}}/PackageInfo + MarkPackageAsShipped: true + ArtifactName: ${{parameters.PipelineArtifactName}} + SourceRootPath: $(System.DefaultWorkingDirectory) + PackageName: ${{artifact.name}} + + # Apply the version increment to each library, which updates the Cargo.toml and changelog files. + - task: PowerShell@2 + displayName: Increment ${{artifact.name}} version + inputs: + targetType: filePath + filePath: $(Build.SourcesDirectory)/eng/scripts/Update-PackageVersion.ps1 + arguments: > + -ServiceDirectory '${{parameters.ServiceDirectory}}' + -PackageName '${{artifact.name}}' + + - template: /eng/common/pipelines/templates/steps/create-pull-request.yml + parameters: + PRBranchName: increment-package-version-${{parameters.ServiceDirectory}}-$(Build.BuildId) + CommitMsg: "Increment package version after release of ${{ artifact.name }}" + PRTitle: "Increment versions for ${{parameters.ServiceDirectory}} releases" + CloseAfterOpenForTesting: '${{parameters.TestPipeline}}' + ${{ if startsWith(variables['Build.SourceBranch'], 'refs/pull/') }}: + BaseBranchName: main From 71f05861bbc012aab7feeb9c89b627f397f688d7 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Wed, 1 Oct 2025 03:23:37 +0000 Subject: [PATCH 58/87] 'False' --- eng/pipelines/templates/jobs/pack.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/eng/pipelines/templates/jobs/pack.yml b/eng/pipelines/templates/jobs/pack.yml index 4ebeaf6620..e64a88d349 100644 --- a/eng/pipelines/templates/jobs/pack.yml +++ b/eng/pipelines/templates/jobs/pack.yml @@ -70,7 +70,9 @@ jobs: - pwsh: | $artifacts = '${{ convertToJson(parameters.Artifacts) }}' | ConvertFrom-Json $requireDependencies = $true - $artifactsToBuild = $artifacts | Where-Object { $_.releaseInBatch } + $artifactsToBuild = $artifacts | Where-Object { $_.releaseInBatch -ne 'False' } + + if (!$artifactsToBuild) { Write-Host "No packages to release. Building all packages in the service directory with no dependency validation." From 19cbea109705f9272b403cf8048254bdb32bafc3 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Wed, 1 Oct 2025 16:39:14 +0000 Subject: [PATCH 59/87] More error throwing --- eng/scripts/Language-Settings.ps1 | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index 63ad13c5c7..809f9a67f2 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -151,8 +151,9 @@ function Get-rust-PackageInfoFromPackageFile([IO.FileInfo]$pkg, [string]$working Write-Host "Reading package info from $cargoTomlPath" if (!(Test-Path $cargoTomlPath)) { - LogError "The Cargo.toml file was not found in the package artifact at $cargoTomlPath" - throw "error" + $message = "The Cargo.toml file was not found in the package artifact at $cargoTomlPath" + LogError $message + throw $message } $package = cargo read-manifest --manifest-path $cargoTomlPath | ConvertFrom-Json From 0d00a9eebdea67bc077f6cb234df9d9236c00dd9 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Thu, 2 Oct 2025 03:44:35 +0000 Subject: [PATCH 60/87] Review feedback: Don't enable allowCompoundWords, also release_ names can include name separators --- .vscode/cspell.json | 5 ----- sdk/canary/ci.yml | 10 ++++------ sdk/core/ci.yml | 20 ++++++++------------ sdk/cosmos/ci.yml | 5 ++--- sdk/eventhubs/ci.yml | 10 ++++------ sdk/identity/ci.yml | 5 ++--- sdk/keyvault/ci.yml | 15 ++++++--------- sdk/servicebus/ci.yml | 5 ++--- sdk/storage/ci.yml | 15 ++++++--------- 9 files changed, 34 insertions(+), 56 deletions(-) diff --git a/.vscode/cspell.json b/.vscode/cspell.json index 62aea9d8b9..f66ef10c98 100644 --- a/.vscode/cspell.json +++ b/.vscode/cspell.json @@ -2,7 +2,6 @@ "version": "0.2", "language": "en", "useGitignore": true, - "allowCompoundWords": true, "ignorePaths": [ "**/test-resources.bicep", "**/test-resources.json", @@ -189,10 +188,6 @@ { "filename": "sdk/storage/**", "dictionaries": ["crates", "rust-custom", "storage"] - }, - { - "filename": "sdk/*/ci.yml", - "ignoreWords": ["azurecoreamqp"] } ] } diff --git a/sdk/canary/ci.yml b/sdk/canary/ci.yml index 7e65a7366f..3d9598076d 100644 --- a/sdk/canary/ci.yml +++ b/sdk/canary/ci.yml @@ -11,11 +11,11 @@ trigger: - sdk/canary/ parameters: -- name: release_azurecanarycore +- name: release_azure_canary_core displayName: 'azure_canary_core' type: boolean default: true -- name: release_azurecanary +- name: release_azure_canary displayName: 'azure_canary' type: boolean default: true @@ -26,8 +26,6 @@ extends: ServiceDirectory: canary Artifacts: - name: azure_canary_core - safeName: AzureCanaryCore - releaseInBatch: ${{ parameters.release_azurecanarycore }} + releaseInBatch: ${{ parameters.release_azure_canary_core }} - name: azure_canary - safeName: AzureCanary - releaseInBatch: ${{ parameters.release_azurecanary }} + releaseInBatch: ${{ parameters.release_azure_canary }} diff --git a/sdk/core/ci.yml b/sdk/core/ci.yml index a20be500bf..0eeb17b9aa 100644 --- a/sdk/core/ci.yml +++ b/sdk/core/ci.yml @@ -11,19 +11,19 @@ trigger: - sdk/core/ parameters: -- name: release_azurecore +- name: release_azure_core displayName: 'azure_core' type: boolean default: true -- name: release_azurecoremacros +- name: release_azure_core_macros displayName: 'azure_core_macros' type: boolean default: true -- name: release_azurecoreamqp +- name: release_azure_core_amqp displayName: 'azure_core_amqp' type: boolean default: true -- name: release_azurecoreopentelemetry +- name: release_azure_core_opentelemetry displayName: 'azure_core_opentelemetry' type: boolean default: true @@ -40,14 +40,10 @@ extends: - name: typespec_client_core safeName: TypespecClientCore - name: azure_core - safeName: AzureCore - releaseInBatch: ${{ parameters.release_azurecore }} + releaseInBatch: ${{ parameters.release_azure_core }} - name: azure_core_macros - safeName: AzureCoreMacros - releaseInBatch: ${{ parameters.release_azurecoremacros }} + releaseInBatch: ${{ parameters.release_azure_core_macros }} - name: azure_core_amqp - safeName: AzureCoreAmqp - releaseInBatch: ${{ parameters.release_azurecoreamqp }} + releaseInBatch: ${{ parameters.release_azure_core_amqp }} - name: azure_core_opentelemetry - safeName: AzureCoreOpentelemetry - releaseInBatch: ${{ parameters.release_azurecoreopentelemetry }} + releaseInBatch: ${{ parameters.release_azure_core_opentelemetry }} diff --git a/sdk/cosmos/ci.yml b/sdk/cosmos/ci.yml index 929405cdc2..b9426d0f1e 100644 --- a/sdk/cosmos/ci.yml +++ b/sdk/cosmos/ci.yml @@ -11,7 +11,7 @@ trigger: - sdk/cosmos/ parameters: -- name: release_azuredatacosmos +- name: release_azure_data_cosmos displayName: 'azure_data_cosmos' type: boolean default: true @@ -22,5 +22,4 @@ extends: ServiceDirectory: cosmos Artifacts: - name: azure_data_cosmos - safeName: AzureDataCosmos - releaseInBatch: ${{ parameters.release_azuredatacosmos }} + releaseInBatch: ${{ parameters.release_azure_data_cosmos }} diff --git a/sdk/eventhubs/ci.yml b/sdk/eventhubs/ci.yml index 893ba8d208..bb69b06a76 100644 --- a/sdk/eventhubs/ci.yml +++ b/sdk/eventhubs/ci.yml @@ -11,11 +11,11 @@ trigger: - sdk/eventhubs/ parameters: -- name: release_azuremessagingeventhubs +- name: release_azure_messaging_eventhubs displayName: 'azure_messaging_eventhubs' type: boolean default: true -- name: release_azuremessagingeventhubscheckpointstoreblob +- name: release_azure_messaging_eventhubs_checkpointstore_blob displayName: 'azure_messaging_eventhubs_checkpointstore_blob' type: boolean default: true @@ -26,8 +26,6 @@ extends: ServiceDirectory: eventhubs Artifacts: - name: azure_messaging_eventhubs - safeName: AzureMessagingEventHubs - releaseInBatch: ${{ parameters.release_azuremessagingeventhubs }} + releaseInBatch: ${{ parameters.release_azure_messaging_eventhubs }} - name: azure_messaging_eventhubs_checkpointstore_blob - safeName: AzureMessagingEventHubsBlobCheckpointStore - releaseInBatch: ${{ parameters.release_azuremessagingeventhubscheckpointstoreblob }} + releaseInBatch: ${{ parameters.release_azure_messaging_eventhubs_checkpointstore_blob }} diff --git a/sdk/identity/ci.yml b/sdk/identity/ci.yml index e641b876ed..4dfa38e4c6 100644 --- a/sdk/identity/ci.yml +++ b/sdk/identity/ci.yml @@ -11,7 +11,7 @@ trigger: - sdk/identity/ parameters: -- name: release_azureidentity +- name: release_azure_identity displayName: 'azure_identity' type: boolean default: true @@ -22,8 +22,7 @@ extends: ServiceDirectory: identity Artifacts: - name: azure_identity - safeName: AzureIdentity - releaseInBatch: ${{ parameters.release_azureidentity }} + releaseInBatch: ${{ parameters.release_azure_identity }} ${{ if endsWith(variables['Build.DefinitionName'], 'weekly') }}: Location: uksouth diff --git a/sdk/keyvault/ci.yml b/sdk/keyvault/ci.yml index 6894d4e5b6..0a240b1fa3 100644 --- a/sdk/keyvault/ci.yml +++ b/sdk/keyvault/ci.yml @@ -14,15 +14,15 @@ parameters: displayName: 'Run live tests' type: boolean default: false -- name: release_azuresecuritykeyvaultsecrets +- name: release_azure_security_keyvault_secrets displayName: 'azure_security_keyvault_secrets' type: boolean default: true -- name: release_azuresecuritykeyvaultkeys +- name: release_azure_security_keyvault_keys displayName: 'azure_security_keyvault_keys' type: boolean default: true -- name: release_azuresecuritykeyvaultcertificates +- name: release_azure_security_keyvault_certificates displayName: 'azure_security_keyvault_certificates' type: boolean default: true @@ -34,11 +34,8 @@ extends: RunLiveTests: ${{ or(parameters.RunLiveTests, eq(variables['Build.Reason'], 'Schedule')) }} Artifacts: - name: azure_security_keyvault_secrets - safeName: AzureSecurityKeyvaultSecrets - releaseInBatch: ${{ parameters.release_azuresecuritykeyvaultsecrets }} + releaseInBatch: ${{ parameters.release_azure_security_keyvault_secrets }} - name: azure_security_keyvault_keys - safeName: AzureSecurityKeyvaultKeys - releaseInBatch: ${{ parameters.release_azuresecuritykeyvaultkeys }} + releaseInBatch: ${{ parameters.release_azure_security_keyvault_keys }} - name: azure_security_keyvault_certificates - safeName: AzureSecurityKeyvaultCertificates - releaseInBatch: ${{ parameters.release_azuresecuritykeyvaultcertificates }} + releaseInBatch: ${{ parameters.release_azure_security_keyvault_certificates }} diff --git a/sdk/servicebus/ci.yml b/sdk/servicebus/ci.yml index 9d0c667d60..d1c8778ce3 100644 --- a/sdk/servicebus/ci.yml +++ b/sdk/servicebus/ci.yml @@ -12,7 +12,7 @@ trigger: - sdk/servicebus/ parameters: -- name: release_azuremessagingservicebus +- name: release_azure_messaging_servicebus displayName: 'azure_messaging_servicebus' type: boolean default: true @@ -23,5 +23,4 @@ extends: ServiceDirectory: servicebus Artifacts: - name: azure_messaging_servicebus - safeName: AzureMessagingServiceBus - releaseInBatch: ${{ parameters.release_azuremessagingservicebus }} + releaseInBatch: ${{ parameters.release_azure_messaging_servicebus }} diff --git a/sdk/storage/ci.yml b/sdk/storage/ci.yml index e7775989d2..03fe735ab3 100644 --- a/sdk/storage/ci.yml +++ b/sdk/storage/ci.yml @@ -14,15 +14,15 @@ parameters: displayName: 'Run live tests' type: boolean default: false -- name: release_azurestoragecommon +- name: release_azure_storage_common displayName: 'azure_storage_common' type: boolean default: true -- name: release_azurestorageblob +- name: release_azure_storage_blob displayName: 'azure_storage_blob' type: boolean default: true -- name: release_azurestoragequeue +- name: release_azure_storage_queue displayName: 'azure_storage_queue' type: boolean default: true @@ -35,11 +35,8 @@ extends: TestTimeoutInMinutes: 120 Artifacts: - name: azure_storage_common - safeName: AzureStorageCommon - releaseInBatch: ${{ parameters.release_azurestoragecommon }} + releaseInBatch: ${{ parameters.release_azure_storage_common }} - name: azure_storage_blob - safeName: AzureStorageBlob - releaseInBatch: ${{ parameters.release_azurestorageblob }} + releaseInBatch: ${{ parameters.release_azure_storage_blob }} - name: azure_storage_queue - safeName: AzureStorageQueue - releaseInBatch: ${{ parameters.release_azurestoragequeue }} + releaseInBatch: ${{ parameters.release_azure_storage_queue }} From ea9b3189e4e389f0fec0957be09a9c16fdbb7a54 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Thu, 2 Oct 2025 04:24:21 +0000 Subject: [PATCH 61/87] Artifact ordering for Pack-Crates.ps1 --- sdk/keyvault/ci.yml | 4 ++-- sdk/storage/ci.yml | 7 +++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/sdk/keyvault/ci.yml b/sdk/keyvault/ci.yml index 0a240b1fa3..8375eb0ec4 100644 --- a/sdk/keyvault/ci.yml +++ b/sdk/keyvault/ci.yml @@ -35,7 +35,7 @@ extends: Artifacts: - name: azure_security_keyvault_secrets releaseInBatch: ${{ parameters.release_azure_security_keyvault_secrets }} - - name: azure_security_keyvault_keys - releaseInBatch: ${{ parameters.release_azure_security_keyvault_keys }} - name: azure_security_keyvault_certificates releaseInBatch: ${{ parameters.release_azure_security_keyvault_certificates }} + - name: azure_security_keyvault_keys + releaseInBatch: ${{ parameters.release_azure_security_keyvault_keys }} diff --git a/sdk/storage/ci.yml b/sdk/storage/ci.yml index 03fe735ab3..11b2636ebb 100644 --- a/sdk/storage/ci.yml +++ b/sdk/storage/ci.yml @@ -34,9 +34,12 @@ extends: RunLiveTests: ${{ or(parameters.RunLiveTests, eq(variables['Build.Reason'], 'Schedule')) }} TestTimeoutInMinutes: 120 Artifacts: - - name: azure_storage_common - releaseInBatch: ${{ parameters.release_azure_storage_common }} + # Artifact order in this file maps to order evaluated by Pack-Crates.ps1 + # When the artifacts like blob and queue depend on common, common will need + # to be listed first. - name: azure_storage_blob releaseInBatch: ${{ parameters.release_azure_storage_blob }} + - name: azure_storage_common + releaseInBatch: ${{ parameters.release_azure_storage_common }} - name: azure_storage_queue releaseInBatch: ${{ parameters.release_azure_storage_queue }} From c7d79889bfa68653349d6b47de8e677ff2979d09 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Mon, 6 Oct 2025 22:37:38 +0000 Subject: [PATCH 62/87] Add support for outputting release order of specified packages --- eng/pipelines/templates/jobs/pack.yml | 1 + eng/scripts/Pack-Crates.ps1 | 24 ++++++++---------------- 2 files changed, 9 insertions(+), 16 deletions(-) diff --git a/eng/pipelines/templates/jobs/pack.yml b/eng/pipelines/templates/jobs/pack.yml index e64a88d349..90b2b53c95 100644 --- a/eng/pipelines/templates/jobs/pack.yml +++ b/eng/pipelines/templates/jobs/pack.yml @@ -97,6 +97,7 @@ jobs: -OutputPath '$(Build.ArtifactStagingDirectory)' -PackageNames $(PackageNames) -RequireDependencies:$$(RequireDependencies) + -OutBuildOrderFile '$(Build.ArtifactStagingDirectory)/release-order.json' # TODO: Ensure APIView works given a change to the crates output folder structure - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml diff --git a/eng/scripts/Pack-Crates.ps1 b/eng/scripts/Pack-Crates.ps1 index 0e1fb04d77..131576f57e 100755 --- a/eng/scripts/Pack-Crates.ps1 +++ b/eng/scripts/Pack-Crates.ps1 @@ -10,7 +10,8 @@ param( [switch]$RequireDependencies, [Parameter(ParameterSetName = 'PackageInfo')] [string]$PackageInfoDirectory, - [switch]$NoVerify + [switch]$NoVerify, + [string]$OutBuildOrderFile ) $ErrorActionPreference = 'Stop' @@ -174,21 +175,6 @@ try { Write-Error "Packages in -PackageNames require dependencies that are either not released or not listed for packing: $($unspecifiedPackages -join ', ')" exit 1 } - - $orderMatches = $true - for ($i = 0; $i -lt $PackageNames.Count; $i++) { - if ($packages[$i].name -ne $PackageNames[$i]) { - $orderMatches = $false - break - } - } - - if (!$orderMatches) { - Write-Host "Expected order: $($packages.name -join ', ')" - Write-Host "Provided order: $($PackageNames -join ', ')" - Write-Error "The order of packages in -PackageNames does not match the required build order." - exit 1 - } } Write-Host "Building packages in the following order:" @@ -198,6 +184,12 @@ try { Write-Host " $packageName ($type)" } + if ($OutBuildOrderFile) { + $buildOrder = ConvertTo-Json $packages.name + Write-Host "Writing build order to $OutBuildOrderFile ($buildOrder)" + $buildOrder | Out-File -FilePath $OutBuildOrderFile -Encoding utf8 -Force + } + foreach ($package in $packages) { Write-Host "" From 3e514859b400dbdc3c8a5c624be66d798dddcbcf Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Tue, 7 Oct 2025 03:05:06 +0000 Subject: [PATCH 63/87] Use artifact ordering from artifacts --- .../stages/archetype-rust-release.yml | 50 +++++++++++++------ sdk/canary/ci.yml | 4 +- 2 files changed, 37 insertions(+), 17 deletions(-) diff --git a/eng/pipelines/templates/stages/archetype-rust-release.yml b/eng/pipelines/templates/stages/archetype-rust-release.yml index 1de7ec0860..7ee7dccfef 100644 --- a/eng/pipelines/templates/stages/archetype-rust-release.yml +++ b/eng/pipelines/templates/stages/archetype-rust-release.yml @@ -85,21 +85,36 @@ stages: runOnce: deploy: steps: + - pwsh: | + Write-Host "##vso[task.setvariable variable=ArtifactIndex]0" + displayName: Set ArtifactIndex to 0 + - ${{ each artifact in parameters.Artifacts }}: - pwsh: | - $ArtifactName = '${{artifact.name}}' - $ArtifactRootPath = '$(Pipeline.Workspace)/drop' - $OutDir = '$(Pipeline.Workspace)/esrp-release' + # Read artifact release order from release-order.json + # and use ArtifactIndex to select the right one + $index = [int]'$(ArtifactIndex)' + $artifacts = Get-Content '$(Pipeline.Workspace)/drop/release-order.json' | ConvertFrom-Json + if ($index -ge $artifacts.Count) { + Write-Error "ArtifactIndex $index is out of range (0..$($artifacts.Count - 1))" + exit 1 + } + + $artifactName = $artifacts[$index] + Write-Host "Releasing artifact $artifactName (index $index)" - if (Test-Path $OutDir) { - Write-Host "Cleaning output directory: $OutDir" - Remove-Item -Path $OutDir -Recurse -Force + $artifactRootPath = '$(Pipeline.Workspace)/drop' + $outDir = '$(Pipeline.Workspace)/esrp-release' + + if (Test-Path $outDir) { + Write-Host "Cleaning output directory: $outDir" + Remove-Item -Path $outDir -Recurse -Force } - New-Item -ItemType Directory -Path $OutDir -Force | Out-Null + New-Item -ItemType Directory -Path $outDir -Force | Out-Null - Write-Host "Artifact name: $ArtifactName" + Write-Host "Artifact name: $artifactName" - $packageMetadataPath = "$ArtifactRootPath/PackageInfo/$ArtifactName.json" + $packageMetadataPath = "$artifactRootPath/PackageInfo/$artifactName.json" if (!(Test-Path $packageMetadataPath)) { Write-Error "Package metadata file not found: $packageMetadataPath" exit 1 @@ -109,16 +124,16 @@ stages: $packageVersion = $packageMetadata.version Write-Host "Package version: $packageVersion" - $cratePath = "$ArtifactRootPath/$ArtifactName/$ArtifactName-$packageVersion.crate" + $cratePath = "$artifactRootPath/$artifactName/$artifactName-$packageVersion.crate" Copy-Item ` -Path $cratePath ` - -Destination $OutDir - Write-Host "Contents of $OutDir" - Get-ChildItem -Path $OutDir | ForEach-Object { Write-Host $_.FullName } - displayName: 'Copy crate for ESRP: ${{artifact.name}}' + -Destination $outDir + Write-Host "Contents of $outDir" + Get-ChildItem -Path $outDir | ForEach-Object { Write-Host $_.FullName } + displayName: 'Copy crate for ESRP' - task: EsrpRelease@10 - displayName: 'ESRP Release: ${{artifact.name}}' + displayName: 'ESRP Release' inputs: connectedservicename: 'Azure SDK PME Managed Identity' ClientId: '5f81938c-2544-4f1f-9251-dd9de5b8a81b' @@ -136,6 +151,11 @@ stages: serviceendpointurl: 'https://api.esrp.microsoft.com/' mainpublisher: 'ESRPRELPACMANTEST' + - pwsh: | + $index = [int]'$(ArtifactIndex)' + 1 + Write-Host "##vso[task.setvariable variable=ArtifactIndex]$index" + displayName: Increment ArtifactIndex + - job: UpdatePackageVersion displayName: "API Review and Package Version Update" condition: and(succeeded(), ne(variables['Skip.UpdatePackageVersion'], 'true')) diff --git a/sdk/canary/ci.yml b/sdk/canary/ci.yml index 3d9598076d..9c580cac20 100644 --- a/sdk/canary/ci.yml +++ b/sdk/canary/ci.yml @@ -25,7 +25,7 @@ extends: parameters: ServiceDirectory: canary Artifacts: - - name: azure_canary_core - releaseInBatch: ${{ parameters.release_azure_canary_core }} - name: azure_canary releaseInBatch: ${{ parameters.release_azure_canary }} + - name: azure_canary_core + releaseInBatch: ${{ parameters.release_azure_canary_core }} From 0c9b166739e4cf16f9e2b50fad714c84635a7edf Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Tue, 7 Oct 2025 03:37:50 +0000 Subject: [PATCH 64/87] Log index --- eng/pipelines/templates/stages/archetype-rust-release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eng/pipelines/templates/stages/archetype-rust-release.yml b/eng/pipelines/templates/stages/archetype-rust-release.yml index 7ee7dccfef..e94c9cf41d 100644 --- a/eng/pipelines/templates/stages/archetype-rust-release.yml +++ b/eng/pipelines/templates/stages/archetype-rust-release.yml @@ -153,6 +153,7 @@ stages: - pwsh: | $index = [int]'$(ArtifactIndex)' + 1 + Write-Host "Setting ArtifactIndex to $index" Write-Host "##vso[task.setvariable variable=ArtifactIndex]$index" displayName: Increment ArtifactIndex From ee301822e8aabffafb3a370d6433ac2455831a40 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Tue, 7 Oct 2025 04:29:45 +0000 Subject: [PATCH 65/87] One PR per-release --- .../templates/stages/archetype-rust-release.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/eng/pipelines/templates/stages/archetype-rust-release.yml b/eng/pipelines/templates/stages/archetype-rust-release.yml index e94c9cf41d..c630d1d3bc 100644 --- a/eng/pipelines/templates/stages/archetype-rust-release.yml +++ b/eng/pipelines/templates/stages/archetype-rust-release.yml @@ -196,11 +196,11 @@ stages: -ServiceDirectory '${{parameters.ServiceDirectory}}' -PackageName '${{artifact.name}}' - - template: /eng/common/pipelines/templates/steps/create-pull-request.yml - parameters: - PRBranchName: increment-package-version-${{parameters.ServiceDirectory}}-$(Build.BuildId) - CommitMsg: "Increment package version after release of ${{ artifact.name }}" - PRTitle: "Increment versions for ${{parameters.ServiceDirectory}} releases" - CloseAfterOpenForTesting: '${{parameters.TestPipeline}}' - ${{ if startsWith(variables['Build.SourceBranch'], 'refs/pull/') }}: - BaseBranchName: main + - template: /eng/common/pipelines/templates/steps/create-pull-request.yml + parameters: + PRBranchName: increment-package-version-${{parameters.ServiceDirectory}}-$(Build.BuildId) + CommitMsg: "Increment package version after release of ${{ join(', ', parameters.Artifacts.*.name) }}" + PRTitle: "Increment versions for ${{parameters.ServiceDirectory}} releases" + CloseAfterOpenForTesting: '${{parameters.TestPipeline}}' + ${{ if startsWith(variables['Build.SourceBranch'], 'refs/pull/') }}: + BaseBranchName: main From 164469e8f649e91837f8ac381c4a32e6ebce8b4d Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Tue, 7 Oct 2025 16:55:28 +0000 Subject: [PATCH 66/87] Review feedback: -eq 'True' --- eng/pipelines/templates/jobs/pack.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/eng/pipelines/templates/jobs/pack.yml b/eng/pipelines/templates/jobs/pack.yml index 90b2b53c95..2a56a313eb 100644 --- a/eng/pipelines/templates/jobs/pack.yml +++ b/eng/pipelines/templates/jobs/pack.yml @@ -70,10 +70,8 @@ jobs: - pwsh: | $artifacts = '${{ convertToJson(parameters.Artifacts) }}' | ConvertFrom-Json $requireDependencies = $true - $artifactsToBuild = $artifacts | Where-Object { $_.releaseInBatch -ne 'False' } + $artifactsToBuild = $artifacts | Where-Object { $_.releaseInBatch -eq 'True' } - - if (!$artifactsToBuild) { Write-Host "No packages to release. Building all packages in the service directory with no dependency validation." $artifactsToBuild = $artifacts @@ -84,7 +82,6 @@ jobs: Write-Host "##vso[task.setvariable variable=PackageNames]$packageNames" Write-Host "##vso[task.setvariable variable=RequireDependencies]$requireDependencies" - displayName: Create package list - task: Powershell@2 From 24abf38dd5f0e6a4cc901d1f641fcc4f5ca3bcb8 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Tue, 7 Oct 2025 17:42:33 +0000 Subject: [PATCH 67/87] Test public API change --- sdk/canary/azure_canary/src/constant_example.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/canary/azure_canary/src/constant_example.rs b/sdk/canary/azure_canary/src/constant_example.rs index 0b354b330c..52ee3bd8c3 100644 --- a/sdk/canary/azure_canary/src/constant_example.rs +++ b/sdk/canary/azure_canary/src/constant_example.rs @@ -45,6 +45,7 @@ pub enum TemperatureUnit { Celsius, Fahrenheit, Kelvin, + Rankine, // Testing APIView } impl Temperature { From 067a2ff381f45ca86b28818b60723cc3aaea0d4f Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Tue, 7 Oct 2025 18:01:18 +0000 Subject: [PATCH 68/87] Cover is_below_freezing --- sdk/canary/azure_canary/src/constant_example.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/canary/azure_canary/src/constant_example.rs b/sdk/canary/azure_canary/src/constant_example.rs index 52ee3bd8c3..ce2465eb72 100644 --- a/sdk/canary/azure_canary/src/constant_example.rs +++ b/sdk/canary/azure_canary/src/constant_example.rs @@ -61,6 +61,7 @@ impl Temperature { TemperatureUnit::Celsius => self.value < 0.0, TemperatureUnit::Fahrenheit => self.value < 32.0, TemperatureUnit::Kelvin => self.value < 273.15, + TemperatureUnit::Rankine => self.value < 491.67, } } } From a1c1e4d68e4ac1cf59cc3204f38efaa4b8bcb9d3 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Tue, 7 Oct 2025 18:37:22 +0000 Subject: [PATCH 69/87] Remove TODO, APIView behavior is unchanged --- eng/pipelines/templates/jobs/pack.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/eng/pipelines/templates/jobs/pack.yml b/eng/pipelines/templates/jobs/pack.yml index 2a56a313eb..729c6f0f6f 100644 --- a/eng/pipelines/templates/jobs/pack.yml +++ b/eng/pipelines/templates/jobs/pack.yml @@ -96,7 +96,6 @@ jobs: -RequireDependencies:$$(RequireDependencies) -OutBuildOrderFile '$(Build.ArtifactStagingDirectory)/release-order.json' - # TODO: Ensure APIView works given a change to the crates output folder structure - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml parameters: ArtifactPath: $(Build.ArtifactStagingDirectory) From 439f70361fa550d9921349adc83bc19ebbb597de Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Tue, 7 Oct 2025 18:38:29 +0000 Subject: [PATCH 70/87] Formatting --- eng/pipelines/templates/stages/archetype-sdk-client.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/stages/archetype-sdk-client.yml b/eng/pipelines/templates/stages/archetype-sdk-client.yml index 4f1d8ee413..9cd7dd8817 100644 --- a/eng/pipelines/templates/stages/archetype-sdk-client.yml +++ b/eng/pipelines/templates/stages/archetype-sdk-client.yml @@ -84,7 +84,7 @@ extends: ServiceDirectory: ${{ parameters.ServiceDirectory }} PipelineArtifactName: packages Artifacts: ${{ parameters.Artifacts }} - TestPipeline: ${{ eq(parameters.ServiceDirectory, 'canary') }} + TestPipeline: ${{ eq(parameters.ServiceDirectory, 'canary') }} TestTimeoutInMinutes: ${{ parameters.TestTimeoutInMinutes }} TestProxy: ${{ parameters.TestProxy }} MatrixConfigs: From a169a49f97d9fbc7033c31ffbdada607956d8ac7 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Tue, 7 Oct 2025 18:51:17 +0000 Subject: [PATCH 71/87] Format --- sdk/canary/azure_canary/src/constant_example.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/canary/azure_canary/src/constant_example.rs b/sdk/canary/azure_canary/src/constant_example.rs index ce2465eb72..deb42ad18f 100644 --- a/sdk/canary/azure_canary/src/constant_example.rs +++ b/sdk/canary/azure_canary/src/constant_example.rs @@ -45,7 +45,7 @@ pub enum TemperatureUnit { Celsius, Fahrenheit, Kelvin, - Rankine, // Testing APIView + Rankine, // Testing APIView } impl Temperature { From 32451433a7229dd0347b582cb9ea36bd5badebbe Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Tue, 7 Oct 2025 20:18:35 +0000 Subject: [PATCH 72/87] Revert constant_example.rs --- sdk/canary/azure_canary/src/constant_example.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/sdk/canary/azure_canary/src/constant_example.rs b/sdk/canary/azure_canary/src/constant_example.rs index deb42ad18f..0b354b330c 100644 --- a/sdk/canary/azure_canary/src/constant_example.rs +++ b/sdk/canary/azure_canary/src/constant_example.rs @@ -45,7 +45,6 @@ pub enum TemperatureUnit { Celsius, Fahrenheit, Kelvin, - Rankine, // Testing APIView } impl Temperature { @@ -61,7 +60,6 @@ impl Temperature { TemperatureUnit::Celsius => self.value < 0.0, TemperatureUnit::Fahrenheit => self.value < 32.0, TemperatureUnit::Kelvin => self.value < 273.15, - TemperatureUnit::Rankine => self.value < 491.67, } } } From f265701f23d0e0051552662fcf68c621b981c785 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Tue, 7 Oct 2025 22:05:33 +0000 Subject: [PATCH 73/87] Review feedback: quotes, Join-Path --- eng/pipelines/templates/jobs/pack.yml | 4 ++-- eng/scripts/Language-Settings.ps1 | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/eng/pipelines/templates/jobs/pack.yml b/eng/pipelines/templates/jobs/pack.yml index 729c6f0f6f..9f1b1654e4 100644 --- a/eng/pipelines/templates/jobs/pack.yml +++ b/eng/pipelines/templates/jobs/pack.yml @@ -57,7 +57,7 @@ jobs: - ${{ if eq('auto', parameters.ServiceDirectory) }}: - task: Powershell@2 - displayName: "Pack Crates (PR build)" + displayName: Pack Crates condition: and(succeeded(), ne(variables['NoPackagesChanged'],'true')) inputs: pwsh: true @@ -85,7 +85,7 @@ jobs: displayName: Create package list - task: Powershell@2 - displayName: "Pack Crates" + displayName: Pack Crates condition: and(succeeded(), ne(variables['NoPackagesChanged'],'true')) inputs: pwsh: true diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index 809f9a67f2..d085a398fc 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -142,12 +142,12 @@ function Get-rust-AdditionalValidationPackagesFromPackageSet ($packagesWithChang # $GetPackageInfoFromPackageFileFn = "Get-${Language}-PackageInfoFromPackageFile" function Get-rust-PackageInfoFromPackageFile([IO.FileInfo]$pkg, [string]$workingDirectory) { # Create a temporary folder for extraction - $extractionPath = Join-Path ([System.IO.Path]::GetTempPath()) ([System.IO.Path]::GetRandomFileName()) + $extractionPath = [System.IO.Path]::Combine([System.IO.Path]::GetTempPath(), [System.IO.Path]::GetRandomFileName()) New-Item -ItemType Directory -Path $extractionPath | Out-Null # Extract the .crate file (which is a tarball) to the temporary folder tar -xvf $pkg.FullName -C $extractionPath - $cargoTomlPath = Join-Path $extractionPath $pkg.BaseName 'Cargo.toml' + $cargoTomlPath = [System.IO.Path]::Combine($extractionPath, $pkg.BaseName, 'Cargo.toml') Write-Host "Reading package info from $cargoTomlPath" if (!(Test-Path $cargoTomlPath)) { @@ -161,7 +161,7 @@ function Get-rust-PackageInfoFromPackageFile([IO.FileInfo]$pkg, [string]$working $packageName = $package.name $packageVersion = $package.version - $packageAssetPath = Join-Path $extractionPath "$packageName-$packageVersion" + $packageAssetPath = [System.IO.Path]::Combine($extractionPath, "$packageName-$packageVersion") $changeLogLoc = Get-ChildItem -Path $packageAssetPath -Filter "CHANGELOG.md" | Select-Object -First 1 $readmeContentLoc = Get-ChildItem -Path $packageAssetPath -Filter "README.md" | Select-Object -First 1 From c405dfaaaf9c5db735133143f1e09dbfb0ffa64c Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Mon, 13 Oct 2025 19:56:31 +0000 Subject: [PATCH 74/87] Build dependencies from source allowing exceptions for crates that have already published to crates.io --- eng/scripts/Pack-Crates.ps1 | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/eng/scripts/Pack-Crates.ps1 b/eng/scripts/Pack-Crates.ps1 index 131576f57e..c26ba677f7 100755 --- a/eng/scripts/Pack-Crates.ps1 +++ b/eng/scripts/Pack-Crates.ps1 @@ -57,13 +57,30 @@ function Get-OutputPackageNames($workspacePackages) { } } - return $names + return , $names } function Get-CargoMetadata() { cargo metadata --no-deps --format-version 1 --manifest-path "$RepoRoot/Cargo.toml" | ConvertFrom-Json -Depth 100 -AsHashtable } +function Test-PublishedOnCratesIo($name, $version) { + try { + Invoke-WebRequest -Uri "https://crates.io/api/v1/crates/$name/$version" + return $true + } + catch { + if ($_.Exception.Response.StatusCode -eq 404) { + # 404 means package version is not found and doesn't exist + return $false + } + else { + # Re-throw other exceptions + throw $_ + } + } +} + function Get-CargoPackages() { $metadata = Get-CargoMetadata @@ -97,7 +114,7 @@ function Get-PackagesToBuild() { foreach ($dependency in $package.UnreleasedDependencies) { if (!$packagesToBuild.Contains($dependency) -and !$toProcess.Contains($dependency)) { if ($RequireDependencies -and $dependency.name -notin $PackageNames) { - Write-Warning "Package $($package.name) depends on unreleased or unspecified dependency: $($dependency.name)" + Write-Warning "Package $($package.name) depends on potentially unreleased or unspecified dependency: $($dependency.name)`@$($dependency.version)" } $packagesToBuild += $dependency @@ -170,7 +187,7 @@ try { [array]$packages = Get-PackagesToBuild if ($RequireDependencies) { - $unspecifiedPackages = $packages.name | Where-Object { $_ -notin $PackageNames } + $unspecifiedPackages = $packages | Where-Object { $_.name -notin $PackageNames -and !(Test-PublishedOnCratesIo -name $_.name -version $_.version) } | Foreach-Object { "$($_.name)`@$($_.version)" } if ($unspecifiedPackages.Count -gt 0) { Write-Error "Packages in -PackageNames require dependencies that are either not released or not listed for packing: $($unspecifiedPackages -join ', ')" exit 1 From 09356e418a628654ff6e4af8dedd7658e74d9eb8 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Mon, 13 Oct 2025 20:45:55 +0000 Subject: [PATCH 75/87] Better single package handling --- eng/scripts/Pack-Crates.ps1 | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/eng/scripts/Pack-Crates.ps1 b/eng/scripts/Pack-Crates.ps1 index c26ba677f7..db1ec8528e 100755 --- a/eng/scripts/Pack-Crates.ps1 +++ b/eng/scripts/Pack-Crates.ps1 @@ -202,9 +202,15 @@ try { } if ($OutBuildOrderFile) { - $buildOrder = ConvertTo-Json $packages.name - Write-Host "Writing build order to $OutBuildOrderFile ($buildOrder)" - $buildOrder | Out-File -FilePath $OutBuildOrderFile -Encoding utf8 -Force + $buildOrder = @() + foreach ($package in $packages) { + if ($package.OutputPackage) { + $buildOrder += $package.name + } + } + $buildOrderJson = ConvertTo-Json $buildOrder + Write-Host "Writing build order to $OutBuildOrderFile ($buildOrderJson)" + $buildOrderJson | Out-File -FilePath $OutBuildOrderFile -Encoding utf8 -Force } foreach ($package in $packages) { From c8457fcb7d703683991b80ee47ccc94e9b3731bd Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Wed, 15 Oct 2025 04:46:22 +0000 Subject: [PATCH 76/87] Use simplified packing process for release --- eng/pipelines/templates/jobs/pack.yml | 25 +++++---- eng/scripts/Pack-Common.ps1 | 15 ++++++ eng/scripts/Pack-Crates.ps1 | 78 +++++---------------------- eng/scripts/Pack-ReleaseCrates.ps1 | 58 ++++++++++++++++++++ 4 files changed, 99 insertions(+), 77 deletions(-) create mode 100644 eng/scripts/Pack-Common.ps1 create mode 100755 eng/scripts/Pack-ReleaseCrates.ps1 diff --git a/eng/pipelines/templates/jobs/pack.yml b/eng/pipelines/templates/jobs/pack.yml index 9f1b1654e4..575d1dbfd5 100644 --- a/eng/pipelines/templates/jobs/pack.yml +++ b/eng/pipelines/templates/jobs/pack.yml @@ -80,21 +80,20 @@ jobs: $packageNames = $artifactsToBuild.name -join ',' - Write-Host "##vso[task.setvariable variable=PackageNames]$packageNames" - Write-Host "##vso[task.setvariable variable=RequireDependencies]$requireDependencies" - displayName: Create package list - - - task: Powershell@2 + if ($requireDependencies) { + Write-Host "Building crates with dependency validation for reelase" + & $(Build.SourcesDirectory)/eng/scripts/Pack-ReleaseCrates.ps1 ` + -OutputPath '$(Build.ArtifactStagingDirectory)' ` + -PackageNames $packageNames ` + -OutBuildOrderFile '$(Build.ArtifactStagingDirectory)/release-order.json' + } else { + Write-Host "Building crates without validating dependencies" + & $(Build.SourcesDirectory)/eng/scripts/Pack-Crates.ps1 ` + -OutputPath '$(Build.ArtifactStagingDirectory)' ` + -PackageNames $packageNames + } displayName: Pack Crates condition: and(succeeded(), ne(variables['NoPackagesChanged'],'true')) - inputs: - pwsh: true - filePath: $(Build.SourcesDirectory)/eng/scripts/Pack-Crates.ps1 - arguments: > - -OutputPath '$(Build.ArtifactStagingDirectory)' - -PackageNames $(PackageNames) - -RequireDependencies:$$(RequireDependencies) - -OutBuildOrderFile '$(Build.ArtifactStagingDirectory)/release-order.json' - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml parameters: diff --git a/eng/scripts/Pack-Common.ps1 b/eng/scripts/Pack-Common.ps1 new file mode 100644 index 0000000000..92fa5768e7 --- /dev/null +++ b/eng/scripts/Pack-Common.ps1 @@ -0,0 +1,15 @@ +. (Join-Path $PSScriptRoot '..' 'common' 'scripts' 'common.ps1') + +function Get-CargoMetadata() { + cargo metadata --no-deps --format-version 1 --manifest-path "$RepoRoot/Cargo.toml" | ConvertFrom-Json -Depth 100 -AsHashtable +} + +function Create-ApiViewFile($package) { + $packageName = $package.name + $command = "cargo run --manifest-path $RepoRoot/eng/tools/generate_api_report/Cargo.toml -- --package $packageName" + Invoke-LoggedCommand $command -GroupOutput | Out-Host + + $packagePath = Split-Path -Path $package.manifest_path -Parent + + "$packagePath/review/$packageName.rust.json" +} diff --git a/eng/scripts/Pack-Crates.ps1 b/eng/scripts/Pack-Crates.ps1 index db1ec8528e..392745b1b4 100755 --- a/eng/scripts/Pack-Crates.ps1 +++ b/eng/scripts/Pack-Crates.ps1 @@ -6,17 +6,16 @@ param( [string]$OutputPath, [Parameter(ParameterSetName = 'Named')] [string[]]$PackageNames, - [Parameter(ParameterSetName = 'Named')] - [switch]$RequireDependencies, [Parameter(ParameterSetName = 'PackageInfo')] [string]$PackageInfoDirectory, - [switch]$NoVerify, - [string]$OutBuildOrderFile + [switch]$NoVerify ) $ErrorActionPreference = 'Stop' -. (Join-Path $PSScriptRoot '..' 'common' 'scripts' 'common.ps1') +# TODO: Don't use Join-Path +. ([System.IO.Path]::Combine($PSScriptRoot, '..', 'common', 'scripts', 'common.ps1')) +. ([System.IO.Path]::Combine($PSScriptRoot, 'Pack-Common.ps1')) Write-Host @" Packing crates with @@ -57,28 +56,7 @@ function Get-OutputPackageNames($workspacePackages) { } } - return , $names -} - -function Get-CargoMetadata() { - cargo metadata --no-deps --format-version 1 --manifest-path "$RepoRoot/Cargo.toml" | ConvertFrom-Json -Depth 100 -AsHashtable -} - -function Test-PublishedOnCratesIo($name, $version) { - try { - Invoke-WebRequest -Uri "https://crates.io/api/v1/crates/$name/$version" - return $true - } - catch { - if ($_.Exception.Response.StatusCode -eq 404) { - # 404 means package version is not found and doesn't exist - return $false - } - else { - # Re-throw other exceptions - throw $_ - } - } + return $names } function Get-CargoPackages() { @@ -113,10 +91,6 @@ function Get-PackagesToBuild() { foreach ($dependency in $package.UnreleasedDependencies) { if (!$packagesToBuild.Contains($dependency) -and !$toProcess.Contains($dependency)) { - if ($RequireDependencies -and $dependency.name -notin $PackageNames) { - Write-Warning "Package $($package.name) depends on potentially unreleased or unspecified dependency: $($dependency.name)`@$($dependency.version)" - } - $packagesToBuild += $dependency $toProcess += $dependency } @@ -170,30 +144,12 @@ function Add-CrateToLocalRegistry($LocalRegistryPath, $Package) { '{"files":{}}' | Out-File -FilePath "$LocalRegistryPath/$packageName-$packageVersion/.cargo-checksum.json" -Encoding utf8 } -function Create-ApiViewFile($package) { - $packageName = $package.name - $command = "cargo run --manifest-path $RepoRoot/eng/tools/generate_api_report/Cargo.toml -- --package $packageName" - Invoke-LoggedCommand $command -GroupOutput | Out-Host - - $packagePath = Split-Path -Path $package.manifest_path -Parent - - "$packagePath/review/$packageName.rust.json" -} - Push-Location $RepoRoot try { $localRegistryPath = Initialize-VendorDirectory [array]$packages = Get-PackagesToBuild - if ($RequireDependencies) { - $unspecifiedPackages = $packages | Where-Object { $_.name -notin $PackageNames -and !(Test-PublishedOnCratesIo -name $_.name -version $_.version) } | Foreach-Object { "$($_.name)`@$($_.version)" } - if ($unspecifiedPackages.Count -gt 0) { - Write-Error "Packages in -PackageNames require dependencies that are either not released or not listed for packing: $($unspecifiedPackages -join ', ')" - exit 1 - } - } - Write-Host "Building packages in the following order:" foreach ($package in $packages) { $packageName = $package.name @@ -201,18 +157,6 @@ try { Write-Host " $packageName ($type)" } - if ($OutBuildOrderFile) { - $buildOrder = @() - foreach ($package in $packages) { - if ($package.OutputPackage) { - $buildOrder += $package.name - } - } - $buildOrderJson = ConvertTo-Json $buildOrder - Write-Host "Writing build order to $OutBuildOrderFile ($buildOrderJson)" - $buildOrderJson | Out-File -FilePath $OutBuildOrderFile -Encoding utf8 -Force - } - foreach ($package in $packages) { Write-Host "" @@ -227,6 +171,7 @@ try { Invoke-LoggedCommand -Command $command -GroupOutput + # copy the package to the local registry Add-CrateToLocalRegistry ` -LocalRegistryPath $localRegistryPath ` @@ -235,11 +180,16 @@ try { if ($OutputPath -and $package.OutputPackage) { $sourcePath = "$RepoRoot/target/package/$packageName-$packageVersion" $targetPath = "$OutputPath/$packageName" + $targetContentsPath = "$targetPath/contents" $targetApiReviewFile = "$targetPath/$packageName.rust.json" - Write-Host "Copying package '$packageName' to '$targetPath'" - New-Item -ItemType Directory -Path $targetPath -Force | Out-Null - Copy-Item -Path "$sourcePath.crate" -Destination $targetPath + if (Test-Path -Path $targetContentsPath) { + Remove-Item -Path $targetContentsPath -Recurse -Force + } + + Write-Host "Copying package '$packageName' to '$targetContentsPath'" + New-Item -ItemType Directory -Path $targetContentsPath -Force | Out-Null + Copy-Item -Path $sourcePath/* -Destination $targetContentsPath -Recurse -Exclude "Cargo.toml.orig" Write-Host "Creating API review file" $apiReviewFile = Create-ApiViewFile $package diff --git a/eng/scripts/Pack-ReleaseCrates.ps1 b/eng/scripts/Pack-ReleaseCrates.ps1 new file mode 100755 index 0000000000..a10eb5bb57 --- /dev/null +++ b/eng/scripts/Pack-ReleaseCrates.ps1 @@ -0,0 +1,58 @@ +#!/usr/bin/env pwsh + +#Requires -Version 7.0 +[CmdletBinding(DefaultParameterSetName = "none")] +param( + [string]$OutputPath, + [string[]]$PackageNames, + [string]$OutBuildOrderFile = 'build-order.json' +) + +. ([System.IO.Path]::Combine($PSScriptRoot, 'Pack-Common.ps1')) + +# TODO: Ensure this works +$RepoRoot = [System.IO.Path]::Combine($PSScriptRoot, '../..') + +$metadata = Get-CargoMetadata +$packagesToBuild = $metadata.packages | Where-Object { $PackageNames.Contains($_.name) } + +$packageParams = @() +foreach ($package in $packagesToBuild) { + $packageParams += "--package", $package.name +} + +Write-Host "cargo publish --dry-run $($packageParams -join ' ') --target-dir $OutputPath --allow-dirty" +cargo publish --dry-run @packageParams --target-dir $OutputPath --allow-dirty 2>&1 | Tee-Object -Variable result + +if ($LASTEXITCODE) { + Write-Host "cargo publish failed with exit code $LASTEXITCODE" + exit $LASTEXITCODE +} + +foreach ($package in $packagesToBuild) { + $sourcePath = [System.IO.Path]::Combine($RepoRoot, "target", "package", "$($package.name)-$($package.version)") + $targetPath = [System.IO.Path]::Combine($OutputPath, $package.name) + $targetApiReviewFile = [System.IO.Path]::Combine($targetPath, "$($package.name).rust.json") + + Write-Host "Copying package '$($package.name)' to '$targetPath'" + New-Item -ItemType Directory -Path $targetPath -Force | Out-Null + Copy-Item -Path "$sourcePath.crate" -Destination $targetPath + + Write-Host "Creating API review file" + $apiReviewFile = Create-ApiViewFile $package + + Write-Host "Copying API review file to '$targetApiReviewFile'" + Copy-Item -Path $apiReviewFile -Destination $targetApiReviewFile -Force +} + +if ($OutBuildOrderFile) { + $buildOrder = @() + foreach ($line in $result) { + if ($line -match '^\s*Packaging (\w*) ([\w\d\.-]*)') { + $buildOrder += $matches[1] + } + } + + Write-Host "Build Order: $($buildOrder -join ', ')" + $buildOrder | ConvertTo-Json -Depth 100 | Set-Content $OutBuildOrderFile +} From 2a2774546695e81afab9d5bc3bbdb5349b6be02a Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Wed, 15 Oct 2025 04:49:28 +0000 Subject: [PATCH 77/87] Output path --- eng/scripts/Pack-ReleaseCrates.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/scripts/Pack-ReleaseCrates.ps1 b/eng/scripts/Pack-ReleaseCrates.ps1 index a10eb5bb57..fb65c21538 100755 --- a/eng/scripts/Pack-ReleaseCrates.ps1 +++ b/eng/scripts/Pack-ReleaseCrates.ps1 @@ -21,8 +21,8 @@ foreach ($package in $packagesToBuild) { $packageParams += "--package", $package.name } -Write-Host "cargo publish --dry-run $($packageParams -join ' ') --target-dir $OutputPath --allow-dirty" -cargo publish --dry-run @packageParams --target-dir $OutputPath --allow-dirty 2>&1 | Tee-Object -Variable result +Write-Host "cargo publish --dry-run $($packageParams -join ' ') --allow-dirty" +cargo publish --dry-run @packageParams --allow-dirty 2>&1 | Tee-Object -Variable result if ($LASTEXITCODE) { Write-Host "cargo publish failed with exit code $LASTEXITCODE" From 06dd993c7e1a28dab6d224ea39a78ad6de7d2b42 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Wed, 15 Oct 2025 05:02:40 +0000 Subject: [PATCH 78/87] Syntax --- eng/pipelines/templates/jobs/pack.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/templates/jobs/pack.yml b/eng/pipelines/templates/jobs/pack.yml index 575d1dbfd5..4d187b348c 100644 --- a/eng/pipelines/templates/jobs/pack.yml +++ b/eng/pipelines/templates/jobs/pack.yml @@ -78,10 +78,10 @@ jobs: $requireDependencies = $false } - $packageNames = $artifactsToBuild.name -join ',' + $packageNames = $artifactsToBuild.name if ($requireDependencies) { - Write-Host "Building crates with dependency validation for reelase" + Write-Host "Building crates with dependency validation for release" & $(Build.SourcesDirectory)/eng/scripts/Pack-ReleaseCrates.ps1 ` -OutputPath '$(Build.ArtifactStagingDirectory)' ` -PackageNames $packageNames ` From 474f902e79da401e6b24b211d0eba376397b641a Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Wed, 15 Oct 2025 05:32:29 +0000 Subject: [PATCH 79/87] Comments --- eng/scripts/Pack-Crates.ps1 | 1 - eng/scripts/Pack-ReleaseCrates.ps1 | 2 -- 2 files changed, 3 deletions(-) diff --git a/eng/scripts/Pack-Crates.ps1 b/eng/scripts/Pack-Crates.ps1 index 392745b1b4..9e8965e00f 100755 --- a/eng/scripts/Pack-Crates.ps1 +++ b/eng/scripts/Pack-Crates.ps1 @@ -13,7 +13,6 @@ param( $ErrorActionPreference = 'Stop' -# TODO: Don't use Join-Path . ([System.IO.Path]::Combine($PSScriptRoot, '..', 'common', 'scripts', 'common.ps1')) . ([System.IO.Path]::Combine($PSScriptRoot, 'Pack-Common.ps1')) diff --git a/eng/scripts/Pack-ReleaseCrates.ps1 b/eng/scripts/Pack-ReleaseCrates.ps1 index fb65c21538..51bec159e1 100755 --- a/eng/scripts/Pack-ReleaseCrates.ps1 +++ b/eng/scripts/Pack-ReleaseCrates.ps1 @@ -9,8 +9,6 @@ param( ) . ([System.IO.Path]::Combine($PSScriptRoot, 'Pack-Common.ps1')) - -# TODO: Ensure this works $RepoRoot = [System.IO.Path]::Combine($PSScriptRoot, '../..') $metadata = Get-CargoMetadata From 0d6f13d66b76aed06c5d3a5a0f5b1db6839b10b5 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Wed, 15 Oct 2025 17:58:57 +0000 Subject: [PATCH 80/87] Change releases to default: false --- sdk/canary/ci.yml | 4 ++-- sdk/core/ci.yml | 8 ++++---- sdk/cosmos/ci.yml | 2 +- sdk/eventhubs/ci.yml | 4 ++-- sdk/identity/ci.yml | 2 +- sdk/keyvault/ci.yml | 6 +++--- sdk/servicebus/ci.yml | 2 +- sdk/storage/ci.yml | 6 +++--- 8 files changed, 17 insertions(+), 17 deletions(-) diff --git a/sdk/canary/ci.yml b/sdk/canary/ci.yml index 9c580cac20..42067d7110 100644 --- a/sdk/canary/ci.yml +++ b/sdk/canary/ci.yml @@ -14,11 +14,11 @@ parameters: - name: release_azure_canary_core displayName: 'azure_canary_core' type: boolean - default: true + default: false - name: release_azure_canary displayName: 'azure_canary' type: boolean - default: true + default: false extends: template: /eng/pipelines/templates/stages/archetype-sdk-client.yml diff --git a/sdk/core/ci.yml b/sdk/core/ci.yml index 0eeb17b9aa..66821011eb 100644 --- a/sdk/core/ci.yml +++ b/sdk/core/ci.yml @@ -14,19 +14,19 @@ parameters: - name: release_azure_core displayName: 'azure_core' type: boolean - default: true + default: false - name: release_azure_core_macros displayName: 'azure_core_macros' type: boolean - default: true + default: false - name: release_azure_core_amqp displayName: 'azure_core_amqp' type: boolean - default: true + default: false - name: release_azure_core_opentelemetry displayName: 'azure_core_opentelemetry' type: boolean - default: true + default: false extends: template: /eng/pipelines/templates/stages/archetype-sdk-client.yml diff --git a/sdk/cosmos/ci.yml b/sdk/cosmos/ci.yml index b9426d0f1e..b309507535 100644 --- a/sdk/cosmos/ci.yml +++ b/sdk/cosmos/ci.yml @@ -14,7 +14,7 @@ parameters: - name: release_azure_data_cosmos displayName: 'azure_data_cosmos' type: boolean - default: true + default: false extends: template: /eng/pipelines/templates/stages/archetype-sdk-client.yml diff --git a/sdk/eventhubs/ci.yml b/sdk/eventhubs/ci.yml index bb69b06a76..c434185ef5 100644 --- a/sdk/eventhubs/ci.yml +++ b/sdk/eventhubs/ci.yml @@ -14,11 +14,11 @@ parameters: - name: release_azure_messaging_eventhubs displayName: 'azure_messaging_eventhubs' type: boolean - default: true + default: false - name: release_azure_messaging_eventhubs_checkpointstore_blob displayName: 'azure_messaging_eventhubs_checkpointstore_blob' type: boolean - default: true + default: false extends: template: /eng/pipelines/templates/stages/archetype-sdk-client.yml diff --git a/sdk/identity/ci.yml b/sdk/identity/ci.yml index 4dfa38e4c6..c7fd3b51a4 100644 --- a/sdk/identity/ci.yml +++ b/sdk/identity/ci.yml @@ -14,7 +14,7 @@ parameters: - name: release_azure_identity displayName: 'azure_identity' type: boolean - default: true + default: false extends: template: /eng/pipelines/templates/stages/archetype-sdk-client.yml diff --git a/sdk/keyvault/ci.yml b/sdk/keyvault/ci.yml index 8375eb0ec4..89e75475d9 100644 --- a/sdk/keyvault/ci.yml +++ b/sdk/keyvault/ci.yml @@ -17,15 +17,15 @@ parameters: - name: release_azure_security_keyvault_secrets displayName: 'azure_security_keyvault_secrets' type: boolean - default: true + default: false - name: release_azure_security_keyvault_keys displayName: 'azure_security_keyvault_keys' type: boolean - default: true + default: false - name: release_azure_security_keyvault_certificates displayName: 'azure_security_keyvault_certificates' type: boolean - default: true + default: false extends: template: /eng/pipelines/templates/stages/archetype-sdk-client.yml diff --git a/sdk/servicebus/ci.yml b/sdk/servicebus/ci.yml index d1c8778ce3..854e147db3 100644 --- a/sdk/servicebus/ci.yml +++ b/sdk/servicebus/ci.yml @@ -15,7 +15,7 @@ parameters: - name: release_azure_messaging_servicebus displayName: 'azure_messaging_servicebus' type: boolean - default: true + default: false extends: template: /eng/pipelines/templates/stages/archetype-sdk-client.yml diff --git a/sdk/storage/ci.yml b/sdk/storage/ci.yml index 11b2636ebb..ab077e74a7 100644 --- a/sdk/storage/ci.yml +++ b/sdk/storage/ci.yml @@ -17,15 +17,15 @@ parameters: - name: release_azure_storage_common displayName: 'azure_storage_common' type: boolean - default: true + default: false - name: release_azure_storage_blob displayName: 'azure_storage_blob' type: boolean - default: true + default: false - name: release_azure_storage_queue displayName: 'azure_storage_queue' type: boolean - default: true + default: false extends: template: /eng/pipelines/templates/stages/archetype-sdk-client.yml From 37e78b6dbce94eaf43ec276b1cf61c739a384f0e Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Wed, 15 Oct 2025 18:08:52 +0000 Subject: [PATCH 81/87] use cspell.json from main --- .vscode/cspell.json | 69 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 56 insertions(+), 13 deletions(-) diff --git a/.vscode/cspell.json b/.vscode/cspell.json index f66ef10c98..29996f7f5f 100644 --- a/.vscode/cspell.json +++ b/.vscode/cspell.json @@ -146,48 +146,91 @@ "overrides": [ { "filename": "**/Cargo.toml", - "dictionaries": ["crates", "rust-custom"] + "dictionaries": [ + "crates", + "rust-custom" + ] }, { "filename": "**/*.md", - "dictionaries": ["crates", "rust", "rust-custom"], - "ignoreRegExpList": ["@\\w+"] + "dictionaries": [ + "crates", + "rust", + "rust-custom" + ], + "ignoreRegExpList": [ + "@\\w+" + ] }, { "filename": "**/*.rs", - "dictionaries": ["crates", "rust-custom"] + "dictionaries": [ + "crates", + "rust-custom" + ] }, { "filename": "sdk/core/azure_core_amqp/**", - "dictionaries": ["crates", "rust-custom", "azure_core_amqp"] + "dictionaries": [ + "crates", + "rust-custom", + "azure_core_amqp" + ] }, { "filename": "sdk/eventhubs/**", - "dictionaries": ["crates", "rust-custom", "eventhubs"] + "dictionaries": [ + "crates", + "rust-custom", + "eventhubs" + ] }, { "filename": "sdk/identity/**", - "dictionaries": ["crates", "rust-custom", "identity"] + "dictionaries": [ + "crates", + "rust-custom", + "identity" + ] }, { "filename": "sdk/keyvault/**", - "dictionaries": ["crates", "rust-custom", "keyvault"] + "dictionaries": [ + "crates", + "rust-custom", + "keyvault" + ] }, { "filename": "sdk/core/typespec*/**", - "flagWords": ["azure", "azurite"] + "flagWords": [ + "azure", + "azurite" + ] }, { "filename": "sdk/cosmos/**", - "dictionaries": ["crates", "rust-custom", "cosmos"] + "dictionaries": [ + "crates", + "rust-custom", + "cosmos" + ] }, { "filename": "sdk/servicebus/**", - "dictionaries": ["crates", "rust-custom", "servicebus"] + "dictionaries": [ + "crates", + "rust-custom", + "servicebus" + ] }, { "filename": "sdk/storage/**", - "dictionaries": ["crates", "rust-custom", "storage"] + "dictionaries": [ + "crates", + "rust-custom", + "storage" + ] } ] -} +} \ No newline at end of file From 85f5cef04e765f4f67a89dca1852fc020d8ccd33 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Wed, 15 Oct 2025 18:13:19 +0000 Subject: [PATCH 82/87] Incorporate typespec into core/ci.yml --- sdk/core/ci.yml | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/sdk/core/ci.yml b/sdk/core/ci.yml index 66821011eb..c7dc313f70 100644 --- a/sdk/core/ci.yml +++ b/sdk/core/ci.yml @@ -10,21 +10,33 @@ trigger: include: - sdk/core/ -parameters: +parameters: +- name: typespec + displayName: typespec + type: boolean + default: false +- name: typespec_macros + displayName: typespec_macros + type: boolean + default: false +- name: typespec_client_core + displayName: typespec_client_core + type: boolean + default: false - name: release_azure_core - displayName: 'azure_core' + displayName: azure_core type: boolean default: false - name: release_azure_core_macros - displayName: 'azure_core_macros' + displayName: azure_core_macros type: boolean default: false - name: release_azure_core_amqp - displayName: 'azure_core_amqp' + displayName: azure_core_amqp type: boolean default: false - name: release_azure_core_opentelemetry - displayName: 'azure_core_opentelemetry' + displayName: azure_core_opentelemetry type: boolean default: false @@ -34,16 +46,16 @@ extends: ServiceDirectory: core Artifacts: - name: typespec - safeName: Typespec + releaseInBatch: ${{ parameters.typespec }} - name: typespec_macros - safeName: TypespecMacros + releaseInBatch: ${{ parameters.typespec_macros }} - name: typespec_client_core - safeName: TypespecClientCore + releaseInBatch: ${{ parameters.typespec_client_core }} - name: azure_core - releaseInBatch: ${{ parameters.release_azure_core }} + releaseInBatch: ${{ parameters.azure_core }} - name: azure_core_macros - releaseInBatch: ${{ parameters.release_azure_core_macros }} + releaseInBatch: ${{ parameters.azure_core_macros }} - name: azure_core_amqp - releaseInBatch: ${{ parameters.release_azure_core_amqp }} + releaseInBatch: ${{ parameters.azure_core_amqp }} - name: azure_core_opentelemetry - releaseInBatch: ${{ parameters.release_azure_core_opentelemetry }} + releaseInBatch: ${{ parameters.azure_core_opentelemetry }} From 530dddf996400cdbd07a05a8d96cc0d67c7a7fe8 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Wed, 15 Oct 2025 18:14:32 +0000 Subject: [PATCH 83/87] Remove redundant single quotes in yaml --- sdk/canary/ci.yml | 4 ++-- sdk/cosmos/ci.yml | 2 +- sdk/eventhubs/ci.yml | 4 ++-- sdk/identity/ci.yml | 2 +- sdk/keyvault/ci.yml | 8 ++++---- sdk/servicebus/ci.yml | 2 +- sdk/storage/ci.yml | 8 ++++---- 7 files changed, 15 insertions(+), 15 deletions(-) diff --git a/sdk/canary/ci.yml b/sdk/canary/ci.yml index 42067d7110..55e0c848fd 100644 --- a/sdk/canary/ci.yml +++ b/sdk/canary/ci.yml @@ -12,11 +12,11 @@ trigger: parameters: - name: release_azure_canary_core - displayName: 'azure_canary_core' + displayName: azure_canary_core type: boolean default: false - name: release_azure_canary - displayName: 'azure_canary' + displayName: azure_canary type: boolean default: false diff --git a/sdk/cosmos/ci.yml b/sdk/cosmos/ci.yml index b309507535..73bccf9941 100644 --- a/sdk/cosmos/ci.yml +++ b/sdk/cosmos/ci.yml @@ -12,7 +12,7 @@ trigger: parameters: - name: release_azure_data_cosmos - displayName: 'azure_data_cosmos' + displayName: azure_data_cosmos type: boolean default: false diff --git a/sdk/eventhubs/ci.yml b/sdk/eventhubs/ci.yml index c434185ef5..7b4ed58c71 100644 --- a/sdk/eventhubs/ci.yml +++ b/sdk/eventhubs/ci.yml @@ -12,11 +12,11 @@ trigger: parameters: - name: release_azure_messaging_eventhubs - displayName: 'azure_messaging_eventhubs' + displayName: azure_messaging_eventhubs type: boolean default: false - name: release_azure_messaging_eventhubs_checkpointstore_blob - displayName: 'azure_messaging_eventhubs_checkpointstore_blob' + displayName: azure_messaging_eventhubs_checkpointstore_blob type: boolean default: false diff --git a/sdk/identity/ci.yml b/sdk/identity/ci.yml index c7fd3b51a4..9908060d08 100644 --- a/sdk/identity/ci.yml +++ b/sdk/identity/ci.yml @@ -12,7 +12,7 @@ trigger: parameters: - name: release_azure_identity - displayName: 'azure_identity' + displayName: azure_identity type: boolean default: false diff --git a/sdk/keyvault/ci.yml b/sdk/keyvault/ci.yml index 89e75475d9..6ecd0157ac 100644 --- a/sdk/keyvault/ci.yml +++ b/sdk/keyvault/ci.yml @@ -11,19 +11,19 @@ trigger: parameters: - name: RunLiveTests - displayName: 'Run live tests' + displayName: Run live tests type: boolean default: false - name: release_azure_security_keyvault_secrets - displayName: 'azure_security_keyvault_secrets' + displayName: azure_security_keyvault_secrets type: boolean default: false - name: release_azure_security_keyvault_keys - displayName: 'azure_security_keyvault_keys' + displayName: azure_security_keyvault_keys type: boolean default: false - name: release_azure_security_keyvault_certificates - displayName: 'azure_security_keyvault_certificates' + displayName: azure_security_keyvault_certificates type: boolean default: false diff --git a/sdk/servicebus/ci.yml b/sdk/servicebus/ci.yml index 854e147db3..c9d7068834 100644 --- a/sdk/servicebus/ci.yml +++ b/sdk/servicebus/ci.yml @@ -13,7 +13,7 @@ trigger: parameters: - name: release_azure_messaging_servicebus - displayName: 'azure_messaging_servicebus' + displayName: azure_messaging_servicebus type: boolean default: false diff --git a/sdk/storage/ci.yml b/sdk/storage/ci.yml index ab077e74a7..7bdc72d1b1 100644 --- a/sdk/storage/ci.yml +++ b/sdk/storage/ci.yml @@ -11,19 +11,19 @@ trigger: parameters: - name: RunLiveTests - displayName: 'Run live tests' + displayName: Run live tests type: boolean default: false - name: release_azure_storage_common - displayName: 'azure_storage_common' + displayName: azure_storage_common type: boolean default: false - name: release_azure_storage_blob - displayName: 'azure_storage_blob' + displayName: azure_storage_blob type: boolean default: false - name: release_azure_storage_queue - displayName: 'azure_storage_queue' + displayName: azure_storage_queue type: boolean default: false From c15eaf2ec2e27ff1a961271e4646d2f9f0139249 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Wed, 15 Oct 2025 20:20:57 +0000 Subject: [PATCH 84/87] Refactor all packing to Pack-Crates.ps1 --- eng/pipelines/templates/jobs/pack.yml | 30 ++-- .../stages/archetype-rust-release.yml | 6 + eng/scripts/Pack-Common.ps1 | 15 -- eng/scripts/Pack-Crates.ps1 | 143 +++++++----------- eng/scripts/Pack-ReleaseCrates.ps1 | 56 ------- 5 files changed, 81 insertions(+), 169 deletions(-) delete mode 100644 eng/scripts/Pack-Common.ps1 delete mode 100755 eng/scripts/Pack-ReleaseCrates.ps1 diff --git a/eng/pipelines/templates/jobs/pack.yml b/eng/pipelines/templates/jobs/pack.yml index 4d187b348c..3d83f89b26 100644 --- a/eng/pipelines/templates/jobs/pack.yml +++ b/eng/pipelines/templates/jobs/pack.yml @@ -69,31 +69,37 @@ jobs: - ${{ else }}: - pwsh: | $artifacts = '${{ convertToJson(parameters.Artifacts) }}' | ConvertFrom-Json - $requireDependencies = $true + $isReleaseBuild = $true $artifactsToBuild = $artifacts | Where-Object { $_.releaseInBatch -eq 'True' } if (!$artifactsToBuild) { Write-Host "No packages to release. Building all packages in the service directory with no dependency validation." $artifactsToBuild = $artifacts - $requireDependencies = $false + $isReleaseBuild = $false } $packageNames = $artifactsToBuild.name - if ($requireDependencies) { - Write-Host "Building crates with dependency validation for release" - & $(Build.SourcesDirectory)/eng/scripts/Pack-ReleaseCrates.ps1 ` - -OutputPath '$(Build.ArtifactStagingDirectory)' ` - -PackageNames $packageNames ` - -OutBuildOrderFile '$(Build.ArtifactStagingDirectory)/release-order.json' + Write-Host "##vso[task.setvariable variable=PackageNames]$($packageNames -join ',')" + if ($isReleaseBuild) { + Write-Host "##vso[task.setvariable variable=AdditionalPackageParams]-Release" } else { - Write-Host "Building crates without validating dependencies" - & $(Build.SourcesDirectory)/eng/scripts/Pack-Crates.ps1 ` - -OutputPath '$(Build.ArtifactStagingDirectory)' ` - -PackageNames $packageNames + Write-Host "##vso[task.setvariable variable=AdditionalPackageParams]" } + displayName: Configure crate packing + condition: and(succeeded(), ne(variables['NoPackagesChanged'],'true')) + + - task: Powershell@2 displayName: Pack Crates condition: and(succeeded(), ne(variables['NoPackagesChanged'],'true')) + inputs: + pwsh: true + filePath: $(Build.SourcesDirectory)/eng/scripts/Pack-Crates.ps1 + arguments: > + -OutputPath '$(Build.ArtifactStagingDirectory)' + -PackageNames $(PackageNames) + $(AdditionalPackageParams) + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml parameters: diff --git a/eng/pipelines/templates/stages/archetype-rust-release.yml b/eng/pipelines/templates/stages/archetype-rust-release.yml index c630d1d3bc..aaaa2ff318 100644 --- a/eng/pipelines/templates/stages/archetype-rust-release.yml +++ b/eng/pipelines/templates/stages/archetype-rust-release.yml @@ -95,6 +95,12 @@ stages: # and use ArtifactIndex to select the right one $index = [int]'$(ArtifactIndex)' $artifacts = Get-Content '$(Pipeline.Workspace)/drop/release-order.json' | ConvertFrom-Json + + # Force $artifacts to be an array (PowerShell unrolls single-item arrays) + if ($artifacts -isnot [Array]) { + $artifacts = @($artifacts) + } + if ($index -ge $artifacts.Count) { Write-Error "ArtifactIndex $index is out of range (0..$($artifacts.Count - 1))" exit 1 diff --git a/eng/scripts/Pack-Common.ps1 b/eng/scripts/Pack-Common.ps1 deleted file mode 100644 index 92fa5768e7..0000000000 --- a/eng/scripts/Pack-Common.ps1 +++ /dev/null @@ -1,15 +0,0 @@ -. (Join-Path $PSScriptRoot '..' 'common' 'scripts' 'common.ps1') - -function Get-CargoMetadata() { - cargo metadata --no-deps --format-version 1 --manifest-path "$RepoRoot/Cargo.toml" | ConvertFrom-Json -Depth 100 -AsHashtable -} - -function Create-ApiViewFile($package) { - $packageName = $package.name - $command = "cargo run --manifest-path $RepoRoot/eng/tools/generate_api_report/Cargo.toml -- --package $packageName" - Invoke-LoggedCommand $command -GroupOutput | Out-Host - - $packagePath = Split-Path -Path $package.manifest_path -Parent - - "$packagePath/review/$packageName.rust.json" -} diff --git a/eng/scripts/Pack-Crates.ps1 b/eng/scripts/Pack-Crates.ps1 index 9e8965e00f..78c1b6c8e0 100755 --- a/eng/scripts/Pack-Crates.ps1 +++ b/eng/scripts/Pack-Crates.ps1 @@ -8,13 +8,14 @@ param( [string[]]$PackageNames, [Parameter(ParameterSetName = 'PackageInfo')] [string]$PackageInfoDirectory, - [switch]$NoVerify + [switch]$Release, + [switch]$NoVerify, + [string]$OutBuildOrderFile ) $ErrorActionPreference = 'Stop' . ([System.IO.Path]::Combine($PSScriptRoot, '..', 'common', 'scripts', 'common.ps1')) -. ([System.IO.Path]::Combine($PSScriptRoot, 'Pack-Common.ps1')) Write-Host @" Packing crates with @@ -61,8 +62,8 @@ function Get-OutputPackageNames($workspacePackages) { function Get-CargoPackages() { $metadata = Get-CargoMetadata - # path based depdenencies are assumed to be unreleased package versions - # they must be included in this build and build before packages that depend on them + # Path based dependencies are assumed to be unreleased package versions. In + # non-release builds these should be packed as well. foreach ($package in $metadata.packages) { $package.UnreleasedDependencies = @() foreach ($dependency in $package.dependencies) { @@ -80,9 +81,13 @@ function Get-PackagesToBuild() { $packages = Get-CargoPackages $outputPackageNames = Get-OutputPackageNames $packages - # We start with output packages, then recursively add unreleased dependencies to the list of packages that need to be built [array]$packagesToBuild = $packages | Where-Object { $outputPackageNames.Contains($_.name) } + if ($Release) { + return $packagesToBuild + } + + # If not releasing, expand dependencies into list of packages to build $toProcess = $packagesToBuild while ($toProcess.Length -gt 0) { $package = $toProcess[0] @@ -96,111 +101,77 @@ function Get-PackagesToBuild() { } } - $buildOrder = @() - - # Then we order the packages to that dependencies are built first - while ($packagesToBuild.Count -gt 0) { - # Pick any package with no unreleased dependencies, add it to the build order and remove it from the list of other packages' unreleased dependencies - $package = $packagesToBuild | Where-Object { $_.UnreleasedDependencies.Count -eq 0 } | Select-Object -First 1 - - if (-not $package) { - Write-Error "These packages cannot be built because they depend on unreleased dependencies that aren't being built." -ErrorAction Continue - foreach ($package in $packagesToBuild) { - Write-Error " $($package.name) -> $($package.UnreleasedDependencies -join ', ')" -ErrorAction Continue - } - exit 1 - } - - $package.OutputPackage = $outputPackageNames.Contains($package.name) - $buildOrder += $package - $packagesToBuild = @($packagesToBuild -ne $package) - - foreach ($otherPackage in $packagesToBuild) { - $otherPackage.UnreleasedDependencies = $otherPackage.UnreleasedDependencies -ne $package - } - } - - return $buildOrder + return $packagesToBuild } -function Initialize-VendorDirectory() { - $path = "$RepoRoot/target/vendor" - Invoke-LoggedCommand "cargo vendor $path" -GroupOutput | Out-Host - return $path +function Get-CargoMetadata() { + cargo metadata --no-deps --format-version 1 --manifest-path "$RepoRoot/Cargo.toml" | ConvertFrom-Json -Depth 100 -AsHashtable } -function Add-CrateToLocalRegistry($LocalRegistryPath, $Package) { - $packageName = $Package.name - $packageVersion = $Package.version - - # create an index entry for the package - $packagePath = "$RepoRoot/target/package/$packageName-$packageVersion" +function Create-ApiViewFile($package) { + $packageName = $package.name + $command = "cargo run --manifest-path $RepoRoot/eng/tools/generate_api_report/Cargo.toml -- --package $packageName" + Invoke-LoggedCommand $command -GroupOutput | Out-Host - Write-Host "Copying package '$packageName' to vendor directory '$LocalRegistryPath'" - Copy-Item -Path $packagePath -Destination $LocalRegistryPath -Recurse + $packagePath = Split-Path -Path $package.manifest_path -Parent - #write an empty checksum file - '{"files":{}}' | Out-File -FilePath "$LocalRegistryPath/$packageName-$packageVersion/.cargo-checksum.json" -Encoding utf8 + "$packagePath/review/$packageName.rust.json" } -Push-Location $RepoRoot +$originalLocation = Get-Location try { - $localRegistryPath = Initialize-VendorDirectory + Set-Location $RepoRoot [array]$packages = Get-PackagesToBuild - - Write-Host "Building packages in the following order:" + $packageParams = @() foreach ($package in $packages) { - $packageName = $package.name - $type = if ($package.OutputPackage) { "output" } else { "dependency" } - Write-Host " $packageName ($type)" + $packageParams += "--package", $package.name } - foreach ($package in $packages) { - Write-Host "" + if ($NoVerify) { + $packageParams += "--no-verify" + } - $packageName = $package.name - $packageVersion = $package.version + Write-Host "> cargo publish --locked --dry-run --allow-dirty $($packageParams -join ' ')" + & cargo publish --locked --dry-run --allow-dirty @packageParams 2>&1 | Tee-Object -Variable packResult + if ($LASTEXITCODE) { + Write-Host "cargo publish failed with exit code $LASTEXITCODE" + exit $LASTEXITCODE + } - $command = "cargo publish --locked --dry-run --package $packageName --registry crates-io --config `"source.crates-io.replace-with='local'`" --config `"source.local.directory='$localRegistryPath'`" --allow-dirty" + if ($OutputPath -and $package.OutputPackage) { + $sourcePath = [System.IO.Path]::Combine($RepoRoot, "target", "package", "$packageName-$packageVersion") + $targetPath = [System.IO.Path]::Combine($OutputPath, $packageName) + $targetContentsPath = [System.IO.Path]::Combine($targetPath, "contents") + $targetApiReviewFile = [System.IO.Path]::Combine($targetPath, "$packageName.rust.json") - if ($NoVerify) { - $command += " --no-verify" + if (Test-Path -Path $targetContentsPath) { + Remove-Item -Path $targetContentsPath -Recurse -Force } - Invoke-LoggedCommand -Command $command -GroupOutput + Write-Host "Copying package '$packageName' to '$targetContentsPath'" + New-Item -ItemType Directory -Path $targetContentsPath -Force | Out-Null + Copy-Item -Path $sourcePath/* -Destination $targetContentsPath -Recurse -Exclude "Cargo.toml.orig" + Write-Host "Creating API review file" + $apiReviewFile = Create-ApiViewFile $package + + Write-Host "Copying API review file to '$targetApiReviewFile'" + Copy-Item -Path $apiReviewFile -Destination $targetApiReviewFile -Force + } - # copy the package to the local registry - Add-CrateToLocalRegistry ` - -LocalRegistryPath $localRegistryPath ` - -Package $package - - if ($OutputPath -and $package.OutputPackage) { - $sourcePath = "$RepoRoot/target/package/$packageName-$packageVersion" - $targetPath = "$OutputPath/$packageName" - $targetContentsPath = "$targetPath/contents" - $targetApiReviewFile = "$targetPath/$packageName.rust.json" - - if (Test-Path -Path $targetContentsPath) { - Remove-Item -Path $targetContentsPath -Recurse -Force + if ($OutBuildOrderFile) { + $buildOrder = @() + foreach ($line in $packResult) { + if ($line -match '^\s*Packaging (\w*) ([\w\d\.-]*)') { + $buildOrder += $matches[1] } - - Write-Host "Copying package '$packageName' to '$targetContentsPath'" - New-Item -ItemType Directory -Path $targetContentsPath -Force | Out-Null - Copy-Item -Path $sourcePath/* -Destination $targetContentsPath -Recurse -Exclude "Cargo.toml.orig" - - Write-Host "Creating API review file" - $apiReviewFile = Create-ApiViewFile $package - - Write-Host "Copying API review file to '$targetApiReviewFile'" - Copy-Item -Path $apiReviewFile -Destination $targetApiReviewFile -Force } - } - Write-Host "Removing local registry" - Remove-Item -Path $localRegistryPath -Recurse -Force | Out-Null + Write-Host "Build Order: $($buildOrder -join ', ')" + $buildOrder | ConvertTo-Json -Depth 100 | Set-Content $OutBuildOrderFile + } } finally { - Pop-Location + Set-Location $originalLocation } diff --git a/eng/scripts/Pack-ReleaseCrates.ps1 b/eng/scripts/Pack-ReleaseCrates.ps1 deleted file mode 100755 index 51bec159e1..0000000000 --- a/eng/scripts/Pack-ReleaseCrates.ps1 +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env pwsh - -#Requires -Version 7.0 -[CmdletBinding(DefaultParameterSetName = "none")] -param( - [string]$OutputPath, - [string[]]$PackageNames, - [string]$OutBuildOrderFile = 'build-order.json' -) - -. ([System.IO.Path]::Combine($PSScriptRoot, 'Pack-Common.ps1')) -$RepoRoot = [System.IO.Path]::Combine($PSScriptRoot, '../..') - -$metadata = Get-CargoMetadata -$packagesToBuild = $metadata.packages | Where-Object { $PackageNames.Contains($_.name) } - -$packageParams = @() -foreach ($package in $packagesToBuild) { - $packageParams += "--package", $package.name -} - -Write-Host "cargo publish --dry-run $($packageParams -join ' ') --allow-dirty" -cargo publish --dry-run @packageParams --allow-dirty 2>&1 | Tee-Object -Variable result - -if ($LASTEXITCODE) { - Write-Host "cargo publish failed with exit code $LASTEXITCODE" - exit $LASTEXITCODE -} - -foreach ($package in $packagesToBuild) { - $sourcePath = [System.IO.Path]::Combine($RepoRoot, "target", "package", "$($package.name)-$($package.version)") - $targetPath = [System.IO.Path]::Combine($OutputPath, $package.name) - $targetApiReviewFile = [System.IO.Path]::Combine($targetPath, "$($package.name).rust.json") - - Write-Host "Copying package '$($package.name)' to '$targetPath'" - New-Item -ItemType Directory -Path $targetPath -Force | Out-Null - Copy-Item -Path "$sourcePath.crate" -Destination $targetPath - - Write-Host "Creating API review file" - $apiReviewFile = Create-ApiViewFile $package - - Write-Host "Copying API review file to '$targetApiReviewFile'" - Copy-Item -Path $apiReviewFile -Destination $targetApiReviewFile -Force -} - -if ($OutBuildOrderFile) { - $buildOrder = @() - foreach ($line in $result) { - if ($line -match '^\s*Packaging (\w*) ([\w\d\.-]*)') { - $buildOrder += $matches[1] - } - } - - Write-Host "Build Order: $($buildOrder -join ', ')" - $buildOrder | ConvertTo-Json -Depth 100 | Set-Content $OutBuildOrderFile -} From 6eee1c8de02fdc7c2efb6b77cf1e2b7a4d073e1e Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Wed, 15 Oct 2025 20:56:10 +0000 Subject: [PATCH 85/87] Pack-Crates.ps1 artifact output --- eng/scripts/Pack-Crates.ps1 | 43 ++++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/eng/scripts/Pack-Crates.ps1 b/eng/scripts/Pack-Crates.ps1 index 78c1b6c8e0..0592719f3f 100755 --- a/eng/scripts/Pack-Crates.ps1 +++ b/eng/scripts/Pack-Crates.ps1 @@ -22,10 +22,6 @@ Packing crates with RUSTFLAGS: '${env:RUSTFLAGS}' "@ -if ($OutputPath) { - $OutputPath = New-Item -ItemType Directory -Path $OutputPath -Force | Select-Object -ExpandProperty FullName -} - function Get-OutputPackageNames($workspacePackages) { $packablePackages = $workspacePackages | Where-Object -Property publish -NE -Value @() $packablePackageNames = $packablePackages.name @@ -139,25 +135,32 @@ try { exit $LASTEXITCODE } - if ($OutputPath -and $package.OutputPackage) { - $sourcePath = [System.IO.Path]::Combine($RepoRoot, "target", "package", "$packageName-$packageVersion") - $targetPath = [System.IO.Path]::Combine($OutputPath, $packageName) - $targetContentsPath = [System.IO.Path]::Combine($targetPath, "contents") - $targetApiReviewFile = [System.IO.Path]::Combine($targetPath, "$packageName.rust.json") + if ($OutputPath) { + $OutputPath = New-Item -ItemType Directory -Path $OutputPath -Force | Select-Object -ExpandProperty FullName - if (Test-Path -Path $targetContentsPath) { - Remove-Item -Path $targetContentsPath -Recurse -Force - } + foreach ($package in $packages) { + $sourcePath = [System.IO.Path]::Combine($RepoRoot, "target", "package", "$($package.name)-$($package.version)") + $targetPath = [System.IO.Path]::Combine($OutputPath, $package.name) + $targetContentsPath = [System.IO.Path]::Combine($targetPath, "contents") + $targetApiReviewFile = [System.IO.Path]::Combine($targetPath, "$($package.name).rust.json") - Write-Host "Copying package '$packageName' to '$targetContentsPath'" - New-Item -ItemType Directory -Path $targetContentsPath -Force | Out-Null - Copy-Item -Path $sourcePath/* -Destination $targetContentsPath -Recurse -Exclude "Cargo.toml.orig" + if (Test-Path -Path $targetContentsPath) { + Remove-Item -Path $targetContentsPath -Recurse -Force + } - Write-Host "Creating API review file" - $apiReviewFile = Create-ApiViewFile $package - - Write-Host "Copying API review file to '$targetApiReviewFile'" - Copy-Item -Path $apiReviewFile -Destination $targetApiReviewFile -Force + Write-Host "Copying package contents '$($package.name)' to '$targetContentsPath'" + New-Item -ItemType Directory -Path $targetContentsPath -Force | Out-Null + Copy-Item -Path $sourcePath/* -Destination $targetContentsPath -Recurse + + Write-Host "Copying .crate file for '$($package.name)' to '$targetPath'" + Copy-Item -Path "$sourcePath.crate" -Destination $targetPath -Force + + Write-Host "Creating API review file" + $apiReviewFile = Create-ApiViewFile $package + + Write-Host "Copying API review file to '$targetApiReviewFile'" + Copy-Item -Path $apiReviewFile -Destination $targetApiReviewFile -Force + } } if ($OutBuildOrderFile) { From cc1182f5fcfc9a92a03c11b260fbe0e4453458f0 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Wed, 15 Oct 2025 21:33:39 +0000 Subject: [PATCH 86/87] release-order.json --- eng/pipelines/templates/jobs/pack.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eng/pipelines/templates/jobs/pack.yml b/eng/pipelines/templates/jobs/pack.yml index 3d83f89b26..cbd258b83a 100644 --- a/eng/pipelines/templates/jobs/pack.yml +++ b/eng/pipelines/templates/jobs/pack.yml @@ -98,6 +98,7 @@ jobs: arguments: > -OutputPath '$(Build.ArtifactStagingDirectory)' -PackageNames $(PackageNames) + -OutBuildOrderFile '$(Build.ArtifactStagingDirectory)/release-order.json' $(AdditionalPackageParams) From 94e2da47b2424188280a9577d52531de3c6234a7 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Wed, 15 Oct 2025 18:22:08 +0000 Subject: [PATCH 87/87] Add dependency on azure_core@0.30.0 which has not been released --- sdk/canary/azure_canary_core/Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/canary/azure_canary_core/Cargo.toml b/sdk/canary/azure_canary_core/Cargo.toml index 18c2d595a9..156a62dab6 100644 --- a/sdk/canary/azure_canary_core/Cargo.toml +++ b/sdk/canary/azure_canary_core/Cargo.toml @@ -9,6 +9,7 @@ repository.workspace = true rust-version.workspace = true [dependencies] +azure_core = { workspace = true } [dev-dependencies] azure_canary.path = "../azure_canary"