From 56518ec7c9586599051101033caa9045ffe9bfb7 Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Wed, 25 Jun 2025 16:33:26 +0530 Subject: [PATCH 01/19] build pipeline publishing pdb as artifacts --- eng/pipelines/build-whl-pipeline.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/eng/pipelines/build-whl-pipeline.yml b/eng/pipelines/build-whl-pipeline.yml index 1926611a..7662abd2 100644 --- a/eng/pipelines/build-whl-pipeline.yml +++ b/eng/pipelines/build-whl-pipeline.yml @@ -158,6 +158,15 @@ jobs: publishLocation: 'Container' displayName: 'Publish all PYDs as artifacts' + # Publish the collected .pdb file(s) as build artifacts + - task: PublishBuildArtifacts@1 + condition: succeededOrFailed() + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)\all-pdbs' + ArtifactName: 'mssql-python-symbols' + publishLocation: 'Container' + displayName: 'Publish all PDBs as build artifacts' + # Publish the python arm64 libraries as build artifacts for next builds if ARM64 # We publish them only for ARM64 builds since we cannot install arm64 Python on x64 host # This allows us to reuse the libraries in future ARM64 builds @@ -169,7 +178,7 @@ jobs: ArtifactName: 'mssql-python-arm64-libs' publishLocation: 'Container' displayName: 'Publish arm64 libs as artifacts' - + # Publish the collected wheel file(s) as build artifacts - task: PublishBuildArtifacts@1 condition: succeededOrFailed() From 00a2e00b92687e312f1b41076c7b99a3fb7b2f86 Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Wed, 25 Jun 2025 17:10:10 +0530 Subject: [PATCH 02/19] added a job to publish in dummy release pipeline --- eng/pipelines/dummy-release-pipeline.yml | 203 ++++++++++++++++++----- 1 file changed, 164 insertions(+), 39 deletions(-) diff --git a/eng/pipelines/dummy-release-pipeline.yml b/eng/pipelines/dummy-release-pipeline.yml index 728ce88b..73c53586 100644 --- a/eng/pipelines/dummy-release-pipeline.yml +++ b/eng/pipelines/dummy-release-pipeline.yml @@ -4,48 +4,173 @@ variables: - group: 'ESRP Federated Creds (AME)' jobs: -- job: ReleaseESRPPackage - displayName: 'Release ESRP Package' +- job: PublishSymbols + # Use the latest Windows image for building pool: vmImage: 'windows-latest' - + displayName: 'Publish Symbols - Windows' + # Strategy matrix to build all combinations + + parameters: + - name: SymAccount + type: string + default: 'mssql-python' + + - name: symbolsVersion + type: string + # Default version is the build ID, which is unique for each build + # This will be used to identify the symbols in the symbol server + default: '$(Build.BuildId)' + + - name: symbolServer + type: string + default: '$(SymbolServer)' + + - name: symbolTokenUri + type: string + default: '$(SymbolTokenUri)' + + - name: requestName + type: string + # Default artifact name is the build definition name and build ID + # This will be used to identify the symbols in the symbol server + default: '$(Build.DefinitionName)-$(Build.BuildId)' + + - name: publishToServers + type: object + default: + internal: true + public: true + + - name: product + default: mssql-python + values: + - mssql-python + steps: - - task: DownloadPipelineArtifact@2 - inputs: - buildType: 'specific' - project: '$(System.TeamProject)' - definition: 2162 - buildVersionToDownload: 'latest' - branchName: '$(Build.SourceBranch)' - artifactName: 'mssql-python-wheels-dist' - targetPath: '$(Build.SourcesDirectory)\dist' - displayName: 'Download release wheel files artifact from latest successful run on main branch' - - # Show content of the downloaded artifact - script: | - echo "Contents of the dist directory:" - dir "$(Build.SourcesDirectory)\dist" - displayName: 'List contents of dist directory' + # Check what PDB files are available in the artifact staging directory + echo "Contents of the $(Build.ArtifactStagingDirectory) directory:" + dir "$(Build.ArtifactStagingDirectory)\all-pdbs" + displayName: 'List contents of all-pdbs directory' + + - powershell: 'Write-Host "##vso[task.setvariable variable=ArtifactServices.Symbol.AccountName;]${{parameters.SymAccount}}"' + displayName: 'Update Symbol.AccountName with ${{parameters.SymAccount}}' + + - task: PublishSymbols@2 + displayName: 'Upload symbols to ${{parameters.SymAccount }} org' + inputs: + SymbolsFolder: '$(Build.ArtifactStagingDirectory)\all-pdbs' + SearchPattern: '**/*.pdb' + IndexSources: false + SymbolServerType: TeamServices + SymbolsMaximumWaitTime: 60 + SymbolExpirationInDays: 1825 # 5 years + SymbolsProduct: ${{parameters.product}} + SymbolsVersion: ${{parameters.symbolsVersion}} + SymbolsArtifactName: ${{parameters.requestName}} + Pat: $(System.AccessToken) - - task: EsrpRelease@9 - displayName: 'ESRP Release' + - task: AzureCLI@2 + displayName: 'Publish symbols' inputs: - connectedservicename: '$(ESRPConnectedServiceName)' - usemanagedidentity: true - keyvaultname: '$(AuthAKVName)' - signcertname: '$(AuthSignCertName)' - clientid: '$(EsrpClientId)' - Intent: 'PackageDistribution' - # Changing content type to Maven release (NOT PyPI) since we want to do dummy release - # for ESRP testing purposes, not for actual PyPI distribution. - # This is a workaround to allow ESRP to process the release without actual PyPI content - # and to avoid ESRP validation errors. - ContentType: 'Maven' - ContentSource: 'Folder' - FolderLocation: '$(Build.SourcesDirectory)/dist' - WaitForReleaseCompletion: true - Owners: '$(owner)' - Approvers: '$(approver)' - ServiceEndpointUrl: 'https://api.esrp.microsoft.com' - MainPublisher: 'ESRPRELPACMAN' - DomainTenantId: '$(DomainTenantId)' + azureSubscription: 'Symbols publishing Workload Identity federation service-ADO.Net' + scriptType: ps + scriptLocation: inlineScript + inlineScript: | + $publishToInternalServer = "${{parameters.publishToServers.internal }}".ToLower() + $publishToPublicServer = "${{parameters.publishToServers.public }}".ToLower() + + echo "Publishing request name: ${{parameters.requestName }}" + echo "Publish to internal server: $publishToInternalServer" + echo "Publish to public server: $publishToPublicServer" + + $symbolServer = "${{parameters.symbolServer }}" + $tokenUri = "${{parameters.symbolTokenUri }}" + $projectName = "${{parameters.SymAccount }}" + + # Get the access token for the symbol publishing service + $symbolPublishingToken = az account get-access-token --resource $tokenUri --query accessToken -o tsv + + echo "> 1.Symbol publishing token acquired." + + echo "Registering the request name ..." + $requestName = "${{parameters.requestName }}" + $requestNameRegistrationBody = "{'requestName': '$requestName'}" + Invoke-RestMethod -Method POST -Uri "https://$symbolServer.trafficmanager.net/projects/$projectName/requests" -Headers @{ Authorization = "Bearer $symbolPublishingToken" } -ContentType "application/json" -Body $requestNameRegistrationBody + + echo "> 2.Registration of request name succeeded." + + echo "Publishing the symbols ..." + $publishSymbolsBody = "{'publishToInternalServer': $publishToInternalServer, 'publishToPublicServer': $publishToPublicServer}" + echo "Publishing symbols request body: $publishSymbolsBody" + Invoke-RestMethod -Method POST -Uri "https://$symbolServer.trafficmanager.net/projects/$projectName/requests/$requestName" -Headers @{ Authorization = "Bearer $symbolPublishingToken" } -ContentType "application/json" -Body $publishSymbolsBody + + echo "> 3.Request to publish symbols succeeded." + + # The following REST calls are used to check publishing status. + echo "> 4.Checking the status of the request ..." + + Invoke-RestMethod -Method GET -Uri "https://$symbolServer.trafficmanager.net/projects/$projectName/requests/$requestName" -Headers @{ Authorization = "Bearer $symbolPublishingToken" } -ContentType "application/json" + + echo "Use below tables to interpret the values of xxxServerStatus and xxxServerResult fields from the response." + + echo "PublishingStatus" + echo "-----------------" + echo "0 NotRequested; The request has not been requested to publish." + echo "1 Submitted; The request is submitted to be published" + echo "2 Processing; The request is still being processed" + echo "3 Completed; The request has been completed processing. It can be failed or successful. Check PublishingResult to get more details" + + echo "PublishingResult" + echo "-----------------" + echo "0 Pending; The request has not completed or has not been requested." + echo "1 Succeeded; The request has published successfully" + echo "2 Failed; The request has failed to publish" + echo "3 Cancelled; The request was cancelled" + +# - job: ReleaseESRPPackage +# displayName: 'Release ESRP Package' +# pool: +# vmImage: 'windows-latest' + +# steps: +# - task: DownloadPipelineArtifact@2 +# inputs: +# buildType: 'specific' +# project: '$(System.TeamProject)' +# definition: 2162 +# buildVersionToDownload: 'latest' +# branchName: '$(Build.SourceBranch)' +# artifactName: 'mssql-python-wheels-dist' +# targetPath: '$(Build.SourcesDirectory)\dist' +# displayName: 'Download release wheel files artifact from latest successful run on main branch' + +# # Show content of the downloaded artifact +# - script: | +# echo "Contents of the dist directory:" +# dir "$(Build.SourcesDirectory)\dist" +# displayName: 'List contents of dist directory' + +# - task: EsrpRelease@9 +# displayName: 'ESRP Release' +# inputs: +# connectedservicename: '$(ESRPConnectedServiceName)' +# usemanagedidentity: true +# keyvaultname: '$(AuthAKVName)' +# signcertname: '$(AuthSignCertName)' +# clientid: '$(EsrpClientId)' +# Intent: 'PackageDistribution' +# # Changing content type to Maven release (NOT PyPI) since we want to do dummy release +# # for ESRP testing purposes, not for actual PyPI distribution. +# # This is a workaround to allow ESRP to process the release without actual PyPI content +# # and to avoid ESRP validation errors. +# ContentType: 'Maven' +# ContentSource: 'Folder' +# FolderLocation: '$(Build.SourcesDirectory)/dist' +# WaitForReleaseCompletion: true +# Owners: '$(owner)' +# Approvers: '$(approver)' +# ServiceEndpointUrl: 'https://api.esrp.microsoft.com' +# MainPublisher: 'ESRPRELPACMAN' +# DomainTenantId: '$(DomainTenantId)' From 8e80e7def6be3643b97b0f6a9b2d8d438875adfa Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Wed, 25 Jun 2025 17:14:59 +0530 Subject: [PATCH 03/19] fixed pdb dir --- eng/pipelines/build-whl-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/build-whl-pipeline.yml b/eng/pipelines/build-whl-pipeline.yml index 7662abd2..2fec310d 100644 --- a/eng/pipelines/build-whl-pipeline.yml +++ b/eng/pipelines/build-whl-pipeline.yml @@ -128,7 +128,7 @@ jobs: # Copy the built .pdb files to staging folder for artifacts - task: CopyFiles@2 inputs: - SourceFolder: '$(Build.SourcesDirectory)\mssql_python\pybind\build\$(targetArch)\py$(shortPyVer)\Release' + SourceFolder: '$(Build.SourcesDirectory)\mssql_python' Contents: 'ddbc_bindings.cp$(shortPyVer)-*.pdbs' TargetFolder: '$(Build.ArtifactStagingDirectory)\all-pdbs' displayName: 'Place PDB file into artifacts directory' From 06379285622b64f97f5f9f094986d17f3151361f Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Wed, 25 Jun 2025 17:20:15 +0530 Subject: [PATCH 04/19] fixed pdb ext --- eng/pipelines/build-whl-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/build-whl-pipeline.yml b/eng/pipelines/build-whl-pipeline.yml index 2fec310d..187525b3 100644 --- a/eng/pipelines/build-whl-pipeline.yml +++ b/eng/pipelines/build-whl-pipeline.yml @@ -129,7 +129,7 @@ jobs: - task: CopyFiles@2 inputs: SourceFolder: '$(Build.SourcesDirectory)\mssql_python' - Contents: 'ddbc_bindings.cp$(shortPyVer)-*.pdbs' + Contents: 'ddbc_bindings.cp$(shortPyVer)-*.pdb' TargetFolder: '$(Build.ArtifactStagingDirectory)\all-pdbs' displayName: 'Place PDB file into artifacts directory' From e2a10376562cee906c39d55ae8d96044a02da5aa Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Wed, 25 Jun 2025 17:35:11 +0530 Subject: [PATCH 05/19] fixed parameters and made them into env --- eng/pipelines/dummy-release-pipeline.yml | 73 ++++++++---------------- 1 file changed, 23 insertions(+), 50 deletions(-) diff --git a/eng/pipelines/dummy-release-pipeline.yml b/eng/pipelines/dummy-release-pipeline.yml index 73c53586..8b06cfa7 100644 --- a/eng/pipelines/dummy-release-pipeline.yml +++ b/eng/pipelines/dummy-release-pipeline.yml @@ -11,42 +11,6 @@ jobs: displayName: 'Publish Symbols - Windows' # Strategy matrix to build all combinations - parameters: - - name: SymAccount - type: string - default: 'mssql-python' - - - name: symbolsVersion - type: string - # Default version is the build ID, which is unique for each build - # This will be used to identify the symbols in the symbol server - default: '$(Build.BuildId)' - - - name: symbolServer - type: string - default: '$(SymbolServer)' - - - name: symbolTokenUri - type: string - default: '$(SymbolTokenUri)' - - - name: requestName - type: string - # Default artifact name is the build definition name and build ID - # This will be used to identify the symbols in the symbol server - default: '$(Build.DefinitionName)-$(Build.BuildId)' - - - name: publishToServers - type: object - default: - internal: true - public: true - - - name: product - default: mssql-python - values: - - mssql-python - steps: - script: | # Check what PDB files are available in the artifact staging directory @@ -54,11 +18,11 @@ jobs: dir "$(Build.ArtifactStagingDirectory)\all-pdbs" displayName: 'List contents of all-pdbs directory' - - powershell: 'Write-Host "##vso[task.setvariable variable=ArtifactServices.Symbol.AccountName;]${{parameters.SymAccount}}"' - displayName: 'Update Symbol.AccountName with ${{parameters.SymAccount}}' + - powershell: 'Write-Host "##vso[task.setvariable variable=ArtifactServices.Symbol.AccountName;]mssql-python"' + displayName: 'Update Symbol.AccountName with mssql-python' - task: PublishSymbols@2 - displayName: 'Upload symbols to ${{parameters.SymAccount }} org' + displayName: 'Upload symbols to mssql-python org' inputs: SymbolsFolder: '$(Build.ArtifactStagingDirectory)\all-pdbs' SearchPattern: '**/*.pdb' @@ -66,28 +30,37 @@ jobs: SymbolServerType: TeamServices SymbolsMaximumWaitTime: 60 SymbolExpirationInDays: 1825 # 5 years - SymbolsProduct: ${{parameters.product}} - SymbolsVersion: ${{parameters.symbolsVersion}} - SymbolsArtifactName: ${{parameters.requestName}} + SymbolsProduct: mssql-python + # Have kept the default version as the build ID, which is unique for each build + # This will be used to identify the symbols in the symbol server + SymbolsVersion: $(Build.BuildId) + # Default artifact name is the build definition name and build ID + # This will be used to identify the symbols in the symbol server + SymbolsArtifactName: $(Build.DefinitionName)-$(Build.BuildId) Pat: $(System.AccessToken) - task: AzureCLI@2 displayName: 'Publish symbols' + env: + SymbolServer: '$(SymbolServer)' + SymbolTokenUri: '$(SymbolTokenUri)' + requestName: '$(Build.DefinitionName)-$(Build.BuildId)' inputs: azureSubscription: 'Symbols publishing Workload Identity federation service-ADO.Net' scriptType: ps scriptLocation: inlineScript inlineScript: | - $publishToInternalServer = "${{parameters.publishToServers.internal }}".ToLower() - $publishToPublicServer = "${{parameters.publishToServers.public }}".ToLower() + # Should be true by default for both internal and public servers + $publishToInternalServer = $true + $publishToPublicServer = $false - echo "Publishing request name: ${{parameters.requestName }}" + echo "Publishing request name: $requestName" echo "Publish to internal server: $publishToInternalServer" echo "Publish to public server: $publishToPublicServer" - $symbolServer = "${{parameters.symbolServer }}" - $tokenUri = "${{parameters.symbolTokenUri }}" - $projectName = "${{parameters.SymAccount }}" + $symbolServer = '$(SymbolServer)' + $tokenUri = '$(SymbolTokenUri)' + $projectName = "mssql-python" # Get the access token for the symbol publishing service $symbolPublishingToken = az account get-access-token --resource $tokenUri --query accessToken -o tsv @@ -95,7 +68,7 @@ jobs: echo "> 1.Symbol publishing token acquired." echo "Registering the request name ..." - $requestName = "${{parameters.requestName }}" + $requestName = '$(requestName)' $requestNameRegistrationBody = "{'requestName': '$requestName'}" Invoke-RestMethod -Method POST -Uri "https://$symbolServer.trafficmanager.net/projects/$projectName/requests" -Headers @{ Authorization = "Bearer $symbolPublishingToken" } -ContentType "application/json" -Body $requestNameRegistrationBody @@ -127,7 +100,7 @@ jobs: echo "0 Pending; The request has not completed or has not been requested." echo "1 Succeeded; The request has published successfully" echo "2 Failed; The request has failed to publish" - echo "3 Cancelled; The request was cancelled" + echo "3 Cancelled; The request was cancelled" # - job: ReleaseESRPPackage # displayName: 'Release ESRP Package' From 632f13b777fcb4eb938b572a49bf00e814ced4be Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Wed, 25 Jun 2025 17:39:27 +0530 Subject: [PATCH 06/19] fixed input azureSubscription --- eng/pipelines/dummy-release-pipeline.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/pipelines/dummy-release-pipeline.yml b/eng/pipelines/dummy-release-pipeline.yml index 8b06cfa7..e1a3bf7b 100644 --- a/eng/pipelines/dummy-release-pipeline.yml +++ b/eng/pipelines/dummy-release-pipeline.yml @@ -1,4 +1,4 @@ -name: mssql-python-official-release-pipeline +name: mssql-python-dummy-release-pipeline variables: - group: 'ESRP Federated Creds (AME)' @@ -46,11 +46,11 @@ jobs: SymbolTokenUri: '$(SymbolTokenUri)' requestName: '$(Build.DefinitionName)-$(Build.BuildId)' inputs: - azureSubscription: 'Symbols publishing Workload Identity federation service-ADO.Net' + azureSubscription: 'SymbolsPublishing' scriptType: ps scriptLocation: inlineScript inlineScript: | - # Should be true by default for both internal and public servers + # Should be true by default for internal server $publishToInternalServer = $true $publishToPublicServer = $false From 080af4f8bd61726fef6e15d43430b6d0d8e7e4b2 Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Wed, 25 Jun 2025 17:49:10 +0530 Subject: [PATCH 07/19] fixed input azureSubscription as mssql-python --- eng/pipelines/dummy-release-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/dummy-release-pipeline.yml b/eng/pipelines/dummy-release-pipeline.yml index e1a3bf7b..2ca389d2 100644 --- a/eng/pipelines/dummy-release-pipeline.yml +++ b/eng/pipelines/dummy-release-pipeline.yml @@ -46,7 +46,7 @@ jobs: SymbolTokenUri: '$(SymbolTokenUri)' requestName: '$(Build.DefinitionName)-$(Build.BuildId)' inputs: - azureSubscription: 'SymbolsPublishing' + azureSubscription: 'mssql-python' scriptType: ps scriptLocation: inlineScript inlineScript: | From 06caf7e5d2db14a9cc985bbd253e1a42e792abd8 Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Wed, 25 Jun 2025 17:50:37 +0530 Subject: [PATCH 08/19] fixed input azureSubscription as mssql-python-service --- eng/pipelines/dummy-release-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/dummy-release-pipeline.yml b/eng/pipelines/dummy-release-pipeline.yml index 2ca389d2..32d6359b 100644 --- a/eng/pipelines/dummy-release-pipeline.yml +++ b/eng/pipelines/dummy-release-pipeline.yml @@ -46,7 +46,7 @@ jobs: SymbolTokenUri: '$(SymbolTokenUri)' requestName: '$(Build.DefinitionName)-$(Build.BuildId)' inputs: - azureSubscription: 'mssql-python' + azureSubscription: 'mssql-python-service' scriptType: ps scriptLocation: inlineScript inlineScript: | From 1e7b29655ef838d59a04b788d7c523ac8255a86e Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Wed, 25 Jun 2025 17:58:14 +0530 Subject: [PATCH 09/19] fix download pipeline artifact --- eng/pipelines/dummy-release-pipeline.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/eng/pipelines/dummy-release-pipeline.yml b/eng/pipelines/dummy-release-pipeline.yml index 32d6359b..679b3308 100644 --- a/eng/pipelines/dummy-release-pipeline.yml +++ b/eng/pipelines/dummy-release-pipeline.yml @@ -12,11 +12,15 @@ jobs: # Strategy matrix to build all combinations steps: - - script: | - # Check what PDB files are available in the artifact staging directory - echo "Contents of the $(Build.ArtifactStagingDirectory) directory:" - dir "$(Build.ArtifactStagingDirectory)\all-pdbs" - displayName: 'List contents of all-pdbs directory' + - task: DownloadPipelineArtifact@2 + inputs: + buildType: 'specific' + project: '$(System.TeamProject)' + definition: 2162 + buildVersionToDownload: 'latest' + artifactName: 'mssql-python-symbols' + targetPath: '$(Build.ArtifactStagingDirectory)\all-pdbs' + displayName: 'Download PDB files artifact from latest successful run' - powershell: 'Write-Host "##vso[task.setvariable variable=ArtifactServices.Symbol.AccountName;]mssql-python"' displayName: 'Update Symbol.AccountName with mssql-python' From 026f9a8eb0ee9e4726d06a739d4eeba7cf94d752 Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Wed, 25 Jun 2025 18:05:51 +0530 Subject: [PATCH 10/19] rename SymbolsArtifactName --- eng/pipelines/dummy-release-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/dummy-release-pipeline.yml b/eng/pipelines/dummy-release-pipeline.yml index 679b3308..1daa4972 100644 --- a/eng/pipelines/dummy-release-pipeline.yml +++ b/eng/pipelines/dummy-release-pipeline.yml @@ -40,7 +40,7 @@ jobs: SymbolsVersion: $(Build.BuildId) # Default artifact name is the build definition name and build ID # This will be used to identify the symbols in the symbol server - SymbolsArtifactName: $(Build.DefinitionName)-$(Build.BuildId) + SymbolsArtifactName: mssql-python-symbols Pat: $(System.AccessToken) - task: AzureCLI@2 From c6372d1872f32924ffc23bffc58b12036b03743e Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Wed, 25 Jun 2025 18:20:51 +0530 Subject: [PATCH 11/19] changed symbolsArtifactName --- eng/pipelines/dummy-release-pipeline.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/eng/pipelines/dummy-release-pipeline.yml b/eng/pipelines/dummy-release-pipeline.yml index 1daa4972..5a9b7cf5 100644 --- a/eng/pipelines/dummy-release-pipeline.yml +++ b/eng/pipelines/dummy-release-pipeline.yml @@ -38,9 +38,9 @@ jobs: # Have kept the default version as the build ID, which is unique for each build # This will be used to identify the symbols in the symbol server SymbolsVersion: $(Build.BuildId) - # Default artifact name is the build definition name and build ID - # This will be used to identify the symbols in the symbol server - SymbolsArtifactName: mssql-python-symbols + # Ensuring the symbols are uniquely identified + # MDS uses symbolsArtifactName as mds_symbols_$(System.TeamProject)_$(Build.Repository.Name)_$(Build.SourceBranchName)_$(NuGetPackageVersion)_$(System.TimelineId) + SymbolsArtifactName: $(System.TeamProject)-$(Build.SourceBranchName)-$(Build.DefinitionName)-$(Build.BuildId) Pat: $(System.AccessToken) - task: AzureCLI@2 @@ -48,7 +48,7 @@ jobs: env: SymbolServer: '$(SymbolServer)' SymbolTokenUri: '$(SymbolTokenUri)' - requestName: '$(Build.DefinitionName)-$(Build.BuildId)' + requestName: '$(System.TeamProject)-$(Build.SourceBranchName)-$(Build.DefinitionName)-$(Build.BuildId)' inputs: azureSubscription: 'mssql-python-service' scriptType: ps From 640bf3ac8e2a8d4ce25178fa3108247216d09e08 Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Thu, 10 Jul 2025 10:41:00 +0530 Subject: [PATCH 12/19] code changes from review Co-authored-by: David Engel --- eng/pipelines/dummy-release-pipeline.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/eng/pipelines/dummy-release-pipeline.yml b/eng/pipelines/dummy-release-pipeline.yml index 5a9b7cf5..3c29d860 100644 --- a/eng/pipelines/dummy-release-pipeline.yml +++ b/eng/pipelines/dummy-release-pipeline.yml @@ -22,11 +22,11 @@ jobs: targetPath: '$(Build.ArtifactStagingDirectory)\all-pdbs' displayName: 'Download PDB files artifact from latest successful run' - - powershell: 'Write-Host "##vso[task.setvariable variable=ArtifactServices.Symbol.AccountName;]mssql-python"' - displayName: 'Update Symbol.AccountName with mssql-python' + - powershell: 'Write-Host "##vso[task.setvariable variable=ArtifactServices.Symbol.AccountName;]SqlClientDrivers"' + displayName: 'Update Symbol.AccountName with SqlClientDrivers' - task: PublishSymbols@2 - displayName: 'Upload symbols to mssql-python org' + displayName: 'Upload symbols to SqlClientDrivers org' inputs: SymbolsFolder: '$(Build.ArtifactStagingDirectory)\all-pdbs' SearchPattern: '**/*.pdb' @@ -50,7 +50,7 @@ jobs: SymbolTokenUri: '$(SymbolTokenUri)' requestName: '$(System.TeamProject)-$(Build.SourceBranchName)-$(Build.DefinitionName)-$(Build.BuildId)' inputs: - azureSubscription: 'mssql-python-service' + azureSubscription: 'SymbolsPublishing-msodbcsql-mssql-python' scriptType: ps scriptLocation: inlineScript inlineScript: | From dab82e869fddafe7d5bb4495760ffba0cd97996d Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Thu, 10 Jul 2025 11:11:36 +0530 Subject: [PATCH 13/19] add symbol generation and copy step in dummy release --- eng/pipelines/dummy-release-pipeline.yml | 47 +++++++++++++++++++----- 1 file changed, 38 insertions(+), 9 deletions(-) diff --git a/eng/pipelines/dummy-release-pipeline.yml b/eng/pipelines/dummy-release-pipeline.yml index 3c29d860..ed2f4dca 100644 --- a/eng/pipelines/dummy-release-pipeline.yml +++ b/eng/pipelines/dummy-release-pipeline.yml @@ -12,15 +12,44 @@ jobs: # Strategy matrix to build all combinations steps: - - task: DownloadPipelineArtifact@2 + - task: UsePythonVersion@0 inputs: - buildType: 'specific' - project: '$(System.TeamProject)' - definition: 2162 - buildVersionToDownload: 'latest' - artifactName: 'mssql-python-symbols' - targetPath: '$(Build.ArtifactStagingDirectory)\all-pdbs' - displayName: 'Download PDB files artifact from latest successful run' + versionSpec: '3.13' + architecture: 'x64' + addToPath: true + displayName: 'Use Python 3.13' + + # Install required packages: pip, CMake, pybind11 + - script: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install cmake pybind11 + displayName: 'Install dependencies' + + # Generate the symbols for the mssql-python package using build.bat + - script: | + echo "Generating symbols for mssql-python package..." + cd mssql_python\pybind + build.bat + cd .. + dir /s /b + displayName: 'Generate symbols for mssql-python package' + + # Copy the generated symbols to the staging folder for artifacts + - task: CopyFiles@2 + inputs: + SourceFolder: '$(Build.SourcesDirectory)\mssql_python' + Contents: '**\*.pdb' + TargetFolder: '$(Build.ArtifactStagingDirectory)\mssql-python-symbols' + displayName: 'Copy symbols to staging folder' + + # Publish the symbols (only pdbs) as an artifact + - task: PublishBuildArtifact@1 + displayName: 'Publish symbols as build artifact' + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)\mssql-python-symbols' + ArtifactName: 'mssql-python-symbols' + publishLocation: 'Container' - powershell: 'Write-Host "##vso[task.setvariable variable=ArtifactServices.Symbol.AccountName;]SqlClientDrivers"' displayName: 'Update Symbol.AccountName with SqlClientDrivers' @@ -28,7 +57,7 @@ jobs: - task: PublishSymbols@2 displayName: 'Upload symbols to SqlClientDrivers org' inputs: - SymbolsFolder: '$(Build.ArtifactStagingDirectory)\all-pdbs' + SymbolsFolder: '$(Build.ArtifactStagingDirectory)\mssql-python-symbols' SearchPattern: '**/*.pdb' IndexSources: false SymbolServerType: TeamServices From 3d68d94c1f3e9af94ec18113b2008226c2f7016b Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Thu, 10 Jul 2025 11:12:58 +0530 Subject: [PATCH 14/19] syntax --- eng/pipelines/dummy-release-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/dummy-release-pipeline.yml b/eng/pipelines/dummy-release-pipeline.yml index ed2f4dca..1418da37 100644 --- a/eng/pipelines/dummy-release-pipeline.yml +++ b/eng/pipelines/dummy-release-pipeline.yml @@ -44,7 +44,7 @@ jobs: displayName: 'Copy symbols to staging folder' # Publish the symbols (only pdbs) as an artifact - - task: PublishBuildArtifact@1 + - task: PublishBuildArtifacts@1 displayName: 'Publish symbols as build artifact' inputs: PathtoPublish: '$(Build.ArtifactStagingDirectory)\mssql-python-symbols' From b987d0dee9f5a042f2e9c8e564d8d16a870b1fd1 Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Thu, 10 Jul 2025 14:02:40 +0530 Subject: [PATCH 15/19] use onebranch temp --- eng/pipelines/dummy-release-pipeline.yml | 270 ++++++++++++----------- 1 file changed, 139 insertions(+), 131 deletions(-) diff --git a/eng/pipelines/dummy-release-pipeline.yml b/eng/pipelines/dummy-release-pipeline.yml index 1418da37..c774a2b8 100644 --- a/eng/pipelines/dummy-release-pipeline.yml +++ b/eng/pipelines/dummy-release-pipeline.yml @@ -1,139 +1,147 @@ +trigger: none + name: mssql-python-dummy-release-pipeline variables: - group: 'ESRP Federated Creds (AME)' -jobs: -- job: PublishSymbols - # Use the latest Windows image for building - pool: - vmImage: 'windows-latest' - displayName: 'Publish Symbols - Windows' - # Strategy matrix to build all combinations - - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.13' - architecture: 'x64' - addToPath: true - displayName: 'Use Python 3.13' - - # Install required packages: pip, CMake, pybind11 - - script: | - python -m pip install --upgrade pip - pip install -r requirements.txt - pip install cmake pybind11 - displayName: 'Install dependencies' - - # Generate the symbols for the mssql-python package using build.bat - - script: | - echo "Generating symbols for mssql-python package..." - cd mssql_python\pybind - build.bat - cd .. - dir /s /b - displayName: 'Generate symbols for mssql-python package' - - # Copy the generated symbols to the staging folder for artifacts - - task: CopyFiles@2 - inputs: - SourceFolder: '$(Build.SourcesDirectory)\mssql_python' - Contents: '**\*.pdb' - TargetFolder: '$(Build.ArtifactStagingDirectory)\mssql-python-symbols' - displayName: 'Copy symbols to staging folder' - - # Publish the symbols (only pdbs) as an artifact - - task: PublishBuildArtifacts@1 - displayName: 'Publish symbols as build artifact' - inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)\mssql-python-symbols' - ArtifactName: 'mssql-python-symbols' - publishLocation: 'Container' - - - powershell: 'Write-Host "##vso[task.setvariable variable=ArtifactServices.Symbol.AccountName;]SqlClientDrivers"' - displayName: 'Update Symbol.AccountName with SqlClientDrivers' - - - task: PublishSymbols@2 - displayName: 'Upload symbols to SqlClientDrivers org' - inputs: - SymbolsFolder: '$(Build.ArtifactStagingDirectory)\mssql-python-symbols' - SearchPattern: '**/*.pdb' - IndexSources: false - SymbolServerType: TeamServices - SymbolsMaximumWaitTime: 60 - SymbolExpirationInDays: 1825 # 5 years - SymbolsProduct: mssql-python - # Have kept the default version as the build ID, which is unique for each build - # This will be used to identify the symbols in the symbol server - SymbolsVersion: $(Build.BuildId) - # Ensuring the symbols are uniquely identified - # MDS uses symbolsArtifactName as mds_symbols_$(System.TeamProject)_$(Build.Repository.Name)_$(Build.SourceBranchName)_$(NuGetPackageVersion)_$(System.TimelineId) - SymbolsArtifactName: $(System.TeamProject)-$(Build.SourceBranchName)-$(Build.DefinitionName)-$(Build.BuildId) - Pat: $(System.AccessToken) - - - task: AzureCLI@2 - displayName: 'Publish symbols' - env: - SymbolServer: '$(SymbolServer)' - SymbolTokenUri: '$(SymbolTokenUri)' - requestName: '$(System.TeamProject)-$(Build.SourceBranchName)-$(Build.DefinitionName)-$(Build.BuildId)' - inputs: - azureSubscription: 'SymbolsPublishing-msodbcsql-mssql-python' - scriptType: ps - scriptLocation: inlineScript - inlineScript: | - # Should be true by default for internal server - $publishToInternalServer = $true - $publishToPublicServer = $false - - echo "Publishing request name: $requestName" - echo "Publish to internal server: $publishToInternalServer" - echo "Publish to public server: $publishToPublicServer" - - $symbolServer = '$(SymbolServer)' - $tokenUri = '$(SymbolTokenUri)' - $projectName = "mssql-python" - - # Get the access token for the symbol publishing service - $symbolPublishingToken = az account get-access-token --resource $tokenUri --query accessToken -o tsv - - echo "> 1.Symbol publishing token acquired." - - echo "Registering the request name ..." - $requestName = '$(requestName)' - $requestNameRegistrationBody = "{'requestName': '$requestName'}" - Invoke-RestMethod -Method POST -Uri "https://$symbolServer.trafficmanager.net/projects/$projectName/requests" -Headers @{ Authorization = "Bearer $symbolPublishingToken" } -ContentType "application/json" -Body $requestNameRegistrationBody - - echo "> 2.Registration of request name succeeded." - - echo "Publishing the symbols ..." - $publishSymbolsBody = "{'publishToInternalServer': $publishToInternalServer, 'publishToPublicServer': $publishToPublicServer}" - echo "Publishing symbols request body: $publishSymbolsBody" - Invoke-RestMethod -Method POST -Uri "https://$symbolServer.trafficmanager.net/projects/$projectName/requests/$requestName" -Headers @{ Authorization = "Bearer $symbolPublishingToken" } -ContentType "application/json" -Body $publishSymbolsBody - - echo "> 3.Request to publish symbols succeeded." - - # The following REST calls are used to check publishing status. - echo "> 4.Checking the status of the request ..." - - Invoke-RestMethod -Method GET -Uri "https://$symbolServer.trafficmanager.net/projects/$projectName/requests/$requestName" -Headers @{ Authorization = "Bearer $symbolPublishingToken" } -ContentType "application/json" - - echo "Use below tables to interpret the values of xxxServerStatus and xxxServerResult fields from the response." - - echo "PublishingStatus" - echo "-----------------" - echo "0 NotRequested; The request has not been requested to publish." - echo "1 Submitted; The request is submitted to be published" - echo "2 Processing; The request is still being processed" - echo "3 Completed; The request has been completed processing. It can be failed or successful. Check PublishingResult to get more details" - - echo "PublishingResult" - echo "-----------------" - echo "0 Pending; The request has not completed or has not been requested." - echo "1 Succeeded; The request has published successfully" - echo "2 Failed; The request has failed to publish" - echo "3 Cancelled; The request was cancelled" +extends: + template: v2/OneBranch.Official.CrossPlat.yml@templates + parameters: + featureFlags: + WindowsHostVersion: 1ESWindows2022 + + jobs: + - job: PublishSymbols + # Use the latest Windows image for building + pool: + vmImage: 'windows-latest' + displayName: 'Publish Symbols - Windows' + # Strategy matrix to build all combinations + + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.13' + architecture: 'x64' + addToPath: true + displayName: 'Use Python 3.13' + + # Install required packages: pip, CMake, pybind11 + - script: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install cmake pybind11 + displayName: 'Install dependencies' + + # Generate the symbols for the mssql-python package using build.bat + - script: | + echo "Generating symbols for mssql-python package..." + cd mssql_python\pybind + build.bat + cd .. + dir /s /b + displayName: 'Generate symbols for mssql-python package' + + # Copy the generated symbols to the staging folder for artifacts + - task: CopyFiles@2 + inputs: + SourceFolder: '$(Build.SourcesDirectory)\mssql_python' + Contents: '**\*.pdb' + TargetFolder: '$(Build.ArtifactStagingDirectory)\mssql-python-symbols' + displayName: 'Copy symbols to staging folder' + + # Publish the symbols (only pdbs) as an artifact + - task: PublishBuildArtifacts@1 + displayName: 'Publish symbols as build artifact' + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)\mssql-python-symbols' + ArtifactName: 'mssql-python-symbols' + publishLocation: 'Container' + + - powershell: 'Write-Host "##vso[task.setvariable variable=ArtifactServices.Symbol.AccountName;]SqlClientDrivers"' + displayName: 'Update Symbol.AccountName with SqlClientDrivers' + + - task: PublishSymbols@2 + displayName: 'Upload symbols to SqlClientDrivers org' + inputs: + SymbolsFolder: '$(Build.ArtifactStagingDirectory)\mssql-python-symbols' + SearchPattern: '**/*.pdb' + IndexSources: false + SymbolServerType: TeamServices + SymbolsMaximumWaitTime: 60 + SymbolExpirationInDays: 1825 # 5 years + SymbolsProduct: mssql-python + # Have kept the default version as the build ID, which is unique for each build + # This will be used to identify the symbols in the symbol server + SymbolsVersion: $(Build.BuildId) + # Ensuring the symbols are uniquely identified + # MDS uses symbolsArtifactName as mds_symbols_$(System.TeamProject)_$(Build.Repository.Name)_$(Build.SourceBranchName)_$(NuGetPackageVersion)_$(System.TimelineId) + SymbolsArtifactName: $(System.TeamProject)-$(Build.SourceBranchName)-$(Build.DefinitionName)-$(Build.BuildId) + Pat: $(System.AccessToken) + + - task: AzureCLI@2 + displayName: 'Publish symbols' + env: + SymbolServer: '$(SymbolServer)' + SymbolTokenUri: '$(SymbolTokenUri)' + requestName: '$(System.TeamProject)-$(Build.SourceBranchName)-$(Build.DefinitionName)-$(Build.BuildId)' + inputs: + azureSubscription: 'SymbolsPublishing-msodbcsql-mssql-python' + scriptType: ps + scriptLocation: inlineScript + inlineScript: | + # Should be true by default for internal server + $publishToInternalServer = $true + $publishToPublicServer = $false + + echo "Publishing request name: $requestName" + echo "Publish to internal server: $publishToInternalServer" + echo "Publish to public server: $publishToPublicServer" + + $symbolServer = '$(SymbolServer)' + $tokenUri = '$(SymbolTokenUri)' + $projectName = "mssql-python" + + # Get the access token for the symbol publishing service + $symbolPublishingToken = az account get-access-token --resource $tokenUri --query accessToken -o tsv + + echo "> 1.Symbol publishing token acquired." + + echo "Registering the request name ..." + $requestName = '$(requestName)' + $requestNameRegistrationBody = "{'requestName': '$requestName'}" + Invoke-RestMethod -Method POST -Uri "https://$symbolServer.trafficmanager.net/projects/$projectName/requests" -Headers @{ Authorization = "Bearer $symbolPublishingToken" } -ContentType "application/json" -Body $requestNameRegistrationBody + + echo "> 2.Registration of request name succeeded." + + echo "Publishing the symbols ..." + $publishSymbolsBody = "{'publishToInternalServer': $publishToInternalServer, 'publishToPublicServer': $publishToPublicServer}" + echo "Publishing symbols request body: $publishSymbolsBody" + Invoke-RestMethod -Method POST -Uri "https://$symbolServer.trafficmanager.net/projects/$projectName/requests/$requestName" -Headers @{ Authorization = "Bearer $symbolPublishingToken" } -ContentType "application/json" -Body $publishSymbolsBody + + echo "> 3.Request to publish symbols succeeded." + + # The following REST calls are used to check publishing status. + echo "> 4.Checking the status of the request ..." + + Invoke-RestMethod -Method GET -Uri "https://$symbolServer.trafficmanager.net/projects/$projectName/requests/$requestName" -Headers @{ Authorization = "Bearer $symbolPublishingToken" } -ContentType "application/json" + + echo "Use below tables to interpret the values of xxxServerStatus and xxxServerResult fields from the response." + + echo "PublishingStatus" + echo "-----------------" + echo "0 NotRequested; The request has not been requested to publish." + echo "1 Submitted; The request is submitted to be published" + echo "2 Processing; The request is still being processed" + echo "3 Completed; The request has been completed processing. It can be failed or successful. Check PublishingResult to get more details" + + echo "PublishingResult" + echo "-----------------" + echo "0 Pending; The request has not completed or has not been requested." + echo "1 Succeeded; The request has published successfully" + echo "2 Failed; The request has failed to publish" + echo "3 Cancelled; The request was cancelled" # - job: ReleaseESRPPackage # displayName: 'Release ESRP Package' From bfab7cfe1114f547c90fc9aa5e41d1054b1aa356 Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Thu, 10 Jul 2025 14:04:44 +0530 Subject: [PATCH 16/19] use onebranch temp --- eng/pipelines/dummy-release-pipeline.yml | 265 ++++++++++++----------- 1 file changed, 134 insertions(+), 131 deletions(-) diff --git a/eng/pipelines/dummy-release-pipeline.yml b/eng/pipelines/dummy-release-pipeline.yml index c774a2b8..d22465bd 100644 --- a/eng/pipelines/dummy-release-pipeline.yml +++ b/eng/pipelines/dummy-release-pipeline.yml @@ -11,137 +11,140 @@ extends: featureFlags: WindowsHostVersion: 1ESWindows2022 - jobs: - - job: PublishSymbols - # Use the latest Windows image for building - pool: - vmImage: 'windows-latest' - displayName: 'Publish Symbols - Windows' - # Strategy matrix to build all combinations - - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.13' - architecture: 'x64' - addToPath: true - displayName: 'Use Python 3.13' - - # Install required packages: pip, CMake, pybind11 - - script: | - python -m pip install --upgrade pip - pip install -r requirements.txt - pip install cmake pybind11 - displayName: 'Install dependencies' - - # Generate the symbols for the mssql-python package using build.bat - - script: | - echo "Generating symbols for mssql-python package..." - cd mssql_python\pybind - build.bat - cd .. - dir /s /b - displayName: 'Generate symbols for mssql-python package' - - # Copy the generated symbols to the staging folder for artifacts - - task: CopyFiles@2 - inputs: - SourceFolder: '$(Build.SourcesDirectory)\mssql_python' - Contents: '**\*.pdb' - TargetFolder: '$(Build.ArtifactStagingDirectory)\mssql-python-symbols' - displayName: 'Copy symbols to staging folder' - - # Publish the symbols (only pdbs) as an artifact - - task: PublishBuildArtifacts@1 - displayName: 'Publish symbols as build artifact' - inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)\mssql-python-symbols' - ArtifactName: 'mssql-python-symbols' - publishLocation: 'Container' - - - powershell: 'Write-Host "##vso[task.setvariable variable=ArtifactServices.Symbol.AccountName;]SqlClientDrivers"' - displayName: 'Update Symbol.AccountName with SqlClientDrivers' - - - task: PublishSymbols@2 - displayName: 'Upload symbols to SqlClientDrivers org' - inputs: - SymbolsFolder: '$(Build.ArtifactStagingDirectory)\mssql-python-symbols' - SearchPattern: '**/*.pdb' - IndexSources: false - SymbolServerType: TeamServices - SymbolsMaximumWaitTime: 60 - SymbolExpirationInDays: 1825 # 5 years - SymbolsProduct: mssql-python - # Have kept the default version as the build ID, which is unique for each build - # This will be used to identify the symbols in the symbol server - SymbolsVersion: $(Build.BuildId) - # Ensuring the symbols are uniquely identified - # MDS uses symbolsArtifactName as mds_symbols_$(System.TeamProject)_$(Build.Repository.Name)_$(Build.SourceBranchName)_$(NuGetPackageVersion)_$(System.TimelineId) - SymbolsArtifactName: $(System.TeamProject)-$(Build.SourceBranchName)-$(Build.DefinitionName)-$(Build.BuildId) - Pat: $(System.AccessToken) - - - task: AzureCLI@2 - displayName: 'Publish symbols' - env: - SymbolServer: '$(SymbolServer)' - SymbolTokenUri: '$(SymbolTokenUri)' - requestName: '$(System.TeamProject)-$(Build.SourceBranchName)-$(Build.DefinitionName)-$(Build.BuildId)' - inputs: - azureSubscription: 'SymbolsPublishing-msodbcsql-mssql-python' - scriptType: ps - scriptLocation: inlineScript - inlineScript: | - # Should be true by default for internal server - $publishToInternalServer = $true - $publishToPublicServer = $false - - echo "Publishing request name: $requestName" - echo "Publish to internal server: $publishToInternalServer" - echo "Publish to public server: $publishToPublicServer" - - $symbolServer = '$(SymbolServer)' - $tokenUri = '$(SymbolTokenUri)' - $projectName = "mssql-python" - - # Get the access token for the symbol publishing service - $symbolPublishingToken = az account get-access-token --resource $tokenUri --query accessToken -o tsv - - echo "> 1.Symbol publishing token acquired." - - echo "Registering the request name ..." - $requestName = '$(requestName)' - $requestNameRegistrationBody = "{'requestName': '$requestName'}" - Invoke-RestMethod -Method POST -Uri "https://$symbolServer.trafficmanager.net/projects/$projectName/requests" -Headers @{ Authorization = "Bearer $symbolPublishingToken" } -ContentType "application/json" -Body $requestNameRegistrationBody - - echo "> 2.Registration of request name succeeded." - - echo "Publishing the symbols ..." - $publishSymbolsBody = "{'publishToInternalServer': $publishToInternalServer, 'publishToPublicServer': $publishToPublicServer}" - echo "Publishing symbols request body: $publishSymbolsBody" - Invoke-RestMethod -Method POST -Uri "https://$symbolServer.trafficmanager.net/projects/$projectName/requests/$requestName" -Headers @{ Authorization = "Bearer $symbolPublishingToken" } -ContentType "application/json" -Body $publishSymbolsBody - - echo "> 3.Request to publish symbols succeeded." - - # The following REST calls are used to check publishing status. - echo "> 4.Checking the status of the request ..." - - Invoke-RestMethod -Method GET -Uri "https://$symbolServer.trafficmanager.net/projects/$projectName/requests/$requestName" -Headers @{ Authorization = "Bearer $symbolPublishingToken" } -ContentType "application/json" - - echo "Use below tables to interpret the values of xxxServerStatus and xxxServerResult fields from the response." - - echo "PublishingStatus" - echo "-----------------" - echo "0 NotRequested; The request has not been requested to publish." - echo "1 Submitted; The request is submitted to be published" - echo "2 Processing; The request is still being processed" - echo "3 Completed; The request has been completed processing. It can be failed or successful. Check PublishingResult to get more details" - - echo "PublishingResult" - echo "-----------------" - echo "0 Pending; The request has not completed or has not been requested." - echo "1 Succeeded; The request has published successfully" - echo "2 Failed; The request has failed to publish" - echo "3 Cancelled; The request was cancelled" + stages: + - stage: PublishSymbols + displayName: 'Publish Symbols' + jobs: + - job: PublishSymbols + # Use the latest Windows image for building + pool: + vmImage: 'windows-latest' + displayName: 'Publish Symbols - Windows' + # Strategy matrix to build all combinations + + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.13' + architecture: 'x64' + addToPath: true + displayName: 'Use Python 3.13' + + # Install required packages: pip, CMake, pybind11 + - script: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install cmake pybind11 + displayName: 'Install dependencies' + + # Generate the symbols for the mssql-python package using build.bat + - script: | + echo "Generating symbols for mssql-python package..." + cd mssql_python\pybind + build.bat + cd .. + dir /s /b + displayName: 'Generate symbols for mssql-python package' + + # Copy the generated symbols to the staging folder for artifacts + - task: CopyFiles@2 + inputs: + SourceFolder: '$(Build.SourcesDirectory)\mssql_python' + Contents: '**\*.pdb' + TargetFolder: '$(Build.ArtifactStagingDirectory)\mssql-python-symbols' + displayName: 'Copy symbols to staging folder' + + # Publish the symbols (only pdbs) as an artifact + - task: PublishBuildArtifacts@1 + displayName: 'Publish symbols as build artifact' + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)\mssql-python-symbols' + ArtifactName: 'mssql-python-symbols' + publishLocation: 'Container' + + - powershell: 'Write-Host "##vso[task.setvariable variable=ArtifactServices.Symbol.AccountName;]SqlClientDrivers"' + displayName: 'Update Symbol.AccountName with SqlClientDrivers' + + - task: PublishSymbols@2 + displayName: 'Upload symbols to SqlClientDrivers org' + inputs: + SymbolsFolder: '$(Build.ArtifactStagingDirectory)\mssql-python-symbols' + SearchPattern: '**/*.pdb' + IndexSources: false + SymbolServerType: TeamServices + SymbolsMaximumWaitTime: 60 + SymbolExpirationInDays: 1825 # 5 years + SymbolsProduct: mssql-python + # Have kept the default version as the build ID, which is unique for each build + # This will be used to identify the symbols in the symbol server + SymbolsVersion: $(Build.BuildId) + # Ensuring the symbols are uniquely identified + # MDS uses symbolsArtifactName as mds_symbols_$(System.TeamProject)_$(Build.Repository.Name)_$(Build.SourceBranchName)_$(NuGetPackageVersion)_$(System.TimelineId) + SymbolsArtifactName: $(System.TeamProject)-$(Build.SourceBranchName)-$(Build.DefinitionName)-$(Build.BuildId) + Pat: $(System.AccessToken) + + - task: AzureCLI@2 + displayName: 'Publish symbols' + env: + SymbolServer: '$(SymbolServer)' + SymbolTokenUri: '$(SymbolTokenUri)' + requestName: '$(System.TeamProject)-$(Build.SourceBranchName)-$(Build.DefinitionName)-$(Build.BuildId)' + inputs: + azureSubscription: 'SymbolsPublishing-msodbcsql-mssql-python' + scriptType: ps + scriptLocation: inlineScript + inlineScript: | + # Should be true by default for internal server + $publishToInternalServer = $true + $publishToPublicServer = $false + + echo "Publishing request name: $requestName" + echo "Publish to internal server: $publishToInternalServer" + echo "Publish to public server: $publishToPublicServer" + + $symbolServer = '$(SymbolServer)' + $tokenUri = '$(SymbolTokenUri)' + $projectName = "mssql-python" + + # Get the access token for the symbol publishing service + $symbolPublishingToken = az account get-access-token --resource $tokenUri --query accessToken -o tsv + + echo "> 1.Symbol publishing token acquired." + + echo "Registering the request name ..." + $requestName = '$(requestName)' + $requestNameRegistrationBody = "{'requestName': '$requestName'}" + Invoke-RestMethod -Method POST -Uri "https://$symbolServer.trafficmanager.net/projects/$projectName/requests" -Headers @{ Authorization = "Bearer $symbolPublishingToken" } -ContentType "application/json" -Body $requestNameRegistrationBody + + echo "> 2.Registration of request name succeeded." + + echo "Publishing the symbols ..." + $publishSymbolsBody = "{'publishToInternalServer': $publishToInternalServer, 'publishToPublicServer': $publishToPublicServer}" + echo "Publishing symbols request body: $publishSymbolsBody" + Invoke-RestMethod -Method POST -Uri "https://$symbolServer.trafficmanager.net/projects/$projectName/requests/$requestName" -Headers @{ Authorization = "Bearer $symbolPublishingToken" } -ContentType "application/json" -Body $publishSymbolsBody + + echo "> 3.Request to publish symbols succeeded." + + # The following REST calls are used to check publishing status. + echo "> 4.Checking the status of the request ..." + + Invoke-RestMethod -Method GET -Uri "https://$symbolServer.trafficmanager.net/projects/$projectName/requests/$requestName" -Headers @{ Authorization = "Bearer $symbolPublishingToken" } -ContentType "application/json" + + echo "Use below tables to interpret the values of xxxServerStatus and xxxServerResult fields from the response." + + echo "PublishingStatus" + echo "-----------------" + echo "0 NotRequested; The request has not been requested to publish." + echo "1 Submitted; The request is submitted to be published" + echo "2 Processing; The request is still being processed" + echo "3 Completed; The request has been completed processing. It can be failed or successful. Check PublishingResult to get more details" + + echo "PublishingResult" + echo "-----------------" + echo "0 Pending; The request has not completed or has not been requested." + echo "1 Succeeded; The request has published successfully" + echo "2 Failed; The request has failed to publish" + echo "3 Cancelled; The request was cancelled" # - job: ReleaseESRPPackage # displayName: 'Release ESRP Package' From 115b2626a4d009e37b1c8a395e2bfac80f932f7e Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Thu, 10 Jul 2025 14:07:02 +0530 Subject: [PATCH 17/19] use onebranch temp --- eng/pipelines/dummy-release-pipeline.yml | 297 ++++++++++++----------- 1 file changed, 162 insertions(+), 135 deletions(-) diff --git a/eng/pipelines/dummy-release-pipeline.yml b/eng/pipelines/dummy-release-pipeline.yml index d22465bd..01509d3b 100644 --- a/eng/pipelines/dummy-release-pipeline.yml +++ b/eng/pipelines/dummy-release-pipeline.yml @@ -10,141 +10,168 @@ extends: parameters: featureFlags: WindowsHostVersion: 1ESWindows2022 - - stages: - - stage: PublishSymbols - displayName: 'Publish Symbols' - jobs: - - job: PublishSymbols - # Use the latest Windows image for building - pool: - vmImage: 'windows-latest' - displayName: 'Publish Symbols - Windows' - # Strategy matrix to build all combinations - - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.13' - architecture: 'x64' - addToPath: true - displayName: 'Use Python 3.13' - - # Install required packages: pip, CMake, pybind11 - - script: | - python -m pip install --upgrade pip - pip install -r requirements.txt - pip install cmake pybind11 - displayName: 'Install dependencies' - - # Generate the symbols for the mssql-python package using build.bat - - script: | - echo "Generating symbols for mssql-python package..." - cd mssql_python\pybind - build.bat - cd .. - dir /s /b - displayName: 'Generate symbols for mssql-python package' - - # Copy the generated symbols to the staging folder for artifacts - - task: CopyFiles@2 - inputs: - SourceFolder: '$(Build.SourcesDirectory)\mssql_python' - Contents: '**\*.pdb' - TargetFolder: '$(Build.ArtifactStagingDirectory)\mssql-python-symbols' - displayName: 'Copy symbols to staging folder' - - # Publish the symbols (only pdbs) as an artifact - - task: PublishBuildArtifacts@1 - displayName: 'Publish symbols as build artifact' - inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)\mssql-python-symbols' - ArtifactName: 'mssql-python-symbols' - publishLocation: 'Container' - - - powershell: 'Write-Host "##vso[task.setvariable variable=ArtifactServices.Symbol.AccountName;]SqlClientDrivers"' - displayName: 'Update Symbol.AccountName with SqlClientDrivers' - - - task: PublishSymbols@2 - displayName: 'Upload symbols to SqlClientDrivers org' - inputs: - SymbolsFolder: '$(Build.ArtifactStagingDirectory)\mssql-python-symbols' - SearchPattern: '**/*.pdb' - IndexSources: false - SymbolServerType: TeamServices - SymbolsMaximumWaitTime: 60 - SymbolExpirationInDays: 1825 # 5 years - SymbolsProduct: mssql-python - # Have kept the default version as the build ID, which is unique for each build - # This will be used to identify the symbols in the symbol server - SymbolsVersion: $(Build.BuildId) - # Ensuring the symbols are uniquely identified - # MDS uses symbolsArtifactName as mds_symbols_$(System.TeamProject)_$(Build.Repository.Name)_$(Build.SourceBranchName)_$(NuGetPackageVersion)_$(System.TimelineId) - SymbolsArtifactName: $(System.TeamProject)-$(Build.SourceBranchName)-$(Build.DefinitionName)-$(Build.BuildId) - Pat: $(System.AccessToken) - - - task: AzureCLI@2 - displayName: 'Publish symbols' - env: - SymbolServer: '$(SymbolServer)' - SymbolTokenUri: '$(SymbolTokenUri)' - requestName: '$(System.TeamProject)-$(Build.SourceBranchName)-$(Build.DefinitionName)-$(Build.BuildId)' - inputs: - azureSubscription: 'SymbolsPublishing-msodbcsql-mssql-python' - scriptType: ps - scriptLocation: inlineScript - inlineScript: | - # Should be true by default for internal server - $publishToInternalServer = $true - $publishToPublicServer = $false - - echo "Publishing request name: $requestName" - echo "Publish to internal server: $publishToInternalServer" - echo "Publish to public server: $publishToPublicServer" - - $symbolServer = '$(SymbolServer)' - $tokenUri = '$(SymbolTokenUri)' - $projectName = "mssql-python" - - # Get the access token for the symbol publishing service - $symbolPublishingToken = az account get-access-token --resource $tokenUri --query accessToken -o tsv - - echo "> 1.Symbol publishing token acquired." - - echo "Registering the request name ..." - $requestName = '$(requestName)' - $requestNameRegistrationBody = "{'requestName': '$requestName'}" - Invoke-RestMethod -Method POST -Uri "https://$symbolServer.trafficmanager.net/projects/$projectName/requests" -Headers @{ Authorization = "Bearer $symbolPublishingToken" } -ContentType "application/json" -Body $requestNameRegistrationBody - - echo "> 2.Registration of request name succeeded." - - echo "Publishing the symbols ..." - $publishSymbolsBody = "{'publishToInternalServer': $publishToInternalServer, 'publishToPublicServer': $publishToPublicServer}" - echo "Publishing symbols request body: $publishSymbolsBody" - Invoke-RestMethod -Method POST -Uri "https://$symbolServer.trafficmanager.net/projects/$projectName/requests/$requestName" -Headers @{ Authorization = "Bearer $symbolPublishingToken" } -ContentType "application/json" -Body $publishSymbolsBody - - echo "> 3.Request to publish symbols succeeded." - - # The following REST calls are used to check publishing status. - echo "> 4.Checking the status of the request ..." - - Invoke-RestMethod -Method GET -Uri "https://$symbolServer.trafficmanager.net/projects/$projectName/requests/$requestName" -Headers @{ Authorization = "Bearer $symbolPublishingToken" } -ContentType "application/json" - - echo "Use below tables to interpret the values of xxxServerStatus and xxxServerResult fields from the response." - - echo "PublishingStatus" - echo "-----------------" - echo "0 NotRequested; The request has not been requested to publish." - echo "1 Submitted; The request is submitted to be published" - echo "2 Processing; The request is still being processed" - echo "3 Completed; The request has been completed processing. It can be failed or successful. Check PublishingResult to get more details" - - echo "PublishingResult" - echo "-----------------" - echo "0 Pending; The request has not completed or has not been requested." - echo "1 Succeeded; The request has published successfully" - echo "2 Failed; The request has failed to publish" - echo "3 Cancelled; The request was cancelled" + # SDL should have been performed before release. Disabling SDL in release pipeline. Note that some static analysis can't be disabled by default. + # More info at https://eng.ms/docs/cloud-ai-platform/azure-edge-platform-aep/aep-engineering-systems/productivity-and-experiences/onebranch-platform-services/onebranch/securitycompliancegovernanceandpolicies/sdlforcontainerizedworkflows. + globalSdl: + baseline: + baselineFile: $(Build.SourcesDirectory)\.gdn\.gdnbaselines + suppressionSet: default + suppression: + suppressionFile: $(Build.SourcesDirectory)\.gdn\.gdnsuppress + suppressionSet: default + asyncSdl: + enabled: false + tsa: + enabled: false + credscan: + enabled: false + apiscan: + enabled: false + codeql: + compiled: + enabled: false + binskim: + enabled: false + policheck: + enabled: false + eslint: + enabled: false + tsaOptionsFile: .config/tsaoptions.json + + stages: + - stage: PublishSymbols + displayName: 'Publish Symbols' + jobs: + - job: PublishSymbols + # Use the latest Windows image for building + pool: + vmImage: 'windows-latest' + displayName: 'Publish Symbols - Windows' + # Strategy matrix to build all combinations + + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.13' + architecture: 'x64' + addToPath: true + displayName: 'Use Python 3.13' + + # Install required packages: pip, CMake, pybind11 + - script: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install cmake pybind11 + displayName: 'Install dependencies' + + # Generate the symbols for the mssql-python package using build.bat + - script: | + echo "Generating symbols for mssql-python package..." + cd mssql_python\pybind + build.bat + cd .. + dir /s /b + displayName: 'Generate symbols for mssql-python package' + + # Copy the generated symbols to the staging folder for artifacts + - task: CopyFiles@2 + inputs: + SourceFolder: '$(Build.SourcesDirectory)\mssql_python' + Contents: '**\*.pdb' + TargetFolder: '$(Build.ArtifactStagingDirectory)\mssql-python-symbols' + displayName: 'Copy symbols to staging folder' + + # Publish the symbols (only pdbs) as an artifact + - task: PublishBuildArtifacts@1 + displayName: 'Publish symbols as build artifact' + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)\mssql-python-symbols' + ArtifactName: 'mssql-python-symbols' + publishLocation: 'Container' + + - powershell: 'Write-Host "##vso[task.setvariable variable=ArtifactServices.Symbol.AccountName;]SqlClientDrivers"' + displayName: 'Update Symbol.AccountName with SqlClientDrivers' + + - task: PublishSymbols@2 + displayName: 'Upload symbols to SqlClientDrivers org' + inputs: + SymbolsFolder: '$(Build.ArtifactStagingDirectory)\mssql-python-symbols' + SearchPattern: '**/*.pdb' + IndexSources: false + SymbolServerType: TeamServices + SymbolsMaximumWaitTime: 60 + SymbolExpirationInDays: 1825 # 5 years + SymbolsProduct: mssql-python + # Have kept the default version as the build ID, which is unique for each build + # This will be used to identify the symbols in the symbol server + SymbolsVersion: $(Build.BuildId) + # Ensuring the symbols are uniquely identified + # MDS uses symbolsArtifactName as mds_symbols_$(System.TeamProject)_$(Build.Repository.Name)_$(Build.SourceBranchName)_$(NuGetPackageVersion)_$(System.TimelineId) + SymbolsArtifactName: $(System.TeamProject)-$(Build.SourceBranchName)-$(Build.DefinitionName)-$(Build.BuildId) + Pat: $(System.AccessToken) + + - task: AzureCLI@2 + displayName: 'Publish symbols' + env: + SymbolServer: '$(SymbolServer)' + SymbolTokenUri: '$(SymbolTokenUri)' + requestName: '$(System.TeamProject)-$(Build.SourceBranchName)-$(Build.DefinitionName)-$(Build.BuildId)' + inputs: + azureSubscription: 'SymbolsPublishing-msodbcsql-mssql-python' + scriptType: ps + scriptLocation: inlineScript + inlineScript: | + # Should be true by default for internal server + $publishToInternalServer = $true + $publishToPublicServer = $false + + echo "Publishing request name: $requestName" + echo "Publish to internal server: $publishToInternalServer" + echo "Publish to public server: $publishToPublicServer" + + $symbolServer = '$(SymbolServer)' + $tokenUri = '$(SymbolTokenUri)' + $projectName = "mssql-python" + + # Get the access token for the symbol publishing service + $symbolPublishingToken = az account get-access-token --resource $tokenUri --query accessToken -o tsv + + echo "> 1.Symbol publishing token acquired." + + echo "Registering the request name ..." + $requestName = '$(requestName)' + $requestNameRegistrationBody = "{'requestName': '$requestName'}" + Invoke-RestMethod -Method POST -Uri "https://$symbolServer.trafficmanager.net/projects/$projectName/requests" -Headers @{ Authorization = "Bearer $symbolPublishingToken" } -ContentType "application/json" -Body $requestNameRegistrationBody + + echo "> 2.Registration of request name succeeded." + + echo "Publishing the symbols ..." + $publishSymbolsBody = "{'publishToInternalServer': $publishToInternalServer, 'publishToPublicServer': $publishToPublicServer}" + echo "Publishing symbols request body: $publishSymbolsBody" + Invoke-RestMethod -Method POST -Uri "https://$symbolServer.trafficmanager.net/projects/$projectName/requests/$requestName" -Headers @{ Authorization = "Bearer $symbolPublishingToken" } -ContentType "application/json" -Body $publishSymbolsBody + + echo "> 3.Request to publish symbols succeeded." + + # The following REST calls are used to check publishing status. + echo "> 4.Checking the status of the request ..." + + Invoke-RestMethod -Method GET -Uri "https://$symbolServer.trafficmanager.net/projects/$projectName/requests/$requestName" -Headers @{ Authorization = "Bearer $symbolPublishingToken" } -ContentType "application/json" + + echo "Use below tables to interpret the values of xxxServerStatus and xxxServerResult fields from the response." + + echo "PublishingStatus" + echo "-----------------" + echo "0 NotRequested; The request has not been requested to publish." + echo "1 Submitted; The request is submitted to be published" + echo "2 Processing; The request is still being processed" + echo "3 Completed; The request has been completed processing. It can be failed or successful. Check PublishingResult to get more details" + + echo "PublishingResult" + echo "-----------------" + echo "0 Pending; The request has not completed or has not been requested." + echo "1 Succeeded; The request has published successfully" + echo "2 Failed; The request has failed to publish" + echo "3 Cancelled; The request was cancelled" # - job: ReleaseESRPPackage # displayName: 'Release ESRP Package' From 864a7ffb98af311f6886f8abb836423bdab7cbe2 Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Thu, 10 Jul 2025 15:35:16 +0530 Subject: [PATCH 18/19] use onebranch template - change pool to windows --- eng/pipelines/dummy-release-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/dummy-release-pipeline.yml b/eng/pipelines/dummy-release-pipeline.yml index 01509d3b..2facfd07 100644 --- a/eng/pipelines/dummy-release-pipeline.yml +++ b/eng/pipelines/dummy-release-pipeline.yml @@ -45,7 +45,7 @@ extends: - job: PublishSymbols # Use the latest Windows image for building pool: - vmImage: 'windows-latest' + type: 'windows' displayName: 'Publish Symbols - Windows' # Strategy matrix to build all combinations From 4faedbc011f4f11039b4749c0b9095b90917a9cd Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Thu, 10 Jul 2025 15:40:48 +0530 Subject: [PATCH 19/19] use onebranch template - remove globalSdl --- eng/pipelines/dummy-release-pipeline.yml | 27 ------------------------ 1 file changed, 27 deletions(-) diff --git a/eng/pipelines/dummy-release-pipeline.yml b/eng/pipelines/dummy-release-pipeline.yml index 2facfd07..d376b6b3 100644 --- a/eng/pipelines/dummy-release-pipeline.yml +++ b/eng/pipelines/dummy-release-pipeline.yml @@ -10,33 +10,6 @@ extends: parameters: featureFlags: WindowsHostVersion: 1ESWindows2022 - # SDL should have been performed before release. Disabling SDL in release pipeline. Note that some static analysis can't be disabled by default. - # More info at https://eng.ms/docs/cloud-ai-platform/azure-edge-platform-aep/aep-engineering-systems/productivity-and-experiences/onebranch-platform-services/onebranch/securitycompliancegovernanceandpolicies/sdlforcontainerizedworkflows. - globalSdl: - baseline: - baselineFile: $(Build.SourcesDirectory)\.gdn\.gdnbaselines - suppressionSet: default - suppression: - suppressionFile: $(Build.SourcesDirectory)\.gdn\.gdnsuppress - suppressionSet: default - asyncSdl: - enabled: false - tsa: - enabled: false - credscan: - enabled: false - apiscan: - enabled: false - codeql: - compiled: - enabled: false - binskim: - enabled: false - policheck: - enabled: false - eslint: - enabled: false - tsaOptionsFile: .config/tsaoptions.json stages: - stage: PublishSymbols