build: add a OneBranch Official release pipeline (#16081)

This pipeline does everything the existing release pipeline does, except
it does it using the OneBranch official templates.

Most of our existing build infrastructure has been reused, with the
following changes:

- We are no longer using `job-submit-windows-vpack`, as OneBranch does
this for us.
- `job-merge-msix-into-bundle` now supports afterBuildSteps, which we
use to stage the msixbundle into the right place for the vpack
- `job-build-project` supports deleting all non-signed files (which the
OneBranch post-build validation requires)
- `job-build-project` now deletes `console.dll`, which is unused in any
of our builds, because XFGCheck blows up on it for some reason on x86
- `job-publish-symbols` now supports two different types of PAT
ingestion
- I have pulled out the NuGet filename variables into a shared variables
template

I have also introduced a TSA config (which files bugs on us for binary
analysis failures as well as using the word 'sucks' and stuff.)

I have also baselined a number of control flow guard/binary analysis
failures.
This commit is contained in:
Dustin L. Howett 2023-10-02 14:52:54 -05:00 committed by GitHub
parent 1669036e59
commit 6489f6b39d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 1577 additions and 24 deletions

File diff suppressed because it is too large Load Diff

6
build/config/tsa.json Normal file
View File

@ -0,0 +1,6 @@
{
"instanceUrl": "https://microsoft.visualstudio.com",
"projectName": "OS",
"areaPath": "OS\\Windows Client and Services\\ADEPT\\E4D-Engineered for Developers\\SHINE\\Terminal",
"notificationAliases": ["condev@microsoft.com", "duhowett@microsoft.com"]
}

View File

@ -0,0 +1,47 @@
trigger: none
pr: none
schedules:
- cron: "30 3 * * 2-6" # Run at 03:30 UTC Tuesday through Saturday (After the work day in Pacific, Mon-Fri)
displayName: "Nightly Terminal Build"
branches:
include:
- main
always: false # only run if there's code changes!
parameters:
- name: publishToAzure
displayName: "Deploy to **PUBLIC** Azure Storage"
type: boolean
default: true
name: $(BuildDefinitionName)_$(date:yyMM).$(date:dd)$(rev:rrr)
variables:
- template: templates-v2/variables-nuget-package-version.yml
- template: templates-v2/variables-onebranch-config.yml
extends:
template: templates-v2/pipeline-onebranch-full-release-build.yml
parameters:
official: true
branding: Canary
buildTerminal: true
pgoBuildMode: Optimize
codeSign: true
publishSymbolsToPublic: true
publishVpackToWindows: false
symbolExpiryTime: 15
${{ if eq(true, parameters.publishToAzure) }}:
extraPublishJobs:
- template: job-deploy-to-azure-storage.yml
parameters:
pool: { type: windows }
dependsOn: [PublishSymbols]
storagePublicRootURL: $(AppInstallerRootURL)
subscription: $(AzureSubscriptionName)
storageAccount: $(AzureStorageAccount)
storageContainer: $(AzureStorageContainer)
buildConfiguration: Release
buildPlatforms: [x64, x86, arm64]
environment: production-canary

View File

@ -0,0 +1,81 @@
trigger: none
pr: none
# Expose all of these parameters for user configuration.
parameters:
- name: branding
displayName: "Branding (Build Type)"
type: string
default: Release
values:
- Release
- Preview
- Canary
- Dev
- name: buildTerminal
displayName: "Build Windows Terminal MSIX"
type: boolean
default: true
- name: buildConPTY
displayName: "Build ConPTY NuGet"
type: boolean
default: false
- name: buildWPF
displayName: "Build Terminal WPF Control"
type: boolean
default: false
- name: pgoBuildMode
displayName: "PGO Build Mode"
type: string
default: Optimize
values:
- Optimize
- Instrument
- None
- name: buildConfigurations
displayName: "Build Configurations"
type: object
default:
- Release
- name: buildPlatforms
displayName: "Build Platforms"
type: object
default:
- x64
- x86
- arm64
- name: terminalInternalPackageVersion
displayName: "Terminal Internal Package Version"
type: string
default: '0.0.8'
- name: publishSymbolsToPublic
displayName: "Publish Symbols to MSDL"
type: boolean
default: true
- name: publishVpackToWindows
displayName: "Publish VPack to Windows"
type: boolean
default: false
name: $(BuildDefinitionName)_$(date:yyMM).$(date:dd)$(rev:rrr)
variables:
- template: templates-v2/variables-nuget-package-version.yml
- template: templates-v2/variables-onebranch-config.yml
extends:
template: templates-v2/pipeline-onebranch-full-release-build.yml
parameters:
official: true
branding: ${{ parameters.branding }}
buildTerminal: ${{ parameters.buildTerminal }}
buildConPTY: ${{ parameters.buildConPTY }}
buildWPF: ${{ parameters.buildWPF }}
pgoBuildMode: ${{ parameters.pgoBuildMode }}
buildConfigurations: ${{ parameters.buildConfigurations }}
buildPlatforms: ${{ parameters.buildPlatforms }}
codeSign: true
terminalInternalPackageVersion: ${{ parameters.terminalInternalPackageVersion }}
publishSymbolsToPublic: ${{ parameters.publishSymbolsToPublic }}
publishVpackToWindows: ${{ parameters.publishVpackToWindows }}

View File

@ -62,6 +62,9 @@ parameters:
- name: publishArtifacts
type: boolean
default: true
- name: removeAllNonSignedFiles
type: boolean
default: false
jobs:
- job: ${{ parameters.jobName }}
@ -173,6 +176,7 @@ jobs:
# - Directories ending in Lib (static lib projects that we fully linked into DLLs which may also contain unnecessary resources)
# - All LocalTests_ project outputs, as they were subsumed into TestHostApp
# - All PDB files inside the WindowsTerminal/ output, which do not belong there.
# - console.dll, which apparently breaks XFGCheck? lol.
- pwsh: |-
$binDir = '$(Terminal.BinDir)'
$ImportLibs = Get-ChildItem $binDir -Recurse -File -Filter '*.exp' | ForEach-Object { $_.FullName -Replace "exp$","lib" }
@ -189,6 +193,8 @@ jobs:
$Items += Get-ChildItem '$(Terminal.BinDir)' -Filter '*.pdb' -Recurse
}
$Items += Get-ChildItem $binDir -Filter 'console.dll'
$Items | Remove-Item -Recurse -Force -Verbose -ErrorAction:Ignore
displayName: Clean up static libs and extra symbols
errorActionPreference: silentlyContinue # It's OK if this silently fails
@ -246,6 +252,14 @@ jobs:
Write-Host "##vso[task.setvariable variable=WindowsTerminalPackagePath]${PackageFilename}"
displayName: Re-pack the new Terminal package after signing
# Some of our governed pipelines explicitly fail builds that have *any* non-codesigned filed (!)
- ${{ if eq(parameters.removeAllNonSignedFiles, true) }}:
- pwsh: |-
Get-ChildItem "$(Terminal.BinDir)" -Recurse -Include "*.dll","*.exe" |
Where-Object { (Get-AuthenticodeSignature $_).Status -Ne "Valid" } |
Remove-Item -Verbose -Force
displayName: Remove all non-signed output files
- ${{ else }}: # No Signing
- ${{ if or(parameters.buildTerminal, parameters.buildEverything) }}:
- pwsh: |-

View File

@ -29,6 +29,9 @@ parameters:
- name: publishArtifacts
type: boolean
default: true
- name: afterBuildSteps
type: stepList
default: []
jobs:
- job: ${{ parameters.jobName }}
@ -85,7 +88,9 @@ jobs:
$Components[0] = ([int]$Components[0] + $VersionEpoch)
$BundleVersion = $Components -Join "."
New-Item -Type Directory "$(System.ArtifactsDirectory)/bundle"
.\build\scripts\Create-AppxBundle.ps1 -InputPath 'bin/' -ProjectName CascadiaPackage -BundleVersion $BundleVersion -OutputPath "$(System.ArtifactsDirectory)\bundle\$(BundleStemName)_$(XES_APPXMANIFESTVERSION)_8wekyb3d8bbwe.msixbundle"
$BundlePath = "$(System.ArtifactsDirectory)\bundle\$(BundleStemName)_$(XES_APPXMANIFESTVERSION)_8wekyb3d8bbwe.msixbundle"
.\build\scripts\Create-AppxBundle.ps1 -InputPath 'bin/' -ProjectName CascadiaPackage -BundleVersion $BundleVersion -OutputPath $BundlePath
Write-Host "##vso[task.setvariable variable=MsixBundlePath]${BundlePath}"
displayName: Create msixbundle
- ${{ if eq(parameters.codeSign, true) }}:
@ -139,6 +144,8 @@ jobs:
ValidateSignature: true
Verbosity: 'Verbose'
- ${{ parameters.afterBuildSteps }}
- ${{ if eq(parameters.publishArtifacts, true) }}:
- publish: $(JobOutputDirectory)
artifact: $(JobOutputArtifactName)

View File

@ -17,6 +17,12 @@ parameters:
- name: symbolExpiryTime
type: string
default: 36530 # This is the default from PublishSymbols@2
- name: variables
type: object
default: {}
- name: symbolPatGoesInTaskInputs
type: boolean
default: false
jobs:
- job: ${{ parameters.jobName }}
@ -27,6 +33,8 @@ jobs:
${{ else }}:
displayName: Publish Symbols Internally
dependsOn: ${{ parameters.dependsOn }}
variables:
${{ insert }}: ${{ parameters.variables }}
steps:
- checkout: self
clean: true
@ -76,6 +84,8 @@ jobs:
SymbolsProduct: 'Windows Terminal Converged Symbols'
SymbolsVersion: '$(XES_APPXMANIFESTVERSION)'
SymbolExpirationInDays: ${{ parameters.symbolExpiryTime }}
${{ if eq(parameters.symbolPatGoesInTaskInputs, true) }}:
Pat: $(ADO_microsoftpublicsymbols_PAT)
# The ADO task does not support indexing of GitHub sources.
# There is a bug which causes this task to fail if LIB includes an inaccessible path (even though it does not depend on it).
# To work around this issue, we just force LIB to be any dir that we know exists.
@ -83,4 +93,5 @@ jobs:
env:
LIB: $(Build.SourcesDirectory)
ArtifactServices_Symbol_AccountName: microsoftpublicsymbols
ArtifactServices_Symbol_PAT: $(ADO_microsoftpublicsymbols_PAT)
${{ if ne(parameters.symbolPatGoesInTaskInputs, true) }}:
ArtifactServices_Symbol_PAT: $(ADO_microsoftpublicsymbols_PAT)

View File

@ -61,28 +61,7 @@ parameters:
demands: ImageOverride -equals SHINE-VS17-Latest
variables:
# If we are building a branch called "release-*", change the NuGet suffix
# to "preview". If we don't do that, XES will set the suffix to "release1"
# because it truncates the value after the first period.
# We also want to disable the suffix entirely if we're Release branded while
# on a release branch.
# main is special, however. XES ignores main. Since we never produce actual
# shipping builds from main, we want to force it to have a beta label as
# well.
#
# In effect:
# BRANCH / BRANDING | Release | Preview
# ------------------|----------------------------|-----------------------------
# release-* | 1.12.20220427 | 1.13.20220427-preview
# main | 1.14.20220427-experimental | 1.14.20220427-experimental
# all others | 1.14.20220427-mybranch | 1.14.20220427-mybranch
${{ if startsWith(variables['Build.SourceBranchName'], 'release-') }}:
${{ if eq(parameters.branding, 'Release') }}:
NoNuGetPackBetaVersion: true
${{ else }}:
NuGetPackBetaVersion: preview
${{ elseif eq(variables['Build.SourceBranchName'], 'main') }}:
NuGetPackBetaVersion: experimental
- template: variables-nuget-package-version.yml
resources:
repositories:

View File

@ -0,0 +1,256 @@
parameters:
- name: official
type: boolean
default: false
- name: branding
type: string
default: Release
values:
- Release
- Preview
- Canary
- Dev
- name: buildTerminal
type: boolean
default: true
- name: buildConPTY
type: boolean
default: false
- name: buildWPF
type: boolean
default: false
- name: pgoBuildMode
type: string
default: Optimize
values:
- Optimize
- Instrument
- None
- name: buildConfigurations
type: object
default:
- Release
- name: buildPlatforms
type: object
default:
- x64
- x86
- arm64
- name: codeSign
type: boolean
default: true
- name: terminalInternalPackageVersion
type: string
default: '0.0.8'
- name: publishSymbolsToPublic
type: boolean
default: true
- name: publishVpackToWindows
type: boolean
default: false
- name: extraPublishJobs
type: object
default: []
resources:
repositories:
- repository: templates
type: git
name: OneBranch.Pipelines/GovernedTemplates
ref: refs/heads/main
extends:
${{ if eq(parameters.official, true) }}:
template: v2/Microsoft.Official.yml@templates # https://aka.ms/obpipelines/templates
${{ else }}:
template: v2/Microsoft.NonOfficial.yml@templates
parameters:
featureFlags:
WindowsHostVersion: 1ESWindows2022
platform:
name: 'windows_undocked'
product: 'Windows Terminal'
cloudvault: # https://aka.ms/obpipelines/cloudvault
enabled: false
globalSdl: # https://aka.ms/obpipelines/sdl
tsa:
enabled: true
configFile: '$(Build.SourcesDirectory)\build\config\tsa.json'
binskim:
break: false
scanOutputDirectoryOnly: true
policheck:
break: false
severity: Note
baseline:
baselineFile: '$(Build.SourcesDirectory)\build\config\release.gdnbaselines'
suppressionSet: default
stages:
- stage: Build
displayName: Build
dependsOn: []
jobs:
- template: ./build/pipelines/templates-v2/job-build-project.yml@self
parameters:
pool: { type: windows }
variables:
ob_git_checkout: false # This job checks itself out
ob_git_skip_checkout_none: true
ob_outputDirectory: $(JobOutputDirectory)
ob_artifactBaseName: $(JobOutputArtifactName)
publishArtifacts: false # Handled by OneBranch
branding: ${{ parameters.branding }}
buildTerminal: ${{ parameters.buildTerminal }}
buildConPTY: ${{ parameters.buildConPTY }}
buildWPF: ${{ parameters.buildWPF }}
pgoBuildMode: ${{ parameters.pgoBuildMode }}
buildConfigurations: ${{ parameters.buildConfigurations }}
buildPlatforms: ${{ parameters.buildPlatforms }}
generateSbom: false # this is handled by onebranch
removeAllNonSignedFiles: true # appease the overlords
codeSign: ${{ parameters.codeSign }}
beforeBuildSteps: # Right before we build, lay down the universal package and localizations
- task: PkgESSetupBuild@12
displayName: Package ES - Setup Build
inputs:
disableOutputRedirect: true
- task: UniversalPackages@0
displayName: Download terminal-internal Universal Package
inputs:
feedListDownload: 2b3f8893-a6e8-411f-b197-a9e05576da48
packageListDownload: e82d490c-af86-4733-9dc4-07b772033204
versionListDownload: ${{ parameters.terminalInternalPackageVersion }}
- template: ./build/pipelines/templates-v2/steps-fetch-and-prepare-localizations.yml@self
parameters:
includePseudoLoc: true
- ${{ if eq(parameters.buildWPF, true) }}:
# Add an Any CPU build flavor for the WPF control bits
- template: ./build/pipelines/templates-v2/job-build-project.yml@self
parameters:
pool: { type: windows }
variables:
ob_git_checkout: false # This job checks itself out
ob_git_skip_checkout_none: true
ob_outputDirectory: $(JobOutputDirectory)
ob_artifactBaseName: $(JobOutputArtifactName)
publishArtifacts: false # Handled by OneBranch
jobName: BuildWPF
branding: ${{ parameters.branding }}
buildTerminal: false
buildWPFDotNetComponents: true
buildConfigurations: ${{ parameters.buildConfigurations }}
buildPlatforms:
- Any CPU
generateSbom: false # this is handled by onebranch
removeAllNonSignedFiles: true # appease the overlords
codeSign: ${{ parameters.codeSign }}
beforeBuildSteps:
- task: PkgESSetupBuild@12
displayName: Package ES - Setup Build
inputs:
disableOutputRedirect: true
# WPF doesn't need the localizations or the universal package, but if it does... put them here.
- stage: Package
displayName: Package
dependsOn: [Build]
jobs:
- ${{ if eq(parameters.buildTerminal, true) }}:
- template: ./build/pipelines/templates-v2/job-merge-msix-into-bundle.yml@self
parameters:
pool: { type: windows }
variables:
ob_git_checkout: false # This job checks itself out
ob_git_skip_checkout_none: true
ob_outputDirectory: $(JobOutputDirectory)
ob_artifactBaseName: $(JobOutputArtifactName)
### This job is also in charge of submitting the vpack to Windows if it's enabled
ob_createvpack_enabled: ${{ and(parameters.buildTerminal, parameters.publishVpackToWindows) }}
ob_updateOSManifest_enabled: ${{ and(parameters.buildTerminal, parameters.publishVpackToWindows) }}
### If enabled above, these options are in play.
ob_createvpack_packagename: 'WindowsTerminal.app'
ob_createvpack_owneralias: 'conhost@microsoft.com'
ob_createvpack_description: 'VPack for the Windows Terminal Application'
ob_createvpack_targetDestinationDirectory: '$(Destination)'
ob_createvpack_propsFile: false
ob_createvpack_provData: true
ob_createvpack_metadata: '$(Build.SourceVersion)'
ob_createvpack_topLevelRetries: 0
ob_createvpack_failOnStdErr: true
ob_createvpack_taskLogVerbosity: Detailed
ob_createvpack_verbose: true
ob_createvpack_vpackdirectory: '$(JobOutputDirectory)\vpack'
ob_updateOSManifest_gitcheckinConfigPath: '$(Build.SourcesDirectory)\build\config\GitCheckin.json'
# We're skipping the 'fetch' part of the OneBranch rules, but that doesn't mean
# that it doesn't expect to have downloaded a manifest directly to some 'destination'
# folder that it can then update and upload.
# Effectively: it says "destination" but it means "source"
# DH: Don't ask why.
ob_updateOSManifest_destination: $(XES_VPACKMANIFESTDIRECTORY)
ob_updateOSManifest_skipFetch: true
publishArtifacts: false # Handled by OneBranch
jobName: Bundle
branding: ${{ parameters.branding }}
buildConfigurations: ${{ parameters.buildConfigurations }}
buildPlatforms: ${{ parameters.buildPlatforms }}
generateSbom: false # Handled by onebranch
codeSign: ${{ parameters.codeSign }}
afterBuildSteps:
- pwsh: |-
$d = New-Item "$(JobOutputDirectory)/vpack" -Type Directory
Copy-Item -Verbose -Path "$(MsixBundlePath)" -Destination (Join-Path $d 'Microsoft.WindowsTerminal_8wekyb3d8bbwe.msixbundle')
displayName: Stage msixbundle for vpack
- ${{ if eq(parameters.buildConPTY, true) }}:
- template: ./build/pipelines/templates-v2/job-package-conpty.yml@self
parameters:
pool: { type: windows }
variables:
ob_git_checkout: false # This job checks itself out
ob_git_skip_checkout_none: true
ob_outputDirectory: $(JobOutputDirectory)
ob_artifactBaseName: $(JobOutputArtifactName)
publishArtifacts: false # Handled by OneBranch
buildConfigurations: ${{ parameters.buildConfigurations }}
buildPlatforms: ${{ parameters.buildPlatforms }}
generateSbom: false # this is handled by onebranch
codeSign: ${{ parameters.codeSign }}
- ${{ if eq(parameters.buildWPF, true) }}:
- template: ./build/pipelines/templates-v2/job-build-package-wpf.yml@self
parameters:
pool: { type: windows }
variables:
ob_git_checkout: false # This job checks itself out
ob_git_skip_checkout_none: true
ob_outputDirectory: $(JobOutputDirectory)
ob_artifactBaseName: $(JobOutputArtifactName)
publishArtifacts: false # Handled by OneBranch
buildConfigurations: ${{ parameters.buildConfigurations }}
buildPlatforms: ${{ parameters.buildPlatforms }}
generateSbom: false # this is handled by onebranch
codeSign: ${{ parameters.codeSign }}
- stage: Publish
displayName: Publish
dependsOn: [Build, Package]
jobs:
- template: ./build/pipelines/templates-v2/job-publish-symbols.yml@self
parameters:
pool: { type: windows }
includePublicSymbolServer: ${{ parameters.publishSymbolsToPublic }}
symbolPatGoesInTaskInputs: true # onebranch tries to muck with the PAT variable, so we need to change how it get the PAT
variables:
ob_git_checkout: false # This job checks itself out
ob_git_skip_checkout_none: true
ob_outputDirectory: $(Build.ArtifactStagingDirectory)
# Without this, OneBranch will nerf our symbol tasks
ob_symbolsPublishing_enabled: true
- ${{ parameters.extraPublishJobs }}

View File

@ -0,0 +1,23 @@
variables:
# If we are building a branch called "release-*", change the NuGet suffix
# to "preview". If we don't do that, XES will set the suffix to "release1"
# because it truncates the value after the first period.
# We also want to disable the suffix entirely if we're Release branded while
# on a release branch.
# main is special, however. XES ignores main. Since we never produce actual
# shipping builds from main, we want to force it to have a beta label as
# well.
#
# In effect:
# BRANCH / BRANDING | Release | Preview
# ------------------|----------------------------|-----------------------------
# release-* | 1.12.20220427 | 1.13.20220427-preview
# main | 1.14.20220427-experimental | 1.14.20220427-experimental
# all others | 1.14.20220427-mybranch | 1.14.20220427-mybranch
${{ if startsWith(variables['Build.SourceBranchName'], 'release-') }}:
${{ if eq(parameters.branding, 'Release') }}:
NoNuGetPackBetaVersion: true
${{ else }}:
NuGetPackBetaVersion: preview
${{ elseif eq(variables['Build.SourceBranchName'], 'main') }}:
NuGetPackBetaVersion: experimental

View File

@ -0,0 +1,2 @@
variables:
WindowsContainerImage: 'onebranch.azurecr.io/windows/ltsc2022/vse2022:latest'