diff --git a/src/arcade/eng/common/BuildConfiguration/build-configuration.json b/src/arcade/eng/common/BuildConfiguration/build-configuration.json deleted file mode 100644 index 3d1cc89894c..00000000000 --- a/src/arcade/eng/common/BuildConfiguration/build-configuration.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "RetryCountLimit": 1, - "RetryByAnyError": false -} diff --git a/src/arcade/eng/common/PSScriptAnalyzerSettings.psd1 b/src/arcade/eng/common/PSScriptAnalyzerSettings.psd1 deleted file mode 100644 index 4c1ea7c98ea..00000000000 --- a/src/arcade/eng/common/PSScriptAnalyzerSettings.psd1 +++ /dev/null @@ -1,11 +0,0 @@ -@{ - IncludeRules=@('PSAvoidUsingCmdletAliases', - 'PSAvoidUsingWMICmdlet', - 'PSAvoidUsingPositionalParameters', - 'PSAvoidUsingInvokeExpression', - 'PSUseDeclaredVarsMoreThanAssignments', - 'PSUseCmdletCorrectly', - 'PSStandardDSCFunctionsInResource', - 'PSUseIdenticalMandatoryParametersForDSC', - 'PSUseIdenticalParametersForDSC') -} \ No newline at end of file diff --git a/src/arcade/eng/common/README.md b/src/arcade/eng/common/README.md deleted file mode 100644 index ff49c371527..00000000000 --- a/src/arcade/eng/common/README.md +++ /dev/null @@ -1,28 +0,0 @@ -# Don't touch this folder - - uuuuuuuuuuuuuuuuuuuu - u" uuuuuuuuuuuuuuuuuu "u - u" u$$$$$$$$$$$$$$$$$$$$u "u - u" u$$$$$$$$$$$$$$$$$$$$$$$$u "u - u" u$$$$$$$$$$$$$$$$$$$$$$$$$$$$u "u - u" u$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$u "u - u" u$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$u "u - $ $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ $ - $ $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ $ - $ $$$" ... "$... ...$" ... "$$$ ... "$$$ $ - $ $$$u `"$$$$$$$ $$$ $$$$$ $$ $$$ $$$ $ - $ $$$$$$uu "$$$$ $$$ $$$$$ $$ """ u$$$ $ - $ $$$""$$$ $$$$ $$$u "$$$" u$$ $$$$$$$$ $ - $ $$$$....,$$$$$..$$$$$....,$$$$..$$$$$$$$ $ - $ $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ $ - "u "$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$" u" - "u "$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$" u" - "u "$$$$$$$$$$$$$$$$$$$$$$$$$$$$" u" - "u "$$$$$$$$$$$$$$$$$$$$$$$$" u" - "u "$$$$$$$$$$$$$$$$$$$$" u" - "u """""""""""""""""" u" - """""""""""""""""""" - -!!! Changes made in this directory are subject to being overwritten by automation !!! - -The files in this directory are shared by all Arcade repos and managed by automation. If you need to make changes to these files, open an issue or submit a pull request to https://github.com/dotnet/arcade first. diff --git a/src/arcade/eng/common/cross/arm/tizen/tizen.patch b/src/arcade/eng/common/cross/arm/tizen/tizen.patch deleted file mode 100644 index fb12ade7250..00000000000 --- a/src/arcade/eng/common/cross/arm/tizen/tizen.patch +++ /dev/null @@ -1,9 +0,0 @@ -diff -u -r a/usr/lib/libc.so b/usr/lib/libc.so ---- a/usr/lib/libc.so 2016-12-30 23:00:08.284951863 +0900 -+++ b/usr/lib/libc.so 2016-12-30 23:00:32.140951815 +0900 -@@ -2,4 +2,4 @@ - Use the shared library, but some functions are only in - the static library, so try that secondarily. */ - OUTPUT_FORMAT(elf32-littlearm) --GROUP ( /lib/libc.so.6 /usr/lib/libc_nonshared.a AS_NEEDED ( /lib/ld-linux-armhf.so.3 ) ) -+GROUP ( libc.so.6 libc_nonshared.a AS_NEEDED ( ld-linux-armhf.so.3 ) ) diff --git a/src/arcade/eng/common/cross/armel/tizen/tizen.patch b/src/arcade/eng/common/cross/armel/tizen/tizen.patch deleted file mode 100644 index ca7c7c1ff75..00000000000 --- a/src/arcade/eng/common/cross/armel/tizen/tizen.patch +++ /dev/null @@ -1,9 +0,0 @@ -diff -u -r a/usr/lib/libc.so b/usr/lib/libc.so ---- a/usr/lib/libc.so 2016-12-30 23:00:08.284951863 +0900 -+++ b/usr/lib/libc.so 2016-12-30 23:00:32.140951815 +0900 -@@ -2,4 +2,4 @@ - Use the shared library, but some functions are only in - the static library, so try that secondarily. */ - OUTPUT_FORMAT(elf32-littlearm) --GROUP ( /lib/libc.so.6 /usr/lib/libc_nonshared.a AS_NEEDED ( /lib/ld-linux.so.3 ) ) -+GROUP ( libc.so.6 libc_nonshared.a AS_NEEDED ( ld-linux.so.3 ) ) diff --git a/src/arcade/eng/common/dotnet-install.cmd b/src/arcade/eng/common/dotnet-install.cmd deleted file mode 100644 index b1c2642e76f..00000000000 --- a/src/arcade/eng/common/dotnet-install.cmd +++ /dev/null @@ -1,2 +0,0 @@ -@echo off -powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0dotnet-install.ps1""" %*" \ No newline at end of file diff --git a/src/arcade/eng/common/dotnet-install.ps1 b/src/arcade/eng/common/dotnet-install.ps1 deleted file mode 100644 index 811f0f717f7..00000000000 --- a/src/arcade/eng/common/dotnet-install.ps1 +++ /dev/null @@ -1,28 +0,0 @@ -[CmdletBinding(PositionalBinding=$false)] -Param( - [string] $verbosity = 'minimal', - [string] $architecture = '', - [string] $version = 'Latest', - [string] $runtime = 'dotnet', - [string] $RuntimeSourceFeed = '', - [string] $RuntimeSourceFeedKey = '' -) - -. $PSScriptRoot\tools.ps1 - -$dotnetRoot = Join-Path $RepoRoot '.dotnet' - -$installdir = $dotnetRoot -try { - if ($architecture -and $architecture.Trim() -eq 'x86') { - $installdir = Join-Path $installdir 'x86' - } - InstallDotNet $installdir $version $architecture $runtime $true -RuntimeSourceFeed $RuntimeSourceFeed -RuntimeSourceFeedKey $RuntimeSourceFeedKey -} -catch { - Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Category 'InitializeToolset' -Message $_ - ExitWithExitCode 1 -} - -ExitWithExitCode 0 diff --git a/src/arcade/eng/common/enable-cross-org-publishing.ps1 b/src/arcade/eng/common/enable-cross-org-publishing.ps1 deleted file mode 100644 index da09da4f1fc..00000000000 --- a/src/arcade/eng/common/enable-cross-org-publishing.ps1 +++ /dev/null @@ -1,13 +0,0 @@ -param( - [string] $token -) - - -. $PSScriptRoot\pipeline-logging-functions.ps1 - -# Write-PipelineSetVariable will no-op if a variable named $ci is not defined -# Since this script is only ever called in AzDO builds, just universally set it -$ci = $true - -Write-PipelineSetVariable -Name 'VSS_NUGET_ACCESSTOKEN' -Value $token -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'VSS_NUGET_URI_PREFIXES' -Value 'https://dnceng.pkgs.visualstudio.com/;https://pkgs.dev.azure.com/dnceng/;https://devdiv.pkgs.visualstudio.com/;https://pkgs.dev.azure.com/devdiv/' -IsMultiJobVariable $false diff --git a/src/arcade/eng/common/generate-locproject.ps1 b/src/arcade/eng/common/generate-locproject.ps1 deleted file mode 100644 index 524aaa57f2b..00000000000 --- a/src/arcade/eng/common/generate-locproject.ps1 +++ /dev/null @@ -1,189 +0,0 @@ -Param( - [Parameter(Mandatory=$true)][string] $SourcesDirectory, # Directory where source files live; if using a Localize directory it should live in here - [string] $LanguageSet = 'VS_Main_Languages', # Language set to be used in the LocProject.json - [switch] $UseCheckedInLocProjectJson, # When set, generates a LocProject.json and compares it to one that already exists in the repo; otherwise just generates one - [switch] $CreateNeutralXlfs # Creates neutral xlf files. Only set to false when running locally -) - -# Generates LocProject.json files for the OneLocBuild task. OneLocBuildTask is described here: -# https://ceapex.visualstudio.com/CEINTL/_wiki/wikis/CEINTL.wiki/107/Localization-with-OneLocBuild-Task - -Set-StrictMode -Version 2.0 -$ErrorActionPreference = "Stop" -. $PSScriptRoot\pipeline-logging-functions.ps1 - -$exclusionsFilePath = "$SourcesDirectory\eng\Localize\LocExclusions.json" -$exclusions = @{ Exclusions = @() } -if (Test-Path -Path $exclusionsFilePath) -{ - $exclusions = Get-Content "$exclusionsFilePath" | ConvertFrom-Json -} - -Push-Location "$SourcesDirectory" # push location for Resolve-Path -Relative to work - -# Template files -$jsonFiles = @() -$jsonTemplateFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\.template\.config\\localize\\.+\.en\.json" } # .NET templating pattern -$jsonTemplateFiles | ForEach-Object { - $null = $_.Name -Match "(.+)\.[\w-]+\.json" # matches '[filename].[langcode].json - - $destinationFile = "$($_.Directory.FullName)\$($Matches.1).json" - $jsonFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru -} - -$jsonWinformsTemplateFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\\strings\.json" } # current winforms pattern - -$wxlFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\\.+\.wxl" -And -Not( $_.Directory.Name -Match "\d{4}" ) } # localized files live in four digit lang ID directories; this excludes them -if (-not $wxlFiles) { - $wxlEnFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\\1033\\.+\.wxl" } # pick up en files (1033 = en) specifically so we can copy them to use as the neutral xlf files - if ($wxlEnFiles) { - $wxlFiles = @() - $wxlEnFiles | ForEach-Object { - $destinationFile = "$($_.Directory.Parent.FullName)\$($_.Name)" - $wxlFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru - } - } -} - -$macosHtmlEnFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\.lproj\\.+\.html$" } # add installer HTML files -$macosHtmlFiles = @() -if ($macosHtmlEnFiles) { - $macosHtmlEnFiles | ForEach-Object { - $destinationFile = "$($_.Directory.Parent.FullName)\$($_.Name)" - $macosHtmlFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru - } -} - -$xlfFiles = @() - -$allXlfFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory\*\*.xlf" -$langXlfFiles = @() -if ($allXlfFiles) { - $null = $allXlfFiles[0].FullName -Match "\.([\w-]+)\.xlf" # matches '[langcode].xlf' - $firstLangCode = $Matches.1 - $langXlfFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory\*\*.$firstLangCode.xlf" -} -$langXlfFiles | ForEach-Object { - $null = $_.Name -Match "(.+)\.[\w-]+\.xlf" # matches '[filename].[langcode].xlf - - $destinationFile = "$($_.Directory.FullName)\$($Matches.1).xlf" - $xlfFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru -} - -$locFiles = $jsonFiles + $jsonWinformsTemplateFiles + $xlfFiles - -$locJson = @{ - Projects = @( - @{ - LanguageSet = $LanguageSet - LocItems = @( - $locFiles | ForEach-Object { - $outputPath = "$(($_.DirectoryName | Resolve-Path -Relative) + "\")" - $continue = $true - foreach ($exclusion in $exclusions.Exclusions) { - if ($_.FullName.Contains($exclusion)) - { - $continue = $false - } - } - $sourceFile = ($_.FullName | Resolve-Path -Relative) - if (!$CreateNeutralXlfs -and $_.Extension -eq '.xlf') { - Remove-Item -Path $sourceFile - } - if ($continue) - { - if ($_.Directory.Name -eq 'en' -and $_.Extension -eq '.json') { - return @{ - SourceFile = $sourceFile - CopyOption = "LangIDOnPath" - OutputPath = "$($_.Directory.Parent.FullName | Resolve-Path -Relative)\" - } - } else { - return @{ - SourceFile = $sourceFile - CopyOption = "LangIDOnName" - OutputPath = $outputPath - } - } - } - } - ) - }, - @{ - LanguageSet = $LanguageSet - CloneLanguageSet = "WiX_CloneLanguages" - LssFiles = @( "wxl_loc.lss" ) - LocItems = @( - $wxlFiles | ForEach-Object { - $outputPath = "$($_.Directory.FullName | Resolve-Path -Relative)\" - $continue = $true - foreach ($exclusion in $exclusions.Exclusions) { - if ($_.FullName.Contains($exclusion)) { - $continue = $false - } - } - $sourceFile = ($_.FullName | Resolve-Path -Relative) - if ($continue) - { - return @{ - SourceFile = $sourceFile - CopyOption = "LangIDOnPath" - OutputPath = $outputPath - } - } - } - ) - }, - @{ - LanguageSet = $LanguageSet - CloneLanguageSet = "VS_macOS_CloneLanguages" - LssFiles = @( ".\eng\common\loc\P22DotNetHtmlLocalization.lss" ) - LocItems = @( - $macosHtmlFiles | ForEach-Object { - $outputPath = "$($_.Directory.FullName | Resolve-Path -Relative)\" - $continue = $true - foreach ($exclusion in $exclusions.Exclusions) { - if ($_.FullName.Contains($exclusion)) { - $continue = $false - } - } - $sourceFile = ($_.FullName | Resolve-Path -Relative) - $lciFile = $sourceFile + ".lci" - if ($continue) { - $result = @{ - SourceFile = $sourceFile - CopyOption = "LangIDOnPath" - OutputPath = $outputPath - } - if (Test-Path $lciFile -PathType Leaf) { - $result["LciFile"] = $lciFile - } - return $result - } - } - ) - } - ) -} - -$json = ConvertTo-Json $locJson -Depth 5 -Write-Host "LocProject.json generated:`n`n$json`n`n" -Pop-Location - -if (!$UseCheckedInLocProjectJson) { - New-Item "$SourcesDirectory\eng\Localize\LocProject.json" -Force # Need this to make sure the Localize directory is created - Set-Content "$SourcesDirectory\eng\Localize\LocProject.json" $json -} -else { - New-Item "$SourcesDirectory\eng\Localize\LocProject-generated.json" -Force # Need this to make sure the Localize directory is created - Set-Content "$SourcesDirectory\eng\Localize\LocProject-generated.json" $json - - if ((Get-FileHash "$SourcesDirectory\eng\Localize\LocProject-generated.json").Hash -ne (Get-FileHash "$SourcesDirectory\eng\Localize\LocProject.json").Hash) { - Write-PipelineTelemetryError -Category "OneLocBuild" -Message "Existing LocProject.json differs from generated LocProject.json. Download LocProject-generated.json and compare them." - - exit 1 - } - else { - Write-Host "Generated LocProject.json and current LocProject.json are identical." - } -} diff --git a/src/arcade/eng/common/generate-sbom-prep.ps1 b/src/arcade/eng/common/generate-sbom-prep.ps1 deleted file mode 100644 index a0c7d792a76..00000000000 --- a/src/arcade/eng/common/generate-sbom-prep.ps1 +++ /dev/null @@ -1,29 +0,0 @@ -Param( - [Parameter(Mandatory=$true)][string] $ManifestDirPath # Manifest directory where sbom will be placed -) - -. $PSScriptRoot\pipeline-logging-functions.ps1 - -# Normally - we'd listen to the manifest path given, but 1ES templates will overwrite if this level gets uploaded directly -# with their own overwriting ours. So we create it as a sub directory of the requested manifest path. -$ArtifactName = "${env:SYSTEM_STAGENAME}_${env:AGENT_JOBNAME}_SBOM" -$SafeArtifactName = $ArtifactName -replace '["/:<>\\|?@*"() ]', '_' -$SbomGenerationDir = Join-Path $ManifestDirPath $SafeArtifactName - -Write-Host "Artifact name before : $ArtifactName" -Write-Host "Artifact name after : $SafeArtifactName" - -Write-Host "Creating dir $ManifestDirPath" - -# create directory for sbom manifest to be placed -if (!(Test-Path -path $SbomGenerationDir)) -{ - New-Item -ItemType Directory -path $SbomGenerationDir - Write-Host "Successfully created directory $SbomGenerationDir" -} -else{ - Write-PipelineTelemetryError -category 'Build' "Unable to create sbom folder." -} - -Write-Host "Updating artifact name" -Write-Host "##vso[task.setvariable variable=ARTIFACT_NAME]$SafeArtifactName" diff --git a/src/arcade/eng/common/init-tools-native.cmd b/src/arcade/eng/common/init-tools-native.cmd deleted file mode 100644 index 438cd548c45..00000000000 --- a/src/arcade/eng/common/init-tools-native.cmd +++ /dev/null @@ -1,3 +0,0 @@ -@echo off -powershell -NoProfile -NoLogo -ExecutionPolicy ByPass -command "& """%~dp0init-tools-native.ps1""" %*" -exit /b %ErrorLevel% \ No newline at end of file diff --git a/src/arcade/eng/common/init-tools-native.ps1 b/src/arcade/eng/common/init-tools-native.ps1 deleted file mode 100644 index 27ccdb9ecc9..00000000000 --- a/src/arcade/eng/common/init-tools-native.ps1 +++ /dev/null @@ -1,203 +0,0 @@ -<# -.SYNOPSIS -Entry point script for installing native tools - -.DESCRIPTION -Reads $RepoRoot\global.json file to determine native assets to install -and executes installers for those tools - -.PARAMETER BaseUri -Base file directory or Url from which to acquire tool archives - -.PARAMETER InstallDirectory -Directory to install native toolset. This is a command-line override for the default -Install directory precedence order: -- InstallDirectory command-line override -- NETCOREENG_INSTALL_DIRECTORY environment variable -- (default) %USERPROFILE%/.netcoreeng/native - -.PARAMETER Clean -Switch specifying to not install anything, but cleanup native asset folders - -.PARAMETER Force -Clean and then install tools - -.PARAMETER DownloadRetries -Total number of retry attempts - -.PARAMETER RetryWaitTimeInSeconds -Wait time between retry attempts in seconds - -.PARAMETER GlobalJsonFile -File path to global.json file - -.PARAMETER PathPromotion -Optional switch to enable either promote native tools specified in the global.json to the path (in Azure Pipelines) -or break the build if a native tool is not found on the path (on a local dev machine) - -.NOTES -#> -[CmdletBinding(PositionalBinding=$false)] -Param ( - [string] $BaseUri = 'https://netcorenativeassets.blob.core.windows.net/resource-packages/external', - [string] $InstallDirectory, - [switch] $Clean = $False, - [switch] $Force = $False, - [int] $DownloadRetries = 5, - [int] $RetryWaitTimeInSeconds = 30, - [string] $GlobalJsonFile, - [switch] $PathPromotion -) - -if (!$GlobalJsonFile) { - $GlobalJsonFile = Join-Path (Get-Item $PSScriptRoot).Parent.Parent.FullName 'global.json' -} - -Set-StrictMode -version 2.0 -$ErrorActionPreference='Stop' - -. $PSScriptRoot\pipeline-logging-functions.ps1 -Import-Module -Name (Join-Path $PSScriptRoot 'native\CommonLibrary.psm1') - -try { - # Define verbose switch if undefined - $Verbose = $VerbosePreference -Eq 'Continue' - - $EngCommonBaseDir = Join-Path $PSScriptRoot 'native\' - $NativeBaseDir = $InstallDirectory - if (!$NativeBaseDir) { - $NativeBaseDir = CommonLibrary\Get-NativeInstallDirectory - } - $Env:CommonLibrary_NativeInstallDir = $NativeBaseDir - $InstallBin = Join-Path $NativeBaseDir 'bin' - $InstallerPath = Join-Path $EngCommonBaseDir 'install-tool.ps1' - - # Process tools list - Write-Host "Processing $GlobalJsonFile" - If (-Not (Test-Path $GlobalJsonFile)) { - Write-Host "Unable to find '$GlobalJsonFile'" - exit 0 - } - $NativeTools = Get-Content($GlobalJsonFile) -Raw | - ConvertFrom-Json | - Select-Object -Expand 'native-tools' -ErrorAction SilentlyContinue - if ($NativeTools) { - if ($PathPromotion -eq $True) { - $ArcadeToolsDirectory = "$env:SYSTEMDRIVE\arcade-tools" - if (Test-Path $ArcadeToolsDirectory) { # if this directory exists, we should use native tools on machine - $NativeTools.PSObject.Properties | ForEach-Object { - $ToolName = $_.Name - $ToolVersion = $_.Value - $InstalledTools = @{} - - if ((Get-Command "$ToolName" -ErrorAction SilentlyContinue) -eq $null) { - if ($ToolVersion -eq "latest") { - $ToolVersion = "" - } - $ToolDirectories = (Get-ChildItem -Path "$ArcadeToolsDirectory" -Filter "$ToolName-$ToolVersion*" | Sort-Object -Descending) - if ($ToolDirectories -eq $null) { - Write-Error "Unable to find directory for $ToolName $ToolVersion; please make sure the tool is installed on this image." - exit 1 - } - $ToolDirectory = $ToolDirectories[0] - $BinPathFile = "$($ToolDirectory.FullName)\binpath.txt" - if (-not (Test-Path -Path "$BinPathFile")) { - Write-Error "Unable to find binpath.txt in '$($ToolDirectory.FullName)' ($ToolName $ToolVersion); artifact is either installed incorrectly or is not a bootstrappable tool." - exit 1 - } - $BinPath = Get-Content "$BinPathFile" - $ToolPath = Convert-Path -Path $BinPath - Write-Host "Adding $ToolName to the path ($ToolPath)..." - Write-Host "##vso[task.prependpath]$ToolPath" - $env:PATH = "$ToolPath;$env:PATH" - $InstalledTools += @{ $ToolName = $ToolDirectory.FullName } - } - } - return $InstalledTools - } else { - $NativeTools.PSObject.Properties | ForEach-Object { - $ToolName = $_.Name - $ToolVersion = $_.Value - - if ((Get-Command "$ToolName" -ErrorAction SilentlyContinue) -eq $null) { - Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message "$ToolName not found on path. Please install $ToolName $ToolVersion before proceeding." - Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message "If this is running on a build machine, the arcade-tools directory was not found, which means there's an error with the image." - } - } - exit 0 - } - } else { - $NativeTools.PSObject.Properties | ForEach-Object { - $ToolName = $_.Name - $ToolVersion = $_.Value - $LocalInstallerArguments = @{ ToolName = "$ToolName" } - $LocalInstallerArguments += @{ InstallPath = "$InstallBin" } - $LocalInstallerArguments += @{ BaseUri = "$BaseUri" } - $LocalInstallerArguments += @{ CommonLibraryDirectory = "$EngCommonBaseDir" } - $LocalInstallerArguments += @{ Version = "$ToolVersion" } - - if ($Verbose) { - $LocalInstallerArguments += @{ Verbose = $True } - } - if (Get-Variable 'Force' -ErrorAction 'SilentlyContinue') { - if($Force) { - $LocalInstallerArguments += @{ Force = $True } - } - } - if ($Clean) { - $LocalInstallerArguments += @{ Clean = $True } - } - - Write-Verbose "Installing $ToolName version $ToolVersion" - Write-Verbose "Executing '$InstallerPath $($LocalInstallerArguments.Keys.ForEach({"-$_ '$($LocalInstallerArguments.$_)'"}) -join ' ')'" - & $InstallerPath @LocalInstallerArguments - if ($LASTEXITCODE -Ne "0") { - $errMsg = "$ToolName installation failed" - if ((Get-Variable 'DoNotAbortNativeToolsInstallationOnFailure' -ErrorAction 'SilentlyContinue') -and $DoNotAbortNativeToolsInstallationOnFailure) { - $showNativeToolsWarning = $true - if ((Get-Variable 'DoNotDisplayNativeToolsInstallationWarnings' -ErrorAction 'SilentlyContinue') -and $DoNotDisplayNativeToolsInstallationWarnings) { - $showNativeToolsWarning = $false - } - if ($showNativeToolsWarning) { - Write-Warning $errMsg - } - $toolInstallationFailure = $true - } else { - # We cannot change this to Write-PipelineTelemetryError because of https://github.com/dotnet/arcade/issues/4482 - Write-Host $errMsg - exit 1 - } - } - } - - if ((Get-Variable 'toolInstallationFailure' -ErrorAction 'SilentlyContinue') -and $toolInstallationFailure) { - # We cannot change this to Write-PipelineTelemetryError because of https://github.com/dotnet/arcade/issues/4482 - Write-Host 'Native tools bootstrap failed' - exit 1 - } - } - } - else { - Write-Host 'No native tools defined in global.json' - exit 0 - } - - if ($Clean) { - exit 0 - } - if (Test-Path $InstallBin) { - Write-Host 'Native tools are available from ' (Convert-Path -Path $InstallBin) - Write-Host "##vso[task.prependpath]$(Convert-Path -Path $InstallBin)" - return $InstallBin - } - elseif (-not ($PathPromotion)) { - Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message 'Native tools install directory does not exist, installation failed' - exit 1 - } - exit 0 -} -catch { - Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message $_ - ExitWithExitCode 1 -} diff --git a/src/arcade/eng/common/init-tools-native.sh b/src/arcade/eng/common/init-tools-native.sh deleted file mode 100644 index 3e6a8d6acf2..00000000000 --- a/src/arcade/eng/common/init-tools-native.sh +++ /dev/null @@ -1,238 +0,0 @@ -#!/usr/bin/env bash - -source="${BASH_SOURCE[0]}" -scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - -base_uri='https://netcorenativeassets.blob.core.windows.net/resource-packages/external' -install_directory='' -clean=false -force=false -download_retries=5 -retry_wait_time_seconds=30 -global_json_file="$(dirname "$(dirname "${scriptroot}")")/global.json" -declare -a native_assets - -. $scriptroot/pipeline-logging-functions.sh -. $scriptroot/native/common-library.sh - -while (($# > 0)); do - lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")" - case $lowerI in - --baseuri) - base_uri=$2 - shift 2 - ;; - --installdirectory) - install_directory=$2 - shift 2 - ;; - --clean) - clean=true - shift 1 - ;; - --force) - force=true - shift 1 - ;; - --donotabortonfailure) - donotabortonfailure=true - shift 1 - ;; - --donotdisplaywarnings) - donotdisplaywarnings=true - shift 1 - ;; - --downloadretries) - download_retries=$2 - shift 2 - ;; - --retrywaittimeseconds) - retry_wait_time_seconds=$2 - shift 2 - ;; - --help) - echo "Common settings:" - echo " --installdirectory Directory to install native toolset." - echo " This is a command-line override for the default" - echo " Install directory precedence order:" - echo " - InstallDirectory command-line override" - echo " - NETCOREENG_INSTALL_DIRECTORY environment variable" - echo " - (default) %USERPROFILE%/.netcoreeng/native" - echo "" - echo " --clean Switch specifying not to install anything, but cleanup native asset folders" - echo " --donotabortonfailure Switch specifiying whether to abort native tools installation on failure" - echo " --donotdisplaywarnings Switch specifiying whether to display warnings during native tools installation on failure" - echo " --force Clean and then install tools" - echo " --help Print help and exit" - echo "" - echo "Advanced settings:" - echo " --baseuri Base URI for where to download native tools from" - echo " --downloadretries Number of times a download should be attempted" - echo " --retrywaittimeseconds Wait time between download attempts" - echo "" - exit 0 - ;; - esac -done - -function ReadGlobalJsonNativeTools { - # happy path: we have a proper JSON parsing tool `jq(1)` in PATH! - if command -v jq &> /dev/null; then - - # jq: read each key/value pair under "native-tools" entry and emit: - # KEY="" VALUE="" - # followed by a null byte. - # - # bash: read line with null byte delimeter and push to array (for later `eval`uation). - - while IFS= read -rd '' line; do - native_assets+=("$line") - done < <(jq -r '. | - select(has("native-tools")) | - ."native-tools" | - keys[] as $k | - @sh "KEY=\($k) VALUE=\(.[$k])\u0000"' "$global_json_file") - - return - fi - - # Warning: falling back to manually parsing JSON, which is not recommended. - - # Following routine matches the output and escaping logic of jq(1)'s @sh formatter used above. - # It has been tested with several weird strings with escaped characters in entries (key and value) - # and results were compared with the output of jq(1) in binary representation using xxd(1); - # just before the assignment to 'native_assets' array (above and below). - - # try to capture the section under "native-tools". - if [[ ! "$(cat "$global_json_file")" =~ \"native-tools\"[[:space:]\:\{]*([^\}]+) ]]; then - return - fi - - section="${BASH_REMATCH[1]}" - - parseStarted=0 - possibleEnd=0 - escaping=0 - escaped=0 - isKey=1 - - for (( i=0; i<${#section}; i++ )); do - char="${section:$i:1}" - if ! ((parseStarted)) && [[ "$char" =~ [[:space:],:] ]]; then continue; fi - - if ! ((escaping)) && [[ "$char" == "\\" ]]; then - escaping=1 - elif ((escaping)) && ! ((escaped)); then - escaped=1 - fi - - if ! ((parseStarted)) && [[ "$char" == "\"" ]]; then - parseStarted=1 - possibleEnd=0 - elif [[ "$char" == "'" ]]; then - token="$token'\\\''" - possibleEnd=0 - elif ((escaping)) || [[ "$char" != "\"" ]]; then - token="$token$char" - possibleEnd=1 - fi - - if ((possibleEnd)) && ! ((escaping)) && [[ "$char" == "\"" ]]; then - # Use printf to unescape token to match jq(1)'s @sh formatting rules. - # do not use 'token="$(printf "$token")"' syntax, as $() eats the trailing linefeed. - printf -v token "'$token'" - - if ((isKey)); then - KEY="$token" - isKey=0 - else - line="KEY=$KEY VALUE=$token" - native_assets+=("$line") - isKey=1 - fi - - # reset for next token - parseStarted=0 - token= - elif ((escaping)) && ((escaped)); then - escaping=0 - escaped=0 - fi - done -} - -native_base_dir=$install_directory -if [[ -z $install_directory ]]; then - native_base_dir=$(GetNativeInstallDirectory) -fi - -install_bin="${native_base_dir}/bin" -installed_any=false - -ReadGlobalJsonNativeTools - -if [[ ${#native_assets[@]} -eq 0 ]]; then - echo "No native tools defined in global.json" - exit 0; -else - native_installer_dir="$scriptroot/native" - for index in "${!native_assets[@]}"; do - eval "${native_assets["$index"]}" - - installer_path="$native_installer_dir/install-$KEY.sh" - installer_command="$installer_path" - installer_command+=" --baseuri $base_uri" - installer_command+=" --installpath $install_bin" - installer_command+=" --version $VALUE" - echo $installer_command - - if [[ $force = true ]]; then - installer_command+=" --force" - fi - - if [[ $clean = true ]]; then - installer_command+=" --clean" - fi - - if [[ -a $installer_path ]]; then - $installer_command - if [[ $? != 0 ]]; then - if [[ $donotabortonfailure = true ]]; then - if [[ $donotdisplaywarnings != true ]]; then - Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Execution Failed" - fi - else - Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Execution Failed" - exit 1 - fi - else - $installed_any = true - fi - else - if [[ $donotabortonfailure == true ]]; then - if [[ $donotdisplaywarnings != true ]]; then - Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Execution Failed: no install script" - fi - else - Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Execution Failed: no install script" - exit 1 - fi - fi - done -fi - -if [[ $clean = true ]]; then - exit 0 -fi - -if [[ -d $install_bin ]]; then - echo "Native tools are available from $install_bin" - echo "##vso[task.prependpath]$install_bin" -else - if [[ $installed_any = true ]]; then - Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Native tools install directory does not exist, installation failed" - exit 1 - fi -fi - -exit 0 diff --git a/src/arcade/eng/common/internal-feed-operations.ps1 b/src/arcade/eng/common/internal-feed-operations.ps1 deleted file mode 100644 index 92b77347d99..00000000000 --- a/src/arcade/eng/common/internal-feed-operations.ps1 +++ /dev/null @@ -1,132 +0,0 @@ -param( - [Parameter(Mandatory=$true)][string] $Operation, - [string] $AuthToken, - [string] $CommitSha, - [string] $RepoName, - [switch] $IsFeedPrivate -) - -$ErrorActionPreference = 'Stop' -Set-StrictMode -Version 2.0 -. $PSScriptRoot\tools.ps1 - -# Sets VSS_NUGET_EXTERNAL_FEED_ENDPOINTS based on the "darc-int-*" feeds defined in NuGet.config. This is needed -# in build agents by CredProvider to authenticate the restore requests to internal feeds as specified in -# https://github.com/microsoft/artifacts-credprovider/blob/0f53327cd12fd893d8627d7b08a2171bf5852a41/README.md#environment-variables. This should ONLY be called from identified -# internal builds -function SetupCredProvider { - param( - [string] $AuthToken - ) - - # Install the Cred Provider NuGet plugin - Write-Host 'Setting up Cred Provider NuGet plugin in the agent...' - Write-Host "Getting 'installcredprovider.ps1' from 'https://github.com/microsoft/artifacts-credprovider'..." - - $url = 'https://raw.githubusercontent.com/microsoft/artifacts-credprovider/master/helpers/installcredprovider.ps1' - - Write-Host "Writing the contents of 'installcredprovider.ps1' locally..." - Invoke-WebRequest $url -OutFile installcredprovider.ps1 - - Write-Host 'Installing plugin...' - .\installcredprovider.ps1 -Force - - Write-Host "Deleting local copy of 'installcredprovider.ps1'..." - Remove-Item .\installcredprovider.ps1 - - if (-Not("$env:USERPROFILE\.nuget\plugins\netcore")) { - Write-PipelineTelemetryError -Category 'Arcade' -Message 'CredProvider plugin was not installed correctly!' - ExitWithExitCode 1 - } - else { - Write-Host 'CredProvider plugin was installed correctly!' - } - - # Then, we set the 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' environment variable to restore from the stable - # feeds successfully - - $nugetConfigPath = Join-Path $RepoRoot "NuGet.config" - - if (-Not (Test-Path -Path $nugetConfigPath)) { - Write-PipelineTelemetryError -Category 'Build' -Message 'NuGet.config file not found in repo root!' - ExitWithExitCode 1 - } - - $endpoints = New-Object System.Collections.ArrayList - $nugetConfigPackageSources = Select-Xml -Path $nugetConfigPath -XPath "//packageSources/add[contains(@key, 'darc-int-')]/@value" | foreach{$_.Node.Value} - - if (($nugetConfigPackageSources | Measure-Object).Count -gt 0 ) { - foreach ($stableRestoreResource in $nugetConfigPackageSources) { - $trimmedResource = ([string]$stableRestoreResource).Trim() - [void]$endpoints.Add(@{endpoint="$trimmedResource"; password="$AuthToken"}) - } - } - - if (($endpoints | Measure-Object).Count -gt 0) { - $endpointCredentials = @{endpointCredentials=$endpoints} | ConvertTo-Json -Compress - - # Create the environment variables the AzDo way - Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data $endpointCredentials -Properties @{ - 'variable' = 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' - 'issecret' = 'false' - } - - # We don't want sessions cached since we will be updating the endpoints quite frequently - Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data 'False' -Properties @{ - 'variable' = 'NUGET_CREDENTIALPROVIDER_SESSIONTOKENCACHE_ENABLED' - 'issecret' = 'false' - } - } - else - { - Write-Host 'No internal endpoints found in NuGet.config' - } -} - -#Workaround for https://github.com/microsoft/msbuild/issues/4430 -function InstallDotNetSdkAndRestoreArcade { - $dotnetTempDir = Join-Path $RepoRoot "dotnet" - $dotnetSdkVersion="2.1.507" # After experimentation we know this version works when restoring the SDK (compared to 3.0.*) - $dotnet = "$dotnetTempDir\dotnet.exe" - $restoreProjPath = "$PSScriptRoot\restore.proj" - - Write-Host "Installing dotnet SDK version $dotnetSdkVersion to restore Arcade SDK..." - InstallDotNetSdk "$dotnetTempDir" "$dotnetSdkVersion" - - '' | Out-File "$restoreProjPath" - - & $dotnet restore $restoreProjPath - - Write-Host 'Arcade SDK restored!' - - if (Test-Path -Path $restoreProjPath) { - Remove-Item $restoreProjPath - } - - if (Test-Path -Path $dotnetTempDir) { - Remove-Item $dotnetTempDir -Recurse - } -} - -try { - Push-Location $PSScriptRoot - - if ($Operation -like 'setup') { - SetupCredProvider $AuthToken - } - elseif ($Operation -like 'install-restore') { - InstallDotNetSdkAndRestoreArcade - } - else { - Write-PipelineTelemetryError -Category 'Arcade' -Message "Unknown operation '$Operation'!" - ExitWithExitCode 1 - } -} -catch { - Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Category 'Arcade' -Message $_ - ExitWithExitCode 1 -} -finally { - Pop-Location -} diff --git a/src/arcade/eng/common/internal-feed-operations.sh b/src/arcade/eng/common/internal-feed-operations.sh deleted file mode 100644 index 9378223ba09..00000000000 --- a/src/arcade/eng/common/internal-feed-operations.sh +++ /dev/null @@ -1,141 +0,0 @@ -#!/usr/bin/env bash - -set -e - -# Sets VSS_NUGET_EXTERNAL_FEED_ENDPOINTS based on the "darc-int-*" feeds defined in NuGet.config. This is needed -# in build agents by CredProvider to authenticate the restore requests to internal feeds as specified in -# https://github.com/microsoft/artifacts-credprovider/blob/0f53327cd12fd893d8627d7b08a2171bf5852a41/README.md#environment-variables. -# This should ONLY be called from identified internal builds -function SetupCredProvider { - local authToken=$1 - - # Install the Cred Provider NuGet plugin - echo "Setting up Cred Provider NuGet plugin in the agent..."... - echo "Getting 'installcredprovider.ps1' from 'https://github.com/microsoft/artifacts-credprovider'..." - - local url="https://raw.githubusercontent.com/microsoft/artifacts-credprovider/master/helpers/installcredprovider.sh" - - echo "Writing the contents of 'installcredprovider.ps1' locally..." - local installcredproviderPath="installcredprovider.sh" - if command -v curl > /dev/null; then - curl $url > "$installcredproviderPath" - else - wget -q -O "$installcredproviderPath" "$url" - fi - - echo "Installing plugin..." - . "$installcredproviderPath" - - echo "Deleting local copy of 'installcredprovider.sh'..." - rm installcredprovider.sh - - if [ ! -d "$HOME/.nuget/plugins" ]; then - Write-PipelineTelemetryError -category 'Build' 'CredProvider plugin was not installed correctly!' - ExitWithExitCode 1 - else - echo "CredProvider plugin was installed correctly!" - fi - - # Then, we set the 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' environment variable to restore from the stable - # feeds successfully - - local nugetConfigPath="{$repo_root}NuGet.config" - - if [ ! "$nugetConfigPath" ]; then - Write-PipelineTelemetryError -category 'Build' "NuGet.config file not found in repo's root!" - ExitWithExitCode 1 - fi - - local endpoints='[' - local nugetConfigPackageValues=`cat "$nugetConfigPath" | grep "key=\"darc-int-"` - local pattern="value=\"(.*)\"" - - for value in $nugetConfigPackageValues - do - if [[ $value =~ $pattern ]]; then - local endpoint="${BASH_REMATCH[1]}" - endpoints+="{\"endpoint\": \"$endpoint\", \"password\": \"$authToken\"}," - fi - done - - endpoints=${endpoints%?} - endpoints+=']' - - if [ ${#endpoints} -gt 2 ]; then - local endpointCredentials="{\"endpointCredentials\": "$endpoints"}" - - echo "##vso[task.setvariable variable=VSS_NUGET_EXTERNAL_FEED_ENDPOINTS]$endpointCredentials" - echo "##vso[task.setvariable variable=NUGET_CREDENTIALPROVIDER_SESSIONTOKENCACHE_ENABLED]False" - else - echo "No internal endpoints found in NuGet.config" - fi -} - -# Workaround for https://github.com/microsoft/msbuild/issues/4430 -function InstallDotNetSdkAndRestoreArcade { - local dotnetTempDir="$repo_root/dotnet" - local dotnetSdkVersion="2.1.507" # After experimentation we know this version works when restoring the SDK (compared to 3.0.*) - local restoreProjPath="$repo_root/eng/common/restore.proj" - - echo "Installing dotnet SDK version $dotnetSdkVersion to restore Arcade SDK..." - echo "" > "$restoreProjPath" - - InstallDotNetSdk "$dotnetTempDir" "$dotnetSdkVersion" - - local res=`$dotnetTempDir/dotnet restore $restoreProjPath` - echo "Arcade SDK restored!" - - # Cleanup - if [ "$restoreProjPath" ]; then - rm "$restoreProjPath" - fi - - if [ "$dotnetTempDir" ]; then - rm -r $dotnetTempDir - fi -} - -source="${BASH_SOURCE[0]}" -operation='' -authToken='' -repoName='' - -while [[ $# > 0 ]]; do - opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")" - case "$opt" in - --operation) - operation=$2 - shift - ;; - --authtoken) - authToken=$2 - shift - ;; - *) - echo "Invalid argument: $1" - usage - exit 1 - ;; - esac - - shift -done - -while [[ -h "$source" ]]; do - scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - source="$(readlink "$source")" - # if $source was a relative symlink, we need to resolve it relative to the path where the - # symlink file was located - [[ $source != /* ]] && source="$scriptroot/$source" -done -scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - -. "$scriptroot/tools.sh" - -if [ "$operation" = "setup" ]; then - SetupCredProvider $authToken -elif [ "$operation" = "install-restore" ]; then - InstallDotNetSdkAndRestoreArcade -else - echo "Unknown operation '$operation'!" -fi diff --git a/src/arcade/eng/common/internal/NuGet.config b/src/arcade/eng/common/internal/NuGet.config deleted file mode 100644 index 19d3d311b16..00000000000 --- a/src/arcade/eng/common/internal/NuGet.config +++ /dev/null @@ -1,7 +0,0 @@ - - - - - - - diff --git a/src/arcade/eng/common/loc/P22DotNetHtmlLocalization.lss b/src/arcade/eng/common/loc/P22DotNetHtmlLocalization.lss deleted file mode 100644 index 5d892d61939..00000000000 --- a/src/arcade/eng/common/loc/P22DotNetHtmlLocalization.lss +++ /dev/null @@ -1,29 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/src/arcade/eng/common/msbuild.ps1 b/src/arcade/eng/common/msbuild.ps1 deleted file mode 100644 index f041e5ddd95..00000000000 --- a/src/arcade/eng/common/msbuild.ps1 +++ /dev/null @@ -1,28 +0,0 @@ -[CmdletBinding(PositionalBinding=$false)] -Param( - [string] $verbosity = 'minimal', - [bool] $warnAsError = $true, - [bool] $nodeReuse = $true, - [switch] $ci, - [switch] $prepareMachine, - [switch] $excludePrereleaseVS, - [string] $msbuildEngine = $null, - [Parameter(ValueFromRemainingArguments=$true)][String[]]$extraArgs -) - -. $PSScriptRoot\tools.ps1 - -try { - if ($ci) { - $nodeReuse = $false - } - - MSBuild @extraArgs -} -catch { - Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Category 'Build' -Message $_ - ExitWithExitCode 1 -} - -ExitWithExitCode 0 \ No newline at end of file diff --git a/src/arcade/eng/common/msbuild.sh b/src/arcade/eng/common/msbuild.sh deleted file mode 100644 index 20d3dad5435..00000000000 --- a/src/arcade/eng/common/msbuild.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env bash - -source="${BASH_SOURCE[0]}" - -# resolve $source until the file is no longer a symlink -while [[ -h "$source" ]]; do - scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - source="$(readlink "$source")" - # if $source was a relative symlink, we need to resolve it relative to the path where the - # symlink file was located - [[ $source != /* ]] && source="$scriptroot/$source" -done -scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - -verbosity='minimal' -warn_as_error=true -node_reuse=true -prepare_machine=false -extra_args='' - -while (($# > 0)); do - lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")" - case $lowerI in - --verbosity) - verbosity=$2 - shift 2 - ;; - --warnaserror) - warn_as_error=$2 - shift 2 - ;; - --nodereuse) - node_reuse=$2 - shift 2 - ;; - --ci) - ci=true - shift 1 - ;; - --preparemachine) - prepare_machine=true - shift 1 - ;; - *) - extra_args="$extra_args $1" - shift 1 - ;; - esac -done - -. "$scriptroot/tools.sh" - -if [[ "$ci" == true ]]; then - node_reuse=false -fi - -MSBuild $extra_args -ExitWithExitCode 0 diff --git a/src/arcade/eng/common/native/common-library.sh b/src/arcade/eng/common/native/common-library.sh deleted file mode 100644 index 080c2c283ae..00000000000 --- a/src/arcade/eng/common/native/common-library.sh +++ /dev/null @@ -1,172 +0,0 @@ -#!/usr/bin/env bash - -function GetNativeInstallDirectory { - local install_dir - - if [[ -z $NETCOREENG_INSTALL_DIRECTORY ]]; then - install_dir=$HOME/.netcoreeng/native/ - else - install_dir=$NETCOREENG_INSTALL_DIRECTORY - fi - - echo $install_dir - return 0 -} - -function GetTempDirectory { - - echo $(GetNativeInstallDirectory)temp/ - return 0 -} - -function ExpandZip { - local zip_path=$1 - local output_directory=$2 - local force=${3:-false} - - echo "Extracting $zip_path to $output_directory" - if [[ -d $output_directory ]] && [[ $force = false ]]; then - echo "Directory '$output_directory' already exists, skipping extract" - return 0 - fi - - if [[ -d $output_directory ]]; then - echo "'Force flag enabled, but '$output_directory' exists. Removing directory" - rm -rf $output_directory - if [[ $? != 0 ]]; then - Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Unable to remove '$output_directory'" - return 1 - fi - fi - - echo "Creating directory: '$output_directory'" - mkdir -p $output_directory - - echo "Extracting archive" - tar -xf $zip_path -C $output_directory - if [[ $? != 0 ]]; then - Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Unable to extract '$zip_path'" - return 1 - fi - - return 0 -} - -function GetCurrentOS { - local unameOut="$(uname -s)" - case $unameOut in - Linux*) echo "Linux";; - Darwin*) echo "MacOS";; - esac - return 0 -} - -function GetFile { - local uri=$1 - local path=$2 - local force=${3:-false} - local download_retries=${4:-5} - local retry_wait_time_seconds=${5:-30} - - if [[ -f $path ]]; then - if [[ $force = false ]]; then - echo "File '$path' already exists. Skipping download" - return 0 - else - rm -rf $path - fi - fi - - if [[ -f $uri ]]; then - echo "'$uri' is a file path, copying file to '$path'" - cp $uri $path - return $? - fi - - echo "Downloading $uri" - # Use curl if available, otherwise use wget - if command -v curl > /dev/null; then - curl "$uri" -sSL --retry $download_retries --retry-delay $retry_wait_time_seconds --create-dirs -o "$path" --fail - else - wget -q -O "$path" "$uri" --tries="$download_retries" - fi - - return $? -} - -function GetTempPathFileName { - local path=$1 - - local temp_dir=$(GetTempDirectory) - local temp_file_name=$(basename $path) - echo $temp_dir$temp_file_name - return 0 -} - -function DownloadAndExtract { - local uri=$1 - local installDir=$2 - local force=${3:-false} - local download_retries=${4:-5} - local retry_wait_time_seconds=${5:-30} - - local temp_tool_path=$(GetTempPathFileName $uri) - - echo "downloading to: $temp_tool_path" - - # Download file - GetFile "$uri" "$temp_tool_path" $force $download_retries $retry_wait_time_seconds - if [[ $? != 0 ]]; then - Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Failed to download '$uri' to '$temp_tool_path'." - return 1 - fi - - # Extract File - echo "extracting from $temp_tool_path to $installDir" - ExpandZip "$temp_tool_path" "$installDir" $force $download_retries $retry_wait_time_seconds - if [[ $? != 0 ]]; then - Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Failed to extract '$temp_tool_path' to '$installDir'." - return 1 - fi - - return 0 -} - -function NewScriptShim { - local shimpath=$1 - local tool_file_path=$2 - local force=${3:-false} - - echo "Generating '$shimpath' shim" - if [[ -f $shimpath ]]; then - if [[ $force = false ]]; then - echo "File '$shimpath' already exists." >&2 - return 1 - else - rm -rf $shimpath - fi - fi - - if [[ ! -f $tool_file_path ]]; then - # try to see if the path is lower cased - tool_file_path="$(echo $tool_file_path | tr "[:upper:]" "[:lower:]")" - if [[ ! -f $tool_file_path ]]; then - Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Specified tool file path:'$tool_file_path' does not exist" - return 1 - fi - fi - - local shim_contents=$'#!/usr/bin/env bash\n' - shim_contents+="SHIMARGS="$'$1\n' - shim_contents+="$tool_file_path"$' $SHIMARGS\n' - - # Write shim file - echo "$shim_contents" > $shimpath - - chmod +x $shimpath - - echo "Finished generating shim '$shimpath'" - - return $? -} - diff --git a/src/arcade/eng/common/native/install-cmake-test.sh b/src/arcade/eng/common/native/install-cmake-test.sh deleted file mode 100644 index 8a5e7cf0db5..00000000000 --- a/src/arcade/eng/common/native/install-cmake-test.sh +++ /dev/null @@ -1,117 +0,0 @@ -#!/usr/bin/env bash - -source="${BASH_SOURCE[0]}" -scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - -. $scriptroot/common-library.sh - -base_uri= -install_path= -version= -clean=false -force=false -download_retries=5 -retry_wait_time_seconds=30 - -while (($# > 0)); do - lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")" - case $lowerI in - --baseuri) - base_uri=$2 - shift 2 - ;; - --installpath) - install_path=$2 - shift 2 - ;; - --version) - version=$2 - shift 2 - ;; - --clean) - clean=true - shift 1 - ;; - --force) - force=true - shift 1 - ;; - --downloadretries) - download_retries=$2 - shift 2 - ;; - --retrywaittimeseconds) - retry_wait_time_seconds=$2 - shift 2 - ;; - --help) - echo "Common settings:" - echo " --baseuri Base file directory or Url wrom which to acquire tool archives" - echo " --installpath Base directory to install native tool to" - echo " --clean Don't install the tool, just clean up the current install of the tool" - echo " --force Force install of tools even if they previously exist" - echo " --help Print help and exit" - echo "" - echo "Advanced settings:" - echo " --downloadretries Total number of retry attempts" - echo " --retrywaittimeseconds Wait time between retry attempts in seconds" - echo "" - exit 0 - ;; - esac -done - -tool_name="cmake-test" -tool_os=$(GetCurrentOS) -tool_folder="$(echo $tool_os | tr "[:upper:]" "[:lower:]")" -tool_arch="x86_64" -tool_name_moniker="$tool_name-$version-$tool_os-$tool_arch" -tool_install_directory="$install_path/$tool_name/$version" -tool_file_path="$tool_install_directory/$tool_name_moniker/bin/$tool_name" -shim_path="$install_path/$tool_name.sh" -uri="${base_uri}/$tool_folder/$tool_name/$tool_name_moniker.tar.gz" - -# Clean up tool and installers -if [[ $clean = true ]]; then - echo "Cleaning $tool_install_directory" - if [[ -d $tool_install_directory ]]; then - rm -rf $tool_install_directory - fi - - echo "Cleaning $shim_path" - if [[ -f $shim_path ]]; then - rm -rf $shim_path - fi - - tool_temp_path=$(GetTempPathFileName $uri) - echo "Cleaning $tool_temp_path" - if [[ -f $tool_temp_path ]]; then - rm -rf $tool_temp_path - fi - - exit 0 -fi - -# Install tool -if [[ -f $tool_file_path ]] && [[ $force = false ]]; then - echo "$tool_name ($version) already exists, skipping install" - exit 0 -fi - -DownloadAndExtract $uri $tool_install_directory $force $download_retries $retry_wait_time_seconds - -if [[ $? != 0 ]]; then - Write-PipelineTelemetryError -category 'NativeToolsBootstrap' 'Installation failed' - exit 1 -fi - -# Generate Shim -# Always rewrite shims so that we are referencing the expected version -NewScriptShim $shim_path $tool_file_path true - -if [[ $? != 0 ]]; then - Write-PipelineTelemetryError -category 'NativeToolsBootstrap' 'Shim generation failed' - exit 1 -fi - -exit 0 diff --git a/src/arcade/eng/common/native/install-cmake.sh b/src/arcade/eng/common/native/install-cmake.sh deleted file mode 100644 index de496beebc5..00000000000 --- a/src/arcade/eng/common/native/install-cmake.sh +++ /dev/null @@ -1,117 +0,0 @@ -#!/usr/bin/env bash - -source="${BASH_SOURCE[0]}" -scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - -. $scriptroot/common-library.sh - -base_uri= -install_path= -version= -clean=false -force=false -download_retries=5 -retry_wait_time_seconds=30 - -while (($# > 0)); do - lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")" - case $lowerI in - --baseuri) - base_uri=$2 - shift 2 - ;; - --installpath) - install_path=$2 - shift 2 - ;; - --version) - version=$2 - shift 2 - ;; - --clean) - clean=true - shift 1 - ;; - --force) - force=true - shift 1 - ;; - --downloadretries) - download_retries=$2 - shift 2 - ;; - --retrywaittimeseconds) - retry_wait_time_seconds=$2 - shift 2 - ;; - --help) - echo "Common settings:" - echo " --baseuri Base file directory or Url wrom which to acquire tool archives" - echo " --installpath Base directory to install native tool to" - echo " --clean Don't install the tool, just clean up the current install of the tool" - echo " --force Force install of tools even if they previously exist" - echo " --help Print help and exit" - echo "" - echo "Advanced settings:" - echo " --downloadretries Total number of retry attempts" - echo " --retrywaittimeseconds Wait time between retry attempts in seconds" - echo "" - exit 0 - ;; - esac -done - -tool_name="cmake" -tool_os=$(GetCurrentOS) -tool_folder="$(echo $tool_os | tr "[:upper:]" "[:lower:]")" -tool_arch="x86_64" -tool_name_moniker="$tool_name-$version-$tool_os-$tool_arch" -tool_install_directory="$install_path/$tool_name/$version" -tool_file_path="$tool_install_directory/$tool_name_moniker/bin/$tool_name" -shim_path="$install_path/$tool_name.sh" -uri="${base_uri}/$tool_folder/$tool_name/$tool_name_moniker.tar.gz" - -# Clean up tool and installers -if [[ $clean = true ]]; then - echo "Cleaning $tool_install_directory" - if [[ -d $tool_install_directory ]]; then - rm -rf $tool_install_directory - fi - - echo "Cleaning $shim_path" - if [[ -f $shim_path ]]; then - rm -rf $shim_path - fi - - tool_temp_path=$(GetTempPathFileName $uri) - echo "Cleaning $tool_temp_path" - if [[ -f $tool_temp_path ]]; then - rm -rf $tool_temp_path - fi - - exit 0 -fi - -# Install tool -if [[ -f $tool_file_path ]] && [[ $force = false ]]; then - echo "$tool_name ($version) already exists, skipping install" - exit 0 -fi - -DownloadAndExtract $uri $tool_install_directory $force $download_retries $retry_wait_time_seconds - -if [[ $? != 0 ]]; then - Write-PipelineTelemetryError -category 'NativeToolsBootstrap' 'Installation failed' - exit 1 -fi - -# Generate Shim -# Always rewrite shims so that we are referencing the expected version -NewScriptShim $shim_path $tool_file_path true - -if [[ $? != 0 ]]; then - Write-PipelineTelemetryError -category 'NativeToolsBootstrap' 'Shim generation failed' - exit 1 -fi - -exit 0 diff --git a/src/arcade/eng/common/native/install-tool.ps1 b/src/arcade/eng/common/native/install-tool.ps1 deleted file mode 100644 index 78f2d84a4e4..00000000000 --- a/src/arcade/eng/common/native/install-tool.ps1 +++ /dev/null @@ -1,132 +0,0 @@ -<# -.SYNOPSIS -Install native tool - -.DESCRIPTION -Install cmake native tool from Azure blob storage - -.PARAMETER InstallPath -Base directory to install native tool to - -.PARAMETER BaseUri -Base file directory or Url from which to acquire tool archives - -.PARAMETER CommonLibraryDirectory -Path to folder containing common library modules - -.PARAMETER Force -Force install of tools even if they previously exist - -.PARAMETER Clean -Don't install the tool, just clean up the current install of the tool - -.PARAMETER DownloadRetries -Total number of retry attempts - -.PARAMETER RetryWaitTimeInSeconds -Wait time between retry attempts in seconds - -.NOTES -Returns 0 if install succeeds, 1 otherwise -#> -[CmdletBinding(PositionalBinding=$false)] -Param ( - [Parameter(Mandatory=$True)] - [string] $ToolName, - [Parameter(Mandatory=$True)] - [string] $InstallPath, - [Parameter(Mandatory=$True)] - [string] $BaseUri, - [Parameter(Mandatory=$True)] - [string] $Version, - [string] $CommonLibraryDirectory = $PSScriptRoot, - [switch] $Force = $False, - [switch] $Clean = $False, - [int] $DownloadRetries = 5, - [int] $RetryWaitTimeInSeconds = 30 -) - -. $PSScriptRoot\..\pipeline-logging-functions.ps1 - -# Import common library modules -Import-Module -Name (Join-Path $CommonLibraryDirectory "CommonLibrary.psm1") - -try { - # Define verbose switch if undefined - $Verbose = $VerbosePreference -Eq "Continue" - - $Arch = CommonLibrary\Get-MachineArchitecture - $ToolOs = "win64" - if($Arch -Eq "x32") { - $ToolOs = "win32" - } - $ToolNameMoniker = "$ToolName-$Version-$ToolOs-$Arch" - $ToolInstallDirectory = Join-Path $InstallPath "$ToolName\$Version\" - $Uri = "$BaseUri/windows/$ToolName/$ToolNameMoniker.zip" - $ShimPath = Join-Path $InstallPath "$ToolName.exe" - - if ($Clean) { - Write-Host "Cleaning $ToolInstallDirectory" - if (Test-Path $ToolInstallDirectory) { - Remove-Item $ToolInstallDirectory -Force -Recurse - } - Write-Host "Cleaning $ShimPath" - if (Test-Path $ShimPath) { - Remove-Item $ShimPath -Force - } - $ToolTempPath = CommonLibrary\Get-TempPathFilename -Path $Uri - Write-Host "Cleaning $ToolTempPath" - if (Test-Path $ToolTempPath) { - Remove-Item $ToolTempPath -Force - } - exit 0 - } - - # Install tool - if ((Test-Path $ToolInstallDirectory) -And (-Not $Force)) { - Write-Verbose "$ToolName ($Version) already exists, skipping install" - } - else { - $InstallStatus = CommonLibrary\DownloadAndExtract -Uri $Uri ` - -InstallDirectory $ToolInstallDirectory ` - -Force:$Force ` - -DownloadRetries $DownloadRetries ` - -RetryWaitTimeInSeconds $RetryWaitTimeInSeconds ` - -Verbose:$Verbose - - if ($InstallStatus -Eq $False) { - Write-PipelineTelemetryError "Installation failed" -Category "NativeToolsetBootstrapping" - exit 1 - } - } - - $ToolFilePath = Get-ChildItem $ToolInstallDirectory -Recurse -Filter "$ToolName.exe" | % { $_.FullName } - if (@($ToolFilePath).Length -Gt 1) { - Write-Error "There are multiple copies of $ToolName in $($ToolInstallDirectory): `n$(@($ToolFilePath | out-string))" - exit 1 - } elseif (@($ToolFilePath).Length -Lt 1) { - Write-Host "$ToolName was not found in $ToolInstallDirectory." - exit 1 - } - - # Generate shim - # Always rewrite shims so that we are referencing the expected version - $GenerateShimStatus = CommonLibrary\New-ScriptShim -ShimName $ToolName ` - -ShimDirectory $InstallPath ` - -ToolFilePath "$ToolFilePath" ` - -BaseUri $BaseUri ` - -Force:$Force ` - -Verbose:$Verbose - - if ($GenerateShimStatus -Eq $False) { - Write-PipelineTelemetryError "Generate shim failed" -Category "NativeToolsetBootstrapping" - return 1 - } - - exit 0 -} -catch { - Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Category "NativeToolsetBootstrapping" -Message $_ - exit 1 -} diff --git a/src/arcade/eng/common/pipeline-logging-functions.ps1 b/src/arcade/eng/common/pipeline-logging-functions.ps1 deleted file mode 100644 index 8e422c561e4..00000000000 --- a/src/arcade/eng/common/pipeline-logging-functions.ps1 +++ /dev/null @@ -1,260 +0,0 @@ -# Source for this file was taken from https://github.com/microsoft/azure-pipelines-task-lib/blob/11c9439d4af17e6475d9fe058e6b2e03914d17e6/powershell/VstsTaskSdk/LoggingCommandFunctions.ps1 and modified. - -# NOTE: You should not be calling these method directly as they are likely to change. Instead you should be calling the Write-Pipeline* functions defined in tools.ps1 - -$script:loggingCommandPrefix = '##vso[' -$script:loggingCommandEscapeMappings = @( # TODO: WHAT ABOUT "="? WHAT ABOUT "%"? - New-Object psobject -Property @{ Token = ';' ; Replacement = '%3B' } - New-Object psobject -Property @{ Token = "`r" ; Replacement = '%0D' } - New-Object psobject -Property @{ Token = "`n" ; Replacement = '%0A' } - New-Object psobject -Property @{ Token = "]" ; Replacement = '%5D' } -) -# TODO: BUG: Escape % ??? -# TODO: Add test to verify don't need to escape "=". - -# Specify "-Force" to force pipeline formatted output even if "$ci" is false or not set -function Write-PipelineTelemetryError { - [CmdletBinding()] - param( - [Parameter(Mandatory = $true)] - [string]$Category, - [Parameter(Mandatory = $true)] - [string]$Message, - [Parameter(Mandatory = $false)] - [string]$Type = 'error', - [string]$ErrCode, - [string]$SourcePath, - [string]$LineNumber, - [string]$ColumnNumber, - [switch]$AsOutput, - [switch]$Force) - - $PSBoundParameters.Remove('Category') | Out-Null - - if ($Force -Or ((Test-Path variable:ci) -And $ci)) { - $Message = "(NETCORE_ENGINEERING_TELEMETRY=$Category) $Message" - } - $PSBoundParameters.Remove('Message') | Out-Null - $PSBoundParameters.Add('Message', $Message) - Write-PipelineTaskError @PSBoundParameters -} - -# Specify "-Force" to force pipeline formatted output even if "$ci" is false or not set -function Write-PipelineTaskError { - [CmdletBinding()] - param( - [Parameter(Mandatory = $true)] - [string]$Message, - [Parameter(Mandatory = $false)] - [string]$Type = 'error', - [string]$ErrCode, - [string]$SourcePath, - [string]$LineNumber, - [string]$ColumnNumber, - [switch]$AsOutput, - [switch]$Force - ) - - if (!$Force -And (-Not (Test-Path variable:ci) -Or !$ci)) { - if ($Type -eq 'error') { - Write-Host $Message -ForegroundColor Red - return - } - elseif ($Type -eq 'warning') { - Write-Host $Message -ForegroundColor Yellow - return - } - } - - if (($Type -ne 'error') -and ($Type -ne 'warning')) { - Write-Host $Message - return - } - $PSBoundParameters.Remove('Force') | Out-Null - if (-not $PSBoundParameters.ContainsKey('Type')) { - $PSBoundParameters.Add('Type', 'error') - } - Write-LogIssue @PSBoundParameters -} - -function Write-PipelineSetVariable { - [CmdletBinding()] - param( - [Parameter(Mandatory = $true)] - [string]$Name, - [string]$Value, - [switch]$Secret, - [switch]$AsOutput, - [bool]$IsMultiJobVariable = $true) - - if ((Test-Path variable:ci) -And $ci) { - Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data $Value -Properties @{ - 'variable' = $Name - 'isSecret' = $Secret - 'isOutput' = $IsMultiJobVariable - } -AsOutput:$AsOutput - } -} - -function Write-PipelinePrependPath { - [CmdletBinding()] - param( - [Parameter(Mandatory = $true)] - [string]$Path, - [switch]$AsOutput) - - if ((Test-Path variable:ci) -And $ci) { - Write-LoggingCommand -Area 'task' -Event 'prependpath' -Data $Path -AsOutput:$AsOutput - } -} - -function Write-PipelineSetResult { - [CmdletBinding()] - param( - [ValidateSet("Succeeded", "SucceededWithIssues", "Failed", "Cancelled", "Skipped")] - [Parameter(Mandatory = $true)] - [string]$Result, - [string]$Message) - if ((Test-Path variable:ci) -And $ci) { - Write-LoggingCommand -Area 'task' -Event 'complete' -Data $Message -Properties @{ - 'result' = $Result - } - } -} - -<######################################## -# Private functions. -########################################> -function Format-LoggingCommandData { - [CmdletBinding()] - param([string]$Value, [switch]$Reverse) - - if (!$Value) { - return '' - } - - if (!$Reverse) { - foreach ($mapping in $script:loggingCommandEscapeMappings) { - $Value = $Value.Replace($mapping.Token, $mapping.Replacement) - } - } - else { - for ($i = $script:loggingCommandEscapeMappings.Length - 1 ; $i -ge 0 ; $i--) { - $mapping = $script:loggingCommandEscapeMappings[$i] - $Value = $Value.Replace($mapping.Replacement, $mapping.Token) - } - } - - return $Value -} - -function Format-LoggingCommand { - [CmdletBinding()] - param( - [Parameter(Mandatory = $true)] - [string]$Area, - [Parameter(Mandatory = $true)] - [string]$Event, - [string]$Data, - [hashtable]$Properties) - - # Append the preamble. - [System.Text.StringBuilder]$sb = New-Object -TypeName System.Text.StringBuilder - $null = $sb.Append($script:loggingCommandPrefix).Append($Area).Append('.').Append($Event) - - # Append the properties. - if ($Properties) { - $first = $true - foreach ($key in $Properties.Keys) { - [string]$value = Format-LoggingCommandData $Properties[$key] - if ($value) { - if ($first) { - $null = $sb.Append(' ') - $first = $false - } - else { - $null = $sb.Append(';') - } - - $null = $sb.Append("$key=$value") - } - } - } - - # Append the tail and output the value. - $Data = Format-LoggingCommandData $Data - $sb.Append(']').Append($Data).ToString() -} - -function Write-LoggingCommand { - [CmdletBinding(DefaultParameterSetName = 'Parameters')] - param( - [Parameter(Mandatory = $true, ParameterSetName = 'Parameters')] - [string]$Area, - [Parameter(Mandatory = $true, ParameterSetName = 'Parameters')] - [string]$Event, - [Parameter(ParameterSetName = 'Parameters')] - [string]$Data, - [Parameter(ParameterSetName = 'Parameters')] - [hashtable]$Properties, - [Parameter(Mandatory = $true, ParameterSetName = 'Object')] - $Command, - [switch]$AsOutput) - - if ($PSCmdlet.ParameterSetName -eq 'Object') { - Write-LoggingCommand -Area $Command.Area -Event $Command.Event -Data $Command.Data -Properties $Command.Properties -AsOutput:$AsOutput - return - } - - $command = Format-LoggingCommand -Area $Area -Event $Event -Data $Data -Properties $Properties - if ($AsOutput) { - $command - } - else { - Write-Host $command - } -} - -function Write-LogIssue { - [CmdletBinding()] - param( - [ValidateSet('warning', 'error')] - [Parameter(Mandatory = $true)] - [string]$Type, - [string]$Message, - [string]$ErrCode, - [string]$SourcePath, - [string]$LineNumber, - [string]$ColumnNumber, - [switch]$AsOutput) - - $command = Format-LoggingCommand -Area 'task' -Event 'logissue' -Data $Message -Properties @{ - 'type' = $Type - 'code' = $ErrCode - 'sourcepath' = $SourcePath - 'linenumber' = $LineNumber - 'columnnumber' = $ColumnNumber - } - if ($AsOutput) { - return $command - } - - if ($Type -eq 'error') { - $foregroundColor = $host.PrivateData.ErrorForegroundColor - $backgroundColor = $host.PrivateData.ErrorBackgroundColor - if ($foregroundColor -isnot [System.ConsoleColor] -or $backgroundColor -isnot [System.ConsoleColor]) { - $foregroundColor = [System.ConsoleColor]::Red - $backgroundColor = [System.ConsoleColor]::Black - } - } - else { - $foregroundColor = $host.PrivateData.WarningForegroundColor - $backgroundColor = $host.PrivateData.WarningBackgroundColor - if ($foregroundColor -isnot [System.ConsoleColor] -or $backgroundColor -isnot [System.ConsoleColor]) { - $foregroundColor = [System.ConsoleColor]::Yellow - $backgroundColor = [System.ConsoleColor]::Black - } - } - - Write-Host $command -ForegroundColor $foregroundColor -BackgroundColor $backgroundColor -} diff --git a/src/arcade/eng/common/pipeline-logging-functions.sh b/src/arcade/eng/common/pipeline-logging-functions.sh deleted file mode 100644 index 6a0b2255e91..00000000000 --- a/src/arcade/eng/common/pipeline-logging-functions.sh +++ /dev/null @@ -1,206 +0,0 @@ -#!/usr/bin/env bash - -function Write-PipelineTelemetryError { - local telemetry_category='' - local force=false - local function_args=() - local message='' - while [[ $# -gt 0 ]]; do - opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")" - case "$opt" in - -category|-c) - telemetry_category=$2 - shift - ;; - -force|-f) - force=true - ;; - -*) - function_args+=("$1 $2") - shift - ;; - *) - message=$* - ;; - esac - shift - done - - if [[ $force != true ]] && [[ "$ci" != true ]]; then - echo "$message" >&2 - return - fi - - if [[ $force == true ]]; then - function_args+=("-force") - fi - message="(NETCORE_ENGINEERING_TELEMETRY=$telemetry_category) $message" - function_args+=("$message") - Write-PipelineTaskError ${function_args[@]} -} - -function Write-PipelineTaskError { - local message_type="error" - local sourcepath='' - local linenumber='' - local columnnumber='' - local error_code='' - local force=false - - while [[ $# -gt 0 ]]; do - opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")" - case "$opt" in - -type|-t) - message_type=$2 - shift - ;; - -sourcepath|-s) - sourcepath=$2 - shift - ;; - -linenumber|-ln) - linenumber=$2 - shift - ;; - -columnnumber|-cn) - columnnumber=$2 - shift - ;; - -errcode|-e) - error_code=$2 - shift - ;; - -force|-f) - force=true - ;; - *) - break - ;; - esac - - shift - done - - if [[ $force != true ]] && [[ "$ci" != true ]]; then - echo "$@" >&2 - return - fi - - local message="##vso[task.logissue" - - message="$message type=$message_type" - - if [ -n "$sourcepath" ]; then - message="$message;sourcepath=$sourcepath" - fi - - if [ -n "$linenumber" ]; then - message="$message;linenumber=$linenumber" - fi - - if [ -n "$columnnumber" ]; then - message="$message;columnnumber=$columnnumber" - fi - - if [ -n "$error_code" ]; then - message="$message;code=$error_code" - fi - - message="$message]$*" - echo "$message" -} - -function Write-PipelineSetVariable { - if [[ "$ci" != true ]]; then - return - fi - - local name='' - local value='' - local secret=false - local as_output=false - local is_multi_job_variable=true - - while [[ $# -gt 0 ]]; do - opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")" - case "$opt" in - -name|-n) - name=$2 - shift - ;; - -value|-v) - value=$2 - shift - ;; - -secret|-s) - secret=true - ;; - -as_output|-a) - as_output=true - ;; - -is_multi_job_variable|-i) - is_multi_job_variable=$2 - shift - ;; - esac - shift - done - - value=${value/;/%3B} - value=${value/\\r/%0D} - value=${value/\\n/%0A} - value=${value/]/%5D} - - local message="##vso[task.setvariable variable=$name;isSecret=$secret;isOutput=$is_multi_job_variable]$value" - - if [[ "$as_output" == true ]]; then - $message - else - echo "$message" - fi -} - -function Write-PipelinePrependPath { - local prepend_path='' - - while [[ $# -gt 0 ]]; do - opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")" - case "$opt" in - -path|-p) - prepend_path=$2 - shift - ;; - esac - shift - done - - export PATH="$prepend_path:$PATH" - - if [[ "$ci" == true ]]; then - echo "##vso[task.prependpath]$prepend_path" - fi -} - -function Write-PipelineSetResult { - local result='' - local message='' - - while [[ $# -gt 0 ]]; do - opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")" - case "$opt" in - -result|-r) - result=$2 - shift - ;; - -message|-m) - message=$2 - shift - ;; - esac - shift - done - - if [[ "$ci" == true ]]; then - echo "##vso[task.complete result=$result;]$message" - fi -} diff --git a/src/arcade/eng/common/retain-build.ps1 b/src/arcade/eng/common/retain-build.ps1 deleted file mode 100644 index e7ba975adeb..00000000000 --- a/src/arcade/eng/common/retain-build.ps1 +++ /dev/null @@ -1,45 +0,0 @@ - -Param( -[Parameter(Mandatory=$true)][int] $buildId, -[Parameter(Mandatory=$true)][string] $azdoOrgUri, -[Parameter(Mandatory=$true)][string] $azdoProject, -[Parameter(Mandatory=$true)][string] $token -) - -$ErrorActionPreference = 'Stop' -Set-StrictMode -Version 2.0 - -function Get-AzDOHeaders( - [string] $token) -{ - $base64AuthInfo = [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes(":${token}")) - $headers = @{"Authorization"="Basic $base64AuthInfo"} - return $headers -} - -function Update-BuildRetention( - [string] $azdoOrgUri, - [string] $azdoProject, - [int] $buildId, - [string] $token) -{ - $headers = Get-AzDOHeaders -token $token - $requestBody = "{ - `"keepForever`": `"true`" - }" - - $requestUri = "${azdoOrgUri}/${azdoProject}/_apis/build/builds/${buildId}?api-version=6.0" - write-Host "Attempting to retain build using the following URI: ${requestUri} ..." - - try { - Invoke-RestMethod -Uri $requestUri -Method Patch -Body $requestBody -Header $headers -contentType "application/json" - Write-Host "Updated retention settings for build ${buildId}." - } - catch { - Write-Error "Failed to update retention settings for build: $_.Exception.Response.StatusDescription" - exit 1 - } -} - -Update-BuildRetention -azdoOrgUri $azdoOrgUri -azdoProject $azdoProject -buildId $buildId -token $token -exit 0 diff --git a/src/arcade/eng/common/sdl/configure-sdl-tool.ps1 b/src/arcade/eng/common/sdl/configure-sdl-tool.ps1 deleted file mode 100644 index 27f5a4115fc..00000000000 --- a/src/arcade/eng/common/sdl/configure-sdl-tool.ps1 +++ /dev/null @@ -1,130 +0,0 @@ -Param( - [string] $GuardianCliLocation, - [string] $WorkingDirectory, - [string] $TargetDirectory, - [string] $GdnFolder, - # The list of Guardian tools to configure. For each object in the array: - # - If the item is a [hashtable], it must contain these entries: - # - Name = The tool name as Guardian knows it. - # - Scenario = (Optional) Scenario-specific name for this configuration entry. It must be unique - # among all tool entries with the same Name. - # - Args = (Optional) Array of Guardian tool configuration args, like '@("Target > C:\temp")' - # - If the item is a [string] $v, it is treated as '@{ Name="$v" }' - [object[]] $ToolsList, - [string] $GuardianLoggerLevel='Standard', - # Optional: Additional params to add to any tool using CredScan. - [string[]] $CrScanAdditionalRunConfigParams, - # Optional: Additional params to add to any tool using PoliCheck. - [string[]] $PoliCheckAdditionalRunConfigParams, - # Optional: Additional params to add to any tool using CodeQL/Semmle. - [string[]] $CodeQLAdditionalRunConfigParams, - # Optional: Additional params to add to any tool using Binskim. - [string[]] $BinskimAdditionalRunConfigParams -) - -$ErrorActionPreference = 'Stop' -Set-StrictMode -Version 2.0 -$disableConfigureToolsetImport = $true -$global:LASTEXITCODE = 0 - -try { - # `tools.ps1` checks $ci to perform some actions. Since the SDL - # scripts don't necessarily execute in the same agent that run the - # build.ps1/sh script this variable isn't automatically set. - $ci = $true - . $PSScriptRoot\..\tools.ps1 - - # Normalize tools list: all in [hashtable] form with defined values for each key. - $ToolsList = $ToolsList | - ForEach-Object { - if ($_ -is [string]) { - $_ = @{ Name = $_ } - } - - if (-not ($_['Scenario'])) { $_.Scenario = "" } - if (-not ($_['Args'])) { $_.Args = @() } - $_ - } - - Write-Host "List of tools to configure:" - $ToolsList | ForEach-Object { $_ | Out-String | Write-Host } - - # We store config files in the r directory of .gdn - $gdnConfigPath = Join-Path $GdnFolder 'r' - $ValidPath = Test-Path $GuardianCliLocation - - if ($ValidPath -eq $False) - { - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Invalid Guardian CLI Location." - ExitWithExitCode 1 - } - - foreach ($tool in $ToolsList) { - # Put together the name and scenario to make a unique key. - $toolConfigName = $tool.Name - if ($tool.Scenario) { - $toolConfigName += "_" + $tool.Scenario - } - - Write-Host "=== Configuring $toolConfigName..." - - $gdnConfigFile = Join-Path $gdnConfigPath "$toolConfigName-configure.gdnconfig" - - # For some tools, add default and automatic args. - switch -Exact ($tool.Name) { - 'credscan' { - if ($targetDirectory) { - $tool.Args += "`"TargetDirectory < $TargetDirectory`"" - } - $tool.Args += "`"OutputType < pre`"" - $tool.Args += $CrScanAdditionalRunConfigParams - } - 'policheck' { - if ($targetDirectory) { - $tool.Args += "`"Target < $TargetDirectory`"" - } - $tool.Args += $PoliCheckAdditionalRunConfigParams - } - {$_ -in 'semmle', 'codeql'} { - if ($targetDirectory) { - $tool.Args += "`"SourceCodeDirectory < $TargetDirectory`"" - } - $tool.Args += $CodeQLAdditionalRunConfigParams - } - 'binskim' { - if ($targetDirectory) { - # Binskim crashes due to specific PDBs. GitHub issue: https://github.com/microsoft/binskim/issues/924. - # We are excluding all `_.pdb` files from the scan. - $tool.Args += "`"Target < $TargetDirectory\**;-:file|$TargetDirectory\**\_.pdb`"" - } - $tool.Args += $BinskimAdditionalRunConfigParams - } - } - - # Create variable pointing to the args array directly so we can use splat syntax later. - $toolArgs = $tool.Args - - # Configure the tool. If args array is provided or the current tool has some default arguments - # defined, add "--args" and splat each element on the end. Arg format is "{Arg id} < {Value}", - # one per parameter. Doc page for "guardian configure": - # https://dev.azure.com/securitytools/SecurityIntegration/_wiki/wikis/Guardian/1395/configure - Exec-BlockVerbosely { - & $GuardianCliLocation configure ` - --working-directory $WorkingDirectory ` - --tool $tool.Name ` - --output-path $gdnConfigFile ` - --logger-level $GuardianLoggerLevel ` - --noninteractive ` - --force ` - $(if ($toolArgs) { "--args" }) @toolArgs - Exit-IfNZEC "Sdl" - } - - Write-Host "Created '$toolConfigName' configuration file: $gdnConfigFile" - } -} -catch { - Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ - ExitWithExitCode 1 -} diff --git a/src/arcade/eng/common/sdl/extract-artifact-archives.ps1 b/src/arcade/eng/common/sdl/extract-artifact-archives.ps1 deleted file mode 100644 index 68da4fbf257..00000000000 --- a/src/arcade/eng/common/sdl/extract-artifact-archives.ps1 +++ /dev/null @@ -1,63 +0,0 @@ -# This script looks for each archive file in a directory and extracts it into the target directory. -# For example, the file "$InputPath/bin.tar.gz" extracts to "$ExtractPath/bin.tar.gz.extracted/**". -# Uses the "tar" utility added to Windows 10 / Windows 2019 that supports tar.gz and zip. -param( - # Full path to directory where archives are stored. - [Parameter(Mandatory=$true)][string] $InputPath, - # Full path to directory to extract archives into. May be the same as $InputPath. - [Parameter(Mandatory=$true)][string] $ExtractPath -) - -$ErrorActionPreference = 'Stop' -Set-StrictMode -Version 2.0 - -$disableConfigureToolsetImport = $true - -try { - # `tools.ps1` checks $ci to perform some actions. Since the SDL - # scripts don't necessarily execute in the same agent that run the - # build.ps1/sh script this variable isn't automatically set. - $ci = $true - . $PSScriptRoot\..\tools.ps1 - - Measure-Command { - $jobs = @() - - # Find archive files for non-Windows and Windows builds. - $archiveFiles = @( - Get-ChildItem (Join-Path $InputPath "*.tar.gz") - Get-ChildItem (Join-Path $InputPath "*.zip") - ) - - foreach ($targzFile in $archiveFiles) { - $jobs += Start-Job -ScriptBlock { - $file = $using:targzFile - $fileName = [System.IO.Path]::GetFileName($file) - $extractDir = Join-Path $using:ExtractPath "$fileName.extracted" - - New-Item $extractDir -ItemType Directory -Force | Out-Null - - Write-Host "Extracting '$file' to '$extractDir'..." - - # Pipe errors to stdout to prevent PowerShell detecting them and quitting the job early. - # This type of quit skips the catch, so we wouldn't be able to tell which file triggered the - # error. Save output so it can be stored in the exception string along with context. - $output = tar -xf $file -C $extractDir 2>&1 - # Handle NZEC manually rather than using Exit-IfNZEC: we are in a background job, so we - # don't have access to the outer scope. - if ($LASTEXITCODE -ne 0) { - throw "Error extracting '$file': non-zero exit code ($LASTEXITCODE). Output: '$output'" - } - - Write-Host "Extracted to $extractDir" - } - } - - Receive-Job $jobs -Wait - } -} -catch { - Write-Host $_ - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ - ExitWithExitCode 1 -} diff --git a/src/arcade/eng/common/sdl/extract-artifact-packages.ps1 b/src/arcade/eng/common/sdl/extract-artifact-packages.ps1 deleted file mode 100644 index f031ed5b25e..00000000000 --- a/src/arcade/eng/common/sdl/extract-artifact-packages.ps1 +++ /dev/null @@ -1,82 +0,0 @@ -param( - [Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where artifact packages are stored - [Parameter(Mandatory=$true)][string] $ExtractPath # Full path to directory where the packages will be extracted -) - -$ErrorActionPreference = 'Stop' -Set-StrictMode -Version 2.0 - -$disableConfigureToolsetImport = $true - -function ExtractArtifacts { - if (!(Test-Path $InputPath)) { - Write-Host "Input Path does not exist: $InputPath" - ExitWithExitCode 0 - } - $Jobs = @() - Get-ChildItem "$InputPath\*.nupkg" | - ForEach-Object { - $Jobs += Start-Job -ScriptBlock $ExtractPackage -ArgumentList $_.FullName - } - - foreach ($Job in $Jobs) { - Wait-Job -Id $Job.Id | Receive-Job - } -} - -try { - # `tools.ps1` checks $ci to perform some actions. Since the SDL - # scripts don't necessarily execute in the same agent that run the - # build.ps1/sh script this variable isn't automatically set. - $ci = $true - . $PSScriptRoot\..\tools.ps1 - - $ExtractPackage = { - param( - [string] $PackagePath # Full path to a NuGet package - ) - - if (!(Test-Path $PackagePath)) { - Write-PipelineTelemetryError -Category 'Build' -Message "Input file does not exist: $PackagePath" - ExitWithExitCode 1 - } - - $RelevantExtensions = @('.dll', '.exe', '.pdb') - Write-Host -NoNewLine 'Extracting ' ([System.IO.Path]::GetFileName($PackagePath)) '...' - - $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath) - $ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageId - - Add-Type -AssemblyName System.IO.Compression.FileSystem - - [System.IO.Directory]::CreateDirectory($ExtractPath); - - try { - $zip = [System.IO.Compression.ZipFile]::OpenRead($PackagePath) - - $zip.Entries | - Where-Object {$RelevantExtensions -contains [System.IO.Path]::GetExtension($_.Name)} | - ForEach-Object { - $TargetPath = Join-Path -Path $ExtractPath -ChildPath (Split-Path -Path $_.FullName) - [System.IO.Directory]::CreateDirectory($TargetPath); - - $TargetFile = Join-Path -Path $ExtractPath -ChildPath $_.FullName - [System.IO.Compression.ZipFileExtensions]::ExtractToFile($_, $TargetFile) - } - } - catch { - Write-Host $_ - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ - ExitWithExitCode 1 - } - finally { - $zip.Dispose() - } - } - Measure-Command { ExtractArtifacts } -} -catch { - Write-Host $_ - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ - ExitWithExitCode 1 -} diff --git a/src/arcade/eng/common/sdl/packages.config b/src/arcade/eng/common/sdl/packages.config deleted file mode 100644 index e5f543ea68c..00000000000 --- a/src/arcade/eng/common/sdl/packages.config +++ /dev/null @@ -1,4 +0,0 @@ - - - - diff --git a/src/arcade/eng/common/sdl/run-sdl.ps1 b/src/arcade/eng/common/sdl/run-sdl.ps1 deleted file mode 100644 index 2eac8c78f10..00000000000 --- a/src/arcade/eng/common/sdl/run-sdl.ps1 +++ /dev/null @@ -1,49 +0,0 @@ -Param( - [string] $GuardianCliLocation, - [string] $WorkingDirectory, - [string] $GdnFolder, - [string] $UpdateBaseline, - [string] $GuardianLoggerLevel='Standard' -) - -$ErrorActionPreference = 'Stop' -Set-StrictMode -Version 2.0 -$disableConfigureToolsetImport = $true -$global:LASTEXITCODE = 0 - -try { - # `tools.ps1` checks $ci to perform some actions. Since the SDL - # scripts don't necessarily execute in the same agent that run the - # build.ps1/sh script this variable isn't automatically set. - $ci = $true - . $PSScriptRoot\..\tools.ps1 - - # We store config files in the r directory of .gdn - $gdnConfigPath = Join-Path $GdnFolder 'r' - $ValidPath = Test-Path $GuardianCliLocation - - if ($ValidPath -eq $False) - { - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Invalid Guardian CLI Location." - ExitWithExitCode 1 - } - - $gdnConfigFiles = Get-ChildItem $gdnConfigPath -Recurse -Include '*.gdnconfig' - Write-Host "Discovered Guardian config files:" - $gdnConfigFiles | Out-String | Write-Host - - Exec-BlockVerbosely { - & $GuardianCliLocation run ` - --working-directory $WorkingDirectory ` - --baseline mainbaseline ` - --update-baseline $UpdateBaseline ` - --logger-level $GuardianLoggerLevel ` - --config @gdnConfigFiles - Exit-IfNZEC "Sdl" - } -} -catch { - Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ - ExitWithExitCode 1 -} diff --git a/src/arcade/eng/common/templates-official/variables/pool-providers.yml b/src/arcade/eng/common/templates-official/variables/pool-providers.yml deleted file mode 100644 index 1f308b24efc..00000000000 --- a/src/arcade/eng/common/templates-official/variables/pool-providers.yml +++ /dev/null @@ -1,45 +0,0 @@ -# Select a pool provider based off branch name. Anything with branch name containing 'release' must go into an -Svc pool, -# otherwise it should go into the "normal" pools. This separates out the queueing and billing of released branches. - -# Motivation: -# Once a given branch of a repository's output has been officially "shipped" once, it is then considered to be COGS -# (Cost of goods sold) and should be moved to a servicing pool provider. This allows both separation of queueing -# (allowing release builds and main PR builds to not intefere with each other) and billing (required for COGS. -# Additionally, the pool provider name itself may be subject to change when the .NET Core Engineering Services -# team needs to move resources around and create new and potentially differently-named pools. Using this template -# file from an Arcade-ified repo helps guard against both having to update one's release/* branches and renaming. - -# How to use: -# This yaml assumes your shipped product branches use the naming convention "release/..." (which many do). -# If we find alternate naming conventions in broad usage it can be added to the condition below. -# -# First, import the template in an arcade-ified repo to pick up the variables, e.g.: -# -# variables: -# - template: /eng/common/templates-official/variables/pool-providers.yml -# -# ... then anywhere specifying the pool provider use the runtime variables, -# $(DncEngInternalBuildPool) -# -# pool: -# name: $(DncEngInternalBuildPool) -# image: 1es-windows-2022 - -variables: - # Coalesce the target and source branches so we know when a PR targets a release branch - # If these variables are somehow missing, fall back to main (tends to have more capacity) - - # Any new -Svc alternative pools should have variables added here to allow for splitting work - - - name: DncEngInternalBuildPool - value: $[ - replace( - replace( - eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), - True, - 'NetCore1ESPool-Svc-Internal' - ), - False, - 'NetCore1ESPool-Internal' - ) - ] \ No newline at end of file diff --git a/src/arcade/eng/common/templates-official/variables/sdl-variables.yml b/src/arcade/eng/common/templates-official/variables/sdl-variables.yml deleted file mode 100644 index dbdd66d4a4b..00000000000 --- a/src/arcade/eng/common/templates-official/variables/sdl-variables.yml +++ /dev/null @@ -1,7 +0,0 @@ -variables: -# The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in -# sync with the packages.config file. -- name: DefaultGuardianVersion - value: 0.109.0 -- name: GuardianPackagesConfigFile - value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config \ No newline at end of file