[release/dev17.8] Update dependencies from dotnet/arcade

eng/Version.Details.xml

@@ -98,9 +98,9 @@
       <Uri>https://dev.azure.com/dnceng/internal/_git/dotnet-runtime</Uri>
       <Sha>3a25a7f1cc446b60678ed25c9d829420d6321eba</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="8.0.0-beta.24570.5">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="8.0.0-beta.25562.3">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>3c7e11bf80279cde53a6251c4d0fa10e613fc739</Sha>
+      <Sha>e8483fe03c7d3257c68f6013441da5d72eeb8392</Sha>
       <SourceBuild RepoName="arcade" ManagedOnly="true" />
     </Dependency>
     <Dependency Name="Microsoft.DotNet.XliffTasks" Version="1.0.0-beta.23475.1" CoherentParentDependency="Microsoft.DotNet.Arcade.Sdk">

eng/common/SetupNugetSources.ps1

@@ -17,8 +17,8 @@
 #    displayName: Setup Private Feeds Credentials
 #    condition: eq(variables['Agent.OS'], 'Windows_NT')
 #    inputs:
-#      filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1
-#      arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token
+#      filePath: $(System.DefaultWorkingDirectory)/eng/common/SetupNugetSources.ps1
+#      arguments: -ConfigFile $(System.DefaultWorkingDirectory)/NuGet.config -Password $Env:Token
 #    env:
 #      Token: $(dn-bot-dnceng-artifact-feeds-rw)
 

eng/common/SetupNugetSources.sh

@@ -18,8 +18,8 @@
 #  - task: Bash@3
 #    displayName: Setup Private Feeds Credentials
 #    inputs:
-#      filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh
-#      arguments: $(Build.SourcesDirectory)/NuGet.config $Token
+#      filePath: $(System.DefaultWorkingDirectory)/eng/common/SetupNugetSources.sh
+#      arguments: $(System.DefaultWorkingDirectory)/NuGet.config $Token
 #    condition: ne(variables['Agent.OS'], 'Windows_NT')
 #    env:
 #      Token: $(dn-bot-dnceng-artifact-feeds-rw)

eng/common/cross/toolchain.cmake

@@ -40,7 +40,7 @@ if(TARGET_ARCH_NAME STREQUAL "arm")
     set(TOOLCHAIN "arm-linux-gnueabihf")
   endif()
   if(TIZEN)
-    set(TIZEN_TOOLCHAIN "armv7hl-tizen-linux-gnueabihf/9.2.0")
+    set(TIZEN_TOOLCHAIN "armv7hl-tizen-linux-gnueabihf")
   endif()
 elseif(TARGET_ARCH_NAME STREQUAL "arm64")
   set(CMAKE_SYSTEM_PROCESSOR aarch64)
@@ -49,7 +49,7 @@ elseif(TARGET_ARCH_NAME STREQUAL "arm64")
   elseif(LINUX)
     set(TOOLCHAIN "aarch64-linux-gnu")
     if(TIZEN)
-      set(TIZEN_TOOLCHAIN "aarch64-tizen-linux-gnu/9.2.0")
+      set(TIZEN_TOOLCHAIN "aarch64-tizen-linux-gnu")
     endif()
   elseif(FREEBSD)
     set(triple "aarch64-unknown-freebsd12")
@@ -58,7 +58,7 @@ elseif(TARGET_ARCH_NAME STREQUAL "armel")
   set(CMAKE_SYSTEM_PROCESSOR armv7l)
   set(TOOLCHAIN "arm-linux-gnueabi")
   if(TIZEN)
-    set(TIZEN_TOOLCHAIN "armv7l-tizen-linux-gnueabi/9.2.0")
+    set(TIZEN_TOOLCHAIN "armv7l-tizen-linux-gnueabi")
   endif()
 elseif(TARGET_ARCH_NAME STREQUAL "armv6")
   set(CMAKE_SYSTEM_PROCESSOR armv6l)
@@ -95,7 +95,7 @@ elseif(TARGET_ARCH_NAME STREQUAL "x64")
   elseif(LINUX)
     set(TOOLCHAIN "x86_64-linux-gnu")
     if(TIZEN)
-      set(TIZEN_TOOLCHAIN "x86_64-tizen-linux-gnu/9.2.0")
+      set(TIZEN_TOOLCHAIN "x86_64-tizen-linux-gnu")
     endif()
   elseif(FREEBSD)
     set(triple "x86_64-unknown-freebsd12")
@@ -112,7 +112,7 @@ elseif(TARGET_ARCH_NAME STREQUAL "x86")
     set(TOOLCHAIN "i686-linux-gnu")
   endif()
   if(TIZEN)
-    set(TIZEN_TOOLCHAIN "i586-tizen-linux-gnu/9.2.0")
+    set(TIZEN_TOOLCHAIN "i586-tizen-linux-gnu")
   endif()
 else()
   message(FATAL_ERROR "Arch is ${TARGET_ARCH_NAME}. Only arm, arm64, armel, armv6, ppc64le, riscv64, s390x, x64 and x86 are supported!")
@@ -124,26 +124,25 @@ endif()
 
 # Specify include paths
 if(TIZEN)
-  if(TARGET_ARCH_NAME STREQUAL "arm")
-    include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/)
-    include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/armv7hl-tizen-linux-gnueabihf)
-  endif()
-  if(TARGET_ARCH_NAME STREQUAL "armel")
-    include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/)
-    include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/armv7l-tizen-linux-gnueabi)
-  endif()
-  if(TARGET_ARCH_NAME STREQUAL "arm64")
-    include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}/include/c++/)
-    include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}/include/c++/aarch64-tizen-linux-gnu)
-  endif()
-  if(TARGET_ARCH_NAME STREQUAL "x86")
-    include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/)
-    include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/i586-tizen-linux-gnu)
-  endif()
-  if(TARGET_ARCH_NAME STREQUAL "x64")
-    include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}/include/c++/)
-    include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}/include/c++/x86_64-tizen-linux-gnu)
+  function(find_toolchain_dir prefix)
+    # Dynamically find the version subdirectory
+    file(GLOB DIRECTORIES "${prefix}/*")
+    list(GET DIRECTORIES 0 FIRST_MATCH)
+    get_filename_component(TOOLCHAIN_VERSION ${FIRST_MATCH} NAME)
+
+    set(TIZEN_TOOLCHAIN_PATH "${prefix}/${TOOLCHAIN_VERSION}" PARENT_SCOPE)
+  endfunction()
+
+  if(TARGET_ARCH_NAME MATCHES "^(arm|armel|x86)$")
+    find_toolchain_dir("${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}")
+  else()
+    find_toolchain_dir("${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}")
   endif()
+
+  message(STATUS "TIZEN_TOOLCHAIN_PATH set to: ${TIZEN_TOOLCHAIN_PATH}")
+
+  include_directories(SYSTEM ${TIZEN_TOOLCHAIN_PATH}/include/c++)
+  include_directories(SYSTEM ${TIZEN_TOOLCHAIN_PATH}/include/c++/${TIZEN_TOOLCHAIN})
 endif()
 
 if(ANDROID)
@@ -265,33 +264,35 @@ endif()
 
 if(TARGET_ARCH_NAME MATCHES "^(arm|armel)$")
   if(TIZEN)
-    add_toolchain_linker_flag("-B${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}")
+    add_toolchain_linker_flag("-B${TIZEN_TOOLCHAIN_PATH}")
     add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib")
     add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib")
-    add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}")
+    add_toolchain_linker_flag("-L${TIZEN_TOOLCHAIN_PATH}")
   endif()
 elseif(TARGET_ARCH_NAME MATCHES "^(arm64|x64)$")
   if(TIZEN)
-    add_toolchain_linker_flag("-B${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}")
+    add_toolchain_linker_flag("-B${TIZEN_TOOLCHAIN_PATH}")
     add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib64")
     add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib64")
-    add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}")
+    add_toolchain_linker_flag("-L${TIZEN_TOOLCHAIN_PATH}")
 
     add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/lib64")
     add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/usr/lib64")
-    add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}")
+    add_toolchain_linker_flag("-Wl,--rpath-link=${TIZEN_TOOLCHAIN_PATH}")
   endif()
+elseif(TARGET_ARCH_NAME STREQUAL "s390x")
+  add_toolchain_linker_flag("--target=${TOOLCHAIN}")
 elseif(TARGET_ARCH_NAME STREQUAL "x86")
   if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/i586-alpine-linux-musl)
     add_toolchain_linker_flag("--target=${TOOLCHAIN}")
     add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/usr/lib/gcc/${TOOLCHAIN}")
   endif()
   add_toolchain_linker_flag(-m32)
   if(TIZEN)
-    add_toolchain_linker_flag("-B${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}")
+    add_toolchain_linker_flag("-B${TIZEN_TOOLCHAIN_PATH}")
     add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib")
     add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib")
-    add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}")
+    add_toolchain_linker_flag("-L${TIZEN_TOOLCHAIN_PATH}")
   endif()
 elseif(ILLUMOS)
   add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib/amd64")
@@ -328,6 +329,8 @@ if(TARGET_ARCH_NAME MATCHES "^(arm|armel)$")
   if(TARGET_ARCH_NAME STREQUAL "armel")
     add_compile_options(-mfloat-abi=softfp)
   endif()
+elseif(TARGET_ARCH_NAME STREQUAL "s390x")
+  add_compile_options("--target=${TOOLCHAIN}")
 elseif(TARGET_ARCH_NAME STREQUAL "x86")
   if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/i586-alpine-linux-musl)
     add_compile_options(--target=${TOOLCHAIN})

eng/common/cross/x64/sources.list.bionic

@@ -0,0 +1,11 @@
+deb http://archive.ubuntu.com/ubuntu/ bionic main restricted universe
+deb-src http://archive.ubuntu.com/ubuntu/ bionic main restricted universe
+
+deb http://archive.ubuntu.com/ubuntu/ bionic-updates main restricted universe
+deb-src http://archive.ubuntu.com/ubuntu/ bionic-updates main restricted universe
+
+deb http://archive.ubuntu.com/ubuntu/ bionic-backports main restricted
+deb-src http://archive.ubuntu.com/ubuntu/ bionic-backports main restricted
+
+deb http://archive.ubuntu.com/ubuntu/ bionic-security main restricted universe multiverse
+deb-src http://archive.ubuntu.com/ubuntu/ bionic-security main restricted universe multiverse

eng/common/cross/x64/sources.list.xenial

@@ -0,0 +1,11 @@
+deb http://archive.ubuntu.com/ubuntu/ xenial main restricted universe
+deb-src http://archive.ubuntu.com/ubuntu/ xenial main restricted universe
+
+deb http://archive.ubuntu.com/ubuntu/ xenial-updates main restricted universe
+deb-src http://archive.ubuntu.com/ubuntu/ xenial-updates main restricted universe
+
+deb http://archive.ubuntu.com/ubuntu/ xenial-backports main restricted
+deb-src http://archive.ubuntu.com/ubuntu/ xenial-backports main restricted
+
+deb http://archive.ubuntu.com/ubuntu/ xenial-security main restricted universe multiverse
+deb-src http://archive.ubuntu.com/ubuntu/ xenial-security main restricted universe multiverse

eng/common/cross/x64/tizen/tizen.patch

@@ -0,0 +1,9 @@
+diff -u -r a/usr/lib64/libc.so b/usr/lib64/libc.so
+--- a/usr/lib64/libc.so	2016-12-30 23:00:08.284951863 +0900
++++ b/usr/lib64/libc.so	2016-12-30 23:00:32.140951815 +0900
+@@ -2,4 +2,4 @@
+    Use the shared library, but some functions are only in
+    the static library, so try that secondarily.  */
+ OUTPUT_FORMAT(elf64-x86-64)
+-GROUP ( /lib64/libc.so.6 /usr/lib64/libc_nonshared.a  AS_NEEDED ( /lib64/ld-linux-x86-64.so.2 ) )
++GROUP ( libc.so.6 libc_nonshared.a  AS_NEEDED ( ld-linux-x86-64.so.2 ) )

eng/common/generate-sbom-prep.ps1

@@ -4,18 +4,26 @@ Param(
 
 . $PSScriptRoot\pipeline-logging-functions.ps1
 
+# Normally - we'd listen to the manifest path given, but 1ES templates will overwrite if this level gets uploaded directly
+# with their own overwriting ours. So we create it as a sub directory of the requested manifest path.
+$ArtifactName = "${env:SYSTEM_STAGENAME}_${env:AGENT_JOBNAME}_SBOM"
+$SafeArtifactName = $ArtifactName -replace '["/:<>\\|?@*"() ]', '_'
+$SbomGenerationDir = Join-Path $ManifestDirPath $SafeArtifactName
+
+Write-Host "Artifact name before : $ArtifactName"
+Write-Host "Artifact name after : $SafeArtifactName"
+
 Write-Host "Creating dir $ManifestDirPath"
+
 # create directory for sbom manifest to be placed
-if (!(Test-Path -path $ManifestDirPath))
+if (!(Test-Path -path $SbomGenerationDir))
 {
-  New-Item -ItemType Directory -path $ManifestDirPath
-  Write-Host "Successfully created directory $ManifestDirPath"
+  New-Item -ItemType Directory -path $SbomGenerationDir
+  Write-Host "Successfully created directory $SbomGenerationDir"
 }
 else{
   Write-PipelineTelemetryError -category 'Build'  "Unable to create sbom folder."
 }
 
 Write-Host "Updating artifact name"
-$artifact_name = "${env:SYSTEM_STAGENAME}_${env:AGENT_JOBNAME}_SBOM" -replace '["/:<>\\|?@*"() ]', '_'
-Write-Host "Artifact name $artifact_name"
-Write-Host "##vso[task.setvariable variable=ARTIFACT_NAME]$artifact_name"
+Write-Host "##vso[task.setvariable variable=ARTIFACT_NAME]$SafeArtifactName"

eng/common/generate-sbom-prep.sh

@@ -14,19 +14,24 @@ done
 scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
 . $scriptroot/pipeline-logging-functions.sh
 
+# replace all special characters with _, some builds use special characters like : in Agent.Jobname, that is not a permissible name while uploading artifacts.
+artifact_name=$SYSTEM_STAGENAME"_"$AGENT_JOBNAME"_SBOM"
+safe_artifact_name="${artifact_name//["/:<>\\|?@*$" ]/_}"
+
 manifest_dir=$1
 
-if [ ! -d "$manifest_dir" ] ; then
-  mkdir -p "$manifest_dir"
-  echo "Sbom directory created." $manifest_dir
+# Normally - we'd listen to the manifest path given, but 1ES templates will overwrite if this level gets uploaded directly
+# with their own overwriting ours. So we create it as a sub directory of the requested manifest path.
+sbom_generation_dir="$manifest_dir/$safe_artifact_name"
+
+if [ ! -d "$sbom_generation_dir" ] ; then
+  mkdir -p "$sbom_generation_dir"
+  echo "Sbom directory created." $sbom_generation_dir
 else
   Write-PipelineTelemetryError -category 'Build'  "Unable to create sbom folder."
 fi
 
-artifact_name=$SYSTEM_STAGENAME"_"$AGENT_JOBNAME"_SBOM"
 echo "Artifact name before : "$artifact_name
-# replace all special characters with _, some builds use special characters like : in Agent.Jobname, that is not a permissible name while uploading artifacts.
-safe_artifact_name="${artifact_name//["/:<>\\|?@*$" ]/_}"
 echo "Artifact name after : "$safe_artifact_name
 export ARTIFACT_NAME=$safe_artifact_name
 echo "##vso[task.setvariable variable=ARTIFACT_NAME]$safe_artifact_name"

eng/common/post-build/nuget-validation.ps1

@@ -2,20 +2,13 @@
 # tool: https://github.com/NuGet/NuGetGallery/tree/jver-verify/src/VerifyMicrosoftPackage
 
 param(
-  [Parameter(Mandatory=$true)][string] $PackagesPath,           # Path to where the packages to be validated are
-  [Parameter(Mandatory=$true)][string] $ToolDestinationPath     # Where the validation tool should be downloaded to
+  [Parameter(Mandatory=$true)][string] $PackagesPath # Path to where the packages to be validated are
 )
 
 try {
   . $PSScriptRoot\post-build-utils.ps1
 
-  $url = 'https://raw.githubusercontent.com/NuGet/NuGetGallery/3e25ad135146676bcab0050a516939d9958bfa5d/src/VerifyMicrosoftPackage/verify.ps1'
-
-  New-Item -ItemType 'directory' -Path ${ToolDestinationPath} -Force
-
-  Invoke-WebRequest $url -OutFile ${ToolDestinationPath}\verify.ps1 
-
-  & ${ToolDestinationPath}\verify.ps1 ${PackagesPath}\*.nupkg
+  & $PSScriptRoot\nuget-verification.ps1 ${PackagesPath}\*.nupkg
 } 
 catch {
   Write-Host $_.ScriptStackTrace