Merge pull request #8028 from dotnet/feature/CacheTaskUsage

Update cache pipeline to use AzureFileCopy task

Author: MiYanni

eng/SearchCache/InstallAzureCLI.sh

@@ -18,6 +18,10 @@ parameters:
   type: boolean
   default: true
 
+variables:
+# Variables used: DncEngInternalBuildPool
+- template: /eng/common/templates-official/variables/pool-providers.yml
+
 resources:
   repositories:
   - repository: 1ESPipelineTemplates
@@ -30,81 +34,78 @@ extends:
   parameters:
     sdl:
       sourceAnalysisPool:
-        name: NetCore1ESPool-Internal
+        name: $(DncEngInternalBuildPool)
         image: 1es-windows-2022
         os: windows
-    pool:
-      name: NetCore1ESPool-Internal
-      image: 1es-ubuntu-2204
-      os: linux
     stages:
-    - stage: stage
+    - stage: Build
       jobs:
-      - job: jobs
+      - job: Create
+        pool:
+          name: $(DncEngInternalBuildPool)
+          image: 1es-ubuntu-2204
+          os: linux
         templateContext:
           outputs:
           - output: pipelineArtifact
-            displayName: 'Publish Test Log and Results'
-            condition: always()
-            targetPath: '$(Build.ArtifactStagingDirectory)'
+            targetPath: $(Build.ArtifactStagingDirectory)
             artifactName: Test_LogResults
-            publishLocation: Container
           - output: pipelineArtifact
-            displayName: 'Publish Artifacts'
             targetPath: $(System.DefaultWorkingDirectory)/ArtifactsToPublish/
-            artifactName: outputs
+            artifactName: ArtifactsToPublish
 
         steps:
-        - checkout: self
-        - script: '$(Build.SourcesDirectory)/build.sh'
+        - script: $(Build.SourcesDirectory)/build.sh
+          displayName: Build
+
         - task: CopyFiles@2
           displayName: Gather Test Log and Results
           inputs:
-            SourceFolder: '$(Build.SourcesDirectory)/artifacts'
+            SourceFolder: $(Build.SourcesDirectory)/artifacts
             Contents: |
               log/**/*
               TestResults/**/*
-            TargetFolder: '$(Build.ArtifactStagingDirectory)'
+            TargetFolder: $(Build.ArtifactStagingDirectory)
           continueOnError: true
           condition: always()
 
         - task: UseDotNet@2
-          displayName: 'Use .NET 6.0.100'
+          displayName: Use .NET 6.0.100
           inputs:
             packageType: sdk
             version: 6.0.100
             installationPath: $(Build.SourcesDirectory)/.dotnet
 
         - task: UseDotNet@2
-          displayName: 'Use .NET 6.0.300'
+          displayName: Use .NET 6.0.300
           inputs:
             packageType: sdk
             version: 6.0.300
             installationPath: $(Build.SourcesDirectory)/.dotnet
 
         - task: UseDotNet@2
-          displayName: 'Use .NET 6.0.400'
+          displayName: Use .NET 6.0.400
           inputs:
             packageType: sdk
             version: 6.0.400
             installationPath: $(Build.SourcesDirectory)/.dotnet
 
         - task: UseDotNet@2
-          displayName: 'Use .NET 7.0.100'
+          displayName: Use .NET 7.0.100
           inputs:
             packageType: sdk
             version: 7.0.100
             installationPath: $(Build.SourcesDirectory)/.dotnet
 
         - task: UseDotNet@2
-          displayName: 'Use .NET 7.0.200'
+          displayName: Use .NET 7.0.200
           inputs:
             packageType: sdk
             version: 7.0.200
             installationPath: $(Build.SourcesDirectory)/.dotnet
 
         - task: UseDotNet@2
-          displayName: 'Use .NET 8.0.200'
+          displayName: Use .NET 8.0.200
           inputs:
             packageType: sdk
             version: 8.0.200
@@ -131,51 +132,37 @@ extends:
           displayName: Run Cache Updater
 
         - task: CopyFiles@2
+          displayName: Copy artifacts to publish
           inputs:
             SourceFolder: $(System.DefaultWorkingDirectory)/NugetDownloadDirectory/SearchCache/
             Contents: |
               NuGetTemplateSearchInfo*.json
               nonTemplatePacks.json
             TargetFolder: $(System.DefaultWorkingDirectory)/ArtifactsToPublish/
-        - ${{ if eq(parameters.publishToBlob, true) }}:
-          - script: '$(Build.SourcesDirectory)/eng/SearchCache/InstallAzureCLI.sh'
-            displayName: Install Azure CLI
-
-          - bash: az config set extension.use_dynamic_install=yes_without_prompt
-            displayName: Disable Azure CLI prompts
-
-          - bash: >
-              az storage azcopy blob upload
-              -c $(CacheFileStorageContainer)
-              --account-name $(CacheFileStorageAccount)
-              -s '$(System.DefaultWorkingDirectory)/ArtifactsToPublish/NuGetTemplateSearchInfoVer2.json'
-              --sas-token "$(CacheFileStorageSasToken)"
-              -d NuGetTemplateSearchInfoVer2.json
-              --verbose
-              | tee upload.log
-              && grep ".*Number of Transfers Completed: 1" upload.log || (echo ; echo "Cache file upload failed"; false)
-            displayName: Upload to blob storage
 
-          - bash: >
-              az storage azcopy blob upload
-              -c $(LegacyCacheFileStorageContainer)
-              --account-name $(LegacyCacheFileStorageAccount)
-              -s '$(System.DefaultWorkingDirectory)/ArtifactsToPublish/NuGetTemplateSearchInfo.json'
-              --sas-token "$(LegacyCacheFileStorageSasToken)"
-              -d NuGetTemplateSearchInfo.json
-              --verbose
-              | tee upload-legacy.log
-              && grep ".*Number of Transfers Completed: 1" upload-legacy.log || (echo ; echo "Legacy cache file upload failed"; false)
-            displayName: Upload legacy file to blob storage
-
-          - bash: >
-              az storage azcopy blob upload
-              -c $(NonTemplatePacksFileStorageContainer)
-              --account-name $(NonTemplatePacksFileStorageAccount)
-              -s '$(System.DefaultWorkingDirectory)/ArtifactsToPublish/nonTemplatePacks.json'
-              --sas-token "$(NonTemplatePacksFileStorageSasToken)"
-              -d nonTemplatePacks.json
-              --verbose
-              | tee upload-non-template-packs.log
-              && grep ".*Number of Transfers Completed: 1" upload-non-template-packs.log || (echo ; echo "Legacy cache file upload failed"; false)
-            displayName: Upload non template packages file to blob storage
+      - ${{ if eq(parameters.publishToBlob, true) }}:
+        # An entirely separate job is required to run AzureFileCopy@6, which is a Windows-only task.
+        # If the Create job was rewritten to use PowerShell instead of Bash, Create and Publish could be combined into a single Windows job.
+        - job: Publish
+          dependsOn: Create
+          pool:
+            name: $(DncEngInternalBuildPool)
+            image: 1es-windows-2022
+            os: windows
+          templateContext:
+            inputs:
+            - input: pipelineArtifact
+              targetPath: $(System.DefaultWorkingDirectory)/ArtifactsToPublish/
+              artifactName: ArtifactsToPublish
+
+          steps:
+          - task: AzureFileCopy@6
+            displayName: Upload to blob storage
+            inputs:
+              SourcePath: $(System.DefaultWorkingDirectory)/ArtifactsToPublish/*.json
+              # Service connection: https://dnceng.visualstudio.com/internal/_settings/adminservices?resourceId=010b0bdc-9487-484f-af2d-ca3ae3235b84
+              azureSubscription: DOTNET-Templating - PME
+              Destination: AzureBlob
+              # These variables are defined in the pipeline's Variables UI.
+              storage: $(CacheFileStorageAccount)
+              ContainerName: $(CacheFileStorageContainer)