Fixed pattern

build/nightly-E2E-test-pipelines.yml

@@ -100,33 +100,35 @@ stages:
       ASPNETCORE_URLS: https://localhost:44331
     jobs:
       # E2E Tests
-      - job:
+      - job: e2ETestsSQLite
         displayName: E2E Tests (SQLite)
         timeoutInMinutes: 180
         variables:
           # Connection string
           CONNECTIONSTRINGS__UMBRACODBDSN: Data Source=Umbraco;Mode=Memory;Cache=Shared;Foreign Keys=True;Pooling=True
           CONNECTIONSTRINGS__UMBRACODBDSN_PROVIDERNAME: Microsoft.Data.Sqlite
+          # The amount of sharding for our playwright, if the shard index is increased to 4 this should be updated to 4 as well
+          shardTotal: 3
         strategy:
           matrix:
             LinuxPart1Of3:
               vmImage: "ubuntu-latest"
-              testCommand: "npm run test -- --shard=1/3"
+              shardIndex: 1
             LinuxPart2Of3:
               vmImage: "ubuntu-latest"
-              testCommand: "npm run test -- --shard=2/3"
+              shardIndex: 2
             LinuxPart3Of3:
               vmImage: "ubuntu-latest"
-              testCommand: "npm run test -- --shard=3/3"
+              shardIndex: 3
             WindowsPart1Of3:
               vmImage: "windows-latest"
-              testCommand: "npm run test -- --shard=1/3"
+              shardIndex: 1
             WindowsPart2Of3:
               vmImage: "windows-latest"
-              testCommand: "npm run test -- --shard=2/3"
+              shardIndex: 2
             WindowsPart3Of3:
               vmImage: "windows-latest"
-              testCommand: "npm run test -- --shard=3/3"
+              shardIndex: 3
         pool:
           vmImage: $(vmImage)
         steps:
@@ -152,7 +154,8 @@ stages:
               "UMBRACO_USER_LOGIN=$(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSEREMAIL)
               UMBRACO_USER_PASSWORD=$(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSERPASSWORD)
               URL=$(ASPNETCORE_URLS)
-              STORAGE_STAGE_PATH=$(Build.SourcesDirectory)/tests/Umbraco.Tests.AcceptanceTest/playwright/.auth/user.json" | Out-File .env
+              STORAGE_STAGE_PATH=$(Build.SourcesDirectory)/tests/Umbraco.Tests.AcceptanceTest/playwright/.auth/user.json
+              PLAYWRIGHT_BLOB_OUTPUT_NAME=report-$(vmImage)-attempt-$(System.JobAttempt)-shard-$(shardIndex).zip" | Out-File .env
             displayName: Generate .env
             workingDirectory: $(Build.SourcesDirectory)/tests/Umbraco.Tests.AcceptanceTest
 
@@ -214,7 +217,8 @@ stages:
             workingDirectory: tests/Umbraco.Tests.AcceptanceTest
 
           # Test
-          - pwsh: $(testCommand)
+          # Change testCommand to run different tests
+          - pwsh: "npm run smokeTestSqlite -- --shard=$(shardIndex)/$(shardTotal) --reporter=line,blob"
             displayName: Run Playwright tests
             continueOnError: true
             workingDirectory: tests/Umbraco.Tests.AcceptanceTest
@@ -234,10 +238,10 @@ stages:
 
           # Copy artifacts
           - pwsh: |
-              if (Test-Path tests/Umbraco.Tests.AcceptanceTest/results/*) {
-                Copy-Item tests/Umbraco.Tests.AcceptanceTest/results/* $(Build.ArtifactStagingDirectory) -Recurse
+              if (Test-Path tests/Umbraco.Tests.AcceptanceTest/blob-report/*) {
+                Copy-Item tests/Umbraco.Tests.AcceptanceTest/blob-report/* $(Build.ArtifactStagingDirectory) -Recurse
               }
-            displayName: Copy Playwright results
+            displayName: Copy Playwright blob report
             condition: succeededOrFailed()
 
           # Publish
@@ -246,175 +250,254 @@ stages:
             condition: succeededOrFailed()
             inputs:
               targetPath: $(Build.ArtifactStagingDirectory)
-              artifact: "Acceptance Test Results - $(Agent.JobName) - Attempt #$(System.JobAttempt)"
+              artifact: blob-report-sql-lite-$(Agent.OS)-shard-$(shardIndex)-attempt-$(System.JobAttempt)
 
-      - job:
-        displayName: E2E Tests (SQL Server)
-        timeoutInMinutes: 180
+      - job: mergePlaywrightReportsSQLite
+        dependsOn: e2ETestsSQLite
+        displayName: Merge Playwright SQLite reports
         variables:
-          # Connection string
-          CONNECTIONSTRINGS__UMBRACODBDSN: Data Source=(localdb)\MSSQLLocalDB;AttachDbFilename=|DataDirectory|\Umbraco.mdf;Integrated Security=True
-          CONNECTIONSTRINGS__UMBRACODBDSN_PROVIDERNAME: Microsoft.Data.SqlClient
-        strategy:
-          matrix:
-            LinuxPart1Of3:
-              testCommand: "npm run testSqlite -- --shard=1/3"
-              vmImage: "ubuntu-latest"
-              SA_PASSWORD: $(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSERPASSWORD)
-              CONNECTIONSTRINGS__UMBRACODBDSN: "Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);TrustServerCertificate=True"
-            LinuxPart2Of3:
-              testCommand: "npm run testSqlite -- --shard=2/3"
-              vmImage: "ubuntu-latest"
-              SA_PASSWORD: $(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSERPASSWORD)
-              CONNECTIONSTRINGS__UMBRACODBDSN: "Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);TrustServerCertificate=True"
-            LinuxPart3Of3:
-              testCommand: "npm run testSqlite -- --shard=3/3"
-              vmImage: "ubuntu-latest"
-              SA_PASSWORD: $(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSERPASSWORD)
-              CONNECTIONSTRINGS__UMBRACODBDSN: "Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);TrustServerCertificate=True"
-            WindowsPart1Of3:
-              vmImage: "windows-latest"
-              testCommand: "npm run testSqlite -- --shard=1/3"
-            WindowsPart2Of3:
-              vmImage: "windows-latest"
-              testCommand: "npm run testSqlite -- --shard=2/3"
-            WindowsPart3Of3:
-              vmImage: "windows-latest"
-              testCommand: "npm run testSqlite -- --shard=3/3"
-        pool:
-          vmImage: $(vmImage)
+          linuxResultsPath: "$(Agent.BuildDirectory)/linuxResults"
+          allLinuxArtifactsPath: "$(Agent.BuildDirectory)/linuxResults/all-artifacts-with-pattern"
+          allLinuxBlobReportsPath: "$(Agent.BuildDirectory)/linuxResults/all-blob-reports-with-pattern"
+          linuxMergedResultsPath: "$(Agent.BuildDirectory)/linuxResults/merged-acceptance-test-results"
+          linuxArtifactPattern: "*/report-ubuntu-latest-attempt-$(System.JobAttempt)*.zip"
+          linuxArtifactName: "merged-acceptance-test-results-linux-$(System.JobAttempt)"
         steps:
-          # Setup test environment
+          # Create necessary Linux directories
+          - pwsh: |
+              mkdir -p "$(allLinuxArtifactsPath)"
+              mkdir -p "$(allLinuxBlobReportsPath)"
+              mkdir -p "$(linuxMergedResultsPath)"
+            displayName: Create necessary Linux directories
+
           - task: DownloadPipelineArtifact@2
-            displayName: Download NuGet artifacts
+            displayName: Download Linux Playwright results
             inputs:
-              artifact: nupkg
-              path: $(Agent.BuildDirectory)/app/nupkg
+              patterns: "$(linuxArtifactPattern)"
+              path: "$(allLinuxArtifactsPath)"
 
-          - task: NodeTool@0
-            displayName: Use Node.js $(nodeVersion)
-            inputs:
-              versionSpec: $(nodeVersion)
-
-          - task: UseDotNet@2
-            displayName: Use .NET SDK from global.json
-            inputs:
-              useGlobalJson: true
+          - pwsh: ls
+            displayName: Check downloaded artifacts
+            workingDirectory: "$(allLinuxArtifactsPath)"
 
           - pwsh: |
-              "UMBRACO_USER_LOGIN=$(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSEREMAIL)
-              UMBRACO_USER_PASSWORD=$(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSERPASSWORD)
-              URL=$(ASPNETCORE_URLS)
-              STORAGE_STAGE_PATH=$(Build.SourcesDirectory)/tests/Umbraco.Tests.AcceptanceTest/playwright/.auth/user.json" | Out-File .env
-            displayName: Generate .env
-            workingDirectory: $(Build.SourcesDirectory)/tests/Umbraco.Tests.AcceptanceTest
-
-          # Cache and restore NPM packages
-          - task: Cache@2
-            displayName: Cache NPM packages
-            inputs:
-              key: 'npm_e2e | "$(Agent.OS)" | $(Build.SourcesDirectory)/tests/Umbraco.Tests.AcceptanceTest/package-lock.json'
-              restoreKeys: |
-                npm_e2e | "$(Agent.OS)"
-                npm_e2e
-              path: $(npm_config_cache)
-
-          - script: npm ci --no-fund --no-audit --prefer-offline
-            workingDirectory: $(Build.SourcesDirectory)/tests/Umbraco.Tests.AcceptanceTest
-            displayName: Restore NPM packages
-
-          # Build application
-          - pwsh: |
-              $cmsVersion = "$(Build.BuildNumber)" -replace "\+",".g"
-              dotnet new nugetconfig
-              dotnet nuget add source ./nupkg --name Local
-              dotnet new install Umbraco.Templates::$cmsVersion
-              dotnet new umbraco --name UmbracoProject --version $cmsVersion --exclude-gitignore --no-restore --no-update-check
-              dotnet restore UmbracoProject
-              cp $(Build.SourcesDirectory)/tests/Umbraco.Tests.AcceptanceTest.UmbracoProject/*.cs UmbracoProject
-              dotnet build UmbracoProject --configuration $(buildConfiguration) --no-restore
-              dotnet dev-certs https
-            displayName: Build application
-            workingDirectory: $(Agent.BuildDirectory)/app
-
-          # Start SQL Server
-          - powershell: docker run --name mssql -d -p 1433:1433 -e "ACCEPT_EULA=Y" -e "MSSQL_SA_PASSWORD=$(SA_PASSWORD)" mcr.microsoft.com/mssql/server:2022-latest
-            displayName: Start SQL Server Docker image (Linux)
-            condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux'))
-
-          - pwsh: SqlLocalDB start MSSQLLocalDB
-            displayName: Start SQL Server LocalDB (Windows)
-            condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))
-
-          # Run application
-          - bash: |
-              nohup dotnet run --project UmbracoProject --configuration $(buildConfiguration) --no-build --no-launch-profile > $(Build.ArtifactStagingDirectory)/playwright.log 2>&1 &
-              echo "##vso[task.setvariable variable=AcceptanceTestProcessId]$!"
-            displayName: Run application (Linux)
-            condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux'))
-            workingDirectory: $(Agent.BuildDirectory)/app
-
-          - pwsh: |
-              $process = Start-Process dotnet "run --project UmbracoProject --configuration $(buildConfiguration) --no-build --no-launch-profile 2>&1" -PassThru -NoNewWindow -RedirectStandardOutput $(Build.ArtifactStagingDirectory)/playwright.log
-              Write-Host "##vso[task.setvariable variable=AcceptanceTestProcessId]$($process.Id)"
-            displayName: Run application (Windows)
-            condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))
-            workingDirectory: $(Agent.BuildDirectory)/app
-
-          # Ensures we have the package wait-on installed
-          - pwsh: npm install wait-on
-            displayName: Install wait-on package
-
-          # Wait for application to start responding to requests
-          - pwsh: npx wait-on -v --interval 1000 --timeout 120000 $(ASPNETCORE_URLS)
-            displayName: Wait for application
-            workingDirectory: tests/Umbraco.Tests.AcceptanceTest
+              Get-ChildItem "$(allLinuxArtifactsPath)" -Recurse -Filter "$(linuxArtifactPattern)" -File |
+              ForEach-Object {
+                Move-Item -Path $_.FullName -Destination "$(allLinuxBlobReportsPath)" -Force
+              }
+            displayName: Move linux blob reports to specific folder
 
           # Install Playwright and dependencies
           - pwsh: npx playwright install --with-deps
             displayName: Install Playwright
-            workingDirectory: tests/Umbraco.Tests.AcceptanceTest
+            workingDirectory: "$(linuxResultsPath)"
 
-          # Test
-          - pwsh: $(testCommand)
-            displayName: Run Playwright tests
-            continueOnError: true
-            workingDirectory: tests/Umbraco.Tests.AcceptanceTest
-            env:
-              CI: true
-              CommitId: $(Build.SourceVersion)
-              AgentOs: $(Agent.OS)
+          # Merge Linux Playwright reports
+          - pwsh: npx playwright merge-reports --reporter html "$(allLinuxBlobReportsPath)"
+            displayName: Merge Linux reports
+            workingDirectory: "$(linuxMergedResultsPath)"
 
-          # Stop application
-          - bash: kill -15 $(AcceptanceTestProcessId)
-            displayName: Stop application (Linux)
-            condition: and(succeeded(), ne(variables.AcceptanceTestProcessId, ''), eq(variables['Agent.OS'], 'Linux'))
-
-          - pwsh: Stop-Process -Id $(AcceptanceTestProcessId)
-            displayName: Stop application (Windows)
-            condition: and(succeeded(), ne(variables.AcceptanceTestProcessId, ''), eq(variables['Agent.OS'], 'Windows_NT'))
-
-          # Stop SQL Server
-          - pwsh: docker stop mssql
-            displayName: Stop SQL Server Docker image (Linux)
-            condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux'))
-
-          - pwsh: SqlLocalDB stop MSSQLLocalDB
-            displayName: Stop SQL Server LocalDB (Windows)
-            condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))
-
-          # Copy artifacts
+          # Copy merged Linux reports if they exist
           - pwsh: |
-              if (Test-Path tests/Umbraco.Tests.AcceptanceTest/results/*) {
-                Copy-Item tests/Umbraco.Tests.AcceptanceTest/results/* $(Build.ArtifactStagingDirectory) -Recurse
+              if (Test-Path "$(linuxMergedResultsPath)/*") {
+                Copy-Item "$(linuxMergedResultsPath)/*" "$(Build.ArtifactStagingDirectory)" -Recurse
               }
-            displayName: Copy Playwright results
-            condition: succeededOrFailed()
+            displayName: Copy merged Linux reports
 
-          # Publish
+          # Publish merged Linux reports
           - task: PublishPipelineArtifact@1
-            displayName: Publish test artifacts
+            displayName: Publish merged test artifacts
             condition: succeededOrFailed()
             inputs:
-              targetPath: $(Build.ArtifactStagingDirectory)
-              artifact: "Acceptance Test Results - $(Agent.JobName) - Attempt #$(System.JobAttempt)"
+              targetPath: "$(Build.ArtifactStagingDirectory)"
+              artifact: "$(linuxArtifactName)"
+
+
+
+#          - task: DownloadPipelineArtifact@2
+#            displayName: Download Windows playwright results
+#            inputs:
+#              patterns: blob-report-sql-lite-Windows-*
+#              path: $(Build.ArtifactStagingDirectory)
+#
+#          - pwsh: "npx playwright merge-reports --reporter html $(Agent.BuildDirectory)/all-acceptance-test-results/windows"
+#            displayName: Merge Windows reports
+#            workingDirectory: $(Agent.BuildDirectory)/merged-acceptance-test-results/windows
+#
+#          - task: PublishPipelineArtifact@1
+#            displayName: Publish merged test artifacts
+#            condition: succeededOrFailed()
+#            inputs:
+#              targetPath: $(Build.ArtifactStagingDirectory)
+#              artifact: merged-acceptance-test-results-windows-$(System.JobAttempt)
+
+#      - job:
+#        displayName: E2E Tests (SQL Server)
+#        timeoutInMinutes: 180
+#        variables:
+#          # Connection string
+#          CONNECTIONSTRINGS__UMBRACODBDSN: Data Source=(localdb)\MSSQLLocalDB;AttachDbFilename=|DataDirectory|\Umbraco.mdf;Integrated Security=True
+#          CONNECTIONSTRINGS__UMBRACODBDSN_PROVIDERNAME: Microsoft.Data.SqlClient
+#        strategy:
+#          matrix:
+#            LinuxPart1Of3:
+#              testCommand: "npm run testSqlite -- --shard=1/3"
+#              vmImage: "ubuntu-latest"
+#              SA_PASSWORD: $(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSERPASSWORD)
+#              CONNECTIONSTRINGS__UMBRACODBDSN: "Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);TrustServerCertificate=True"
+#            LinuxPart2Of3:
+#              testCommand: "npm run testSqlite -- --shard=2/3"
+#              vmImage: "ubuntu-latest"
+#              SA_PASSWORD: $(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSERPASSWORD)
+#              CONNECTIONSTRINGS__UMBRACODBDSN: "Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);TrustServerCertificate=True"
+#            LinuxPart3Of3:
+#              testCommand: "npm run testSqlite -- --shard=3/3"
+#              vmImage: "ubuntu-latest"
+#              SA_PASSWORD: $(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSERPASSWORD)
+#              CONNECTIONSTRINGS__UMBRACODBDSN: "Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);TrustServerCertificate=True"
+#            WindowsPart1Of3:
+#              vmImage: "windows-latest"
+#              testCommand: "npm run testSqlite -- --shard=1/3"
+#            WindowsPart2Of3:
+#              vmImage: "windows-latest"
+#              testCommand: "npm run testSqlite -- --shard=2/3"
+#            WindowsPart3Of3:
+#              vmImage: "windows-latest"
+#              testCommand: "npm run testSqlite -- --shard=3/3"
+#        pool:
+#          vmImage: $(vmImage)
+#        steps:
+#          # Setup test environment
+#          - task: DownloadPipelineArtifact@2
+#            displayName: Download NuGet artifacts
+#            inputs:
+#              artifact: nupkg
+#              path: $(Agent.BuildDirectory)/app/nupkg
+#
+#          - task: NodeTool@0
+#            displayName: Use Node.js $(nodeVersion)
+#            inputs:
+#              versionSpec: $(nodeVersion)
+#
+#          - task: UseDotNet@2
+#            displayName: Use .NET SDK from global.json
+#            inputs:
+#              useGlobalJson: true
+#
+#          - pwsh: |
+#              "UMBRACO_USER_LOGIN=$(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSEREMAIL)
+#              UMBRACO_USER_PASSWORD=$(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSERPASSWORD)
+#              URL=$(ASPNETCORE_URLS)
+#              STORAGE_STAGE_PATH=$(Build.SourcesDirectory)/tests/Umbraco.Tests.AcceptanceTest/playwright/.auth/user.json" | Out-File .env
+#            displayName: Generate .env
+#            workingDirectory: $(Build.SourcesDirectory)/tests/Umbraco.Tests.AcceptanceTest
+#
+#          # Cache and restore NPM packages
+#          - task: Cache@2
+#            displayName: Cache NPM packages
+#            inputs:
+#              key: 'npm_e2e | "$(Agent.OS)" | $(Build.SourcesDirectory)/tests/Umbraco.Tests.AcceptanceTest/package-lock.json'
+#              restoreKeys: |
+#                npm_e2e | "$(Agent.OS)"
+#                npm_e2e
+#              path: $(npm_config_cache)
+#
+#          - script: npm ci --no-fund --no-audit --prefer-offline
+#            workingDirectory: $(Build.SourcesDirectory)/tests/Umbraco.Tests.AcceptanceTest
+#            displayName: Restore NPM packages
+#
+#          # Build application
+#          - pwsh: |
+#              $cmsVersion = "$(Build.BuildNumber)" -replace "\+",".g"
+#              dotnet new nugetconfig
+#              dotnet nuget add source ./nupkg --name Local
+#              dotnet new install Umbraco.Templates::$cmsVersion
+#              dotnet new umbraco --name UmbracoProject --version $cmsVersion --exclude-gitignore --no-restore --no-update-check
+#              dotnet restore UmbracoProject
+#              cp $(Build.SourcesDirectory)/tests/Umbraco.Tests.AcceptanceTest.UmbracoProject/*.cs UmbracoProject
+#              dotnet build UmbracoProject --configuration $(buildConfiguration) --no-restore
+#              dotnet dev-certs https
+#            displayName: Build application
+#            workingDirectory: $(Agent.BuildDirectory)/app
+#
+#          # Start SQL Server
+#          - powershell: docker run --name mssql -d -p 1433:1433 -e "ACCEPT_EULA=Y" -e "MSSQL_SA_PASSWORD=$(SA_PASSWORD)" mcr.microsoft.com/mssql/server:2022-latest
+#            displayName: Start SQL Server Docker image (Linux)
+#            condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux'))
+#
+#          - pwsh: SqlLocalDB start MSSQLLocalDB
+#            displayName: Start SQL Server LocalDB (Windows)
+#            condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))
+#
+#          # Run application
+#          - bash: |
+#              nohup dotnet run --project UmbracoProject --configuration $(buildConfiguration) --no-build --no-launch-profile > $(Build.ArtifactStagingDirectory)/playwright.log 2>&1 &
+#              echo "##vso[task.setvariable variable=AcceptanceTestProcessId]$!"
+#            displayName: Run application (Linux)
+#            condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux'))
+#            workingDirectory: $(Agent.BuildDirectory)/app
+#
+#          - pwsh: |
+#              $process = Start-Process dotnet "run --project UmbracoProject --configuration $(buildConfiguration) --no-build --no-launch-profile 2>&1" -PassThru -NoNewWindow -RedirectStandardOutput $(Build.ArtifactStagingDirectory)/playwright.log
+#              Write-Host "##vso[task.setvariable variable=AcceptanceTestProcessId]$($process.Id)"
+#            displayName: Run application (Windows)
+#            condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))
+#            workingDirectory: $(Agent.BuildDirectory)/app
+#
+#          # Ensures we have the package wait-on installed
+#          - pwsh: npm install wait-on
+#            displayName: Install wait-on package
+#
+#          # Wait for application to start responding to requests
+#          - pwsh: npx wait-on -v --interval 1000 --timeout 120000 $(ASPNETCORE_URLS)
+#            displayName: Wait for application
+#            workingDirectory: tests/Umbraco.Tests.AcceptanceTest
+#
+#          # Install Playwright and dependencies
+#          - pwsh: npx playwright install --with-deps
+#            displayName: Install Playwright
+#            workingDirectory: tests/Umbraco.Tests.AcceptanceTest
+#
+#          # Test
+#          - pwsh: $(testCommand)
+#            displayName: Run Playwright tests
+#            continueOnError: true
+#            workingDirectory: tests/Umbraco.Tests.AcceptanceTest
+#            env:
+#              CI: true
+#              CommitId: $(Build.SourceVersion)
+#              AgentOs: $(Agent.OS)
+#
+#          # Stop application
+#          - bash: kill -15 $(AcceptanceTestProcessId)
+#            displayName: Stop application (Linux)
+#            condition: and(succeeded(), ne(variables.AcceptanceTestProcessId, ''), eq(variables['Agent.OS'], 'Linux'))
+#
+#          - pwsh: Stop-Process -Id $(AcceptanceTestProcessId)
+#            displayName: Stop application (Windows)
+#            condition: and(succeeded(), ne(variables.AcceptanceTestProcessId, ''), eq(variables['Agent.OS'], 'Windows_NT'))
+#
+#          # Stop SQL Server
+#          - pwsh: docker stop mssql
+#            displayName: Stop SQL Server Docker image (Linux)
+#            condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux'))
+#
+#          - pwsh: SqlLocalDB stop MSSQLLocalDB
+#            displayName: Stop SQL Server LocalDB (Windows)
+#            condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))
+#
+#          # Copy artifacts
+#          - pwsh: |
+#              if (Test-Path tests/Umbraco.Tests.AcceptanceTest/results/*) {
+#                Copy-Item tests/Umbraco.Tests.AcceptanceTest/results/* $(Build.ArtifactStagingDirectory) -Recurse
+#              }
+#            displayName: Copy Playwright results
+#            condition: succeededOrFailed()
+#
+#          # Publish
+#          - task: PublishPipelineArtifact@1
+#            displayName: Publish test artifacts
+#            condition: succeededOrFailed()
+#            inputs:
+#              targetPath: $(Build.ArtifactStagingDirectory)
+#              artifact: "Acceptance Test Results - $(Agent.JobName) - Attempt #$(System.JobAttempt)"

tests/Umbraco.Tests.AcceptanceTest/playwright.config.ts

@@ -23,7 +23,7 @@ export default defineConfig({
   // We don't want to run parallel, as tests might differ in state
   workers: 1,
   /* Reporter to use. See https://playwright.dev/docs/test-reporters */
-  reporter: process.env.CI ? 'line' : 'html',
+  reporter: process.env.CI ? 'blob' : 'html',
   outputDir: "./results",
   /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
   use: {