@@ -5,29 +5,29 @@ parameters:
55- name : testMatrix
66 type : object
77- name : dependsOn
8- type : string
8+ type : string
99 default : ' EndToEndBuildTests'
1010
1111jobs :
1212- ${{ each item in parameters.testMatrix }} :
1313 - job : RegressionTest_${{ replace(item.displayName, '-', '_') }}_${{ replace(replace(item.repo, '/', '_'), '-', '_') }}
14- displayName : ' ${{ item.displayName }} Regression Test'
14+ displayName : ' ${{ item.displayName }} Regression Test'
1515 dependsOn : ${{ parameters.dependsOn }}
1616 ${{ if item.useVmImage }} :
1717 pool :
1818 vmImage : ${{ item.useVmImage }}
1919 ${{ else }} :
2020 pool :
2121 name : $(DncEngPublicBuildPool)
22- demands : ImageOverride -equals $(_WindowsMachineQueueName)
22+ demands : ImageOverride -equals $(_WindowsMachineQueueName)
2323 timeoutInMinutes : 60
2424 steps :
2525 - checkout : self
2626 displayName : Checkout F# compiler repo (for scripts)
2727
2828 - task : DownloadPipelineArtifact@2
2929 displayName : Download F# Compiler FSC Artifacts
30- inputs :
30+ inputs :
3131 artifactName : ' FSharpCompilerFscArtifacts'
3232 downloadPath : ' $(Pipeline.Workspace)/FSharpCompiler/artifacts/bin/fsc'
3333
@@ -38,17 +38,17 @@ jobs:
3838 downloadPath : ' $(Pipeline.Workspace)/FSharpCompiler/artifacts/bin/FSharp.Core'
3939
4040 - task : DownloadPipelineArtifact@2
41- displayName : Download UseLocalCompiler props
41+ displayName : Download UseLocalCompiler props
4242 inputs :
4343 artifactName : ' UseLocalCompilerProps'
4444 downloadPath : ' $(Pipeline.Workspace)/Props'
4545
4646 - pwsh : |
47- Write-Host "Cloning repository: ${{ item.repo }}"
47+ Write-Host "Cloning repository: ${{ item.repo }}"
4848 git clone --recursive https://github.com/${{ item.repo }}.git $(Pipeline.Workspace)/TestRepo
4949 Set-Location $(Pipeline.Workspace)/TestRepo
5050
51- Write-Host "Checking out commit: ${{ item.commit }}"
51+ Write-Host "Checking out commit: ${{ item.commit }}"
5252 git checkout ${{ item.commit }}
5353
5454 Write-Host "Initializing submodules (if any)..."
@@ -60,25 +60,22 @@ jobs:
6060 Write-Host "Repository structure:"
6161 Get-ChildItem -Name
6262
63- # Check if buildScript is a built-in dotnet command or a file-based script
64- # Note: buildScript comes from the pipeline YAML configuration (testMatrix parameter),
65- # not from external user input, so it's safe to use
66- $buildScript = "${{ item.buildScript }}"
63+ $buildScript = '${{ item.buildScript }}'
6764 if ($buildScript -like "dotnet*") {
68- Write-Host "Build command is a built-in dotnet command: $buildScript"
65+ Write-Host "Build command is a built-in dotnet command: $buildScript"
6966 Write-Host "Skipping file existence check for built-in command"
7067 } else {
71- Write-Host "Verifying build script exists: $buildScript"
68+ Write-Host "Verifying build script exists: $buildScript"
7269 if (Test-Path $buildScript) {
7370 Write-Host "Build script found: $buildScript"
7471 } else {
75- Write-Host "Build script not found: $buildScript"
72+ Write-Host "Build script not found: $buildScript"
7673 Write-Host "Available files in root:"
7774 Get-ChildItem
7875 exit 1
7976 }
8077 }
81- displayName: Checkout ${{ item.displayName }} at specific commit
78+ displayName: Checkout ${{ item.displayName }} at specific commit
8279
8380 - pwsh : |
8481 Set-Location $(Pipeline.Workspace)/TestRepo
@@ -92,14 +89,14 @@ jobs:
9289 displayName: Remove global.json to use latest SDK
9390
9491 - task : UseDotNet@2
95- displayName : Install .NET SDK 8.0.x for ${{ item.displayName }}
92+ displayName : Install .NET SDK 8.0.x for ${{ item.displayName }}
9693 inputs :
9794 packageType : sdk
9895 version : ' 8.0.x'
9996 installationPath : $(Pipeline.Workspace)/TestRepo/.dotnet
10097
10198 - task : UseDotNet@2
102- displayName : Install .NET SDK 10.0.100 for ${{ item.displayName }}
99+ displayName : Install .NET SDK 10.0.100 for ${{ item.displayName }}
103100 inputs :
104101 packageType : sdk
105102 version : ' 10.0.100'
@@ -153,23 +150,47 @@ jobs:
153150 Write-Host "============================================"
154151 Write-Host ""
155152
156- $buildScript = "${{ item.buildScript }}"
153+ $buildScript = '${{ item.buildScript }}'
154+ $errorLogPath = "$(Pipeline.Workspace)/build-errors.log"
155+ $fullLogPath = "$(Pipeline.Workspace)/build-full.log"
156+
157+ # Function to filter and capture important lines
158+ function Process-BuildOutput {
159+ param([string]$line)
160+ Write-Host $line
161+ if ($line -match ': error ' -or
162+ $line -match ': warning ' -or
163+ $line -match 'FAILED|Failed!' -or
164+ $line -match '^\s+X\s+' -or
165+ $line -match 'Exception: ' -or
166+ $line -match '^\s+at\s+.*\(\)' -or
167+ $line -match 'Build FAILED') {
168+ Add-Content -Path $errorLogPath -Value $line
169+ }
170+ }
157171
158- # Check if it's a built-in dotnet command or a file-based script
159- # Note: buildScript comes from the pipeline YAML configuration (testMatrix parameter),
160- # not from external user input, so using Invoke-Expression is safe here
161172 if ($buildScript -like "dotnet*") {
162- Write-Host "Executing built-in command: $buildScript"
163- Invoke-Expression $buildScript
173+ Write-Host "Executing built-in command: $buildScript"
174+ if ($IsWindows) {
175+ cmd /c $buildScript 2>&1 | Tee-Object -FilePath $fullLogPath | ForEach-Object {
176+ Process-BuildOutput $_
177+ }
178+ } else {
179+ bash -c "$buildScript" 2>&1 | Tee-Object -FilePath $fullLogPath | ForEach-Object {
180+ Process-BuildOutput $_
181+ }
182+ }
164183 } elseif ($IsWindows) {
165- # Execute the provided script on Windows
166184 Write-Host "Executing file-based script: $buildScript"
167- & ".\$buildScript"
185+ & ".\$buildScript" 2>&1 | Tee-Object -FilePath $fullLogPath | ForEach-Object {
186+ Process-BuildOutput $_
187+ }
168188 } else {
169- # Execute the provided script on Linux
170189 Write-Host "Executing file-based script: $buildScript"
171190 chmod +x "$buildScript"
172- bash -c "./$buildScript"
191+ bash -c "./$buildScript" 2>&1 | Tee-Object -FilePath $fullLogPath | ForEach-Object {
192+ Process-BuildOutput $_
193+ }
173194 }
174195 $exitCode = $LASTEXITCODE
175196
@@ -182,14 +203,14 @@ jobs:
182203 if ($exitCode -ne 0) {
183204 exit $exitCode
184205 }
185- displayName: Build ${{ item.displayName }} with local F# compiler
206+ displayName: Build ${{ item.displayName }} with local F# compiler
186207 env:
187208 LocalFSharpCompilerPath: $(Pipeline.Workspace)/FSharpCompiler
188209 LoadLocalFSharpBuild: 'True'
189210 LocalFSharpCompilerConfiguration: Release
190- timeoutInMinutes: 45
211+ timeoutInMinutes: 120
191212
192- - pwsh : |
213+ - pwsh : |
193214 Set-Location $(Pipeline.Workspace)/TestRepo
194215 $binlogDir = "$(Pipeline.Workspace)/BinaryLogs"
195216 New-Item -ItemType Directory -Force -Path $binlogDir | Out-Null
@@ -200,18 +221,18 @@ jobs:
200221 Write-Host "No .binlog files found"
201222 } else {
202223 foreach ($binlog in $binlogs) {
203- Write-Host "Copying: $($binlog.FullName)"
224+ Write-Host "Copying: $($binlog.FullName)"
204225 Copy-Item $binlog.FullName -Destination $binlogDir
205226 }
206227 Write-Host "Collected $($binlogs.Count) .binlog files"
207228 }
208- displayName: Collect Binary Logs
229+ displayName: Collect Binary Logs
209230 condition: always()
210231 continueOnError: true
211232
212233 - task : PublishPipelineArtifact@1
213- displayName : Publish ${{ item.displayName }} Binary Logs
214- inputs :
234+ displayName : Publish ${{ item.displayName }} Binary Logs
235+ inputs :
215236 targetPath : ' $(Pipeline.Workspace)/BinaryLogs'
216237 artifactName : ' ${{ item.displayName }}_BinaryLogs'
217238 publishLocation : pipeline
@@ -226,51 +247,80 @@ jobs:
226247 Write-Host "Repository: ${{ item.repo }}"
227248 Write-Host "Commit: ${{ item.commit }}"
228249 Write-Host "Build Script: ${{ item.buildScript }}"
250+
229251 if ($env:AGENT_JOBSTATUS -eq "Succeeded") {
230252 Write-Host "Status: SUCCESS"
231253 Write-Host "The ${{ item.displayName }} library builds successfully with the new F# compiler"
232254 } else {
233255 Write-Host "Status: FAILED"
234256 Write-Host "The ${{ item.displayName }} library failed to build with the new F# compiler"
235257
236- # Build multiline error message with reproduction steps
237- $lines = @(
238- "Regression test FAILED for ${{ item.displayName }} (${{ item.repo }}@${{ item.commit }})",
239- "",
240- "LOCAL REPRODUCTION STEPS (from fsharp repo root):",
241- "==========================================",
242- "# 1. Build the F# compiler",
243- "./build.sh -c Release",
244- "",
245- "# 2. Clone and checkout the failing library",
246- "cd ..",
247- "git clone --recursive https://github.com/${{ item.repo }}.git TestRepo",
248- "cd TestRepo",
249- "git checkout ${{ item.commit }}",
250- "git submodule update --init --recursive",
251- "rm -f global.json",
252- "",
253- "# 3. Prepare the repo for local compiler",
254- "dotnet fsi ../fsharp/eng/scripts/PrepareRepoForRegressionTesting.fsx `"../fsharp/UseLocalCompiler.Directory.Build.props`"",
255- "",
256- "# 4. Build with local compiler",
257- "export LocalFSharpCompilerPath=`$PWD/../fsharp",
258- "export LoadLocalFSharpBuild=True",
259- "export LocalFSharpCompilerConfiguration=Release",
260- "./${{ item.buildScript }}",
261- "=========================================="
262- )
263-
264- # Report using VSO error format - each line separately
265- foreach ($line in $lines) {
266- Write-Host "##[error]$line"
258+ $errorLogPath = "$(Pipeline.Workspace)/build-errors.log"
259+ if (Test-Path $errorLogPath) {
260+ Write-Host ""
261+ Write-Host "##[error]============================================"
262+ Write-Host "##[error]FAILURE SUMMARY for ${{ item.displayName }}"
263+ Write-Host "##[error]============================================"
264+
265+ $errors = Get-Content $errorLogPath | Select-Object -First 100
266+ $errorCount = 0
267+ $warningCount = 0
268+ $testFailCount = 0
269+
270+ foreach ($line in $errors) {
271+ if ($line -match ': error ') {
272+ Write-Host "##[error]$line"
273+ $errorCount++
274+ } elseif ($line -match ': warning ') {
275+ Write-Host "##[warning]$line"
276+ $warningCount++
277+ } elseif ($line -match 'FAILED|Failed! |^\s+X\s+') {
278+ Write-Host "##[error][TEST] $line"
279+ $testFailCount++
280+ } else {
281+ Write-Host "##[error]$line"
282+ }
283+ }
284+
285+ $totalLines = (Get-Content $errorLogPath | Measure-Object -Line).Lines
286+ if ($totalLines -gt 100) {
287+ Write-Host "##[warning]... and $($totalLines - 100) more lines (see full log or binary logs)"
288+ }
289+
290+ Write-Host ""
291+ Write-Host "##[error]Summary: $errorCount error(s), $warningCount warning(s), $testFailCount test failure(s)"
292+ Write-Host "##[error]============================================"
293+ } else {
294+ Write-Host "##[warning]No error log captured - check full build output above"
267295 }
268296
269- # Also log as VSO issue for Azure DevOps integration
297+ Write-Host ""
298+ Write-Host "##[section]LOCAL REPRODUCTION STEPS (from fsharp repo root):"
299+ Write-Host "# 1. Build the F# compiler"
300+ Write-Host "./build.sh -c Release"
301+ Write-Host ""
302+ Write-Host "# 2. Clone and checkout the failing library"
303+ Write-Host "cd .."
304+ Write-Host "git clone --recursive https://github.com/${{ item.repo }}.git TestRepo"
305+ Write-Host "cd TestRepo"
306+ Write-Host "git checkout ${{ item.commit }}"
307+ Write-Host "git submodule update --init --recursive"
308+ Write-Host "rm -f global.json"
309+ Write-Host ""
310+ Write-Host "# 3. Prepare the repo for local compiler"
311+ Write-Host "dotnet fsi ../fsharp/eng/scripts/PrepareRepoForRegressionTesting.fsx `"../fsharp/UseLocalCompiler.Directory.Build.props`""
312+ Write-Host ""
313+ Write-Host "# 4. Build with local compiler"
314+ Write-Host "export LocalFSharpCompilerPath=`$PWD/../fsharp"
315+ Write-Host "export LoadLocalFSharpBuild=True"
316+ Write-Host "export LocalFSharpCompilerConfiguration=Release"
317+ Write-Host "${{ item.buildScript }}"
318+
270319 Write-Host "##vso[task.logissue type=error;sourcepath=azure-pipelines-PR.yml]Regression test failed: ${{ item.displayName }}"
271320 }
272321 Write-Host "============================================"
273322
323+ Write-Host ""
274324 Write-Host "Binary logs found:"
275325 Get-ChildItem "*.binlog" -Recurse -ErrorAction SilentlyContinue | ForEach-Object { Write-Host $_.FullName }
276326 displayName: Report ${{ item.displayName }} test result
0 commit comments