From cf1b12a640f5bc5f5b0d0e9061d1569e060d83e4 Mon Sep 17 00:00:00 2001 From: Gator96100 Date: Fri, 18 Sep 2020 16:37:56 +0200 Subject: [PATCH 1/8] Appveyor run WSL installation in parallel --- appveyor.yml | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 4eecb27f7..fb96b2a54 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -60,7 +60,8 @@ clone_script: break } - Start-Sleep -s 1 + Start-Sleep -s 5 + Receive-Job -Job $WSLjob } } @@ -70,6 +71,18 @@ clone_script: Write-Host "[ OK ]" -ForegroundColor Green } + $WSLjob = Start-Job -Name WSLInstall -ScriptBlock { + $WSLInstallTime=[System.Environment]::TickCount + WSLExec "WSL update..." "sudo apt-get update 1>/dev/null" + WSLExec "WSL upgrade..." "sudo apt-get upgrade -y 1>/dev/null" + WSLExec "WSL cleanup..." "sudo apt-get auto-remove -y 1>/dev/null" + WSLExec "WSL install..." "sudo apt-get -y install --reinstall --no-install-recommends git ca-certificates build-essential pkg-config libreadline-dev gcc-arm-none-eabi libnewlib-dev libbz2-dev qtbase5-dev cmake 1>/dev/null" + WSLExec "WSL QT fix..." "sudo strip --remove-section=.note.ABI-tag /usr/lib/x86_64-linux-gnu/libQt5Core.so.5" + Add-AppveyorMessage -Message "WSL setup took $(([System.Environment]::TickCount-$WSLInstallTime) / 1000) sec" -Category Information + } + + Receive-Job -Job $WSLjob + Write-Host "Removing ProxSpace..." -NoNewLine $PSInstallTime=[System.Environment]::TickCount @@ -113,20 +126,6 @@ clone_script: Add-AppveyorMessage -Message "ProxSpace download and update took $(([System.Environment]::TickCount-$PSInstallTime) / 1000) sec" -Category Information - $WSLInstallTime=[System.Environment]::TickCount - - WSLExec "WSL update..." "sudo apt-get update 1>/dev/null" - - WSLExec "WSL upgrade..." "sudo apt-get upgrade -y 1>/dev/null" - - WSLExec "WSL cleanup..." "sudo apt-get auto-remove -y 1>/dev/null" - - WSLExec "WSL install..." "sudo apt-get -y install --reinstall --no-install-recommends git ca-certificates build-essential pkg-config libreadline-dev gcc-arm-none-eabi libnewlib-dev libbz2-dev qtbase5-dev cmake 1>/dev/null" - - WSLExec "WSL QT fix..." "sudo strip --remove-section=.note.ABI-tag /usr/lib/x86_64-linux-gnu/libQt5Core.so.5" - - Add-AppveyorMessage -Message "WSL setup took $(([System.Environment]::TickCount-$WSLInstallTime) / 1000) sec" -Category Information - Write-Host "Cloning repository <$env:appveyor_repo_name> to $env:appveyor_build_folder ..." -NoNewLine if(-not $env:appveyor_pull_request_number) { @@ -142,6 +141,8 @@ clone_script: Write-Host "[ OK ]" -ForegroundColor Green + Receive-Job -Wait -Job $WSLjob + install: build_script: From 2d6a6ba5d92edaa03960e335b7d624c4def01559 Mon Sep 17 00:00:00 2001 From: Gator96100 Date: Fri, 18 Sep 2020 16:45:09 +0200 Subject: [PATCH 2/8] Appveyor moved function into scriptblock --- appveyor.yml | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index fb96b2a54..3a6f783b3 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -65,13 +65,13 @@ clone_script: } } - Function WSLExec($Text, $Cmd) { - Write-Host "$Text" -NoNewLine - wsl -- bash -c $Cmd - Write-Host "[ OK ]" -ForegroundColor Green - } - $WSLjob = Start-Job -Name WSLInstall -ScriptBlock { + Function WSLExec($Text, $Cmd) { + Write-Host "$Text" + wsl -- bash -c $Cmd + Write-Host "$Text[ OK ]" -ForegroundColor Green + } + $WSLInstallTime=[System.Environment]::TickCount WSLExec "WSL update..." "sudo apt-get update 1>/dev/null" WSLExec "WSL upgrade..." "sudo apt-get upgrade -y 1>/dev/null" @@ -81,8 +81,6 @@ clone_script: Add-AppveyorMessage -Message "WSL setup took $(([System.Environment]::TickCount-$WSLInstallTime) / 1000) sec" -Category Information } - Receive-Job -Job $WSLjob - Write-Host "Removing ProxSpace..." -NoNewLine $PSInstallTime=[System.Environment]::TickCount @@ -92,6 +90,8 @@ clone_script: Remove-Item -Recurse -Force -Path $env:proxspace_path Write-Host "[ OK ]" -ForegroundColor Green + + Receive-Job -Job $WSLjob Write-Host "Download ProxSpace..." -NoNewLine @@ -101,6 +101,8 @@ clone_script: Write-Host "[ OK ]" -ForegroundColor Green + Receive-Job -Job $WSLjob + Write-Host "Extracting ProxSpace..." -NoNewLine Expand-Archive -LiteralPath "$env:proxspace_zip_file" -DestinationPath "\" @@ -109,6 +111,8 @@ clone_script: Write-Host "[ OK ]" -ForegroundColor Green + Receive-Job -Job $WSLjob + Write-Host "Renaming ProxSpace folder..." -NoNewLine Get-ChildItem -Path "\$env:proxspace_zip_folder_name" | Rename-Item -NewName (Split-Path $env:proxspace_path -Leaf) From 3719fee8a34ccb2c61d9207f31824f2a0d59ffbf Mon Sep 17 00:00:00 2001 From: Gator96100 Date: Fri, 18 Sep 2020 16:47:24 +0200 Subject: [PATCH 3/8] Fixed color output --- appveyor.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 3a6f783b3..769be4630 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -69,7 +69,8 @@ clone_script: Function WSLExec($Text, $Cmd) { Write-Host "$Text" wsl -- bash -c $Cmd - Write-Host "$Text[ OK ]" -ForegroundColor Green + Write-Host "$Text" -NoNewLine + Write-Host "[ OK ]" -ForegroundColor Green } $WSLInstallTime=[System.Environment]::TickCount From cbfa6d8876d15262dd96b6765aa697ad11d46474 Mon Sep 17 00:00:00 2001 From: Gator96100 Date: Fri, 18 Sep 2020 17:14:59 +0200 Subject: [PATCH 4/8] Fixed text output --- appveyor.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 769be4630..131ac10d8 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -34,7 +34,7 @@ clone_script: - ps: >- Function ExecUpdate($Text, $firstStart) { - Write-Host "$Text" -NoNewLine + Write-Host "$Text" Start-Process "cmd.exe" "/c ""cd /D $env:proxspace_path && runme64.bat -c ""exit""""" $StartTime=[System.Environment]::TickCount Start-Sleep -s 10 @@ -42,6 +42,7 @@ clone_script: $cmdprocess = Get-Process "cmd" -ErrorAction SilentlyContinue if (!$cmdprocess -Or $cmdprocess.HasExited) { + Write-Host "$Text" -NoNewLine Write-Host "[ OK ]" -ForegroundColor Green break } @@ -51,11 +52,13 @@ clone_script: $tmp = $cmdprocess.CloseMainWindow() Start-Sleep -s 5 Stop-Process -Name "cmd" -Force -ErrorAction SilentlyContinue + Write-Host "$Text" -NoNewLine Write-Host "Exit by pacman.conf" -ForegroundColor Green break } if ([System.Environment]::TickCount-$StartTime -gt 1000000) { + Write-Host "$Text" -NoNewLine Write-host "Exit by timeout" -ForegroundColor Yellow break } From cdd860a0488f4536ba314286f8c5bdb209949a36 Mon Sep 17 00:00:00 2001 From: Gator96100 Date: Fri, 18 Sep 2020 18:00:16 +0200 Subject: [PATCH 5/8] Appveyor run compilation in parallel --- appveyor.yml | 149 ++++++++++++++++++++++++++------------------------- 1 file changed, 75 insertions(+), 74 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 131ac10d8..3ac706140 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -7,6 +7,7 @@ environment: proxspace_zip_folder_name: ProxSpace-* proxspace_path: \ProxSpace proxspace_home_path: \ProxSpace\pm3 + wsl_git_path: C:\wslgit init: - ps: >- @@ -85,7 +86,22 @@ clone_script: Add-AppveyorMessage -Message "WSL setup took $(([System.Environment]::TickCount-$WSLInstallTime) / 1000) sec" -Category Information } - Write-Host "Removing ProxSpace..." -NoNewLine + Function GitClone($Text, $Folder) { + Write-Host "$Text" -NoNewLine + if(-not $env:appveyor_pull_request_number) { + git clone -q --branch=$env:appveyor_repo_branch https://github.com/$env:appveyor_repo_name.git $Folder + cd $Folder + git checkout -qf $env:appveyor_repo_commit + } else { + git clone -q https://github.com/$env:appveyor_repo_name.git $Folder + cd $Folder + git fetch -q origin +refs/pull/$env:appveyor_pull_request_number/merge: + git checkout -qf FETCH_HEAD + } + Write-Host "[ OK ]" -ForegroundColor Green + } + + Write-Host "ProxSpace: Removing folder..." -NoNewLine $PSInstallTime=[System.Environment]::TickCount @@ -97,7 +113,7 @@ clone_script: Receive-Job -Job $WSLjob - Write-Host "Download ProxSpace..." -NoNewLine + Write-Host "ProxSpace: downloading..." -NoNewLine [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 @@ -107,7 +123,7 @@ clone_script: Receive-Job -Job $WSLjob - Write-Host "Extracting ProxSpace..." -NoNewLine + Write-Host "ProxSpace: extracting..." -NoNewLine Expand-Archive -LiteralPath "$env:proxspace_zip_file" -DestinationPath "\" @@ -117,15 +133,15 @@ clone_script: Receive-Job -Job $WSLjob - Write-Host "Renaming ProxSpace folder..." -NoNewLine + Write-Host "ProxSpace: renaming folder..." -NoNewLine Get-ChildItem -Path "\$env:proxspace_zip_folder_name" | Rename-Item -NewName (Split-Path $env:proxspace_path -Leaf) Write-Host "[ OK ]" -ForegroundColor Gree - ExecUpdate "Initial msys2 startup..." $true + ExecUpdate "ProxSpace: initial msys2 startup..." $true - ExecUpdate "Installing required packages..." $false + ExecUpdate "ProxSpace: installing required packages..." $false $psversion = (Select-String -Pattern 'PSVERSION=' -SimpleMatch -Path "$env:proxspace_path\msys2\ps\09-proxspace_setup.post").Line.Split("""")[1] @@ -134,23 +150,12 @@ clone_script: Add-AppveyorMessage -Message "ProxSpace download and update took $(([System.Environment]::TickCount-$PSInstallTime) / 1000) sec" -Category Information - Write-Host "Cloning repository <$env:appveyor_repo_name> to $env:appveyor_build_folder ..." -NoNewLine - - if(-not $env:appveyor_pull_request_number) { - git clone -q --branch=$env:appveyor_repo_branch https://github.com/$env:appveyor_repo_name.git $env:appveyor_build_folder - cd $env:appveyor_build_folder - git checkout -qf $env:appveyor_repo_commit - } else { - git clone -q https://github.com/$env:appveyor_repo_name.git $env:appveyor_build_folder - cd $env:appveyor_build_folder - git fetch -q origin +refs/pull/$env:appveyor_pull_request_number/merge: - git checkout -qf FETCH_HEAD - } - - Write-Host "[ OK ]" -ForegroundColor Green + GitClone "ProxSpace: Cloning repository <$env:appveyor_repo_name> to $env:appveyor_build_folder ..." $env:appveyor_build_folder Receive-Job -Wait -Job $WSLjob + GitClone "WSL: Cloning repository <$env:appveyor_repo_name> to $env:wsl_git_path ..." $env:wsl_git_path + install: build_script: @@ -163,10 +168,6 @@ build_script: ./runme64.bat -c "cd $pmfolder && $Cmd" } - Function ExecWSLCmd($Cmd) { - wsl -- bash -c $Cmd - } - Function ExecCheck($Name) { $testspass = ($LASTEXITCODE -eq 0) @@ -182,6 +183,55 @@ build_script: } } + $WSLjob = Start-Job -Name WSLCompile -ScriptBlock { + Function ExecWSLCmd($Cmd) { + wsl -- bash -c $Cmd + } + + Function ExecCheck($Name) { + $testspass = ($LASTEXITCODE -eq 0) + + $global:TestsPassed=$testspass + + if ($testspass) { + Add-AppveyorTest -Name $Name -Framework NUnit -Filename $Name -Outcome Passed -Duration "$([System.Environment]::TickCount-$TestTime)" + Write-Host "$Name [ OK ]" -ForegroundColor Green + } else { + Add-AppveyorTest -Name $Name -Framework NUnit -Filename $Name -Outcome Failed -Duration "$([System.Environment]::TickCount-$TestTime)" + Write-Host "$Name [ ERROR ]" -ForegroundColor Red + throw "Tests error." + } + } + + #Windows Subsystem for Linux (WSL) + Write-Host "---------- WSL make ----------" -ForegroundColor Yellow + cd $env:appveyor_build_folder + $TestTime=[System.Environment]::TickCount + ExecWSLCmd "make clean;make V=1" + #some checks + if(!(Test-Path "$env:proxspace_home_path\$pmfolder\client\proxmark3")){ + throw "Main file proxmark3 not exists." + } + + ExecWSLCmd "make check" + ExecCheck "WSL make Tests" + Start-Sleep -s 2 + Write-Host "---------- WSL btaddon ----------" -ForegroundColor Yellow + $TestTime=[System.Environment]::TickCount + ExecWSLCmd "make clean;make V=1 PLATFORM_EXTRAS=BTADDON" + ExecWSLCmd "make check" + ExecCheck "WSL BTaddon Tests" + Start-Sleep -s 2 + Write-Host "---------- WSL make clean ----------" -ForegroundColor Yellow + ExecWSLCmd 'make clean' + Write-Host "---------- WSL cmake ----------" -ForegroundColor Yellow + $TestTime=[System.Environment]::TickCount + ExecWSLCmd 'rm -rf client/build;mkdir -p client/build; cd client/build; cmake ..; make VERBOSE=1;' + Write-Host "---------- WSL cmake tests ----------" -ForegroundColor Yellow + ExecWSLCmd './tools/pm3_tests.sh --clientbin client/build/proxmark3 client' + ExecCheck "WSL cmake Tests" + } + #ProxSpace Write-Host "---------- PS make ----------" -ForegroundColor Yellow @@ -226,57 +276,8 @@ build_script: ExecCheck "PS cmake Tests" - #Windows Subsystem for Linux (WSL) - - Write-Host "---------- WSL make ----------" -ForegroundColor Yellow - - cd $env:appveyor_build_folder - - $TestTime=[System.Environment]::TickCount - - ExecWSLCmd "make clean;make V=1" - - #some checks + Receive-Job -Wait -Job $WSLjob -ErrorAction Stop - if(!(Test-Path "$env:proxspace_home_path\$pmfolder\client\proxmark3")){ - - throw "Main file proxmark3 not exists." - - } - - ExecWSLCmd "make check" - - ExecCheck "WSL make Tests" - - Start-Sleep -s 2 - - Write-Host "---------- WSL btaddon ----------" -ForegroundColor Yellow - - $TestTime=[System.Environment]::TickCount - - ExecWSLCmd "make clean;make V=1 PLATFORM_EXTRAS=BTADDON" - - ExecWSLCmd "make check" - - ExecCheck "WSL BTaddon Tests" - - Start-Sleep -s 2 - - Write-Host "---------- WSL make clean ----------" -ForegroundColor Yellow - - ExecWSLCmd 'make clean' - - Write-Host "---------- WSL cmake ----------" -ForegroundColor Yellow - - $TestTime=[System.Environment]::TickCount - - ExecWSLCmd 'rm -rf client/build;mkdir -p client/build; cd client/build; cmake ..; make VERBOSE=1;' - - Write-Host "---------- WSL cmake tests ----------" -ForegroundColor Yellow - - ExecWSLCmd './tools/pm3_tests.sh --clientbin client/build/proxmark3 client' - - ExecCheck "WSL cmake Tests" test_script: - ps: >- From 93848d5a1cb5e6e809b7a905fde0ea9ff50db1d7 Mon Sep 17 00:00:00 2001 From: Gator96100 Date: Fri, 18 Sep 2020 18:24:19 +0200 Subject: [PATCH 6/8] Fixed paths --- appveyor.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 3ac706140..81dd22f8f 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -205,11 +205,11 @@ build_script: #Windows Subsystem for Linux (WSL) Write-Host "---------- WSL make ----------" -ForegroundColor Yellow - cd $env:appveyor_build_folder + cd $env:wsl_git_path $TestTime=[System.Environment]::TickCount ExecWSLCmd "make clean;make V=1" #some checks - if(!(Test-Path "$env:proxspace_home_path\$pmfolder\client\proxmark3")){ + if(!(Test-Path "$env:wsl_git_path\client\proxmark3")){ throw "Main file proxmark3 not exists." } @@ -226,7 +226,7 @@ build_script: ExecWSLCmd 'make clean' Write-Host "---------- WSL cmake ----------" -ForegroundColor Yellow $TestTime=[System.Environment]::TickCount - ExecWSLCmd 'rm -rf client/build;mkdir -p client/build; cd client/build; cmake ..; make VERBOSE=1;' + ExecWSLCmd 'mkdir -p client/build; cd client/build; cmake ..; make VERBOSE=1;' Write-Host "---------- WSL cmake tests ----------" -ForegroundColor Yellow ExecWSLCmd './tools/pm3_tests.sh --clientbin client/build/proxmark3 client' ExecCheck "WSL cmake Tests" From a5c67ea4e62584302707f9cef301d31a2772e0a5 Mon Sep 17 00:00:00 2001 From: Gator96100 Date: Fri, 18 Sep 2020 18:51:12 +0200 Subject: [PATCH 7/8] Make sure throw works inside a job --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 81dd22f8f..ecc9646d1 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -209,7 +209,7 @@ build_script: $TestTime=[System.Environment]::TickCount ExecWSLCmd "make clean;make V=1" #some checks - if(!(Test-Path "$env:wsl_git_path\client\proxmark3")){ + if(!(Test-Path "$env:wsl_git_path\client\proxmark3fail")){ throw "Main file proxmark3 not exists." } From 23c81357c2d2fe8aac268c66f2d471ec37df4dd1 Mon Sep 17 00:00:00 2001 From: Gator96100 Date: Fri, 18 Sep 2020 19:13:13 +0200 Subject: [PATCH 8/8] Moved wsl git path --- appveyor.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index ecc9646d1..2f145c598 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -7,7 +7,7 @@ environment: proxspace_zip_folder_name: ProxSpace-* proxspace_path: \ProxSpace proxspace_home_path: \ProxSpace\pm3 - wsl_git_path: C:\wslgit + wsl_git_path: C:\proxmark init: - ps: >- @@ -209,7 +209,7 @@ build_script: $TestTime=[System.Environment]::TickCount ExecWSLCmd "make clean;make V=1" #some checks - if(!(Test-Path "$env:wsl_git_path\client\proxmark3fail")){ + if(!(Test-Path "$env:wsl_git_path\client\proxmark3")){ throw "Main file proxmark3 not exists." }