From ae2dfe404b9ac43cfca3a8e52113ad731b0cc3e4 Mon Sep 17 00:00:00 2001 From: Dmitri Smirnov Date: Tue, 10 Nov 2015 17:23:16 -0800 Subject: [PATCH 1/4] Try running db_test during integration build --- appveyor.yml | 14 +++++++++++++- appveyordailytests.yml | 2 +- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index eb2f63172..d257751e8 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -8,4 +8,16 @@ build: project: build\rocksdb.sln parallel: true verbosity: minimal -test: off +test: +test_script: +- ps: build_tools\run_ci_db_test.ps1 -Run db_test -Concurrency 16 +notifications: + - provider: Email + to: + - svmtrocksdb@microsoft.com + subject: "Build {{status}}" + message: "{{message}}, {{commitId}}, ..." + on_build_success: false + on_build_failure: true + on_build_status_changed: true + diff --git a/appveyordailytests.yml b/appveyordailytests.yml index 0a35ac68b..681b5f9ea 100644 --- a/appveyordailytests.yml +++ b/appveyordailytests.yml @@ -10,7 +10,7 @@ build: verbosity: minimal test: test_script: -- ps: build_tools\run_ci_db_test.ps1 +- ps: build_tools\run_ci_db_test.ps1 -Run db_test -Concurrency 16 notifications: - provider: Email to: From 7f59e33b1971fdb20d37ea3aff19f9e31366e3f1 Mon Sep 17 00:00:00 2001 From: Dmitri Smirnov Date: Tue, 10 Nov 2015 17:28:56 -0800 Subject: [PATCH 2/4] Make CI build debug/optimized --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index d257751e8..2bfad53af 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -2,7 +2,7 @@ version: 1.0.{build} before_build: - md %APPVEYOR_BUILD_FOLDER%\build - cd %APPVEYOR_BUILD_FOLDER%\build -- cmake -G "Visual Studio 12 Win64" .. +- cmake -G "Visual Studio 12 Win64" -DOPTDBG=1 .. - cd .. build: project: build\rocksdb.sln From 935d1495c5ff13d1266d73550f0fcab4e9a190e9 Mon Sep 17 00:00:00 2001 From: yuslepukhin Date: Thu, 12 Nov 2015 14:42:38 -0800 Subject: [PATCH 3/4] Run tests imporvements Add sequential rerun for any failed tests. Add env_test case. Limit concurrency Allow to specify individual tests Take $Limit into account when displaying number of tests --- appveyor.yml | 12 +- appveyordailytests.yml | 22 ---- build_tools/run_ci_db_test.ps1 | 221 ++++++++++++++++++++------------- 3 files changed, 134 insertions(+), 121 deletions(-) delete mode 100644 appveyordailytests.yml diff --git a/appveyor.yml b/appveyor.yml index 2bfad53af..986a82526 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -10,14 +10,6 @@ build: verbosity: minimal test: test_script: -- ps: build_tools\run_ci_db_test.ps1 -Run db_test -Concurrency 16 -notifications: - - provider: Email - to: - - svmtrocksdb@microsoft.com - subject: "Build {{status}}" - message: "{{message}}, {{commitId}}, ..." - on_build_success: false - on_build_failure: true - on_build_status_changed: true +- ps: build_tools\run_ci_db_test.ps1 -EnableRerun -Run db_test -Exclude DBTest.Randomized -Concurrency 16 +- ps: build_tools\run_ci_db_test.ps1 -Run env_test -Concurrency 1 diff --git a/appveyordailytests.yml b/appveyordailytests.yml deleted file mode 100644 index 681b5f9ea..000000000 --- a/appveyordailytests.yml +++ /dev/null @@ -1,22 +0,0 @@ -version: 1.0.{build} -before_build: -- md %APPVEYOR_BUILD_FOLDER%\build -- cd %APPVEYOR_BUILD_FOLDER%\build -- cmake -G "Visual Studio 12 Win64" -DOPTDBG=1 .. -- cd .. -build: - project: build\rocksdb.sln - parallel: true - verbosity: minimal -test: -test_script: -- ps: build_tools\run_ci_db_test.ps1 -Run db_test -Concurrency 16 -notifications: - - provider: Email - to: - - svmtrocksdb@microsoft.com - subject: "Build {{status}}" - message: "{{message}}, {{commitId}}, ..." - on_build_success: false - on_build_failure: true - on_build_status_changed: true diff --git a/build_tools/run_ci_db_test.ps1 b/build_tools/run_ci_db_test.ps1 index 5cb4ac2c1..c646f194e 100644 --- a/build_tools/run_ci_db_test.ps1 +++ b/build_tools/run_ci_db_test.ps1 @@ -5,13 +5,15 @@ # Run the script from the enlistment Param( [switch]$EnableJE = $false, # Use je executable + [switch]$EnableRerun = $false, # Rerun failed tests sequentially at the end [string]$WorkFolder = "", # Direct tests to use that folder [int]$Limit = -1, # -1 means run all otherwise limit for testing purposes [string]$Exclude = "", # Expect a comma separated list, no spaces - [string]$Run = "db_test", # Run db_test|tests + [string]$Run = "db_test", # Run db_test|tests|testname1,testname2... # Number of async tasks that would run concurrently. Recommend a number below 64. # However, CPU utlization really depends on the storage media. Recommend ram based disk. - [int]$Concurrency = 62 + # a value of 1 will run everything serially + [int]$Concurrency = 16 ) # Folders and commands must be fullpath to run assuming @@ -52,7 +54,7 @@ if($EnableJE) { } Write-Output "Root: $RootFolder, WorkFolder: $WorkFolder" -Write-Output "Binaries: $BinariesFolder exe: $db_test" +Write-Output "Binaries: $BinariesFolder db_test: $db_test" #Exclusions that we do not want to run $ExcludeTests = New-Object System.Collections.Generic.HashSet[string] @@ -60,7 +62,7 @@ $ExcludeTests = New-Object System.Collections.Generic.HashSet[string] if($Exclude -ne "") { Write-Host "Exclude: $Exclude" - $l = $Exclude -split ',' + $l = $Exclude -split ' ' ForEach($t in $l) { $ExcludeTests.Add($t) | Out-Null } } @@ -119,41 +121,60 @@ function Normalize-DbTests($HashTable) { } } +# The function removes trailing .exe siffix if any, +# creates a name for the log file +function MakeAndAdd([string]$token, $HashTable) { + $test_name = $token -replace '.exe$', '' + $log_name = -join ($test_name, ".log") + if(!$ExcludeTests.Contains($test)) { + $HashTable.Add($test_name, $log_name) + } else { + Write-Warning "Test $test_name is excluded" + } +} + # The function scans build\Debug folder to discover # Test executables. It then populates a table with # Test executable name -> Log file -function Discover-TestBinaries($HashTable) { +function Discover-TestBinaries([string]$Pattern, $HashTable) { $Exclusions = @("db_test*", "db_sanity_test*") - if($EnableJE) { - $p = -join ($BinariesFolder, "*_test_je.exe") - } else { - $p = -join ($BinariesFolder, "*_test.exe") - } + + $p = -join ($BinariesFolder, $pattern) + + Write-Host "Path: $p" dir -Path $p -Exclude $Exclusions | ForEach-Object { - $t = ($_.Name) -replace '.exe$', '' - if($ExcludeTests.Contains($t)) { - continue - } - $test_log = -join ($t, ".log") - $HashTable.Add($t, $test_log) + MakeAndAdd -token ($_.Name) -HashTable $HashTable } } -$TestToLog = [ordered]@{} +$TestsToRun = [ordered]@{} if($Run -ceq "db_test") { - Normalize-DbTests -HashTable $TestToLog + Normalize-DbTests -HashTable $TestsToRun } elseif($Run -ceq "tests") { - Discover-TestBinaries -HashTable $TestToLog + if($EnableJE) { + $pattern = "*_test_je.exe" + } else { + $pattern = "*_test.exe" + } + Discover-TestBinaries -Pattern $pattern -HashTable $TestsToRun } else { - Write-Warning "Invalid -Run option value" - exit 2 + + $test_list = $Run -split ' ' + + ForEach($t in $test_list) { + MakeAndAdd -token $t -HashTable $TestsToRun + } } +$NumTestsToStart = $TestsToRun.Count +if($Limit -ge 0 -and $NumTestsToStart -gt $Limit) { + $NumTestsToStart = $Limit +} -Write-Host "Attempting to start: " ($TestToLog.Count) " tests" +Write-Host "Attempting to start: $NumTestsToStart tests" # Invoke a test with a filter and redirect all output $InvokeTestCase = { @@ -167,101 +188,123 @@ $InvokeTestAsync = { &$exe > $log 2>&1 } -$jobs = @() -$JobToLog = @{} +# Hash that contains tests to rerun if any failed +# Those tests will be rerun sequentially +$Rerun = [ordered]@{} # Test limiting factor here $count = 0 - +# Overall status [bool]$success = $true; -# Wait for all to finish and get the results -while(($JobToLog.Count -gt 0) -or - ($TestToLog.Count -gt 0)) { +function RunJobs($TestToLog, [int]$ConcurrencyVal, [bool]$AddForRerun) +{ + # Array to wait for any of the running jobs + $jobs = @() + # Hash JobToLog + $JobToLog = @{} - # Make sure we have maximum concurrent jobs running if anything - # and the $Limit either not set or allows to proceed - while(($JobToLog.Count -lt $Concurrency) -and - (($TestToLog.Count -gt 0) -and - (($Limit -lt 0) -or ($count -lt $Limit)))) { + # Wait for all to finish and get the results + while(($JobToLog.Count -gt 0) -or + ($TestToLog.Count -gt 0)) { + # Make sure we have maximum concurrent jobs running if anything + # and the $Limit either not set or allows to proceed + while(($JobToLog.Count -lt $ConcurrencyVal) -and + (($TestToLog.Count -gt 0) -and + (($Limit -lt 0) -or ($count -lt $Limit)))) { - # We only need the first key - foreach($key in $TestToLog.keys) { - $k = $key - break - } - Write-Host "Starting $k" - $log_path = -join ($LogFolder, ($TestToLog.$k)) - - if($Run -ceq "db_test") { - $job = Start-Job -Name $k -ScriptBlock $InvokeTestCase -ArgumentList @($db_test,$k,$log_path) - } else { - [string]$Exe = -Join ($BinariesFolder, $k) - $job = Start-Job -Name $k -ScriptBlock $InvokeTestAsync -ArgumentList @($exe,$log_path) - } + # We only need the first key + foreach($key in $TestToLog.keys) { + $k = $key + break + } - $JobToLog.Add($job, $log_path) - $TestToLog.Remove($k) + Write-Host "Starting $k" + $log_path = -join ($LogFolder, ($TestToLog.$k)) - ++$count - } + if($Run -ceq "db_test") { + $job = Start-Job -Name $k -ScriptBlock $InvokeTestCase -ArgumentList @($db_test,$k,$log_path) + } else { + [string]$Exe = -Join ($BinariesFolder, $k) + $job = Start-Job -Name $k -ScriptBlock $InvokeTestAsync -ArgumentList @($exe,$log_path) + } - if($JobToLog.Count -lt 1) { - break - } + $JobToLog.Add($job, $log_path) + $TestToLog.Remove($k) - $jobs = @() - foreach($k in $JobToLog.Keys) { $jobs += $k } + ++$count + } - $completed = Wait-Job -Job $jobs -Any - $log = $JobToLog[$completed] - $JobToLog.Remove($completed) + if($JobToLog.Count -lt 1) { + break + } - $message = -join @($completed.Name, " State: ", ($completed.State)) + $jobs = @() + foreach($k in $JobToLog.Keys) { $jobs += $k } - $log_content = @(Get-Content $log) + $completed = Wait-Job -Job $jobs -Any + $log = $JobToLog[$completed] + $JobToLog.Remove($completed) - if($completed.State -ne "Completed") { - $success = $false - Write-Warning $message - $log_content | Write-Warning - } else { - # Scan the log. If we find PASSED and no occurence of FAILED - # then it is a success - [bool]$pass_found = $false - ForEach($l in $log_content) { - - if(($l -match "^\[\s+FAILED") -or - ($l -match "Assertion failed:")) { - $pass_found = $false - break - } + $message = -join @($completed.Name, " State: ", ($completed.State)) - if(($l -match "^\[\s+PASSED") -or - ($l -match " : PASSED$") -or - ($l -match "^PASSED") -or - ($l -match "Passed all tests!") ) { - $pass_found = $true - } - } + $log_content = @(Get-Content $log) - if(!$pass_found) { - $success = $false; + if($completed.State -ne "Completed") { + $success = $false Write-Warning $message $log_content | Write-Warning } else { - Write-Host $message + # Scan the log. If we find PASSED and no occurence of FAILED + # then it is a success + [bool]$pass_found = $false + ForEach($l in $log_content) { + + if(($l -match "^\[\s+FAILED") -or + ($l -match "Assertion failed:")) { + $pass_found = $false + break + } + + if(($l -match "^\[\s+PASSED") -or + ($l -match " : PASSED$") -or + ($l -match "^PASS$") -or # Special c_test case + ($l -match "Passed all tests!") ) { + $pass_found = $true + } + } + + if(!$pass_found) { + $success = $false; + Write-Warning $message + $log_content | Write-Warning + if($AddForRerun) { + MakeAndAdd -token $completed.Name -HashTable $Rerun + } + } else { + Write-Host $message + } } + + # Remove cached job info from the system + # Should be no output + Receive-Job -Job $completed | Out-Null } +} - # Remove cached job info from the system - # Should be no output - Receive-Job -Job $completed | Out-Null +RunJobs -TestToLog $TestsToRun -ConcurrencyVal $Concurrency -AddForRerun $EnableRerun + +if($Rerun.Count -gt 0) { + Write-Host "Rerunning " ($Rerun.Count) " tests sequentially" + $success = $true + $count = 0 + RunJobs -TestToLog $Rerun -ConcurrencyVal $Concurrency -AddForRerun $false } Get-Date + if(!$success) { # This does not succeed killing off jobs quick # So we simply exit From 2ab3e2df2c025cb322973b7d9dcc6311befae5cd Mon Sep 17 00:00:00 2001 From: yuslepukhin Date: Thu, 12 Nov 2015 14:53:19 -0800 Subject: [PATCH 4/4] Fix a build break so tests can run --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ed9fde0b0..9cf1bbaa3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -383,7 +383,7 @@ set(TESTS utilities/geodb/geodb_test.cc utilities/memory/memory_test.cc utilities/merge_operators/string_append/stringappend_test.cc - utilities/options_util_test.cc + utilities/options/options_util_test.cc utilities/redis/redis_lists_test.cc utilities/spatialdb/spatial_db_test.cc utilities/table_properties_collectors/compact_on_deletion_collector_test.cc