NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Rework test running script.

Browse Source 
Summary:
Rework test running script.
  New options SuiteRun - runs specified executables as google suite
  test cases in parallel.
  Run - this option now runs executables in parallel the same as 'tests'
  RunAll - scans for test executables and attempts to run them all
  as suites except those mentiones in RunOnly (hardcoded in the script)
  or specified either in $ExcludeTestCases $ExcludeTestExe
Closes https://github.com/facebook/rocksdb/pull/2089

Differential Revision: D4832212

Pulled By: yiwu-arbug

fbshipit-source-id: 954990c
main
Dmitri Smirnov 8 years ago committed by Facebook Github Bot
parent d659faad54
commit e5a1372b24
2 changed files with 235 additions and 115 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 5
      appveyor.yml
  2. 325
      build_tools/run_ci_db_test.ps1

5
appveyor.yml
Unescape View File

@ -11,8 +11,5 @@ build:
verbosity: minimal verbosity: minimal
test: test:
test_script: test_script:
- ps: build_tools\run_ci_db_test.ps1 -Run db_basic_test -Concurrency 8 - ps: build_tools\run_ci_db_test.ps1 -SuiteRun db_basic_test,db_test2,db_test,env_basic_test,env_test -Concurrency 8
- ps: build_tools\run_ci_db_test.ps1 -Run db_test2 -Concurrency 8
- ps: build_tools\run_ci_db_test.ps1 -Run db_test -Concurrency 8
- ps: build_tools\run_ci_db_test.ps1 -Run env_test,env_basic_test -Concurrency 2

325
build_tools/run_ci_db_test.ps1
Unescape View File

@ -1,24 +1,50 @@
# This script enables you running RocksDB tests by running # This script enables you running RocksDB tests by running
# All the tests in parallel and utilizing all the cores # All the tests concurrently and utilizing all the cores
# For db_test the script first lists and parses the tests
# and then fires them up in parallel using async PS Job functionality
# Run the script from the enlistment
Param( Param(
[switch]$EnableJE = $false, # Use je executable [switch]$EnableJE = $false, # Look for and use _je executable, append _je to listed exclusions
[switch]$EnableRerun = $false, # Rerun failed tests sequentially at the end [switch]$RunAll = $false, # Will attempt discover all *_test[_je].exe binaries and run all
[string]$WorkFolder = "", # Direct tests to use that folder # of them as Google suites. I.e. It will run test cases concurrently
[int]$Limit = -1, # -1 means run all otherwise limit for testing purposes # except those mentioned as $Run, those will run as individual test cases
[string]$Exclude = "", # Expect a comma separated list, no spaces # And any execlued with $ExcludeExes or $ExcludeCases
[string]$Run = "db_test", # Run db_test|db_test2|tests|testname1,testname2... # It will also not run any individual test cases
# excluded but $ExcludeCasese
[string]$SuiteRun = "", # Split test suites in test cases and run in parallel, not compatible with $RunAll
[string]$Run = "", # Run specified executables in parallel but do not split to test cases
[string]$ExcludeCases = "", # Exclude test cases, expects a comma separated list, no spaces
# Takes effect when $RunAll or $SuiteRun is specified. Must have full
# Test cases name including a group and a parameter if any
[string]$ExcludeExes = "", # Exclude exes from consideration, expects a comma separated list,
# no spaces. Takes effect only when $RunAll is specified
[string]$WorkFolder = "", # Direct tests to use that folder. SSD or Ram drive are better options.
# Number of async tasks that would run concurrently. Recommend a number below 64. # Number of async tasks that would run concurrently. Recommend a number below 64.
# However, CPU utlization really depends on the storage media. Recommend ram based disk. # However, CPU utlization really depends on the storage media. Recommend ram based disk.
# a value of 1 will run everything serially # a value of 1 will run everything serially
[int]$Concurrency = 16 [int]$Concurrency = 8,
[int]$Limit = -1 # -1 means do not limit for test purposes
) )
# Folders and commands must be fullpath to run assuming # Folders and commands must be fullpath to run assuming
# the current folder is at the root of the git enlistment # the current folder is at the root of the git enlistment
Get-Date $StartDate = (Get-Date)
$StartDate
$DebugPreference = "Continue"
# These tests are not google test suites and we should guard
# Against running them as suites
$RunOnly = New-Object System.Collections.Generic.HashSet[string]
$RunOnly.Add("c_test") | Out-Null
$RunOnly.Add("compact_on_deletion_collector_test") | Out-Null
$RunOnly.Add("merge_test") | Out-Null
$RunOnly.Add("stringappend_test") | Out-Null # Apparently incorrectly written
$RunOnly.Add("backupable_db_test") | Out-Null # Disabled
if($RunAll -and $SuiteRun -ne "") {
Write-Error "$RunAll and $SuiteRun are not compatible"
exit 1
}
# If running under Appveyor assume that root # If running under Appveyor assume that root
[string]$Appveyor = $Env:APPVEYOR_BUILD_FOLDER [string]$Appveyor = $Env:APPVEYOR_BUILD_FOLDER
@ -47,39 +73,31 @@ if($WorkFolder -eq "") {
} }
Write-Output "Root: $RootFolder, WorkFolder: $WorkFolder" Write-Output "Root: $RootFolder, WorkFolder: $WorkFolder"
Write-Output "BinariesFolder: $BinariesFolder, LogFolder: $LogFolder"
# Use JEMALLOC executables # Create test directories in the current folder
if($Run -ceq "db_test" -or md -Path $WorkFolder -ErrorAction Ignore | Out-Null
$Run -ceq "db_test2" -or md -Path $LogFolder -ErrorAction Ignore | Out-Null
$Run -ceq "db_basic_test") {
$file_name = $Run
if($EnableJE) { $ExcludeCasesSet = New-Object System.Collections.Generic.HashSet[string]
$file_name += "_je" if($ExcludeCases -ne "") {
Write-Host "ExcludeCases: $ExcludeCases"
$l = $ExcludeCases -split ' '
ForEach($t in $l) {
$ExcludeCasesSet.Add($t) | Out-Null
} }
$file_name += ".exe"
$db_test = -Join ($BinariesFolder, $file_name)
Write-Output "Binaries: $BinariesFolder db_test: $db_test"
} }
$ExcludeExesSet = New-Object System.Collections.Generic.HashSet[string]
#Exclusions that we do not want to run if($ExcludeExes -ne "") {
$ExcludeTests = New-Object System.Collections.Generic.HashSet[string] Write-Host "ExcludeExe: $ExcludeExes"
$l = $ExcludeExes -split ' '
ForEach($t in $l) {
if($Exclude -ne "") { $ExcludeExesSet.Add($t) | Out-Null
Write-Host "Exclude: $Exclude" }
$l = $Exclude -split ' '
ForEach($t in $l) { $ExcludeTests.Add($t) | Out-Null }
} }
# Create test directories in the current folder
md -Path $WorkFolder -ErrorAction Ignore | Out-Null
md -Path $LogFolder -ErrorAction Ignore | Out-Null
# Extract the names of its tests by running db_test with --gtest_list_tests. # Extract the names of its tests by running db_test with --gtest_list_tests.
# This filter removes the "#"-introduced comments, and expands to # This filter removes the "#"-introduced comments, and expands to
@ -98,28 +116,37 @@ md -Path $LogFolder -ErrorAction Ignore | Out-Null
# DBTest.WriteEmptyBatch # DBTest.WriteEmptyBatch
# MultiThreaded/MultiThreadedDBTest.MultiThreaded/0 # MultiThreaded/MultiThreadedDBTest.MultiThreaded/0
# MultiThreaded/MultiThreadedDBTest.MultiThreaded/1 # MultiThreaded/MultiThreadedDBTest.MultiThreaded/1
#
# Output into the parameter in a form TestName -> Log File Name # Output into the parameter in a form TestName -> Log File Name
function Normalize-DbTests($HashTable) { function ExtractTestCases([string]$GTestExe, $HashTable) {
$Tests = @() $Tests = @()
# Run db_test to get a list of tests and store it into $a array # Run db_test to get a list of tests and store it into $a array
&$db_test --gtest_list_tests | tee -Variable Tests | Out-Null &$GTestExe --gtest_list_tests | tee -Variable Tests | Out-Null
# Current group # Current group
$Group="" $Group=""
ForEach( $l in $Tests) { ForEach( $l in $Tests) {
# Trailing dot is a test group
if( $l -match "\.$") { # Leading whitespace is fine
$l = $l -replace '^\s+',''
# but no whitespace any other place
if($l -match "\s+") {
continue
}
# Trailing dot is a test group but no whitespace
elseif ( $l -match "\.$" ) {
$Group = $l $Group = $l
} else { } else {
# Otherwise it is a test name, remove leading space # Otherwise it is a test name, remove leading space
$test = $l -replace '^\s+','' $test = $l
# remove trailing comment if any and create a log name # remove trailing comment if any and create a log name
$test = $test -replace '\s+\#.*','' $test = $test -replace '\s+\#.*',''
$test = "$Group$test" $test = "$Group$test"
if($ExcludeTests.Contains($test)) { if($ExcludeCasesSet.Contains($test)) {
Write-Warning "$test case is excluded"
continue continue
} }
@ -135,60 +162,131 @@ function Normalize-DbTests($HashTable) {
# The function removes trailing .exe siffix if any, # The function removes trailing .exe siffix if any,
# creates a name for the log file # creates a name for the log file
# Then adds the test name if it was not excluded into
# a HashTable in a form of test_name -> log_path
function MakeAndAdd([string]$token, $HashTable) { function MakeAndAdd([string]$token, $HashTable) {
$test_name = $token -replace '.exe$', '' $test_name = $token -replace '.exe$', ''
$log_name = -join ($test_name, ".log") $log_name = -join ($test_name, ".log")
$log_path = -join ($LogFolder, $log_name) $log_path = -join ($LogFolder, $log_name)
if(!$ExcludeTests.Contains($test_name)) {
$HashTable.Add($test_name, $log_path) $HashTable.Add($test_name, $log_path)
} else { }
Write-Warning "Test $test_name is excluded"
# This function takes a list of Suites to run
# Lists all the test cases in each of the suite
# and populates HashOfHashes
# Ordered by suite(exe) @{ Exe = @{ TestCase = LogName }}
function ProcessSuites($ListOfSuites, $HashOfHashes) {
$suite_list = $ListOfSuites
# Problem: if you run --gtest_list_tests on
# a non Google Test executable then it will start executing
# and we will get nowhere
ForEach($suite in $suite_list) {
if($RunOnly.Contains($suite)) {
Write-Warning "$suite is excluded from running as Google test suite"
continue
} }
if($EnableJE) {
$suite += "_je"
}
$Cases = [ordered]@{}
$Cases.Clear()
$suite_exe = -Join ($BinariesFolder, $suite)
ExtractTestCases -GTestExe $suite_exe -HashTable $Cases
if($Cases.Count -gt 0) {
$HashOfHashes.Add($suite, $Cases);
}
}
# Make logs and run
if($CasesToRun.Count -lt 1) {
Write-Error "Failed to extract tests from $SuiteRun"
exit 1
}
} }
# The function scans build\Debug folder to discover # This will contain all test executables to run
# Test executables. It then populates a table with
# Test executable name -> Log file # Hash table that contains all non suite
function Discover-TestBinaries([string]$Pattern, $HashTable) { # Test executable to run
$TestExes = [ordered]@{}
$Exclusions = @("db_test*", "db_sanity_test*", "db_basic_test*") # Check for test exe that are not
# Google Test Suites
# Since this is explicitely mentioned it is not subject
# for exclusions
if($Run -ne "") {
$p = -join ($BinariesFolder, $pattern) $test_list = $Run -split ' '
ForEach($t in $test_list) {
Write-Host "Path: $p" if($EnableJE) {
$t += "_je"
}
dir -Path $p -Exclude $Exclusions | ForEach-Object { MakeAndAdd -token $t -HashTable $TestExes
MakeAndAdd -token ($_.Name) -HashTable $HashTable }
if($TestExes.Count -lt 1) {
Write-Error "Failed to extract tests from $Run"
exit 1
} }
} }
$TestsToRun = [ordered]@{} # Ordered by exe @{ Exe = @{ TestCase = LogName }}
$CasesToRun = [ordered]@{}
if($SuiteRun -ne "") {
$suite_list = $SuiteRun -split ' '
ProcessSuites -ListOfSuites $suite_list -HashOfHashes $CasesToRun
}
if($Run -ceq "db_test" -or if($RunAll) {
$Run -ceq "db_test2" -or # Discover all the test binaries
$Run -ceq "db_basic_test") {
Normalize-DbTests -HashTable $TestsToRun
} elseif($Run -ceq "tests") {
if($EnableJE) { if($EnableJE) {
$pattern = "*_test_je.exe" $pattern = "*_test_je.exe"
} else { } else {
$pattern = "*_test.exe" $pattern = "*_test.exe"
} }
Discover-TestBinaries -Pattern $pattern -HashTable $TestsToRun
} else {
$test_list = $Run -split ' '
ForEach($t in $test_list) { $search_path = -join ($BinariesFolder, $pattern)
MakeAndAdd -token $t -HashTable $TestsToRun Write-Host "Binaries Search Path: $search_path"
$ListOfExe = @()
dir -Path $search_path | ForEach-Object {
$ListOfExe += ($_.Name)
} }
}
$NumTestsToStart = $TestsToRun.Count # Exclude those in RunOnly from running as suites
if($Limit -ge 0 -and $NumTestsToStart -gt $Limit) { $ListOfSuites = @()
$NumTestsToStart = $Limit ForEach($e in $ListOfExe) {
$e = $e -replace '.exe$', ''
$bare_name = $e -replace '_je$', ''
if($ExcludeExesSet.Contains($bare_name)) {
Write-Warning "Test $e is excluded"
continue
}
if($RunOnly.Contains($e)) {
MakeAndAdd -token $e -HashTable $TestExes
} else {
$ListOfSuites += $bare_name
}
}
ProcessSuites -ListOfSuites $ListOfSuites -HashOfHashes $CasesToRun
} }
Write-Host "Attempting to start: $NumTestsToStart tests" Write-Host "Attempting to start: $NumTestsToStart tests"
# Invoke a test with a filter and redirect all output # Invoke a test with a filter and redirect all output
@ -205,13 +303,13 @@ $InvokeTestAsync = {
# Hash that contains tests to rerun if any failed # Hash that contains tests to rerun if any failed
# Those tests will be rerun sequentially # Those tests will be rerun sequentially
$Rerun = [ordered]@{} # $Rerun = [ordered]@{}
# Test limiting factor here # Test limiting factor here
$count = 0 [int]$count = 0
# Overall status # Overall status
[bool]$success = $true; [bool]$success = $true;
function RunJobs($TestToLog, [int]$ConcurrencyVal, [bool]$AddForRerun) function RunJobs($Suites, $TestCmds, [int]$ConcurrencyVal)
{ {
# Array to wait for any of the running jobs # Array to wait for any of the running jobs
$jobs = @() $jobs = @()
@ -220,39 +318,67 @@ function RunJobs($TestToLog, [int]$ConcurrencyVal, [bool]$AddForRerun)
# Wait for all to finish and get the results # Wait for all to finish and get the results
while(($JobToLog.Count -gt 0) -or while(($JobToLog.Count -gt 0) -or
($TestToLog.Count -gt 0)) { ($TestCmds.Count -gt 0) -or
($Suites.Count -gt 0)) {
# Make sure we have maximum concurrent jobs running if anything # Make sure we have maximum concurrent jobs running if anything
# and the $Limit either not set or allows to proceed # and the $Limit either not set or allows to proceed
while(($JobToLog.Count -lt $ConcurrencyVal) -and while(($JobToLog.Count -lt $ConcurrencyVal) -and
(($TestToLog.Count -gt 0) -and ((($TestCmds.Count -gt 0) -or ($Suites.Count -gt 0)) -and
(($Limit -lt 0) -or ($count -lt $Limit)))) { (($Limit -lt 0) -or ($count -lt $Limit)))) {
# We always favore suites to run if available
[string]$exe_name = ""
[string]$log_path = ""
$Cases = @{}
# We only need the first key if($Suites.Count -gt 0) {
foreach($key in $TestToLog.keys) { # Will teh first one
$k = $key ForEach($e in $Suites.Keys) {
$exe_name = $e
$Cases = $Suites[$e]
break
}
[string]$test_case = ""
[string]$log_path = ""
ForEach($c in $Cases.Keys) {
$test_case = $c
$log_path = $Cases[$c]
break break
} }
Write-Host "Starting $k" Write-Host "Starting $exe_name::$test_case"
$log_path = ($TestToLog.$k) [string]$Exe = -Join ($BinariesFolder, $exe_name)
$job = Start-Job -Name "$exe_name::$test_case" -ArgumentList @($Exe,$test_case,$log_path) -ScriptBlock $InvokeTestCase
$JobToLog.Add($job, $log_path)
if($Run -ceq "db_test" -or $Cases.Remove($test_case)
$Run -ceq "db_test2" -or if($Cases.Count -lt 1) {
$Run -ceq "db_basic_test") { $Suites.Remove($exe_name)
$job = Start-Job -Name $k -ScriptBlock $InvokeTestCase -ArgumentList @($db_test,$k,$log_path) }
} else {
[string]$Exe = -Join ($BinariesFolder, $k) } elseif ($TestCmds.Count -gt 0) {
$job = Start-Job -Name $k -ScriptBlock $InvokeTestAsync -ArgumentList @($exe,$log_path)
ForEach($e in $TestCmds.Keys) {
$exe_name = $e
$log_path = $TestCmds[$e]
break
} }
[string]$Exe = -Join ($BinariesFolder, $exe_name)
$job = Start-Job -Name $exe_name -ScriptBlock $InvokeTestAsync -ArgumentList @($Exe,$log_path)
$JobToLog.Add($job, $log_path) $JobToLog.Add($job, $log_path)
$TestToLog.Remove($k)
++$count $TestCmds.Remove($exe_name)
} else {
Write-Error "In the job loop but nothing to run"
exit 1
} }
++$count
} # End of Job starting loop
if($JobToLog.Count -lt 1) { if($JobToLog.Count -lt 1) {
break break
} }
@ -296,9 +422,6 @@ function RunJobs($TestToLog, [int]$ConcurrencyVal, [bool]$AddForRerun)
$success = $false; $success = $false;
Write-Warning $message Write-Warning $message
$log_content | Write-Warning $log_content | Write-Warning
if($AddForRerun) {
$Rerun.Add($completed.Name, $log)
}
} else { } else {
Write-Host $message Write-Host $message
} }
@ -310,16 +433,14 @@ function RunJobs($TestToLog, [int]$ConcurrencyVal, [bool]$AddForRerun)
} }
} }
RunJobs -TestToLog $TestsToRun -ConcurrencyVal $Concurrency -AddForRerun $EnableRerun RunJobs -Suites $CasesToRun -TestCmds $TestExes -ConcurrencyVal $Concurrency
if($Rerun.Count -gt 0) { $EndDate = (Get-Date)
Write-Host "Rerunning " ($Rerun.Count) " tests sequentially"
$success = $true
$count = 0
RunJobs -TestToLog $Rerun -ConcurrencyVal 1 -AddForRerun $false
}
Get-Date New-TimeSpan -Start $StartDate -End $EndDate |
ForEach-Object {
"Elapsed time: {0:g}" -f $_
}
if(!$success) { if(!$success) {
@ -327,7 +448,9 @@ if(!$success) {
# So we simply exit # So we simply exit
# Remove-Job -Job $jobs -Force # Remove-Job -Job $jobs -Force
# indicate failure using this exit code # indicate failure using this exit code
exit 12345 exit 1
} }
exit 0

Write Preview
Loading…
Cancel
Save