Rework test running script.

Summary:
Rework test running script.
  New options SuiteRun - runs specified executables as google suite
  test cases in parallel.
  Run - this option now runs executables in parallel the same as 'tests'
  RunAll - scans for test executables and attempts to run them all
  as suites except those mentiones in RunOnly (hardcoded in the script)
  or specified either in $ExcludeTestCases $ExcludeTestExe
Closes https://github.com/facebook/rocksdb/pull/2089

Differential Revision: D4832212

Pulled By: yiwu-arbug

fbshipit-source-id: 954990c

appveyor.yml

@@ -11,8 +11,5 @@ build:
   verbosity: minimal
 test:
 test_script:
-- ps: build_tools\run_ci_db_test.ps1 -Run db_basic_test -Concurrency 8
+- ps: build_tools\run_ci_db_test.ps1 -SuiteRun db_basic_test,db_test2,db_test,env_basic_test,env_test -Concurrency 8
-- ps: build_tools\run_ci_db_test.ps1 -Run db_test2 -Concurrency 8
-- ps: build_tools\run_ci_db_test.ps1 -Run db_test -Concurrency 8
-- ps: build_tools\run_ci_db_test.ps1 -Run env_test,env_basic_test -Concurrency 2
 

build_tools/run_ci_db_test.ps1

@@ -1,24 +1,50 @@
 # This script enables you running RocksDB tests by running
-# All the tests in parallel and utilizing all the cores
+# All the tests concurrently and utilizing all the cores
-# For db_test the script first lists and parses the tests
-# and then fires them up in parallel using async PS Job functionality
-# Run the script from the enlistment
 Param(
-  [switch]$EnableJE = $false,  # Use je executable
+  [switch]$EnableJE = $false,  # Look for and use _je executable, append _je to listed exclusions
-  [switch]$EnableRerun = $false, # Rerun failed tests sequentially at the end
+  [switch]$RunAll = $false,    # Will attempt discover all *_test[_je].exe binaries and run all
-  [string]$WorkFolder = "",  # Direct tests to use that folder
+                               # of them as Google suites. I.e. It will run test cases concurrently
-  [int]$Limit = -1, # -1 means run all otherwise limit for testing purposes
+                               # except those mentioned as $Run, those will run as individual test cases
-  [string]$Exclude = "", # Expect a comma separated list, no spaces
+                               # And any execlued with $ExcludeExes or $ExcludeCases
-  [string]$Run = "db_test",  # Run db_test|db_test2|tests|testname1,testname2...
+                               # It will also not run any individual test cases
+                               # excluded but $ExcludeCasese
+  [string]$SuiteRun = "",      # Split test suites in test cases and run in parallel, not compatible with $RunAll
+  [string]$Run = "",           # Run specified executables in parallel but do not split to test cases
+  [string]$ExcludeCases = "",  # Exclude test cases, expects a comma separated list, no spaces
+                               # Takes effect when $RunAll or $SuiteRun is specified. Must have full
+                               # Test cases name including a group and a parameter if any
+  [string]$ExcludeExes = "",   # Exclude exes from consideration, expects a comma separated list,
+                               # no spaces. Takes effect only when $RunAll is specified
+  [string]$WorkFolder = "",    # Direct tests to use that folder. SSD or Ram drive are better options.
    # Number of async tasks that would run concurrently. Recommend a number below 64.
    # However, CPU utlization really depends on the storage media. Recommend ram based disk.
    # a value of 1 will run everything serially
-  [int]$Concurrency = 16
+  [int]$Concurrency = 8,
+  [int]$Limit = -1 # -1 means do not limit for test purposes
 )
 
 # Folders and commands must be fullpath to run assuming
 # the current folder is at the root of the git enlistment
-Get-Date
+$StartDate = (Get-Date)
+$StartDate
+
+
+$DebugPreference = "Continue"
+
+# These tests are not google test suites and we should guard
+# Against running them as suites
+$RunOnly = New-Object System.Collections.Generic.HashSet[string]
+$RunOnly.Add("c_test") | Out-Null
+$RunOnly.Add("compact_on_deletion_collector_test") | Out-Null
+$RunOnly.Add("merge_test") | Out-Null
+$RunOnly.Add("stringappend_test") | Out-Null # Apparently incorrectly written
+$RunOnly.Add("backupable_db_test") | Out-Null # Disabled
+
+
+if($RunAll -and $SuiteRun -ne "") {
+    Write-Error "$RunAll and $SuiteRun are not compatible"
+    exit 1
+}
 
 # If running under Appveyor assume that root
 [string]$Appveyor = $Env:APPVEYOR_BUILD_FOLDER
@@ -47,39 +73,31 @@ if($WorkFolder -eq "") {
 }
 
 Write-Output "Root: $RootFolder, WorkFolder: $WorkFolder"
+Write-Output "BinariesFolder: $BinariesFolder, LogFolder: $LogFolder"
 
-# Use JEMALLOC executables
+# Create test directories in the current folder
-if($Run -ceq "db_test" -or
+md -Path $WorkFolder -ErrorAction Ignore | Out-Null
-   $Run -ceq "db_test2" -or
+md -Path $LogFolder -ErrorAction Ignore | Out-Null
-   $Run -ceq "db_basic_test") {
-
-   $file_name = $Run
-
-   if($EnableJE) {
-     $file_name += "_je"
-   }
-
-   $file_name += ".exe"
 
-   $db_test = -Join ($BinariesFolder, $file_name)
 
-   Write-Output "Binaries: $BinariesFolder db_test: $db_test"
+$ExcludeCasesSet = New-Object System.Collections.Generic.HashSet[string]
+if($ExcludeCases -ne "") {
+    Write-Host "ExcludeCases: $ExcludeCases"
+    $l = $ExcludeCases -split ' '
+    ForEach($t in $l) { 
+      $ExcludeCasesSet.Add($t) | Out-Null
+    }
 }
 
-
+$ExcludeExesSet = New-Object System.Collections.Generic.HashSet[string]
-#Exclusions that we do not want to run
+if($ExcludeExes -ne "") {
-$ExcludeTests = New-Object System.Collections.Generic.HashSet[string]
+    Write-Host "ExcludeExe: $ExcludeExes"
-
+    $l = $ExcludeExes -split ' '
-
+    ForEach($t in $l) { 
-if($Exclude -ne "") {
+      $ExcludeExesSet.Add($t) | Out-Null
-    Write-Host "Exclude: $Exclude"
+    }
-    $l = $Exclude -split ' '
-    ForEach($t in $l) { $ExcludeTests.Add($t) | Out-Null }
 }
 
-# Create test directories in the current folder
-md -Path $WorkFolder -ErrorAction Ignore | Out-Null
-md -Path $LogFolder -ErrorAction Ignore | Out-Null
 
 # Extract the names of its tests by running db_test with --gtest_list_tests.
 # This filter removes the "#"-introduced comments, and expands to
@@ -98,28 +116,37 @@ md -Path $LogFolder -ErrorAction Ignore | Out-Null
 #   DBTest.WriteEmptyBatch
 #   MultiThreaded/MultiThreadedDBTest.MultiThreaded/0
 #   MultiThreaded/MultiThreadedDBTest.MultiThreaded/1
+#
 # Output into the parameter in a form TestName -> Log File Name
-function Normalize-DbTests($HashTable) {
+function ExtractTestCases([string]$GTestExe, $HashTable) {
 
     $Tests = @()
 # Run db_test to get a list of tests and store it into $a array
-    &$db_test --gtest_list_tests | tee -Variable Tests | Out-Null
+    &$GTestExe --gtest_list_tests | tee -Variable Tests | Out-Null
 
     # Current group
     $Group=""
 
     ForEach( $l in $Tests) {
-      # Trailing dot is a test group
+
-      if( $l -match "\.$") {
+      # Leading whitespace is fine
+      $l = $l -replace '^\s+',''
+      # but no whitespace any other place
+      if($l -match "\s+") {
+        continue
+      }
+      # Trailing dot is a test group but no whitespace
+      elseif ( $l -match "\.$" ) {
         $Group = $l
       }  else {
         # Otherwise it is a test name, remove leading space
-        $test = $l -replace '^\s+',''
+        $test = $l
         # remove trailing comment if any and create a log name
         $test = $test -replace '\s+\#.*',''
         $test = "$Group$test"
 
-        if($ExcludeTests.Contains($test)) {
+        if($ExcludeCasesSet.Contains($test)) {
+            Write-Warning "$test case is excluded"
             continue
         }
 
@@ -135,60 +162,131 @@ function Normalize-DbTests($HashTable) {
 
 # The function removes trailing .exe siffix if any,
 # creates a name for the log file
+# Then adds the test name if it was not excluded into
+# a HashTable in a form of test_name -> log_path
 function MakeAndAdd([string]$token, $HashTable) {
+
     $test_name = $token -replace '.exe$', ''
     $log_name =  -join ($test_name, ".log")
     $log_path = -join ($LogFolder, $log_name)
-    if(!$ExcludeTests.Contains($test_name)) {
+    $HashTable.Add($test_name, $log_path)
-        $HashTable.Add($test_name, $log_path)
-    } else {
-        Write-Warning "Test $test_name is excluded"
-    }
 }
 
-# The function scans build\Debug folder to discover
+# This function takes a list of Suites to run
-# Test executables. It then populates a table with
+# Lists all the test cases in each of the suite
-# Test executable name -> Log file
+# and populates HashOfHashes
-function Discover-TestBinaries([string]$Pattern, $HashTable) {
+# Ordered by suite(exe) @{ Exe = @{ TestCase = LogName }}
+function ProcessSuites($ListOfSuites, $HashOfHashes) {
+
+  $suite_list = $ListOfSuites
+  # Problem: if you run --gtest_list_tests on
+  # a non Google Test executable then it will start executing
+  # and we will get nowhere
+  ForEach($suite in $suite_list) {
+
+    if($RunOnly.Contains($suite)) {
+      Write-Warning "$suite is excluded from running as Google test suite"
+      continue
+    }
 
-    $Exclusions = @("db_test*", "db_sanity_test*", "db_basic_test*")
+    if($EnableJE) {
+      $suite += "_je"
+    }
 
-    $p = -join ($BinariesFolder, $pattern)
+    $Cases = [ordered]@{}
+    $Cases.Clear()
+    $suite_exe = -Join ($BinariesFolder, $suite)
+    ExtractTestCases -GTestExe $suite_exe -HashTable $Cases
+    if($Cases.Count -gt 0) {
+      $HashOfHashes.Add($suite, $Cases);
+    }
+  }
 
-    Write-Host "Path: $p"
+  # Make logs and run
+  if($CasesToRun.Count -lt 1) {
+     Write-Error "Failed to extract tests from $SuiteRun"
+     exit 1
+  }
 
-    dir -Path $p -Exclude $Exclusions | ForEach-Object {
-       MakeAndAdd -token ($_.Name) -HashTable $HashTable
-    }
 }
 
-$TestsToRun = [ordered]@{}
+# This will contain all test executables to run
+
+# Hash table that contains all non suite
+# Test executable to run
+$TestExes = [ordered]@{}
+
+# Check for test exe that are not
+# Google Test Suites
+# Since this is explicitely mentioned it is not subject
+# for exclusions
+if($Run -ne "") {
+
+  $test_list = $Run -split ' '
+
+  ForEach($t in $test_list) {
 
-if($Run -ceq "db_test" -or
-   $Run -ceq "db_test2" -or
-   $Run -ceq "db_basic_test") {
-    Normalize-DbTests -HashTable $TestsToRun
-} elseif($Run -ceq "tests") {
     if($EnableJE) {
-        $pattern = "*_test_je.exe"
+      $t += "_je"
-    } else {
-        $pattern = "*_test.exe"
     }
-    Discover-TestBinaries -Pattern $pattern -HashTable $TestsToRun
-} else {
 
-    $test_list = $Run -split ' '
+    MakeAndAdd -token $t -HashTable $TestExes
+  }
 
-    ForEach($t in $test_list) {
+  if($TestExes.Count -lt 1) {
-       MakeAndAdd -token $t -HashTable $TestsToRun
+     Write-Error "Failed to extract tests from $Run"
-    }
+     exit 1
+  }
 }
 
-$NumTestsToStart = $TestsToRun.Count
+# Ordered by exe @{ Exe = @{ TestCase = LogName }}
-if($Limit -ge 0 -and $NumTestsToStart -gt $Limit) {
+$CasesToRun = [ordered]@{}
-    $NumTestsToStart = $Limit
+
+if($SuiteRun -ne "") {
+  $suite_list = $SuiteRun -split ' '
+  ProcessSuites -ListOfSuites $suite_list -HashOfHashes $CasesToRun
+}
+
+if($RunAll) {
+# Discover all the test binaries
+  if($EnableJE) {
+    $pattern = "*_test_je.exe"
+  } else {
+    $pattern = "*_test.exe"
+  }
+
+
+  $search_path = -join ($BinariesFolder, $pattern)
+  Write-Host "Binaries Search Path: $search_path"
+
+  $ListOfExe = @()
+  dir -Path $search_path | ForEach-Object {
+     $ListOfExe += ($_.Name)     
+  }
+
+  # Exclude those in RunOnly from running as suites
+  $ListOfSuites = @()
+  ForEach($e in $ListOfExe) {
+
+    $e = $e -replace '.exe$', ''
+    $bare_name = $e -replace '_je$', ''
+
+    if($ExcludeExesSet.Contains($bare_name)) {
+      Write-Warning "Test $e is excluded"
+      continue
+    }
+
+    if($RunOnly.Contains($e)) {
+      MakeAndAdd -token $e -HashTable $TestExes
+    } else {
+      $ListOfSuites += $bare_name
+    }
+  }
+
+  ProcessSuites -ListOfSuites $ListOfSuites -HashOfHashes $CasesToRun
 }
 
+
 Write-Host "Attempting to start: $NumTestsToStart tests"
 
 # Invoke a test with a filter and redirect all output
@@ -205,13 +303,13 @@ $InvokeTestAsync = {
 
 # Hash that contains tests to rerun if any failed
 # Those tests will be rerun sequentially
-$Rerun = [ordered]@{}
+# $Rerun = [ordered]@{}
 # Test limiting factor here
-$count = 0
+[int]$count = 0
 # Overall status
 [bool]$success = $true;
 
-function RunJobs($TestToLog, [int]$ConcurrencyVal, [bool]$AddForRerun)
+function RunJobs($Suites, $TestCmds, [int]$ConcurrencyVal)
 {
     # Array to wait for any of the running jobs
     $jobs = @()
@@ -220,38 +318,66 @@ function RunJobs($TestToLog, [int]$ConcurrencyVal, [bool]$AddForRerun)
 
     # Wait for all to finish and get the results
     while(($JobToLog.Count -gt 0) -or
-          ($TestToLog.Count -gt 0)) {
+          ($TestCmds.Count -gt 0) -or 
+           ($Suites.Count -gt 0)) {
 
         # Make sure we have maximum concurrent jobs running if anything
         # and the $Limit either not set or allows to proceed
         while(($JobToLog.Count -lt $ConcurrencyVal) -and
-              (($TestToLog.Count -gt 0) -and
+              ((($TestCmds.Count -gt 0) -or ($Suites.Count -gt 0)) -and
               (($Limit -lt 0) -or ($count -lt $Limit)))) {
 
+            # We always favore suites to run if available
+            [string]$exe_name = ""
+            [string]$log_path = ""
+            $Cases = @{}
 
-            # We only need the first key
+            if($Suites.Count -gt 0) {
-            foreach($key in $TestToLog.keys) {
+              # Will teh first one
-                $k = $key
+              ForEach($e in $Suites.Keys) {
+                $exe_name = $e
+                $Cases = $Suites[$e]
                 break
-            }
+              }
-
+              [string]$test_case = ""
-            Write-Host "Starting $k"
+              [string]$log_path = ""
-            $log_path = ($TestToLog.$k)
+              ForEach($c in $Cases.Keys) {
+                 $test_case = $c
+                 $log_path = $Cases[$c]
+                 break
+              }
+
+              Write-Host "Starting $exe_name::$test_case"
+              [string]$Exe =  -Join ($BinariesFolder, $exe_name)
+              $job = Start-Job -Name "$exe_name::$test_case" -ArgumentList @($Exe,$test_case,$log_path) -ScriptBlock $InvokeTestCase
+              $JobToLog.Add($job, $log_path)
+
+              $Cases.Remove($test_case)
+              if($Cases.Count -lt 1) {
+                $Suites.Remove($exe_name)
+              }
+
+            } elseif ($TestCmds.Count -gt 0) {
+
+               ForEach($e in $TestCmds.Keys) {
+                 $exe_name = $e
+                 $log_path = $TestCmds[$e]
+                 break
+               }
+
+              [string]$Exe =  -Join ($BinariesFolder, $exe_name)
+              $job = Start-Job -Name $exe_name -ScriptBlock $InvokeTestAsync -ArgumentList @($Exe,$log_path)
+              $JobToLog.Add($job, $log_path)
+
+              $TestCmds.Remove($exe_name)
 
-            if($Run -ceq "db_test" -or
-               $Run -ceq "db_test2" -or
-               $Run -ceq "db_basic_test") {
-              $job = Start-Job -Name $k -ScriptBlock $InvokeTestCase -ArgumentList @($db_test,$k,$log_path)
             } else {
-              [string]$Exe =  -Join ($BinariesFolder, $k)
+                Write-Error "In the job loop but nothing to run"
-               $job = Start-Job -Name $k -ScriptBlock $InvokeTestAsync -ArgumentList @($exe,$log_path)
+                exit 1
             }
 
-            $JobToLog.Add($job, $log_path)
-            $TestToLog.Remove($k)
-
             ++$count
-        }
+        } # End of Job starting loop
 
         if($JobToLog.Count -lt 1) {
           break
@@ -296,9 +422,6 @@ function RunJobs($TestToLog, [int]$ConcurrencyVal, [bool]$AddForRerun)
                 $success = $false;
                 Write-Warning $message
                 $log_content | Write-Warning
-                if($AddForRerun) {
-                    $Rerun.Add($completed.Name, $log)
-                }
             } else {
                 Write-Host $message
             }
@@ -310,16 +433,14 @@ function RunJobs($TestToLog, [int]$ConcurrencyVal, [bool]$AddForRerun)
     }
 }
 
-RunJobs -TestToLog $TestsToRun -ConcurrencyVal $Concurrency -AddForRerun $EnableRerun
+RunJobs -Suites $CasesToRun -TestCmds $TestExes -ConcurrencyVal $Concurrency
 
-if($Rerun.Count -gt 0) {
+$EndDate = (Get-Date)
-    Write-Host "Rerunning " ($Rerun.Count) " tests sequentially"
-    $success = $true
-    $count = 0
-    RunJobs -TestToLog $Rerun -ConcurrencyVal 1 -AddForRerun $false
-}
 
-Get-Date
+New-TimeSpan -Start $StartDate -End $EndDate | 
+  ForEach-Object { 
+    "Elapsed time: {0:g}" -f $_
+  }
 
 
 if(!$success) {
@@ -327,7 +448,9 @@ if(!$success) {
 # So we simply exit
 #    Remove-Job -Job $jobs -Force
 # indicate failure using this exit code
-    exit 12345
+    exit 1
  }
 
+ exit 0
+