From 3277d172bacf2a38343f3393898cc49d79e6208e Mon Sep 17 00:00:00 2001
From: Dmitri Smirnov <dmitrism@microsoft.com>
Date: Thu, 5 Nov 2015 14:03:41 -0800
Subject: [PATCH] Improve concurrency when running tests   PowerShell seems to
 have a hard time when a flood of async tasks is   scheduled at the same time.
   I speculated that WaitForMultipleObjects() in Windows can only take up   to
 64 process handles and if you want to handle more than you should write  
 some additional code which can be sub-optimal. I.e to implement Wait-Job
 -Any.   I decided to test that suggestion and introduced a $Concurrency
 parameter with a default value of 62.   So in the new version the script
 fires up up to $Concurrency value   and wait for anything to complete before
 starting any more processes.   This improved matters greatly. Individual
 tests against ramdrive now   run in 8 minutes and all of the 200+ db_tests
 run in 9 minutes with concurrency   values of 8-16. About 48 is required to
 load a CPU on my box running against HD   but that does not improve running
 times much.

  Other changes include respect -EnableJE for the individual test exes.
  Enforce exclusions for the individual tests.
---
 build_tools/run_ci_db_test.ps1 | 73 ++++++++++++++++++++++------------
 1 file changed, 47 insertions(+), 26 deletions(-)

diff --git a/build_tools/run_ci_db_test.ps1 b/build_tools/run_ci_db_test.ps1
index 5f47f3d87..5cb4ac2c1 100644
--- a/build_tools/run_ci_db_test.ps1
+++ b/build_tools/run_ci_db_test.ps1
@@ -8,7 +8,10 @@ Param(
   [string]$WorkFolder = "",  # Direct tests to use that folder
   [int]$Limit = -1, # -1 means run all otherwise limit for testing purposes
   [string]$Exclude = "", # Expect a comma separated list, no spaces
-  [string]$Run = "db_test"  # Run db_test|tests
+  [string]$Run = "db_test",  # Run db_test|tests
+   # Number of async tasks that would run concurrently. Recommend a number below 64.
+   # However, CPU utlization really depends on the storage media. Recommend ram based disk.
+  [int]$Concurrency = 62
 )
 
 # Folders and commands must be fullpath to run assuming
@@ -122,10 +125,17 @@ function Normalize-DbTests($HashTable) {
 function Discover-TestBinaries($HashTable) {
 
     $Exclusions = @("db_test*", "db_sanity_test*")
-    $p = -join ($BinariesFolder, "*_test*.exe")
+    if($EnableJE) {
+        $p = -join ($BinariesFolder, "*_test_je.exe")
+    } else {
+        $p = -join ($BinariesFolder, "*_test.exe")
+    }
 
     dir -Path $p -Exclude $Exclusions | ForEach-Object {
        $t = ($_.Name) -replace '.exe$', ''
+       if($ExcludeTests.Contains($t)) {
+            continue
+       }
        $test_log = -join ($t, ".log")
        $HashTable.Add($t, $test_log)
     }
@@ -137,6 +147,9 @@ if($Run -ceq "db_test") {
     Normalize-DbTests -HashTable $TestToLog
 } elseif($Run -ceq "tests") {
     Discover-TestBinaries -HashTable $TestToLog
+} else {
+    Write-Warning "Invalid -Run option value"
+    exit 2
 }
 
 
@@ -159,39 +172,47 @@ $JobToLog = @{}
 # Test limiting factor here
 $count = 0
 
-ForEach($k in $TestToLog.keys) {
+[bool]$success = $true;
 
-    Write-Host "Starting $k"
-    $log_path = -join ($LogFolder, ($TestToLog.$k))
+# Wait for all to finish and get the results
+while(($JobToLog.Count -gt 0) -or
+      ($TestToLog.Count -gt 0)) {
 
-    if($Run -ceq "db_test") {
-        $job = Start-Job -Name $k -ScriptBlock $InvokeTestCase -ArgumentList @($db_test,$k,$log_path)
-    } else {
-        [string]$Exe =  -Join ($BinariesFolder, $k)
-        $job = Start-Job -Name $k -ScriptBlock $InvokeTestAsync -ArgumentList @($exe,$log_path)
-    }
+    # Make sure we have maximum concurrent jobs running if anything
+    # and the $Limit either not set or allows to proceed
+    while(($JobToLog.Count -lt $Concurrency) -and
+          (($TestToLog.Count -gt 0) -and
+          (($Limit -lt 0) -or ($count -lt $Limit)))) {
 
-    $JobToLog.Add($job, $log_path)
 
-    # Limiting trial runs
-    if(($Limit -gt 0) -and (++$count -ge $Limit)) {
-         break
-    }
-}
+        # We only need the first key
+        foreach($key in $TestToLog.keys) {
+            $k = $key
+            break
+        }
 
-[bool]$success = $true;
+        Write-Host "Starting $k"
+        $log_path = -join ($LogFolder, ($TestToLog.$k))
 
-# Wait for all to finish and get the results
-while($JobToLog.Count -gt 0) {
+        if($Run -ceq "db_test") {
+          $job = Start-Job -Name $k -ScriptBlock $InvokeTestCase -ArgumentList @($db_test,$k,$log_path)
+        } else {
+          [string]$Exe =  -Join ($BinariesFolder, $k)
+           $job = Start-Job -Name $k -ScriptBlock $InvokeTestAsync -ArgumentList @($exe,$log_path)
+        }
 
-    $jobs = @()
-    foreach($k in $JobToLog.Keys) { $jobs += $k }
+        $JobToLog.Add($job, $log_path)
+        $TestToLog.Remove($k)
 
-<#
-    if(!$success) {
-        break
+        ++$count
     }
-#>
+
+    if($JobToLog.Count -lt 1) {
+      break
+    }
+
+    $jobs = @()
+    foreach($k in $JobToLog.Keys) { $jobs += $k }
 
     $completed = Wait-Job -Job $jobs -Any
     $log = $JobToLog[$completed]