Fix crashes, address test issues and adjust windows test script

Summary:
Add per-exe execution capability
  Add fix parsing of groups/tests
  Add timer test exclusion

 Fix unit tests
  Ifdef threadpool specific tests that do not pass on Vista threadpool.
  Remove spurious outout from prefix_test so test case listing works
  properly.
  Fix not using standard test directories results in file creation errors
  in sst_dump_test.

  BlobDb fixes:
    In C++ end() iterators can not be dereferenced. They are not valid.
	When deleting blob_db_ set it to nullptr before any other code executes.
	Not fixed:. On Windows you can not delete a file while it is open.
	[ RUN      ] BlobDBTest.ReadWhileGC
	d:\dev\rocksdb\rocksdb\utilities\blob_db\blob_db_test.cc(75): error: DestroyBlobDB(dbname_, options, bdb_options)
	IO error: Failed to delete: d:/mnt/db\testrocksdb-17444/blob_db_test/blob_dir/000001.blob: Permission denied
	d:\dev\rocksdb\rocksdb\utilities\blob_db\blob_db_test.cc(75): error: DestroyBlobDB(dbname_, options, bdb_options)
	IO error: Failed to delete: d:/mnt/db\testrocksdb-17444/blob_db_test/blob_dir/000001.blob: Permission denied

  write_batch
    Should not call front() if there is a chance the container is empty
Closes https://github.com/facebook/rocksdb/pull/3152

Differential Revision: D6293274

Pulled By: sagar0

fbshipit-source-id: 318c3717c22087fae13b18715dffb24565dbd956

build_tools/run_ci_db_test.ps1

@@ -1,13 +1,16 @@
 # This script enables you running RocksDB tests by running
 # All the tests concurrently and utilizing all the cores
 Param(
-  [switch]$EnableJE = $false,  # Look for and use _je executable, append _je to listed exclusions
+  [switch]$EnableJE = $false,  # Look for and use test executable, append _je to listed exclusions
   [switch]$RunAll = $false,    # Will attempt discover all *_test[_je].exe binaries and run all
                                # of them as Google suites. I.e. It will run test cases concurrently
                                # except those mentioned as $Run, those will run as individual test cases
                                # And any execlued with $ExcludeExes or $ExcludeCases
                                # It will also not run any individual test cases
                                # excluded but $ExcludeCasese
+  [switch]$RunAllExe = $false, # Look for and use test exdcutables, append _je to exclusions automatically
+                               # It will attempt to run them in parallel w/o breaking them up on individual
+                               # test cases. Those listed with $ExcludeExes will be excluded
   [string]$SuiteRun = "",      # Split test suites in test cases and run in parallel, not compatible with $RunAll
   [string]$Run = "",           # Run specified executables in parallel but do not split to test cases
   [string]$ExcludeCases = "",  # Exclude test cases, expects a comma separated list, no spaces
@@ -39,13 +42,18 @@ $RunOnly.Add("compact_on_deletion_collector_test") | Out-Null
 $RunOnly.Add("merge_test") | Out-Null
 $RunOnly.Add("stringappend_test") | Out-Null # Apparently incorrectly written
 $RunOnly.Add("backupable_db_test") | Out-Null # Disabled
-
+$RunOnly.Add("timer_queue_test") | Out-Null # Not a gtest
 
 if($RunAll -and $SuiteRun -ne "") {
     Write-Error "$RunAll and $SuiteRun are not compatible"
     exit 1
 }
 
+if($RunAllExe -and $Run -ne "") {
+    Write-Error "$RunAllExe and $Run are not compatible"
+    exit 1
+}
+
 # If running under Appveyor assume that root
 [string]$Appveyor = $Env:APPVEYOR_BUILD_FOLDER
 if($Appveyor -ne "") {
@@ -131,12 +139,8 @@ function ExtractTestCases([string]$GTestExe, $HashTable) {
 
       # Leading whitespace is fine
       $l = $l -replace '^\s+',''
-      # but no whitespace any other place
-      if($l -match "\s+") {
-        continue
-      }
       # Trailing dot is a test group but no whitespace
-      elseif ( $l -match "\.$" ) {
+      if ($l -match "\.$" -and $l -notmatch "\s+") {
         $Group = $l
       }  else {
         # Otherwise it is a test name, remove leading space
@@ -223,13 +227,11 @@ $TestExes = [ordered]@{}
 if($Run -ne "") {
 
   $test_list = $Run -split ' '
-
   ForEach($t in $test_list) {
 
     if($EnableJE) {
       $t += "_je"
     }
-
     MakeAndAdd -token $t -HashTable $TestExes
   }
 
@@ -237,6 +239,38 @@ if($Run -ne "") {
      Write-Error "Failed to extract tests from $Run"
      exit 1
   }
+} elseif($RunAllExe) {
+  # Discover all the test binaries
+  if($EnableJE) {
+    $pattern = "*_test_je.exe"
+  } else {
+    $pattern = "*_test.exe"
+  }
+
+  $search_path = -join ($BinariesFolder, $pattern)
+  Write-Host "Binaries Search Path: $search_path"
+
+  $DiscoveredExe = @()
+  dir -Path $search_path | ForEach-Object {
+     $DiscoveredExe += ($_.Name)     
+  }
+
+  # Remove exclusions
+  ForEach($e in $DiscoveredExe) {
+    $e = $e -replace '.exe$', ''
+    $bare_name = $e -replace '_je$', ''
+
+    if($ExcludeExesSet.Contains($bare_name)) {
+      Write-Warning "Test $e is excluded"
+      continue
+    }
+    MakeAndAdd -token $e -HashTable $TestExes
+  }
+
+  if($TestExes.Count -lt 1) {
+     Write-Error "Failed to discover test executables"
+     exit 1
+  }
 }
 
 # Ordered by exe @{ Exe = @{ TestCase = LogName }}
@@ -245,9 +279,7 @@ $CasesToRun = [ordered]@{}
 if($SuiteRun -ne "") {
   $suite_list = $SuiteRun -split ' '
   ProcessSuites -ListOfSuites $suite_list -HashOfHashes $CasesToRun
-}
-
-if($RunAll) {
+} elseif ($RunAll) {
 # Discover all the test binaries
   if($EnableJE) {
     $pattern = "*_test_je.exe"
@@ -255,7 +287,6 @@ if($RunAll) {
     $pattern = "*_test.exe"
   }
 
-
   $search_path = -join ($BinariesFolder, $pattern)
   Write-Host "Binaries Search Path: $search_path"
 
@@ -287,8 +318,6 @@ if($RunAll) {
 }
 
 
-Write-Host "Attempting to start: $NumTestsToStart tests"
-
 # Invoke a test with a filter and redirect all output
 $InvokeTestCase = {
     param($exe, $test, $log);
@@ -365,6 +394,7 @@ function RunJobs($Suites, $TestCmds, [int]$ConcurrencyVal)
                  break
                }
 
+              Write-Host "Starting $exe_name"
               [string]$Exe =  -Join ($BinariesFolder, $exe_name)
               $job = Start-Job -Name $exe_name -ScriptBlock $InvokeTestAsync -ArgumentList @($Exe,$log_path)
               $JobToLog.Add($job, $log_path)

db/prefix_test.cc

@@ -879,8 +879,6 @@ TEST_F(PrefixTest, PrefixSeekModePrev3) {
 int main(int argc, char** argv) {
   ::testing::InitGoogleTest(&argc, argv);
   ParseCommandLineFlags(&argc, &argv, true);
-  std::cout << kDbName << "\n";
-
   return RUN_ALL_TESTS();
 }
 

env/env_test.cc

@@ -171,6 +171,11 @@ TEST_P(EnvPosixTestWithParam, UnSchedule) {
   WaitThreadPoolsEmpty();
 }
 
+// This tests assumes that the last scheduled
+// task will run last. In fact, in the allotted
+// sleeping time nothing may actually run or they may
+// run in any order. The purpose of the test is unclear.
+#ifndef OS_WIN
 TEST_P(EnvPosixTestWithParam, RunMany) {
   std::atomic<int> last_id(0);
 
@@ -203,6 +208,7 @@ TEST_P(EnvPosixTestWithParam, RunMany) {
   ASSERT_EQ(4, cur);
   WaitThreadPoolsEmpty();
 }
+#endif
 
 struct State {
   port::Mutex mu;

utilities/blob_db/blob_db_impl.cc

@@ -288,13 +288,13 @@ Status BlobDBImpl::OpenAllFiles() {
                  "BlobDir files path: %s count: %d min: %" PRIu64
                  " max: %" PRIu64,
                  blob_dir_.c_str(), static_cast<int>(file_nums.size()),
-                 (file_nums.empty()) ? -1 : (file_nums.begin())->first,
-                 (file_nums.empty()) ? -1 : (file_nums.end())->first);
+                 (file_nums.empty()) ? -1 : file_nums.cbegin()->first,
+                 (file_nums.empty()) ? -1 : file_nums.crbegin()->first);
 
   if (!file_nums.empty())
     next_file_number_.store((file_nums.rbegin())->first + 1);
 
-  for (auto f_iter : file_nums) {
+  for (auto& f_iter : file_nums) {
     std::string bfpath = BlobFileName(blob_dir_, f_iter.first);
     uint64_t size_bytes;
     Status s1 = env_->GetFileSize(bfpath, &size_bytes);

utilities/blob_db/blob_db_test.cc

@@ -71,8 +71,8 @@ class BlobDBTest : public testing::Test {
       Options options = blob_db_->GetOptions();
       BlobDBOptions bdb_options = blob_db_->GetBlobDBOptions();
       delete blob_db_;
-      ASSERT_OK(DestroyBlobDB(dbname_, options, bdb_options));
       blob_db_ = nullptr;
+      ASSERT_OK(DestroyBlobDB(dbname_, options, bdb_options));
     }
   }
 

utilities/write_batch_with_index/write_batch_with_index.cc

@@ -604,7 +604,8 @@ void WriteBatchWithIndex::Rep::AddNewEntry(uint32_t column_family_id) {
       size_t last_entry_offset = input.data() - write_batch.Data().data();
       s = ReadRecordFromWriteBatch(&input, &tag, &column_family_id, &key,
                                    &value, &blob, &xid);
-      if (rep->obsolete_offsets.front() == last_entry_offset) {
+      if (!rep->obsolete_offsets.empty() && 
+        rep->obsolete_offsets.front() == last_entry_offset) {
         rep->obsolete_offsets.erase(rep->obsolete_offsets.begin());
         continue;
       }