Print stack traces on frozen tests in CI (#10828)

Summary:
Instead of existing calls to ps from gnu_parallel, call a new wrapper that does ps, looks for unit test like processes, and uses pstack or gdb to print thread stack traces. Also, using `ps -wwf` instead of `ps -wf` ensures output is not cut off.

For security, CircleCI runs with security restrictions on ptrace (/proc/sys/kernel/yama/ptrace_scope = 1), and this change adds a work-around to `InstallStackTraceHandler()` (only used by testing tools) to allow any process from the same user to debug it. (I've also touched >100 files to ensure all the unit tests call this function.)

Pull Request resolved: https://github.com/facebook/rocksdb/pull/10828

Test Plan: local manual + temporary infinite loop in a unit test to observe in CircleCI

Reviewed By: hx235

Differential Revision: D40447634

Pulled By: pdillinger

fbshipit-source-id: 718a4c4a5b54fa0f9af2d01a446162b45e5e84e1

Makefile

@@ -1832,7 +1832,7 @@ memtable_list_test: $(OBJ_DIR)/db/memtable_list_test.o $(TEST_LIBRARY) $(LIBRARY
 write_callback_test: $(OBJ_DIR)/db/write_callback_test.o $(TEST_LIBRARY) $(LIBRARY)
 	$(AM_LINK)
 
-heap_test: $(OBJ_DIR)/util/heap_test.o $(GTEST)
+heap_test: $(OBJ_DIR)/util/heap_test.o $(TEST_LIBRARY) $(LIBRARY)
 	$(AM_LINK)
 
 point_lock_manager_test: utilities/transactions/lock/point/point_lock_manager_test.o $(TEST_LIBRARY) $(LIBRARY)

build_tools/gnu_parallel

@@ -1916,7 +1916,8 @@ sub drain_job_queue {
                 } elsif (not $ps_reported and (time() - $last_progress_time) >= 60) {
                     # No progress in at least 60 seconds: run ps
                     print $Global::original_stderr "\n";
-                    system("ps", "-wf");
+                    my $script_dir = ::dirname($0);
+                    system("$script_dir/ps_with_stack || ps -wwf");
                     $ps_reported = 1;
                 }
                 $last_left = $Global::left;

build_tools/ps_with_stack

@@ -0,0 +1,38 @@
+#!/usr/bin/env perl
+
+use strict;
+
+open(my $ps, "-|", "ps -wwf");
+my $cols_known = 0;
+my $cmd_col = 0;
+my $pid_col = 0;
+while (<$ps>) {
+  print;
+  my @cols = split(/\s+/);
+
+  if (!$cols_known && /CMD/) {
+    # Parse relevant ps column headers
+    for (my $i = 0; $i <= $#cols; $i++) {
+      if ($cols[$i] eq "CMD") {
+        $cmd_col = $i;
+      }
+      if ($cols[$i] eq "PID") {
+        $pid_col = $i;
+      }
+    }
+    $cols_known = 1;
+  } else {
+    my $pid = $cols[$pid_col];
+    my $cmd = $cols[$cmd_col];
+    # Match numeric PID and relative path command
+    # -> The intention is only to dump stack traces for hangs in code under
+    # test, which means we probably just built it and are executing by
+    # relative path (e.g. ./my_test or foo/bar_test) rather then by absolute
+    # path (e.g. /usr/bin/time) or PATH search (e.g. grep).
+    if ($pid =~ /^[0-9]+$/ && $cmd =~ /^[^\/ ]+[\/]/) {
+      print "Dumping stacks for $pid...\n";
+      system("pstack $pid || gdb -batch -p $pid -ex 'thread apply all bt'");
+    }
+  }
+}
+close $ps;

cache/cache_reservation_manager_test.cc

@@ -463,6 +463,7 @@ TEST(CacheReservationHandleTest, HandleTest) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

cache/compressed_secondary_cache_test.cc

@@ -999,6 +999,7 @@ TEST_F(CompressedSecondaryCacheTest, SplictValueAndMergeChunksTest) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

cache/lru_cache_test.cc

@@ -2703,6 +2703,7 @@ TEST_F(DBSecondaryCacheTest, TestSecondaryCacheOptionTwoDB) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

db/blob/blob_counting_iterator_test.cc

@@ -321,6 +321,7 @@ TEST(BlobCountingIteratorTest, CorruptBlobIndex) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

db/blob/blob_file_addition_test.cc

@@ -205,6 +205,7 @@ TEST_F(BlobFileAdditionTest, ForwardIncompatibleCustomField) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

db/blob/blob_file_builder_test.cc

@@ -674,6 +674,7 @@ TEST_P(BlobFileBuilderIOErrorTest, IOError) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

db/blob/blob_file_cache_test.cc

@@ -263,6 +263,7 @@ TEST_F(BlobFileCacheTest, GetBlobFileReader_CacheFull) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

db/blob/blob_file_garbage_test.cc

@@ -168,6 +168,7 @@ TEST_F(BlobFileGarbageTest, ForwardIncompatibleCustomField) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

db/blob/blob_file_reader_test.cc

@@ -1018,6 +1018,7 @@ TEST_P(BlobFileReaderDecodingErrorTest, DecodingError) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

db/blob/blob_garbage_meter_test.cc

@@ -191,6 +191,7 @@ TEST(BlobGarbageMeterTest, InlinedTTLBlobIndex) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

db/blob/blob_source_test.cc

@@ -1618,6 +1618,7 @@ TEST_F(BlobSourceCacheReservationTest, IncreaseCacheReservationOnFullCache) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

db/compact_files_test.cc

@@ -490,6 +490,7 @@ TEST_F(CompactFilesTest, GetCompactionJobInfo) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

db/compaction/clipping_iterator_test.cc

@@ -253,6 +253,7 @@ INSTANTIATE_TEST_CASE_P(
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

db/compaction/compaction_iterator_test.cc

@@ -1497,6 +1497,7 @@ INSTANTIATE_TEST_CASE_P(CompactionIteratorTsGcTestInstance,
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

db/compaction/compaction_job_test.cc

@@ -2434,6 +2434,7 @@ TEST_F(CompactionJobIOPriorityTest, GetRateLimiterPriority) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   RegisterCustomObjects(argc, argv);
   return RUN_ALL_TESTS();

db/compaction/compaction_picker_test.cc

@@ -3721,6 +3721,7 @@ INSTANTIATE_TEST_CASE_P(PerKeyPlacementCompactionPickerTest,
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

db/comparator_db_test.cc

@@ -673,6 +673,7 @@ TEST_P(ComparatorDBTest, SeparatorSuccessorRandomizeTest) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

db/cuckoo_table_db_test.cc

@@ -345,6 +345,7 @@ TEST_F(CuckooTableDBTest, AdaptiveTable) {
 
 int main(int argc, char** argv) {
   if (ROCKSDB_NAMESPACE::port::kLittleEndian) {
+    ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
     ::testing::InitGoogleTest(&argc, argv);
     return RUN_ALL_TESTS();
   } else {

db/db_iter_stress_test.cc

@@ -652,6 +652,7 @@ TEST_F(DBIteratorStressTest, StressTest) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   ParseCommandLineFlags(&argc, &argv, true);
   return RUN_ALL_TESTS();

db/db_iter_test.cc

@@ -3187,6 +3187,7 @@ TEST_F(DBIteratorTest, ReverseToForwardWithDisappearingKeys) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

db/db_kv_checksum_test.cc

@@ -879,6 +879,7 @@ TEST_P(DbMemtableKVChecksumTest, FlushWithCorruptAfterMemtableInsert) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

db/db_logical_block_size_cache_test.cc

@@ -515,6 +515,7 @@ TEST_F(DBLogicalBlockSizeCacheTest, MultiDBWithSamePaths) {
 #endif  // OS_LINUX
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

db/dbformat_test.cc

@@ -206,6 +206,7 @@ TEST_F(FormatTest, RangeTombstoneSerializeEndKey) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   RegisterCustomObjects(argc, argv);
   return RUN_ALL_TESTS();

db/fault_injection_test.cc

@@ -631,6 +631,7 @@ INSTANTIATE_TEST_CASE_P(
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   RegisterCustomObjects(argc, argv);
   return RUN_ALL_TESTS();

db/filename_test.cc

@@ -237,6 +237,7 @@ TEST_F(FileNameTest, NormalizePath) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

db/flush_job_test.cc

@@ -739,6 +739,7 @@ TEST_F(FlushJobTimestampTest, NoKeyExpired) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

db/listener_test.cc

@@ -1591,6 +1591,7 @@ TEST_F(EventListenerTest, BlobDBFileTest) {
 #endif  // ROCKSDB_LITE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

db/log_test.cc

@@ -1062,6 +1062,7 @@ INSTANTIATE_TEST_CASE_P(
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

db/manual_compaction_test.cc

@@ -304,6 +304,7 @@ TEST_F(ManualCompactionTest, SkipLevel) {
 }  // anonymous namespace
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

db/memtable_list_test.cc

@@ -1009,6 +1009,7 @@ TEST_F(MemTableListTest, AtomicFlusTest) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

db/merge_helper_test.cc

@@ -291,6 +291,7 @@ TEST_F(MergeHelperTest, DontFilterMergeOperandsBeforeSnapshotTest) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

db/options_file_test.cc

@@ -102,6 +102,7 @@ TEST_F(OptionsFileTest, OptionsFileName) {
 
 int main(int argc, char** argv) {
 #if !(defined NDEBUG) || !defined(OS_WIN)
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 #else

db/perf_context_test.cc

@@ -959,6 +959,7 @@ TEST_F(PerfContextTest, CPUTimer) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
 
   for (int i = 1; i < argc; i++) {

db/periodic_task_scheduler_test.cc

@@ -224,6 +224,7 @@ TEST_F(PeriodicTaskSchedulerTest, MultiEnv) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
 
   return RUN_ALL_TESTS();

db/plain_table_db_test.cc

@@ -1352,6 +1352,7 @@ INSTANTIATE_TEST_CASE_P(PlainTableDBTest, PlainTableDBTest, ::testing::Bool());
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

db/prefix_test.cc

@@ -890,6 +890,7 @@ TEST_F(PrefixTest, PrefixSeekModePrev3) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   ParseCommandLineFlags(&argc, &argv, true);
   return RUN_ALL_TESTS();

db/range_del_aggregator_test.cc

@@ -709,6 +709,7 @@ TEST_F(RangeDelAggregatorTest,
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

db/range_tombstone_fragmenter_test.cc

@@ -549,6 +549,7 @@ TEST_F(RangeTombstoneFragmenterTest, SeekOutOfBounds) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

db/repair_test.cc

@@ -427,6 +427,7 @@ TEST_F(RepairTest, DbNameContainsTrailingSlash) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

db/table_properties_collector_test.cc

@@ -507,6 +507,7 @@ INSTANTIATE_TEST_CASE_P(CustomizedTablePropertiesCollector, TablePropertiesTest,
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

db/version_builder_test.cc

@@ -1689,6 +1689,7 @@ TEST_F(VersionBuilderTest, EstimatedActiveKeys) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

db/version_edit_test.cc

@@ -724,6 +724,7 @@ TEST(FileMetaDataTest, UpdateBoundariesBlobIndex) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

db/version_set_test.cc

@@ -3579,6 +3579,7 @@ TEST_P(ChargeFileMetadataTestWithParam, Basic) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

db/wal_manager_test.cc

@@ -329,6 +329,7 @@ TEST_F(WalManagerTest, TransactionLogIteratorNewFileWhileScanning) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

db/wide/wide_column_serialization_test.cc

@@ -303,6 +303,7 @@ TEST(WideColumnSerializationTest, DeserializeColumnsOutOfOrder) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

db/write_batch_test.cc

@@ -1107,6 +1107,7 @@ TEST_F(WriteBatchTest, CommitWithTimestamp) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

db/write_callback_test.cc

@@ -447,6 +447,7 @@ TEST_F(WriteCallbackTest, WriteCallBackTest) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

db/write_controller_test.cc

@@ -242,6 +242,7 @@ TEST_F(WriteControllerTest, CreditAccumulation) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

env/env_basic_test.cc

@@ -395,6 +395,7 @@ TEST_P(EnvMoreTestWithParam, GetChildrenIgnoresDotAndDotDot) {
 
 }  // namespace ROCKSDB_NAMESPACE
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

env/io_posix_test.cc

@@ -135,6 +135,7 @@ TEST_F(LogicalBlockSizeCacheTest, Ref) {
 #endif
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

env/mock_env_test.cc

@@ -78,6 +78,7 @@ TEST_F(MockEnvTest, FakeSleeping) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

file/delete_scheduler_test.cc

@@ -712,6 +712,7 @@ TEST_F(DeleteSchedulerTest, IsTrashCheck) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

file/prefetch_test.cc

@@ -2079,6 +2079,7 @@ namespace {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
 
   return RUN_ALL_TESTS();

logging/auto_roll_logger_test.cc

@@ -728,6 +728,7 @@ TEST_F(AutoRollLoggerTest, RenameError) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

logging/env_logger_test.cc

@@ -156,6 +156,7 @@ TEST_F(EnvLoggerTest, ConcurrentLogging) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

logging/event_logger_test.cc

@@ -38,6 +38,7 @@ TEST_F(EventLoggerTest, SimpleTest) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

memory/arena_test.cc

@@ -259,6 +259,7 @@ TEST(MmapTest, AllocateLazyZeroed) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

memory/memory_allocator_test.cc

@@ -234,6 +234,7 @@ INSTANTIATE_TEST_CASE_P(
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

memtable/inlineskiplist_test.cc

@@ -658,6 +658,7 @@ TEST_F(InlineSkipTest, ConcurrentInsertWithHint3) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

memtable/skiplist_test.cc

@@ -383,6 +383,7 @@ TEST_F(SkipTest, Concurrent5) { RunConcurrent(5); }
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

memtable/write_buffer_manager_test.cc

@@ -298,6 +298,7 @@ TEST_F(ChargeWriteBufferTest, BasicWithCacheFull) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

monitoring/histogram_test.cc

@@ -248,6 +248,7 @@ TEST_F(HistogramTest, LostUpdateStandardDeviation) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

monitoring/iostats_context_test.cc

@@ -24,6 +24,7 @@ TEST(IOStatsContextTest, ToString) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

options/configurable_test.cc

@@ -872,6 +872,7 @@ INSTANTIATE_TEST_CASE_P(
 }  // namespace test
 }  // namespace ROCKSDB_NAMESPACE
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
 #ifdef GFLAGS
   ParseCommandLineFlags(&argc, &argv, true);

options/options_settable_test.cc

@@ -611,6 +611,7 @@ TEST_F(OptionsSettableTest, ColumnFamilyOptionsAllFieldsSettable) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
 #ifdef GFLAGS
   ParseCommandLineFlags(&argc, &argv, true);

options/options_test.cc

@@ -4985,6 +4985,7 @@ INSTANTIATE_TEST_CASE_P(OptionsSanityCheckTest, OptionsSanityCheckTest,
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
 #ifdef GFLAGS
   ParseCommandLineFlags(&argc, &argv, true);

port/stack_trace.cc

@@ -35,6 +35,9 @@ void* SaveStack(int* /*num_frames*/, int /*first_frames_to_skip*/) {
 #if defined(OS_FREEBSD)
 #include <sys/sysctl.h>
 #endif
+#ifdef OS_LINUX
+#include <sys/prctl.h>
+#endif
 
 #include "port/lang.h"
 
@@ -187,6 +190,10 @@ void InstallStackTraceHandler() {
   signal(SIGSEGV, StackTraceHandler);
   signal(SIGBUS, StackTraceHandler);
   signal(SIGABRT, StackTraceHandler);
+  // Allow ouside debugger to attach, even with Yama security restrictions
+#ifdef PR_SET_PTRACER_ANY
+  (void)prctl(PR_SET_PTRACER, PR_SET_PTRACER_ANY, 0, 0, 0);
+#endif
 }
 
 }  // namespace port

port/stack_trace.h

@@ -12,7 +12,10 @@ namespace port {
 
 // Install a signal handler to print callstack on the following signals:
 // SIGILL SIGSEGV SIGBUS SIGABRT
-// Currently supports linux only. No-op otherwise.
+// And also (Linux ony for now) overrides security settings to allow outside
+// processes to attach to this one as a debugger. ONLY USE FOR NON-SECURITY
+// CRITICAL PROCESSES such as unit tests or benchmarking tools.
+// Currently supports only some POSIX implementations. No-op otherwise.
 void InstallStackTraceHandler();
 
 // Prints stack, skips skip_first_frames frames

table/block_based/block_test.cc

@@ -621,6 +621,7 @@ INSTANTIATE_TEST_CASE_P(P, IndexBlockTest,
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char **argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

table/block_based/data_block_hash_index_test.cc

@@ -711,6 +711,7 @@ TEST(DataBlockHashIndex, BlockBoundary) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

table/block_based/full_filter_block_test.cc

@@ -334,6 +334,7 @@ TEST_F(FullFilterBlockTest, SingleChunk) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

table/block_based/partitioned_filter_block_test.cc

@@ -429,6 +429,7 @@ TEST_P(PartitionedFilterBlockTest, PartitionCount) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

table/cuckoo/cuckoo_table_builder_test.cc

@@ -630,6 +630,7 @@ TEST_F(CuckooBuilderTest, FailWhenSameKeyInserted) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

table/cuckoo/cuckoo_table_reader_test.cc

@@ -556,6 +556,7 @@ TEST_F(CuckooReaderTest, TestReadPerformance) {
 
 int main(int argc, char** argv) {
   if (ROCKSDB_NAMESPACE::port::kLittleEndian) {
+    ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
     ::testing::InitGoogleTest(&argc, argv);
     ParseCommandLineFlags(&argc, &argv, true);
     return RUN_ALL_TESTS();

table/merger_test.cc

@@ -176,6 +176,7 @@ TEST_F(MergerTest, SeekToLastTest) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

test_util/testharness.h

@@ -52,6 +52,8 @@
   } while (false) /* user ; */
 
 #include <string>
+
+#include "port/stack_trace.h"
 #include "rocksdb/env.h"
 
 namespace ROCKSDB_NAMESPACE {

tools/block_cache_analyzer/block_cache_trace_analyzer_test.cc

@@ -711,6 +711,7 @@ TEST_F(BlockCacheTracerTest, MixedBlocks) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

tools/db_bench_tool_test.cc

@@ -320,6 +320,7 @@ TEST_F(DBBenchTest, OptionsFileFromFile) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true);
   return RUN_ALL_TESTS();

tools/io_tracer_parser_test.cc

@@ -176,6 +176,7 @@ TEST_F(IOTracerParserTest, NoRecordingBeforeStartIOTrace) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

tools/reduce_levels_test.cc

@@ -206,6 +206,7 @@ TEST_F(ReduceLevelTest, All_Levels) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

tools/trace_analyzer_test.cc

@@ -874,6 +874,7 @@ TEST_F(TraceAnalyzerTest, MultiGet) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

trace_replay/block_cache_tracer_test.cc

@@ -379,6 +379,7 @@ TEST_F(BlockCacheTracerTest, HumanReadableTrace) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

trace_replay/io_tracer_test.cc

@@ -347,6 +347,7 @@ TEST_F(IOTracerTest, AtomicMultipleWrites) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

util/autovector_test.cc

@@ -325,6 +325,7 @@ TEST_F(AutoVectorTest, PerfBench) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

util/bloom_test.cc

@@ -1169,6 +1169,7 @@ TEST(RibbonTest, RibbonTestLevelThreshold) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   ParseCommandLineFlags(&argc, &argv, true);
 

util/coding_test.cc

@@ -212,6 +212,7 @@ TEST(Coding, Strings) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

util/crc32c_test.cc

@@ -204,6 +204,7 @@ inline uint64_t fnv64_buf(const void* buf,
 }
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
 
   // Populate a buffer with a deterministic pattern

util/dynamic_bloom_test.cc

@@ -315,6 +315,7 @@ TEST_F(DynamicBloomTest, concurrent_with_perf) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   ParseCommandLineFlags(&argc, &argv, true);
 

util/file_reader_writer_test.cc

@@ -1060,6 +1060,7 @@ TEST_F(WritableFileWriterIOPriorityTest, BasicOp) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

util/filelock_test.cc

@@ -147,6 +147,7 @@ TEST_F(LockTest, LockBySameThread) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

util/hash_test.cc

@@ -846,6 +846,7 @@ TEST(MathTest, CodingGeneric) {
 int main(int argc, char** argv) {
   fprintf(stderr, "NPHash64 id: %x\n",
           static_cast<int>(ROCKSDB_NAMESPACE::GetSliceNPHash64("RocksDB")));
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
 
   return RUN_ALL_TESTS();

util/heap_test.cc

@@ -3,15 +3,16 @@
 //  COPYING file in the root directory) and Apache 2.0 License
 //  (found in the LICENSE.Apache file in the root directory).
 
+#include "util/heap.h"
+
 #include <gtest/gtest.h>
 
 #include <climits>
-
 #include <queue>
 #include <random>
 #include <utility>
 
-#include "util/heap.h"
+#include "port/stack_trace.h"
 
 #ifndef GFLAGS
 const int64_t FLAGS_iters = 100000;
@@ -131,6 +132,7 @@ INSTANTIATE_TEST_CASE_P(
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
 #ifdef GFLAGS
   GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true);

util/random_test.cc

@@ -99,6 +99,7 @@ TEST(RandomTest, PercentTrue) {
 }
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
 
   return RUN_ALL_TESTS();

util/rate_limiter_test.cc

@@ -470,6 +470,7 @@ TEST_F(RateLimiterTest, AutoTuneIncreaseWhenFull) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

util/repeatable_thread_test.cc

@@ -104,6 +104,7 @@ TEST_F(RepeatableThreadTest, MockEnvTest) {
 }
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
 
   return RUN_ALL_TESTS();

util/ribbon_test.cc

@@ -1298,6 +1298,7 @@ TYPED_TEST(RibbonTypeParamTest, OptimizeHomogAtScale) {
 }
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
 #ifdef GFLAGS
   ParseCommandLineFlags(&argc, &argv, true);

util/slice_transform_test.cc

@@ -148,6 +148,7 @@ TEST_F(SliceTransformDBTest, CapPrefix) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }