From cf756ed916e74cf4296ace05f5a408129f96768a Mon Sep 17 00:00:00 2001 From: Peter Dillinger Date: Fri, 3 Feb 2023 15:28:52 -0800 Subject: [PATCH] Use LIB_MODE=shared build by default with make (#11168) Summary: With https://github.com/facebook/rocksdb/issues/11150 this becomes a practical change that I think is overall good for developer efficiency. Pull Request resolved: https://github.com/facebook/rocksdb/pull/11168 Test Plan: More efficient build of all unit tests and tools: ``` $ git clean -fdx $ du -sh . 522M . $ /usr/bin/time make -j32 LIB_MODE=static ... 14270.63user 1043.33system 11:19.85elapsed 2252%CPU (0avgtext+0avgdata 1929944maxresident)k ... $ du -sh . 62G . $ ``` Vs. ``` $ git clean -fdx $ du -sh . 522M . $ /usr/bin/time make -j32 LIB_MODE=shared ... 9479.87user 478.26system 7:20.82elapsed 2258%CPU (0avgtext+0avgdata 1929272maxresident)k ... $ du -sh . 5.4G . $ ``` So 1/3 less build time and >90% less space usage. Individual unit test edit-compile-run is not too different. Modifying an average unit test source file: ``` $ touch db/version_builder_test.cc $ /usr/bin/time make -j32 LIB_MODE=static version_builder_test ... 34.74user 3.37system 0:38.29elapsed 99%CPU (0avgtext+0avgdata 945520maxresident)k ``` Vs. ``` $ touch db/version_builder_test.cc $ /usr/bin/time make -j32 LIB_MODE=shared version_builder_test ... 116.26user 43.91system 0:28.65elapsed 559%CPU (0avgtext+0avgdata 675160maxresident)k ``` A little faster with shared. However, modifying an average DB implementation file has an extra linking step with shared lib: ``` $ touch db/db_impl/db_impl_files.cc $ /usr/bin/time make -j32 LIB_MODE=static version_builder_test ... 33.17user 5.13system 0:39.70elapsed 96%CPU (0avgtext+0avgdata 945544maxresident)k ``` Vs. ``` $ touch db/db_impl/db_impl_files.cc $ /usr/bin/time make -j32 LIB_MODE=shared version_builder_test ... 40.80user 4.66system 0:45.54elapsed 99%CPU (0avgtext+0avgdata 1056340maxresident)k ``` A little slower with shared. On the whole, should be faster and lighter weight because of the many unit test files case Reviewed By: cbi42 Differential Revision: D42894004 Pulled By: pdillinger fbshipit-source-id: 9e827e52ace79b86f849b6a24466e318b4b605a7 --- .circleci/config.yml | 20 +++++++++++++------- HISTORY.md | 3 +++ Makefile | 4 ++-- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 12d4a94fc..9505ee14f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -273,12 +273,12 @@ jobs: ./sst_dump --help | grep -E -q 'Supported compression types: kNoCompression$' # Verify no compiled in compression - post-steps - build-linux-shared_lib-alt_namespace-status_checked: + build-linux-static_lib-alt_namespace-status_checked: executor: linux-docker resource_class: 2xlarge steps: - pre-steps - - run: ASSERT_STATUS_CHECKED=1 TEST_UINT128_COMPAT=1 ROCKSDB_MODIFY_NPHASH=1 LIB_MODE=shared OPT="-DROCKSDB_NAMESPACE=alternative_rocksdb_ns" make V=1 -j32 check + - run: ASSERT_STATUS_CHECKED=1 TEST_UINT128_COMPAT=1 ROCKSDB_MODIFY_NPHASH=1 LIB_MODE=static OPT="-DROCKSDB_NAMESPACE=alternative_rocksdb_ns" make V=1 -j32 check - post-steps build-linux-release: @@ -286,11 +286,17 @@ jobs: resource_class: 2xlarge steps: - checkout # check out the code in the project directory - - run: make V=1 -j32 release + - run: make V=1 -j32 LIB_MODE=shared release + - run: ./db_stress --version # ensure with gflags + - run: make clean + - run: make V=1 -j32 LIB_MODE=static release - run: ./db_stress --version # ensure with gflags - run: make clean - run: apt-get remove -y libgflags-dev - - run: make V=1 -j32 release + - run: make V=1 -j32 LIB_MODE=shared release + - run: if ./db_stress --version; then false; else true; fi # ensure without gflags + - run: make clean + - run: make V=1 -j32 LIB_MODE=static release - run: if ./db_stress --version; then false; else true; fi # ensure without gflags - post-steps @@ -421,7 +427,7 @@ jobs: - pre-steps - setup-folly - build-folly - - run: USE_FOLLY=1 CC=gcc-7 CXX=g++-7 V=1 make -j32 check + - run: USE_FOLLY=1 LIB_MODE=static CC=gcc-7 CXX=g++-7 V=1 make -j32 check # TODO: LIB_MODE only to work around unresolved linker failures - post-steps build-linux-gcc-7-with-folly-lite-no-test: @@ -467,7 +473,7 @@ jobs: resource_class: 2xlarge steps: - pre-steps - - run: CC=gcc-11 CXX=g++-11 V=1 make -j32 all microbench + - run: LIB_MODE=static CC=gcc-11 CXX=g++-11 V=1 make -j32 all microbench # TODO: LIB_MODE only to work around unresolved linker failures - post-steps build-linux-clang-13-no_test_run: @@ -809,7 +815,7 @@ workflows: - build-linux-clang10-asan - build-linux-clang10-ubsan - build-linux-clang10-mini-tsan - - build-linux-shared_lib-alt_namespace-status_checked + - build-linux-static_lib-alt_namespace-status_checked jobs-linux-no-test-run: jobs: - build-linux-release diff --git a/HISTORY.md b/HISTORY.md index fc259b92b..0b4634885 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -24,6 +24,9 @@ * Completely removed the following deprecated/obsolete statistics: the tickers `BLOCK_CACHE_INDEX_BYTES_EVICT`, `BLOCK_CACHE_FILTER_BYTES_EVICT`, `BLOOM_FILTER_MICROS`, `NO_FILE_CLOSES`, `STALL_L0_SLOWDOWN_MICROS`, `STALL_MEMTABLE_COMPACTION_MICROS`, `STALL_L0_NUM_FILES_MICROS`, `RATE_LIMIT_DELAY_MILLIS`, `NO_ITERATORS`, `NUMBER_FILTERED_DELETES`, `WRITE_TIMEDOUT`, `BLOB_DB_GC_NUM_KEYS_OVERWRITTEN`, `BLOB_DB_GC_NUM_KEYS_EXPIRED`, `BLOB_DB_GC_BYTES_OVERWRITTEN`, `BLOB_DB_GC_BYTES_EXPIRED`, `BLOCK_CACHE_COMPRESSION_DICT_BYTES_EVICT` as well as the histograms `STALL_L0_SLOWDOWN_COUNT`, `STALL_MEMTABLE_COMPACTION_COUNT`, `STALL_L0_NUM_FILES_COUNT`, `HARD_RATE_LIMIT_DELAY_COUNT`, `SOFT_RATE_LIMIT_DELAY_COUNT`, `BLOB_DB_GC_MICROS`, and `NUM_DATA_BLOCKS_READ_PER_LEVEL`. Note that as a result, the C++ enum values of the still supported statistics have changed. Developers are advised to not rely on the actual numeric values. * Deprecated IngestExternalFileOptions::write_global_seqno and change default to false. This option only needs to be set to true to generate a DB compatible with RocksDB versions before 5.16.0. +### Build Changes +* The `make` build now builds a shared library by default instead of a static library. Use `LIB_MODE=static` to override. + ## 7.10.0 (01/23/2023) ### Behavior changes * Make best-efforts recovery verify SST unique ID before Version construction (#10962) diff --git a/Makefile b/Makefile index 9964d7d3e..54751bde9 100644 --- a/Makefile +++ b/Makefile @@ -48,8 +48,8 @@ DEBUG_LEVEL?=1 # Mode "static" means to link against static libraries (.a) # Mode "shared" means to link against shared libraries (.so, .sl, .dylib, etc) # -# Set the default LIB_MODE to static -LIB_MODE?=static +# Set the default LIB_MODE to shared for efficient `make check` etc. +LIB_MODE?=shared # OBJ_DIR is where the object files reside. Default to the current directory OBJ_DIR?=.