Fix and detect headers with missing dependencies (#8893)

Summary:
It's always annoying to find a header does not include its own
dependencies and only works when included after other includes. This
change adds `make check-headers` which validates that each header can
be included at the top of a file. Some headers are excluded e.g. because
of platform or external dependencies.

rocksdb_namespace.h had to be re-worked slightly to enable checking for
failure to include it. (ROCKSDB_NAMESPACE is a valid namespace name.)

Fixes mostly involve adding and cleaning up #includes, but for
FileTraceWriter, a constructor was out-of-lined to make a forward
declaration sufficient.

This check is not currently run with `make check` but is added to
CircleCI build-linux-unity since that one is already relatively fast.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/8893

Test Plan: existing tests and resolving issues detected by new check

Reviewed By: mrambacher

Differential Revision: D30823300

Pulled By: pdillinger

fbshipit-source-id: 9fff223944994c83c105e2e6496d24845dc8e572
main
Peter Dillinger 3 years ago committed by Facebook GitHub Bot
parent dc0dc90cf5
commit bda8d93ba9
  1. 7
      .circleci/config.yml
  2. 27
      Makefile
  3. 1
      cache/cache_entry_stats.h
  4. 2
      db/compaction/compaction_iteration_stats.h
  5. 3
      db/job_context.h
  6. 3
      db/pre_release_callback.h
  7. 1
      db/read_callback.h
  8. 1
      db/snapshot_impl.h
  9. 2
      db_stress_tool/db_stress_compaction_filter.h
  10. 3
      db_stress_tool/db_stress_listener.h
  11. 1
      db_stress_tool/db_stress_table_properties_collector.h
  12. 1
      include/rocksdb/functor_wrapper.h
  13. 6
      include/rocksdb/rocksdb_namespace.h
  14. 4
      include/rocksdb/thread_status.h
  15. 4
      include/rocksdb/universal_compaction.h
  16. 4
      include/rocksdb/utilities/ldb_cmd_execute_result.h
  17. 3
      memory/memory_usage.h
  18. 2
      table/block_based/block_type.h
  19. 2
      table/table_properties_internal.h
  20. 2
      table/table_reader_caller.h
  21. 3
      util/cast_util.h
  22. 2
      util/channel.h
  23. 3
      util/crc32c_ppc.h
  24. 2
      util/defer.h
  25. 3
      util/duplicate_detector.h
  26. 2
      util/fastrange.h
  27. 2
      util/math.h
  28. 2
      util/set_comparator.h
  29. 2
      util/work_queue.h
  30. 1
      util/xxph3.h
  31. 4
      utilities/blob_db/blob_db_gc_stats.h
  32. 5
      utilities/cassandra/cassandra_options.h
  33. 5
      utilities/cassandra/serialize.h
  34. 4
      utilities/trace/file_trace_reader_writer.cc
  35. 3
      utilities/trace/file_trace_reader_writer.h

@ -328,7 +328,7 @@ jobs:
- run: (mkdir build && cd build && cmake -DWITH_GFLAGS=1 -DWITH_BENCHMARK=1 .. && make V=1 -j20 && ctest -j20 && make microbench) | .circleci/cat_ignore_eagain - run: (mkdir build && cd build && cmake -DWITH_GFLAGS=1 -DWITH_BENCHMARK=1 .. && make V=1 -j20 && ctest -j20 && make microbench) | .circleci/cat_ignore_eagain
- post-steps - post-steps
build-linux-unity: build-linux-unity-and-headers:
docker: # executor type docker: # executor type
- image: gcc:latest - image: gcc:latest
resource_class: large resource_class: large
@ -336,6 +336,7 @@ jobs:
- checkout # check out the code in the project directory - checkout # check out the code in the project directory
- run: apt-get update -y && apt-get install -y libgflags-dev - run: apt-get update -y && apt-get install -y libgflags-dev
- run: TEST_TMPDIR=/dev/shm && make V=1 -j8 unity_test | .circleci/cat_ignore_eagain - run: TEST_TMPDIR=/dev/shm && make V=1 -j8 unity_test | .circleci/cat_ignore_eagain
- run: make V=1 -j8 -k check-headers | .circleci/cat_ignore_eagain # could be moved to a different build
- post-steps - post-steps
build-linux-gcc-4_8-no_test_run: build-linux-gcc-4_8-no_test_run:
@ -730,9 +731,9 @@ workflows:
build-linux-clang10-clang-analyze: build-linux-clang10-clang-analyze:
jobs: jobs:
- build-linux-clang10-clang-analyze - build-linux-clang10-clang-analyze
build-linux-unity: build-linux-unity-and-headers:
jobs: jobs:
- build-linux-unity - build-linux-unity-and-headers
build-windows-vs2019: build-windows-vs2019:
jobs: jobs:
- build-windows: - build-windows:

@ -516,6 +516,29 @@ ifeq ($(USE_FOLLY_DISTRIBUTED_MUTEX),1)
ALL_SOURCES += third-party/folly/folly/synchronization/test/DistributedMutexTest.cc ALL_SOURCES += third-party/folly/folly/synchronization/test/DistributedMutexTest.cc
endif endif
# `make check-headers` to very that each header file includes its own
# dependencies
ifneq ($(filter check-headers, $(MAKECMDGOALS)),)
# TODO: add/support JNI headers
DEV_HEADER_DIRS := $(sort include/ hdfs/ $(dir $(ALL_SOURCES)))
# Some headers like in port/ are platform-specific
DEV_HEADERS := $(shell $(FIND) $(DEV_HEADER_DIRS) -type f -name '*.h' | egrep -v 'port/|plugin/|lua/|range_tree/|tools/rdb/db_wrapper.h|include/rocksdb/utilities/env_librados.h')
else
DEV_HEADERS :=
endif
HEADER_OK_FILES = $(patsubst %.h, %.h.ok, $(DEV_HEADERS))
AM_V_CCH = $(am__v_CCH_$(V))
am__v_CCH_ = $(am__v_CCH_$(AM_DEFAULT_VERBOSITY))
am__v_CCH_0 = @echo " CC.h " $<;
am__v_CCH_1 =
%.h.ok: %.h # .h.ok not actually created, so re-checked on each invocation
# -DROCKSDB_NAMESPACE=42 ensures the namespace header is included
$(AM_V_CCH) echo '#include "$<"' | $(CXX) $(CXXFLAGS) -DROCKSDB_NAMESPACE=42 -x c++ -c - -o /dev/null
check-headers: $(HEADER_OK_FILES)
# options_settable_test doesn't pass with UBSAN as we use hack in the test # options_settable_test doesn't pass with UBSAN as we use hack in the test
ifdef COMPILE_WITH_UBSAN ifdef COMPILE_WITH_UBSAN
TESTS := $(shell echo $(TESTS) | sed 's/\boptions_settable_test\b//g') TESTS := $(shell echo $(TESTS) | sed 's/\boptions_settable_test\b//g')
@ -716,7 +739,7 @@ endif # PLATFORM_SHARED_EXT
.PHONY: blackbox_crash_test check clean coverage crash_test ldb_tests package \ .PHONY: blackbox_crash_test check clean coverage crash_test ldb_tests package \
release tags tags0 valgrind_check whitebox_crash_test format static_lib shared_lib all \ release tags tags0 valgrind_check whitebox_crash_test format static_lib shared_lib all \
dbg rocksdbjavastatic rocksdbjava gen-pc install install-static install-shared uninstall \ dbg rocksdbjavastatic rocksdbjava gen-pc install install-static install-shared uninstall \
analyze tools tools_lib \ analyze tools tools_lib check-headers \
blackbox_crash_test_with_atomic_flush whitebox_crash_test_with_atomic_flush \ blackbox_crash_test_with_atomic_flush whitebox_crash_test_with_atomic_flush \
blackbox_crash_test_with_txn whitebox_crash_test_with_txn \ blackbox_crash_test_with_txn whitebox_crash_test_with_txn \
blackbox_crash_test_with_best_efforts_recovery \ blackbox_crash_test_with_best_efforts_recovery \
@ -2368,7 +2391,7 @@ build_subset_tests: $(ROCKSDBTESTS_SUBSET)
# Remove the rules for which dependencies should not be generated and see if any are left. # Remove the rules for which dependencies should not be generated and see if any are left.
#If so, include the dependencies; if not, do not include the dependency files #If so, include the dependencies; if not, do not include the dependency files
ROCKS_DEP_RULES=$(filter-out clean format check-format check-buck-targets check-sources jclean jtest package analyze tags rocksdbjavastatic% unity.% unity_test, $(MAKECMDGOALS)) ROCKS_DEP_RULES=$(filter-out clean format check-format check-buck-targets check-headers check-sources jclean jtest package analyze tags rocksdbjavastatic% unity.% unity_test, $(MAKECMDGOALS))
ifneq ("$(ROCKS_DEP_RULES)", "") ifneq ("$(ROCKS_DEP_RULES)", "")
-include $(DEPFILES) -include $(DEPFILES)
endif endif

@ -15,6 +15,7 @@
#include "rocksdb/cache.h" #include "rocksdb/cache.h"
#include "rocksdb/status.h" #include "rocksdb/status.h"
#include "rocksdb/system_clock.h" #include "rocksdb/system_clock.h"
#include "test_util/sync_point.h"
#include "util/coding_lean.h" #include "util/coding_lean.h"
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {

@ -5,6 +5,8 @@
#pragma once #pragma once
#include <cstdint>
#include "rocksdb/rocksdb_namespace.h" #include "rocksdb/rocksdb_namespace.h"
struct CompactionIterationStats { struct CompactionIterationStats {

@ -12,8 +12,9 @@
#include <string> #include <string>
#include <vector> #include <vector>
#include "db/log_writer.h"
#include "db/column_family.h" #include "db/column_family.h"
#include "db/log_writer.h"
#include "db/version_set.h"
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {

@ -6,11 +6,10 @@
#pragma once #pragma once
#include "rocksdb/status.h" #include "rocksdb/status.h"
#include "rocksdb/types.h"
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {
class DB;
class PreReleaseCallback { class PreReleaseCallback {
public: public:
virtual ~PreReleaseCallback() {} virtual ~PreReleaseCallback() {}

@ -5,6 +5,7 @@
#pragma once #pragma once
#include "db/dbformat.h"
#include "rocksdb/types.h" #include "rocksdb/types.h"
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {

@ -10,6 +10,7 @@
#pragma once #pragma once
#include <vector> #include <vector>
#include "db/dbformat.h"
#include "rocksdb/db.h" #include "rocksdb/db.h"
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {

@ -9,6 +9,8 @@
#pragma once #pragma once
#include "db_stress_tool/db_stress_common.h"
#include "db_stress_tool/db_stress_shared_state.h"
#include "rocksdb/compaction_filter.h" #include "rocksdb/compaction_filter.h"
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {

@ -3,11 +3,14 @@
// COPYING file in the root directory) and Apache 2.0 License // COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory). // (found in the LICENSE.Apache file in the root directory).
#include "file/filename.h"
#ifdef GFLAGS #ifdef GFLAGS
#pragma once #pragma once
#include "rocksdb/db.h"
#include "rocksdb/listener.h" #include "rocksdb/listener.h"
#include "util/gflags_compat.h" #include "util/gflags_compat.h"
#include "util/random.h"
DECLARE_int32(compact_files_one_in); DECLARE_int32(compact_files_one_in);

@ -7,6 +7,7 @@
#include "rocksdb/table.h" #include "rocksdb/table.h"
#include "util/gflags_compat.h" #include "util/gflags_compat.h"
#include "util/random.h"
DECLARE_int32(mark_for_compaction_one_file_in); DECLARE_int32(mark_for_compaction_one_file_in);

@ -5,6 +5,7 @@
#pragma once #pragma once
#include <functional>
#include <memory> #include <memory>
#include <utility> #include <utility>

@ -5,6 +5,12 @@
#pragma once #pragma once
// For testing purposes
#if ROCKSDB_NAMESPACE == 42
#undef ROCKSDB_NAMESPACE
#endif
// Normal logic
#ifndef ROCKSDB_NAMESPACE #ifndef ROCKSDB_NAMESPACE
#define ROCKSDB_NAMESPACE rocksdb #define ROCKSDB_NAMESPACE rocksdb
#endif #endif

@ -13,13 +13,15 @@
#pragma once #pragma once
#include <stdint.h>
#include <cstddef> #include <cstddef>
#include <cstdint>
#include <map> #include <map>
#include <string> #include <string>
#include <utility> #include <utility>
#include <vector> #include <vector>
#include "rocksdb/rocksdb_namespace.h"
#if !defined(ROCKSDB_LITE) && !defined(NROCKSDB_THREAD_STATUS) && \ #if !defined(ROCKSDB_LITE) && !defined(NROCKSDB_THREAD_STATUS) && \
defined(ROCKSDB_SUPPORT_THREAD_LOCAL) defined(ROCKSDB_SUPPORT_THREAD_LOCAL)
#define ROCKSDB_USING_THREAD_STATUS #define ROCKSDB_USING_THREAD_STATUS

@ -5,10 +5,12 @@
#pragma once #pragma once
#include <stdint.h>
#include <climits> #include <climits>
#include <cstdint>
#include <vector> #include <vector>
#include "rocksdb/rocksdb_namespace.h"
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {
// //

@ -5,6 +5,10 @@
// //
#pragma once #pragma once
#include <string>
#include "rocksdb/rocksdb_namespace.h"
#ifdef FAILED #ifdef FAILED
#undef FAILED #undef FAILED
#endif #endif

@ -5,8 +5,11 @@
#pragma once #pragma once
#include <cstddef>
#include <unordered_map> #include <unordered_map>
#include "rocksdb/rocksdb_namespace.h"
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {
// Helper methods to estimate memroy usage by std containers. // Helper methods to estimate memroy usage by std containers.

@ -7,6 +7,8 @@
#include <cstdint> #include <cstdint>
#include "rocksdb/rocksdb_namespace.h"
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {
// Represents the types of blocks used in the block based table format. // Represents the types of blocks used in the block based table format.

@ -6,7 +6,7 @@
#pragma once #pragma once
#include "rocksdb/status.h" #include "rocksdb/status.h"
#include "rocksdb/iterator.h" #include "table/internal_iterator.h"
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {

@ -5,6 +5,8 @@
#pragma once #pragma once
#include "rocksdb/rocksdb_namespace.h"
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {
// A list of callers for a table reader. It is used to trace the caller that // A list of callers for a table reader. It is used to trace the caller that
// accesses on a block. This is only used for block cache tracing and analysis. // accesses on a block. This is only used for block cache tracing and analysis.

@ -4,6 +4,9 @@
// (found in the LICENSE.Apache file in the root directory). // (found in the LICENSE.Apache file in the root directory).
#pragma once #pragma once
#include "rocksdb/rocksdb_namespace.h"
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {
// The helper function to assert the move from dynamic_cast<> to // The helper function to assert the move from dynamic_cast<> to
// static_cast<> is correct. This function is to deal with legacy code. // static_cast<> is correct. This function is to deal with legacy code.

@ -10,6 +10,8 @@
#include <queue> #include <queue>
#include <utility> #include <utility>
#include "rocksdb/rocksdb_namespace.h"
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {
template <class T> template <class T>

@ -7,6 +7,9 @@
#pragma once #pragma once
#include <cstddef>
#include <cstdint>
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif

@ -7,6 +7,8 @@
#include <functional> #include <functional>
#include "rocksdb/rocksdb_namespace.h"
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {
// Defers the execution of the provided function until the Defer // Defers the execution of the provided function until the Defer

@ -5,8 +5,7 @@
#pragma once #pragma once
#include <cinttypes> #include "db/db_impl/db_impl.h"
#include "util/set_comparator.h" #include "util/set_comparator.h"
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {

@ -22,6 +22,8 @@
#include <cstdint> #include <cstdint>
#include <type_traits> #include <type_traits>
#include "rocksdb/rocksdb_namespace.h"
#ifdef TEST_UINT128_COMPAT #ifdef TEST_UINT128_COMPAT
#undef HAVE_UINT128_EXTENSION #undef HAVE_UINT128_EXTENSION
#endif #endif

@ -13,6 +13,8 @@
#include <cstdint> #include <cstdint>
#include <type_traits> #include <type_traits>
#include "rocksdb/rocksdb_namespace.h"
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {
// Fast implementation of floor(log2(v)). Undefined for 0 or negative // Fast implementation of floor(log2(v)). Undefined for 0 or negative

@ -5,6 +5,8 @@
#pragma once #pragma once
#include "rocksdb/comparator.h"
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {
// A comparator to be used in std::set // A comparator to be used in std::set
struct SetComparator { struct SetComparator {

@ -21,6 +21,8 @@
#include <mutex> #include <mutex>
#include <queue> #include <queue>
#include "rocksdb/rocksdb_namespace.h"
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {
/// Unbounded thread-safe work queue. /// Unbounded thread-safe work queue.

@ -51,6 +51,7 @@
#endif #endif
#define XXPH_NAMESPACE ROCKSDB_ #define XXPH_NAMESPACE ROCKSDB_
#define XXPH_INLINE_ALL #define XXPH_INLINE_ALL
#include <cstring>
/* END RocksDB customizations */ /* END RocksDB customizations */
#if defined (__cplusplus) #if defined (__cplusplus)

@ -5,6 +5,10 @@
// //
#pragma once #pragma once
#include <cstdint>
#include "rocksdb/rocksdb_namespace.h"
#ifndef ROCKSDB_LITE #ifndef ROCKSDB_LITE
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {

@ -4,7 +4,10 @@
// (found in the LICENSE.Apache file in the root directory). // (found in the LICENSE.Apache file in the root directory).
#pragma once #pragma once
#include <cinttypes>
#include <cstddef>
#include <cstdint>
#include <string>
#include "rocksdb/rocksdb_namespace.h" #include "rocksdb/rocksdb_namespace.h"

@ -10,6 +10,11 @@
#pragma once #pragma once
#include <cstdint>
#include <string>
#include "rocksdb/rocksdb_namespace.h"
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {
namespace cassandra { namespace cassandra {
namespace { namespace {

@ -84,6 +84,10 @@ Status FileTraceReader::Read(std::string* data) {
return s; return s;
} }
FileTraceWriter::FileTraceWriter(
std::unique_ptr<WritableFileWriter>&& file_writer)
: file_writer_(std::move(file_writer)) {}
FileTraceWriter::~FileTraceWriter() { Close().PermitUncheckedError(); } FileTraceWriter::~FileTraceWriter() { Close().PermitUncheckedError(); }
Status FileTraceWriter::Close() { Status FileTraceWriter::Close() {

@ -34,8 +34,7 @@ class FileTraceReader : public TraceReader {
// FileTraceWriter allows writing RocksDB traces to a file. // FileTraceWriter allows writing RocksDB traces to a file.
class FileTraceWriter : public TraceWriter { class FileTraceWriter : public TraceWriter {
public: public:
explicit FileTraceWriter(std::unique_ptr<WritableFileWriter>&& file_writer) explicit FileTraceWriter(std::unique_ptr<WritableFileWriter>&& file_writer);
: file_writer_(std::move(file_writer)) {}
~FileTraceWriter(); ~FileTraceWriter();
virtual Status Write(const Slice& data) override; virtual Status Write(const Slice& data) override;

Loading…
Cancel
Save