diff --git a/CMakeLists.txt b/CMakeLists.txt index 1fbf1cef1..fcf24fbed 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,8 +1,9 @@ # This cmake build is for Windows only. # # Prerequisites: -# You must have Visual Studio 2013 installed. Start the Developer Command Prompt window that is a part of Visual Studio installation. +# You must have Visual Studio 2013 Update 4 installed. Start the Developer Command Prompt window that is a part of Visual Studio installation. # Run the build commands from within the Developer Command Prompt window to have paths to the compiler and runtime libraries set. +# Make sure that Git is in your PATH # # To build Rocksdb for Windows is as easy as 1-2-3-4-5: # diff --git a/INSTALL.md b/INSTALL.md index c1b13986f..d10d82a7c 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -80,3 +80,6 @@ your make commands, like this: `PORTABLE=1 make static_lib` * **iOS**: * Run: `TARGET_OS=IOS make static_lib`. When building the project which uses rocksdb iOS library, make sure to define two important pre-processing macros: `ROCKSDB_LITE` and `IOS_CROSS_COMPILE`. + +* **Windows**: + * Read the follow the instructions at CMakeLists.txt diff --git a/WINDOWS_PORT.md b/WINDOWS_PORT.md index 7f5d3e040..a0fe1fe11 100644 --- a/WINDOWS_PORT.md +++ b/WINDOWS_PORT.md @@ -15,7 +15,7 @@ These notes describe some decisions and changes we had to make with regards to p We are open for comments and improvements. ## OS specifics -All of the porting, testing and benchmarking was done on Windows Server 2012 R2 Datacenter but to the best of our knowledge there is not a specific API we used during porting that is unsupported on other Windows OS after Vista. +All of the porting, testing and benchmarking was done on Windows Server 2012 R2 Datacenter 64-bit but to the best of our knowledge there is not a specific API we used during porting that is unsupported on other Windows OS after Vista. ## Porting goals We strive to achieve the following goals: @@ -34,6 +34,8 @@ At the same time it generates Visual Studio projects that are both usable from a The top-level CMakeLists.txt file contains description of all targets and build rules. It also provides brief instructions on how to build the software for Windows. One more build related file is thirdparty.inc that also resides on the top level. This file must be edited to point to actual third party libraries location. We think that it would be beneficial to merge the existing make-based build system and the new cmake-based build system into a single one to use on all platforms. +All building and testing was done for 64-bit. We have not conducted any testing for 32-bit and early reports indicate that it will not run on 32-bit. + ## C++ and STL notes We had to make some minimum changes within the portable files that either account for OS differences or the shortcomings of C++11 support in the current version of the MS compiler. Most or all of them are expected to be fixed in the upcoming compiler releases. diff --git a/build_tools/build_detect_version.bat b/build_tools/build_detect_version.bat deleted file mode 100644 index 41066af9e..000000000 --- a/build_tools/build_detect_version.bat +++ /dev/null @@ -1,24 +0,0 @@ -@echo off - -REM Record the version of the source that we are compiling. -REM We keep a record of the git revision in util/version.cc. This source file -REM is then built as a regular source file as part of the compilation process. -REM One can run "strings executable_filename | grep _build_" to find the version of -REM the source that we used to build the executable file. - -set CONFIGURATION=%1 - -pushd "%~dp0" -set "OUTFILE="..\util\build_version_%CONFIGURATION%.cc" - -REM GIT_SHA="" -REM if command -v git >/dev/null 2>&1; then -REM GIT_SHA=$(git rev-parse HEAD 2>/dev/null) -REM fi - -@echo #include "build_version.h" > %OUTFILE% -@echo const char* rocksdb_build_git_sha = "rocksdb_build_git_sha:${GIT_SHA}"; >> %OUTFILE% -@echo const char* rocksdb_build_git_datetime = "rocksdb_build_git_datetime:$(date)"; >> %OUTFILE% -@echo const char* rocksdb_build_compile_date = __DATE__; >> %OUTFILE% - -@popd diff --git a/build_tools/runall.bat b/build_tools/runall.bat deleted file mode 100644 index 039fb690c..000000000 --- a/build_tools/runall.bat +++ /dev/null @@ -1,99 +0,0 @@ -@echo off -call :init -call :runtest arena_test.exe -call :runtest autovector_test.exe -call :runtest auto_roll_logger_test.exe -call :runtest backupable_db_test.exe -rem call :runtest benchharness_test.exe -call :runtest block_based_filter_block_test.exe -call :runtest block_hash_index_test.exe -call :runtest block_test.exe -call :runtest bloom_test.exe -call :runtest cache_test.exe -call :runtest coding_test.exe -call :runtest column_family_test.exe -call :runtest compaction_job_test.exe -call :runtest compaction_picker_test.exe -call :runtest comparator_db_test.exe -call :runtest corruption_test.exe -call :runtest crc32c_test.exe -call :runtest cuckoo_table_builder_test.exe -call :runtest cuckoo_table_db_test.exe -call :runtest cuckoo_table_reader_test.exe -call :runtest dbformat_test.exe -call :runtest db_iter_test.exe -call :runtest db_test.exe -call :runtest deletefile_test.exe -call :runtest dynamic_bloom_test.exe -call :runtest env_test.exe -call :runtest fault_injection_test.exe -call :runtest filelock_test.exe -call :runtest filename_test.exe -call :runtest file_indexer_test.exe -call :runtest full_filter_block_test.exe -call :runtest histogram_test.exe -call :runtest listener_test.exe -call :runtest log_test.exe -call :runtest manual_compaction_test.exe -call :runtest memenv_test.exe -call :runtest merger_test.exe -call :runtest merge_test.exe -call :runtest mock_env_test.exe -call :runtest options_test.exe -call :runtest perf_context_test.exe -call :runtest plain_table_db_test.exe -call :runtest prefix_test.exe -call :runtest rate_limiter_test.exe -call :runtest redis_lists_test.exe -rem call :runtest signal_test.exe -call :runtest skiplist_test.exe -call :runtest slice_transform_test.exe -call :runtest sst_dump_test.exe -call :runtest stringappend_test.exe -call :runtest table_properties_collector_test.exe -call :runtest table_test.exe -call :runtest thread_list_test.exe -call :runtest thread_local_test.exe -call :runtest ttl_test.exe -call :runtest version_builder_test.exe -call :runtest version_edit_test.exe -call :runtest version_set_test.exe -call :runtest wal_manager_test.exe -call :runtest write_batch_test.exe -rem call :runtest write_batch_with_index_test.exe -call :runtest write_controller_test.exe -call :stat -goto :eof - -:init -set tests=0 -set passed=0 -set failed=0 -goto :eof - -:runtest -set /A tests=%tests% + 1 -echo|set /p=Running %1... -%1 > %1.log 2>&1 -findstr /C:"PASSED" %1.log > nul 2>&1 -IF ERRORLEVEL 1 ( - findstr /C:"Passed all tests" %1.log > nul 2>&1 - IF ERRORLEVEL 1 ( - echo ***FAILED*** - set /A failed=%failed% + 1 - ) ELSE ( - echo OK - set /A passed=%passed% + 1 - ) -) ELSE ( - echo OK - set /A passed=%passed% + 1 -) -goto :eof - -:stat -echo ================= -echo Total tests : %tests% -echo Passed : %passed% -echo Failed : %failed% -goto :eof diff --git a/db/c.cc b/db/c.cc index 96a5882d8..318b92f2e 100644 --- a/db/c.cc +++ b/db/c.cc @@ -484,7 +484,7 @@ static bool SaveError(char** errptr, const Status& s) { *errptr = strdup(s.ToString().c_str()); } else { // TODO(sanjay): Merge with existing error? - // This is a bug if *errptr is not create by malloc() + // This is a bug if *errptr is not created by malloc() free(*errptr); *errptr = strdup(s.ToString().c_str()); } diff --git a/db/file_indexer.h b/db/file_indexer.h index 748669a82..617634192 100644 --- a/db/file_indexer.h +++ b/db/file_indexer.h @@ -12,6 +12,7 @@ #include #include #include +#include "port/port.h" #include "util/arena.h" #include "util/autovector.h" @@ -59,7 +60,7 @@ class FileIndexer { enum { // MSVC version 1800 still does not have constexpr for ::max() - kLevelMaxIndex = INT32_MAX + kLevelMaxIndex = rocksdb::port::LevelMaxIndex }; private: diff --git a/include/rocksdb/c.h b/include/rocksdb/c.h index fc36372b2..42641c3ea 100644 --- a/include/rocksdb/c.h +++ b/include/rocksdb/c.h @@ -192,13 +192,13 @@ extern ROCKSDB_LIBRARY_API rocksdb_t* rocksdb_open_for_read_only_column_families unsigned char error_if_log_file_exist, char** errptr); -ROCKSDB_LIBRARY_API char** rocksdb_list_column_families( +extern ROCKSDB_LIBRARY_API char** rocksdb_list_column_families( const rocksdb_options_t* options, const char* name, size_t* lencf, char** errptr); -ROCKSDB_LIBRARY_API void rocksdb_list_column_families_destroy(char** list, size_t len); +extern ROCKSDB_LIBRARY_API void rocksdb_list_column_families_destroy(char** list, size_t len); extern ROCKSDB_LIBRARY_API rocksdb_column_family_handle_t* rocksdb_create_column_family( rocksdb_t* db, @@ -236,7 +236,7 @@ extern ROCKSDB_LIBRARY_API void rocksdb_delete( const char* key, size_t keylen, char** errptr); -void ROCKSDB_LIBRARY_API rocksdb_delete_cf( +extern ROCKSDB_LIBRARY_API void rocksdb_delete_cf( rocksdb_t* db, const rocksdb_writeoptions_t* options, rocksdb_column_family_handle_t* column_family, @@ -467,16 +467,16 @@ extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_delete_cf( rocksdb_writebatch_t*, rocksdb_column_family_handle_t* column_family, const char* key, size_t klen); -ROCKSDB_LIBRARY_API void rocksdb_writebatch_deletev( +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_deletev( rocksdb_writebatch_t* b, int num_keys, const char* const* keys_list, const size_t* keys_list_sizes); -ROCKSDB_LIBRARY_API void rocksdb_writebatch_deletev_cf( +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_deletev_cf( rocksdb_writebatch_t* b, rocksdb_column_family_handle_t* column_family, int num_keys, const char* const* keys_list, const size_t* keys_list_sizes); -ROCKSDB_LIBRARY_API extern void rocksdb_writebatch_put_log_data( +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_put_log_data( rocksdb_writebatch_t*, const char* blob, size_t len); extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_iterate( @@ -986,7 +986,7 @@ extern ROCKSDB_LIBRARY_API void rocksdb_get_options_from_string( rocksdb_options_t* new_options, char** errptr); -// refering to convention (3), this should be used by client +// referring to convention (3), this should be used by client // to free memory that was malloc()ed extern ROCKSDB_LIBRARY_API void rocksdb_free( void* ptr); diff --git a/include/rocksdb/slice.h b/include/rocksdb/slice.h index c07f62755..084e846e7 100644 --- a/include/rocksdb/slice.h +++ b/include/rocksdb/slice.h @@ -24,12 +24,6 @@ #include #include #include -#include - -// Do not want to include the whole /port/port.h here for one define -#ifdef OS_WIN -# define snprintf _snprintf -#endif namespace rocksdb { @@ -80,19 +74,7 @@ class Slice { } // Return a string that contains the copy of the referenced data. - std::string ToString(bool hex = false) const { - if (hex) { - std::string result; - char buf[10]; - for (size_t i = 0; i < size_; i++) { - snprintf(buf, 10, "%02X", (unsigned char)data_[i]); - result += buf; - } - return result; - } else { - return std::string(data_, size_); - } - } + std::string ToString(bool hex = false) const; // Three-way comparison. Returns value: // < 0 iff "*this" < "b", diff --git a/include/rocksdb/thread_status.h b/include/rocksdb/thread_status.h index 282c9eb2e..4d53b28aa 100644 --- a/include/rocksdb/thread_status.h +++ b/include/rocksdb/thread_status.h @@ -32,15 +32,11 @@ namespace rocksdb { // TODO(yhchiang): remove this function once c++14 is available // as std::max will be able to cover this. -#ifndef OS_WIN -constexpr int constexpr_max(int a, int b) { return a > b ? a : b; } -#else // Current MS compiler does not support constexpr template struct constexpr_max { static const int result = (A > B) ? A : B; }; -#endif // A structure that describes the current status of a thread. // The status of active threads can be fetched using @@ -100,11 +96,7 @@ struct ThreadStatus { // The maximum number of properties of an operation. // This number should be set to the biggest NUM_XXX_PROPERTIES. static const int kNumOperationProperties = -#ifndef OS_WIN - constexpr_max(NUM_COMPACTION_PROPERTIES, NUM_FLUSH_PROPERTIES); -#else constexpr_max::result; -#endif // The type used to refer to a thread state. // A state describes lower-level action of a thread diff --git a/include/rocksdb/utilities/spatial_db.h b/include/rocksdb/utilities/spatial_db.h index 979968cb6..894930a68 100644 --- a/include/rocksdb/utilities/spatial_db.h +++ b/include/rocksdb/utilities/spatial_db.h @@ -80,7 +80,7 @@ struct Variant { const std::string& get_string() const { return *GetStringPtr(data_); } bool operator==(const Variant& other) const; - bool operator!=(const Variant& rhs) const { return !(*this == rhs); } + bool operator!=(const Variant& other) const { return !(*this == other); } private: diff --git a/include/utilities/document_db.h b/include/utilities/document_db.h index 1d1330bca..9c4a388ea 100644 --- a/include/utilities/document_db.h +++ b/include/utilities/document_db.h @@ -4,5 +4,9 @@ // of patent rights can be found in the PATENTS file in the same directory. #pragma once -#warning This file was moved to rocksdb/utilities/document_db.h + +#include "pragma_error.h" + +ROCKSDB_WARNING("This file was moved to rocksdb/utilities/document_db.h") + #include "rocksdb/utilities/document_db.h" diff --git a/include/utilities/geo_db.h b/include/utilities/geo_db.h index 48957d407..ae2e8c47f 100644 --- a/include/utilities/geo_db.h +++ b/include/utilities/geo_db.h @@ -4,5 +4,9 @@ // of patent rights can be found in the PATENTS file in the same directory. #pragma once -#warning This file was moved to rocksdb/utilities/geo_db.h + +#include "pragma_error.h" + +ROCKSDB_WARNING("This file was moved to rocksdb/utilities/geo_db.h") + #include "rocksdb/utilities/geo_db.h" diff --git a/include/utilities/json_document.h b/include/utilities/json_document.h index f3f93969d..fb8b7d304 100644 --- a/include/utilities/json_document.h +++ b/include/utilities/json_document.h @@ -3,5 +3,9 @@ // LICENSE file in the root directory of this source tree. An additional grant // of patent rights can be found in the PATENTS file in the same directory. #pragma once -#warning This file was moved to rocksdb/utilities/json_document.h + +#include "pragma_error.h" + +ROCKSDB_WARNING("This file was moved to rocksdb/utilities/json_document.h") + #include "rocksdb/utilities/json_document.h" diff --git a/include/utilities/stackable_db.h b/include/utilities/stackable_db.h index 435818d2b..8f72b04a6 100644 --- a/include/utilities/stackable_db.h +++ b/include/utilities/stackable_db.h @@ -3,5 +3,8 @@ // found in the LICENSE file. See the AUTHORS file for names of contributors. #pragma once -#warning This file was moved to rocksdb/utilities/stackable_db.h +#include "pragma_error.h" + +ROCKSDB_WARNING("This file was moved to rocksdb/utilities/stackable_db.h") + #include "rocksdb/utilities/stackable_db.h" diff --git a/port/port_posix.h b/port/port_posix.h index e9af0978d..75ae996c0 100644 --- a/port/port_posix.h +++ b/port/port_posix.h @@ -71,9 +71,14 @@ #define fdatasync fsync #endif +#include + namespace rocksdb { namespace port { +// For use at db/file_indexer.h kLevelMaxIndex +const int LevelMaxIndex = std::numeric_limits::max(); + static const bool kLittleEndian = PLATFORM_IS_LITTLE_ENDIAN; #undef PLATFORM_IS_LITTLE_ENDIAN diff --git a/port/sys_time.h b/port/sys_time.h index f416159e2..0ebf298aa 100644 --- a/port/sys_time.h +++ b/port/sys_time.h @@ -12,7 +12,7 @@ #ifndef STORAGE_LEVELDB_PORT_SYS_TIME_H_ #define STORAGE_LEVELDB_PORT_SYS_TIME_H_ -#if defined(_WIN32) && defined(_MSC_VER) +#if defined(OS_WIN) && defined(_MSC_VER) #include diff --git a/port/win/env_win.cc b/port/win/env_win.cc index c56873ce2..27c5c3fbd 100644 --- a/port/win/env_win.cc +++ b/port/win/env_win.cc @@ -229,10 +229,9 @@ size_t Roundup(size_t x, size_t y) { } -// Can only truncate or reserve to a sector size aligned if -// used on files that are opened with Unbuffered I/O -// Normally it does not present a problem since in memory mapped files -// we do not disable buffering +// SetFileInformationByHandle() is capable of fast pre-allocates. +// However, this does not change the file end position unless the file is +// truncated and the pre-allocated space is not considered filled with zeros. inline Status fallocate(const std::string& filename, HANDLE hFile, uint64_t to_size) { @@ -394,7 +393,7 @@ class WinMmapReadableFile : public RandomAccessFile { const size_t length_; public: - // base[0,length-1] contains the mmapped contents of the file. + // mapped_region_[0,length-1] contains the mmapped contents of the file. WinMmapReadableFile(const std::string &fileName, HANDLE hFile, HANDLE hMap, const void* mapped_region, size_t length) : fileName_(fileName), hFile_(hFile), hMap_(hMap), mapped_region_(mapped_region), length_(length) { @@ -1523,7 +1522,7 @@ public: NULL); } - if (hFile == INVALID_HANDLE_VALUE) { + if (INVALID_HANDLE_VALUE == hFile) { auto lastError = GetLastError(); s = IOErrorFromWindowsError("Failed to open NewSequentialFile" + fname, lastError); } else { @@ -1565,7 +1564,7 @@ public: /// Shared access is necessary for corruption test to pass // almost all tests would work with a possible exception of fault_injection - HANDLE hFile; + HANDLE hFile = 0; { IOSTATS_TIMER_GUARD(open_nanos); hFile = CreateFileA( @@ -1717,8 +1716,8 @@ public: FILE_ATTRIBUTE_NORMAL, NULL); } - - if (hFile == INVALID_HANDLE_VALUE) { + + if (INVALID_HANDLE_VALUE == hFile) { auto lastError = GetLastError(); s = IOErrorFromWindowsError("Failed to Open/Create NewRandomRWFile" + fname, lastError); } @@ -2018,7 +2017,7 @@ public: NULL); } - if (hFile == INVALID_HANDLE_VALUE) { + if (INVALID_HANDLE_VALUE == hFile) { auto lastError = GetLastError(); s = IOErrorFromWindowsError("Failed to open LogFile" + fname, lastError); } else { diff --git a/port/win/port_win.h b/port/win/port_win.h index 1c638878b..6e1a40bcd 100644 --- a/port/win/port_win.h +++ b/port/win/port_win.h @@ -62,7 +62,7 @@ typedef SSIZE_T ssize_t; #endif #ifdef SNAPPY -#include "snappy.h" +#include #endif // Thread local storage on Linux @@ -80,6 +80,9 @@ namespace rocksdb { namespace port { +// For use at db/file_indexer.h kLevelMaxIndex +const int LevelMaxIndex = INT32_MAX; + const bool kLittleEndian = true; class CondVar; @@ -87,8 +90,7 @@ class CondVar; class Mutex { public: - /* implicit */ - Mutex(bool adaptive = false); + /* implicit */ Mutex(bool adaptive = false); ~Mutex(); void Lock(); @@ -97,8 +99,8 @@ public: // this will assert if the mutex is not locked // it does NOT verify that mutex is held by a calling thread void AssertHeld(); - std::unique_lock& getLock() - { + + std::unique_lock& getLock() { return lock; } @@ -117,38 +119,36 @@ private: class RWMutex { -private: - SRWLOCK srwLock_; public: - RWMutex(){ - InitializeSRWLock(&srwLock_); + RWMutex() { + InitializeSRWLock(&srwLock_); } void ReadLock() { - AcquireSRWLockShared(&srwLock_); + AcquireSRWLockShared(&srwLock_); } void WriteLock() { - AcquireSRWLockExclusive(&srwLock_); + AcquireSRWLockExclusive(&srwLock_); } void ReadUnlock() { - ReleaseSRWLockShared(&srwLock_); + ReleaseSRWLockShared(&srwLock_); } void WriteUnlock() { - ReleaseSRWLockExclusive(&srwLock_); + ReleaseSRWLockExclusive(&srwLock_); } - void AssertHeld() { - //TODO: psrao - should be implemented - } + // Empty as in POSIX + void AssertHeld() { } private: - // No copying allowed - RWMutex(const RWMutex&); - void operator=(const RWMutex&); + SRWLOCK srwLock_; + // No copying allowed + RWMutex(const RWMutex&); + void operator=(const RWMutex&); }; class CondVar @@ -520,7 +520,7 @@ int pthread_key_create(pthread_key_t *key, void(*destructor)(void*)) { (void)destructor; pthread_key_t k = TlsAlloc(); - if (k == TLS_OUT_OF_INDEXES) { + if (TLS_OUT_OF_INDEXES == k) { return ENOMEM; } @@ -530,7 +530,7 @@ int pthread_key_create(pthread_key_t *key, void(*destructor)(void*)) { inline int pthread_key_delete(pthread_key_t key) { - if(!TlsFree(key)) { + if (!TlsFree(key)) { return EINVAL; } return 0; @@ -538,7 +538,7 @@ int pthread_key_delete(pthread_key_t key) { inline int pthread_setspecific(pthread_key_t key, const void *value) { - if(!TlsSetValue(key, const_cast(value))) { + if (!TlsSetValue(key, const_cast(value))) { return ENOMEM; } return 0; @@ -547,8 +547,8 @@ int pthread_setspecific(pthread_key_t key, const void *value) { inline void* pthread_getspecific(pthread_key_t key) { void* result = TlsGetValue(key); - if(!result) { - if(GetLastError() != ERROR_SUCCESS) { + if (!result) { + if (GetLastError() != ERROR_SUCCESS) { errno = EINVAL; } else { errno = NOERROR; diff --git a/port/win/win_logger.cc b/port/win/win_logger.cc index 1ef58e21b..44c279af3 100644 --- a/port/win/win_logger.cc +++ b/port/win/win_logger.cc @@ -24,8 +24,6 @@ namespace rocksdb { -//const int kDebugLogChunkSize = 128 * 1024; - WinLogger::WinLogger(uint64_t (*gettid)(), Env* env, FILE * file, const InfoLogLevel log_level) : Logger(log_level), gettid_(gettid), @@ -61,91 +59,93 @@ void WinLogger::Flush() { } void WinLogger::Logv(const char* format, va_list ap) { - const uint64_t thread_id = (*gettid_)(); - - IOSTATS_TIMER_GUARD(logger_nanos); - // We try twice: the first time with a fixed-size stack allocated buffer, - // and the second time with a much larger dynamically allocated buffer. - char buffer[500]; - std::unique_ptr largeBuffer; - for (int iter = 0; iter < 2; ++iter) { - char* base; - int bufsize; - if (iter == 0) { - bufsize = sizeof(buffer); - base = buffer; - } else { - bufsize = 30000; - largeBuffer.reset(new char[bufsize]); - base = largeBuffer.get(); - } - - char* p = base; - char* limit = base + bufsize; - - struct timeval now_tv; - gettimeofday(&now_tv, nullptr); - const time_t seconds = now_tv.tv_sec; - struct tm t; - localtime_s(&t, &seconds); - p += snprintf(p, limit - p, "%04d/%02d/%02d-%02d:%02d:%02d.%06d %llx ", t.tm_year + 1900, t.tm_mon + 1, t.tm_mday, - t.tm_hour, - t.tm_min, - t.tm_sec, - static_cast(now_tv.tv_usec), - static_cast(thread_id)); - - // Print the message - if (p < limit) { - va_list backup_ap; - va_copy(backup_ap, ap); - int done = vsnprintf(p, limit - p, format, backup_ap); - if (done > 0){ - p += done; - } else { - continue; - } - va_end(backup_ap); - } - - // Truncate to available space if necessary - if (p >= limit) { - if (iter == 0) - { - continue; // Try again with larger buffer - } else { - p = limit - 1; - } - } - - // Add newline if necessary - if (p == base || p[-1] != '\n') { - *p++ = '\n'; - } - - assert(p <= limit); - const size_t write_size = p - base; - - size_t sz = fwrite(base, 1, write_size, file_); - if (sz == 0) { - perror("fwrite .. [BAD]"); - } - - flush_pending_ = true; - assert(sz == write_size); - if (sz > 0) { - log_size_ += write_size; - } - - uint64_t now_micros = static_cast(now_tv.tv_sec) * 1000000 + - now_tv.tv_usec; - if (now_micros - last_flush_micros_ >= flush_every_seconds_ * 1000000) { - flush_pending_ = false; - fflush(file_); - last_flush_micros_ = now_micros; - } - break; - } + + IOSTATS_TIMER_GUARD(logger_nanos); + + const uint64_t thread_id = (*gettid_)(); + + // We try twice: the first time with a fixed-size stack allocated buffer, + // and the second time with a much larger dynamically allocated buffer. + char buffer[500]; + std::unique_ptr largeBuffer; + for (int iter = 0; iter < 2; ++iter) { + char* base; + int bufsize; + if (iter == 0) { + bufsize = sizeof(buffer); + base = buffer; + } else { + bufsize = 30000; + largeBuffer.reset(new char[bufsize]); + base = largeBuffer.get(); + } + + char* p = base; + char* limit = base + bufsize; + + struct timeval now_tv; + gettimeofday(&now_tv, nullptr); + const time_t seconds = now_tv.tv_sec; + struct tm t; + localtime_s(&t, &seconds); + p += snprintf(p, limit - p, "%04d/%02d/%02d-%02d:%02d:%02d.%06d %llx ", t.tm_year + 1900, t.tm_mon + 1, t.tm_mday, + t.tm_hour, + t.tm_min, + t.tm_sec, + static_cast(now_tv.tv_usec), + static_cast(thread_id)); + + // Print the message + if (p < limit) { + va_list backup_ap; + va_copy(backup_ap, ap); + int done = vsnprintf(p, limit - p, format, backup_ap); + if (done > 0){ + p += done; + } else { + continue; + } + va_end(backup_ap); + } + + // Truncate to available space if necessary + if (p >= limit) { + if (iter == 0) + { + continue; // Try again with larger buffer + } else { + p = limit - 1; + } + } + + // Add newline if necessary + if (p == base || p[-1] != '\n') { + *p++ = '\n'; + } + + assert(p <= limit); + const size_t write_size = p - base; + + size_t sz = fwrite(base, 1, write_size, file_); + if (sz == 0) { + perror("fwrite .. [BAD]"); + } + + flush_pending_ = true; + assert(sz == write_size); + if (sz > 0) { + log_size_ += write_size; + } + + uint64_t now_micros = static_cast(now_tv.tv_sec) * 1000000 + + now_tv.tv_usec; + if (now_micros - last_flush_micros_ >= flush_every_seconds_ * 1000000) { + flush_pending_ = false; + fflush(file_); + last_flush_micros_ = now_micros; + } + break; + } } size_t WinLogger::GetLogFileSize() const { diff --git a/port/win/win_logger.h b/port/win/win_logger.h index 0a9dabf4a..b7c85100b 100644 --- a/port/win/win_logger.h +++ b/port/win/win_logger.h @@ -23,30 +23,36 @@ class Env; const int kDebugLogChunkSize = 128 * 1024; class WinLogger : public rocksdb::Logger { -private: - FILE* file_; - uint64_t (*gettid_)(); // Return the thread id for the current thread - std::atomic_size_t log_size_; - std::atomic_uint_fast64_t last_flush_micros_; - Env* env_; - bool flush_pending_; +public: + WinLogger(uint64_t(*gettid)(), Env* env, FILE * file, + const InfoLogLevel log_level = InfoLogLevel::ERROR_LEVEL); - const static uint64_t flush_every_seconds_ = 5; + virtual ~WinLogger(); -public: - WinLogger(uint64_t(*gettid)(), Env* env, FILE * file, const InfoLogLevel log_level = InfoLogLevel::ERROR_LEVEL); + WinLogger(const WinLogger&) = delete; + + WinLogger& operator=(const WinLogger&) = delete; - virtual ~WinLogger(); + void close(); - void close(); + void Flush() override; - void Flush() override; + void Logv(const char* format, va_list ap) override; - void Logv(const char* format, va_list ap) override; + size_t GetLogFileSize() const override; + + void DebugWriter(const char* str, int len); + +private: - size_t GetLogFileSize() const override; + FILE* file_; + uint64_t(*gettid_)(); // Return the thread id for the current thread + std::atomic_size_t log_size_; + std::atomic_uint_fast64_t last_flush_micros_; + Env* env_; + bool flush_pending_; - void DebugWriter(const char* str, int len); + const static uint64_t flush_every_seconds_ = 5; }; } // namespace rocksdb diff --git a/table/block_based_table_builder.cc b/table/block_based_table_builder.cc index 006be6fde..4b58e9382 100644 --- a/table/block_based_table_builder.cc +++ b/table/block_based_table_builder.cc @@ -374,7 +374,7 @@ Slice CompressBlock(const Slice& raw, // kBlockBasedTableMagicNumber was picked by running // echo rocksdb.table.block_based | sha1sum // and taking the leading 64 bits. -// Please note that kBlockBasedTableMagicNumber may also be accessed by +// Please note that kBlockBasedTableMagicNumber may also be accessed by other .cc files // for that reason we declare it extern in the header but to get the space allocated // it must be not extern in one place. const uint64_t kBlockBasedTableMagicNumber = 0x88e241b785f4cff7ull; diff --git a/table/format.h b/table/format.h index 06eef47cd..7b231da28 100644 --- a/table/format.h +++ b/table/format.h @@ -182,26 +182,26 @@ struct BlockContents { BlockContents() : cachable(false), compression_type(kNoCompression) {} BlockContents(const Slice& _data, bool _cachable, - CompressionType _compression_type) - : data(_data), cachable(_cachable), compression_type(_compression_type) {} + CompressionType _compression_type) + : data(_data), cachable(_cachable), compression_type(_compression_type) {} BlockContents(std::unique_ptr&& _data, size_t _size, bool _cachable, - CompressionType _compression_type) - : data(_data.get(), _size), - cachable(_cachable), - compression_type(_compression_type), - allocation(std::move(_data)) {} + CompressionType _compression_type) + : data(_data.get(), _size), + cachable(_cachable), + compression_type(_compression_type), + allocation(std::move(_data)) {} BlockContents(BlockContents&& other) { - *this = std::move(other); + *this = std::move(other); } BlockContents& operator=(BlockContents&& other) { - data = std::move(other.data); - cachable = other.cachable; - compression_type = other.compression_type; - allocation = std::move(other.allocation); - return *this; + data = std::move(other.data); + cachable = other.cachable; + compression_type = other.compression_type; + allocation = std::move(other.allocation); + return *this; } }; diff --git a/third-party/fbson/FbsonStream.h b/third-party/fbson/FbsonStream.h index 6ac132bae..bd59ea172 100644 --- a/third-party/fbson/FbsonStream.h +++ b/third-party/fbson/FbsonStream.h @@ -30,6 +30,11 @@ #define __STDC_FORMAT_MACROS #endif +#if defined OS_WIN && !defined snprintf +# define snprintf _snprintf +#endif + + #include #include diff --git a/util/arena.cc b/util/arena.cc index 0a0acd55d..09c2bcedc 100644 --- a/util/arena.cc +++ b/util/arena.cc @@ -17,6 +17,11 @@ namespace rocksdb { +// MSVC complains that it is already defined since it is static in the header. +#ifndef OS_WIN +const size_t Arena::kInlineSize; +#endif + const size_t Arena::kMinBlockSize = 4096; const size_t Arena::kMaxBlockSize = 2 << 30; static const int kAlignUnit = sizeof(void*); @@ -54,8 +59,8 @@ Arena::~Arena() { for (const auto& block : blocks_) { delete[] block; } -// yuslepukhin: this needs to be addressed as it previously was under #ifdef -#ifndef OS_WIN + +#ifdef MAP_HUGETLB for (const auto& mmap_info : huge_blocks_) { auto ret = munmap(mmap_info.addr_, mmap_info.length_); if (ret != 0) { diff --git a/util/autovector.h b/util/autovector.h index 3d0820bf4..11fc4667b 100644 --- a/util/autovector.h +++ b/util/autovector.h @@ -240,7 +240,6 @@ class autovector { } void push_back(const T& item) { - //psrao: causes infinite recursion with VC if (num_stack_items_ < kSize) { values_[num_stack_items_++] = item; } diff --git a/util/dynamic_bloom.h b/util/dynamic_bloom.h index 58cad7c86..e2ac56e76 100644 --- a/util/dynamic_bloom.h +++ b/util/dynamic_bloom.h @@ -9,8 +9,7 @@ #include "rocksdb/slice.h" - -#include +#include "port/port.h" #include #include diff --git a/util/env.cc b/util/env.cc index 95b54c383..d98d926d8 100644 --- a/util/env.cc +++ b/util/env.cc @@ -10,7 +10,7 @@ #include "rocksdb/env.h" #include -#include +#include "port/sys_time.h" #include "rocksdb/options.h" #include "util/arena.h" diff --git a/util/log_buffer.cc b/util/log_buffer.cc index f4f016b84..7d15cf22e 100644 --- a/util/log_buffer.cc +++ b/util/log_buffer.cc @@ -5,8 +5,8 @@ #include "util/log_buffer.h" -#include -#include +#include "port/sys_time.h" +#include "port/port.h" namespace rocksdb { diff --git a/util/options_helper.cc b/util/options_helper.cc index 4de39f652..fdbedc7cd 100644 --- a/util/options_helper.cc +++ b/util/options_helper.cc @@ -417,6 +417,8 @@ bool ParseColumnFamilyOption(const std::string& name, const std::string& value, new_options->memtable_factory.reset(new_mem_factory); } else if (name == "min_write_buffer_number_to_merge") { new_options->min_write_buffer_number_to_merge = ParseInt(value); + } else if (name == "max_write_buffer_number_to_maintain") { + new_options->max_write_buffer_number_to_maintain = ParseInt(value); } else if (name == "compression") { new_options->compression = ParseCompressionType(value); } else if (name == "compression_per_level") { diff --git a/util/posix_logger.h b/util/posix_logger.h index 1d495973b..55cb34a86 100644 --- a/util/posix_logger.h +++ b/util/posix_logger.h @@ -13,7 +13,7 @@ #pragma once #include #include -#include +#include "port/sys_time.h" #include #include #ifdef OS_LINUX diff --git a/util/slice.cc b/util/slice.cc index 371272e0e..03d89e27f 100644 --- a/util/slice.cc +++ b/util/slice.cc @@ -11,6 +11,7 @@ #include "rocksdb/slice_transform.h" #include "rocksdb/slice.h" #include "util/string_util.h" +#include namespace rocksdb { @@ -93,6 +94,28 @@ class NoopTransform : public SliceTransform { } +// Do not want to include the whole /port/port.h here for one define +#ifdef OS_WIN +# define snprintf _snprintf +#endif + +// Return a string that contains the copy of the referenced data. +std::string Slice::ToString(bool hex) const { + std::string result; // RVO/NRVO/move + if (hex) { + char buf[10]; + for (size_t i = 0; i < size_; i++) { + snprintf(buf, 10, "%02X", (unsigned char)data_[i]); + result += buf; + } + return result; + } else { + result.assign(data_, size_); + return result; + } +} + + const SliceTransform* NewFixedPrefixTransform(size_t prefix_len) { return new FixedPrefixTransform(prefix_len); } diff --git a/util/thread_local.cc b/util/thread_local.cc index ca64a77d9..ab70c4a6f 100644 --- a/util/thread_local.cc +++ b/util/thread_local.cc @@ -42,15 +42,15 @@ pthread_key_t thread_local_key = -1; // Static callback function to call with each thread termination. void NTAPI WinOnThreadExit(PVOID module, DWORD reason, PVOID reserved) { - // We decided to punt on PROCESS_EXIT - if (DLL_THREAD_DETACH == reason) { - if (thread_local_key != -1 && thread_local_inclass_routine != nullptr) { - void* tls = pthread_getspecific(thread_local_key); - if(tls != nullptr) { - thread_local_inclass_routine(tls); - } - } - } + // We decided to punt on PROCESS_EXIT + if (DLL_THREAD_DETACH == reason) { + if (thread_local_key != -1 && thread_local_inclass_routine != nullptr) { + void* tls = pthread_getspecific(thread_local_key); + if(tls != nullptr) { + thread_local_inclass_routine(tls); + } + } + } } } // wintlscleanup diff --git a/utilities/checkpoint/checkpoint_test.cc b/utilities/checkpoint/checkpoint_test.cc index c143831c8..b01fe647e 100644 --- a/utilities/checkpoint/checkpoint_test.cc +++ b/utilities/checkpoint/checkpoint_test.cc @@ -8,7 +8,7 @@ // found in the LICENSE file. See the AUTHORS file for names of contributors. // Syncpoint prevents us building and running tests in release -#if !defined( NDEBUG) || !defined (OS_WIN) +#if !defined(NDEBUG) || !defined (OS_WIN) #ifndef OS_WIN # include @@ -351,7 +351,7 @@ TEST_F(DBTest, CheckpointCF) { #endif int main(int argc, char** argv) { -#if !defined( NDEBUG) || !defined (OS_WIN) +#if !defined(NDEBUG) || !defined (OS_WIN) rocksdb::port::InstallStackTraceHandler(); ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS();