NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix Windows build issues (#1253)

Browse Source
main
Dmitri Smirnov 8 years ago committed by Siying Dong
parent 2306167d30
commit e4609a749b
9 changed files with 45 additions and 30 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 3
      CMakeLists.txt
  2. 2
      utilities/col_buf_decoder.cc
  3. 6
      utilities/column_aware_encoding_exp.cc
  4. 10
      utilities/column_aware_encoding_util.cc
  5. 9
      utilities/persistent_cache/block_cache_tier.h
  6. 7
      utilities/persistent_cache/block_cache_tier_file.cc
  7. 4
      utilities/persistent_cache/block_cache_tier_file.h
  8. 32
      utilities/persistent_cache/persistent_cache_test.cc
  9. 2
      utilities/persistent_cache/persistent_cache_tier.h

3
CMakeLists.txt
Unescape View File

@ -265,6 +265,9 @@ set(SOURCES
utilities/merge_operators/max.cc utilities/merge_operators/max.cc
utilities/merge_operators/uint64add.cc utilities/merge_operators/uint64add.cc
utilities/options/options_util.cc utilities/options/options_util.cc
utilities/persistent_cache/block_cache_tier.cc
utilities/persistent_cache/block_cache_tier_file.cc
utilities/persistent_cache/block_cache_tier_metadata.cc
utilities/persistent_cache/persistent_cache_tier.cc utilities/persistent_cache/persistent_cache_tier.cc
utilities/persistent_cache/volatile_tier_impl.cc utilities/persistent_cache/volatile_tier_impl.cc
utilities/redis/redis_lists.cc utilities/redis/redis_lists.cc

2
utilities/col_buf_decoder.cc
Unescape View File

@ -229,7 +229,7 @@ size_t VariableChunkColBufDecoder::Decode(const char* src, char** dest) {
} }
memcpy(*dest, reinterpret_cast<char*>(&chunk_buf), 8); memcpy(*dest, reinterpret_cast<char*>(&chunk_buf), 8);
*dest += 8; *dest += 8;
uint8_t mask = 0xFF - 8 + chunk_size; uint8_t mask = ((0xFF - 8) + chunk_size) & 0xFF;
memcpy(*dest, reinterpret_cast<char*>(&mask), 1); memcpy(*dest, reinterpret_cast<char*>(&mask), 1);
*dest += 1; *dest += 1;
} }

6
utilities/column_aware_encoding_exp.cc
Unescape View File

@ -106,10 +106,10 @@ class ColumnAwareEncodingExp {
if (encoded_out_file != nullptr) { if (encoded_out_file != nullptr) {
uint64_t size = 0; uint64_t size = 0;
env->GetFileSize(FLAGS_encoded_file, &size); env->GetFileSize(FLAGS_encoded_file, &size);
fprintf(stdout, "File size: %lu\n", size); fprintf(stdout, "File size: %llu\n", size);
} }
uint64_t encode_time = sw.ElapsedNanosSafe(false /* reset */); uint64_t encode_time = sw.ElapsedNanosSafe(false /* reset */);
fprintf(stdout, "Encode time:%lu\n", encode_time); fprintf(stdout, "Encode time:%llu\n", encode_time);
if (decode) { if (decode) {
unique_ptr<WritableFile> decoded_out_file; unique_ptr<WritableFile> decoded_out_file;
if (!FLAGS_decoded_file.empty()) { if (!FLAGS_decoded_file.empty()) {
@ -124,7 +124,7 @@ class ColumnAwareEncodingExp {
&encoded_blocks); &encoded_blocks);
} }
uint64_t decode_time = sw.ElapsedNanosSafe(true /* reset */); uint64_t decode_time = sw.ElapsedNanosSafe(true /* reset */);
fprintf(stdout, "Decode time:%lu\n", decode_time); fprintf(stdout, "Decode time:%llu\n", decode_time);
} }
} else { } else {
fprintf(stdout, "Unsupported compression type: %s.\n", fprintf(stdout, "Unsupported compression type: %s.\n",

10
utilities/column_aware_encoding_util.cc
Unescape View File

@ -29,6 +29,8 @@
#include "utilities/col_buf_decoder.h" #include "utilities/col_buf_decoder.h"
#include "utilities/col_buf_encoder.h" #include "utilities/col_buf_encoder.h"
#include "port/port.h"
namespace rocksdb { namespace rocksdb {
ColumnAwareEncodingReader::ColumnAwareEncodingReader( ColumnAwareEncodingReader::ColumnAwareEncodingReader(
@ -195,7 +197,7 @@ void ColumnAwareEncodingReader::DumpDataColumns(
FILE* fp = fopen(filename.c_str(), "w"); FILE* fp = fopen(filename.c_str(), "w");
size_t block_id = 1; size_t block_id = 1;
for (auto& kv_pairs : kv_pair_blocks) { for (auto& kv_pairs : kv_pair_blocks) {
fprintf(fp, "---------------- Block: %-4lu ----------------\n", block_id); fprintf(fp, "---------------- Block: %-4" ROCKSDB_PRIszt " ----------------\n", block_id);
for (auto& kv_pair : kv_pairs) { for (auto& kv_pair : kv_pairs) {
const auto& key = kv_pair.first; const auto& key = kv_pair.first;
const auto& value = kv_pair.second; const auto& value = kv_pair.second;
@ -421,12 +423,12 @@ Status ColumnAwareEncodingReader::EncodeBlocks(
total_size += value_checksum_size; total_size += value_checksum_size;
for (size_t i = 0; i < key_col_sizes.size(); ++i) for (size_t i = 0; i < key_col_sizes.size(); ++i)
printf("Key col %lu size: %lu percentage %lf%%\n", i, key_col_sizes[i], printf("Key col %" ROCKSDB_PRIszt " size: %" ROCKSDB_PRIszt " percentage %lf%%\n", i, key_col_sizes[i],
100.0 * key_col_sizes[i] / total_size); 100.0 * key_col_sizes[i] / total_size);
for (size_t i = 0; i < value_col_sizes.size(); ++i) for (size_t i = 0; i < value_col_sizes.size(); ++i)
printf("Value col %lu size: %lu percentage %lf%%\n", i, printf("Value col %" ROCKSDB_PRIszt " size: %" ROCKSDB_PRIszt " percentage %lf%%\n", i,
value_col_sizes[i], 100.0 * value_col_sizes[i] / total_size); value_col_sizes[i], 100.0 * value_col_sizes[i] / total_size);
printf("Value checksum size: %lu percentage %lf%%\n", value_checksum_size, printf("Value checksum size: %" ROCKSDB_PRIszt " percentage %lf%%\n", value_checksum_size,
100.0 * value_checksum_size / total_size); 100.0 * value_checksum_size / total_size);
} }
return Status::OK(); return Status::OK();

9
utilities/persistent_cache/block_cache_tier.h vendored
Unescape View File

@ -4,7 +4,10 @@
// of patent rights can be found in the PATENTS file in the same directory. // of patent rights can be found in the PATENTS file in the same directory.
#pragma once #pragma once
#ifndef OS_WIN
#include <unistd.h> #include <unistd.h>
#endif // ! OS_WIN
#include <list> #include <list>
#include <memory> #include <memory>
#include <set> #include <set>
@ -22,7 +25,7 @@
#include "utilities/persistent_cache/persistent_cache_util.h" #include "utilities/persistent_cache/persistent_cache_util.h"
#include "db/skiplist.h" #include "db/skiplist.h"
#include "port/port_posix.h" #include "port/port.h"
#include "util/arena.h" #include "util/arena.h"
#include "util/coding.h" #include "util/coding.h"
#include "util/crc32c.h" #include "util/crc32c.h"
@ -65,7 +68,11 @@ class BlockCacheTier : public PersistentCacheTier {
void TEST_Flush() override { void TEST_Flush() override {
while (insert_ops_.Size()) { while (insert_ops_.Size()) {
#ifdef OS_WIN
Sleep(1000);
#else
/* sleep override */ sleep(1); /* sleep override */ sleep(1);
#endif
} }
} }

7
utilities/persistent_cache/block_cache_tier_file.cc vendored
Unescape View File

@ -6,7 +6,9 @@
#include "utilities/persistent_cache/block_cache_tier_file.h" #include "utilities/persistent_cache/block_cache_tier_file.h"
#ifndef OS_WIN
#include <unistd.h> #include <unistd.h>
#endif
#include <memory> #include <memory>
#include <vector> #include <vector>
@ -353,7 +355,7 @@ bool WriteableCacheFile::ExpandBuffer(const size_t size) {
return false; return false;
} }
size_ += buf->Free(); size_ += static_cast<uint32_t>(buf->Free());
free += buf->Free(); free += buf->Free();
bufs_.push_back(buf); bufs_.push_back(buf);
} }
@ -550,7 +552,8 @@ void ThreadedWriter::ThreadMain() {
while (!cache_->Reserve(io.buf_->Used())) { while (!cache_->Reserve(io.buf_->Used())) {
// We can fail to reserve space if every file in the system // We can fail to reserve space if every file in the system
// is being currently accessed // is being currently accessed
/* sleep override */ sleep(1); /* sleep override */
Env::Default()->SleepForMicroseconds(1000000);
} }
DispatchIO(io); DispatchIO(io);

4
utilities/persistent_cache/block_cache_tier_file.h vendored
Unescape View File

@ -19,7 +19,7 @@
#include "utilities/persistent_cache/persistent_cache_tier.h" #include "utilities/persistent_cache/persistent_cache_tier.h"
#include "utilities/persistent_cache/persistent_cache_util.h" #include "utilities/persistent_cache/persistent_cache_util.h"
#include "port/port_posix.h" #include "port/port.h"
#include "util/crc32c.h" #include "util/crc32c.h"
#include "util/mutexlock.h" #include "util/mutexlock.h"
@ -200,7 +200,7 @@ class WriteableCacheFile : public RandomAccessCacheFile {
} }
// append data to end of file // append data to end of file
bool Append(const Slice&, const Slice&, LBA*) override; bool Append(const Slice&, const Slice&, LBA* const) override;
// End-of-file // End-of-file
bool Eof() const { return eof_; } bool Eof() const { return eof_; }

32
utilities/persistent_cache/persistent_cache_test.cc vendored
Unescape View File

@ -78,7 +78,7 @@ std::unique_ptr<PersistentCacheTier> NewBlockCache(
Env* env, const std::string& path, Env* env, const std::string& path,
const uint64_t max_size = std::numeric_limits<uint64_t>::max(), const uint64_t max_size = std::numeric_limits<uint64_t>::max(),
const bool enable_direct_writes = false) { const bool enable_direct_writes = false) {
const uint32_t max_file_size = 12 * 1024 * 1024 * kStressFactor; const uint32_t max_file_size = static_cast<uint32_t>(12 * 1024 * 1024 * kStressFactor);
auto log = std::make_shared<ConsoleLogger>(); auto log = std::make_shared<ConsoleLogger>();
PersistentCacheConfig opt(env, path, max_size, log); PersistentCacheConfig opt(env, path, max_size, log);
opt.cache_file_size = max_file_size; opt.cache_file_size = max_file_size;
@ -95,7 +95,7 @@ std::unique_ptr<PersistentTieredCache> NewTieredCache(
Env* env, const std::string& path, const uint64_t max_volatile_cache_size, Env* env, const std::string& path, const uint64_t max_volatile_cache_size,
const uint64_t max_block_cache_size = const uint64_t max_block_cache_size =
std::numeric_limits<uint64_t>::max()) { std::numeric_limits<uint64_t>::max()) {
const uint32_t max_file_size = 12 * 1024 * 1024 * kStressFactor; const uint32_t max_file_size = static_cast<uint32_t>(12 * 1024 * 1024 * kStressFactor);
auto log = std::make_shared<ConsoleLogger>(); auto log = std::make_shared<ConsoleLogger>();
auto opt = PersistentCacheConfig(env, path, max_block_cache_size, log); auto opt = PersistentCacheConfig(env, path, max_block_cache_size, log);
opt.cache_file_size = max_file_size; opt.cache_file_size = max_file_size;
@ -122,7 +122,7 @@ TEST_F(PersistentCacheTierTest, VolatileCacheInsert) {
for (auto max_keys : for (auto max_keys :
{10 * 1024 * kStressFactor, 1 * 1024 * 1024 * kStressFactor}) { {10 * 1024 * kStressFactor, 1 * 1024 * 1024 * kStressFactor}) {
cache_ = std::make_shared<VolatileCacheTier>(); cache_ = std::make_shared<VolatileCacheTier>();
RunInsertTest(nthreads, max_keys); RunInsertTest(nthreads, static_cast<size_t>(max_keys));
} }
} }
} }
@ -131,8 +131,8 @@ TEST_F(PersistentCacheTierTest, VolatileCacheInsertWithEviction) {
for (auto nthreads : {1, 5}) { for (auto nthreads : {1, 5}) {
for (auto max_keys : {1 * 1024 * 1024 * kStressFactor}) { for (auto max_keys : {1 * 1024 * 1024 * kStressFactor}) {
cache_ = std::make_shared<VolatileCacheTier>( cache_ = std::make_shared<VolatileCacheTier>(
/*compressed=*/true, /*size=*/1 * 1024 * 1024 * kStressFactor); /*compressed=*/true, /*size=*/static_cast<size_t>(1 * 1024 * 1024 * kStressFactor));
RunInsertTestWithEviction(nthreads, max_keys); RunInsertTestWithEviction(nthreads, static_cast<size_t>(max_keys));
} }
} }
} }
@ -146,7 +146,7 @@ TEST_F(PersistentCacheTierTest, BlockCacheInsert) {
cache_ = NewBlockCache(Env::Default(), path_, cache_ = NewBlockCache(Env::Default(), path_,
/*size=*/std::numeric_limits<uint64_t>::max(), /*size=*/std::numeric_limits<uint64_t>::max(),
direct_writes); direct_writes);
RunInsertTest(nthreads, max_keys); RunInsertTest(nthreads, static_cast<size_t>(max_keys));
} }
} }
} }
@ -156,8 +156,8 @@ TEST_F(PersistentCacheTierTest, BlockCacheInsertWithEviction) {
for (auto nthreads : {1, 5}) { for (auto nthreads : {1, 5}) {
for (auto max_keys : {1 * 1024 * 1024 * kStressFactor}) { for (auto max_keys : {1 * 1024 * 1024 * kStressFactor}) {
cache_ = NewBlockCache(Env::Default(), path_, cache_ = NewBlockCache(Env::Default(), path_,
/*max_size=*/200 * 1024 * 1024 * kStressFactor); /*max_size=*/static_cast<size_t>(200 * 1024 * 1024 * kStressFactor));
RunInsertTestWithEviction(nthreads, max_keys); RunInsertTestWithEviction(nthreads, static_cast<size_t>(max_keys));
} }
} }
} }
@ -168,8 +168,8 @@ TEST_F(PersistentCacheTierTest, TieredCacheInsert) {
for (auto max_keys : for (auto max_keys :
{10 * 1024 * kStressFactor, 1 * 1024 * 1024 * kStressFactor}) { {10 * 1024 * kStressFactor, 1 * 1024 * 1024 * kStressFactor}) {
cache_ = NewTieredCache(Env::Default(), path_, cache_ = NewTieredCache(Env::Default(), path_,
/*memory_size=*/1 * 1024 * 1024 * kStressFactor); /*memory_size=*/static_cast<size_t>(1 * 1024 * 1024 * kStressFactor));
RunInsertTest(nthreads, max_keys); RunInsertTest(nthreads, static_cast<size_t>(max_keys));
} }
} }
} }
@ -179,9 +179,9 @@ TEST_F(PersistentCacheTierTest, TieredCacheInsertWithEviction) {
for (auto max_keys : {1 * 1024 * 1024 * kStressFactor}) { for (auto max_keys : {1 * 1024 * 1024 * kStressFactor}) {
cache_ = NewTieredCache( cache_ = NewTieredCache(
Env::Default(), path_, Env::Default(), path_,
/*memory_size=*/1 * 1024 * 1024 * kStressFactor, /*memory_size=*/static_cast<size_t>(1 * 1024 * 1024 * kStressFactor),
/*block_cache_size*/ 200 * 1024 * 1024 * kStressFactor); /*block_cache_size*/ static_cast<size_t>(200 * 1024 * 1024 * kStressFactor));
RunInsertTestWithEviction(nthreads, max_keys); RunInsertTestWithEviction(nthreads, static_cast<size_t>(max_keys));
} }
} }
} }
@ -198,7 +198,7 @@ std::shared_ptr<PersistentCacheTier> MakeBlockCache(const std::string& dbname) {
std::shared_ptr<PersistentCacheTier> MakeTieredCache( std::shared_ptr<PersistentCacheTier> MakeTieredCache(
const std::string& dbname) { const std::string& dbname) {
const auto memory_size = 1 * 1024 * 1024 * kStressFactor; const auto memory_size = 1 * 1024 * 1024 * kStressFactor;
return NewTieredCache(Env::Default(), dbname, memory_size); return NewTieredCache(Env::Default(), dbname, static_cast<size_t>(memory_size));
} }
#ifdef OS_LINUX #ifdef OS_LINUX
@ -233,12 +233,12 @@ void PersistentCacheDBTest::RunTest(
} }
// number of insertion interations // number of insertion interations
int num_iter = 100 * 1024 * kStressFactor; int num_iter = static_cast<int>(100 * 1024 * kStressFactor);
for (int iter = 0; iter < 5; iter++) { for (int iter = 0; iter < 5; iter++) {
Options options; Options options;
options.write_buffer_size = options.write_buffer_size =
64 * 1024 * kStressFactor; // small write buffer static_cast<size_t>(64 * 1024 * kStressFactor); // small write buffer
options.statistics = rocksdb::CreateDBStatistics(); options.statistics = rocksdb::CreateDBStatistics();
options = CurrentOptions(options); options = CurrentOptions(options);

2
utilities/persistent_cache/persistent_cache_tier.h vendored
Unescape View File

@ -200,7 +200,7 @@ struct PersistentCacheConfig {
// file size in order to avoid dead lock. // file size in order to avoid dead lock.
size_t write_buffer_count() const { size_t write_buffer_count() const {
assert(write_buffer_size); assert(write_buffer_size);
return (writer_qdepth + 1.2) * cache_file_size / write_buffer_size; return (writer_qdepth + writer_qdepth/5) * cache_file_size / write_buffer_size;
} }
// writer-dispatch-size // writer-dispatch-size

Write Preview
Loading…
Cancel
Save