Remove bulk loading and auto_roll_logger in rocksdb_lite

Summary:
shrink lite size
Closes https://github.com/facebook/rocksdb/pull/1929

Differential Revision: D4622059

Pulled By: siying

fbshipit-source-id: 050b796
main
Aaron Gao 8 years ago committed by Facebook Github Bot
parent 90d8355075
commit e877afa08b
  1. 23
      db/auto_roll_logger.cc
  2. 2
      db/auto_roll_logger.h
  3. 14
      db/auto_roll_logger_test.cc
  4. 1
      db/db_impl.cc
  5. 2
      db/db_impl.h
  6. 4
      db/external_sst_file_ingestion_job.cc
  7. 18
      db/external_sst_file_test.cc
  8. 1
      include/rocksdb/options.h
  9. 4
      include/rocksdb/sst_file_writer.h
  10. 5
      table/sst_file_writer.cc
  11. 2
      util/env_posix.cc
  12. 5
      util/env_test.cc

@ -8,6 +8,7 @@
namespace rocksdb {
#ifndef ROCKSDB_LITE
// -- AutoRollLogger
Status AutoRollLogger::ResetLogger() {
TEST_SYNC_POINT("AutoRollLogger::ResetLogger:BeforeNewLogger");
@ -130,6 +131,7 @@ bool AutoRollLogger::LogExpired() {
++cached_now_access_count;
return cached_now >= ctime_ + kLogFileTimeToRoll;
}
#endif // !ROCKSDB_LITE
Status CreateLoggerFromOptions(const std::string& dbname,
const DBOptions& options,
@ -147,6 +149,7 @@ Status CreateLoggerFromOptions(const std::string& dbname,
env->CreateDirIfMissing(dbname); // In case it does not exist
// Currently we only support roll by time-to-roll and log size
#ifndef ROCKSDB_LITE
if (options.log_file_time_to_roll > 0 || options.max_log_file_size > 0) {
AutoRollLogger* result = new AutoRollLogger(
env, dbname, options.db_log_dir, options.max_log_file_size,
@ -158,17 +161,17 @@ Status CreateLoggerFromOptions(const std::string& dbname,
logger->reset(result);
}
return s;
} else {
// Open a log file in the same directory as the db
env->RenameFile(
fname, OldInfoLogFileName(dbname, env->NowMicros(), db_absolute_path,
options.db_log_dir));
auto s = env->NewLogger(fname, logger);
if (logger->get() != nullptr) {
(*logger)->SetInfoLogLevel(options.info_log_level);
}
return s;
}
#endif // !ROCKSDB_LITE
// Open a log file in the same directory as the db
env->RenameFile(fname,
OldInfoLogFileName(dbname, env->NowMicros(), db_absolute_path,
options.db_log_dir));
auto s = env->NewLogger(fname, logger);
if (logger->get() != nullptr) {
(*logger)->SetInfoLogLevel(options.info_log_level);
}
return s;
}
} // namespace rocksdb

@ -18,6 +18,7 @@
namespace rocksdb {
#ifndef ROCKSDB_LITE
// Rolls the log file by size and/or time
class AutoRollLogger : public Logger {
public:
@ -120,6 +121,7 @@ class AutoRollLogger : public Logger {
uint64_t call_NowMicros_every_N_records_;
mutable port::Mutex mutex_;
};
#endif // !ROCKSDB_LITE
// Facade to craete logger automatically
Status CreateLoggerFromOptions(const std::string& dbname,

@ -3,6 +3,9 @@
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
//
#ifndef ROCKSDB_LITE
#include <string>
#include <thread>
#include <vector>
@ -469,3 +472,14 @@ int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
#else
#include <stdio.h>
int main(int argc, char** argv) {
fprintf(stderr,
"SKIPPED as AutoRollLogger is not supported in ROCKSDB_LITE\n");
return 0;
}
#endif // !ROCKSDB_LITE

@ -148,6 +148,7 @@ DBOptions SanitizeOptions(const std::string& dbname, const DBOptions& src) {
result.info_log = nullptr;
}
}
if (!result.write_buffer_manager) {
result.write_buffer_manager.reset(
new WriteBufferManager(result.db_write_buffer_size));

@ -565,8 +565,10 @@ class DBImpl : public DB {
void NotifyOnMemTableSealed(ColumnFamilyData* cfd,
const MemTableInfo& mem_table_info);
#ifndef ROCKSDB_LITE
void NotifyOnExternalFileIngested(
ColumnFamilyData* cfd, const ExternalSstFileIngestionJob& ingestion_job);
#endif // !ROCKSDB_LITE
void NewThreadStatusCfInfo(ColumnFamilyData* cfd) const;

@ -3,6 +3,8 @@
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
#ifndef ROCKSDB_LITE
#include "db/external_sst_file_ingestion_job.h"
#define __STDC_FORMAT_MACROS
@ -525,3 +527,5 @@ bool ExternalSstFileIngestionJob::IngestedFileFitInLevel(
}
} // namespace rocksdb
#endif // !ROCKSDB_LITE

@ -3,8 +3,9 @@
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
#include <functional>
#ifndef ROCKSDB_LITE
#include <functional>
#include "db/db_test_util.h"
#include "port/port.h"
#include "port/stack_trace.h"
@ -13,7 +14,6 @@
namespace rocksdb {
#ifndef ROCKSDB_LITE
class ExternalSSTFileTest : public DBTestBase {
public:
ExternalSSTFileTest() : DBTestBase("/external_sst_file_test") {
@ -1986,8 +1986,6 @@ TEST_F(ExternalSSTFileTest, FadviseTrigger) {
rocksdb::SyncPoint::GetInstance()->DisableProcessing();
}
#endif // ROCKSDB_LITE
} // namespace rocksdb
int main(int argc, char** argv) {
@ -1995,3 +1993,15 @@ int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
#else
#include <stdio.h>
int main(int argc, char** argv) {
fprintf(stderr,
"SKIPPED as External SST File Writer and Ingestion are not supported "
"in ROCKSDB_LITE\n");
return 0;
}
#endif // !ROCKSDB_LITE

@ -514,6 +514,7 @@ struct DBOptions {
// If specified with non-zero value, log file will be rolled
// if it has been active longer than `log_file_time_to_roll`.
// Default: 0 (disabled)
// Not supported in ROCKSDB_LITE mode!
size_t log_file_time_to_roll = 0;
// Maximal info log files to be kept.

@ -3,6 +3,8 @@
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
#ifndef ROCKSDB_LITE
#pragma once
#include <string>
#include "rocksdb/env.h"
@ -79,3 +81,5 @@ class SstFileWriter {
Rep* rep_;
};
} // namespace rocksdb
#endif // !ROCKSDB_LITE

@ -19,6 +19,9 @@ const std::string ExternalSstFilePropertyNames::kVersion =
"rocksdb.external_sst_file.version";
const std::string ExternalSstFilePropertyNames::kGlobalSeqno =
"rocksdb.external_sst_file.global_seqno";
#ifndef ROCKSDB_LITE
const size_t kFadviseTrigger = 1024 * 1024; // 1MB
struct SstFileWriter::Rep {
@ -223,4 +226,6 @@ void SstFileWriter::InvalidatePageCache(bool closing) {
uint64_t SstFileWriter::FileSize() {
return rep_->file_info.file_size;
}
#endif // !ROCKSDB_LITE
} // namespace rocksdb

@ -269,7 +269,7 @@ class PosixEnv : public Env {
// More info here: https://linux.die.net/man/2/pwrite
#ifdef ROCKSDB_LITE
return Status::IOError(fname, "Direct I/O not supported in RocksDB lite");
#endif // !ROCKSDB_LITE
#endif // ROCKSDB_LITE
flags |= O_WRONLY;
#ifndef OS_MACOSX
flags |= O_DIRECT;

@ -690,16 +690,14 @@ class IoctlFriendlyTmpdir {
std::string dir_;
};
#ifndef ROCKSDB_LITE
TEST_F(EnvPosixTest, PositionedAppend) {
unique_ptr<WritableFile> writable_file;
EnvOptions options;
options.use_direct_writes = true;
options.use_mmap_writes = false;
IoctlFriendlyTmpdir ift;
ASSERT_OK(env_->NewWritableFile(ift.name() + "/f", &writable_file, options));
const size_t kBlockSize = 4096;
const size_t kPageSize = 4096;
const size_t kDataSize = kPageSize;
@ -724,6 +722,7 @@ TEST_F(EnvPosixTest, PositionedAppend) {
ASSERT_EQ('a', result[kBlockSize - 1]);
ASSERT_EQ('b', result[kBlockSize]);
}
#endif // !ROCKSDB_LITE
// Only works in linux platforms
TEST_P(EnvPosixTestWithParam, RandomAccessUniqueID) {

Loading…
Cancel
Save