Add appropriate parameters to make bulk-load go faster.

Summary:
1. Create only 2 levels so that manual compactions are fast.
2. Set target file size to a large value

Test Plan: make clean check

Reviewers: kailiu, zshao

Reviewed By: zshao

CC: leveldb

Differential Revision: https://reviews.facebook.net/D9231
main
Dhruba Borthakur 12 years ago
parent 3b6653b1f8
commit 469724be7f
  1. 2
      include/leveldb/options.h
  2. 23
      util/options.cc

@ -347,7 +347,7 @@ struct Options {
// Set appropriate parameters for bulk loading. // Set appropriate parameters for bulk loading.
// The reason that this is a function that returns "this" instead of a // The reason that this is a function that returns "this" instead of a
// constructure is to enable chaining of multiple similar calls in the future. // constructor is to enable chaining of multiple similar calls in the future.
// //
// All data will be in level 0 without any automatic compaction. // All data will be in level 0 without any automatic compaction.
// It's recommended to manually call CompactRange(NULL, NULL) before reading // It's recommended to manually call CompactRange(NULL, NULL) before reading

@ -161,17 +161,40 @@ Options::Dump(Logger* log) const
manifest_preallocation_size); manifest_preallocation_size);
} // Options::Dump } // Options::Dump
//
// The goal of this method is to create a configuration that
// allows an application to write all files into L0 and
// then do a single compaction to output all files into L1.
Options* Options*
Options::PrepareForBulkLoad() Options::PrepareForBulkLoad()
{ {
// never slowdown ingest.
level0_file_num_compaction_trigger = (1<<30); level0_file_num_compaction_trigger = (1<<30);
level0_slowdown_writes_trigger = (1<<30); level0_slowdown_writes_trigger = (1<<30);
level0_stop_writes_trigger = (1<<30); level0_stop_writes_trigger = (1<<30);
// no auto compactions please. The application should issue a
// manual compaction after all data is loaded into L0.
disable_auto_compactions = true; disable_auto_compactions = true;
disable_seek_compaction = true; disable_seek_compaction = true;
disableDataSync = true; disableDataSync = true;
// A manual compaction run should pick all files in L0 in
// a single compaction run.
source_compaction_factor = (1<<30); source_compaction_factor = (1<<30);
// It is better to have only 2 levels, otherwise a manual
// compaction would compact at every possible level, thereby
// increasing the total time needed for compactions.
num_levels = 2;
// Prevent a memtable flush to automatically promote files
// to L1. This is helpful so that all files that are
// input to the manual compaction are all at L0.
max_background_compactions = 2;
// The compaction would create large files in L1.
target_file_size_base = 256 * 1024 * 1024;
return this; return this;
} }

Loading…
Cancel
Save