Bulk load: use megabytes as unit

pull/190/head
Tpt 3 years ago
parent 49804d3b46
commit 54c6f2c19f
  1. 6
      lib/src/storage/mod.rs
  2. 10
      lib/src/store.rs

@ -1178,7 +1178,7 @@ impl StorageBulkLoader {
self self
} }
pub fn set_max_memory_size_in_bytes(mut self, max_memory_size: usize) -> Self { pub fn set_max_memory_size_in_megabytes(mut self, max_memory_size: usize) -> Self {
self.max_memory_size = Some(max_memory_size); self.max_memory_size = Some(max_memory_size);
self self
} }
@ -1198,7 +1198,7 @@ impl StorageBulkLoader {
} else if let Some(max_memory_size) = self.max_memory_size { } else if let Some(max_memory_size) = self.max_memory_size {
min( min(
num_cpus::get(), num_cpus::get(),
max_memory_size / 1000 / DEFAULT_BULK_LOAD_BATCH_SIZE, max_memory_size * 1000 / DEFAULT_BULK_LOAD_BATCH_SIZE,
) )
} else { } else {
num_cpus::get() num_cpus::get()
@ -1206,7 +1206,7 @@ impl StorageBulkLoader {
2, 2,
); );
let batch_size = if let Some(max_memory_size) = self.max_memory_size { let batch_size = if let Some(max_memory_size) = self.max_memory_size {
max(1000, max_memory_size / 1000 / num_threads) max(1000, max_memory_size * 1000 / num_threads)
} else { } else {
DEFAULT_BULK_LOAD_BATCH_SIZE DEFAULT_BULK_LOAD_BATCH_SIZE
}; };

@ -1272,7 +1272,7 @@ impl Iterator for GraphNameIter {
/// Results might get weird if you delete data during the loading process. /// Results might get weird if you delete data during the loading process.
/// ///
/// Warning: It is optimized for speed. /// Warning: It is optimized for speed.
/// Memory usage is configurable using [`BulkLoader::set_max_memory_size_in_bytes`] /// Memory usage is configurable using [`BulkLoader::set_max_memory_size_in_megabytes`]
/// and the number of used threads with [`BulkLoader::set_num_threads`]. /// and the number of used threads with [`BulkLoader::set_num_threads`].
/// By default the memory consumption target (excluding the system and RocksDB internal consumption) /// By default the memory consumption target (excluding the system and RocksDB internal consumption)
/// is 1GB per thread and the number of threads is set to the number of logical CPU cores provided by the system. /// is 1GB per thread and the number of threads is set to the number of logical CPU cores provided by the system.
@ -1307,7 +1307,7 @@ impl BulkLoader {
/// This number must be at last 2 (one for parsing and one for loading). /// This number must be at last 2 (one for parsing and one for loading).
/// ///
/// By default this is the number of logical CPU cores provided by the system except if /// By default this is the number of logical CPU cores provided by the system except if
/// [`BulkLoader::set_max_memory_size_in_bytes`] is set. In this case at least one 1GB is reserved /// [`BulkLoader::set_max_memory_size_in_megabytes`] is set. In this case at least one 1GB is reserved
/// per used thread. /// per used thread.
pub fn set_num_threads(self, num_threads: usize) -> Self { pub fn set_num_threads(self, num_threads: usize) -> Self {
Self { Self {
@ -1324,9 +1324,11 @@ impl BulkLoader {
/// ///
/// By default, at most 1GB per used thread is used /// By default, at most 1GB per used thread is used
/// (i.e. at most GBs at the number of available logical CPU cores in total). /// (i.e. at most GBs at the number of available logical CPU cores in total).
pub fn set_max_memory_size_in_bytes(self, max_memory_size: usize) -> Self { pub fn set_max_memory_size_in_megabytes(self, max_memory_size: usize) -> Self {
Self { Self {
storage: self.storage.set_max_memory_size_in_bytes(max_memory_size), storage: self
.storage
.set_max_memory_size_in_megabytes(max_memory_size),
} }
} }

Loading…
Cancel
Save