From 54c6f2c19f95e6f2f08b0919f414e0e6c087a82f Mon Sep 17 00:00:00 2001 From: Tpt Date: Sat, 12 Feb 2022 19:59:39 +0100 Subject: [PATCH] Bulk load: use megabytes as unit --- lib/src/storage/mod.rs | 6 +++--- lib/src/store.rs | 10 ++++++---- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/lib/src/storage/mod.rs b/lib/src/storage/mod.rs index 0de9658f..bb4a0315 100644 --- a/lib/src/storage/mod.rs +++ b/lib/src/storage/mod.rs @@ -1178,7 +1178,7 @@ impl StorageBulkLoader { self } - pub fn set_max_memory_size_in_bytes(mut self, max_memory_size: usize) -> Self { + pub fn set_max_memory_size_in_megabytes(mut self, max_memory_size: usize) -> Self { self.max_memory_size = Some(max_memory_size); self } @@ -1198,7 +1198,7 @@ impl StorageBulkLoader { } else if let Some(max_memory_size) = self.max_memory_size { min( num_cpus::get(), - max_memory_size / 1000 / DEFAULT_BULK_LOAD_BATCH_SIZE, + max_memory_size * 1000 / DEFAULT_BULK_LOAD_BATCH_SIZE, ) } else { num_cpus::get() @@ -1206,7 +1206,7 @@ impl StorageBulkLoader { 2, ); let batch_size = if let Some(max_memory_size) = self.max_memory_size { - max(1000, max_memory_size / 1000 / num_threads) + max(1000, max_memory_size * 1000 / num_threads) } else { DEFAULT_BULK_LOAD_BATCH_SIZE }; diff --git a/lib/src/store.rs b/lib/src/store.rs index ec374535..215479e3 100644 --- a/lib/src/store.rs +++ b/lib/src/store.rs @@ -1272,7 +1272,7 @@ impl Iterator for GraphNameIter { /// Results might get weird if you delete data during the loading process. /// /// Warning: It is optimized for speed. -/// Memory usage is configurable using [`BulkLoader::set_max_memory_size_in_bytes`] +/// Memory usage is configurable using [`BulkLoader::set_max_memory_size_in_megabytes`] /// and the number of used threads with [`BulkLoader::set_num_threads`]. /// By default the memory consumption target (excluding the system and RocksDB internal consumption) /// is 1GB per thread and the number of threads is set to the number of logical CPU cores provided by the system. @@ -1307,7 +1307,7 @@ impl BulkLoader { /// This number must be at last 2 (one for parsing and one for loading). /// /// By default this is the number of logical CPU cores provided by the system except if - /// [`BulkLoader::set_max_memory_size_in_bytes`] is set. In this case at least one 1GB is reserved + /// [`BulkLoader::set_max_memory_size_in_megabytes`] is set. In this case at least one 1GB is reserved /// per used thread. pub fn set_num_threads(self, num_threads: usize) -> Self { Self { @@ -1324,9 +1324,11 @@ impl BulkLoader { /// /// By default, at most 1GB per used thread is used /// (i.e. at most GBs at the number of available logical CPU cores in total). - pub fn set_max_memory_size_in_bytes(self, max_memory_size: usize) -> Self { + pub fn set_max_memory_size_in_megabytes(self, max_memory_size: usize) -> Self { Self { - storage: self.storage.set_max_memory_size_in_bytes(max_memory_size), + storage: self + .storage + .set_max_memory_size_in_megabytes(max_memory_size), } }