Optimize for non-int and non-dup-sort types of databases using features

Signed-off-by: Victor Porof <victor.porof@gmail.com>

Cargo.toml

@@ -14,8 +14,10 @@ categories = ["database"]
 exclude = ["/tests/envs/*"]
 
 [features]
-default = []
+default = ["db-dup-sort", "db-int-key"]
 backtrace = ["failure/backtrace", "failure/std"]
+db-dup-sort = []
+db-int-key = []
 with-asan = ["lmdb-rkv/with-asan"]
 with-fuzzer = ["lmdb-rkv/with-fuzzer"]
 with-fuzzer-no-link = ["lmdb-rkv/with-fuzzer-no-link"]

src/backend/common.rs

@@ -25,7 +25,9 @@ pub enum EnvironmentFlags {
 
 pub enum DatabaseFlags {
     REVERSE_KEY,
+    #[cfg(feature = "db-dup-sort")]
     DUP_SORT,
+    #[cfg(feature = "db-int-key")]
     INTEGER_KEY,
     DUP_FIXED,
     INTEGER_DUP,

src/backend/impl_lmdb/flags.rs

@@ -84,7 +84,9 @@ impl Into<lmdb::DatabaseFlags> for DatabaseFlags {
     fn into(self) -> lmdb::DatabaseFlags {
         match self {
             DatabaseFlags::REVERSE_KEY => lmdb::DatabaseFlags::REVERSE_KEY,
+            #[cfg(feature = "db-dup-sort")]
             DatabaseFlags::DUP_SORT => lmdb::DatabaseFlags::DUP_SORT,
+            #[cfg(feature = "db-int-key")]
             DatabaseFlags::INTEGER_KEY => lmdb::DatabaseFlags::INTEGER_KEY,
             DatabaseFlags::DUP_FIXED => lmdb::DatabaseFlags::DUP_FIXED,
             DatabaseFlags::INTEGER_DUP => lmdb::DatabaseFlags::INTEGER_DUP,

src/backend/impl_lmdb/transaction.rs

@@ -63,6 +63,12 @@ impl<'env> BackendRwTransaction for RwTransactionImpl<'env> {
         self.0.put(db.0, &key, &value, flags.0).map_err(ErrorImpl)
     }
 
+    #[cfg(not(feature = "db-dup-sort"))]
+    fn del(&mut self, db: &Self::Database, key: &[u8]) -> Result<(), Self::Error> {
+        self.0.del(db.0, &key, None).map_err(ErrorImpl)
+    }
+
+    #[cfg(feature = "db-dup-sort")]
     fn del(&mut self, db: &Self::Database, key: &[u8], value: Option<&[u8]>) -> Result<(), Self::Error> {
         self.0.del(db.0, &key, value).map_err(ErrorImpl)
     }

src/backend/impl_safe.rs

@@ -15,6 +15,7 @@ mod error;
 mod flags;
 mod info;
 mod iter;
+mod snapshot;
 mod stat;
 mod transaction;
 

src/backend/impl_safe/cursor.rs

@@ -9,7 +9,7 @@
 // specific language governing permissions and limitations under the License.
 
 use super::{
-    database::Snapshot,
+    snapshot::Snapshot,
     IterImpl,
 };
 use crate::backend::traits::BackendRoCursor;

src/backend/impl_safe/database.rs

@@ -8,18 +8,13 @@
 // CONDITIONS OF ANY KIND, either express or implied. See the License for the
 // specific language governing permissions and limitations under the License.
 
-use std::collections::{
-    BTreeMap,
-    BTreeSet,
-};
-use std::sync::Arc;
-
 use id_arena::Id;
 use serde_derive::{
     Deserialize,
     Serialize,
 };
 
+use super::snapshot::Snapshot;
 use super::DatabaseFlagsImpl;
 use crate::backend::traits::BackendDatabase;
 
@@ -47,89 +42,3 @@ impl DatabaseImpl {
         std::mem::replace(&mut self.snapshot, snapshot)
     }
 }
-
-type Key = Box<[u8]>;
-type Value = Box<[u8]>;
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct Snapshot {
-    flags: DatabaseFlagsImpl,
-    map: Arc<BTreeMap<Key, BTreeSet<Value>>>,
-}
-
-impl Snapshot {
-    pub(crate) fn new(flags: Option<DatabaseFlagsImpl>) -> Snapshot {
-        Snapshot {
-            flags: flags.unwrap_or_else(DatabaseFlagsImpl::default),
-            map: Default::default(),
-        }
-    }
-
-    pub(crate) fn flags(&self) -> &DatabaseFlagsImpl {
-        &self.flags
-    }
-
-    pub(crate) fn get(&self, key: &[u8]) -> Option<&[u8]> {
-        self.map.get(key).and_then(|v| v.iter().next()).map(|v| v.as_ref())
-    }
-
-    pub(crate) fn put_one(&mut self, key: &[u8], value: &[u8]) {
-        let map = Arc::make_mut(&mut self.map);
-        match map.get_mut(key) {
-            None => {
-                let mut values = BTreeSet::new();
-                values.insert(Box::from(value));
-                map.insert(Box::from(key), values);
-            },
-            Some(values) => {
-                values.clear();
-                values.insert(Box::from(value));
-            },
-        }
-    }
-
-    pub(crate) fn put_dup(&mut self, key: &[u8], value: &[u8]) {
-        let map = Arc::make_mut(&mut self.map);
-        match map.get_mut(key) {
-            None => {
-                let mut values = BTreeSet::new();
-                values.insert(Box::from(value));
-                map.insert(Box::from(key), values);
-            },
-            Some(values) => {
-                values.insert(Box::from(value));
-            },
-        }
-    }
-
-    pub(crate) fn del_exact(&mut self, key: &[u8], value: &[u8]) -> Option<()> {
-        let map = Arc::make_mut(&mut self.map);
-        match map.get_mut(key) {
-            None => None,
-            Some(values) => {
-                let was_removed = values.remove(value);
-                Some(()).filter(|_| was_removed)
-            },
-        }
-    }
-
-    pub(crate) fn del_all(&mut self, key: &[u8]) -> Option<()> {
-        let map = Arc::make_mut(&mut self.map);
-        match map.get_mut(key) {
-            None => None,
-            Some(values) => {
-                let was_empty = values.is_empty();
-                values.clear();
-                Some(()).filter(|_| !was_empty)
-            },
-        }
-    }
-
-    pub(crate) fn clear(&mut self) {
-        self.map = Default::default();
-    }
-
-    pub(crate) fn iter(&self) -> impl Iterator<Item = (&[u8], &[u8])> {
-        self.map.iter().flat_map(|(key, values)| values.iter().map(move |value| (key.as_ref(), value.as_ref())))
-    }
-}

src/backend/impl_safe/flags.rs

@@ -66,7 +66,10 @@ impl Into<EnvironmentFlagsImpl> for EnvironmentFlags {
 bitflags! {
     #[derive(Default, Serialize, Deserialize)]
     pub struct DatabaseFlagsImpl: u32 {
+        const NIL = 0b0000_0000;
+        #[cfg(feature = "db-dup-sort")]
         const DUP_SORT = 0b0000_0001;
+        #[cfg(feature = "db-int-key")]
         const INTEGER_KEY = 0b0000_0010;
     }
 }
@@ -87,7 +90,9 @@ impl Into<DatabaseFlagsImpl> for DatabaseFlags {
     fn into(self) -> DatabaseFlagsImpl {
         match self {
             DatabaseFlags::REVERSE_KEY => unimplemented!(),
+            #[cfg(feature = "db-dup-sort")]
             DatabaseFlags::DUP_SORT => DatabaseFlagsImpl::DUP_SORT,
+            #[cfg(feature = "db-int-key")]
             DatabaseFlags::INTEGER_KEY => DatabaseFlagsImpl::INTEGER_KEY,
             DatabaseFlags::DUP_FIXED => unimplemented!(),
             DatabaseFlags::INTEGER_DUP => unimplemented!(),

src/backend/impl_safe/snapshot.rs

@@ -0,0 +1,138 @@
+// Copyright 2018-2019 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+use std::collections::{
+    BTreeMap,
+    BTreeSet,
+};
+use std::sync::Arc;
+
+use serde_derive::{
+    Deserialize,
+    Serialize,
+};
+
+use super::DatabaseFlagsImpl;
+
+type Key = Box<[u8]>;
+type Value = Box<[u8]>;
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct Snapshot {
+    flags: DatabaseFlagsImpl,
+    #[cfg(not(feature = "db-dup-sort"))]
+    map: Arc<BTreeMap<Key, Value>>,
+    #[cfg(feature = "db-dup-sort")]
+    map: Arc<BTreeMap<Key, BTreeSet<Value>>>,
+}
+
+impl Snapshot {
+    pub(crate) fn new(flags: Option<DatabaseFlagsImpl>) -> Snapshot {
+        Snapshot {
+            flags: flags.unwrap_or_else(DatabaseFlagsImpl::default),
+            map: Default::default(),
+        }
+    }
+
+    pub(crate) fn flags(&self) -> &DatabaseFlagsImpl {
+        &self.flags
+    }
+
+    pub(crate) fn clear(&mut self) {
+        self.map = Default::default();
+    }
+}
+
+#[cfg(not(feature = "db-dup-sort"))]
+impl Snapshot {
+    pub(crate) fn get(&self, key: &[u8]) -> Option<&[u8]> {
+        self.map.get(key).map(|value| value.as_ref())
+    }
+
+    pub(crate) fn put(&mut self, key: &[u8], value: &[u8]) {
+        let map = Arc::make_mut(&mut self.map);
+        map.insert(Box::from(key), Box::from(value));
+    }
+
+    pub(crate) fn del(&mut self, key: &[u8]) -> Option<()> {
+        let map = Arc::make_mut(&mut self.map);
+        map.remove(key).map(|_| ())
+    }
+
+    pub(crate) fn iter(&self) -> impl Iterator<Item = (&[u8], &[u8])> {
+        self.map.iter().map(|(key, value)| (key.as_ref(), value.as_ref()))
+    }
+}
+
+#[cfg(feature = "db-dup-sort")]
+impl Snapshot {
+    pub(crate) fn get(&self, key: &[u8]) -> Option<&[u8]> {
+        self.map.get(key).and_then(|v| v.iter().next()).map(|v| v.as_ref())
+    }
+
+    pub(crate) fn put(&mut self, key: &[u8], value: &[u8]) {
+        let map = Arc::make_mut(&mut self.map);
+        match map.get_mut(key) {
+            None => {
+                let mut values = BTreeSet::new();
+                values.insert(Box::from(value));
+                map.insert(Box::from(key), values);
+            },
+            Some(values) => {
+                values.clear();
+                values.insert(Box::from(value));
+            },
+        }
+    }
+
+    pub(crate) fn del(&mut self, key: &[u8]) -> Option<()> {
+        let map = Arc::make_mut(&mut self.map);
+        match map.get_mut(key) {
+            None => None,
+            Some(values) => {
+                let was_empty = values.is_empty();
+                values.clear();
+                Some(()).filter(|_| !was_empty)
+            },
+        }
+    }
+
+    pub(crate) fn iter(&self) -> impl Iterator<Item = (&[u8], &[u8])> {
+        self.map.iter().flat_map(|(key, values)| values.iter().map(move |value| (key.as_ref(), value.as_ref())))
+    }
+}
+
+#[cfg(feature = "db-dup-sort")]
+impl Snapshot {
+    pub(crate) fn put_dup(&mut self, key: &[u8], value: &[u8]) {
+        let map = Arc::make_mut(&mut self.map);
+        match map.get_mut(key) {
+            None => {
+                let mut values = BTreeSet::new();
+                values.insert(Box::from(value));
+                map.insert(Box::from(key), values);
+            },
+            Some(values) => {
+                values.insert(Box::from(value));
+            },
+        }
+    }
+
+    pub(crate) fn del_exact(&mut self, key: &[u8], value: &[u8]) -> Option<()> {
+        let map = Arc::make_mut(&mut self.map);
+        match map.get_mut(key) {
+            None => None,
+            Some(values) => {
+                let was_removed = values.remove(value);
+                Some(()).filter(|_| was_removed)
+            },
+        }
+    }
+}

src/backend/impl_safe/transaction.rs

@@ -12,8 +12,7 @@ use std::collections::HashMap;
 use std::sync::Arc;
 
 use super::{
-    database::Snapshot,
-    DatabaseFlagsImpl,
+    snapshot::Snapshot,
     DatabaseId,
     EnvironmentImpl,
     ErrorImpl,
@@ -96,21 +95,39 @@ impl<'env> BackendRwTransaction for RwTransactionImpl<'env> {
         snapshot.get(key).ok_or_else(|| ErrorImpl::KeyValuePairNotFound)
     }
 
+    #[cfg(not(feature = "db-dup-sort"))]
     fn put(&mut self, db: &Self::Database, key: &[u8], value: &[u8], _flags: Self::Flags) -> Result<(), Self::Error> {
+        let snapshot = self.snapshots.get_mut(db).ok_or_else(|| ErrorImpl::DbIsForeignError)?;
+        snapshot.put(key, value);
+        Ok(())
+    }
+
+    #[cfg(feature = "db-dup-sort")]
+    fn put(&mut self, db: &Self::Database, key: &[u8], value: &[u8], _flags: Self::Flags) -> Result<(), Self::Error> {
+        use super::DatabaseFlagsImpl;
         let snapshot = self.snapshots.get_mut(db).ok_or_else(|| ErrorImpl::DbIsForeignError)?;
         if snapshot.flags().contains(DatabaseFlagsImpl::DUP_SORT) {
             snapshot.put_dup(key, value);
         } else {
-            snapshot.put_one(key, value);
+            snapshot.put(key, value);
         }
         Ok(())
     }
 
+    #[cfg(not(feature = "db-dup-sort"))]
+    fn del(&mut self, db: &Self::Database, key: &[u8]) -> Result<(), Self::Error> {
+        let snapshot = self.snapshots.get_mut(db).ok_or_else(|| ErrorImpl::DbIsForeignError)?;
+        let deleted = snapshot.del(key);
+        Ok(deleted.ok_or_else(|| ErrorImpl::KeyValuePairNotFound)?)
+    }
+
+    #[cfg(feature = "db-dup-sort")]
     fn del(&mut self, db: &Self::Database, key: &[u8], value: Option<&[u8]>) -> Result<(), Self::Error> {
+        use super::DatabaseFlagsImpl;
         let snapshot = self.snapshots.get_mut(db).ok_or_else(|| ErrorImpl::DbIsForeignError)?;
         let deleted = match (value, snapshot.flags()) {
             (Some(value), flags) if flags.contains(DatabaseFlagsImpl::DUP_SORT) => snapshot.del_exact(key, value),
-            _ => snapshot.del_all(key),
+            _ => snapshot.del(key),
         };
         Ok(deleted.ok_or_else(|| ErrorImpl::KeyValuePairNotFound)?)
     }

src/backend/traits.rs

@@ -133,6 +133,10 @@ pub trait BackendRwTransaction: Debug {
 
     fn put(&mut self, db: &Self::Database, key: &[u8], value: &[u8], flags: Self::Flags) -> Result<(), Self::Error>;
 
+    #[cfg(not(feature = "db-dup-sort"))]
+    fn del(&mut self, db: &Self::Database, key: &[u8]) -> Result<(), Self::Error>;
+
+    #[cfg(feature = "db-dup-sort")]
     fn del(&mut self, db: &Self::Database, key: &[u8], value: Option<&[u8]>) -> Result<(), Self::Error>;
 
     fn clear_db(&mut self, db: &Self::Database) -> Result<(), Self::Error>;

src/env.rs

@@ -14,15 +14,18 @@ use std::path::{
     PathBuf,
 };
 
+#[cfg(any(feature = "db-dup-sort", feature = "db-int-key"))]
 use crate::backend::{
     BackendDatabaseFlags,
+    DatabaseFlags,
+};
+use crate::backend::{
     BackendEnvironment,
     BackendEnvironmentBuilder,
     BackendInfo,
     BackendRoCursorTransaction,
     BackendRwCursorTransaction,
     BackendStat,
-    DatabaseFlags,
     SafeModeError,
 };
 use crate::error::StoreError;
@@ -30,13 +33,20 @@ use crate::readwrite::{
     Reader,
     Writer,
 };
-use crate::store::integer::IntegerStore;
-use crate::store::integermulti::MultiIntegerStore;
-use crate::store::keys::PrimitiveInt;
-use crate::store::multi::MultiStore;
 use crate::store::single::SingleStore;
 use crate::store::Options as StoreOptions;
 
+#[cfg(feature = "db-dup-sort")]
+use crate::store::multi::MultiStore;
+
+#[cfg(feature = "db-int-key")]
+use crate::store::integer::IntegerStore;
+#[cfg(feature = "db-int-key")]
+use crate::store::keys::PrimitiveInt;
+
+#[cfg(all(feature = "db-dup-sort", feature = "db-int-key"))]
+use crate::store::integermulti::MultiIntegerStore;
+
 pub static DEFAULT_MAX_DBS: c_uint = 5;
 
 /// Wrapper around an `Environment` (e.g. an LMDB environment).
@@ -124,6 +134,7 @@ where
     /// Create or Open an existing database in (Integer -> Single Value) mode.
     /// Note: that create=true cannot be called concurrently with other operations
     /// so if you are sure that the database exists, call this with create=false.
+    #[cfg(feature = "db-int-key")]
     pub fn open_integer<'s, T, K>(
         &self,
         name: T,
@@ -140,6 +151,7 @@ where
     /// Create or Open an existing database in (&[u8] -> Multiple Values) mode.
     /// Note: that create=true cannot be called concurrently with other operations
     /// so if you are sure that the database exists, call this with create=false.
+    #[cfg(feature = "db-dup-sort")]
     pub fn open_multi<'s, T>(
         &self,
         name: T,
@@ -155,6 +167,7 @@ where
     /// Create or Open an existing database in (Integer -> Multiple Values) mode.
     /// Note: that create=true cannot be called concurrently with other operations
     /// so if you are sure that the database exists, call this with create=false.
+    #[cfg(all(feature = "db-dup-sort", feature = "db-int-key"))]
     pub fn open_multi_integer<'s, T, K>(
         &self,
         name: T,
@@ -631,6 +644,7 @@ mod tests {
     }
 
     #[test]
+    #[cfg(feature = "db-dup-sort")]
     fn test_multi_put_get_del() {
         let root = Builder::new().prefix("test_multi_put_get_del").tempdir().expect("tempdir");
         fs::create_dir_all(root.path()).expect("dir created");
@@ -668,6 +682,7 @@ mod tests {
     }
 
     #[test]
+    #[cfg(feature = "db-dup-sort")]
     fn test_multiple_store_clear() {
         let root = Builder::new().prefix("test_multiple_store_clear").tempdir().expect("tempdir");
         fs::create_dir_all(root.path()).expect("dir created");
@@ -953,6 +968,7 @@ mod tests {
     }
 
     #[test]
+    #[cfg(feature = "db-int-key")]
     fn test_stat() {
         let root = Builder::new().prefix("test_stat").tempdir().expect("tempdir");
         fs::create_dir_all(root.path()).expect("dir created");
@@ -1673,6 +1689,7 @@ mod tests_safe {
     }
 
     #[test]
+    #[cfg(feature = "db-dup-sort")]
     fn test_multi_put_get_del_safe() {
         let root = Builder::new().prefix("test_multi_put_get_del_safe").tempdir().expect("tempdir");
         fs::create_dir_all(root.path()).expect("dir created");
@@ -1710,6 +1727,7 @@ mod tests_safe {
     }
 
     #[test]
+    #[cfg(feature = "db-dup-sort")]
     fn test_multiple_store_clear_safe() {
         let root = Builder::new().prefix("test_multiple_store_clear_safe").tempdir().expect("tempdir");
         fs::create_dir_all(root.path()).expect("dir created");

src/lib.rs

@@ -229,16 +229,21 @@ pub use readwrite::{
     Reader,
     Writer,
 };
-pub use store::integer::IntegerStore;
-pub use store::integermulti::MultiIntegerStore;
-pub use store::keys::{
-    EncodableKey,
-    PrimitiveInt,
-};
-pub use store::multi::MultiStore;
+pub use store::keys::EncodableKey;
 pub use store::single::SingleStore;
 pub use store::Options as StoreOptions;
 pub use value::{
     OwnedValue,
     Value,
 };
+
+#[cfg(feature = "db-dup-sort")]
+pub use store::multi::MultiStore;
+
+#[cfg(feature = "db-int-key")]
+pub use store::integer::IntegerStore;
+#[cfg(feature = "db-int-key")]
+pub use store::keys::PrimitiveInt;
+
+#[cfg(all(feature = "db-dup-sort", feature = "db-int-key"))]
+pub use store::integermulti::MultiIntegerStore;

src/readwrite.rs

@@ -115,6 +115,15 @@ where
         self.0.put(db, k.as_ref(), &v.to_bytes()?, flags).map_err(|e| e.into())
     }
 
+    #[cfg(not(feature = "db-dup-sort"))]
+    pub(crate) fn delete<K>(&mut self, db: &T::Database, k: &K) -> Result<(), StoreError>
+    where
+        K: AsRef<[u8]>,
+    {
+        self.0.del(db, k.as_ref()).map_err(|e| e.into())
+    }
+
+    #[cfg(feature = "db-dup-sort")]
     pub(crate) fn delete<K>(&mut self, db: &T::Database, k: &K, v: Option<&[u8]>) -> Result<(), StoreError>
     where
         K: AsRef<[u8]>,

src/store.rs

@@ -8,12 +8,18 @@
 // CONDITIONS OF ANY KIND, either express or implied. See the License for the
 // specific language governing permissions and limitations under the License.
 
-pub mod integer;
-pub mod integermulti;
 pub mod keys;
-pub mod multi;
 pub mod single;
 
+#[cfg(feature = "db-dup-sort")]
+pub mod multi;
+
+#[cfg(feature = "db-int-key")]
+pub mod integer;
+
+#[cfg(all(feature = "db-dup-sort", feature = "db-int-key"))]
+pub mod integermulti;
+
 use crate::backend::BackendDatabaseFlags;
 
 #[derive(Default, Debug, Copy, Clone)]

src/store/single.rs

@@ -65,6 +65,16 @@ where
         writer.put(&self.db, &k, v, T::Flags::empty())
     }
 
+    #[cfg(not(feature = "db-dup-sort"))]
+    pub fn delete<T, K>(&self, writer: &mut Writer<T>, k: K) -> EmptyResult
+    where
+        T: BackendRwTransaction<Database = D>,
+        K: AsRef<[u8]>,
+    {
+        writer.delete(&self.db, &k)
+    }
+
+    #[cfg(feature = "db-dup-sort")]
     pub fn delete<T, K>(&self, writer: &mut Writer<T>, k: K) -> EmptyResult
     where
         T: BackendRwTransaction<Database = D>,

tests/integer-store.rs

@@ -7,6 +7,7 @@
 // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
 // CONDITIONS OF ANY KIND, either express or implied. See the License for the
 // specific language governing permissions and limitations under the License.
+#![cfg(feature = "db-int-key")]
 
 use std::fs;
 

tests/multi-integer-store.rs

@@ -7,6 +7,7 @@
 // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
 // CONDITIONS OF ANY KIND, either express or implied. See the License for the
 // specific language governing permissions and limitations under the License.
+#![cfg(all(feature = "db-dup-sort", feature = "db-int-key"))]
 
 use std::fs;
 

tests/test_txn.rs

@@ -7,6 +7,7 @@
 // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
 // CONDITIONS OF ANY KIND, either express or implied. See the License for the
 // specific language governing permissions and limitations under the License.
+#![cfg(feature = "db-dup-sort")]
 
 use std::fs;