@ -1,9 +1,9 @@
#![ allow(clippy::same_name_method) ] #![ allow(clippy::same_name_method) ]
#[ cfg(all(not(target_family = " wasm " ))) ] #[ cfg(all(not(target_family = " wasm " ),not(docsrs) )) ]
use crate ::oxigraph ::model ::Quad ; use crate ::oxigraph ::model ::Quad ;
use crate ::oxigraph ::model ::{ GraphNameRef , NamedOrBlankNodeRef , QuadRef , TermRef } ; use crate ::oxigraph ::model ::{ GraphNameRef , NamedOrBlankNodeRef , QuadRef , TermRef } ;
use crate ::oxigraph ::storage ::backend ::{ Reader , Transaction } ; use crate ::oxigraph ::storage ::backend ::{ Reader , Transaction } ;
#[ cfg(all(not(target_family = " wasm " ))) ] #[ cfg(all(not(target_family = " wasm " ),not(docsrs) )) ]
use crate ::oxigraph ::storage ::binary_encoder ::LATEST_STORAGE_VERSION ; use crate ::oxigraph ::storage ::binary_encoder ::LATEST_STORAGE_VERSION ;
use crate ::oxigraph ::storage ::binary_encoder ::{ use crate ::oxigraph ::storage ::binary_encoder ::{
decode_term , encode_term , encode_term_pair , encode_term_quad , encode_term_triple , decode_term , encode_term , encode_term_pair , encode_term_quad , encode_term_triple ,
@ -14,24 +14,24 @@ use crate::oxigraph::storage::binary_encoder::{
pub use crate ::oxigraph ::storage ::error ::{ pub use crate ::oxigraph ::storage ::error ::{
CorruptionError , LoaderError , SerializerError , StorageError , CorruptionError , LoaderError , SerializerError , StorageError ,
} ; } ;
#[ cfg(all(not(target_family = " wasm " ))) ] #[ cfg(all(not(target_family = " wasm " ),not(docsrs) )) ]
use crate ::oxigraph ::storage ::numeric_encoder ::Decoder ; use crate ::oxigraph ::storage ::numeric_encoder ::Decoder ;
use crate ::oxigraph ::storage ::numeric_encoder ::{ use crate ::oxigraph ::storage ::numeric_encoder ::{
insert_term , EncodedQuad , EncodedTerm , StrHash , StrLookup , insert_term , EncodedQuad , EncodedTerm , StrHash , StrLookup ,
} ; } ;
use backend ::{ ColumnFamily , ColumnFamilyDefinition , Db , Iter } ; use backend ::{ ColumnFamily , ColumnFamilyDefinition , Db , Iter } ;
#[ cfg(all(not(target_family = " wasm " ))) ] #[ cfg(all(not(target_family = " wasm " ),not(docsrs) )) ]
use std ::collections ::VecDeque ; use std ::collections ::VecDeque ;
#[ cfg(all(not(target_family = " wasm " ))) ] #[ cfg(all(not(target_family = " wasm " ),not(docsrs) )) ]
use std ::collections ::{ HashMap , HashSet } ; use std ::collections ::{ HashMap , HashSet } ;
use std ::error ::Error ; use std ::error ::Error ;
#[ cfg(all(not(target_family = " wasm " ))) ] #[ cfg(all(not(target_family = " wasm " ),not(docsrs) )) ]
use std ::mem ::{ swap , take } ; use std ::mem ::{ swap , take } ;
#[ cfg(all(not(target_family = " wasm " ))) ] #[ cfg(all(not(target_family = " wasm " ),not(docsrs) )) ]
use std ::path ::{ Path , PathBuf } ; use std ::path ::{ Path , PathBuf } ;
#[ cfg(all(not(target_family = " wasm " ))) ] #[ cfg(all(not(target_family = " wasm " ),not(docsrs) )) ]
use std ::sync ::Mutex ; use std ::sync ::Mutex ;
#[ cfg(all(not(target_family = " wasm " ))) ] #[ cfg(all(not(target_family = " wasm " ),not(docsrs) )) ]
use std ::{ io , thread } ; use std ::{ io , thread } ;
mod backend ; mod backend ;
@ -51,16 +51,16 @@ const DSPO_CF: &str = "dspo";
const DPOS_CF : & str = "dpos" ; const DPOS_CF : & str = "dpos" ;
const DOSP_CF : & str = "dosp" ; const DOSP_CF : & str = "dosp" ;
const GRAPHS_CF : & str = "graphs" ; const GRAPHS_CF : & str = "graphs" ;
#[ cfg(all(not(target_family = " wasm " ))) ] #[ cfg(all(not(target_family = " wasm " ),not(docsrs) )) ]
const DEFAULT_CF : & str = "default" ; const DEFAULT_CF : & str = "default" ;
#[ cfg(all(not(target_family = " wasm " ))) ] #[ cfg(all(not(target_family = " wasm " ),not(docsrs) )) ]
const DEFAULT_BULK_LOAD_BATCH_SIZE : usize = 1_000_000 ; const DEFAULT_BULK_LOAD_BATCH_SIZE : usize = 1_000_000 ;
/// Low level storage primitives
/// Low level storage primitives
#[ derive(Clone) ] #[ derive(Clone) ]
pub struct Storage { pub struct Storage {
db : Db , db : Db ,
#[ cfg(all(not(target_family = " wasm " ))) ] #[ cfg(all(not(target_family = " wasm " ),not(docsrs) )) ]
default_cf : ColumnFamily , default_cf : ColumnFamily ,
id2str_cf : ColumnFamily , id2str_cf : ColumnFamily ,
spog_cf : ColumnFamily , spog_cf : ColumnFamily ,
@ -80,7 +80,7 @@ impl Storage {
Self ::setup ( Db ::new ( Self ::column_families ( ) ) ? ) Self ::setup ( Db ::new ( Self ::column_families ( ) ) ? )
} }
#[ cfg(all(not(target_family = " wasm " ))) ] #[ cfg(all(not(target_family = " wasm " ),not(docsrs) )) ]
pub fn open ( path : & Path , key : Option < [ u8 ; 32 ] > ) -> Result < Self , StorageError > { pub fn open ( path : & Path , key : Option < [ u8 ; 32 ] > ) -> Result < Self , StorageError > {
Self ::setup ( Db ::open_read_write ( Self ::setup ( Db ::open_read_write (
Some ( path ) , Some ( path ) ,
@ -89,7 +89,7 @@ impl Storage {
) ? ) ) ? )
} }
// #[cfg(all(not(target_family = "wasm")))]
// #[cfg(all(not(target_family = "wasm"),not(docsrs) ))]
// pub fn open_secondary(primary_path: &Path) -> Result<Self, StorageError> {
// pub fn open_secondary(primary_path: &Path) -> Result<Self, StorageError> {
// Self::setup(Db::open_secondary(
// Self::setup(Db::open_secondary(
// primary_path,
// primary_path,
@ -98,7 +98,7 @@ impl Storage {
// )?)
// )?)
// }
// }
// #[cfg(all(not(target_family = "wasm")))]
// #[cfg(all(not(target_family = "wasm"),not(docsrs) ))]
// pub fn open_persistent_secondary(
// pub fn open_persistent_secondary(
// primary_path: &Path,
// primary_path: &Path,
// secondary_path: &Path,
// secondary_path: &Path,
@ -110,7 +110,7 @@ impl Storage {
// )?)
// )?)
// }
// }
#[ cfg(all(not(target_family = " wasm " ))) ] #[ cfg(all(not(target_family = " wasm " ),not(docsrs) )) ]
pub fn open_read_only ( path : & Path , key : Option < [ u8 ; 32 ] > ) -> Result < Self , StorageError > { pub fn open_read_only ( path : & Path , key : Option < [ u8 ; 32 ] > ) -> Result < Self , StorageError > {
Self ::setup ( Db ::open_read_only ( path , Self ::column_families ( ) , key ) ? ) Self ::setup ( Db ::open_read_only ( path , Self ::column_families ( ) , key ) ? )
} }
@ -188,7 +188,7 @@ impl Storage {
fn setup ( db : Db ) -> Result < Self , StorageError > { fn setup ( db : Db ) -> Result < Self , StorageError > {
let this = Self { let this = Self {
#[ cfg(all(not(target_family = " wasm " ))) ] #[ cfg(all(not(target_family = " wasm " ),not(docsrs) )) ]
default_cf : db . column_family ( DEFAULT_CF ) ? , default_cf : db . column_family ( DEFAULT_CF ) ? ,
id2str_cf : db . column_family ( ID2STR_CF ) ? , id2str_cf : db . column_family ( ID2STR_CF ) ? ,
spog_cf : db . column_family ( SPOG_CF ) ? , spog_cf : db . column_family ( SPOG_CF ) ? ,
@ -203,12 +203,12 @@ impl Storage {
graphs_cf : db . column_family ( GRAPHS_CF ) ? , graphs_cf : db . column_family ( GRAPHS_CF ) ? ,
db , db ,
} ; } ;
#[ cfg(all(not(target_family = " wasm " ))) ] #[ cfg(all(not(target_family = " wasm " ),not(docsrs) )) ]
this . migrate ( ) ? ; this . migrate ( ) ? ;
Ok ( this ) Ok ( this )
} }
#[ cfg(all(not(target_family = " wasm " ))) ] #[ cfg(all(not(target_family = " wasm " ),not(docsrs) )) ]
fn migrate ( & self ) -> Result < ( ) , StorageError > { fn migrate ( & self ) -> Result < ( ) , StorageError > {
let mut version = self . ensure_version ( ) ? ; let mut version = self . ensure_version ( ) ? ;
if version = = 0 { if version = = 0 {
@ -248,7 +248,7 @@ impl Storage {
} }
} }
#[ cfg(all(not(target_family = " wasm " ))) ] #[ cfg(all(not(target_family = " wasm " ),not(docsrs) )) ]
fn ensure_version ( & self ) -> Result < u64 , StorageError > { fn ensure_version ( & self ) -> Result < u64 , StorageError > {
Ok ( Ok (
if let Some ( version ) = self . db . get ( & self . default_cf , b" oxversion " ) ? { if let Some ( version ) = self . db . get ( & self . default_cf , b" oxversion " ) ? {
@ -262,7 +262,7 @@ impl Storage {
) )
} }
#[ cfg(all(not(target_family = " wasm " ))) ] #[ cfg(all(not(target_family = " wasm " ),not(docsrs) )) ]
fn update_version ( & self , version : u64 ) -> Result < ( ) , StorageError > { fn update_version ( & self , version : u64 ) -> Result < ( ) , StorageError > {
self . db self . db
. insert ( & self . default_cf , b" oxversion " , & version . to_be_bytes ( ) ) ? ; . insert ( & self . default_cf , b" oxversion " , & version . to_be_bytes ( ) ) ? ;
@ -289,12 +289,12 @@ impl Storage {
} ) } )
} }
#[ cfg(all(not(target_family = " wasm " ))) ] #[ cfg(all(not(target_family = " wasm " ),not(docsrs) )) ]
pub fn flush ( & self ) -> Result < ( ) , StorageError > { pub fn flush ( & self ) -> Result < ( ) , StorageError > {
self . db . flush ( ) self . db . flush ( )
} }
#[ cfg(all(not(target_family = " wasm " ))) ] #[ cfg(all(not(target_family = " wasm " ),not(docsrs) )) ]
pub fn compact ( & self ) -> Result < ( ) , StorageError > { pub fn compact ( & self ) -> Result < ( ) , StorageError > {
self . db . compact ( & self . default_cf ) ? ; self . db . compact ( & self . default_cf ) ? ;
self . db . compact ( & self . gspo_cf ) ? ; self . db . compact ( & self . gspo_cf ) ? ;
@ -309,7 +309,7 @@ impl Storage {
self . db . compact ( & self . id2str_cf ) self . db . compact ( & self . id2str_cf )
} }
#[ cfg(all(not(target_family = " wasm " ))) ] #[ cfg(all(not(target_family = " wasm " ),not(docsrs) )) ]
pub fn backup ( & self , target_directory : & Path ) -> Result < ( ) , StorageError > { pub fn backup ( & self , target_directory : & Path ) -> Result < ( ) , StorageError > {
self . db . backup ( target_directory ) self . db . backup ( target_directory )
} }
@ -634,7 +634,7 @@ impl StorageReader {
} }
} }
#[ cfg(all(not(target_family = " wasm " ))) ] #[ cfg(all(not(target_family = " wasm " ),not(docsrs) )) ]
pub fn get_str ( & self , key : & StrHash ) -> Result < Option < String > , StorageError > { pub fn get_str ( & self , key : & StrHash ) -> Result < Option < String > , StorageError > {
Ok ( self Ok ( self
. storage . storage
@ -645,7 +645,7 @@ impl StorageReader {
. map_err ( CorruptionError ::new ) ? ) . map_err ( CorruptionError ::new ) ? )
} }
#[ cfg(any(target_family = " wasm " )) ] #[ cfg(any(target_family = " wasm " ,docsrs ))]
pub fn get_str ( & self , key : & StrHash ) -> Result < Option < String > , StorageError > { pub fn get_str ( & self , key : & StrHash ) -> Result < Option < String > , StorageError > {
Ok ( self Ok ( self
. reader . reader
@ -655,21 +655,21 @@ impl StorageReader {
. map_err ( CorruptionError ::new ) ? ) . map_err ( CorruptionError ::new ) ? )
} }
#[ cfg(all(not(target_family = " wasm " ))) ] #[ cfg(all(not(target_family = " wasm " ),not(docsrs) )) ]
pub fn contains_str ( & self , key : & StrHash ) -> Result < bool , StorageError > { pub fn contains_str ( & self , key : & StrHash ) -> Result < bool , StorageError > {
self . storage self . storage
. db . db
. contains_key ( & self . storage . id2str_cf , & key . to_be_bytes ( ) ) . contains_key ( & self . storage . id2str_cf , & key . to_be_bytes ( ) )
} }
#[ cfg(any(target_family = " wasm " )) ] #[ cfg(any(target_family = " wasm " ,docsrs ))]
pub fn contains_str ( & self , key : & StrHash ) -> Result < bool , StorageError > { pub fn contains_str ( & self , key : & StrHash ) -> Result < bool , StorageError > {
self . reader self . reader
. contains_key ( & self . storage . id2str_cf , & key . to_be_bytes ( ) ) . contains_key ( & self . storage . id2str_cf , & key . to_be_bytes ( ) )
} }
/// Validates that all the storage invariants held in the data
/// Validates that all the storage invariants held in the data
#[ cfg(all(not(target_family = " wasm " ))) ] #[ cfg(all(not(target_family = " wasm " ),not(docsrs) )) ]
pub fn validate ( & self ) -> Result < ( ) , StorageError > { pub fn validate ( & self ) -> Result < ( ) , StorageError > {
// triples
// triples
let dspo_size = self . dspo_quads ( & [ ] ) . count ( ) ; let dspo_size = self . dspo_quads ( & [ ] ) . count ( ) ;
@ -781,7 +781,7 @@ impl StorageReader {
} }
/// Validates that all the storage invariants held in the data
/// Validates that all the storage invariants held in the data
#[ cfg(any(target_family = " wasm " )) ] #[ cfg(any(target_family = " wasm " ,docsrs ))]
#[ allow(clippy::unused_self, clippy::unnecessary_wraps) ] #[ allow(clippy::unused_self, clippy::unnecessary_wraps) ]
pub fn validate ( & self ) -> Result < ( ) , StorageError > { pub fn validate ( & self ) -> Result < ( ) , StorageError > {
Ok ( ( ) ) // TODO
Ok ( ( ) ) // TODO
@ -1005,7 +1005,7 @@ impl<'a> StorageWriter<'a> {
} }
} }
#[ cfg(all(not(target_family = " wasm " ))) ] #[ cfg(all(not(target_family = " wasm " ),not(docsrs) )) ]
fn insert_str ( & mut self , key : & StrHash , value : & str ) -> Result < ( ) , StorageError > { fn insert_str ( & mut self , key : & StrHash , value : & str ) -> Result < ( ) , StorageError > {
if self if self
. storage . storage
@ -1021,7 +1021,7 @@ impl<'a> StorageWriter<'a> {
) )
} }
#[ cfg(any(target_family = " wasm " )) ] #[ cfg(any(target_family = " wasm " ,docsrs ))]
fn insert_str ( & mut self , key : & StrHash , value : & str ) -> Result < ( ) , StorageError > { fn insert_str ( & mut self , key : & StrHash , value : & str ) -> Result < ( ) , StorageError > {
self . transaction . insert ( self . transaction . insert (
& self . storage . id2str_cf , & self . storage . id2str_cf ,
@ -1186,7 +1186,7 @@ impl<'a> StorageWriter<'a> {
} }
} }
#[ cfg(all(not(target_family = " wasm " ))) ] #[ cfg(all(not(target_family = " wasm " ),not(docsrs) )) ]
#[ must_use ] #[ must_use ]
pub struct StorageBulkLoader { pub struct StorageBulkLoader {
storage : Storage , storage : Storage ,
@ -1195,7 +1195,7 @@ pub struct StorageBulkLoader {
max_memory_size : Option < usize > , max_memory_size : Option < usize > ,
} }
#[ cfg(all(not(target_family = " wasm " ))) ] #[ cfg(all(not(target_family = " wasm " ),not(docsrs) )) ]
impl StorageBulkLoader { impl StorageBulkLoader {
pub fn new ( storage : Storage ) -> Self { pub fn new ( storage : Storage ) -> Self {
Self { Self {
@ -1326,7 +1326,7 @@ impl StorageBulkLoader {
} }
} }
#[ cfg(all(not(target_family = " wasm " ))) ] #[ cfg(all(not(target_family = " wasm " ),not(docsrs) )) ]
struct FileBulkLoader < ' a > { struct FileBulkLoader < ' a > {
storage : & ' a Storage , storage : & ' a Storage ,
id2str : HashMap < StrHash , Box < str > > , id2str : HashMap < StrHash , Box < str > > ,
@ -1335,7 +1335,7 @@ struct FileBulkLoader<'a> {
graphs : HashSet < EncodedTerm > , graphs : HashSet < EncodedTerm > ,
} }
#[ cfg(all(not(target_family = " wasm " ))) ] #[ cfg(all(not(target_family = " wasm " ),not(docsrs) )) ]
impl < ' a > FileBulkLoader < ' a > { impl < ' a > FileBulkLoader < ' a > {
fn new ( storage : & ' a Storage , batch_size : usize ) -> Self { fn new ( storage : & ' a Storage , batch_size : usize ) -> Self {
Self { Self {
@ -1541,7 +1541,7 @@ impl<'a> FileBulkLoader<'a> {
} }
} }
#[ cfg(all(not(target_family = " wasm " ))) ] #[ cfg(all(not(target_family = " wasm " ),not(docsrs) )) ]
fn map_thread_result < R > ( result : thread ::Result < R > ) -> io ::Result < R > { fn map_thread_result < R > ( result : thread ::Result < R > ) -> io ::Result < R > {
result . map_err ( | e | { result . map_err ( | e | {
io ::Error ::new ( io ::Error ::new (
