Add option to open database in read-only mode

With read-only it's not possible to modify the data. Updates to the data are possible via a primary instance of oxigraph, but will not be reflected. The data is frozen at the time the read-only server is started.
2 years ago · f7637ee5a5
parent aa9476b9cc
commit f7637ee5a5
5 changed files with 104 additions and 13 deletions
--- a/lib/src/storage/backend/rocksdb.rs
+++ b/lib/src/storage/backend/rocksdb.rs
@ -29,6 +29,7 @@ use std::{ptr, slice};
 enum OpeningMode {
    InMemory,
    Primary,
+    ReadOnly,
    Secondary(PathBuf),
 }

@ -82,7 +83,7 @@ fn primary_db_handler_or_error(
    match db {
        InnerDbHandler::Primary(inner_db) | InnerDbHandler::InMemory(inner_db) => Ok(inner_db),
        _ => Err(StorageError::Other(
-            "Database is opened as secondary, can not execute this operation.".into(),
+            "Database is opened as secondary or readonly, can not execute this operation.".into(),
        )),
    }
 }
@ -91,9 +92,9 @@ fn secondary_db_handler_or_error(
    db: &InnerDbHandler,
 ) -> Result<&InnerDbHandlerSecondary, StorageError> {
    match db {
-        InnerDbHandler::Secondary(inner_db) => Ok(inner_db),
+        InnerDbHandler::ReadOnly(inner_db) | InnerDbHandler::Secondary(inner_db) => Ok(inner_db),
        _ => Err(StorageError::Other(Box::<dyn Error + Send + Sync>::from(
-            "Expected secondary InnerDbHandler, got primary/in memory one",
+            "Expected secondary/read-only InnerDbHandler, got primary/in memory one",
        ))),
    }
 }
@ -131,19 +132,20 @@ enum InnerDbHandler {
    Primary(InnerDbHandlerPrimary),
    InMemory(InnerDbHandlerPrimary),
    Secondary(InnerDbHandlerSecondary),
+    ReadOnly(InnerDbHandlerSecondary),
 }

 impl InnerDbHandler {
    fn is_null(&self) -> bool {
        match self {
            Self::Primary(s) | Self::InMemory(s) => s.db.is_null(),
-            Self::Secondary(s) => s.db.is_null(),
+            Self::ReadOnly(s) | Self::Secondary(s) => s.db.is_null(),
        }
    }
    fn primary_path(&self) -> &PathBuf {
        match self {
            Self::Primary(s) | Self::InMemory(s) => &s.path,
-            Self::Secondary(s) => &s.primary_path,
+            Self::ReadOnly(s) | Self::Secondary(s) => &s.primary_path,
        }
    }
 }
@ -173,7 +175,7 @@ impl Drop for DbHandler {
                InnerDbHandler::Primary(s) | InnerDbHandler::InMemory(s) => {
                    rocksdb_transactiondb_close(s.db);
                }
-                InnerDbHandler::Secondary(s) => {
+                InnerDbHandler::Secondary(s) | InnerDbHandler::ReadOnly(s) => {
                    rocksdb_close(s.db);
                }
            }
@ -234,6 +236,11 @@ impl Db {
        column_families: Vec<ColumnFamilyDefinition>,
    ) -> Result<Self, StorageError> {
        match options {
+            StoreOpenOptions::OpenAsReadOnly(options) => Ok(Self(Arc::new(Self::do_open(
+                options.path,
+                column_families,
+                &OpeningMode::ReadOnly,
+            )?))),
            StoreOpenOptions::OpenAsSecondary(options) => Ok(Self(Arc::new(Self::do_open(
                options.path,
                column_families,
@ -260,7 +267,7 @@ impl Db {
            );
            if let Some(available_fd) = available_file_descriptors()? {
                let max_open_files = match &open_mode {
-                    OpeningMode::Primary | OpeningMode::InMemory => {
+                    OpeningMode::Primary | OpeningMode::InMemory | OpeningMode::ReadOnly => {
                        if available_fd < 96 {
                            rocksdb_options_destroy(options);
                            return Err(io::Error::new(
@ -413,6 +420,23 @@ impl Db {
                        })
                    })
                }
+                OpeningMode::ReadOnly => {
+                    ffi_result!(rocksdb_open_for_read_only_column_families_with_status(
+                        options,
+                        c_path.as_ptr(),
+                        c_num_column_families,
+                        c_column_family_names.as_ptr(),
+                        cf_options.as_ptr() as *const *const rocksdb_options_t,
+                        cf_handles.as_mut_ptr(),
+                        0, // false
+                    ))
+                    .map(|db| {
+                        InnerDbHandler::ReadOnly(InnerDbHandlerSecondary {
+                            db,
+                            primary_path: path,
+                        })
+                    })
+                }
            }
            .map_err(|e| {
                for cf_option in &cf_options {
@ -516,6 +540,10 @@ impl Db {
                        options,
                    }
                }
+                InnerDbHandler::ReadOnly(_) => Reader {
+                    inner: InnerReader::Secondary(self.0.clone()),
+                    options,
+                },
                InnerDbHandler::Secondary(_) => {
                    ffi_result!(rocksdb_try_catch_up_with_primary_with_status(
                        secondary_db_handler_or_error(&self.0.db).unwrap().db,
@ -610,7 +638,8 @@ impl Db {
    ) -> Result<Option<PinnableSlice>, StorageError> {
        unsafe {
            let slice = match &self.0.db {
-                InnerDbHandler::Secondary(inner_db_handler) => {
+                InnerDbHandler::Secondary(inner_db_handler)
+                | InnerDbHandler::ReadOnly(inner_db_handler) => {
                    ffi_result!(rocksdb_get_pinned_cf_with_status(
                        inner_db_handler.db,
                        self.0.read_options,
@ -745,8 +774,8 @@ impl Db {

    pub fn backup(&self, target_directory: &Path) -> Result<(), StorageError> {
        match &self.0.db {
-            InnerDbHandler::Secondary(_) => Err(StorageError::Other(
-                "It is not possible to backup an database opened as secondary.".into(),
+            InnerDbHandler::Secondary(_) | InnerDbHandler::ReadOnly(_) => Err(StorageError::Other(
+                "It is not possible to backup an database opened as secondary or read-only.".into(),
            )),
            InnerDbHandler::InMemory(_) => Err(StorageError::Other(
                "It is not possible to backup an in-memory database created with `Store::new`"
--- a/lib/src/store.rs
+++ b/lib/src/store.rs
@ -51,8 +51,13 @@ pub struct SecondaryOptions {
    pub secondary_path: PathBuf,
 }

+pub struct ReadOnlyOptions {
+    pub path: PathBuf,
+}
+
 pub enum StoreOpenOptions {
    OpenAsSecondary(SecondaryOptions),
+    OpenAsReadOnly(ReadOnlyOptions),
 }

 /// An on-disk [RDF dataset](https://www.w3.org/TR/rdf11-concepts/#dfn-rdf-dataset).
--- a/oxrocksdb-sys/api/c.cc
+++ b/oxrocksdb-sys/api/c.cc
@ -35,6 +35,38 @@ rocksdb_pinnableslice_t* rocksdb_get_pinned_cf_with_status(
  return v;
 }

+rocksdb_t* rocksdb_open_for_read_only_column_families_with_status(
+    const rocksdb_options_t* db_options, const char* name,
+    int num_column_families, const char* const* column_family_names,
+    const rocksdb_options_t* const* column_family_options,
+    rocksdb_column_family_handle_t** column_family_handles,
+    unsigned char error_if_wal_file_exists, rocksdb_status_t* statusptr) {
+  std::vector<ColumnFamilyDescriptor> column_families;
+  for (int i = 0; i < num_column_families; i++) {
+    column_families.push_back(ColumnFamilyDescriptor(
+        std::string(column_family_names[i]),
+        ColumnFamilyOptions(column_family_options[i]->rep)));
+  }
+
+  DB* db;
+  std::vector<ColumnFamilyHandle*> handles;
+  if (SaveStatus(statusptr,
+                DB::OpenForReadOnly(DBOptions(db_options->rep),
+                                    std::string(name), column_families,
+                                    &handles, &db, error_if_wal_file_exists))) {
+    return nullptr;
+  }
+
+  for (size_t i = 0; i < handles.size(); i++) {
+    rocksdb_column_family_handle_t* c_handle =
+        new rocksdb_column_family_handle_t;
+    c_handle->rep = handles[i];
+    column_family_handles[i] = c_handle;
+  }
+  rocksdb_t* result = new rocksdb_t;
+  result->rep = db;
+  return result;
+}

 void rocksdb_try_catch_up_with_primary_with_status(
        rocksdb_t* db, rocksdb_status_t* statusptr) {
--- a/oxrocksdb-sys/api/c.h
+++ b/oxrocksdb-sys/api/c.h
@ -70,6 +70,15 @@ rocksdb_pinnableslice_t* rocksdb_get_pinned_cf_with_status(
    rocksdb_column_family_handle_t* column_family, const char* key,
    size_t keylen, rocksdb_status_t* statusptr);

+
+
+extern ROCKSDB_LIBRARY_API rocksdb_t* rocksdb_open_for_read_only_column_families_with_status(
+    const rocksdb_options_t* db_options, const char* name,
+    int num_column_families, const char* const* column_family_names,
+    const rocksdb_options_t* const* column_family_options,
+    rocksdb_column_family_handle_t** column_family_handles,
+    unsigned char error_if_wal_file_exists, rocksdb_status_t* statusptr);
+
 extern ROCKSDB_LIBRARY_API void rocksdb_try_catch_up_with_primary_with_status(
        rocksdb_t* db, rocksdb_status_t* statusptr);

--- a/server/src/main.rs
+++ b/server/src/main.rs
@ -6,7 +6,9 @@ use oxhttp::Server;
 use oxigraph::io::{DatasetFormat, DatasetSerializer, GraphFormat, GraphSerializer};
 use oxigraph::model::{GraphName, GraphNameRef, IriParseError, NamedNode, NamedOrBlankNode};
 use oxigraph::sparql::{Query, QueryResults, Update};
-use oxigraph::store::{BulkLoader, LoaderError, SecondaryOptions, Store, StoreOpenOptions};
+use oxigraph::store::{
+    BulkLoader, LoaderError, ReadOnlyOptions, SecondaryOptions, Store, StoreOpenOptions,
+};
 use oxiri::Iri;
 use rand::random;
 use rayon_core::ThreadPoolBuilder;
@ -37,10 +39,20 @@ struct Args {
    /// Directory in which persist the data.
    #[arg(short, long, global = true)]
    location: Option<PathBuf>,
-    /// Open underlying database in readonly mode, specify path to store info logs (LOG)
+    /// Open underlying database in secondary mode, specify path to store info logs (LOG)
    // see https://github.com/facebook/rocksdb/wiki/Read-only-and-Secondary-instances
-    #[arg(short, long, global = true)]
+    #[arg(short, long, global = true, conflicts_with = "readonly")]
    secondary_location: Option<PathBuf>,
+    /// Open underlying database in read only mode
+    // see https://github.com/facebook/rocksdb/wiki/Read-only-and-Secondary-instances
+    #[arg(
+        short,
+        long,
+        global = true,
+        action,
+        conflicts_with = "secondary_location"
+    )]
+    readonly: bool,
    #[command(subcommand)]
    command: Command,
 }
@ -76,6 +88,10 @@ pub fn main() -> anyhow::Result<()> {
                path: path.to_path_buf(),
                secondary_path: secondary_path.to_path_buf(),
            }))
+        } else if matches.readonly {
+            Store::open_with_options(StoreOpenOptions::OpenAsReadOnly(ReadOnlyOptions {
+                path: path.to_path_buf(),
+            }))
        } else {
            Store::open(path)
        }