From d8fa540b97e12c20a2c011590d8e9caf214d8528 Mon Sep 17 00:00:00 2001 From: Tpt Date: Mon, 6 Mar 2023 22:30:16 +0100 Subject: [PATCH] Python: Exposes read-only and secondary store --- python/generate_stubs.py | 33 +++++++++++++++++---- python/src/model.rs | 2 ++ python/src/store.rs | 60 +++++++++++++++++++++++++++++++++++++- python/tests/test_store.py | 25 +++++++++++++++- 4 files changed, 113 insertions(+), 7 deletions(-) diff --git a/python/generate_stubs.py b/python/generate_stubs.py index 58a40676..a865f80e 100644 --- a/python/generate_stubs.py +++ b/python/generate_stubs.py @@ -82,7 +82,13 @@ def module_stubs(module: Any) -> ast.Module: ) elif inspect.isbuiltin(member_value): functions.append( - function_stub(member_name, member_value, element_path, types_to_import) + function_stub( + member_name, + member_value, + element_path, + types_to_import, + in_class=False, + ) ) else: logging.warning(f"Unsupported root construction {member_name}") @@ -107,7 +113,11 @@ def class_stubs( inspect.signature(cls_def) # we check it actually exists methods = [ function_stub( - member_name, cls_def, current_element_path, types_to_import + member_name, + cls_def, + current_element_path, + types_to_import, + in_class=True, ) ] + methods except ValueError as e: @@ -129,7 +139,11 @@ def class_stubs( elif inspect.isroutine(member_value): (magic_methods if member_name.startswith("__") else methods).append( function_stub( - member_name, member_value, current_element_path, types_to_import + member_name, + member_value, + current_element_path, + types_to_import, + in_class=True, ) ) else: @@ -182,18 +196,27 @@ def data_descriptor_stub( def function_stub( - fn_name: str, fn_def: Any, element_path: List[str], types_to_import: Set[str] + fn_name: str, + fn_def: Any, + element_path: List[str], + types_to_import: Set[str], + *, + in_class: bool, ) -> ast.FunctionDef: body: List[ast.AST] = [] doc = inspect.getdoc(fn_def) if doc is not None: body.append(build_doc_comment(doc)) + decorator_list = [] + if in_class and hasattr(fn_def, "__self__"): + decorator_list.append(ast.Name("staticmethod")) + return ast.FunctionDef( fn_name, arguments_stub(fn_name, fn_def, doc or "", element_path, types_to_import), body or [AST_ELLIPSIS], - decorator_list=[], + decorator_list=decorator_list, returns=returns_stub(fn_name, doc, element_path, types_to_import) if doc else None, diff --git a/python/src/model.rs b/python/src/model.rs index 864b68d3..fad46a1a 100644 --- a/python/src/model.rs +++ b/python/src/model.rs @@ -167,6 +167,7 @@ impl From for GraphName { #[pymethods] impl PyBlankNode { #[new] + #[pyo3(signature = (value = None))] fn new(value: Option) -> PyResult { Ok(if let Some(value) = value { BlankNode::new(value).map_err(|e| PyValueError::new_err(e.to_string()))? @@ -739,6 +740,7 @@ impl<'a> From<&'a PyQuad> for QuadRef<'a> { #[pymethods] impl PyQuad { #[new] + #[pyo3(signature = (subject, predicate, object, graph_name = None))] fn new( subject: PySubject, predicate: PyNamedNode, diff --git a/python/src/store.rs b/python/src/store.rs index c7ac180a..ac966f6b 100644 --- a/python/src/store.rs +++ b/python/src/store.rs @@ -20,6 +20,10 @@ use pyo3::{Py, PyRef}; /// been "committed" (i.e. no partial writes) and the exposed state does not change for the complete duration /// of a read operation (e.g. a SPARQL query) or a read/write operation (e.g. a SPARQL update). /// +/// The :py:class:`Store` constructor opens a read-write instance. +/// To open a static read-only instance use :py:func:`Store.read_only` +/// and to open a read-only instance that tracks a read-write instance use :py:func:`Store.secondary`. +/// /// :param path: the path of the directory in which the store should read and write its data. If the directory does not exist, it is created. /// If no directory is provided a temporary one is created and removed when the Python garbage collector removes the store. /// In this case, the store data are kept in memory and never written on disk. @@ -42,6 +46,7 @@ pub struct PyStore { #[pymethods] impl PyStore { #[new] + #[pyo3(signature = (path = None))] fn new(path: Option<&str>, py: Python<'_>) -> PyResult { py.allow_threads(|| { Ok(Self { @@ -55,6 +60,59 @@ impl PyStore { }) } + /// Opens a read-only store from disk. + /// + /// Opening as read-only while having an other process writing the database is undefined behavior. + /// :py:func:`Store.secondary` should be used in this case. + /// + /// :param path: path to the primary read-write instance data. + /// :type path: str + /// :return: the opened store. + /// :rtype: Store + /// :raises IOError: if the target directory contains invalid data or could not be accessed. + #[staticmethod] + fn read_only(path: &str, py: Python<'_>) -> PyResult { + py.allow_threads(|| { + Ok(Self { + inner: Store::open_read_only(path).map_err(map_storage_error)?, + }) + }) + } + + /// Opens a read-only clone of a running read-write store. + /// + /// Changes done while this process is running will be replicated after a possible lag. + /// + /// It should only be used if a primary instance opened with :py:func:`Store` is running at the same time. + /// + /// If you want to simple read-only store use :py:func:`Store.read_only`. + /// + /// :param primary_path: path to the primary read-write instance data. + /// :type primary_path: str + /// :param secondary_path: path to an other directory for the secondary instance cache. If not given a temporary directory will be used. + /// :type secondary_path: str or None, optional + /// :return: the opened store. + /// :rtype: Store + /// :raises IOError: if the target directories contain invalid data or could not be accessed. + #[staticmethod] + #[pyo3(signature = (primary_path, secondary_path = None), text_signature = "(primary_path, secondary_path = None)")] + fn secondary( + primary_path: &str, + secondary_path: Option<&str>, + py: Python<'_>, + ) -> PyResult { + py.allow_threads(|| { + Ok(Self { + inner: if let Some(secondary_path) = secondary_path { + Store::open_persistent_secondary(primary_path, secondary_path) + } else { + Store::open_secondary(primary_path) + } + .map_err(map_storage_error)?, + }) + }) + } + /// Adds a quad to the store. /// /// :param quad: the quad to add. @@ -111,7 +169,7 @@ impl PyStore { /// >>> store.add(Quad(NamedNode('http://example.com'), NamedNode('http://example.com/p'), Literal('1'), NamedNode('http://example.com/g'))) /// >>> list(store.quads_for_pattern(NamedNode('http://example.com'), None, None, None)) /// [ predicate= object=> graph_name=>] - #[pyo3(text_signature = "($self, subject, predicate, object, graph_name = None)")] + #[pyo3(signature = (subject, predicate, object, graph_name = None), text_signature = "($self, subject, predicate, object, graph_name = None)")] fn quads_for_pattern( &self, subject: &PyAny, diff --git a/python/tests/test_store.py b/python/tests/test_store.py index 5b4c409c..f0aeb0f2 100644 --- a/python/tests/test_store.py +++ b/python/tests/test_store.py @@ -1,7 +1,7 @@ import os import unittest from io import BytesIO, UnsupportedOperation -from tempfile import NamedTemporaryFile, TemporaryFile +from tempfile import NamedTemporaryFile, TemporaryDirectory, TemporaryFile from typing import Any from pyoxigraph import * @@ -316,6 +316,29 @@ class TestStore(unittest.TestCase): self.assertEqual(list(store.named_graphs()), []) self.assertEqual(list(store), []) + def test_read_only(self) -> None: + quad = Quad(foo, bar, baz, graph) + with TemporaryDirectory() as dir: + store = Store(dir) + store.add(quad) + del store + store = Store.read_only(dir) + self.assertEqual(list(store), [quad]) + + def test_secondary(self) -> None: + quad = Quad(foo, bar, baz, graph) + with TemporaryDirectory() as dir: + store = Store(dir) + store.add(quad) + store.flush() + + secondary_store = Store.secondary(dir) + self.assertEqual(list(secondary_store), [quad]) + + store.remove(quad) + store.flush() + self.assertEqual(list(secondary_store), []) + if __name__ == "__main__": unittest.main()