diff --git a/.github/workflows/manylinux_build.sh b/.github/workflows/manylinux_build.sh index ae72000d..e7ee8ec1 100644 --- a/.github/workflows/manylinux_build.sh +++ b/.github/workflows/manylinux_build.sh @@ -13,9 +13,9 @@ source venv/bin/activate pip install -r requirements.dev.txt maturin develop --release python generate_stubs.py pyoxigraph pyoxigraph.pyi --black -maturin build --release --features abi3 --compatibility manylinux2014 +maturin build --release --no-default-features --features abi3 --features rustls --compatibility manylinux2014 if [ %for_each_version% ]; then for VERSION in 7 8 9 10 11; do - maturin build --release --interpreter "python3.$VERSION" --compatibility manylinux2014 + maturin build --release --no-default-features --features rustls --interpreter "python3.$VERSION" --compatibility manylinux2014 done fi diff --git a/.github/workflows/musllinux_build.sh b/.github/workflows/musllinux_build.sh index 674be63a..ec6c54e1 100644 --- a/.github/workflows/musllinux_build.sh +++ b/.github/workflows/musllinux_build.sh @@ -11,9 +11,9 @@ source venv/bin/activate pip install -r requirements.dev.txt maturin develop --release python generate_stubs.py pyoxigraph pyoxigraph.pyi --black -maturin build --release --features abi3 --compatibility musllinux_1_2 +maturin build --release --no-default-features --features abi3 --features rustls --compatibility musllinux_1_2 if [ %for_each_version% ]; then for VERSION in 7 8 9 10 11; do - maturin build --release --interpreter "python3.$VERSION" --compatibility musllinux_1_2 + maturin build --release --no-default-features --features rustls --interpreter "python3.$VERSION" --compatibility musllinux_1_2 done fi diff --git a/Cargo.lock b/Cargo.lock index 8fea3245..f8f74cc6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -148,9 +148,9 @@ dependencies = [ [[package]] name = "base64" -version = "0.21.3" +version = "0.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "414dcefbc63d77c526a76b3afcf6fbb9b5e2791c19c3aa2297733208750c6e53" +checksum = "9ba43ea6f343b788c8764558649e08df62f86c6ef251fdaeb1ffd010a9ae50a2" [[package]] name = "bindgen" @@ -576,6 +576,21 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.2.0" @@ -874,6 +889,24 @@ dependencies = [ "adler", ] +[[package]] +name = "native-tls" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "nom" version = "7.1.3" @@ -930,24 +963,62 @@ version = "11.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" +[[package]] +name = "openssl" +version = "0.10.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bac25ee399abb46215765b1cb35bc0212377e58a061560d8b29b024fd0430e7c" +dependencies = [ + "bitflags 2.4.0", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.31", +] + [[package]] name = "openssl-probe" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +[[package]] +name = "openssl-sys" +version = "0.9.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db4d56a4c0478783083cfafcc42493dd4a981d41669da64b4572a2a089b51b1d" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "oxhttp" -version = "0.1.7" +version = "0.2.0-alpha.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64f8f4b616372a7e657a05100d1389325fc2f7d3fee5c9de05166d979cb2b729" +checksum = "525e4922eaf3a36d3d8534226910b6f5b878446d2f7f20b31da7889eb969f984" dependencies = [ "httparse", - "lazy_static", - "rayon-core", + "native-tls", "rustls", "rustls-native-certs", "url", + "webpki-roots", ] [[package]] @@ -1524,9 +1595,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.101.4" +version = "0.101.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d93931baf2d282fff8d3a532bbfd7653f734643161b87e3e01e59a04439bf0d" +checksum = "3c7d5dece342910d9ba34d259310cae3e0154b873b35408b787b59bce53d34fe" dependencies = [ "ring", "untrusted", @@ -1930,6 +2001,12 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.4" @@ -2025,6 +2102,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-roots" +version = "0.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14247bb57be4f377dfb94c72830b8ce8fc6beac03cf4bf7b9732eadd414123fc" + [[package]] name = "which" version = "4.4.2" diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 05c7df74..6b84f16b 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -19,13 +19,17 @@ name = "oxigraph" path = "src/main.rs" [features] +default = ["native-tls"] +native-tls = ["oxigraph/http-client-native-tls"] rocksdb-pkg-config = ["oxigraph/rocksdb-pkg-config"] +rustls-native = ["oxigraph/http-client-rustls-native"] +rustls-webpki = ["oxigraph/http-client-rustls-webpki"] [dependencies] anyhow = "1.0.72" -oxhttp = { version = "0.1.7", features = ["rayon"] } +oxhttp = { version = "0.2.0-alpha.1" } clap = { version = "4.0", features = ["derive"] } -oxigraph = { version = "0.4.0-alpha.1-dev", path = "../lib", features = ["http_client"] } +oxigraph = { version = "0.4.0-alpha.1-dev", path = "../lib" } rand = "0.8" url = "2.4" oxiri = "0.2" diff --git a/cli/Dockerfile b/cli/Dockerfile index abbd6aad..9454410f 100644 --- a/cli/Dockerfile +++ b/cli/Dockerfile @@ -13,10 +13,10 @@ RUN if [ "$BUILDARCH" != "$TARGETARCH" ] && [ "$TARGETARCH" = "arm64" ] ; \ then \ export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc && \ export BINDGEN_EXTRA_CLANG_ARGS="--sysroot /usr/aarch64-linux-gnu" && \ - cargo build --release --target aarch64-unknown-linux-gnu && \ + cargo build --release --target aarch64-unknown-linux-gnu --no-default-features --features rustls-webpki && \ mv /oxigraph/target/aarch64-unknown-linux-gnu/release/oxigraph /oxigraph/target/release/oxigraph ; \ else \ - cargo build --release ; \ + cargo build --release --no-default-features --features rustls-webpki ; \ fi FROM --platform=$TARGETPLATFORM gcr.io/distroless/cc-debian11 diff --git a/cli/README.md b/cli/README.md index 8c650b4f..aaa1f3e4 100644 --- a/cli/README.md +++ b/cli/README.md @@ -40,6 +40,13 @@ There is no need to clone the git repository. To compile the command line tool from source, clone this git repository including its submodules (`git clone --recursive https://github.com/oxigraph/oxigraph.git`), and execute `cargo build --release` in the `cli` directory to compile the full binary after having downloaded its dependencies. It will create a fat binary in `target/release/oxigraph`. +Some build options (cargo features) are available: +- `rocksdb-pkg-config`: links against an already compiled rocksdb shared library found using [pkg-config](https://crates.io/crates/pkg-config). +- `native-tls`: Enables Oxigraph HTTP client for query federation using the host OS TLS stack (enabled by default). +- `rustls-native` Enables Oxigraph HTTP client for query federation using [Rustls](https://crates.io/crates/rustls) and the native certificates. +- `rustls-webpki` Enables Oxigraph HTTP client for query federation using [Rustls](https://crates.io/crates/rustls) and the [Common CA Database](https://www.ccadb.org/) certificates. + + ## Usage Run `oxigraph serve --location my_data_storage_directory` to start the server where `my_data_storage_directory` is the directory where you want Oxigraph data to be stored. It listens by default on `localhost:7878`. diff --git a/cli/src/main.rs b/cli/src/main.rs index 54f79299..16371325 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -879,7 +879,7 @@ fn rdf_format_from_name(name: &str) -> anyhow::Result { } fn serve(store: Store, bind: String, read_only: bool, cors: bool) -> anyhow::Result<()> { - let mut server = if cors { + let server = if cors { Server::new(cors_middleware(move |request| { handle_request(request, store.clone(), read_only) .unwrap_or_else(|(status, message)| error(status, message)) @@ -889,9 +889,10 @@ fn serve(store: Store, bind: String, read_only: bool, cors: bool) -> anyhow::Res handle_request(request, store.clone(), read_only) .unwrap_or_else(|(status, message)| error(status, message)) }) - }; - server.set_global_timeout(HTTP_TIMEOUT); - server.set_server_name(concat!("Oxigraph/", env!("CARGO_PKG_VERSION")))?; + } + .with_global_timeout(HTTP_TIMEOUT) + .with_server_name(concat!("Oxigraph/", env!("CARGO_PKG_VERSION")))? + .with_max_num_threads(available_parallelism()?.get() * 128); #[cfg(target_os = "linux")] systemd_notify_ready()?; eprintln!("Listening for requests at http://{}", &bind); @@ -1191,7 +1192,7 @@ fn handle_request( resolve_with_base(request, &format!("/store/{:x}", random::()))?; web_load_graph(&store, request, format, &graph.clone().into())?; Ok(Response::builder(Status::CREATED) - .with_header(HeaderName::LOCATION, graph.as_str()) + .with_header(HeaderName::LOCATION, graph.into_string()) .unwrap() .build()) } diff --git a/lib/Cargo.toml b/lib/Cargo.toml index e4b4b92a..87b0715d 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -18,9 +18,12 @@ rust-version.workspace = true [features] default = [] js = ["getrandom/js", "oxsdatatypes/js", "js-sys"] -http_client = ["oxhttp", "oxhttp/rustls"] +http-client = ["oxhttp"] +http-client-native-tls = ["http-client", "oxhttp/native-tls"] +http-client-rustls-webpki = ["http-client", "oxhttp/rustls-webpki"] +http-client-rustls-native = ["http-client", "oxhttp/rustls-native"] rocksdb-pkg-config = ["oxrocksdb-sys/pkg-config"] -rocksdb_debug = [] +rocksdb-debug = [] [dependencies] digest = "0.10" @@ -44,7 +47,7 @@ sparopt = { version = "0.1.0-alpha.1-dev", path="sparopt", features = ["rdf-star [target.'cfg(not(target_family = "wasm"))'.dependencies] libc = "0.2.147" oxrocksdb-sys = { version = "0.4.0-alpha.1-dev", path="../oxrocksdb-sys" } -oxhttp = { version = "0.1.7", optional = true } +oxhttp = { version = "0.2.0-alpha.1", optional = true } [target.'cfg(all(target_family = "wasm", target_os = "unknown"))'.dependencies] getrandom = "0.2" @@ -52,7 +55,7 @@ js-sys = { version = "0.3.60", optional = true } [target.'cfg(not(target_family = "wasm"))'.dev-dependencies] criterion = "0.5" -oxhttp = "0.1.7" +oxhttp = "0.2.0-alpha.1" zstd = "0.12" [package.metadata.docs.rs] diff --git a/lib/README.md b/lib/README.md index d2afa1bc..31a3e510 100644 --- a/lib/README.md +++ b/lib/README.md @@ -48,9 +48,14 @@ if let QueryResults::Solutions(mut solutions) = store.query("SELECT ?s WHERE { ``` Some parts of this library are available as standalone crates: -* [`oxrdf`](https://crates.io/crates/oxrdf) provides datastructures encoding RDF basic concepts (the `oxigraph::model` module). -* [`spargebra`](https://crates.io/crates/spargebra) provides a SPARQL parser. -* [`sparesults`](https://crates.io/crates/sparesults) provides parsers and serializers for SPARQL result formats. +* [`oxrdf`](https://crates.io/crates/oxrdf), datastructures encoding RDF basic concepts (the `oxigraph::model` module). +* [`oxrdfio`](https://crates.io/crates/oxrdfio), a unified parser and serializer API for RDF formats. It itself relies on: + * [`oxttl`](https://crates.io/crates/oxttl), N-Triple, N-Quad, Turtle, TriG and N3 parsing and serialization. + * [`oxrdfxml`](https://crates.io/crates/oxrdfxml), RDF/XML parsing and serialization. +* [`spargebra`](https://crates.io/crates/spargebra), a SPARQL parser. +* [`sparesults`](https://crates.io/crates/sparesults), parsers and serializers for SPARQL result formats. +* [`sparopt`](https://crates.io/crates/sparesults), a SPARQL optimizer. +* [`oxsdatatypes`](https://crates.io/crates/oxsdatatypes), an implementation of some XML Schema datatypes. To build the library, don't forget to clone the submodules using `git clone --recursive https://github.com/oxigraph/oxigraph.git` to clone the repository including submodules or `git submodule update --init` to add submodules to the already cloned repository. diff --git a/lib/benches/store.rs b/lib/benches/store.rs index 6f18f8d1..20826041 100644 --- a/lib/benches/store.rs +++ b/lib/benches/store.rs @@ -207,8 +207,7 @@ criterion_main!(store); fn read_data(file: &str) -> impl Read { if !Path::new(file).exists() { - let mut client = oxhttp::Client::new(); - client.set_redirection_limit(5); + let client = oxhttp::Client::new().with_redirection_limit(5); let url = format!("https://github.com/Tpt/bsbm-tools/releases/download/v0.2/{file}"); let request = Request::builder(Method::GET, url.parse().unwrap()).build(); let response = client.request(request).unwrap(); diff --git a/lib/src/sparql/http/dummy.rs b/lib/src/sparql/http/dummy.rs index dc8516a6..7b3a551f 100644 --- a/lib/src/sparql/http/dummy.rs +++ b/lib/src/sparql/http/dummy.rs @@ -11,10 +11,10 @@ impl Client { } #[allow(clippy::unused_self)] - pub fn get(&self, _url: &str, _accept: &str) -> Result<(String, Empty)> { + pub fn get(&self, _url: &str, _accept: &'static str) -> Result<(String, Empty)> { Err(Error::new( ErrorKind::Unsupported, - "HTTP client is not available. Enable the feature 'http_client'", + "HTTP client is not available. Enable the feature 'http-client'", )) } @@ -23,12 +23,12 @@ impl Client { &self, _url: &str, _payload: Vec, - _content_type: &str, - _accept: &str, + _content_type: &'static str, + _accept: &'static str, ) -> Result<(String, Empty)> { Err(Error::new( ErrorKind::Unsupported, - "HTTP client is not available. Enable the feature 'http_client'", + "HTTP client is not available. Enable the feature 'http-client'", )) } } diff --git a/lib/src/sparql/http/mod.rs b/lib/src/sparql/http/mod.rs index 0266d0b3..b309cf55 100644 --- a/lib/src/sparql/http/mod.rs +++ b/lib/src/sparql/http/mod.rs @@ -1,9 +1,9 @@ -#[cfg(not(feature = "http_client"))] +#[cfg(not(feature = "http-client"))] mod dummy; -#[cfg(feature = "http_client")] +#[cfg(feature = "http-client")] mod simple; -#[cfg(not(feature = "http_client"))] +#[cfg(not(feature = "http-client"))] pub use dummy::Client; -#[cfg(feature = "http_client")] +#[cfg(feature = "http-client")] pub use simple::Client; diff --git a/lib/src/sparql/http/simple.rs b/lib/src/sparql/http/simple.rs index 57b9e540..bd81d7ce 100644 --- a/lib/src/sparql/http/simple.rs +++ b/lib/src/sparql/http/simple.rs @@ -8,18 +8,17 @@ pub struct Client { impl Client { pub fn new(timeout: Option, redirection_limit: usize) -> Self { - let mut client = oxhttp::Client::new(); + let mut client = oxhttp::Client::new() + .with_redirection_limit(redirection_limit) + .with_user_agent(concat!("Oxigraph/", env!("CARGO_PKG_VERSION"))) + .unwrap(); if let Some(timeout) = timeout { - client.set_global_timeout(timeout); + client = client.with_global_timeout(timeout); } - client.set_redirection_limit(redirection_limit); - client - .set_user_agent(concat!("Oxigraph/", env!("CARGO_PKG_VERSION"))) - .unwrap(); Self { client } } - pub fn get(&self, url: &str, accept: &str) -> Result<(String, Body)> { + pub fn get(&self, url: &str, accept: &'static str) -> Result<(String, Body)> { let request = Request::builder(Method::GET, url.parse().map_err(invalid_input_error)?) .with_header(HeaderName::ACCEPT, accept) .map_err(invalid_input_error)? @@ -50,8 +49,8 @@ impl Client { &self, url: &str, payload: Vec, - content_type: &str, - accept: &str, + content_type: &'static str, + accept: &'static str, ) -> Result<(String, Body)> { let request = Request::builder(Method::POST, url.parse().map_err(invalid_input_error)?) .with_header(HeaderName::ACCEPT, accept) diff --git a/lib/src/sparql/mod.rs b/lib/src/sparql/mod.rs index 6d3ef593..82944f95 100644 --- a/lib/src/sparql/mod.rs +++ b/lib/src/sparql/mod.rs @@ -138,7 +138,7 @@ pub(crate) fn evaluate_query( /// Options for SPARQL query evaluation. /// /// -/// If the `"http_client"` optional feature is enabled, +/// If the `"http-client"` optional feature is enabled, /// a simple HTTP 1.1 client is used to execute [SPARQL 1.1 Federated Query](https://www.w3.org/TR/sparql11-federated-query/) SERVICE calls. /// /// Usage example disabling the federated query support: @@ -182,7 +182,7 @@ impl QueryOptions { } /// Sets a timeout for HTTP requests done during SPARQL evaluation. - #[cfg(feature = "http_client")] + #[cfg(feature = "http-client")] #[inline] #[must_use] pub fn with_http_timeout(mut self, timeout: Duration) -> Self { @@ -193,7 +193,7 @@ impl QueryOptions { /// Sets an upper bound of the number of HTTP redirection followed per HTTP request done during SPARQL evaluation. /// /// By default this value is `0`. - #[cfg(feature = "http_client")] + #[cfg(feature = "http-client")] #[inline] #[must_use] pub fn with_http_redirection_limit(mut self, redirection_limit: usize) -> Self { @@ -235,7 +235,7 @@ impl QueryOptions { fn service_handler(&self) -> Rc> { self.service_handler.clone().unwrap_or_else(|| { - if cfg!(feature = "http_client") { + if cfg!(feature = "http-client") { Rc::new(service::SimpleServiceHandler::new( self.http_timeout, self.http_redirection_limit, diff --git a/lib/src/storage/backend/rocksdb.rs b/lib/src/storage/backend/rocksdb.rs index 6d065059..5e6f4103 100644 --- a/lib/src/storage/backend/rocksdb.rs +++ b/lib/src/storage/backend/rocksdb.rs @@ -200,7 +200,7 @@ impl Db { 16, ); rocksdb_options_set_block_based_table_factory(options, block_based_table_options); - #[cfg(feature = "rocksdb_debug")] + #[cfg(feature = "rocksdb-debug")] { rocksdb_options_set_info_log_level(options, 0); rocksdb_options_enable_statistics(options); diff --git a/lints/test_debian_compatibility.py b/lints/test_debian_compatibility.py index ec11b319..10f6596a 100644 --- a/lints/test_debian_compatibility.py +++ b/lints/test_debian_compatibility.py @@ -5,7 +5,7 @@ from urllib.request import urlopen TARGET_DEBIAN_VERSIONS = ["sid"] IGNORE_PACKAGES = {"oxigraph-js", "oxigraph-testsuite", "pyoxigraph", "sparql-smith"} -ALLOWED_MISSING_PACKAGES = {"escargot", "quick-xml"} +ALLOWED_MISSING_PACKAGES = {"escargot", "oxhttp", "quick-xml"} base_path = Path(__file__).parent.parent diff --git a/python/Cargo.toml b/python/Cargo.toml index ce5df20a..9ecee7af 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -18,9 +18,12 @@ name = "pyoxigraph" doctest = false [features] +default = ["native-tls"] abi3 = ["pyo3/abi3-py38"] +native-tls = ["oxigraph/http-client-native-tls"] rocksdb-pkg-config = ["oxigraph/rocksdb-pkg-config"] +rustls = ["oxigraph/http-client-rustls-native"] [dependencies] -oxigraph = { version = "0.4.0-alpha.1-dev", path="../lib", features = ["http_client"] } +oxigraph = { version = "0.4.0-alpha.1-dev", path="../lib" } pyo3 = { version = "0.19", features = ["extension-module"] } diff --git a/python/README.md b/python/README.md index 8a2e8247..c8478a60 100644 --- a/python/README.md +++ b/python/README.md @@ -32,6 +32,9 @@ Pyoxigraph documentation is [available on the Oxigraph website](https://pyoxigra To build and install the development version of pyoxigraph you need to clone this git repository including submodules (`git clone --recursive https://github.com/oxigraph/oxigraph.git`) and to run `pip install .` in the `python` directory (the one this README is in). +Note that by default the installation will not use [cpython stable ABI](https://docs.python.org/3/c-api/stable.html) and will rely on the host TLS implementation. +Use `--features abi3` feature to use cpython stable ABI and use `--no-default-features --features rustls` to use [rustls](https://github.com/rustls/rustls) with the system certificates. + ## Help Feel free to use [GitHub discussions](https://github.com/oxigraph/oxigraph/discussions) or [the Gitter chat](https://gitter.im/oxigraph/community) to ask questions or talk about Oxigraph.