Upgrades oxhttp

- Disables HTTPs support by default, the TLS stack is opt-in
- Renames "http_client" feature to "http-client"
- Uses native TLS by default in pyoxigraph and cli
- Uses Rustls for Linux Python wheels and Docker images
pull/636/head
Tpt 1 year ago committed by Thomas Tanon
parent 90b7b128f2
commit c5f02d9263
  1. 4
      .github/workflows/manylinux_build.sh
  2. 4
      .github/workflows/musllinux_build.sh
  3. 99
      Cargo.lock
  4. 8
      cli/Cargo.toml
  5. 4
      cli/Dockerfile
  6. 7
      cli/README.md
  7. 11
      cli/src/main.rs
  8. 11
      lib/Cargo.toml
  9. 11
      lib/README.md
  10. 3
      lib/benches/store.rs
  11. 10
      lib/src/sparql/http/dummy.rs
  12. 8
      lib/src/sparql/http/mod.rs
  13. 17
      lib/src/sparql/http/simple.rs
  14. 8
      lib/src/sparql/mod.rs
  15. 2
      lib/src/storage/backend/rocksdb.rs
  16. 2
      lints/test_debian_compatibility.py
  17. 5
      python/Cargo.toml
  18. 3
      python/README.md

@ -13,9 +13,9 @@ source venv/bin/activate
pip install -r requirements.dev.txt
maturin develop --release
python generate_stubs.py pyoxigraph pyoxigraph.pyi --black
maturin build --release --features abi3 --compatibility manylinux2014
maturin build --release --no-default-features --features abi3 --features rustls --compatibility manylinux2014
if [ %for_each_version% ]; then
for VERSION in 7 8 9 10 11; do
maturin build --release --interpreter "python3.$VERSION" --compatibility manylinux2014
maturin build --release --no-default-features --features rustls --interpreter "python3.$VERSION" --compatibility manylinux2014
done
fi

@ -11,9 +11,9 @@ source venv/bin/activate
pip install -r requirements.dev.txt
maturin develop --release
python generate_stubs.py pyoxigraph pyoxigraph.pyi --black
maturin build --release --features abi3 --compatibility musllinux_1_2
maturin build --release --no-default-features --features abi3 --features rustls --compatibility musllinux_1_2
if [ %for_each_version% ]; then
for VERSION in 7 8 9 10 11; do
maturin build --release --interpreter "python3.$VERSION" --compatibility musllinux_1_2
maturin build --release --no-default-features --features rustls --interpreter "python3.$VERSION" --compatibility musllinux_1_2
done
fi

99
Cargo.lock generated

@ -148,9 +148,9 @@ dependencies = [
[[package]]
name = "base64"
version = "0.21.3"
version = "0.21.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "414dcefbc63d77c526a76b3afcf6fbb9b5e2791c19c3aa2297733208750c6e53"
checksum = "9ba43ea6f343b788c8764558649e08df62f86c6ef251fdaeb1ffd010a9ae50a2"
[[package]]
name = "bindgen"
@ -576,6 +576,21 @@ version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
[[package]]
name = "foreign-types"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
dependencies = [
"foreign-types-shared",
]
[[package]]
name = "foreign-types-shared"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
[[package]]
name = "form_urlencoded"
version = "1.2.0"
@ -874,6 +889,24 @@ dependencies = [
"adler",
]
[[package]]
name = "native-tls"
version = "0.2.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e"
dependencies = [
"lazy_static",
"libc",
"log",
"openssl",
"openssl-probe",
"openssl-sys",
"schannel",
"security-framework",
"security-framework-sys",
"tempfile",
]
[[package]]
name = "nom"
version = "7.1.3"
@ -930,24 +963,62 @@ version = "11.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575"
[[package]]
name = "openssl"
version = "0.10.57"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bac25ee399abb46215765b1cb35bc0212377e58a061560d8b29b024fd0430e7c"
dependencies = [
"bitflags 2.4.0",
"cfg-if",
"foreign-types",
"libc",
"once_cell",
"openssl-macros",
"openssl-sys",
]
[[package]]
name = "openssl-macros"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.31",
]
[[package]]
name = "openssl-probe"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
[[package]]
name = "openssl-sys"
version = "0.9.93"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "db4d56a4c0478783083cfafcc42493dd4a981d41669da64b4572a2a089b51b1d"
dependencies = [
"cc",
"libc",
"pkg-config",
"vcpkg",
]
[[package]]
name = "oxhttp"
version = "0.1.7"
version = "0.2.0-alpha.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "64f8f4b616372a7e657a05100d1389325fc2f7d3fee5c9de05166d979cb2b729"
checksum = "525e4922eaf3a36d3d8534226910b6f5b878446d2f7f20b31da7889eb969f984"
dependencies = [
"httparse",
"lazy_static",
"rayon-core",
"native-tls",
"rustls",
"rustls-native-certs",
"url",
"webpki-roots",
]
[[package]]
@ -1524,9 +1595,9 @@ dependencies = [
[[package]]
name = "rustls-webpki"
version = "0.101.4"
version = "0.101.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7d93931baf2d282fff8d3a532bbfd7653f734643161b87e3e01e59a04439bf0d"
checksum = "3c7d5dece342910d9ba34d259310cae3e0154b873b35408b787b59bce53d34fe"
dependencies = [
"ring",
"untrusted",
@ -1930,6 +2001,12 @@ version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a"
[[package]]
name = "vcpkg"
version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
[[package]]
name = "version_check"
version = "0.9.4"
@ -2025,6 +2102,12 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "webpki-roots"
version = "0.25.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "14247bb57be4f377dfb94c72830b8ce8fc6beac03cf4bf7b9732eadd414123fc"
[[package]]
name = "which"
version = "4.4.2"

@ -19,13 +19,17 @@ name = "oxigraph"
path = "src/main.rs"
[features]
default = ["native-tls"]
native-tls = ["oxigraph/http-client-native-tls"]
rocksdb-pkg-config = ["oxigraph/rocksdb-pkg-config"]
rustls-native = ["oxigraph/http-client-rustls-native"]
rustls-webpki = ["oxigraph/http-client-rustls-webpki"]
[dependencies]
anyhow = "1.0.72"
oxhttp = { version = "0.1.7", features = ["rayon"] }
oxhttp = { version = "0.2.0-alpha.1" }
clap = { version = "4.0", features = ["derive"] }
oxigraph = { version = "0.4.0-alpha.1-dev", path = "../lib", features = ["http_client"] }
oxigraph = { version = "0.4.0-alpha.1-dev", path = "../lib" }
rand = "0.8"
url = "2.4"
oxiri = "0.2"

@ -13,10 +13,10 @@ RUN if [ "$BUILDARCH" != "$TARGETARCH" ] && [ "$TARGETARCH" = "arm64" ] ; \
then \
export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc && \
export BINDGEN_EXTRA_CLANG_ARGS="--sysroot /usr/aarch64-linux-gnu" && \
cargo build --release --target aarch64-unknown-linux-gnu && \
cargo build --release --target aarch64-unknown-linux-gnu --no-default-features --features rustls-webpki && \
mv /oxigraph/target/aarch64-unknown-linux-gnu/release/oxigraph /oxigraph/target/release/oxigraph ; \
else \
cargo build --release ; \
cargo build --release --no-default-features --features rustls-webpki ; \
fi
FROM --platform=$TARGETPLATFORM gcr.io/distroless/cc-debian11

@ -40,6 +40,13 @@ There is no need to clone the git repository.
To compile the command line tool from source, clone this git repository including its submodules (`git clone --recursive https://github.com/oxigraph/oxigraph.git`), and execute `cargo build --release` in the `cli` directory to compile the full binary after having downloaded its dependencies.
It will create a fat binary in `target/release/oxigraph`.
Some build options (cargo features) are available:
- `rocksdb-pkg-config`: links against an already compiled rocksdb shared library found using [pkg-config](https://crates.io/crates/pkg-config).
- `native-tls`: Enables Oxigraph HTTP client for query federation using the host OS TLS stack (enabled by default).
- `rustls-native` Enables Oxigraph HTTP client for query federation using [Rustls](https://crates.io/crates/rustls) and the native certificates.
- `rustls-webpki` Enables Oxigraph HTTP client for query federation using [Rustls](https://crates.io/crates/rustls) and the [Common CA Database](https://www.ccadb.org/) certificates.
## Usage
Run `oxigraph serve --location my_data_storage_directory` to start the server where `my_data_storage_directory` is the directory where you want Oxigraph data to be stored. It listens by default on `localhost:7878`.

@ -879,7 +879,7 @@ fn rdf_format_from_name(name: &str) -> anyhow::Result<RdfFormat> {
}
fn serve(store: Store, bind: String, read_only: bool, cors: bool) -> anyhow::Result<()> {
let mut server = if cors {
let server = if cors {
Server::new(cors_middleware(move |request| {
handle_request(request, store.clone(), read_only)
.unwrap_or_else(|(status, message)| error(status, message))
@ -889,9 +889,10 @@ fn serve(store: Store, bind: String, read_only: bool, cors: bool) -> anyhow::Res
handle_request(request, store.clone(), read_only)
.unwrap_or_else(|(status, message)| error(status, message))
})
};
server.set_global_timeout(HTTP_TIMEOUT);
server.set_server_name(concat!("Oxigraph/", env!("CARGO_PKG_VERSION")))?;
}
.with_global_timeout(HTTP_TIMEOUT)
.with_server_name(concat!("Oxigraph/", env!("CARGO_PKG_VERSION")))?
.with_max_num_threads(available_parallelism()?.get() * 128);
#[cfg(target_os = "linux")]
systemd_notify_ready()?;
eprintln!("Listening for requests at http://{}", &bind);
@ -1191,7 +1192,7 @@ fn handle_request(
resolve_with_base(request, &format!("/store/{:x}", random::<u128>()))?;
web_load_graph(&store, request, format, &graph.clone().into())?;
Ok(Response::builder(Status::CREATED)
.with_header(HeaderName::LOCATION, graph.as_str())
.with_header(HeaderName::LOCATION, graph.into_string())
.unwrap()
.build())
}

@ -18,9 +18,12 @@ rust-version.workspace = true
[features]
default = []
js = ["getrandom/js", "oxsdatatypes/js", "js-sys"]
http_client = ["oxhttp", "oxhttp/rustls"]
http-client = ["oxhttp"]
http-client-native-tls = ["http-client", "oxhttp/native-tls"]
http-client-rustls-webpki = ["http-client", "oxhttp/rustls-webpki"]
http-client-rustls-native = ["http-client", "oxhttp/rustls-native"]
rocksdb-pkg-config = ["oxrocksdb-sys/pkg-config"]
rocksdb_debug = []
rocksdb-debug = []
[dependencies]
digest = "0.10"
@ -44,7 +47,7 @@ sparopt = { version = "0.1.0-alpha.1-dev", path="sparopt", features = ["rdf-star
[target.'cfg(not(target_family = "wasm"))'.dependencies]
libc = "0.2.147"
oxrocksdb-sys = { version = "0.4.0-alpha.1-dev", path="../oxrocksdb-sys" }
oxhttp = { version = "0.1.7", optional = true }
oxhttp = { version = "0.2.0-alpha.1", optional = true }
[target.'cfg(all(target_family = "wasm", target_os = "unknown"))'.dependencies]
getrandom = "0.2"
@ -52,7 +55,7 @@ js-sys = { version = "0.3.60", optional = true }
[target.'cfg(not(target_family = "wasm"))'.dev-dependencies]
criterion = "0.5"
oxhttp = "0.1.7"
oxhttp = "0.2.0-alpha.1"
zstd = "0.12"
[package.metadata.docs.rs]

@ -48,9 +48,14 @@ if let QueryResults::Solutions(mut solutions) = store.query("SELECT ?s WHERE {
```
Some parts of this library are available as standalone crates:
* [`oxrdf`](https://crates.io/crates/oxrdf) provides datastructures encoding RDF basic concepts (the `oxigraph::model` module).
* [`spargebra`](https://crates.io/crates/spargebra) provides a SPARQL parser.
* [`sparesults`](https://crates.io/crates/sparesults) provides parsers and serializers for SPARQL result formats.
* [`oxrdf`](https://crates.io/crates/oxrdf), datastructures encoding RDF basic concepts (the `oxigraph::model` module).
* [`oxrdfio`](https://crates.io/crates/oxrdfio), a unified parser and serializer API for RDF formats. It itself relies on:
* [`oxttl`](https://crates.io/crates/oxttl), N-Triple, N-Quad, Turtle, TriG and N3 parsing and serialization.
* [`oxrdfxml`](https://crates.io/crates/oxrdfxml), RDF/XML parsing and serialization.
* [`spargebra`](https://crates.io/crates/spargebra), a SPARQL parser.
* [`sparesults`](https://crates.io/crates/sparesults), parsers and serializers for SPARQL result formats.
* [`sparopt`](https://crates.io/crates/sparesults), a SPARQL optimizer.
* [`oxsdatatypes`](https://crates.io/crates/oxsdatatypes), an implementation of some XML Schema datatypes.
To build the library, don't forget to clone the submodules using `git clone --recursive https://github.com/oxigraph/oxigraph.git` to clone the repository including submodules or `git submodule update --init` to add submodules to the already cloned repository.

@ -207,8 +207,7 @@ criterion_main!(store);
fn read_data(file: &str) -> impl Read {
if !Path::new(file).exists() {
let mut client = oxhttp::Client::new();
client.set_redirection_limit(5);
let client = oxhttp::Client::new().with_redirection_limit(5);
let url = format!("https://github.com/Tpt/bsbm-tools/releases/download/v0.2/{file}");
let request = Request::builder(Method::GET, url.parse().unwrap()).build();
let response = client.request(request).unwrap();

@ -11,10 +11,10 @@ impl Client {
}
#[allow(clippy::unused_self)]
pub fn get(&self, _url: &str, _accept: &str) -> Result<(String, Empty)> {
pub fn get(&self, _url: &str, _accept: &'static str) -> Result<(String, Empty)> {
Err(Error::new(
ErrorKind::Unsupported,
"HTTP client is not available. Enable the feature 'http_client'",
"HTTP client is not available. Enable the feature 'http-client'",
))
}
@ -23,12 +23,12 @@ impl Client {
&self,
_url: &str,
_payload: Vec<u8>,
_content_type: &str,
_accept: &str,
_content_type: &'static str,
_accept: &'static str,
) -> Result<(String, Empty)> {
Err(Error::new(
ErrorKind::Unsupported,
"HTTP client is not available. Enable the feature 'http_client'",
"HTTP client is not available. Enable the feature 'http-client'",
))
}
}

@ -1,9 +1,9 @@
#[cfg(not(feature = "http_client"))]
#[cfg(not(feature = "http-client"))]
mod dummy;
#[cfg(feature = "http_client")]
#[cfg(feature = "http-client")]
mod simple;
#[cfg(not(feature = "http_client"))]
#[cfg(not(feature = "http-client"))]
pub use dummy::Client;
#[cfg(feature = "http_client")]
#[cfg(feature = "http-client")]
pub use simple::Client;

@ -8,18 +8,17 @@ pub struct Client {
impl Client {
pub fn new(timeout: Option<Duration>, redirection_limit: usize) -> Self {
let mut client = oxhttp::Client::new();
let mut client = oxhttp::Client::new()
.with_redirection_limit(redirection_limit)
.with_user_agent(concat!("Oxigraph/", env!("CARGO_PKG_VERSION")))
.unwrap();
if let Some(timeout) = timeout {
client.set_global_timeout(timeout);
client = client.with_global_timeout(timeout);
}
client.set_redirection_limit(redirection_limit);
client
.set_user_agent(concat!("Oxigraph/", env!("CARGO_PKG_VERSION")))
.unwrap();
Self { client }
}
pub fn get(&self, url: &str, accept: &str) -> Result<(String, Body)> {
pub fn get(&self, url: &str, accept: &'static str) -> Result<(String, Body)> {
let request = Request::builder(Method::GET, url.parse().map_err(invalid_input_error)?)
.with_header(HeaderName::ACCEPT, accept)
.map_err(invalid_input_error)?
@ -50,8 +49,8 @@ impl Client {
&self,
url: &str,
payload: Vec<u8>,
content_type: &str,
accept: &str,
content_type: &'static str,
accept: &'static str,
) -> Result<(String, Body)> {
let request = Request::builder(Method::POST, url.parse().map_err(invalid_input_error)?)
.with_header(HeaderName::ACCEPT, accept)

@ -138,7 +138,7 @@ pub(crate) fn evaluate_query(
/// Options for SPARQL query evaluation.
///
///
/// If the `"http_client"` optional feature is enabled,
/// If the `"http-client"` optional feature is enabled,
/// a simple HTTP 1.1 client is used to execute [SPARQL 1.1 Federated Query](https://www.w3.org/TR/sparql11-federated-query/) SERVICE calls.
///
/// Usage example disabling the federated query support:
@ -182,7 +182,7 @@ impl QueryOptions {
}
/// Sets a timeout for HTTP requests done during SPARQL evaluation.
#[cfg(feature = "http_client")]
#[cfg(feature = "http-client")]
#[inline]
#[must_use]
pub fn with_http_timeout(mut self, timeout: Duration) -> Self {
@ -193,7 +193,7 @@ impl QueryOptions {
/// Sets an upper bound of the number of HTTP redirection followed per HTTP request done during SPARQL evaluation.
///
/// By default this value is `0`.
#[cfg(feature = "http_client")]
#[cfg(feature = "http-client")]
#[inline]
#[must_use]
pub fn with_http_redirection_limit(mut self, redirection_limit: usize) -> Self {
@ -235,7 +235,7 @@ impl QueryOptions {
fn service_handler(&self) -> Rc<dyn ServiceHandler<Error = EvaluationError>> {
self.service_handler.clone().unwrap_or_else(|| {
if cfg!(feature = "http_client") {
if cfg!(feature = "http-client") {
Rc::new(service::SimpleServiceHandler::new(
self.http_timeout,
self.http_redirection_limit,

@ -200,7 +200,7 @@ impl Db {
16,
);
rocksdb_options_set_block_based_table_factory(options, block_based_table_options);
#[cfg(feature = "rocksdb_debug")]
#[cfg(feature = "rocksdb-debug")]
{
rocksdb_options_set_info_log_level(options, 0);
rocksdb_options_enable_statistics(options);

@ -5,7 +5,7 @@ from urllib.request import urlopen
TARGET_DEBIAN_VERSIONS = ["sid"]
IGNORE_PACKAGES = {"oxigraph-js", "oxigraph-testsuite", "pyoxigraph", "sparql-smith"}
ALLOWED_MISSING_PACKAGES = {"escargot", "quick-xml"}
ALLOWED_MISSING_PACKAGES = {"escargot", "oxhttp", "quick-xml"}
base_path = Path(__file__).parent.parent

@ -18,9 +18,12 @@ name = "pyoxigraph"
doctest = false
[features]
default = ["native-tls"]
abi3 = ["pyo3/abi3-py38"]
native-tls = ["oxigraph/http-client-native-tls"]
rocksdb-pkg-config = ["oxigraph/rocksdb-pkg-config"]
rustls = ["oxigraph/http-client-rustls-native"]
[dependencies]
oxigraph = { version = "0.4.0-alpha.1-dev", path="../lib", features = ["http_client"] }
oxigraph = { version = "0.4.0-alpha.1-dev", path="../lib" }
pyo3 = { version = "0.19", features = ["extension-module"] }

@ -32,6 +32,9 @@ Pyoxigraph documentation is [available on the Oxigraph website](https://pyoxigra
To build and install the development version of pyoxigraph you need to clone this git repository including submodules (`git clone --recursive https://github.com/oxigraph/oxigraph.git`)
and to run `pip install .` in the `python` directory (the one this README is in).
Note that by default the installation will not use [cpython stable ABI](https://docs.python.org/3/c-api/stable.html) and will rely on the host TLS implementation.
Use `--features abi3` feature to use cpython stable ABI and use `--no-default-features --features rustls` to use [rustls](https://github.com/rustls/rustls) with the system certificates.
## Help
Feel free to use [GitHub discussions](https://github.com/oxigraph/oxigraph/discussions) or [the Gitter chat](https://gitter.im/oxigraph/community) to ask questions or talk about Oxigraph.

Loading…
Cancel
Save