From ab17138f331d4b90ec26f5a52b45ab4e4d57032d Mon Sep 17 00:00:00 2001 From: Tpt Date: Sun, 11 Jun 2023 18:34:50 +0200 Subject: [PATCH 001/217] Starts 0.4.0-alpha.1 --- .github/workflows/artifacts.yml | 1 + .github/workflows/tests.yml | 1 + Cargo.lock | 20 ++++++++++---------- js/Cargo.toml | 4 ++-- lib/Cargo.toml | 12 ++++++------ lib/oxrdf/Cargo.toml | 4 ++-- lib/oxsdatatypes/Cargo.toml | 2 +- lib/sparesults/Cargo.toml | 4 ++-- lib/spargebra/Cargo.toml | 4 ++-- lib/sparql-smith/Cargo.toml | 2 +- oxrocksdb-sys/Cargo.toml | 2 +- python/Cargo.toml | 4 ++-- server/Cargo.toml | 6 +++--- 13 files changed, 34 insertions(+), 32 deletions(-) diff --git a/.github/workflows/artifacts.yml b/.github/workflows/artifacts.yml index e71b1df8..e544b33e 100644 --- a/.github/workflows/artifacts.yml +++ b/.github/workflows/artifacts.yml @@ -4,6 +4,7 @@ on: push: branches: - main + - next release: types: - published diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 363c26ef..0a23e41b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -4,6 +4,7 @@ on: pull_request: branches: - main + - next schedule: - cron: "12 3 * * *" diff --git a/Cargo.lock b/Cargo.lock index 1b6590b8..8ffeb9bb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -933,7 +933,7 @@ dependencies = [ [[package]] name = "oxigraph" -version = "0.3.18" +version = "0.4.0-alpha.1-dev" dependencies = [ "criterion", "digest", @@ -965,7 +965,7 @@ dependencies = [ [[package]] name = "oxigraph_js" -version = "0.3.18" +version = "0.4.0-alpha.1-dev" dependencies = [ "console_error_panic_hook", "js-sys", @@ -975,7 +975,7 @@ dependencies = [ [[package]] name = "oxigraph_server" -version = "0.3.18" +version = "0.4.0-alpha.1-dev" dependencies = [ "anyhow", "assert_cmd", @@ -1023,7 +1023,7 @@ checksum = "bb175ec8981211357b7b379869c2f8d555881c55ea62311428ec0de46d89bd5c" [[package]] name = "oxrdf" -version = "0.1.7" +version = "0.2.0-alpha.1-dev" dependencies = [ "oxilangtag", "oxiri", @@ -1033,7 +1033,7 @@ dependencies = [ [[package]] name = "oxrocksdb-sys" -version = "0.3.18" +version = "0.4.0-alpha.1-dev" dependencies = [ "bindgen", "cc", @@ -1042,7 +1042,7 @@ dependencies = [ [[package]] name = "oxsdatatypes" -version = "0.1.3" +version = "0.2.0-alpha.1-dev" dependencies = [ "js-sys", ] @@ -1284,7 +1284,7 @@ dependencies = [ [[package]] name = "pyoxigraph" -version = "0.3.18" +version = "0.4.0-alpha.1-dev" dependencies = [ "oxigraph", "pyo3", @@ -1625,7 +1625,7 @@ checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" [[package]] name = "sparesults" -version = "0.1.8" +version = "0.2.0-alpha.1-dev" dependencies = [ "json-event-parser", "oxrdf", @@ -1634,7 +1634,7 @@ dependencies = [ [[package]] name = "spargebra" -version = "0.2.8" +version = "0.3.0-alpha.1-dev" dependencies = [ "oxilangtag", "oxiri", @@ -1645,7 +1645,7 @@ dependencies = [ [[package]] name = "sparql-smith" -version = "0.1.0-alpha.4" +version = "0.1.0-alpha.5-dev" dependencies = [ "arbitrary", ] diff --git a/js/Cargo.toml b/js/Cargo.toml index 55e9d0bc..ee103f79 100644 --- a/js/Cargo.toml +++ b/js/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "oxigraph_js" -version = "0.3.18" +version = "0.4.0-alpha.1-dev" authors = ["Tpt "] license = "MIT OR Apache-2.0" readme = "README.md" @@ -14,7 +14,7 @@ crate-type = ["cdylib"] name = "oxigraph" [dependencies] -oxigraph = { version = "0.3.18", path="../lib" } +oxigraph = { version = "0.4.0-alpha.1-dev", path="../lib" } wasm-bindgen = "0.2" js-sys = "0.3" console_error_panic_hook = "0.1" diff --git a/lib/Cargo.toml b/lib/Cargo.toml index c7fa0d26..40bbc0de 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "oxigraph" -version = "0.3.18" +version = "0.4.0-alpha.1-dev" authors = ["Tpt "] license = "MIT OR Apache-2.0" readme = "README.md" @@ -38,14 +38,14 @@ hex = "0.4" siphasher = "0.3" lazy_static = "1" json-event-parser = "0.1" -oxrdf = { version = "0.1.7", path="oxrdf", features = ["rdf-star", "oxsdatatypes"] } -oxsdatatypes = { version = "0.1.3", path="oxsdatatypes" } -spargebra = { version = "0.2.8", path="spargebra", features = ["rdf-star", "sep-0002", "sep-0006"] } -sparesults = { version = "0.1.8", path="sparesults", features = ["rdf-star"] } +oxrdf = { version = "0.2.0-alpha.1-dev", path="oxrdf", features = ["rdf-star", "oxsdatatypes"] } +oxsdatatypes = { version = "0.2.0-alpha.1-dev", path="oxsdatatypes" } +spargebra = { version = "0.3.0-alpha.1-dev", path="spargebra", features = ["rdf-star", "sep-0002", "sep-0006"] } +sparesults = { version = "0.2.0-alpha.1-dev", path="sparesults", features = ["rdf-star"] } [target.'cfg(not(target_family = "wasm"))'.dependencies] libc = "0.2" -oxrocksdb-sys = { version = "0.3.18", path="../oxrocksdb-sys" } +oxrocksdb-sys = { version = "0.4.0-alpha.1-dev", path="../oxrocksdb-sys" } oxhttp = { version = "0.1", optional = true } [target.'cfg(all(target_family = "wasm", target_os = "unknown"))'.dependencies] diff --git a/lib/oxrdf/Cargo.toml b/lib/oxrdf/Cargo.toml index 3e7789c8..2f7de43a 100644 --- a/lib/oxrdf/Cargo.toml +++ b/lib/oxrdf/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "oxrdf" -version = "0.1.7" +version = "0.2.0-alpha.1-dev" authors = ["Tpt "] license = "MIT OR Apache-2.0" readme = "README.md" @@ -21,7 +21,7 @@ rdf-star = [] rand = "0.8" oxilangtag = "0.1" oxiri = "0.2" -oxsdatatypes = { version = "0.1.3", path="../oxsdatatypes", optional = true } +oxsdatatypes = { version = "0.2.0-alpha.1-dev", path="../oxsdatatypes", optional = true } [package.metadata.docs.rs] all-features = true diff --git a/lib/oxsdatatypes/Cargo.toml b/lib/oxsdatatypes/Cargo.toml index 27f2084e..9e20f50d 100644 --- a/lib/oxsdatatypes/Cargo.toml +++ b/lib/oxsdatatypes/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "oxsdatatypes" -version = "0.1.3" +version = "0.2.0-alpha.1-dev" authors = ["Tpt "] license = "MIT OR Apache-2.0" readme = "README.md" diff --git a/lib/sparesults/Cargo.toml b/lib/sparesults/Cargo.toml index a278e55e..e74ebd81 100644 --- a/lib/sparesults/Cargo.toml +++ b/lib/sparesults/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sparesults" -version = "0.1.8" +version = "0.2.0-alpha.1-dev" authors = ["Tpt "] license = "MIT OR Apache-2.0" readme = "README.md" @@ -19,7 +19,7 @@ rdf-star = ["oxrdf/rdf-star"] [dependencies] json-event-parser = "0.1" -oxrdf = { version = "0.1.6", path="../oxrdf" } +oxrdf = { version = "0.2.0-alpha.1-dev", path="../oxrdf" } quick-xml = "0.28" [package.metadata.docs.rs] diff --git a/lib/spargebra/Cargo.toml b/lib/spargebra/Cargo.toml index c2ff363a..911d861a 100644 --- a/lib/spargebra/Cargo.toml +++ b/lib/spargebra/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "spargebra" -version = "0.2.8" +version = "0.3.0-alpha.1-dev" authors = ["Tpt "] license = "MIT OR Apache-2.0" readme = "README.md" @@ -24,7 +24,7 @@ peg = "0.8" rand = "0.8" oxiri = "0.2" oxilangtag = "0.1" -oxrdf = { version = "0.1.6", path="../oxrdf" } +oxrdf = { version = "0.2.0-alpha.1-dev", path="../oxrdf" } [package.metadata.docs.rs] all-features = true diff --git a/lib/sparql-smith/Cargo.toml b/lib/sparql-smith/Cargo.toml index 8dc8ea57..a36096bb 100644 --- a/lib/sparql-smith/Cargo.toml +++ b/lib/sparql-smith/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sparql-smith" -version = "0.1.0-alpha.4" +version = "0.1.0-alpha.5-dev" authors = ["Tpt "] license = "MIT OR Apache-2.0" readme = "README.md" diff --git a/oxrocksdb-sys/Cargo.toml b/oxrocksdb-sys/Cargo.toml index cda577e8..f3725f25 100644 --- a/oxrocksdb-sys/Cargo.toml +++ b/oxrocksdb-sys/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "oxrocksdb-sys" -version = "0.3.18" +version = "0.4.0-alpha.1-dev" authors = ["Tpt "] license = "GPL-2.0 OR Apache-2.0" repository = "https://github.com/oxigraph/oxigraph/tree/main/oxrocksdb-sys" diff --git a/python/Cargo.toml b/python/Cargo.toml index 89ca09e7..6f3b59d4 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyoxigraph" -version = "0.3.18" +version = "0.4.0-alpha.1-dev" authors = ["Tpt"] license = "MIT OR Apache-2.0" readme = "README.md" @@ -19,5 +19,5 @@ doctest = false abi3 = ["pyo3/abi3-py37"] [dependencies] -oxigraph = { version = "0.3.18", path="../lib", features = ["http_client"] } +oxigraph = { version = "0.4.0-alpha.1-dev", path="../lib", features = ["http_client"] } pyo3 = { version = "0.19", features = ["extension-module"] } diff --git a/server/Cargo.toml b/server/Cargo.toml index 81d62db2..fcb610a0 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "oxigraph_server" -version = "0.3.18" +version = "0.4.0-alpha.1-dev" authors = ["Tpt "] license = "MIT OR Apache-2.0" readme = "README.md" @@ -17,8 +17,8 @@ anyhow = "1" oxhttp = { version = "0.1", features = ["rayon"] } clap = { version = "=4.0", features = ["derive"] } clap_lex = "=0.3.0" -oxigraph = { version = "0.3.18", path = "../lib", features = ["http_client"] } -sparesults = { version = "0.1.8", path = "../lib/sparesults", features = ["rdf-star"] } +oxigraph = { version = "0.4.0-alpha.1-dev", path = "../lib", features = ["http_client"] } +sparesults = { version = "0.2.0-alpha.1-dev", path = "../lib/sparesults", features = ["rdf-star"] } rand = "0.8" url = "2" oxiri = "0.2" From ae294683d6d3020673bdc5955dc14e913b7c8af4 Mon Sep 17 00:00:00 2001 From: Tpt Date: Mon, 12 Jun 2023 16:48:45 +0200 Subject: [PATCH 002/217] Upgrades MSRV to 1.65 This is the MSRV of the dependencies (1.64) bumped to 1.65 to allow GAT --- .github/workflows/tests.yml | 8 +- Cargo.lock | 231 +++++++++++++++--------------------- lib/Cargo.toml | 4 +- lib/oxrdf/Cargo.toml | 2 +- lib/oxsdatatypes/Cargo.toml | 2 +- lib/sparesults/Cargo.toml | 2 +- lib/spargebra/Cargo.toml | 2 +- oxrocksdb-sys/Cargo.toml | 2 +- server/Cargo.toml | 14 +-- testsuite/Cargo.toml | 3 - 10 files changed, 112 insertions(+), 158 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0a23e41b..65b9102a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -66,7 +66,7 @@ jobs: - uses: actions/checkout@v3 with: submodules: true - - run: rustup update && rustup override set 1.60.0 && rustup component add clippy + - run: rustup update && rustup override set 1.65.0 && rustup component add clippy - uses: Swatinem/rust-cache@v2 - run: cargo clippy -- -D warnings -D clippy::all working-directory: ./lib/oxsdatatypes @@ -85,7 +85,7 @@ jobs: - uses: actions/checkout@v3 with: submodules: true - - run: rustup update && rustup override set 1.60.0 && rustup target add wasm32-unknown-unknown && rustup component add clippy + - run: rustup update && rustup override set 1.65.0 && rustup target add wasm32-unknown-unknown && rustup component add clippy - uses: Swatinem/rust-cache@v2 - run: cargo clippy --lib --tests --target wasm32-unknown-unknown -- -D warnings -D clippy::all working-directory: ./js @@ -96,7 +96,7 @@ jobs: - uses: actions/checkout@v3 with: submodules: true - - run: rustup update && rustup override set 1.60.0 && rustup target add wasm32-wasi && rustup component add clippy + - run: rustup update && rustup override set 1.65.0 && rustup target add wasm32-wasi && rustup component add clippy - uses: Swatinem/rust-cache@v2 - run: cargo clippy --lib --tests --target wasm32-wasi -- -D warnings -D clippy::all working-directory: ./lib @@ -191,7 +191,7 @@ jobs: - uses: actions/checkout@v3 with: submodules: true - - run: rustup update && rustup override set 1.60.0 + - run: rustup update && rustup override set 1.65.0 - uses: Swatinem/rust-cache@v2 - run: cargo doc --all-features working-directory: ./lib diff --git a/Cargo.lock b/Cargo.lock index 8ffeb9bb..29d3744b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -32,6 +32,55 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" +[[package]] +name = "anstream" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is-terminal", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41ed9a86bf92ae6580e0a31281f65a1b1d867c0cc68d5346e2ae128dddfa6a7d" + +[[package]] +name = "anstyle-parse" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e765fd216e48e067936442276d1d57399e37bce53c264d6fefbe298080cb57ee" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" +dependencies = [ + "windows-sys 0.48.0", +] + +[[package]] +name = "anstyle-wincon" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "180abfa45703aebe0093f79badacc01b8fd4ea2e35118747e5811127f926e188" +dependencies = [ + "anstyle", + "windows-sys 0.48.0", +] + [[package]] name = "anyhow" version = "1.0.71" @@ -49,10 +98,11 @@ dependencies = [ [[package]] name = "assert_cmd" -version = "2.0.8" +version = "2.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9834fcc22e0874394a010230586367d4a3e9f11b560f469262678547e1d2575e" +checksum = "86d6b683edf8d1119fe420a94f8a7e389239666aa72e65495d91c00462510151" dependencies = [ + "anstyle", "bstr", "doc-comment", "predicates", @@ -63,10 +113,11 @@ dependencies = [ [[package]] name = "assert_fs" -version = "1.0.10" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d94b2a3f3786ff2996a98afbd6b4e5b7e890d685ccf67577f508ee2342c71cc9" +checksum = "f070617a68e5c2ed5d06ee8dd620ee18fb72b99f6c094bed34cf8ab07c875b48" dependencies = [ + "anstyle", "doc-comment", "globwalk", "predicates", @@ -75,17 +126,6 @@ dependencies = [ "tempfile", ] -[[package]] -name = "atty" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" -dependencies = [ - "hermit-abi 0.1.19", - "libc", - "winapi 0.3.9", -] - [[package]] name = "autocfg" version = "1.1.0" @@ -224,61 +264,51 @@ dependencies = [ [[package]] name = "clap" -version = "3.2.25" +version = "4.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" +checksum = "ca8f255e4b8027970e78db75e78831229c9815fdbfa67eb1a1b777a62e24b4a0" dependencies = [ - "bitflags", - "clap_lex 0.2.4", - "indexmap", - "textwrap", + "clap_builder", + "clap_derive", + "once_cell", ] [[package]] -name = "clap" -version = "4.0.32" +name = "clap_builder" +version = "4.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7db700bc935f9e43e88d00b0850dae18a63773cfbec6d8e070fccf7fef89a39" +checksum = "acd4f3c17c83b0ba34ffbc4f8bbd74f079413f747f84a6f89292f138057e36ab" dependencies = [ + "anstream", + "anstyle", "bitflags", - "clap_derive", - "clap_lex 0.3.0", - "is-terminal", - "once_cell", + "clap_lex", "strsim", - "termcolor", ] [[package]] name = "clap_derive" -version = "4.0.21" +version = "4.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0177313f9f02afc995627906bbd8967e2be069f5261954222dac78290c2b9014" +checksum = "b8cd2b2a819ad6eec39e8f1d6b53001af1e5469f8c177579cdaeb313115b825f" dependencies = [ "heck", - "proc-macro-error", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.18", ] [[package]] name = "clap_lex" -version = "0.2.4" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" -dependencies = [ - "os_str_bytes", -] +checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b" [[package]] -name = "clap_lex" -version = "0.3.0" +name = "colorchoice" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d4198f73e42b4936b35b5bb248d81d2b595ecb170da0bac7655c54eedfa8da8" -dependencies = [ - "os_str_bytes", -] +checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" [[package]] name = "console_error_panic_hook" @@ -326,19 +356,19 @@ dependencies = [ [[package]] name = "criterion" -version = "0.4.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7c76e09c1aae2bc52b3d2f29e13c6572553b30c4aa1b8a49fd70de6412654cb" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" dependencies = [ "anes", - "atty", "cast", "ciborium", - "clap 3.2.25", + "clap", "criterion-plot", + "is-terminal", "itertools", - "lazy_static", "num-traits", + "once_cell", "oorandom", "plotters", "rayon", @@ -596,27 +626,12 @@ version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" -[[package]] -name = "hashbrown" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" - [[package]] name = "heck" version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" -[[package]] -name = "hermit-abi" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" -dependencies = [ - "libc", -] - [[package]] name = "hermit-abi" version = "0.2.6" @@ -656,11 +671,10 @@ dependencies = [ [[package]] name = "ignore" -version = "0.4.18" +version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "713f1b139373f96a2e0ce3ac931cd01ee973c3c5dd7c40c0c2efe96ad2b6751d" +checksum = "dbe7873dab538a9a44ad79ede1faf5f30d49f9a5c883ddbab48bce81b64b7492" dependencies = [ - "crossbeam-utils", "globset", "lazy_static", "log", @@ -672,16 +686,6 @@ dependencies = [ "winapi-util", ] -[[package]] -name = "indexmap" -version = "1.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" -dependencies = [ - "autocfg", - "hashbrown", -] - [[package]] name = "indoc" version = "1.0.9" @@ -911,12 +915,6 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" -[[package]] -name = "os_str_bytes" -version = "6.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d5d9eb14b174ee9aa2ef96dc2b94637a2d4b6e7cb873c7e171f0c20c6cf3eac" - [[package]] name = "oxhttp" version = "0.1.6" @@ -980,17 +978,13 @@ dependencies = [ "anyhow", "assert_cmd", "assert_fs", - "clap 4.0.32", - "clap_lex 0.3.0", + "clap", "escargot", "flate2", - "ignore", "oxhttp", "oxigraph", "oxiri", "predicates", - "predicates-core", - "predicates-tree", "rand", "rayon-core", "sparesults", @@ -1002,8 +996,7 @@ name = "oxigraph_testsuite" version = "0.0.0" dependencies = [ "anyhow", - "clap 4.0.32", - "criterion", + "clap", "oxigraph", "text-diff", "time", @@ -1151,10 +1144,11 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "predicates" -version = "2.1.5" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59230a63c37f3e18569bdb90e4a89cbf5bf8b06fea0b84e65ea10cc4df47addd" +checksum = "09963355b9f467184c04017ced4a2ba2d75cbcb4e7462690d388233253d4b1a9" dependencies = [ + "anstyle", "difflib", "float-cmp", "itertools", @@ -1165,15 +1159,15 @@ dependencies = [ [[package]] name = "predicates-core" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72f883590242d3c6fc5bf50299011695fa6590c2c70eac95ee1bdb9a733ad1a2" +checksum = "b794032607612e7abeb4db69adb4e33590fa6cf1149e95fd7cb00e634b92f174" [[package]] name = "predicates-tree" -version = "1.0.7" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54ff541861505aabf6ea722d2131ee980b8276e10a1297b94e896dd8b621850d" +checksum = "368ba315fb8c5052ab692e68a0eefec6ec57b23a36959c14496f0b0df2c0cecf" dependencies = [ "predicates-core", "termtree", @@ -1189,30 +1183,6 @@ dependencies = [ "syn 2.0.18", ] -[[package]] -name = "proc-macro-error" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" -dependencies = [ - "proc-macro-error-attr", - "proc-macro2", - "quote", - "syn 1.0.109", - "version_check", -] - -[[package]] -name = "proc-macro-error-attr" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" -dependencies = [ - "proc-macro2", - "quote", - "version_check", -] - [[package]] name = "proc-macro2" version = "1.0.60" @@ -1714,15 +1684,6 @@ dependencies = [ "winapi 0.2.8", ] -[[package]] -name = "termcolor" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" -dependencies = [ - "winapi-util", -] - [[package]] name = "termtree" version = "0.4.1" @@ -1739,12 +1700,6 @@ dependencies = [ "term", ] -[[package]] -name = "textwrap" -version = "0.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" - [[package]] name = "thread_local" version = "1.1.7" @@ -1863,6 +1818,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "utf8parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" + [[package]] name = "version_check" version = "0.9.4" diff --git a/lib/Cargo.toml b/lib/Cargo.toml index 40bbc0de..d67b1349 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -12,7 +12,7 @@ description = """ a SPARQL database and RDF toolkit """ edition = "2021" -rust-version = "1.60" +rust-version = "1.65" [package.metadata.docs.rs] all-features = true @@ -53,7 +53,7 @@ getrandom = { version = "0.2", features = ["js"] } js-sys = "0.3" [target.'cfg(not(target_family = "wasm"))'.dev-dependencies] -criterion = "0.4" +criterion = "0.5" oxhttp = "0.1" zstd = "0.12" diff --git a/lib/oxrdf/Cargo.toml b/lib/oxrdf/Cargo.toml index 2f7de43a..9e429d7d 100644 --- a/lib/oxrdf/Cargo.toml +++ b/lib/oxrdf/Cargo.toml @@ -11,7 +11,7 @@ description = """ A library providing basic data structures related to RDF """ edition = "2021" -rust-version = "1.60" +rust-version = "1.65" [features] default = [] diff --git a/lib/oxsdatatypes/Cargo.toml b/lib/oxsdatatypes/Cargo.toml index 9e20f50d..86c7d292 100644 --- a/lib/oxsdatatypes/Cargo.toml +++ b/lib/oxsdatatypes/Cargo.toml @@ -11,7 +11,7 @@ description = """ An implementation of some XSD datatypes for SPARQL implementations """ edition = "2021" -rust-version = "1.60" +rust-version = "1.65" [target.'cfg(all(target_family = "wasm", target_os = "unknown"))'.dependencies] js-sys = "0.3" diff --git a/lib/sparesults/Cargo.toml b/lib/sparesults/Cargo.toml index e74ebd81..baddda71 100644 --- a/lib/sparesults/Cargo.toml +++ b/lib/sparesults/Cargo.toml @@ -11,7 +11,7 @@ description = """ SPARQL query results formats parsers and serializers """ edition = "2021" -rust-version = "1.60" +rust-version = "1.65" [features] default = [] diff --git a/lib/spargebra/Cargo.toml b/lib/spargebra/Cargo.toml index 911d861a..5fa672b5 100644 --- a/lib/spargebra/Cargo.toml +++ b/lib/spargebra/Cargo.toml @@ -11,7 +11,7 @@ description = """ A SPARQL parser """ edition = "2021" -rust-version = "1.60" +rust-version = "1.65" [features] default = [] diff --git a/oxrocksdb-sys/Cargo.toml b/oxrocksdb-sys/Cargo.toml index f3725f25..5100e198 100644 --- a/oxrocksdb-sys/Cargo.toml +++ b/oxrocksdb-sys/Cargo.toml @@ -9,7 +9,7 @@ description = """ Rust bindings for RocksDB for Oxigraph usage. """ edition = "2021" -rust-version = "1.60" +rust-version = "1.65" build = "build.rs" links = "rocksdb" diff --git a/server/Cargo.toml b/server/Cargo.toml index fcb610a0..36b5fc00 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -10,13 +10,12 @@ description = """ Oxigraph SPARQL HTTP server """ edition = "2021" -rust-version = "1.60" +rust-version = "1.65" [dependencies] anyhow = "1" oxhttp = { version = "0.1", features = ["rayon"] } -clap = { version = "=4.0", features = ["derive"] } -clap_lex = "=0.3.0" +clap = { version = "4.0", features = ["derive"] } oxigraph = { version = "0.4.0-alpha.1-dev", path = "../lib", features = ["http_client"] } sparesults = { version = "0.2.0-alpha.1-dev", path = "../lib/sparesults", features = ["rdf-star"] } rand = "0.8" @@ -26,10 +25,7 @@ flate2 = "1" rayon-core = "1" [dev-dependencies] -assert_cmd = "=2.0.8" -assert_fs = "=1.0.10" +assert_cmd = "2" +assert_fs = "1" escargot = "0.5" -ignore = "=0.4.18" -predicates = "2" -predicates-core = "=1.0.5" -predicates-tree = "=1.0.7" \ No newline at end of file +predicates = "3" diff --git a/testsuite/Cargo.toml b/testsuite/Cargo.toml index e27113cf..92305371 100644 --- a/testsuite/Cargo.toml +++ b/testsuite/Cargo.toml @@ -17,6 +17,3 @@ clap = { version = "4", features = ["derive"] } time = { version = "0.3", features = ["formatting"] } oxigraph = { path = "../lib" } text-diff = "0.4" - -[dev-dependencies] -criterion = "0.4" From c016116b09b96b5d21ffb277092b06bf881ef363 Mon Sep 17 00:00:00 2001 From: Tpt Date: Mon, 12 Jun 2023 20:41:14 +0200 Subject: [PATCH 003/217] Makes Clippy 1.65 happy --- lib/oxrdf/src/dataset.rs | 15 +++------------ lib/oxrdf/src/interning.rs | 2 +- lib/spargebra/src/parser.rs | 2 +- lib/src/sparql/eval.rs | 16 ++++++++-------- lib/src/sparql/plan_builder.rs | 2 +- lib/src/storage/backend/fallback.rs | 2 +- lib/src/storage/backend/rocksdb.rs | 4 ++-- 7 files changed, 17 insertions(+), 26 deletions(-) diff --git a/lib/oxrdf/src/dataset.rs b/lib/oxrdf/src/dataset.rs index 05ccbb56..95ecef75 100644 --- a/lib/oxrdf/src/dataset.rs +++ b/lib/oxrdf/src/dataset.rs @@ -29,6 +29,7 @@ use crate::interning::*; use crate::SubjectRef; use crate::*; +use std::cmp::min; use std::collections::hash_map::DefaultHasher; use std::collections::BTreeSet; use std::collections::{HashMap, HashSet}; @@ -705,7 +706,7 @@ impl Dataset { InternedTerm, InternedGraphName, )> { - let b_prime = partition.iter().find_map(|(_, b)| (b.len() > 1).then(|| b)); + let b_prime = partition.iter().map(|(_, b)| b).find(|b| b.len() > 1); if let Some(b_prime) = b_prime { b_prime .iter() @@ -715,17 +716,7 @@ impl Dataset { let (hash_prime_prime, partition_prime) = self.hash_bnodes(hash_prime); self.distinguish(&hash_prime_prime, &partition_prime) }) - .fold(None, |a, b| { - Some(if let Some(a) = a { - if a <= b { - a - } else { - b - } - } else { - b - }) - }) + .reduce(min) .unwrap_or_default() } else { self.label(hash) diff --git a/lib/oxrdf/src/interning.rs b/lib/oxrdf/src/interning.rs index 54c8acde..02d229e0 100644 --- a/lib/oxrdf/src/interning.rs +++ b/lib/oxrdf/src/interning.rs @@ -466,7 +466,7 @@ impl InternedTriple { interner .triples .contains_key(&interned_triple) - .then(|| interned_triple) + .then_some(interned_triple) } pub fn next(&self) -> Self { diff --git a/lib/spargebra/src/parser.rs b/lib/spargebra/src/parser.rs index 7779e31c..fbe7af91 100644 --- a/lib/spargebra/src/parser.rs +++ b/lib/spargebra/src/parser.rs @@ -736,7 +736,7 @@ impl ParserState { let aggregates = self.aggregates.last_mut().ok_or("Unexpected aggregate")?; Ok(aggregates .iter() - .find_map(|(v, a)| (a == &agg).then(|| v)) + .find_map(|(v, a)| (a == &agg).then_some(v)) .cloned() .unwrap_or_else(|| { let new_var = variable(); diff --git a/lib/src/sparql/eval.rs b/lib/src/sparql/eval.rs index 8c94853d..f4620f56 100644 --- a/lib/src/sparql/eval.rs +++ b/lib/src/sparql/eval.rs @@ -2321,7 +2321,7 @@ fn to_argument_compatible_strings( ) -> Option<(String, String, Option)> { let (value1, language1) = to_string_and_language(dataset, arg1)?; let (value2, language2) = to_string_and_language(dataset, arg2)?; - (language2.is_none() || language1 == language2).then(|| (value1, value2, language1)) + (language2.is_none() || language1 == language2).then_some((value1, value2, language1)) } pub(super) fn compile_pattern(pattern: &str, flags: Option<&str>) -> Option { @@ -3080,10 +3080,10 @@ fn put_pattern_value( tuple: &mut EncodedTuple, ) -> Option<()> { match selector { - TupleSelector::Constant(c) => (*c == value).then(|| ()), + TupleSelector::Constant(c) => (*c == value).then_some(()), TupleSelector::Variable(v) => { if let Some(old) = tuple.get(*v) { - (value == *old).then(|| ()) + (value == *old).then_some(()) } else { tuple.set(*v, value); Some(()) @@ -3161,7 +3161,7 @@ impl PathEvaluator { .and_then(|middle| { Ok(self .eval_closed_in_graph(b, &middle, end, graph_name)? - .then(|| ())) + .then_some(())) }) .transpose() }) @@ -3232,7 +3232,7 @@ impl PathEvaluator { Box::new(self.eval_from_in_unknown_graph(a, start).flat_map_ok( move |(middle, graph_name)| { eval.eval_closed_in_graph(&b, &middle, &end, &graph_name) - .map(|is_found| is_found.then(|| graph_name)) + .map(|is_found| is_found.then_some(graph_name)) .transpose() }, )) @@ -3252,7 +3252,7 @@ impl PathEvaluator { |e| eval.eval_from_in_graph(&p, &e, &graph_name), &end, ) - .map(|is_found| is_found.then(|| graph_name)) + .map(|is_found| is_found.then_some(graph_name)) .transpose() }) } @@ -3269,7 +3269,7 @@ impl PathEvaluator { |e| eval.eval_from_in_graph(&p, &e, &graph_name), &end, ) - .map(|is_found| is_found.then(|| graph_name)) + .map(|is_found| is_found.then_some(graph_name)) }) .transpose() }), @@ -3285,7 +3285,7 @@ impl PathEvaluator { let p = Rc::clone(p); self.run_if_term_is_a_dataset_node(start, move |graph_name| { eval.eval_closed_in_graph(&p, &start2, &end, &graph_name) - .map(|is_found| is_found.then(|| graph_name)) + .map(|is_found| is_found.then_some(graph_name)) .transpose() }) } diff --git a/lib/src/sparql/plan_builder.rs b/lib/src/sparql/plan_builder.rs index 5e7370c1..7d985504 100644 --- a/lib/src/sparql/plan_builder.rs +++ b/lib/src/sparql/plan_builder.rs @@ -1223,7 +1223,7 @@ impl<'a> PlanBuilder<'a> { let encoded = if let Some(to_id) = to .iter() .enumerate() - .find_map(|(to_id, var)| (*var == from_variable.plain).then(|| to_id)) + .find_map(|(to_id, var)| (*var == from_variable.plain).then_some(to_id)) { to_id } else { diff --git a/lib/src/storage/backend/fallback.rs b/lib/src/storage/backend/fallback.rs index 6000863b..1c156111 100644 --- a/lib/src/storage/backend/fallback.rs +++ b/lib/src/storage/backend/fallback.rs @@ -32,7 +32,7 @@ impl Db { #[allow(clippy::unwrap_in_result)] pub fn column_family(&self, name: &'static str) -> Option { let name = ColumnFamily(name); - (self.0.read().unwrap().contains_key(&name)).then(|| name) + self.0.read().unwrap().contains_key(&name).then_some(name) } #[must_use] diff --git a/lib/src/storage/backend/rocksdb.rs b/lib/src/storage/backend/rocksdb.rs index fc8f4da2..81577551 100644 --- a/lib/src/storage/backend/rocksdb.rs +++ b/lib/src/storage/backend/rocksdb.rs @@ -393,7 +393,7 @@ impl Db { cf_handles, cf_options, is_secondary: true, - path_to_remove: in_memory.then(|| secondary_path), + path_to_remove: in_memory.then_some(secondary_path), })), }) } @@ -1001,7 +1001,7 @@ impl Reader { break; } } - found.then(|| bound) + found.then_some(bound) }; unsafe { From a8abf269135e347b0cb05463c27ace574e302325 Mon Sep 17 00:00:00 2001 From: Tpt Date: Mon, 12 Jun 2023 20:56:09 +0200 Subject: [PATCH 004/217] Enables some extra Clippy lints --- .cargo/config.toml | 24 ++++++++++++------------ js/src/model.rs | 6 +++--- lib/oxrdf/src/blank_node.rs | 2 +- lib/oxrdf/src/interning.rs | 2 +- lib/oxrdf/src/variable.rs | 2 +- lib/src/sparql/http/dummy.rs | 2 +- 6 files changed, 19 insertions(+), 19 deletions(-) diff --git a/.cargo/config.toml b/.cargo/config.toml index e248c473..09e6f6fe 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -5,8 +5,8 @@ rustflags = [ "-Wunsafe-code", "-Wunused-lifetimes", "-Wunused-qualifications", - # TODO: 1.63+ "-Wclippy::as-underscore", - # TODO: 1.65+ ""-Wclippy::bool-to-int-with-if", + "-Wclippy::as-underscore", + "-Wclippy::bool-to-int-with-if", "-Wclippy::borrow-as-ptr", "-Wclippy::case-sensitive-file-extension-comparisons", "-Wclippy::cast-lossless", @@ -23,11 +23,11 @@ rustflags = [ "-Wclippy::decimal-literal-representation", "-Wclippy::default-trait-access", "-Wclippy::default-union-representation", - # TODO: 1.61+ "-Wclippy::deref-by-slicing", - # TODO: 1.63+ "-Wclippy::doc-link-with-quotes", - # TODO: 1.62+ "-Wclippy::empty-drop", + "-Wclippy::deref-by-slicing", + "-Wclippy::doc-link-with-quotes", + "-Wclippy::empty-drop", "-Wclippy::empty-enum", - # TODO: on major version "-Wclippy::empty-structs-with-brackets", + "-Wclippy::empty-structs-with-brackets", "-Wclippy::enum-glob-use", "-Wclippy::exit", "-Wclippy::expect-used", @@ -38,7 +38,7 @@ rustflags = [ "-Wclippy::filter-map-next", "-Wclippy::flat-map-option", "-Wclippy::fn-to-numeric-cast-any", - # TODO: 1.62+ "-Wclippy::format-push-string", + "-Wclippy::format-push-string", "-Wclippy::from-iter-instead-of-collect", "-Wclippy::get-unwrap", "-Wclippy::if-not-else", @@ -62,10 +62,10 @@ rustflags = [ "-Wclippy::lossy-float-literal", "-Wclippy::macro-use-imports", "-Wclippy::manual-assert", - # TODO: 1.65+ "-Wclippy::manual-instant-elapsed", + "-Wclippy::manual-instant-elapsed", # TODO: 1.67+ "-Wclippy::manual-let-else", "-Wclippy::manual-ok-or", - # TODO: 1.65+ "-Wclippy::manual-string-new", + "-Wclippy::manual-string-new", "-Wclippy::many-single-char-names", "-Wclippy::map-unwrap-or", "-Wclippy::match-bool", @@ -73,7 +73,7 @@ rustflags = [ "-Wclippy::match-wildcard-for-single-variants", "-Wclippy::maybe-infinite-iter", "-Wclippy::mem-forget", - # TODO: 1.63+ "-Wclippy::mismatching-type-param-order", + "-Wclippy::mismatching-type-param-order", "-Wclippy::multiple-inherent-impl", "-Wclippy::mut-mut", "-Wclippy::mutex-atomic", @@ -99,7 +99,7 @@ rustflags = [ "-Wclippy::rest-pat-in-fully-bound-structs", "-Wclippy::return-self-not-must-use", "-Wclippy::same-functions-in-if-condition", - # TODO: strange failure on 1.60 "-Wclippy::same-name-method", + "-Wclippy::same-name-method", # TODO: 1.68+ "-Wclippy::semicolon-outside-block", "-Wclippy::single-match-else", "-Wclippy::stable-sort-primitive", @@ -117,7 +117,7 @@ rustflags = [ "-Wclippy::unimplemented", # TODO: 1.66+ "-Wclippy::uninlined-format-args", # TODO: 1.70+ "-Wclippy::unnecessary-box-returns", - # TODO: 1.61+ "-Wclippy::unnecessary-join", + "-Wclippy::unnecessary-join", # TODO: 1.67+ "-Wclippy::unnecessary-safety-comment", # TODO: 1.67+ "-Wclippy::unnecessary-safety-doc", "-Wclippy::unnecessary-self-imports", diff --git a/js/src/model.rs b/js/src/model.rs index 4929068f..753ab66e 100644 --- a/js/src/model.rs +++ b/js/src/model.rs @@ -302,7 +302,7 @@ impl From for Term { #[wasm_bindgen(js_name = DefaultGraph)] #[derive(Eq, PartialEq, Debug, Clone, Hash)] -pub struct JsDefaultGraph {} +pub struct JsDefaultGraph; #[wasm_bindgen(js_class = DefaultGraph)] impl JsDefaultGraph { @@ -313,7 +313,7 @@ impl JsDefaultGraph { #[wasm_bindgen(getter)] pub fn value(&self) -> String { - "".to_owned() + String::new() } #[wasm_bindgen(js_name = toString)] @@ -393,7 +393,7 @@ impl JsQuad { #[wasm_bindgen(getter)] pub fn value(&self) -> String { - "".to_owned() + String::new() } #[wasm_bindgen(getter = subject)] diff --git a/lib/oxrdf/src/blank_node.rs b/lib/oxrdf/src/blank_node.rs index 938461e9..5b9172c2 100644 --- a/lib/oxrdf/src/blank_node.rs +++ b/lib/oxrdf/src/blank_node.rs @@ -343,7 +343,7 @@ fn to_integer_id(id: &str) -> Option { /// An error raised during [`BlankNode`] IDs validation. #[derive(Debug)] -pub struct BlankNodeIdParseError {} +pub struct BlankNodeIdParseError; impl fmt::Display for BlankNodeIdParseError { #[inline] diff --git a/lib/oxrdf/src/interning.rs b/lib/oxrdf/src/interning.rs index 02d229e0..4b7b8705 100644 --- a/lib/oxrdf/src/interning.rs +++ b/lib/oxrdf/src/interning.rs @@ -479,7 +479,7 @@ impl InternedTriple { } #[derive(Default)] -struct IdentityHasherBuilder {} +struct IdentityHasherBuilder; impl BuildHasher for IdentityHasherBuilder { type Hasher = IdentityHasher; diff --git a/lib/oxrdf/src/variable.rs b/lib/oxrdf/src/variable.rs index e1c0d3af..af055bb9 100644 --- a/lib/oxrdf/src/variable.rs +++ b/lib/oxrdf/src/variable.rs @@ -219,7 +219,7 @@ fn validate_variable_identifier(id: &str) -> Result<(), VariableNameParseError> /// An error raised during [`Variable`] name validation. #[derive(Debug)] -pub struct VariableNameParseError {} +pub struct VariableNameParseError; impl fmt::Display for VariableNameParseError { #[inline] diff --git a/lib/src/sparql/http/dummy.rs b/lib/src/sparql/http/dummy.rs index 0517456b..3eb47e62 100644 --- a/lib/src/sparql/http/dummy.rs +++ b/lib/src/sparql/http/dummy.rs @@ -3,7 +3,7 @@ use std::io::{Empty, Error, ErrorKind, Result}; use std::time::Duration; -pub struct Client {} +pub struct Client; impl Client { pub fn new(_timeout: Option, _redirection_limit: usize) -> Self { From 7c0563cb1b8c13df845f905edf9d72a1437bd9a4 Mon Sep 17 00:00:00 2001 From: Tpt Date: Mon, 12 Jun 2023 21:02:43 +0200 Subject: [PATCH 005/217] XSD type structs: use by-value instead of by-ref in methods They are small types --- clippy.toml | 2 +- lib/oxsdatatypes/src/boolean.rs | 2 +- lib/oxsdatatypes/src/date_time.rs | 138 ++++++++++++++--------------- lib/oxsdatatypes/src/decimal.rs | 30 +++---- lib/oxsdatatypes/src/double.rs | 8 +- lib/oxsdatatypes/src/duration.rs | 46 +++++----- lib/oxsdatatypes/src/float.rs | 8 +- lib/oxsdatatypes/src/integer.rs | 22 ++--- lib/src/sparql/model.rs | 1 + lib/src/storage/numeric_encoder.rs | 30 +++---- 10 files changed, 144 insertions(+), 143 deletions(-) diff --git a/clippy.toml b/clippy.toml index 302f26e0..13d3926a 100644 --- a/clippy.toml +++ b/clippy.toml @@ -1,4 +1,4 @@ -avoid-breaking-exported-api = true +avoid-breaking-exported-api = false cognitive-complexity-threshold = 50 too-many-arguments-threshold = 10 type-complexity-threshold = 500 \ No newline at end of file diff --git a/lib/oxsdatatypes/src/boolean.rs b/lib/oxsdatatypes/src/boolean.rs index fd213a90..7ce5d977 100644 --- a/lib/oxsdatatypes/src/boolean.rs +++ b/lib/oxsdatatypes/src/boolean.rs @@ -14,7 +14,7 @@ pub struct Boolean { impl Boolean { /// Checks if the two values are [identical](https://www.w3.org/TR/xmlschema11-2/#identity). #[inline] - pub fn is_identical_with(&self, other: &Self) -> bool { + pub fn is_identical_with(self, other: Self) -> bool { self == other } } diff --git a/lib/oxsdatatypes/src/date_time.rs b/lib/oxsdatatypes/src/date_time.rs index b5763220..8d005a46 100644 --- a/lib/oxsdatatypes/src/date_time.rs +++ b/lib/oxsdatatypes/src/date_time.rs @@ -60,53 +60,53 @@ impl DateTime { /// [fn:year-from-dateTime](https://www.w3.org/TR/xpath-functions-31/#func-year-from-dateTime) #[inline] - pub fn year(&self) -> i64 { + pub fn year(self) -> i64 { self.timestamp.year() } /// [fn:month-from-dateTime](https://www.w3.org/TR/xpath-functions-31/#func-month-from-dateTime) #[inline] - pub fn month(&self) -> u8 { + pub fn month(self) -> u8 { self.timestamp.month() } /// [fn:day-from-dateTime](https://www.w3.org/TR/xpath-functions-31/#func-day-from-dateTime) #[inline] - pub fn day(&self) -> u8 { + pub fn day(self) -> u8 { self.timestamp.day() } /// [fn:hour-from-dateTime](https://www.w3.org/TR/xpath-functions-31/#func-hours-from-dateTime) #[inline] - pub fn hour(&self) -> u8 { + pub fn hour(self) -> u8 { self.timestamp.hour() } /// [fn:minute-from-dateTime](https://www.w3.org/TR/xpath-functions-31/#func-minutes-from-dateTime) #[inline] - pub fn minute(&self) -> u8 { + pub fn minute(self) -> u8 { self.timestamp.minute() } /// [fn:second-from-dateTime](https://www.w3.org/TR/xpath-functions-31/#func-seconds-from-dateTime) #[inline] - pub fn second(&self) -> Decimal { + pub fn second(self) -> Decimal { self.timestamp.second() } /// [fn:timezone-from-dateTime](https://www.w3.org/TR/xpath-functions-31/#func-timezone-from-dateTime) #[inline] - pub fn timezone(&self) -> Option { + pub fn timezone(self) -> Option { Some(self.timezone_offset()?.into()) } #[inline] - pub fn timezone_offset(&self) -> Option { + pub fn timezone_offset(self) -> Option { self.timestamp.timezone_offset() } #[inline] - fn properties(&self) -> DateTimeSevenPropertyModel { + fn properties(self) -> DateTimeSevenPropertyModel { DateTimeSevenPropertyModel { year: Some(self.year()), month: Some(self.month()), @@ -125,14 +125,14 @@ impl DateTime { /// [op:subtract-dateTimes](https://www.w3.org/TR/xpath-functions-31/#func-subtract-dateTimes) #[inline] - pub fn checked_sub(&self, rhs: impl Into) -> Option { + pub fn checked_sub(self, rhs: impl Into) -> Option { self.timestamp.checked_sub(rhs.into().timestamp) } /// [op:add-yearMonthDuration-to-dateTime](https://www.w3.org/TR/xpath-functions-31/#func-add-yearMonthDuration-to-dateTime) #[inline] pub fn checked_add_year_month_duration( - &self, + self, rhs: impl Into, ) -> Option { self.checked_add_duration(Duration::from(rhs.into())) @@ -140,7 +140,7 @@ impl DateTime { /// [op:add-dayTimeDuration-to-dateTime](https://www.w3.org/TR/xpath-functions-31/#func-add-dayTimeDuration-to-dateTime) #[inline] - pub fn checked_add_day_time_duration(&self, rhs: impl Into) -> Option { + pub fn checked_add_day_time_duration(self, rhs: impl Into) -> Option { let rhs = rhs.into(); Some(Self { timestamp: self.timestamp.checked_add_seconds(rhs.all_seconds())?, @@ -149,7 +149,7 @@ impl DateTime { /// [op:add-yearMonthDuration-to-dateTime](https://www.w3.org/TR/xpath-functions-31/#func-add-yearMonthDuration-to-dateTime) and [op:add-dayTimeDuration-to-dateTime](https://www.w3.org/TR/xpath-functions-31/#func-add-dayTimeDuration-to-dateTime) #[inline] - pub fn checked_add_duration(&self, rhs: impl Into) -> Option { + pub fn checked_add_duration(self, rhs: impl Into) -> Option { let rhs = rhs.into(); if let Ok(rhs) = DayTimeDuration::try_from(rhs) { self.checked_add_day_time_duration(rhs) @@ -164,7 +164,7 @@ impl DateTime { /// [op:subtract-yearMonthDuration-from-dateTime](https://www.w3.org/TR/xpath-functions-31/#func-subtract-yearMonthDuration-from-dateTime) #[inline] pub fn checked_sub_year_month_duration( - &self, + self, rhs: impl Into, ) -> Option { self.checked_sub_duration(Duration::from(rhs.into())) @@ -172,7 +172,7 @@ impl DateTime { /// [op:subtract-dayTimeDuration-from-dateTime](https://www.w3.org/TR/xpath-functions-31/#func-subtract-dayTimeDuration-from-dateTime) #[inline] - pub fn checked_sub_day_time_duration(&self, rhs: impl Into) -> Option { + pub fn checked_sub_day_time_duration(self, rhs: impl Into) -> Option { let rhs = rhs.into(); Some(Self { timestamp: self.timestamp.checked_sub_seconds(rhs.all_seconds())?, @@ -181,7 +181,7 @@ impl DateTime { /// [op:subtract-yearMonthDuration-from-dateTime](https://www.w3.org/TR/xpath-functions-31/#func-subtract-yearMonthDuration-from-dateTime) and [op:subtract-dayTimeDuration-from-dateTime](https://www.w3.org/TR/xpath-functions-31/#func-subtract-dayTimeDuration-from-dateTime) #[inline] - pub fn checked_sub_duration(&self, rhs: impl Into) -> Option { + pub fn checked_sub_duration(self, rhs: impl Into) -> Option { let rhs = rhs.into(); if let Ok(rhs) = DayTimeDuration::try_from(rhs) { self.checked_sub_day_time_duration(rhs) @@ -198,7 +198,7 @@ impl DateTime { /// [fn:adjust-dateTime-to-timezone](https://www.w3.org/TR/xpath-functions-31/#func-adjust-dateTime-to-timezone) #[inline] - pub fn adjust(&self, timezone_offset: Option) -> Option { + pub fn adjust(self, timezone_offset: Option) -> Option { Some(Self { timestamp: self.timestamp.adjust(timezone_offset)?, }) @@ -206,8 +206,8 @@ impl DateTime { /// Checks if the two values are [identical](https://www.w3.org/TR/xmlschema11-2/#identity). #[inline] - pub fn is_identical_with(&self, other: &Self) -> bool { - self.timestamp.is_identical_with(&other.timestamp) + pub fn is_identical_with(self, other: Self) -> bool { + self.timestamp.is_identical_with(other.timestamp) } } @@ -314,30 +314,30 @@ impl Time { /// [fn:hour-from-time](https://www.w3.org/TR/xpath-functions-31/#func-hours-from-time) #[inline] - pub fn hour(&self) -> u8 { + pub fn hour(self) -> u8 { self.timestamp.hour() } /// [fn:minute-from-time](https://www.w3.org/TR/xpath-functions-31/#func-minutes-from-time) #[inline] - pub fn minute(&self) -> u8 { + pub fn minute(self) -> u8 { self.timestamp.minute() } /// [fn:second-from-time](https://www.w3.org/TR/xpath-functions-31/#func-seconds-from-time) #[inline] - pub fn second(&self) -> Decimal { + pub fn second(self) -> Decimal { self.timestamp.second() } /// [fn:timezone-from-time](https://www.w3.org/TR/xpath-functions-31/#func-timezone-from-time) #[inline] - pub fn timezone(&self) -> Option { + pub fn timezone(self) -> Option { Some(self.timezone_offset()?.into()) } #[inline] - pub fn timezone_offset(&self) -> Option { + pub fn timezone_offset(self) -> Option { self.timestamp.timezone_offset() } @@ -348,19 +348,19 @@ impl Time { /// [op:subtract-times](https://www.w3.org/TR/xpath-functions-31/#func-subtract-times) #[inline] - pub fn checked_sub(&self, rhs: impl Into) -> Option { + pub fn checked_sub(self, rhs: impl Into) -> Option { self.timestamp.checked_sub(rhs.into().timestamp) } /// [op:add-dayTimeDuration-to-time](https://www.w3.org/TR/xpath-functions-31/#func-add-dayTimeDuration-to-time) #[inline] - pub fn checked_add_day_time_duration(&self, rhs: impl Into) -> Option { + pub fn checked_add_day_time_duration(self, rhs: impl Into) -> Option { self.checked_add_duration(Duration::from(rhs.into())) } /// [op:add-dayTimeDuration-to-time](https://www.w3.org/TR/xpath-functions-31/#func-add-dayTimeDuration-to-time) #[inline] - pub fn checked_add_duration(&self, rhs: impl Into) -> Option { + pub fn checked_add_duration(self, rhs: impl Into) -> Option { DateTime::new( 1972, 12, @@ -378,13 +378,13 @@ impl Time { /// [op:subtract-dayTimeDuration-from-time](https://www.w3.org/TR/xpath-functions-31/#func-subtract-dayTimeDuration-from-time) #[inline] - pub fn checked_sub_day_time_duration(&self, rhs: impl Into) -> Option { + pub fn checked_sub_day_time_duration(self, rhs: impl Into) -> Option { self.checked_sub_duration(Duration::from(rhs.into())) } /// [op:subtract-dayTimeDuration-from-time](https://www.w3.org/TR/xpath-functions-31/#func-subtract-dayTimeDuration-from-time) #[inline] - pub fn checked_sub_duration(&self, rhs: impl Into) -> Option { + pub fn checked_sub_duration(self, rhs: impl Into) -> Option { DateTime::new( 1972, 12, @@ -402,7 +402,7 @@ impl Time { // [fn:adjust-time-to-timezone](https://www.w3.org/TR/xpath-functions-31/#func-adjust-time-to-timezone) #[inline] - pub fn adjust(&self, timezone_offset: Option) -> Option { + pub fn adjust(self, timezone_offset: Option) -> Option { DateTime::new( 1972, 12, @@ -420,8 +420,8 @@ impl Time { /// Checks if the two values are [identical](https://www.w3.org/TR/xmlschema11-2/#identity). #[inline] - pub fn is_identical_with(&self, other: &Self) -> bool { - self.timestamp.is_identical_with(&other.timestamp) + pub fn is_identical_with(self, other: Self) -> bool { + self.timestamp.is_identical_with(other.timestamp) } } @@ -515,30 +515,30 @@ impl Date { /// [fn:year-from-date](https://www.w3.org/TR/xpath-functions-31/#func-year-from-date) #[inline] - pub fn year(&self) -> i64 { + pub fn year(self) -> i64 { self.timestamp.year() } /// [fn:month-from-date](https://www.w3.org/TR/xpath-functions-31/#func-month-from-date) #[inline] - pub fn month(&self) -> u8 { + pub fn month(self) -> u8 { self.timestamp.month() } /// [fn:day-from-date](https://www.w3.org/TR/xpath-functions-31/#func-day-from-date) #[inline] - pub fn day(&self) -> u8 { + pub fn day(self) -> u8 { self.timestamp.day() } /// [fn:timezone-from-date](https://www.w3.org/TR/xpath-functions-31/#func-timezone-from-date) #[inline] - pub fn timezone(&self) -> Option { + pub fn timezone(self) -> Option { Some(self.timezone_offset()?.into()) } #[inline] - pub fn timezone_offset(&self) -> Option { + pub fn timezone_offset(self) -> Option { self.timestamp.timezone_offset() } @@ -549,14 +549,14 @@ impl Date { /// [op:subtract-dates](https://www.w3.org/TR/xpath-functions-31/#func-subtract-dates) #[inline] - pub fn checked_sub(&self, rhs: impl Into) -> Option { + pub fn checked_sub(self, rhs: impl Into) -> Option { self.timestamp.checked_sub(rhs.into().timestamp) } /// [op:add-yearMonthDuration-to-date](https://www.w3.org/TR/xpath-functions-31/#func-add-yearMonthDuration-to-date) #[inline] pub fn checked_add_year_month_duration( - &self, + self, rhs: impl Into, ) -> Option { self.checked_add_duration(Duration::from(rhs.into())) @@ -564,14 +564,14 @@ impl Date { /// [op:add-dayTimeDuration-to-dateTime](https://www.w3.org/TR/xpath-functions-31/#func-add-dayTimeDuration-to-date) #[inline] - pub fn checked_add_day_time_duration(&self, rhs: impl Into) -> Option { + pub fn checked_add_day_time_duration(self, rhs: impl Into) -> Option { self.checked_add_duration(Duration::from(rhs.into())) } /// [op:add-yearMonthDuration-to-date](https://www.w3.org/TR/xpath-functions-31/#func-add-yearMonthDuration-to-date) and [op:add-dayTimeDuration-to-dateTime](https://www.w3.org/TR/xpath-functions-31/#func-add-dayTimeDuration-to-date) #[inline] - pub fn checked_add_duration(&self, rhs: impl Into) -> Option { - DateTime::try_from(*self) + pub fn checked_add_duration(self, rhs: impl Into) -> Option { + DateTime::try_from(self) .ok()? .checked_add_duration(rhs)? .try_into() @@ -581,7 +581,7 @@ impl Date { /// [op:subtract-yearMonthDuration-from-date](https://www.w3.org/TR/xpath-functions-31/#func-subtract-yearMonthDuration-from-date) #[inline] pub fn checked_sub_year_month_duration( - &self, + self, rhs: impl Into, ) -> Option { self.checked_sub_duration(Duration::from(rhs.into())) @@ -589,14 +589,14 @@ impl Date { /// [op:subtract-dayTimeDuration-from-date](https://www.w3.org/TR/xpath-functions-31/#func-subtract-dayTimeDuration-from-date) #[inline] - pub fn checked_sub_day_time_duration(&self, rhs: impl Into) -> Option { + pub fn checked_sub_day_time_duration(self, rhs: impl Into) -> Option { self.checked_sub_duration(Duration::from(rhs.into())) } /// [op:subtract-yearMonthDuration-from-date](https://www.w3.org/TR/xpath-functions-31/#func-subtract-yearMonthDuration-from-date) and [op:subtract-dayTimeDuration-from-date](https://www.w3.org/TR/xpath-functions-31/#func-subtract-dayTimeDuration-from-date) #[inline] - pub fn checked_sub_duration(&self, rhs: impl Into) -> Option { - DateTime::try_from(*self) + pub fn checked_sub_duration(self, rhs: impl Into) -> Option { + DateTime::try_from(self) .ok()? .checked_sub_duration(rhs)? .try_into() @@ -605,7 +605,7 @@ impl Date { // [fn:adjust-date-to-timezone](https://www.w3.org/TR/xpath-functions-31/#func-adjust-date-to-timezone) #[inline] - pub fn adjust(&self, timezone_offset: Option) -> Option { + pub fn adjust(self, timezone_offset: Option) -> Option { DateTime::new( self.year(), self.month(), @@ -623,8 +623,8 @@ impl Date { /// Checks if the two values are [identical](https://www.w3.org/TR/xmlschema11-2/#identity). #[inline] - pub fn is_identical_with(&self, other: &Self) -> bool { - self.timestamp.is_identical_with(&other.timestamp) + pub fn is_identical_with(self, other: Self) -> bool { + self.timestamp.is_identical_with(other.timestamp) } } @@ -703,27 +703,27 @@ impl GYearMonth { } #[inline] - pub fn year(&self) -> i64 { + pub fn year(self) -> i64 { self.timestamp.year() } #[inline] - pub fn month(&self) -> u8 { + pub fn month(self) -> u8 { self.timestamp.month() } #[inline] - pub fn timezone(&self) -> Option { + pub fn timezone(self) -> Option { Some(self.timezone_offset()?.into()) } #[inline] - pub fn timezone_offset(&self) -> Option { + pub fn timezone_offset(self) -> Option { self.timestamp.timezone_offset() } #[inline] - pub fn adjust(&self, timezone_offset: Option) -> Option { + pub fn adjust(self, timezone_offset: Option) -> Option { Some(Self { timestamp: self.timestamp.adjust(timezone_offset)?, }) @@ -736,8 +736,8 @@ impl GYearMonth { /// Checks if the two values are [identical](https://www.w3.org/TR/xmlschema11-2/#identity). #[inline] - pub fn is_identical_with(&self, other: &Self) -> bool { - self.timestamp.is_identical_with(&other.timestamp) + pub fn is_identical_with(self, other: Self) -> bool { + self.timestamp.is_identical_with(other.timestamp) } } @@ -824,22 +824,22 @@ impl GYear { } #[inline] - pub fn year(&self) -> i64 { + pub fn year(self) -> i64 { self.timestamp.year() } #[inline] - pub fn timezone(&self) -> Option { + pub fn timezone(self) -> Option { Some(self.timezone_offset()?.into()) } #[inline] - pub fn timezone_offset(&self) -> Option { + pub fn timezone_offset(self) -> Option { self.timestamp.timezone_offset() } #[inline] - pub fn adjust(&self, timezone_offset: Option) -> Option { + pub fn adjust(self, timezone_offset: Option) -> Option { Some(Self { timestamp: self.timestamp.adjust(timezone_offset)?, }) @@ -852,8 +852,8 @@ impl GYear { /// Checks if the two values are [identical](https://www.w3.org/TR/xmlschema11-2/#identity). #[inline] - pub fn is_identical_with(&self, other: &Self) -> bool { - self.timestamp.is_identical_with(&other.timestamp) + pub fn is_identical_with(self, other: Self) -> bool { + self.timestamp.is_identical_with(other.timestamp) } } @@ -979,8 +979,8 @@ impl GMonthDay { /// Checks if the two values are [identical](https://www.w3.org/TR/xmlschema11-2/#identity). #[inline] - pub fn is_identical_with(&self, other: &Self) -> bool { - self.timestamp.is_identical_with(&other.timestamp) + pub fn is_identical_with(self, other: Self) -> bool { + self.timestamp.is_identical_with(other.timestamp) } } @@ -1091,8 +1091,8 @@ impl GMonth { /// Checks if the two values are [identical](https://www.w3.org/TR/xmlschema11-2/#identity). #[inline] - pub fn is_identical_with(&self, other: &Self) -> bool { - self.timestamp.is_identical_with(&other.timestamp) + pub fn is_identical_with(self, other: Self) -> bool { + self.timestamp.is_identical_with(other.timestamp) } } @@ -1217,8 +1217,8 @@ impl GDay { /// Checks if the two values are [identical](https://www.w3.org/TR/xmlschema11-2/#identity). #[inline] - pub fn is_identical_with(&self, other: &Self) -> bool { - self.timestamp.is_identical_with(&other.timestamp) + pub fn is_identical_with(self, other: Self) -> bool { + self.timestamp.is_identical_with(other.timestamp) } } @@ -1650,7 +1650,7 @@ impl Timestamp { /// Checks if the two values are [identical](https://www.w3.org/TR/xmlschema11-2/#identity). #[inline] - pub fn is_identical_with(&self, other: &Self) -> bool { + pub fn is_identical_with(self, other: Self) -> bool { self.value == other.value && self.timezone_offset == other.timezone_offset } } diff --git a/lib/oxsdatatypes/src/decimal.rs b/lib/oxsdatatypes/src/decimal.rs index 19d93fed..0fd4d3db 100644 --- a/lib/oxsdatatypes/src/decimal.rs +++ b/lib/oxsdatatypes/src/decimal.rs @@ -47,7 +47,7 @@ impl Decimal { /// [op:numeric-add](https://www.w3.org/TR/xpath-functions-31/#func-numeric-add) #[inline] - pub fn checked_add(&self, rhs: impl Into) -> Option { + pub fn checked_add(self, rhs: impl Into) -> Option { Some(Self { value: self.value.checked_add(rhs.into().value)?, }) @@ -55,7 +55,7 @@ impl Decimal { /// [op:numeric-subtract](https://www.w3.org/TR/xpath-functions-31/#func-numeric-subtract) #[inline] - pub fn checked_sub(&self, rhs: impl Into) -> Option { + pub fn checked_sub(self, rhs: impl Into) -> Option { Some(Self { value: self.value.checked_sub(rhs.into().value)?, }) @@ -63,7 +63,7 @@ impl Decimal { /// [op:numeric-multiply](https://www.w3.org/TR/xpath-functions-31/#func-numeric-multiply) #[inline] - pub fn checked_mul(&self, rhs: impl Into) -> Option { + pub fn checked_mul(self, rhs: impl Into) -> Option { // Idea: we shift right as much as possible to keep as much precision as possible // Do the multiplication and do the required left shift let mut left = self.value; @@ -95,7 +95,7 @@ impl Decimal { /// [op:numeric-divide](https://www.w3.org/TR/xpath-functions-31/#func-numeric-divide) #[inline] - pub fn checked_div(&self, rhs: impl Into) -> Option { + pub fn checked_div(self, rhs: impl Into) -> Option { // Idea: we shift the dividend left as much as possible to keep as much precision as possible // And we shift right the divisor as much as possible // Do the multiplication and do the required shift @@ -128,14 +128,14 @@ impl Decimal { /// [op:numeric-mod](https://www.w3.org/TR/xpath-functions-31/#func-numeric-mod) #[inline] - pub fn checked_rem(&self, rhs: impl Into) -> Option { + pub fn checked_rem(self, rhs: impl Into) -> Option { Some(Self { value: self.value.checked_rem(rhs.into().value)?, }) } #[inline] - pub fn checked_rem_euclid(&self, rhs: impl Into) -> Option { + pub fn checked_rem_euclid(self, rhs: impl Into) -> Option { Some(Self { value: self.value.checked_rem_euclid(rhs.into().value)?, }) @@ -143,7 +143,7 @@ impl Decimal { /// [op:numeric-unary-minus](https://www.w3.org/TR/xpath-functions-31/#func-numeric-unary-minus) #[inline] - pub fn checked_neg(&self) -> Option { + pub fn checked_neg(self) -> Option { Some(Self { value: self.value.checked_neg()?, }) @@ -151,7 +151,7 @@ impl Decimal { /// [fn:abs](https://www.w3.org/TR/xpath-functions-31/#func-abs) #[inline] - pub const fn abs(&self) -> Self { + pub const fn abs(self) -> Self { Self { value: self.value.abs(), } @@ -159,7 +159,7 @@ impl Decimal { /// [fn:round](https://www.w3.org/TR/xpath-functions-31/#func-round) #[inline] - pub fn round(&self) -> Self { + pub fn round(self) -> Self { let value = self.value / DECIMAL_PART_POW_MINUS_ONE; Self { value: if value >= 0 { @@ -172,7 +172,7 @@ impl Decimal { /// [fn:ceiling](https://www.w3.org/TR/xpath-functions-31/#func-ceiling) #[inline] - pub fn ceil(&self) -> Self { + pub fn ceil(self) -> Self { Self { value: if self.value >= 0 && self.value % DECIMAL_PART_POW != 0 { (self.value / DECIMAL_PART_POW + 1) * DECIMAL_PART_POW @@ -184,7 +184,7 @@ impl Decimal { /// [fn:floor](https://www.w3.org/TR/xpath-functions-31/#func-floor) #[inline] - pub fn floor(&self) -> Self { + pub fn floor(self) -> Self { Self { value: if self.value >= 0 || self.value % DECIMAL_PART_POW == 0 { (self.value / DECIMAL_PART_POW) * DECIMAL_PART_POW @@ -195,23 +195,23 @@ impl Decimal { } #[inline] - pub const fn is_negative(&self) -> bool { + pub const fn is_negative(self) -> bool { self.value < 0 } #[inline] - pub const fn is_positive(&self) -> bool { + pub const fn is_positive(self) -> bool { self.value > 0 } /// Checks if the two values are [identical](https://www.w3.org/TR/xmlschema11-2/#identity). #[inline] - pub fn is_identical_with(&self, other: &Self) -> bool { + pub fn is_identical_with(self, other: Self) -> bool { self == other } #[inline] - pub(super) const fn as_i128(&self) -> i128 { + pub(super) const fn as_i128(self) -> i128 { self.value / DECIMAL_PART_POW } diff --git a/lib/oxsdatatypes/src/double.rs b/lib/oxsdatatypes/src/double.rs index b1465c9e..64d3c786 100644 --- a/lib/oxsdatatypes/src/double.rs +++ b/lib/oxsdatatypes/src/double.rs @@ -71,7 +71,7 @@ impl Double { /// Checks if the two values are [identical](https://www.w3.org/TR/xmlschema11-2/#identity). #[inline] - pub fn is_identical_with(&self, other: &Self) -> bool { + pub fn is_identical_with(self, other: Self) -> bool { self.value.to_ne_bytes() == other.value.to_ne_bytes() } @@ -291,9 +291,9 @@ mod tests { #[test] fn is_identical_with() { - assert!(Double::from(0.).is_identical_with(&Double::from(0.))); - assert!(Double::NAN.is_identical_with(&Double::NAN)); - assert!(!Double::from(-0.).is_identical_with(&Double::from(0.))); + assert!(Double::from(0.).is_identical_with(Double::from(0.))); + assert!(Double::NAN.is_identical_with(Double::NAN)); + assert!(!Double::from(-0.).is_identical_with(Double::from(0.))); } #[test] diff --git a/lib/oxsdatatypes/src/duration.rs b/lib/oxsdatatypes/src/duration.rs index 96faec7b..14ea6aac 100644 --- a/lib/oxsdatatypes/src/duration.rs +++ b/lib/oxsdatatypes/src/duration.rs @@ -35,47 +35,47 @@ impl Duration { /// [fn:years-from-duration](https://www.w3.org/TR/xpath-functions-31/#func-years-from-duration) #[inline] - pub fn years(&self) -> i64 { + pub fn years(self) -> i64 { self.year_month.years() } /// [fn:months-from-duration](https://www.w3.org/TR/xpath-functions-31/#func-months-from-duration) #[inline] - pub fn months(&self) -> i64 { + pub fn months(self) -> i64 { self.year_month.months() } /// [fn:days-from-duration](https://www.w3.org/TR/xpath-functions-31/#func-days-from-duration) #[inline] - pub fn days(&self) -> i64 { + pub fn days(self) -> i64 { self.day_time.days() } /// [fn:hours-from-duration](https://www.w3.org/TR/xpath-functions-31/#func-hours-from-duration) #[inline] - pub fn hours(&self) -> i64 { + pub fn hours(self) -> i64 { self.day_time.hours() } /// [fn:minutes-from-duration](https://www.w3.org/TR/xpath-functions-31/#func-minutes-from-duration) #[inline] - pub fn minutes(&self) -> i64 { + pub fn minutes(self) -> i64 { self.day_time.minutes() } /// [fn:seconds-from-duration](https://www.w3.org/TR/xpath-functions-31/#func-seconds-from-duration) #[inline] - pub fn seconds(&self) -> Decimal { + pub fn seconds(self) -> Decimal { self.day_time.seconds() } #[inline] - pub(super) const fn all_months(&self) -> i64 { + pub(super) const fn all_months(self) -> i64 { self.year_month.all_months() } #[inline] - pub(super) const fn all_seconds(&self) -> Decimal { + pub(super) const fn all_seconds(self) -> Decimal { self.day_time.all_seconds() } @@ -89,7 +89,7 @@ impl Duration { /// [op:add-yearMonthDurations](https://www.w3.org/TR/xpath-functions-31/#func-add-yearMonthDurations) and [op:add-dayTimeDurations](https://www.w3.org/TR/xpath-functions-31/#func-add-dayTimeDurations) #[inline] - pub fn checked_add(&self, rhs: impl Into) -> Option { + pub fn checked_add(self, rhs: impl Into) -> Option { let rhs = rhs.into(); Some(Self { year_month: self.year_month.checked_add(rhs.year_month)?, @@ -99,7 +99,7 @@ impl Duration { /// [op:subtract-yearMonthDurations](https://www.w3.org/TR/xpath-functions-31/#func-subtract-yearMonthDurations) and [op:subtract-dayTimeDurations](https://www.w3.org/TR/xpath-functions-31/#func-subtract-dayTimeDurations) #[inline] - pub fn checked_sub(&self, rhs: impl Into) -> Option { + pub fn checked_sub(self, rhs: impl Into) -> Option { let rhs = rhs.into(); Some(Self { year_month: self.year_month.checked_sub(rhs.year_month)?, @@ -108,7 +108,7 @@ impl Duration { } #[inline] - pub fn checked_neg(&self) -> Option { + pub fn checked_neg(self) -> Option { Some(Self { year_month: self.year_month.checked_neg()?, day_time: self.day_time.checked_neg()?, @@ -117,7 +117,7 @@ impl Duration { /// Checks if the two values are [identical](https://www.w3.org/TR/xmlschema11-2/#identity). #[inline] - pub fn is_identical_with(&self, other: &Self) -> bool { + pub fn is_identical_with(self, other: Self) -> bool { self == other } } @@ -308,7 +308,7 @@ impl YearMonthDuration { } #[inline] - pub fn checked_neg(&self) -> Option { + pub fn checked_neg(self) -> Option { Some(Self { months: self.months.checked_neg()?, }) @@ -316,7 +316,7 @@ impl YearMonthDuration { /// Checks if the two values are [identical](https://www.w3.org/TR/xmlschema11-2/#identity). #[inline] - pub fn is_identical_with(&self, other: &Self) -> bool { + pub fn is_identical_with(self, other: Self) -> bool { self == other } } @@ -428,32 +428,32 @@ impl DayTimeDuration { /// [fn:days-from-duration](https://www.w3.org/TR/xpath-functions-31/#func-days-from-duration) #[allow(clippy::cast_possible_truncation)] #[inline] - pub fn days(&self) -> i64 { + pub fn days(self) -> i64 { (self.seconds.as_i128() / 86400) as i64 } /// [fn:hours-from-duration](https://www.w3.org/TR/xpath-functions-31/#func-hours-from-duration) #[allow(clippy::cast_possible_truncation)] #[inline] - pub fn hours(&self) -> i64 { + pub fn hours(self) -> i64 { ((self.seconds.as_i128() % 86400) / 3600) as i64 } /// [fn:minutes-from-duration](https://www.w3.org/TR/xpath-functions-31/#func-minutes-from-duration) #[allow(clippy::cast_possible_truncation)] #[inline] - pub fn minutes(&self) -> i64 { + pub fn minutes(self) -> i64 { ((self.seconds.as_i128() % 3600) / 60) as i64 } /// [fn:seconds-from-duration](https://www.w3.org/TR/xpath-functions-31/#func-seconds-from-duration) #[inline] - pub fn seconds(&self) -> Decimal { + pub fn seconds(self) -> Decimal { self.seconds.checked_rem(60).unwrap() } #[inline] - pub(super) const fn all_seconds(&self) -> Decimal { + pub(super) const fn all_seconds(self) -> Decimal { self.seconds } @@ -464,7 +464,7 @@ impl DayTimeDuration { /// [op:add-dayTimeDurations](https://www.w3.org/TR/xpath-functions-31/#func-add-dayTimeDurations) #[inline] - pub fn checked_add(&self, rhs: impl Into) -> Option { + pub fn checked_add(self, rhs: impl Into) -> Option { let rhs = rhs.into(); Some(Self { seconds: self.seconds.checked_add(rhs.seconds)?, @@ -473,7 +473,7 @@ impl DayTimeDuration { /// [op:subtract-dayTimeDurations](https://www.w3.org/TR/xpath-functions-31/#func-subtract-dayTimeDurations) #[inline] - pub fn checked_sub(&self, rhs: impl Into) -> Option { + pub fn checked_sub(self, rhs: impl Into) -> Option { let rhs = rhs.into(); Some(Self { seconds: self.seconds.checked_sub(rhs.seconds)?, @@ -481,7 +481,7 @@ impl DayTimeDuration { } #[inline] - pub fn checked_neg(&self) -> Option { + pub fn checked_neg(self) -> Option { Some(Self { seconds: self.seconds.checked_neg()?, }) @@ -489,7 +489,7 @@ impl DayTimeDuration { /// Checks if the two values are [identical](https://www.w3.org/TR/xmlschema11-2/#identity). #[inline] - pub fn is_identical_with(&self, other: &Self) -> bool { + pub fn is_identical_with(self, other: Self) -> bool { self == other } } diff --git a/lib/oxsdatatypes/src/float.rs b/lib/oxsdatatypes/src/float.rs index 7f7b979b..86616a24 100644 --- a/lib/oxsdatatypes/src/float.rs +++ b/lib/oxsdatatypes/src/float.rs @@ -71,7 +71,7 @@ impl Float { /// Checks if the two values are [identical](https://www.w3.org/TR/xmlschema11-2/#identity). #[inline] - pub fn is_identical_with(&self, other: &Self) -> bool { + pub fn is_identical_with(self, other: Self) -> bool { self.value.to_ne_bytes() == other.value.to_ne_bytes() } @@ -281,9 +281,9 @@ mod tests { #[test] fn is_identical_with() { - assert!(Float::from(0.).is_identical_with(&Float::from(0.))); - assert!(Float::NAN.is_identical_with(&Float::NAN)); - assert!(!Float::from(-0.).is_identical_with(&Float::from(0.))); + assert!(Float::from(0.).is_identical_with(Float::from(0.))); + assert!(Float::NAN.is_identical_with(Float::NAN)); + assert!(!Float::from(-0.).is_identical_with(Float::from(0.))); } #[test] diff --git a/lib/oxsdatatypes/src/integer.rs b/lib/oxsdatatypes/src/integer.rs index d3b5fbc9..87d2dbd7 100644 --- a/lib/oxsdatatypes/src/integer.rs +++ b/lib/oxsdatatypes/src/integer.rs @@ -28,7 +28,7 @@ impl Integer { /// [op:numeric-add](https://www.w3.org/TR/xpath-functions-31/#func-numeric-add) #[inline] - pub fn checked_add(&self, rhs: impl Into) -> Option { + pub fn checked_add(self, rhs: impl Into) -> Option { Some(Self { value: self.value.checked_add(rhs.into().value)?, }) @@ -36,7 +36,7 @@ impl Integer { /// [op:numeric-subtract](https://www.w3.org/TR/xpath-functions-31/#func-numeric-subtract) #[inline] - pub fn checked_sub(&self, rhs: impl Into) -> Option { + pub fn checked_sub(self, rhs: impl Into) -> Option { Some(Self { value: self.value.checked_sub(rhs.into().value)?, }) @@ -44,7 +44,7 @@ impl Integer { /// [op:numeric-multiply](https://www.w3.org/TR/xpath-functions-31/#func-numeric-multiply) #[inline] - pub fn checked_mul(&self, rhs: impl Into) -> Option { + pub fn checked_mul(self, rhs: impl Into) -> Option { Some(Self { value: self.value.checked_mul(rhs.into().value)?, }) @@ -52,7 +52,7 @@ impl Integer { /// [op:numeric-divide](https://www.w3.org/TR/xpath-functions-31/#func-numeric-divide) #[inline] - pub fn checked_div(&self, rhs: impl Into) -> Option { + pub fn checked_div(self, rhs: impl Into) -> Option { Some(Self { value: self.value.checked_div(rhs.into().value)?, }) @@ -60,14 +60,14 @@ impl Integer { /// [op:numeric-mod](https://www.w3.org/TR/xpath-functions-31/#func-numeric-mod) #[inline] - pub fn checked_rem(&self, rhs: impl Into) -> Option { + pub fn checked_rem(self, rhs: impl Into) -> Option { Some(Self { value: self.value.checked_rem(rhs.into().value)?, }) } #[inline] - pub fn checked_rem_euclid(&self, rhs: impl Into) -> Option { + pub fn checked_rem_euclid(self, rhs: impl Into) -> Option { Some(Self { value: self.value.checked_rem_euclid(rhs.into().value)?, }) @@ -75,7 +75,7 @@ impl Integer { /// [op:numeric-unary-minus](https://www.w3.org/TR/xpath-functions-31/#func-numeric-unary-minus) #[inline] - pub fn checked_neg(&self) -> Option { + pub fn checked_neg(self) -> Option { Some(Self { value: self.value.checked_neg()?, }) @@ -83,25 +83,25 @@ impl Integer { /// [fn:abs](https://www.w3.org/TR/xpath-functions-31/#func-abs) #[inline] - pub const fn abs(&self) -> Self { + pub const fn abs(self) -> Self { Self { value: self.value.abs(), } } #[inline] - pub const fn is_negative(&self) -> bool { + pub const fn is_negative(self) -> bool { self.value < 0 } #[inline] - pub const fn is_positive(&self) -> bool { + pub const fn is_positive(self) -> bool { self.value > 0 } /// Checks if the two values are [identical](https://www.w3.org/TR/xmlschema11-2/#identity). #[inline] - pub fn is_identical_with(&self, other: &Self) -> bool { + pub fn is_identical_with(self, other: Self) -> bool { self == other } diff --git a/lib/src/sparql/model.rs b/lib/src/sparql/model.rs index d7c69ca7..9525a260 100644 --- a/lib/src/sparql/model.rs +++ b/lib/src/sparql/model.rs @@ -167,6 +167,7 @@ pub struct QuerySolutionIter { } impl QuerySolutionIter { + #[allow(clippy::rc_buffer)] pub fn new( variables: Rc>, iter: impl Iterator>, EvaluationError>> + 'static, diff --git a/lib/src/storage/numeric_encoder.rs b/lib/src/storage/numeric_encoder.rs index 73b00fe3..92fc1b5b 100644 --- a/lib/src/storage/numeric_encoder.rs +++ b/lib/src/storage/numeric_encoder.rs @@ -183,24 +183,24 @@ impl PartialEq for EncodedTerm { }, ) => value_id_a == value_id_b && datatype_id_a == datatype_id_b, (Self::BooleanLiteral(a), Self::BooleanLiteral(b)) => a == b, - (Self::FloatLiteral(a), Self::FloatLiteral(b)) => a.is_identical_with(b), - (Self::DoubleLiteral(a), Self::DoubleLiteral(b)) => a.is_identical_with(b), - (Self::IntegerLiteral(a), Self::IntegerLiteral(b)) => a.is_identical_with(b), - (Self::DecimalLiteral(a), Self::DecimalLiteral(b)) => a.is_identical_with(b), - (Self::DateTimeLiteral(a), Self::DateTimeLiteral(b)) => a.is_identical_with(b), - (Self::TimeLiteral(a), Self::TimeLiteral(b)) => a.is_identical_with(b), - (Self::DateLiteral(a), Self::DateLiteral(b)) => a.is_identical_with(b), - (Self::GYearMonthLiteral(a), Self::GYearMonthLiteral(b)) => a.is_identical_with(b), - (Self::GYearLiteral(a), Self::GYearLiteral(b)) => a.is_identical_with(b), - (Self::GMonthDayLiteral(a), Self::GMonthDayLiteral(b)) => a.is_identical_with(b), - (Self::GMonthLiteral(a), Self::GMonthLiteral(b)) => a.is_identical_with(b), - (Self::GDayLiteral(a), Self::GDayLiteral(b)) => a.is_identical_with(b), - (Self::DurationLiteral(a), Self::DurationLiteral(b)) => a.is_identical_with(b), + (Self::FloatLiteral(a), Self::FloatLiteral(b)) => a.is_identical_with(*b), + (Self::DoubleLiteral(a), Self::DoubleLiteral(b)) => a.is_identical_with(*b), + (Self::IntegerLiteral(a), Self::IntegerLiteral(b)) => a.is_identical_with(*b), + (Self::DecimalLiteral(a), Self::DecimalLiteral(b)) => a.is_identical_with(*b), + (Self::DateTimeLiteral(a), Self::DateTimeLiteral(b)) => a.is_identical_with(*b), + (Self::TimeLiteral(a), Self::TimeLiteral(b)) => a.is_identical_with(*b), + (Self::DateLiteral(a), Self::DateLiteral(b)) => a.is_identical_with(*b), + (Self::GYearMonthLiteral(a), Self::GYearMonthLiteral(b)) => a.is_identical_with(*b), + (Self::GYearLiteral(a), Self::GYearLiteral(b)) => a.is_identical_with(*b), + (Self::GMonthDayLiteral(a), Self::GMonthDayLiteral(b)) => a.is_identical_with(*b), + (Self::GMonthLiteral(a), Self::GMonthLiteral(b)) => a.is_identical_with(*b), + (Self::GDayLiteral(a), Self::GDayLiteral(b)) => a.is_identical_with(*b), + (Self::DurationLiteral(a), Self::DurationLiteral(b)) => a.is_identical_with(*b), (Self::YearMonthDurationLiteral(a), Self::YearMonthDurationLiteral(b)) => { - a.is_identical_with(b) + a.is_identical_with(*b) } (Self::DayTimeDurationLiteral(a), Self::DayTimeDurationLiteral(b)) => { - a.is_identical_with(b) + a.is_identical_with(*b) } (Self::Triple(a), Self::Triple(b)) => a == b, (_, _) => false, From 40b10cdabc3e2322efeb6838de26a48aa0dd25a9 Mon Sep 17 00:00:00 2001 From: Tpt Date: Sun, 16 Apr 2023 09:46:41 +0200 Subject: [PATCH 006/217] Adds a naive standalone query optimizer This drops some left join optimizations --- .github/workflows/tests.yml | 6 +- Cargo.lock | 12 + Cargo.toml | 1 + lib/Cargo.toml | 1 + lib/spargebra/src/term.rs | 24 + lib/sparopt/Cargo.toml | 28 + lib/sparopt/README.md | 33 + lib/sparopt/src/algebra.rs | 1712 +++++++++++++++++ lib/sparopt/src/lib.rs | 5 + lib/sparopt/src/optimizer.rs | 1022 ++++++++++ lib/sparopt/src/type_inference.rs | 451 +++++ lib/src/sparql/eval.rs | 147 +- lib/src/sparql/mod.rs | 1 - lib/src/sparql/plan.rs | 26 +- lib/src/sparql/plan_builder.rs | 1229 +++--------- testsuite/Cargo.toml | 2 + .../bgp_join_reordering_input.rq | 9 + .../bgp_join_reordering_output.rq | 10 + .../bind_always_false_input.rq | 4 + .../bind_always_false_output.rq | 3 + .../bind_always_true_input.rq | 4 + .../bind_always_true_output.rq | 3 + .../sparql-optimization/empty_union_input.rq | 3 + .../sparql-optimization/empty_union_output.rq | 3 + .../equal_to_same_term_input.rq | 5 + .../equal_to_same_term_output.rq | 5 + .../exists_always_false_input.rq | 4 + .../exists_always_false_output.rq | 3 + .../false_and_something_input.rq | 4 + .../false_and_something_output.rq | 3 + .../false_or_something_input.rq | 4 + .../false_or_something_output.rq | 4 + .../if_always_false_input.rq | 4 + .../if_always_false_output.rq | 4 + .../if_always_true_input.rq | 4 + .../if_always_true_output.rq | 4 + .../sparql-optimization/manifest.ttl | 132 ++ .../sparql-optimization/push_filter_input.rq | 11 + .../sparql-optimization/push_filter_output.rq | 18 + .../push_optional_filter_input.rq | 5 + .../push_optional_filter_output.rq | 5 + .../something_and_false_input.rq | 4 + .../something_and_false_output.rq | 3 + .../something_and_true_input.rq | 4 + .../something_and_true_output.rq | 4 + .../something_or_false_input.rq | 4 + .../something_or_false_output.rq | 4 + .../something_or_true_input.rq | 4 + .../something_or_true_output.rq | 3 + .../true_and_something_input.rq | 4 + .../true_and_something_output.rq | 4 + .../true_or_something_input.rq | 4 + .../true_or_something_output.rq | 3 + .../sparql-optimization/unbound_bind_input.rq | 3 + .../unbound_bind_output.rq | 3 + .../unbound_filter_input.rq | 4 + .../unbound_filter_output.rq | 3 + testsuite/src/sparql_evaluator.rs | 56 + testsuite/tests/oxigraph.rs | 17 +- testsuite/tests/sparql.rs | 1 + 60 files changed, 4029 insertions(+), 1061 deletions(-) create mode 100644 lib/sparopt/Cargo.toml create mode 100644 lib/sparopt/README.md create mode 100644 lib/sparopt/src/algebra.rs create mode 100644 lib/sparopt/src/lib.rs create mode 100644 lib/sparopt/src/optimizer.rs create mode 100644 lib/sparopt/src/type_inference.rs create mode 100644 testsuite/oxigraph-tests/sparql-optimization/bgp_join_reordering_input.rq create mode 100644 testsuite/oxigraph-tests/sparql-optimization/bgp_join_reordering_output.rq create mode 100644 testsuite/oxigraph-tests/sparql-optimization/bind_always_false_input.rq create mode 100644 testsuite/oxigraph-tests/sparql-optimization/bind_always_false_output.rq create mode 100644 testsuite/oxigraph-tests/sparql-optimization/bind_always_true_input.rq create mode 100644 testsuite/oxigraph-tests/sparql-optimization/bind_always_true_output.rq create mode 100644 testsuite/oxigraph-tests/sparql-optimization/empty_union_input.rq create mode 100644 testsuite/oxigraph-tests/sparql-optimization/empty_union_output.rq create mode 100644 testsuite/oxigraph-tests/sparql-optimization/equal_to_same_term_input.rq create mode 100644 testsuite/oxigraph-tests/sparql-optimization/equal_to_same_term_output.rq create mode 100644 testsuite/oxigraph-tests/sparql-optimization/exists_always_false_input.rq create mode 100644 testsuite/oxigraph-tests/sparql-optimization/exists_always_false_output.rq create mode 100644 testsuite/oxigraph-tests/sparql-optimization/false_and_something_input.rq create mode 100644 testsuite/oxigraph-tests/sparql-optimization/false_and_something_output.rq create mode 100644 testsuite/oxigraph-tests/sparql-optimization/false_or_something_input.rq create mode 100644 testsuite/oxigraph-tests/sparql-optimization/false_or_something_output.rq create mode 100644 testsuite/oxigraph-tests/sparql-optimization/if_always_false_input.rq create mode 100644 testsuite/oxigraph-tests/sparql-optimization/if_always_false_output.rq create mode 100644 testsuite/oxigraph-tests/sparql-optimization/if_always_true_input.rq create mode 100644 testsuite/oxigraph-tests/sparql-optimization/if_always_true_output.rq create mode 100644 testsuite/oxigraph-tests/sparql-optimization/manifest.ttl create mode 100644 testsuite/oxigraph-tests/sparql-optimization/push_filter_input.rq create mode 100644 testsuite/oxigraph-tests/sparql-optimization/push_filter_output.rq create mode 100644 testsuite/oxigraph-tests/sparql-optimization/push_optional_filter_input.rq create mode 100644 testsuite/oxigraph-tests/sparql-optimization/push_optional_filter_output.rq create mode 100644 testsuite/oxigraph-tests/sparql-optimization/something_and_false_input.rq create mode 100644 testsuite/oxigraph-tests/sparql-optimization/something_and_false_output.rq create mode 100644 testsuite/oxigraph-tests/sparql-optimization/something_and_true_input.rq create mode 100644 testsuite/oxigraph-tests/sparql-optimization/something_and_true_output.rq create mode 100644 testsuite/oxigraph-tests/sparql-optimization/something_or_false_input.rq create mode 100644 testsuite/oxigraph-tests/sparql-optimization/something_or_false_output.rq create mode 100644 testsuite/oxigraph-tests/sparql-optimization/something_or_true_input.rq create mode 100644 testsuite/oxigraph-tests/sparql-optimization/something_or_true_output.rq create mode 100644 testsuite/oxigraph-tests/sparql-optimization/true_and_something_input.rq create mode 100644 testsuite/oxigraph-tests/sparql-optimization/true_and_something_output.rq create mode 100644 testsuite/oxigraph-tests/sparql-optimization/true_or_something_input.rq create mode 100644 testsuite/oxigraph-tests/sparql-optimization/true_or_something_output.rq create mode 100644 testsuite/oxigraph-tests/sparql-optimization/unbound_bind_input.rq create mode 100644 testsuite/oxigraph-tests/sparql-optimization/unbound_bind_output.rq create mode 100644 testsuite/oxigraph-tests/sparql-optimization/unbound_filter_input.rq create mode 100644 testsuite/oxigraph-tests/sparql-optimization/unbound_filter_output.rq diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 65b9102a..5add21ce 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -36,6 +36,8 @@ jobs: working-directory: ./lib/sparesults - run: cargo clippy working-directory: ./lib/spargebra + - run: cargo clippy + working-directory: ./lib/sparopt - run: cargo clippy --all-targets --all-features clippy_wasm_js: @@ -76,6 +78,8 @@ jobs: working-directory: ./lib/sparesults - run: cargo clippy -- -D warnings -D clippy::all working-directory: ./lib/spargebra + - run: cargo clippy -- -D warnings -D clippy::all + working-directory: ./lib/sparopt - run: cargo clippy --all-targets -- -D warnings -D clippy::all working-directory: ./server @@ -119,7 +123,7 @@ jobs: - run: rustup update - uses: Swatinem/rust-cache@v2 - run: cargo install cargo-semver-checks || true - - run: cargo semver-checks check-release --exclude oxrocksdb-sys --exclude oxigraph_js --exclude pyoxigraph --exclude oxigraph_testsuite --exclude oxigraph_server + - run: cargo semver-checks check-release --exclude oxrocksdb-sys --exclude oxigraph_js --exclude pyoxigraph --exclude oxigraph_testsuite --exclude oxigraph_server --exclude sparopt test_linux: runs-on: ubuntu-latest diff --git a/Cargo.lock b/Cargo.lock index 29d3744b..649bd0aa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -958,6 +958,7 @@ dependencies = [ "siphasher", "sparesults", "spargebra", + "sparopt", "zstd", ] @@ -998,6 +999,8 @@ dependencies = [ "anyhow", "clap", "oxigraph", + "spargebra", + "sparopt", "text-diff", "time", ] @@ -1613,6 +1616,15 @@ dependencies = [ "rand", ] +[[package]] +name = "sparopt" +version = "0.1.0-alpha.1-dev" +dependencies = [ + "oxrdf", + "rand", + "spargebra", +] + [[package]] name = "sparql-smith" version = "0.1.0-alpha.5-dev" diff --git a/Cargo.toml b/Cargo.toml index 75a171d3..041afb39 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ members = [ "lib/oxsdatatypes", "lib/spargebra", "lib/sparesults", + "lib/sparopt", "lib/sparql-smith", "oxrocksdb-sys", "python", diff --git a/lib/Cargo.toml b/lib/Cargo.toml index d67b1349..100be488 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -41,6 +41,7 @@ json-event-parser = "0.1" oxrdf = { version = "0.2.0-alpha.1-dev", path="oxrdf", features = ["rdf-star", "oxsdatatypes"] } oxsdatatypes = { version = "0.2.0-alpha.1-dev", path="oxsdatatypes" } spargebra = { version = "0.3.0-alpha.1-dev", path="spargebra", features = ["rdf-star", "sep-0002", "sep-0006"] } +sparopt = { version = "0.1.0-alpha.1-dev", path="sparopt", features = ["rdf-star", "sep-0002", "sep-0006"] } sparesults = { version = "0.2.0-alpha.1-dev", path="sparesults", features = ["rdf-star"] } [target.'cfg(not(target_family = "wasm"))'.dependencies] diff --git a/lib/spargebra/src/term.rs b/lib/spargebra/src/term.rs index 3ef91f68..8c41fc1b 100644 --- a/lib/spargebra/src/term.rs +++ b/lib/spargebra/src/term.rs @@ -541,6 +541,19 @@ impl From for TermPattern { } } +impl From for TermPattern { + #[inline] + fn from(element: GroundTermPattern) -> Self { + match element { + GroundTermPattern::NamedNode(node) => node.into(), + GroundTermPattern::Literal(literal) => literal.into(), + #[cfg(feature = "rdf-star")] + GroundTermPattern::Triple(t) => TriplePattern::from(*t).into(), + GroundTermPattern::Variable(variable) => variable.into(), + } + } +} + impl TryFrom for Subject { type Error = (); @@ -799,6 +812,17 @@ impl From for TriplePattern { } } +impl From for TriplePattern { + #[inline] + fn from(triple: GroundTriplePattern) -> Self { + Self { + subject: triple.subject.into(), + predicate: triple.predicate, + object: triple.object.into(), + } + } +} + impl TryFrom for Triple { type Error = (); diff --git a/lib/sparopt/Cargo.toml b/lib/sparopt/Cargo.toml new file mode 100644 index 00000000..3e30b5ac --- /dev/null +++ b/lib/sparopt/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "sparopt" +version = "0.1.0-alpha.1-dev" +authors = ["Tpt "] +license = "MIT OR Apache-2.0" +readme = "README.md" +keywords = ["SPARQL"] +repository = "https://github.com/oxigraph/oxigraph/tree/main/lib/sparopt" +homepage = "https://oxigraph.org/" +description = """ +A SPARQL optimizer +""" +edition = "2021" +rust-version = "1.60" + +[features] +default = [] +rdf-star = ["oxrdf/rdf-star", "spargebra/rdf-star"] +sep-0002 = ["spargebra/sep-0002"] +sep-0006 = ["spargebra/sep-0006"] + +[dependencies] +oxrdf = { version = "0.2.0-alpha.1-dev", path="../oxrdf" } +rand = "0.8" +spargebra = { version = "0.3.0-alpha.1-dev", path="../spargebra" } + +[package.metadata.docs.rs] +all-features = true diff --git a/lib/sparopt/README.md b/lib/sparopt/README.md new file mode 100644 index 00000000..7a0ac67b --- /dev/null +++ b/lib/sparopt/README.md @@ -0,0 +1,33 @@ +sparopt +======= + +[![Latest Version](https://img.shields.io/crates/v/sparopt.svg)](https://crates.io/crates/sparopt) +[![Released API docs](https://docs.rs/sparopt/badge.svg)](https://docs.rs/sparopt) +[![Crates.io downloads](https://img.shields.io/crates/d/sparopt)](https://crates.io/crates/sparopt) +[![actions status](https://github.com/oxigraph/oxigraph/workflows/build/badge.svg)](https://github.com/oxigraph/oxigraph/actions) +[![Gitter](https://badges.gitter.im/oxigraph/community.svg)](https://gitter.im/oxigraph/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) + +sparopt is a work in progress [SPARQL Query](https://www.w3.org/TR/sparql11-query/) optimizer. + +It relies on the output of [spargebra](https://crates.io/crates/spargebra). + +Support for [SPARQL-star](https://w3c.github.io/rdf-star/cg-spec/2021-12-17.html#sparql-star) is also available behind the `rdf-star` feature. + +This crate is intended to be a building piece for SPARQL implementations in Rust like [Oxigraph](https://oxigraph.org). + + +## License + +This project is licensed under either of + +* Apache License, Version 2.0, ([LICENSE-APACHE](../LICENSE-APACHE) or + ``) +* MIT license ([LICENSE-MIT](../LICENSE-MIT) or + ``) + +at your option. + + +### Contribution + +Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in Oxigraph by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. diff --git a/lib/sparopt/src/algebra.rs b/lib/sparopt/src/algebra.rs new file mode 100644 index 00000000..65c4c618 --- /dev/null +++ b/lib/sparopt/src/algebra.rs @@ -0,0 +1,1712 @@ +//! [SPARQL 1.1 Query Algebra](https://www.w3.org/TR/sparql11-query/#sparqlQuery) representation. + +use oxrdf::vocab::xsd; +use rand::random; +use spargebra::algebra::{ + AggregateExpression as AlAggregateExpression, Expression as AlExpression, + GraphPattern as AlGraphPattern, OrderExpression as AlOrderExpression, +}; +pub use spargebra::algebra::{Function, PropertyPathExpression}; +use spargebra::term::{BlankNode, GroundSubject, TermPattern, TriplePattern}; +pub use spargebra::term::{ + GroundTerm, GroundTermPattern, Literal, NamedNode, NamedNodePattern, Variable, +}; +#[cfg(feature = "rdf-star")] +use spargebra::term::{GroundTriple, GroundTriplePattern}; +use std::collections::hash_map::DefaultHasher; +use std::collections::{HashMap, HashSet}; +use std::hash::{Hash, Hasher}; +use std::ops::{Add, BitAnd, BitOr, Div, Mul, Neg, Not, Sub}; + +/// An [expression](https://www.w3.org/TR/sparql11-query/#expressions). +#[derive(Eq, PartialEq, Debug, Clone, Hash)] +pub enum Expression { + NamedNode(NamedNode), + Literal(Literal), + Variable(Variable), + /// [Logical-or](https://www.w3.org/TR/sparql11-query/#func-logical-or). + Or(Vec), + /// [Logical-and](https://www.w3.org/TR/sparql11-query/#func-logical-and). + And(Vec), + /// [RDFterm-equal](https://www.w3.org/TR/sparql11-query/#func-RDFterm-equal) and all the XSD equalities. + Equal(Box, Box), + /// [sameTerm](https://www.w3.org/TR/sparql11-query/#func-sameTerm). + SameTerm(Box, Box), + /// [op:numeric-greater-than](https://www.w3.org/TR/xpath-functions-31/#func-numeric-greater-than) and other XSD greater than operators. + Greater(Box, Box), + GreaterOrEqual(Box, Box), + /// [op:numeric-less-than](https://www.w3.org/TR/xpath-functions-31/#func-numeric-less-than) and other XSD greater than operators. + Less(Box, Box), + LessOrEqual(Box, Box), + /// [op:numeric-add](https://www.w3.org/TR/xpath-functions-31/#func-numeric-add) and other XSD additions. + Add(Box, Box), + /// [op:numeric-subtract](https://www.w3.org/TR/xpath-functions-31/#func-numeric-subtract) and other XSD subtractions. + Subtract(Box, Box), + /// [op:numeric-multiply](https://www.w3.org/TR/xpath-functions-31/#func-numeric-multiply) and other XSD multiplications. + Multiply(Box, Box), + /// [op:numeric-divide](https://www.w3.org/TR/xpath-functions-31/#func-numeric-divide) and other XSD divides. + Divide(Box, Box), + /// [op:numeric-unary-plus](https://www.w3.org/TR/xpath-functions-31/#func-numeric-unary-plus) and other XSD unary plus. + UnaryPlus(Box), + /// [op:numeric-unary-minus](https://www.w3.org/TR/xpath-functions-31/#func-numeric-unary-minus) and other XSD unary minus. + UnaryMinus(Box), + /// [fn:not](https://www.w3.org/TR/xpath-functions-31/#func-not). + Not(Box), + /// [EXISTS](https://www.w3.org/TR/sparql11-query/#func-filter-exists). + Exists(Box), + /// [BOUND](https://www.w3.org/TR/sparql11-query/#func-bound). + Bound(Variable), + /// [IF](https://www.w3.org/TR/sparql11-query/#func-if). + If(Box, Box, Box), + /// [COALESCE](https://www.w3.org/TR/sparql11-query/#func-coalesce). + Coalesce(Vec), + /// A regular function call. + FunctionCall(Function, Vec), +} + +impl Expression { + pub fn or_all(args: impl IntoIterator) -> Self { + let args = args.into_iter(); + let mut all = Vec::with_capacity(args.size_hint().0); + for arg in args { + if let Some(ebv) = arg.effective_boolean_value() { + if ebv { + return true.into(); + } + // We ignore false values + } else if let Self::Or(args) = arg { + all.extend(args); + } else { + all.push(arg); + } + } + match all.len() { + 0 => false.into(), + 1 => { + let result = all.pop().unwrap(); + if result.returns_boolean() { + result // It's already casted to boolean + } else { + Self::And(vec![result]) + } + } + _ => Self::Or(order_vec(all)), + } + } + + pub fn and_all(args: impl IntoIterator) -> Self { + let args = args.into_iter(); + let mut all = Vec::with_capacity(args.size_hint().0); + for arg in args { + if let Some(ebv) = arg.effective_boolean_value() { + if !ebv { + return false.into(); + } + // We ignore true values + } else if let Self::And(args) = arg { + all.extend(args); + } else { + all.push(arg); + } + } + match all.len() { + 0 => true.into(), + 1 => { + let result = all.pop().unwrap(); + if result.returns_boolean() { + result + } else { + Self::And(vec![result]) + } + } + _ => Self::And(order_vec(all)), + } + } + + pub fn equal(left: Self, right: Self) -> Self { + match (left, right) { + (Self::NamedNode(left), Self::NamedNode(right)) => (left == right).into(), + (Self::Literal(left), Self::Literal(right)) if left == right => true.into(), + (left, right) => { + let (left, right) = order_pair(left, right); + Self::Equal(Box::new(left), Box::new(right)) + } + } + } + + pub fn same_term(left: Self, right: Self) -> Self { + match (left, right) { + (Self::NamedNode(left), Self::NamedNode(right)) => (left == right).into(), + (Self::Literal(left), Self::Literal(right)) if left == right => true.into(), + (left, right) => { + let (left, right) = order_pair(left, right); + Self::SameTerm(Box::new(left), Box::new(right)) + } + } + } + + pub fn greater(left: Self, right: Self) -> Self { + Self::Greater(Box::new(left), Box::new(right)) + } + + pub fn greater_or_equal(left: Self, right: Self) -> Self { + Self::GreaterOrEqual(Box::new(left), Box::new(right)) + } + + pub fn less(left: Self, right: Self) -> Self { + Self::Less(Box::new(left), Box::new(right)) + } + + pub fn less_or_equal(left: Self, right: Self) -> Self { + Self::LessOrEqual(Box::new(left), Box::new(right)) + } + + pub fn unary_plus(inner: Self) -> Self { + Self::UnaryPlus(Box::new(inner)) + } + + pub fn exists(inner: GraphPattern) -> Self { + if inner.is_empty() { + return false.into(); + } + if inner.is_empty_singleton() { + return true.into(); + } + Self::Exists(Box::new(inner)) + } + + pub fn if_cond(cond: Self, then: Self, els: Self) -> Self { + match cond.effective_boolean_value() { + Some(true) => then, + Some(false) => els, + None => Self::If(Box::new(cond), Box::new(then), Box::new(els)), + } + } + + pub fn coalesce(args: Vec) -> Self { + Self::Coalesce(args) + } + + pub fn call(name: Function, args: Vec) -> Self { + Self::FunctionCall(name, args) + } + + pub fn effective_boolean_value(&self) -> Option { + if let Self::Literal(literal) = self { + match literal.datatype() { + xsd::BOOLEAN => match literal.value() { + "true" | "1" => Some(true), + "false" | "0" => Some(false), + _ => None, //TODO + }, + xsd::STRING => Some(!literal.value().is_empty()), + _ => None, //TODO + } + } else { + None + } + } + + pub fn used_variables(&self) -> HashSet<&Variable> { + let mut variables = HashSet::new(); + self.lookup_used_variables(&mut |v| { + variables.insert(v); + }); + variables + } + + pub fn lookup_used_variables<'a>(&'a self, callback: &mut impl FnMut(&'a Variable)) { + match self { + Self::NamedNode(_) | Self::Literal(_) => {} + Self::Variable(v) | Self::Bound(v) => callback(v), + Self::Or(inner) + | Self::And(inner) + | Self::Coalesce(inner) + | Self::FunctionCall(_, inner) => { + for i in inner { + i.lookup_used_variables(callback); + } + } + Self::Equal(a, b) + | Self::SameTerm(a, b) + | Self::Greater(a, b) + | Self::GreaterOrEqual(a, b) + | Self::Less(a, b) + | Self::LessOrEqual(a, b) + | Self::Add(a, b) + | Self::Subtract(a, b) + | Self::Multiply(a, b) + | Self::Divide(a, b) => { + a.lookup_used_variables(callback); + b.lookup_used_variables(callback); + } + Self::UnaryPlus(i) | Self::UnaryMinus(i) | Self::Not(i) => { + i.lookup_used_variables(callback) + } + Self::Exists(e) => e.lookup_used_variables(callback), + Self::If(a, b, c) => { + a.lookup_used_variables(callback); + b.lookup_used_variables(callback); + c.lookup_used_variables(callback); + } + } + } + + fn from_sparql_algebra( + expression: &AlExpression, + graph_name: Option<&NamedNodePattern>, + ) -> Self { + match expression { + AlExpression::NamedNode(node) => Self::NamedNode(node.clone()), + AlExpression::Literal(literal) => Self::Literal(literal.clone()), + AlExpression::Variable(variable) => Self::Variable(variable.clone()), + AlExpression::Or(left, right) => Self::Or(vec![ + Self::from_sparql_algebra(left, graph_name), + Self::from_sparql_algebra(right, graph_name), + ]), + AlExpression::And(left, right) => Self::And(vec![ + Self::from_sparql_algebra(left, graph_name), + Self::from_sparql_algebra(right, graph_name), + ]), + AlExpression::Equal(left, right) => Self::Equal( + Box::new(Self::from_sparql_algebra(left, graph_name)), + Box::new(Self::from_sparql_algebra(right, graph_name)), + ), + AlExpression::SameTerm(left, right) => Self::SameTerm( + Box::new(Self::from_sparql_algebra(left, graph_name)), + Box::new(Self::from_sparql_algebra(right, graph_name)), + ), + AlExpression::Greater(left, right) => Self::Greater( + Box::new(Self::from_sparql_algebra(left, graph_name)), + Box::new(Self::from_sparql_algebra(right, graph_name)), + ), + AlExpression::GreaterOrEqual(left, right) => Self::GreaterOrEqual( + Box::new(Self::from_sparql_algebra(left, graph_name)), + Box::new(Self::from_sparql_algebra(right, graph_name)), + ), + AlExpression::Less(left, right) => Self::Less( + Box::new(Self::from_sparql_algebra(left, graph_name)), + Box::new(Self::from_sparql_algebra(right, graph_name)), + ), + AlExpression::LessOrEqual(left, right) => Self::LessOrEqual( + Box::new(Self::from_sparql_algebra(left, graph_name)), + Box::new(Self::from_sparql_algebra(right, graph_name)), + ), + AlExpression::In(left, right) => { + let left = Self::from_sparql_algebra(left, graph_name); + match right.len() { + 0 => Self::if_cond(left, false.into(), false.into()), + 1 => Self::Equal( + Box::new(left), + Box::new(Self::from_sparql_algebra(&right[0], graph_name)), + ), + _ => Self::Or( + right + .iter() + .map(|e| { + Self::Equal( + Box::new(left.clone()), + Box::new(Self::from_sparql_algebra(e, graph_name)), + ) + }) + .collect(), + ), + } + } + AlExpression::Add(left, right) => Self::Add( + Box::new(Self::from_sparql_algebra(left, graph_name)), + Box::new(Self::from_sparql_algebra(right, graph_name)), + ), + AlExpression::Subtract(left, right) => Self::Subtract( + Box::new(Self::from_sparql_algebra(left, graph_name)), + Box::new(Self::from_sparql_algebra(right, graph_name)), + ), + AlExpression::Multiply(left, right) => Self::Multiply( + Box::new(Self::from_sparql_algebra(left, graph_name)), + Box::new(Self::from_sparql_algebra(right, graph_name)), + ), + AlExpression::Divide(left, right) => Self::Divide( + Box::new(Self::from_sparql_algebra(left, graph_name)), + Box::new(Self::from_sparql_algebra(right, graph_name)), + ), + AlExpression::UnaryPlus(inner) => { + Self::UnaryPlus(Box::new(Self::from_sparql_algebra(inner, graph_name))) + } + AlExpression::UnaryMinus(inner) => { + Self::UnaryMinus(Box::new(Self::from_sparql_algebra(inner, graph_name))) + } + AlExpression::Not(inner) => { + Self::Not(Box::new(Self::from_sparql_algebra(inner, graph_name))) + } + AlExpression::Exists(inner) => Self::Exists(Box::new( + GraphPattern::from_sparql_algebra(inner, graph_name, &mut HashMap::new()), + )), + AlExpression::Bound(variable) => Self::Bound(variable.clone()), + AlExpression::If(cond, yes, no) => Self::If( + Box::new(Self::from_sparql_algebra(cond, graph_name)), + Box::new(Self::from_sparql_algebra(yes, graph_name)), + Box::new(Self::from_sparql_algebra(no, graph_name)), + ), + AlExpression::Coalesce(inner) => Self::Coalesce( + inner + .iter() + .map(|e| Self::from_sparql_algebra(e, graph_name)) + .collect(), + ), + AlExpression::FunctionCall(name, args) => Self::FunctionCall( + name.clone(), + args.iter() + .map(|e| Self::from_sparql_algebra(e, graph_name)) + .collect(), + ), + } + } + + fn returns_boolean(&self) -> bool { + match self { + Expression::Or(_) + | Expression::And(_) + | Expression::Equal(_, _) + | Expression::SameTerm(_, _) + | Expression::Greater(_, _) + | Expression::GreaterOrEqual(_, _) + | Expression::Less(_, _) + | Expression::LessOrEqual(_, _) + | Expression::Not(_) + | Expression::Exists(_) + | Expression::Bound(_) + | Expression::FunctionCall( + Function::IsBlank | Function::IsIri | Function::IsLiteral | Function::IsNumeric, + _, + ) => true, + #[cfg(feature = "rdf-star")] + Expression::FunctionCall(Function::IsTriple, _) => true, + Expression::Literal(literal) => literal.datatype() == xsd::BOOLEAN, + Expression::If(_, a, b) => a.returns_boolean() && b.returns_boolean(), + _ => false, + } + } +} + +impl From for Expression { + fn from(value: NamedNode) -> Self { + Self::NamedNode(value) + } +} + +impl From for Expression { + fn from(value: Literal) -> Self { + Self::Literal(value) + } +} + +impl From for Expression { + fn from(value: GroundSubject) -> Self { + match value { + GroundSubject::NamedNode(value) => value.into(), + #[cfg(feature = "rdf-star")] + GroundSubject::Triple(value) => (*value).into(), + } + } +} + +impl From for Expression { + fn from(value: GroundTerm) -> Self { + match value { + GroundTerm::NamedNode(value) => value.into(), + GroundTerm::Literal(value) => value.into(), + #[cfg(feature = "rdf-star")] + GroundTerm::Triple(value) => (*value).into(), + } + } +} + +impl From for Expression { + fn from(value: NamedNodePattern) -> Self { + match value { + NamedNodePattern::NamedNode(value) => value.into(), + NamedNodePattern::Variable(variable) => variable.into(), + } + } +} + +impl From for Expression { + fn from(value: GroundTermPattern) -> Self { + match value { + GroundTermPattern::NamedNode(value) => value.into(), + GroundTermPattern::Literal(value) => value.into(), + #[cfg(feature = "rdf-star")] + GroundTermPattern::Triple(value) => (*value).into(), + GroundTermPattern::Variable(variable) => variable.into(), + } + } +} + +#[cfg(feature = "rdf-star")] +impl From for Expression { + fn from(value: GroundTriple) -> Self { + Self::FunctionCall( + Function::Triple, + vec![ + value.subject.into(), + value.predicate.into(), + value.object.into(), + ], + ) + } +} + +#[cfg(feature = "rdf-star")] +impl From for Expression { + fn from(value: GroundTriplePattern) -> Self { + Self::FunctionCall( + Function::Triple, + vec![ + value.subject.into(), + value.predicate.into(), + value.object.into(), + ], + ) + } +} + +impl From for Expression { + fn from(value: Variable) -> Self { + Self::Variable(value) + } +} + +impl From for Expression { + fn from(value: bool) -> Self { + Literal::from(value).into() + } +} + +impl From<&Expression> for AlExpression { + fn from(expression: &Expression) -> Self { + match expression { + Expression::NamedNode(node) => Self::NamedNode(node.clone()), + Expression::Literal(literal) => Self::Literal(literal.clone()), + Expression::Variable(variable) => Self::Variable(variable.clone()), + Expression::Or(inner) => inner + .iter() + .map(Into::into) + .reduce(|a, b| Self::Or(Box::new(a), Box::new(b))) + .unwrap_or_else(|| Literal::from(false).into()), + Expression::And(inner) => inner + .iter() + .map(Into::into) + .reduce(|a, b| Self::And(Box::new(a), Box::new(b))) + .unwrap_or_else(|| Literal::from(true).into()), + Expression::Equal(left, right) => Self::Equal( + Box::new(left.as_ref().into()), + Box::new(right.as_ref().into()), + ), + Expression::SameTerm(left, right) => Self::SameTerm( + Box::new(left.as_ref().into()), + Box::new(right.as_ref().into()), + ), + Expression::Greater(left, right) => Self::Greater( + Box::new(left.as_ref().into()), + Box::new(right.as_ref().into()), + ), + Expression::GreaterOrEqual(left, right) => Self::GreaterOrEqual( + Box::new(left.as_ref().into()), + Box::new(right.as_ref().into()), + ), + Expression::Less(left, right) => Self::Less( + Box::new(left.as_ref().into()), + Box::new(right.as_ref().into()), + ), + Expression::LessOrEqual(left, right) => Self::LessOrEqual( + Box::new(left.as_ref().into()), + Box::new(right.as_ref().into()), + ), + Expression::Add(left, right) => Self::Add( + Box::new(left.as_ref().into()), + Box::new(right.as_ref().into()), + ), + Expression::Subtract(left, right) => Self::Subtract( + Box::new(left.as_ref().into()), + Box::new(right.as_ref().into()), + ), + Expression::Multiply(left, right) => Self::Multiply( + Box::new(left.as_ref().into()), + Box::new(right.as_ref().into()), + ), + Expression::Divide(left, right) => Self::Divide( + Box::new(left.as_ref().into()), + Box::new(right.as_ref().into()), + ), + Expression::UnaryPlus(inner) => Self::UnaryPlus(Box::new(inner.as_ref().into())), + Expression::UnaryMinus(inner) => Self::UnaryMinus(Box::new(inner.as_ref().into())), + Expression::Not(inner) => Self::Not(Box::new(inner.as_ref().into())), + Expression::Exists(inner) => Self::Exists(Box::new(inner.as_ref().into())), + Expression::Bound(variable) => Self::Bound(variable.clone()), + Expression::If(cond, yes, no) => Self::If( + Box::new(cond.as_ref().into()), + Box::new(yes.as_ref().into()), + Box::new(no.as_ref().into()), + ), + Expression::Coalesce(inner) => Self::Coalesce(inner.iter().map(Into::into).collect()), + Expression::FunctionCall(name, args) => { + Self::FunctionCall(name.clone(), args.iter().map(Into::into).collect()) + } + } + } +} + +impl BitAnd for Expression { + type Output = Self; + + fn bitand(self, rhs: Self) -> Self::Output { + Self::and_all([self, rhs]) + } +} + +impl BitOr for Expression { + type Output = Self; + + fn bitor(self, rhs: Self) -> Self { + Self::or_all([self, rhs]) + } +} + +impl Not for Expression { + type Output = Self; + + fn not(self) -> Self { + if let Some(v) = self.effective_boolean_value() { + (!v).into() + } else if let Self::Not(v) = self { + if v.returns_boolean() { + *v + } else { + Self::And(vec![*v]) + } + } else { + Self::Not(Box::new(self)) + } + } +} + +impl Add for Expression { + type Output = Self; + + fn add(self, rhs: Self) -> Self { + let (left, right) = order_pair(self, rhs); + Self::Add(Box::new(left), Box::new(right)) + } +} + +impl Sub for Expression { + type Output = Self; + + fn sub(self, rhs: Self) -> Self { + Self::Subtract(Box::new(self), Box::new(rhs)) + } +} + +impl Mul for Expression { + type Output = Self; + + fn mul(self, rhs: Self) -> Self { + let (left, right) = order_pair(self, rhs); + Self::Multiply(Box::new(left), Box::new(right)) + } +} + +impl Div for Expression { + type Output = Self; + + fn div(self, rhs: Self) -> Self { + Self::Divide(Box::new(self), Box::new(rhs)) + } +} + +impl Neg for Expression { + type Output = Self; + + fn neg(self) -> Self { + Self::UnaryMinus(Box::new(self)) + } +} + +/// A SPARQL query [graph pattern](https://www.w3.org/TR/sparql11-query/#sparqlQuery). +#[derive(Eq, PartialEq, Debug, Clone, Hash)] +pub enum GraphPattern { + /// A [basic graph pattern](https://www.w3.org/TR/sparql11-query/#defn_BasicGraphPattern). + QuadPattern { + subject: GroundTermPattern, + predicate: NamedNodePattern, + object: GroundTermPattern, + graph_name: Option, + }, + /// A [property path pattern](https://www.w3.org/TR/sparql11-query/#defn_evalPP_predicate). + Path { + subject: GroundTermPattern, + path: PropertyPathExpression, + object: GroundTermPattern, + graph_name: Option, + }, + /// [Join](https://www.w3.org/TR/sparql11-query/#defn_algJoin). + Join { + left: Box, + right: Box, + algorithm: JoinAlgorithm, + }, + /// [LeftJoin](https://www.w3.org/TR/sparql11-query/#defn_algLeftJoin). + LeftJoin { + left: Box, + right: Box, + expression: Expression, + }, + /// Lateral join i.e. evaluate right for all result row of left + #[cfg(feature = "sep-0006")] + Lateral { left: Box, right: Box }, + /// [Filter](https://www.w3.org/TR/sparql11-query/#defn_algFilter). + Filter { + expression: Expression, + inner: Box, + }, + /// [Union](https://www.w3.org/TR/sparql11-query/#defn_algUnion). + Union { inner: Vec }, + /// [Extend](https://www.w3.org/TR/sparql11-query/#defn_extend). + Extend { + inner: Box, + variable: Variable, + expression: Expression, + }, + /// [Minus](https://www.w3.org/TR/sparql11-query/#defn_algMinus). + Minus { left: Box, right: Box }, + /// A table used to provide inline values + Values { + variables: Vec, + bindings: Vec>>, + }, + /// [OrderBy](https://www.w3.org/TR/sparql11-query/#defn_algOrdered). + OrderBy { + inner: Box, + expression: Vec, + }, + /// [Project](https://www.w3.org/TR/sparql11-query/#defn_algProjection). + Project { + inner: Box, + variables: Vec, + }, + /// [Distinct](https://www.w3.org/TR/sparql11-query/#defn_algDistinct). + Distinct { inner: Box }, + /// [Reduced](https://www.w3.org/TR/sparql11-query/#defn_algReduced). + Reduced { inner: Box }, + /// [Slice](https://www.w3.org/TR/sparql11-query/#defn_algSlice). + Slice { + inner: Box, + start: usize, + length: Option, + }, + /// [Group](https://www.w3.org/TR/sparql11-query/#aggregateAlgebra). + Group { + inner: Box, + variables: Vec, + aggregates: Vec<(Variable, AggregateExpression)>, + }, + /// [Service](https://www.w3.org/TR/sparql11-federated-query/#defn_evalService). + Service { + name: NamedNodePattern, + inner: Box, + silent: bool, + }, +} + +impl GraphPattern { + pub fn empty() -> Self { + Self::Values { + variables: Vec::new(), + bindings: Vec::new(), + } + } + + /// Check if the pattern is the empty table + fn is_empty(&self) -> bool { + if let Self::Values { bindings, .. } = self { + bindings.is_empty() + } else { + false + } + } + + pub fn empty_singleton() -> Self { + Self::Values { + variables: Vec::new(), + bindings: vec![Vec::new()], + } + } + + pub fn is_empty_singleton(&self) -> bool { + if let Self::Values { bindings, .. } = self { + bindings.len() == 1 && bindings.iter().all(|b| b.iter().all(Option::is_none)) + } else { + false + } + } + + pub fn join(left: Self, right: Self, algorithm: JoinAlgorithm) -> Self { + if left.is_empty() || right.is_empty() { + return Self::empty(); + } + if left.is_empty_singleton() { + return right; + } + if right.is_empty_singleton() { + return left; + } + Self::Join { + left: Box::new(left), + right: Box::new(right), + algorithm, + } + } + + #[cfg(feature = "sep-0006")] + pub fn lateral(left: Self, right: Self) -> Self { + if left.is_empty() || right.is_empty() { + return Self::empty(); + } + if left.is_empty_singleton() { + return right; + } + if right.is_empty_singleton() { + return left; + } + Self::Lateral { + left: Box::new(left), + right: Box::new(right), + } + } + + pub fn left_join(left: Self, right: Self, expression: Expression) -> Self { + let expression_ebv = expression.effective_boolean_value(); + if left.is_empty() + || right.is_empty() + || right.is_empty_singleton() + || expression_ebv == Some(false) + { + return left; + } + Self::LeftJoin { + left: Box::new(left), + right: Box::new(right), + expression: if expression_ebv == Some(true) { + true.into() + } else { + expression + }, + } + } + + pub fn minus(left: Self, right: Self) -> Self { + if left.is_empty() { + return Self::empty(); + } + if right.is_empty() { + return left; + } + Self::Minus { + left: Box::new(left), + right: Box::new(right), + } + } + + pub fn union(left: Self, right: Self) -> Self { + Self::union_all([left, right]) + } + + pub fn union_all(args: impl IntoIterator) -> Self { + let args = args.into_iter(); + let mut all = Vec::with_capacity(args.size_hint().0); + for arg in args { + if arg.is_empty() { + continue; + } + if let Self::Union { inner } = arg { + all.extend(inner); + } else { + all.push(arg); + } + } + if all.is_empty() { + GraphPattern::empty() + } else { + Self::Union { + inner: order_vec(all), + } + } + } + + pub fn filter(inner: Self, expression: Expression) -> Self { + if inner.is_empty() { + return Self::empty(); + } + // We unwrap singleton And + let expression = match expression { + Expression::And(mut l) if l.len() == 1 => l.pop().unwrap(), + e => e, + }; + match expression.effective_boolean_value() { + Some(true) => inner, + Some(false) => Self::empty(), + None => match inner { + Self::Filter { + inner, + expression: e2, + } => Self::Filter { + inner, + expression: expression & e2, + }, + inner => Self::Filter { + inner: Box::new(inner), + expression, + }, + }, + } + } + + pub fn extend(inner: Self, variable: Variable, expression: Expression) -> Self { + if inner.is_empty() { + return Self::empty(); + } + Self::Extend { + inner: Box::new(inner), + variable, + expression, + } + } + + pub fn values( + mut variables: Vec, + mut bindings: Vec>>, + ) -> Self { + let empty_rows = (0..variables.len()) + .filter(|row| !bindings.iter().any(|binding| binding.get(*row).is_some())) + .collect::>(); + if !empty_rows.is_empty() { + // We remove empty rows + variables = variables + .into_iter() + .enumerate() + .filter_map(|(i, v)| { + if empty_rows.contains(&i) { + None + } else { + Some(v) + } + }) + .collect(); + bindings = bindings + .into_iter() + .map(|binding| { + binding + .into_iter() + .enumerate() + .filter_map(|(i, v)| { + if empty_rows.contains(&i) { + None + } else { + Some(v) + } + }) + .collect() + }) + .collect(); + } + Self::Values { + variables, + bindings, + } + } + + pub fn order_by(inner: Self, expression: Vec) -> Self { + if inner.is_empty() { + return Self::empty(); + } + if expression.is_empty() { + return inner; + } + Self::OrderBy { + inner: Box::new(inner), + expression, + } + } + + pub fn project(inner: Self, variables: Vec) -> Self { + Self::Project { + inner: Box::new(inner), + variables, + } + } + + pub fn distinct(inner: Self) -> Self { + if inner.is_empty() { + return Self::empty(); + } + Self::Distinct { + inner: Box::new(inner), + } + } + + pub fn reduced(inner: Self) -> Self { + if inner.is_empty() { + return Self::empty(); + } + Self::Reduced { + inner: Box::new(inner), + } + } + + pub fn slice(inner: Self, start: usize, length: Option) -> Self { + if inner.is_empty() { + return Self::empty(); + } + if start == 0 && length.is_none() { + return inner; + } + Self::Slice { + inner: Box::new(inner), + start, + length, + } + } + + pub fn group( + inner: Self, + variables: Vec, + aggregates: Vec<(Variable, AggregateExpression)>, + ) -> Self { + if inner.is_empty() { + return Self::empty(); + } + Self::Group { + inner: Box::new(inner), + variables, + aggregates, + } + } + + pub fn service(inner: Self, name: NamedNodePattern, silent: bool) -> Self { + if inner.is_empty() { + return Self::empty(); + } + Self::Service { + inner: Box::new(inner), + name, + silent, + } + } + + pub fn lookup_used_variables<'a>(&'a self, callback: &mut impl FnMut(&'a Variable)) { + match self { + Self::Values { variables, .. } | Self::Project { variables, .. } => { + for v in variables { + callback(v); + } + } + Self::QuadPattern { + subject, + predicate, + object, + graph_name, + } => { + lookup_term_pattern_variables(subject, callback); + if let NamedNodePattern::Variable(v) = predicate { + callback(v); + } + lookup_term_pattern_variables(object, callback); + if let Some(NamedNodePattern::Variable(v)) = graph_name { + callback(v); + } + } + Self::Path { + subject, + object, + graph_name, + .. + } => { + lookup_term_pattern_variables(subject, callback); + lookup_term_pattern_variables(object, callback); + if let Some(NamedNodePattern::Variable(v)) = graph_name { + callback(v); + } + } + Self::Filter { inner, expression } => { + expression.lookup_used_variables(callback); + inner.lookup_used_variables(callback); + } + Self::Union { inner } => { + for child in inner { + child.lookup_used_variables(callback); + } + } + Self::Join { left, right, .. } | Self::Minus { left, right } => { + left.lookup_used_variables(callback); + right.lookup_used_variables(callback); + } + #[cfg(feature = "sep-0006")] + Self::Lateral { left, right } => { + left.lookup_used_variables(callback); + right.lookup_used_variables(callback); + } + Self::LeftJoin { + left, + right, + expression, + } => { + expression.lookup_used_variables(callback); + left.lookup_used_variables(callback); + right.lookup_used_variables(callback); + } + Self::Extend { + inner, + variable, + expression, + } => { + callback(variable); + expression.lookup_used_variables(callback); + inner.lookup_used_variables(callback); + } + Self::OrderBy { inner, .. } + | Self::Distinct { inner } + | Self::Reduced { inner } + | Self::Slice { inner, .. } => inner.lookup_used_variables(callback), + Self::Service { inner, name, .. } => { + if let NamedNodePattern::Variable(v) = name { + callback(v); + } + inner.lookup_used_variables(callback); + } + Self::Group { + variables, + aggregates, + .. + } => { + for v in variables { + callback(v); + } + for (v, _) in aggregates { + callback(v); + } + } + } + } + + fn from_sparql_algebra( + pattern: &AlGraphPattern, + graph_name: Option<&NamedNodePattern>, + blank_nodes: &mut HashMap, + ) -> Self { + match pattern { + AlGraphPattern::Bgp { patterns } => patterns + .iter() + .map(|p| { + let (subject, predicate, object) = + Self::triple_pattern_from_algebra(p, blank_nodes); + Self::QuadPattern { + subject, + predicate, + object, + graph_name: graph_name.cloned(), + } + }) + .reduce(|a, b| Self::Join { + left: Box::new(a), + right: Box::new(b), + algorithm: JoinAlgorithm::default(), + }) + .unwrap_or_else(Self::empty_singleton), + AlGraphPattern::Path { + subject, + path, + object, + } => Self::Path { + subject: Self::term_pattern_from_algebra(subject, blank_nodes), + path: path.clone(), + object: Self::term_pattern_from_algebra(object, blank_nodes), + graph_name: graph_name.cloned(), + }, + AlGraphPattern::Join { left, right } => Self::Join { + left: Box::new(Self::from_sparql_algebra(left, graph_name, blank_nodes)), + right: Box::new(Self::from_sparql_algebra(right, graph_name, blank_nodes)), + algorithm: JoinAlgorithm::default(), + }, + AlGraphPattern::LeftJoin { + left, + right, + expression, + } => Self::LeftJoin { + left: Box::new(Self::from_sparql_algebra(left, graph_name, blank_nodes)), + right: Box::new(Self::from_sparql_algebra(right, graph_name, blank_nodes)), + expression: expression.as_ref().map_or_else( + || true.into(), + |e| Expression::from_sparql_algebra(e, graph_name), + ), + }, + #[cfg(feature = "sep-0006")] + AlGraphPattern::Lateral { left, right } => Self::Lateral { + left: Box::new(Self::from_sparql_algebra(left, graph_name, blank_nodes)), + right: Box::new(Self::from_sparql_algebra(right, graph_name, blank_nodes)), + }, + AlGraphPattern::Filter { inner, expr } => Self::Filter { + inner: Box::new(Self::from_sparql_algebra(inner, graph_name, blank_nodes)), + expression: Expression::from_sparql_algebra(expr, graph_name), + }, + AlGraphPattern::Union { left, right } => Self::Union { + inner: vec![ + Self::from_sparql_algebra(left, graph_name, blank_nodes), + Self::from_sparql_algebra(right, graph_name, blank_nodes), + ], + }, + AlGraphPattern::Graph { inner, name } => { + Self::from_sparql_algebra(inner, Some(name), blank_nodes) + } + AlGraphPattern::Extend { + inner, + expression, + variable, + } => Self::Extend { + inner: Box::new(Self::from_sparql_algebra(inner, graph_name, blank_nodes)), + expression: Expression::from_sparql_algebra(expression, graph_name), + variable: variable.clone(), + }, + AlGraphPattern::Minus { left, right } => Self::Minus { + left: Box::new(Self::from_sparql_algebra(left, graph_name, blank_nodes)), + right: Box::new(Self::from_sparql_algebra(right, graph_name, blank_nodes)), + }, + AlGraphPattern::Values { + variables, + bindings, + } => Self::Values { + variables: variables.clone(), + bindings: bindings.clone(), + }, + AlGraphPattern::OrderBy { inner, expression } => Self::OrderBy { + inner: Box::new(Self::from_sparql_algebra(inner, graph_name, blank_nodes)), + expression: expression + .iter() + .map(|e| OrderExpression::from_sparql_algebra(e, graph_name)) + .collect(), + }, + AlGraphPattern::Project { inner, variables } => { + let graph_name = if let Some(NamedNodePattern::Variable(graph_name)) = graph_name { + Some(NamedNodePattern::Variable( + if variables.contains(graph_name) { + graph_name.clone() + } else { + new_var() + }, + )) + } else { + graph_name.cloned() + }; + Self::Project { + inner: Box::new(Self::from_sparql_algebra( + inner, + graph_name.as_ref(), + &mut HashMap::new(), + )), + variables: variables.clone(), + } + } + AlGraphPattern::Distinct { inner } => Self::Distinct { + inner: Box::new(Self::from_sparql_algebra(inner, graph_name, blank_nodes)), + }, + AlGraphPattern::Reduced { inner } => Self::Distinct { + inner: Box::new(Self::from_sparql_algebra(inner, graph_name, blank_nodes)), + }, + AlGraphPattern::Slice { + inner, + start, + length, + } => Self::Slice { + inner: Box::new(Self::from_sparql_algebra(inner, graph_name, blank_nodes)), + start: *start, + length: *length, + }, + AlGraphPattern::Group { + inner, + variables, + aggregates, + } => Self::Group { + inner: Box::new(Self::from_sparql_algebra(inner, graph_name, blank_nodes)), + variables: variables.clone(), + aggregates: aggregates + .iter() + .map(|(var, expr)| { + ( + var.clone(), + AggregateExpression::from_sparql_algebra(expr, graph_name), + ) + }) + .collect(), + }, + AlGraphPattern::Service { + inner, + name, + silent, + } => Self::Service { + inner: Box::new(Self::from_sparql_algebra(inner, graph_name, blank_nodes)), + name: name.clone(), + silent: *silent, + }, + } + } + + fn triple_pattern_from_algebra( + pattern: &TriplePattern, + blank_nodes: &mut HashMap, + ) -> (GroundTermPattern, NamedNodePattern, GroundTermPattern) { + ( + Self::term_pattern_from_algebra(&pattern.subject, blank_nodes), + pattern.predicate.clone(), + Self::term_pattern_from_algebra(&pattern.object, blank_nodes), + ) + } + + fn term_pattern_from_algebra( + pattern: &TermPattern, + blank_nodes: &mut HashMap, + ) -> GroundTermPattern { + match pattern { + TermPattern::NamedNode(node) => node.clone().into(), + TermPattern::BlankNode(node) => blank_nodes + .entry(node.clone()) + .or_insert_with(new_var) + .clone() + .into(), + TermPattern::Literal(literal) => literal.clone().into(), + #[cfg(feature = "rdf-star")] + TermPattern::Triple(pattern) => { + let (subject, predicate, object) = + Self::triple_pattern_from_algebra(pattern, blank_nodes); + GroundTriplePattern { + subject, + predicate, + object, + } + .into() + } + TermPattern::Variable(variable) => variable.clone().into(), + } + } +} + +impl From<&AlGraphPattern> for GraphPattern { + fn from(pattern: &AlGraphPattern) -> Self { + Self::from_sparql_algebra(pattern, None, &mut HashMap::new()) + } +} + +impl From<&GraphPattern> for AlGraphPattern { + fn from(pattern: &GraphPattern) -> Self { + match pattern { + GraphPattern::QuadPattern { + subject, + predicate, + object, + graph_name, + } => { + let pattern = Self::Bgp { + patterns: vec![TriplePattern { + subject: subject.clone().into(), + predicate: predicate.clone(), + object: object.clone().into(), + }], + }; + if let Some(graph_name) = graph_name { + Self::Graph { + inner: Box::new(pattern), + name: graph_name.clone(), + } + } else { + pattern + } + } + GraphPattern::Path { + subject, + path, + object, + graph_name, + } => { + let pattern = Self::Path { + subject: subject.clone().into(), + path: path.clone(), + object: object.clone().into(), + }; + if let Some(graph_name) = graph_name { + Self::Graph { + inner: Box::new(pattern), + name: graph_name.clone(), + } + } else { + pattern + } + } + GraphPattern::Join { left, right, .. } => { + match (left.as_ref().into(), right.as_ref().into()) { + (Self::Bgp { patterns: mut left }, Self::Bgp { patterns: right }) => { + left.extend(right); + Self::Bgp { patterns: left } + } + (left, right) => Self::Join { + left: Box::new(left), + right: Box::new(right), + }, + } + } + GraphPattern::LeftJoin { + left, + right, + expression, + } => { + let empty_expr = if let Expression::Literal(l) = expression { + l.datatype() == xsd::BOOLEAN && l.value() == "true" + } else { + false + }; + Self::LeftJoin { + left: Box::new(left.as_ref().into()), + right: Box::new(right.as_ref().into()), + expression: if empty_expr { + None + } else { + Some(expression.into()) + }, + } + } + #[cfg(feature = "sep-0006")] + GraphPattern::Lateral { left, right } => { + match (left.as_ref().into(), right.as_ref().into()) { + (Self::Bgp { patterns: mut left }, Self::Bgp { patterns: right }) => { + left.extend(right); + Self::Bgp { patterns: left } + } + (left, right) => Self::Lateral { + left: Box::new(left), + right: Box::new(right), + }, + } + } + GraphPattern::Filter { inner, expression } => Self::Filter { + inner: Box::new(inner.as_ref().into()), + expr: expression.into(), + }, + GraphPattern::Union { inner } => inner + .iter() + .map(Into::into) + .reduce(|a, b| Self::Union { + left: Box::new(a), + right: Box::new(b), + }) + .unwrap_or_else(|| Self::Values { + variables: Vec::new(), + bindings: Vec::new(), + }), + GraphPattern::Extend { + inner, + expression, + variable, + } => Self::Extend { + inner: Box::new(inner.as_ref().into()), + expression: expression.into(), + variable: variable.clone(), + }, + GraphPattern::Minus { left, right } => Self::Minus { + left: Box::new(left.as_ref().into()), + right: Box::new(right.as_ref().into()), + }, + GraphPattern::Values { + variables, + bindings, + } => Self::Values { + variables: variables.clone(), + bindings: bindings.clone(), + }, + GraphPattern::OrderBy { inner, expression } => Self::OrderBy { + inner: Box::new(inner.as_ref().into()), + expression: expression.iter().map(Into::into).collect(), + }, + GraphPattern::Project { inner, variables } => Self::Project { + inner: Box::new(inner.as_ref().into()), + variables: variables.clone(), + }, + GraphPattern::Distinct { inner } => Self::Distinct { + inner: Box::new(inner.as_ref().into()), + }, + GraphPattern::Reduced { inner } => Self::Distinct { + inner: Box::new(inner.as_ref().into()), + }, + GraphPattern::Slice { + inner, + start, + length, + } => Self::Slice { + inner: Box::new(inner.as_ref().into()), + start: *start, + length: *length, + }, + GraphPattern::Group { + inner, + variables, + aggregates, + } => Self::Group { + inner: Box::new(inner.as_ref().into()), + variables: variables.clone(), + aggregates: aggregates + .iter() + .map(|(var, expr)| (var.clone(), expr.into())) + .collect(), + }, + GraphPattern::Service { + inner, + name, + silent, + } => Self::Service { + inner: Box::new(inner.as_ref().into()), + name: name.clone(), + silent: *silent, + }, + } + } +} + +/// The join algorithm used (c.f. [`GraphPattern::Join`]). +#[derive(Eq, PartialEq, Debug, Clone, Copy, Hash)] +pub enum JoinAlgorithm { + HashBuildLeftProbeRight, +} + +impl Default for JoinAlgorithm { + fn default() -> Self { + Self::HashBuildLeftProbeRight + } +} + +/// A set function used in aggregates (c.f. [`GraphPattern::Group`]). +#[derive(Eq, PartialEq, Debug, Clone, Hash)] +pub enum AggregateExpression { + /// [Count](https://www.w3.org/TR/sparql11-query/#defn_aggCount). + Count { + expr: Option>, + distinct: bool, + }, + /// [Sum](https://www.w3.org/TR/sparql11-query/#defn_aggSum). + Sum { + expr: Box, + distinct: bool, + }, + /// [Avg](https://www.w3.org/TR/sparql11-query/#defn_aggAvg). + Avg { + expr: Box, + distinct: bool, + }, + /// [Min](https://www.w3.org/TR/sparql11-query/#defn_aggMin). + Min { + expr: Box, + distinct: bool, + }, + /// [Max](https://www.w3.org/TR/sparql11-query/#defn_aggMax). + Max { + expr: Box, + distinct: bool, + }, + /// [GroupConcat](https://www.w3.org/TR/sparql11-query/#defn_aggGroupConcat). + GroupConcat { + expr: Box, + distinct: bool, + separator: Option, + }, + /// [Sample](https://www.w3.org/TR/sparql11-query/#defn_aggSample). + Sample { + expr: Box, + distinct: bool, + }, + /// Custom function. + Custom { + name: NamedNode, + expr: Box, + distinct: bool, + }, +} + +impl AggregateExpression { + fn from_sparql_algebra( + expression: &AlAggregateExpression, + graph_name: Option<&NamedNodePattern>, + ) -> Self { + match expression { + AlAggregateExpression::Count { expr, distinct } => Self::Count { + expr: expr + .as_ref() + .map(|e| Box::new(Expression::from_sparql_algebra(e, graph_name))), + distinct: *distinct, + }, + AlAggregateExpression::Sum { expr, distinct } => Self::Sum { + expr: Box::new(Expression::from_sparql_algebra(expr, graph_name)), + distinct: *distinct, + }, + AlAggregateExpression::Avg { expr, distinct } => Self::Avg { + expr: Box::new(Expression::from_sparql_algebra(expr, graph_name)), + distinct: *distinct, + }, + AlAggregateExpression::Min { expr, distinct } => Self::Min { + expr: Box::new(Expression::from_sparql_algebra(expr, graph_name)), + distinct: *distinct, + }, + AlAggregateExpression::Max { expr, distinct } => Self::Max { + expr: Box::new(Expression::from_sparql_algebra(expr, graph_name)), + distinct: *distinct, + }, + AlAggregateExpression::GroupConcat { + expr, + distinct, + separator, + } => Self::GroupConcat { + expr: Box::new(Expression::from_sparql_algebra(expr, graph_name)), + distinct: *distinct, + separator: separator.clone(), + }, + AlAggregateExpression::Sample { expr, distinct } => Self::Sample { + expr: Box::new(Expression::from_sparql_algebra(expr, graph_name)), + distinct: *distinct, + }, + AlAggregateExpression::Custom { + name, + expr, + distinct, + } => Self::Custom { + name: name.clone(), + expr: Box::new(Expression::from_sparql_algebra(expr, graph_name)), + distinct: *distinct, + }, + } + } +} + +impl From<&AggregateExpression> for AlAggregateExpression { + fn from(expression: &AggregateExpression) -> Self { + match expression { + AggregateExpression::Count { expr, distinct } => Self::Count { + expr: expr.as_ref().map(|e| Box::new(e.as_ref().into())), + distinct: *distinct, + }, + AggregateExpression::Sum { expr, distinct } => Self::Sum { + expr: Box::new(expr.as_ref().into()), + distinct: *distinct, + }, + AggregateExpression::Avg { expr, distinct } => Self::Avg { + expr: Box::new(expr.as_ref().into()), + distinct: *distinct, + }, + AggregateExpression::Min { expr, distinct } => Self::Min { + expr: Box::new(expr.as_ref().into()), + distinct: *distinct, + }, + AggregateExpression::Max { expr, distinct } => Self::Max { + expr: Box::new(expr.as_ref().into()), + distinct: *distinct, + }, + AggregateExpression::GroupConcat { + expr, + distinct, + separator, + } => Self::GroupConcat { + expr: Box::new(expr.as_ref().into()), + distinct: *distinct, + separator: separator.clone(), + }, + AggregateExpression::Sample { expr, distinct } => Self::Sample { + expr: Box::new(expr.as_ref().into()), + distinct: *distinct, + }, + AggregateExpression::Custom { + name, + expr, + distinct, + } => Self::Custom { + name: name.clone(), + expr: Box::new(expr.as_ref().into()), + distinct: *distinct, + }, + } + } +} + +/// An ordering comparator used by [`GraphPattern::OrderBy`]. +#[derive(Eq, PartialEq, Debug, Clone, Hash)] +pub enum OrderExpression { + /// Ascending order + Asc(Expression), + /// Descending order + Desc(Expression), +} + +impl OrderExpression { + fn from_sparql_algebra( + expression: &AlOrderExpression, + graph_name: Option<&NamedNodePattern>, + ) -> Self { + match expression { + AlOrderExpression::Asc(e) => Self::Asc(Expression::from_sparql_algebra(e, graph_name)), + AlOrderExpression::Desc(e) => { + Self::Desc(Expression::from_sparql_algebra(e, graph_name)) + } + } + } +} + +impl From<&OrderExpression> for AlOrderExpression { + fn from(expression: &OrderExpression) -> Self { + match expression { + OrderExpression::Asc(e) => Self::Asc(e.into()), + OrderExpression::Desc(e) => Self::Desc(e.into()), + } + } +} + +fn new_var() -> Variable { + Variable::new_unchecked(format!("{:x}", random::())) +} + +fn order_pair(a: T, b: T) -> (T, T) { + if hash(&a) <= hash(&b) { + (a, b) + } else { + (b, a) + } +} + +fn order_vec(mut vec: Vec) -> Vec { + vec.sort_unstable_by_key(|a| hash(a)); + vec +} + +fn hash(v: impl Hash) -> u64 { + let mut hasher = DefaultHasher::new(); + v.hash(&mut hasher); + hasher.finish() +} + +fn lookup_term_pattern_variables<'a>( + pattern: &'a GroundTermPattern, + callback: &mut impl FnMut(&'a Variable), +) { + if let GroundTermPattern::Variable(v) = pattern { + callback(v); + } + #[cfg(feature = "rdf-star")] + if let GroundTermPattern::Triple(t) = pattern { + lookup_term_pattern_variables(&t.subject, callback); + if let NamedNodePattern::Variable(v) = &t.predicate { + callback(v); + } + lookup_term_pattern_variables(&t.object, callback); + } +} diff --git a/lib/sparopt/src/lib.rs b/lib/sparopt/src/lib.rs new file mode 100644 index 00000000..d6f62207 --- /dev/null +++ b/lib/sparopt/src/lib.rs @@ -0,0 +1,5 @@ +pub use crate::optimizer::Optimizer; + +pub mod algebra; +mod optimizer; +mod type_inference; diff --git a/lib/sparopt/src/optimizer.rs b/lib/sparopt/src/optimizer.rs new file mode 100644 index 00000000..91ff65d0 --- /dev/null +++ b/lib/sparopt/src/optimizer.rs @@ -0,0 +1,1022 @@ +use crate::algebra::{Expression, GraphPattern, JoinAlgorithm, OrderExpression}; +use crate::type_inference::{ + infer_expression_type, infer_graph_pattern_types, VariableType, VariableTypes, +}; +use spargebra::algebra::PropertyPathExpression; +use spargebra::term::{GroundTermPattern, NamedNodePattern}; +use std::cmp::{max, min}; + +pub struct Optimizer; + +impl Optimizer { + pub fn optimize_graph_pattern(pattern: GraphPattern) -> GraphPattern { + let pattern = Self::normalize_pattern(pattern, &VariableTypes::default()); + let pattern = Self::reorder_joins(pattern, &VariableTypes::default()); + Self::push_filters(pattern, Vec::new(), &VariableTypes::default()) + } + + /// Normalize the pattern, discarding any join ordering information + fn normalize_pattern(pattern: GraphPattern, input_types: &VariableTypes) -> GraphPattern { + match pattern { + GraphPattern::QuadPattern { + subject, + predicate, + object, + graph_name, + } => GraphPattern::QuadPattern { + subject, + predicate, + object, + graph_name, + }, + GraphPattern::Path { + subject, + path, + object, + graph_name, + } => GraphPattern::Path { + subject, + path, + object, + graph_name, + }, + GraphPattern::Join { + left, + right, + algorithm, + } => GraphPattern::join( + Self::normalize_pattern(*left, input_types), + Self::normalize_pattern(*right, input_types), + algorithm, + ), + GraphPattern::LeftJoin { + left, + right, + expression, + } => { + let left = Self::normalize_pattern(*left, input_types); + let right = Self::normalize_pattern(*right, input_types); + let mut inner_types = infer_graph_pattern_types(&left, input_types.clone()); + inner_types.intersect_with(infer_graph_pattern_types(&right, input_types.clone())); + GraphPattern::left_join( + left, + right, + Self::normalize_expression(expression, &inner_types), + ) + } + #[cfg(feature = "sep-0006")] + GraphPattern::Lateral { left, right } => { + let left = Self::normalize_pattern(*left, input_types); + let left_types = infer_graph_pattern_types(&left, input_types.clone()); + let right = Self::normalize_pattern(*right, &left_types); + GraphPattern::lateral(left, right) + } + GraphPattern::Filter { inner, expression } => { + let inner = Self::normalize_pattern(*inner, input_types); + let inner_types = infer_graph_pattern_types(&inner, input_types.clone()); + let expression = Self::normalize_expression(expression, &inner_types); + let expression_type = infer_expression_type(&expression, &inner_types); + if expression_type == VariableType::UNDEF { + GraphPattern::empty() + } else { + GraphPattern::filter(inner, expression) + } + } + GraphPattern::Union { inner } => GraphPattern::union_all( + inner + .into_iter() + .map(|e| Self::normalize_pattern(e, input_types)), + ), + GraphPattern::Extend { + inner, + variable, + expression, + } => { + let inner = Self::normalize_pattern(*inner, input_types); + let inner_types = infer_graph_pattern_types(&inner, input_types.clone()); + let expression = Self::normalize_expression(expression, &inner_types); + let expression_type = infer_expression_type(&expression, &inner_types); + if expression_type == VariableType::UNDEF { + //TODO: valid? + inner + } else { + GraphPattern::extend(inner, variable, expression) + } + } + GraphPattern::Minus { left, right } => GraphPattern::minus( + Self::normalize_pattern(*left, input_types), + Self::normalize_pattern(*right, input_types), + ), + GraphPattern::Values { + variables, + bindings, + } => GraphPattern::values(variables, bindings), + GraphPattern::OrderBy { inner, expression } => { + let inner = Self::normalize_pattern(*inner, input_types); + let inner_types = infer_graph_pattern_types(&inner, input_types.clone()); + GraphPattern::order_by( + inner, + expression + .into_iter() + .map(|e| match e { + OrderExpression::Asc(e) => { + OrderExpression::Asc(Self::normalize_expression(e, &inner_types)) + } + OrderExpression::Desc(e) => { + OrderExpression::Desc(Self::normalize_expression(e, &inner_types)) + } + }) + .collect(), + ) + } + GraphPattern::Project { inner, variables } => { + GraphPattern::project(Self::normalize_pattern(*inner, input_types), variables) + } + GraphPattern::Distinct { inner } => { + GraphPattern::distinct(Self::normalize_pattern(*inner, input_types)) + } + GraphPattern::Reduced { inner } => { + GraphPattern::reduced(Self::normalize_pattern(*inner, input_types)) + } + GraphPattern::Slice { + inner, + start, + length, + } => GraphPattern::slice(Self::normalize_pattern(*inner, input_types), start, length), + GraphPattern::Group { + inner, + variables, + aggregates, + } => GraphPattern::group( + Self::normalize_pattern(*inner, input_types), + variables, + aggregates, + ), + GraphPattern::Service { + name, + inner, + silent, + } => GraphPattern::service(Self::normalize_pattern(*inner, input_types), name, silent), + } + } + + fn normalize_expression(expression: Expression, types: &VariableTypes) -> Expression { + match expression { + Expression::NamedNode(node) => node.into(), + Expression::Literal(literal) => literal.into(), + Expression::Variable(variable) => variable.into(), + Expression::Or(inner) => Expression::or_all( + inner + .into_iter() + .map(|e| Self::normalize_expression(e, types)), + ), + Expression::And(inner) => Expression::and_all( + inner + .into_iter() + .map(|e| Self::normalize_expression(e, types)), + ), + Expression::Equal(left, right) => { + let left = Self::normalize_expression(*left, types); + let left_types = infer_expression_type(&left, types); + let right = Self::normalize_expression(*right, types); + let right_types = infer_expression_type(&right, types); + #[allow(unused_mut)] + let mut must_use_equal = left_types.literal && right_types.literal; + #[cfg(feature = "rdf-star")] + { + must_use_equal = must_use_equal || left_types.triple && right_types.triple; + } + if must_use_equal { + Expression::equal(left, right) + } else { + Expression::same_term(left, right) + } + } + Expression::SameTerm(left, right) => Expression::same_term( + Self::normalize_expression(*left, types), + Self::normalize_expression(*right, types), + ), + Expression::Greater(left, right) => Expression::greater( + Self::normalize_expression(*left, types), + Self::normalize_expression(*right, types), + ), + Expression::GreaterOrEqual(left, right) => Expression::greater_or_equal( + Self::normalize_expression(*left, types), + Self::normalize_expression(*right, types), + ), + Expression::Less(left, right) => Expression::less( + Self::normalize_expression(*left, types), + Self::normalize_expression(*right, types), + ), + Expression::LessOrEqual(left, right) => Expression::less_or_equal( + Self::normalize_expression(*left, types), + Self::normalize_expression(*right, types), + ), + Expression::Add(left, right) => { + Self::normalize_expression(*left, types) + Self::normalize_expression(*right, types) + } + Expression::Subtract(left, right) => { + Self::normalize_expression(*left, types) - Self::normalize_expression(*right, types) + } + Expression::Multiply(left, right) => { + Self::normalize_expression(*left, types) * Self::normalize_expression(*right, types) + } + Expression::Divide(left, right) => { + Self::normalize_expression(*left, types) / Self::normalize_expression(*right, types) + } + Expression::UnaryPlus(inner) => { + Expression::unary_plus(Self::normalize_expression(*inner, types)) + } + Expression::UnaryMinus(inner) => -Self::normalize_expression(*inner, types), + Expression::Not(inner) => !Self::normalize_expression(*inner, types), + Expression::Exists(inner) => Expression::exists(Self::normalize_pattern(*inner, types)), + Expression::Bound(variable) => { + let t = types.get(&variable); + if !t.undef { + true.into() + } else if t == VariableType::UNDEF { + false.into() + } else { + Expression::Bound(variable) + } + } + Expression::If(cond, then, els) => Expression::if_cond( + Self::normalize_expression(*cond, types), + Self::normalize_expression(*then, types), + Self::normalize_expression(*els, types), + ), + Expression::Coalesce(inners) => Expression::coalesce( + inners + .into_iter() + .map(|e| Self::normalize_expression(e, types)) + .collect(), + ), + Expression::FunctionCall(name, args) => Expression::call( + name, + args.into_iter() + .map(|e| Self::normalize_expression(e, types)) + .collect(), + ), + } + } + + fn push_filters( + pattern: GraphPattern, + filters: Vec, + input_types: &VariableTypes, + ) -> GraphPattern { + match pattern { + pattern @ (GraphPattern::QuadPattern { .. } + | GraphPattern::Path { .. } + | GraphPattern::Values { .. }) => { + GraphPattern::filter(pattern, Expression::and_all(filters)) + } + GraphPattern::Join { + left, + right, + algorithm, + } => { + let left_types = infer_graph_pattern_types(&left, input_types.clone()); + let right_types = infer_graph_pattern_types(&right, input_types.clone()); + let mut left_filters = Vec::new(); + let mut right_filters = Vec::new(); + let mut final_filters = Vec::new(); + for filter in filters { + let push_left = are_all_expression_variables_bound(&filter, &left_types); + let push_right = are_all_expression_variables_bound(&filter, &right_types); + if push_left { + if push_right { + left_filters.push(filter.clone()); + right_filters.push(filter); + } else { + left_filters.push(filter); + } + } else if push_right { + right_filters.push(filter); + } else { + final_filters.push(filter); + } + } + GraphPattern::filter( + GraphPattern::join( + Self::push_filters(*left, left_filters, input_types), + Self::push_filters(*right, right_filters, input_types), + algorithm, + ), + Expression::and_all(final_filters), + ) + } + #[cfg(feature = "sep-0006")] + GraphPattern::Lateral { left, right } => { + let left_types = infer_graph_pattern_types(&left, input_types.clone()); + let mut left_filters = Vec::new(); + let mut right_filters = Vec::new(); + for filter in filters { + let push_left = are_all_expression_variables_bound(&filter, &left_types); + if push_left { + left_filters.push(filter); + } else { + right_filters.push(filter); + } + } + let left = Self::push_filters(*left, left_filters, input_types); + let right = Self::push_filters(*right, right_filters, &left_types); + if let GraphPattern::Filter { + inner: right, + expression, + } = right + { + // We prefer to have filter out of the lateral rather than inside the right part + GraphPattern::filter(GraphPattern::lateral(left, *right), expression) + } else { + GraphPattern::lateral(left, right) + } + } + GraphPattern::LeftJoin { + left, + right, + expression, + } => { + let left_types = infer_graph_pattern_types(&left, input_types.clone()); + let right_types = infer_graph_pattern_types(&right, input_types.clone()); + let mut left_filters = Vec::new(); + let mut right_filters = Vec::new(); + let mut final_filters = Vec::new(); + for filter in filters { + let push_left = are_all_expression_variables_bound(&filter, &left_types); + if push_left { + left_filters.push(filter); + } else { + final_filters.push(filter); + } + } + let expression = if expression.effective_boolean_value().is_none() + && (are_all_expression_variables_bound(&expression, &right_types) + || are_no_expression_variables_bound(&expression, &left_types)) + { + right_filters.push(expression); + true.into() + } else { + expression + }; + GraphPattern::filter( + GraphPattern::left_join( + Self::push_filters(*left, left_filters, input_types), + Self::push_filters(*right, right_filters, input_types), + expression, + ), + Expression::and_all(final_filters), + ) + } + GraphPattern::Minus { left, right } => GraphPattern::minus( + Self::push_filters(*left, filters, input_types), + Self::push_filters(*right, Vec::new(), input_types), + ), + GraphPattern::Extend { + inner, + expression, + variable, + } => { + //TODO: handle the case where the filter overrides an expression variable (should not happen in SPARQL but allowed in the algebra) + let mut inner_filters = Vec::new(); + let mut final_filters = Vec::new(); + for filter in filters { + let extend_variable_used = + filter.used_variables().into_iter().any(|v| *v == variable); + if extend_variable_used { + final_filters.push(filter); + } else { + inner_filters.push(filter); + } + } + GraphPattern::filter( + GraphPattern::extend( + Self::push_filters(*inner, inner_filters, input_types), + variable, + expression, + ), + Expression::and_all(final_filters), + ) + } + GraphPattern::Filter { inner, expression } => { + let mut filters = filters; + if let Expression::And(expressions) = expression { + filters.extend(expressions) + } else { + filters.push(expression) + }; + Self::push_filters(*inner, filters, input_types) + } + GraphPattern::Union { inner } => GraphPattern::union_all( + inner + .into_iter() + .map(|c| Self::push_filters(c, filters.clone(), input_types)), + ), + GraphPattern::Slice { + inner, + start, + length, + } => GraphPattern::filter( + GraphPattern::slice( + Self::push_filters(*inner, Vec::new(), input_types), + start, + length, + ), + Expression::and_all(filters), + ), + GraphPattern::Distinct { inner } => { + GraphPattern::distinct(Self::push_filters(*inner, filters, input_types)) + } + GraphPattern::Reduced { inner } => { + GraphPattern::reduced(Self::push_filters(*inner, filters, input_types)) + } + GraphPattern::Project { inner, variables } => { + GraphPattern::project(Self::push_filters(*inner, filters, input_types), variables) + } + GraphPattern::OrderBy { inner, expression } => { + GraphPattern::order_by(Self::push_filters(*inner, filters, input_types), expression) + } + GraphPattern::Service { + inner, + name, + silent, + } => GraphPattern::service( + Self::push_filters(*inner, filters, input_types), + name, + silent, + ), + GraphPattern::Group { + inner, + variables, + aggregates, + } => GraphPattern::filter( + GraphPattern::group( + Self::push_filters(*inner, Vec::new(), input_types), + variables, + aggregates, + ), + Expression::and_all(filters), + ), + } + } + + fn reorder_joins(pattern: GraphPattern, input_types: &VariableTypes) -> GraphPattern { + match pattern { + pattern @ (GraphPattern::QuadPattern { .. } + | GraphPattern::Path { .. } + | GraphPattern::Values { .. }) => pattern, + GraphPattern::Join { left, right, .. } => { + // We flatten the join operation + let mut to_reorder = Vec::new(); + let mut todo = vec![*right, *left]; + while let Some(e) = todo.pop() { + if let GraphPattern::Join { left, right, .. } = e { + todo.push(*right); + todo.push(*left); + } else { + to_reorder.push(e); + } + } + + // We do first type inference + let to_reorder_types = to_reorder + .iter() + .map(|p| infer_graph_pattern_types(p, input_types.clone())) + .collect::>(); + + // We do greedy join reordering + let mut output_cartesian_product_joins = Vec::new(); + let mut not_yet_reordered_ids = vec![true; to_reorder.len()]; + // We look for the next connected component to reorder and pick the smallest element + while let Some(next_entry_id) = not_yet_reordered_ids + .iter() + .enumerate() + .filter_map(|(i, v)| v.then(|| i)) + .min_by_key(|i| estimate_graph_pattern_size(&to_reorder[*i], input_types)) + { + not_yet_reordered_ids[next_entry_id] = false; // It's now done + let mut output = to_reorder[next_entry_id].clone(); + let mut output_types = to_reorder_types[next_entry_id].clone(); + // We look for an other child to join with that does not blow up the join cost + while let Some(next_id) = not_yet_reordered_ids + .iter() + .enumerate() + .filter_map(|(i, v)| v.then(|| i)) + .filter(|i| { + count_common_variables( + &output_types, + &to_reorder_types[*i], + input_types, + ) > 0 + }) + .min_by_key(|i| { + // Estimation of the join cost + if cfg!(feature = "sep-0006") + && is_fit_for_for_loop_join( + &to_reorder[*i], + input_types, + &output_types, + ) + { + estimate_lateral_cost( + &output, + &output_types, + &to_reorder[*i], + input_types, + ) + } else { + estimate_join_cost( + &output, + &output_types, + &to_reorder[*i], + &to_reorder_types[*i], + JoinAlgorithm::HashBuildLeftProbeRight, + input_types, + ) + } + }) + { + not_yet_reordered_ids[next_id] = false; // It's now done + let next = to_reorder[next_id].clone(); + #[cfg(feature = "sep-0006")] + { + output = if is_fit_for_for_loop_join(&next, input_types, &output_types) + { + GraphPattern::lateral(output, next) + } else { + GraphPattern::join( + output, + next, + JoinAlgorithm::HashBuildLeftProbeRight, + ) + }; + } + #[cfg(not(feature = "sep-0006"))] + { + output = GraphPattern::join( + output, + next, + JoinAlgorithm::HashBuildLeftProbeRight, + ); + } + output_types.intersect_with(to_reorder_types[next_id].clone()); + } + output_cartesian_product_joins.push(output); + } + output_cartesian_product_joins + .into_iter() + .reduce(|left, right| { + if estimate_graph_pattern_size(&left, input_types) + <= estimate_graph_pattern_size(&right, input_types) + { + GraphPattern::join(left, right, JoinAlgorithm::HashBuildLeftProbeRight) + } else { + GraphPattern::join(right, left, JoinAlgorithm::HashBuildLeftProbeRight) + } + }) + .unwrap() + } + #[cfg(feature = "sep-0006")] + GraphPattern::Lateral { left, right } => { + let left_types = infer_graph_pattern_types(&left, input_types.clone()); + GraphPattern::lateral( + Self::reorder_joins(*left, input_types), + Self::reorder_joins(*right, &left_types), + ) + } + GraphPattern::LeftJoin { + left, + right, + expression, + } => { + let left = Self::reorder_joins(*left, input_types); + let right = Self::reorder_joins(*right, input_types); + #[cfg(feature = "sep-0006")] + { + let left_types = infer_graph_pattern_types(&left, input_types.clone()); + let right_types = infer_graph_pattern_types(&right, input_types.clone()); + if is_fit_for_for_loop_join(&right, input_types, &left_types) + && count_common_variables(&left_types, &right_types, input_types) > 0 + { + return GraphPattern::lateral( + left, + GraphPattern::left_join( + GraphPattern::empty_singleton(), + right, + expression, + ), + ); + } + } + GraphPattern::left_join(left, right, expression) + } + GraphPattern::Minus { left, right } => GraphPattern::minus( + Self::reorder_joins(*left, input_types), + Self::reorder_joins(*right, input_types), + ), + GraphPattern::Extend { + inner, + expression, + variable, + } => GraphPattern::extend( + Self::reorder_joins(*inner, input_types), + variable, + expression, + ), + GraphPattern::Filter { inner, expression } => { + GraphPattern::filter(Self::reorder_joins(*inner, input_types), expression) + } + GraphPattern::Union { inner } => GraphPattern::union_all( + inner + .into_iter() + .map(|c| Self::reorder_joins(c, input_types)), + ), + GraphPattern::Slice { + inner, + start, + length, + } => GraphPattern::slice(Self::reorder_joins(*inner, input_types), start, length), + GraphPattern::Distinct { inner } => { + GraphPattern::distinct(Self::reorder_joins(*inner, input_types)) + } + GraphPattern::Reduced { inner } => { + GraphPattern::reduced(Self::reorder_joins(*inner, input_types)) + } + GraphPattern::Project { inner, variables } => { + GraphPattern::project(Self::reorder_joins(*inner, input_types), variables) + } + GraphPattern::OrderBy { inner, expression } => { + GraphPattern::order_by(Self::reorder_joins(*inner, input_types), expression) + } + GraphPattern::Service { + inner, + name, + silent, + } => GraphPattern::service(Self::reorder_joins(*inner, input_types), name, silent), + GraphPattern::Group { + inner, + variables, + aggregates, + } => GraphPattern::group( + Self::reorder_joins(*inner, input_types), + variables, + aggregates, + ), + } + } +} + +fn is_fit_for_for_loop_join( + pattern: &GraphPattern, + global_input_types: &VariableTypes, + entry_types: &VariableTypes, +) -> bool { + //TODO: think more about it + match pattern { + GraphPattern::Values { .. } + | GraphPattern::QuadPattern { .. } + | GraphPattern::Path { .. } => true, + #[cfg(feature = "sep-0006")] + GraphPattern::Lateral { left, right } => { + is_fit_for_for_loop_join(left, global_input_types, entry_types) + && is_fit_for_for_loop_join(right, global_input_types, entry_types) + } + GraphPattern::LeftJoin { + left, + right, + expression, + } => { + if !is_fit_for_for_loop_join(left, global_input_types, entry_types) { + return false; + } + + // It is not ok to transform into for loop join if right binds a variable also bound by the entry part of the for loop join + let mut left_types = infer_graph_pattern_types(left, global_input_types.clone()); + let right_types = infer_graph_pattern_types(right, global_input_types.clone()); + if right_types.iter().any(|(variable, t)| { + *t != VariableType::UNDEF + && left_types.get(variable).undef + && entry_types.get(variable) != VariableType::UNDEF + }) { + return false; + } + + // We don't forget the final expression + left_types.intersect_with(right_types); + is_expression_fit_for_for_loop_join(expression, &left_types, entry_types) + } + GraphPattern::Union { inner } => inner + .iter() + .all(|i| is_fit_for_for_loop_join(i, global_input_types, entry_types)), + GraphPattern::Filter { inner, expression } => { + is_fit_for_for_loop_join(inner, global_input_types, entry_types) + && is_expression_fit_for_for_loop_join( + expression, + &infer_graph_pattern_types(inner, global_input_types.clone()), + entry_types, + ) + } + GraphPattern::Extend { + inner, + expression, + variable, + } => { + is_fit_for_for_loop_join(inner, global_input_types, entry_types) + && entry_types.get(variable) == VariableType::UNDEF + && is_expression_fit_for_for_loop_join( + expression, + &infer_graph_pattern_types(inner, global_input_types.clone()), + entry_types, + ) + } + GraphPattern::Join { .. } + | GraphPattern::Minus { .. } + | GraphPattern::Service { .. } + | GraphPattern::OrderBy { .. } + | GraphPattern::Distinct { .. } + | GraphPattern::Reduced { .. } + | GraphPattern::Slice { .. } + | GraphPattern::Project { .. } + | GraphPattern::Group { .. } => false, + } +} + +fn are_all_expression_variables_bound( + expression: &Expression, + variable_types: &VariableTypes, +) -> bool { + expression + .used_variables() + .into_iter() + .all(|v| !variable_types.get(v).undef) +} + +fn are_no_expression_variables_bound( + expression: &Expression, + variable_types: &VariableTypes, +) -> bool { + expression + .used_variables() + .into_iter() + .all(|v| variable_types.get(v) == VariableType::UNDEF) +} + +fn is_expression_fit_for_for_loop_join( + expression: &Expression, + input_types: &VariableTypes, + entry_types: &VariableTypes, +) -> bool { + match expression { + Expression::NamedNode(_) | Expression::Literal(_) => true, + Expression::Variable(v) | Expression::Bound(v) => { + !input_types.get(v).undef || entry_types.get(v) == VariableType::UNDEF + } + Expression::Or(inner) + | Expression::And(inner) + | Expression::Coalesce(inner) + | Expression::FunctionCall(_, inner) => inner + .iter() + .all(|e| is_expression_fit_for_for_loop_join(e, input_types, entry_types)), + Expression::Equal(a, b) + | Expression::SameTerm(a, b) + | Expression::Greater(a, b) + | Expression::GreaterOrEqual(a, b) + | Expression::Less(a, b) + | Expression::LessOrEqual(a, b) + | Expression::Add(a, b) + | Expression::Subtract(a, b) + | Expression::Multiply(a, b) + | Expression::Divide(a, b) => { + is_expression_fit_for_for_loop_join(a, input_types, entry_types) + && is_expression_fit_for_for_loop_join(b, input_types, entry_types) + } + Expression::UnaryPlus(e) | Expression::UnaryMinus(e) | Expression::Not(e) => { + is_expression_fit_for_for_loop_join(e, input_types, entry_types) + } + Expression::If(a, b, c) => { + is_expression_fit_for_for_loop_join(a, input_types, entry_types) + && is_expression_fit_for_for_loop_join(b, input_types, entry_types) + && is_expression_fit_for_for_loop_join(c, input_types, entry_types) + } + Expression::Exists(inner) => is_fit_for_for_loop_join(inner, input_types, entry_types), + } +} + +fn count_common_variables( + left: &VariableTypes, + right: &VariableTypes, + input_types: &VariableTypes, +) -> usize { + // TODO: we should be smart and count as shared variables FILTER(?a = ?b) + left.iter() + .filter(|(variable, left_type)| { + !left_type.undef && !right.get(variable).undef && input_types.get(variable).undef + }) + .count() +} + +fn estimate_graph_pattern_size(pattern: &GraphPattern, input_types: &VariableTypes) -> usize { + match pattern { + GraphPattern::Values { bindings, .. } => bindings.len(), + GraphPattern::QuadPattern { + subject, + predicate, + object, + .. + } => estimate_triple_pattern_size( + is_term_pattern_bound(subject, input_types), + is_named_node_pattern_bound(predicate, input_types), + is_term_pattern_bound(object, input_types), + ), + GraphPattern::Path { + subject, + path, + object, + .. + } => estimate_path_size( + is_term_pattern_bound(subject, input_types), + path, + is_term_pattern_bound(object, input_types), + ), + GraphPattern::Join { + left, + right, + algorithm, + } => { + let left_types = infer_graph_pattern_types(left, input_types.clone()); + let right_types = infer_graph_pattern_types(right, input_types.clone()); + estimate_join_cost( + left, + &left_types, + right, + &right_types, + *algorithm, + input_types, + ) + } + GraphPattern::LeftJoin { left, right, .. } => { + let left_size = estimate_graph_pattern_size(left, input_types); + let left_types = infer_graph_pattern_types(left, input_types.clone()); + let right_types = infer_graph_pattern_types(right, input_types.clone()); + max( + left_size, + left_size + .saturating_mul(estimate_graph_pattern_size(right, &right_types)) + .saturating_div( + 1_000_usize.saturating_pow( + count_common_variables(&left_types, &right_types, input_types) + .try_into() + .unwrap(), + ), + ), + ) + } + #[cfg(feature = "sep-0006")] + GraphPattern::Lateral { left, right } => estimate_lateral_cost( + left, + &infer_graph_pattern_types(left, input_types.clone()), + right, + input_types, + ), + GraphPattern::Union { inner } => inner + .iter() + .map(|inner| estimate_graph_pattern_size(inner, input_types)) + .fold(0, usize::saturating_add), + GraphPattern::Minus { left, .. } => estimate_graph_pattern_size(left, input_types), + GraphPattern::Filter { inner, .. } + | GraphPattern::Extend { inner, .. } + | GraphPattern::OrderBy { inner, .. } + | GraphPattern::Project { inner, .. } + | GraphPattern::Distinct { inner, .. } + | GraphPattern::Reduced { inner, .. } + | GraphPattern::Group { inner, .. } + | GraphPattern::Service { inner, .. } => estimate_graph_pattern_size(inner, input_types), + GraphPattern::Slice { + inner, + start, + length, + } => { + let inner = estimate_graph_pattern_size(inner, input_types); + if let Some(length) = length { + min(inner, *length - *start) + } else { + inner + } + } + } +} + +fn estimate_join_cost( + left: &GraphPattern, + left_types: &VariableTypes, + right: &GraphPattern, + right_types: &VariableTypes, + algorithm: JoinAlgorithm, + input_types: &VariableTypes, +) -> usize { + match algorithm { + JoinAlgorithm::HashBuildLeftProbeRight => estimate_graph_pattern_size(left, input_types) + .saturating_mul(estimate_graph_pattern_size(right, input_types)) + .saturating_div( + 1_000_usize.saturating_pow( + count_common_variables(left_types, right_types, input_types) + .try_into() + .unwrap(), + ), + ), + } +} +fn estimate_lateral_cost( + left: &GraphPattern, + left_types: &VariableTypes, + right: &GraphPattern, + input_types: &VariableTypes, +) -> usize { + estimate_graph_pattern_size(left, input_types) + .saturating_mul(estimate_graph_pattern_size(right, left_types)) +} + +fn estimate_triple_pattern_size( + subject_bound: bool, + predicate_bound: bool, + object_bound: bool, +) -> usize { + match (subject_bound, predicate_bound, object_bound) { + (true, true, true) => 1, + (true, true, false) => 10, + (true, false, true) => 2, + (false, true, true) => 10_000, + (true, false, false) => 100, + (false, false, false) => 1_000_000_000, + (false, true, false) => 1_000_000, + (false, false, true) => 100_000, + } +} + +fn estimate_path_size(start_bound: bool, path: &PropertyPathExpression, end_bound: bool) -> usize { + match path { + PropertyPathExpression::NamedNode(_) => { + estimate_triple_pattern_size(start_bound, true, end_bound) + } + PropertyPathExpression::Reverse(p) => estimate_path_size(end_bound, p, start_bound), + PropertyPathExpression::Sequence(a, b) => { + // We do a for loop join in the best direction + min( + estimate_path_size(start_bound, a, false) + .saturating_mul(estimate_path_size(true, b, end_bound)), + estimate_path_size(start_bound, a, true) + .saturating_mul(estimate_path_size(false, b, end_bound)), + ) + } + PropertyPathExpression::Alternative(a, b) => estimate_path_size(start_bound, a, end_bound) + .saturating_add(estimate_path_size(start_bound, b, end_bound)), + PropertyPathExpression::ZeroOrMore(p) => { + if start_bound && end_bound { + 1 + } else if start_bound || end_bound { + estimate_path_size(start_bound, p, end_bound).saturating_mul(1000) + } else { + 1_000_000_000 + } + } + PropertyPathExpression::OneOrMore(p) => { + if start_bound && end_bound { + 1 + } else { + estimate_path_size(start_bound, p, end_bound).saturating_mul(1000) + } + } + PropertyPathExpression::ZeroOrOne(p) => { + if start_bound && end_bound { + 1 + } else if start_bound || end_bound { + estimate_path_size(start_bound, p, end_bound) + } else { + 1_000_000_000 + } + } + PropertyPathExpression::NegatedPropertySet(_) => { + estimate_triple_pattern_size(start_bound, false, end_bound) + } + } +} + +fn is_term_pattern_bound(pattern: &GroundTermPattern, input_types: &VariableTypes) -> bool { + match pattern { + GroundTermPattern::NamedNode(_) | GroundTermPattern::Literal(_) => true, + GroundTermPattern::Variable(v) => !input_types.get(v).undef, + #[cfg(feature = "rdf-star")] + GroundTermPattern::Triple(t) => { + is_term_pattern_bound(&t.subject, input_types) + && is_named_node_pattern_bound(&t.predicate, input_types) + && is_term_pattern_bound(&t.object, input_types) + } + } +} + +fn is_named_node_pattern_bound(pattern: &NamedNodePattern, input_types: &VariableTypes) -> bool { + match pattern { + NamedNodePattern::NamedNode(_) => true, + NamedNodePattern::Variable(v) => !input_types.get(v).undef, + } +} diff --git a/lib/sparopt/src/type_inference.rs b/lib/sparopt/src/type_inference.rs new file mode 100644 index 00000000..421fd756 --- /dev/null +++ b/lib/sparopt/src/type_inference.rs @@ -0,0 +1,451 @@ +use crate::algebra::{Expression, GraphPattern}; +use oxrdf::Variable; +use spargebra::algebra::Function; +use spargebra::term::{GroundTerm, GroundTermPattern, NamedNodePattern}; +use std::collections::HashMap; +use std::ops::{BitAnd, BitOr}; + +pub fn infer_graph_pattern_types( + pattern: &GraphPattern, + mut types: VariableTypes, +) -> VariableTypes { + match pattern { + GraphPattern::QuadPattern { + subject, + predicate, + object, + graph_name, + } => { + add_ground_term_pattern_types(subject, &mut types, false); + if let NamedNodePattern::Variable(v) = predicate { + types.intersect_variable_with(v.clone(), VariableType::NAMED_NODE) + } + add_ground_term_pattern_types(object, &mut types, true); + if let Some(NamedNodePattern::Variable(v)) = graph_name { + types.intersect_variable_with(v.clone(), VariableType::NAMED_NODE) + } + types + } + GraphPattern::Path { + subject, + object, + graph_name, + .. + } => { + add_ground_term_pattern_types(subject, &mut types, false); + add_ground_term_pattern_types(object, &mut types, true); + if let Some(NamedNodePattern::Variable(v)) = graph_name { + types.intersect_variable_with(v.clone(), VariableType::NAMED_NODE) + } + types + } + GraphPattern::Join { left, right, .. } => { + let mut output_types = infer_graph_pattern_types(left, types.clone()); + output_types.intersect_with(infer_graph_pattern_types(right, types)); + output_types + } + #[cfg(feature = "sep-0006")] + GraphPattern::Lateral { left, right } => { + infer_graph_pattern_types(right, infer_graph_pattern_types(left, types)) + } + GraphPattern::LeftJoin { left, right, .. } => { + let mut right_types = infer_graph_pattern_types(right, types.clone()); //TODO: expression + for t in right_types.inner.values_mut() { + t.undef = true; // Right might be unset + } + let mut output_types = infer_graph_pattern_types(left, types); + output_types.intersect_with(right_types); + output_types + } + GraphPattern::Minus { left, .. } => infer_graph_pattern_types(left, types), + GraphPattern::Union { inner } => inner + .iter() + .map(|inner| infer_graph_pattern_types(inner, types.clone())) + .reduce(|mut a, b| { + a.union_with(b); + a + }) + .unwrap_or_default(), + GraphPattern::Extend { + inner, + variable, + expression, + } => { + let mut types = infer_graph_pattern_types(inner, types); + types.intersect_variable_with( + variable.clone(), + infer_expression_type(expression, &types), + ); + types + } + GraphPattern::Filter { inner, .. } => infer_graph_pattern_types(inner, types), + GraphPattern::Project { inner, variables } => VariableTypes { + inner: infer_graph_pattern_types(inner, types) + .inner + .into_iter() + .filter(|(v, _)| variables.contains(v)) + .collect(), + }, + GraphPattern::Distinct { inner } + | GraphPattern::Reduced { inner } + | GraphPattern::OrderBy { inner, .. } + | GraphPattern::Slice { inner, .. } => infer_graph_pattern_types(inner, types), + GraphPattern::Group { + inner, + variables, + aggregates, + } => { + let types = infer_graph_pattern_types(inner, types); + VariableTypes { + inner: infer_graph_pattern_types(inner, types) + .inner + .into_iter() + .filter(|(v, _)| variables.contains(v)) + .chain(aggregates.iter().map(|(v, _)| (v.clone(), VariableType::ANY))) //TODO: guess from aggregate + .collect(), + } + } + GraphPattern::Values { + variables, + bindings, + } => { + for (i, v) in variables.iter().enumerate() { + let mut t = VariableType::default(); + for binding in bindings { + match binding[i] { + Some(GroundTerm::NamedNode(_)) => t.named_node = true, + Some(GroundTerm::Literal(_)) => t.literal = true, + #[cfg(feature = "rdf-star")] + Some(GroundTerm::Triple(_)) => t.triple = true, + None => t.undef = true, + } + } + types.intersect_variable_with(v.clone(), t) + } + types + } + GraphPattern::Service { name, inner, .. } => { + let mut types = infer_graph_pattern_types(inner, types); + if let NamedNodePattern::Variable(v) = name { + types.intersect_variable_with(v.clone(), VariableType::NAMED_NODE) + } + types + } + } +} + +fn add_ground_term_pattern_types( + pattern: &GroundTermPattern, + types: &mut VariableTypes, + is_object: bool, +) { + if let GroundTermPattern::Variable(v) = pattern { + types.intersect_variable_with( + v.clone(), + if is_object { + VariableType::TERM + } else { + VariableType::SUBJECT + }, + ) + } + #[cfg(feature = "rdf-star")] + if let GroundTermPattern::Triple(t) = pattern { + add_ground_term_pattern_types(&t.subject, types, false); + if let NamedNodePattern::Variable(v) = &t.predicate { + types.intersect_variable_with(v.clone(), VariableType::NAMED_NODE) + } + add_ground_term_pattern_types(&t.object, types, true); + } +} + +pub fn infer_expression_type(expression: &Expression, types: &VariableTypes) -> VariableType { + match expression { + Expression::NamedNode(_) => VariableType::NAMED_NODE, + Expression::Literal(_) | Expression::Exists(_) | Expression::Bound(_) => { + VariableType::LITERAL + } + Expression::Variable(v) => types.get(v), + Expression::FunctionCall(Function::Datatype | Function::Iri, _) => { + VariableType::NAMED_NODE | VariableType::UNDEF + } + #[cfg(feature = "rdf-star")] + Expression::FunctionCall(Function::Predicate, _) => { + VariableType::NAMED_NODE | VariableType::UNDEF + } + Expression::FunctionCall(Function::BNode, _) => { + VariableType::BLANK_NODE | VariableType::UNDEF + } + Expression::Or(_) + | Expression::And(_) + | Expression::Equal(_, _) + | Expression::Greater(_, _) + | Expression::GreaterOrEqual(_, _) + | Expression::Less(_, _) + | Expression::LessOrEqual(_, _) + | Expression::Add(_, _) + | Expression::Subtract(_, _) + | Expression::Multiply(_, _) + | Expression::Divide(_, _) + | Expression::UnaryPlus(_) + | Expression::UnaryMinus(_) + | Expression::Not(_) + | Expression::FunctionCall( + Function::Str + | Function::Lang + | Function::LangMatches + | Function::Rand + | Function::Abs + | Function::Ceil + | Function::Floor + | Function::Round + | Function::Concat + | Function::SubStr + | Function::StrLen + | Function::Replace + | Function::UCase + | Function::LCase + | Function::EncodeForUri + | Function::Contains + | Function::StrStarts + | Function::StrEnds + | Function::StrBefore + | Function::StrAfter + | Function::Year + | Function::Month + | Function::Day + | Function::Hours + | Function::Minutes + | Function::Seconds + | Function::Timezone + | Function::Tz + | Function::Now + | Function::Uuid + | Function::StrUuid + | Function::Md5 + | Function::Sha1 + | Function::Sha256 + | Function::Sha384 + | Function::Sha512 + | Function::StrLang + | Function::StrDt + | Function::IsIri + | Function::IsBlank + | Function::IsLiteral + | Function::IsNumeric + | Function::Regex, + _, + ) => VariableType::LITERAL | VariableType::UNDEF, + #[cfg(feature = "sep-0002")] + Expression::FunctionCall(Function::Adjust, _) => { + VariableType::LITERAL | VariableType::UNDEF + } + #[cfg(feature = "rdf-star")] + Expression::FunctionCall(Function::IsTriple, _) => { + VariableType::LITERAL | VariableType::UNDEF + } + Expression::SameTerm(left, right) => { + if infer_expression_type(left, types).undef || infer_expression_type(right, types).undef + { + VariableType::LITERAL | VariableType::UNDEF + } else { + VariableType::LITERAL + } + } + Expression::If(_, then, els) => { + infer_expression_type(then, types) | infer_expression_type(els, types) + } + Expression::Coalesce(inner) => { + let mut t = VariableType::UNDEF; + for e in inner { + let new = infer_expression_type(e, types); + t = t | new; + if !new.undef { + t.undef = false; + return t; + } + } + t + } + #[cfg(feature = "rdf-star")] + Expression::FunctionCall(Function::Triple, _) => VariableType::TRIPLE | VariableType::UNDEF, + #[cfg(feature = "rdf-star")] + Expression::FunctionCall(Function::Subject, _) => { + VariableType::SUBJECT | VariableType::UNDEF + } + #[cfg(feature = "rdf-star")] + Expression::FunctionCall(Function::Object, _) => VariableType::TERM | VariableType::UNDEF, + Expression::FunctionCall(Function::Custom(_), _) => VariableType::ANY, + } +} + +#[derive(Default, Clone, Debug)] +pub struct VariableTypes { + inner: HashMap, +} + +impl VariableTypes { + pub fn get(&self, variable: &Variable) -> VariableType { + self.inner + .get(variable) + .copied() + .unwrap_or(VariableType::UNDEF) + } + + pub fn iter(&self) -> impl Iterator { + self.inner.iter() + } + + pub fn intersect_with(&mut self, other: Self) { + for (v, t) in other.inner { + self.intersect_variable_with(v, t); + } + } + + pub fn union_with(&mut self, other: Self) { + for (v, t) in &mut self.inner { + if other.get(v).undef { + t.undef = true; // Might be undefined + } + } + for (v, mut t) in other.inner { + self.inner + .entry(v) + .and_modify(|ex| *ex = *ex | t) + .or_insert({ + t.undef = true; + t + }); + } + } + + fn intersect_variable_with(&mut self, variable: Variable, t: VariableType) { + let t = self.get(&variable) & t; + if t != VariableType::UNDEF { + self.inner.insert(variable, t); + } + } +} + +#[derive(Clone, Copy, Eq, PartialEq, Debug, Default)] +pub struct VariableType { + pub undef: bool, + pub named_node: bool, + pub blank_node: bool, + pub literal: bool, + #[cfg(feature = "rdf-star")] + pub triple: bool, +} + +impl VariableType { + pub const UNDEF: Self = Self { + undef: true, + named_node: false, + blank_node: false, + literal: false, + #[cfg(feature = "rdf-star")] + triple: false, + }; + + const NAMED_NODE: Self = Self { + undef: false, + named_node: true, + blank_node: false, + literal: false, + #[cfg(feature = "rdf-star")] + triple: false, + }; + + const BLANK_NODE: Self = Self { + undef: false, + named_node: false, + blank_node: true, + literal: false, + #[cfg(feature = "rdf-star")] + triple: false, + }; + + const LITERAL: Self = Self { + undef: false, + named_node: false, + blank_node: false, + literal: true, + #[cfg(feature = "rdf-star")] + triple: false, + }; + + #[cfg(feature = "rdf-star")] + const TRIPLE: Self = Self { + undef: false, + named_node: false, + blank_node: false, + literal: false, + triple: true, + }; + + const SUBJECT: Self = Self { + undef: false, + named_node: true, + blank_node: true, + literal: false, + #[cfg(feature = "rdf-star")] + triple: true, + }; + + const TERM: Self = Self { + undef: false, + named_node: true, + blank_node: true, + literal: true, + #[cfg(feature = "rdf-star")] + triple: true, + }; + + const ANY: Self = Self { + undef: true, + named_node: true, + blank_node: true, + literal: true, + #[cfg(feature = "rdf-star")] + triple: true, + }; +} + +impl BitOr for VariableType { + type Output = Self; + + fn bitor(self, other: Self) -> Self { + Self { + undef: self.undef || other.undef, + named_node: self.named_node || other.named_node, + blank_node: self.blank_node || other.blank_node, + literal: self.literal || other.literal, + #[cfg(feature = "rdf-star")] + triple: self.triple || other.triple, + } + } +} + +impl BitAnd for VariableType { + type Output = Self; + + #[allow(clippy::nonminimal_bool)] + fn bitand(self, other: Self) -> Self { + Self { + undef: self.undef && other.undef, + named_node: self.named_node && other.named_node + || (self.undef && other.named_node) + || (self.named_node && other.undef), + blank_node: self.blank_node && other.blank_node + || (self.undef && other.blank_node) + || (self.blank_node && other.undef), + literal: self.literal && other.literal + || (self.undef && other.literal) + || (self.literal && other.undef), + #[cfg(feature = "rdf-star")] + triple: self.triple && other.triple + || (self.undef && other.triple) + || (self.triple && other.undef), + } + } +} diff --git a/lib/src/sparql/eval.rs b/lib/src/sparql/eval.rs index f4620f56..f9a8690b 100644 --- a/lib/src/sparql/eval.rs +++ b/lib/src/sparql/eval.rs @@ -379,21 +379,24 @@ impl SimpleEvaluator { } }) } - PlanNode::HashJoin { left, right } => { - let join_keys: Vec<_> = left + PlanNode::HashJoin { + probe_child, + build_child, + } => { + let join_keys: Vec<_> = probe_child .always_bound_variables() - .intersection(&right.always_bound_variables()) + .intersection(&build_child.always_bound_variables()) .copied() .collect(); - let (left, left_stats) = self.plan_evaluator(Rc::clone(left)); - stat_children.push(left_stats); - let (right, right_stats) = self.plan_evaluator(Rc::clone(right)); - stat_children.push(right_stats); + let (probe, probe_stats) = self.plan_evaluator(Rc::clone(probe_child)); + stat_children.push(probe_stats); + let (build, build_stats) = self.plan_evaluator(Rc::clone(build_child)); + stat_children.push(build_stats); if join_keys.is_empty() { // Cartesian product Rc::new(move |from| { let mut errors = Vec::default(); - let right_values = right(from.clone()) + let build_values = build(from.clone()) .filter_map(|result| match result { Ok(result) => Some(result), Err(error) => { @@ -403,8 +406,8 @@ impl SimpleEvaluator { }) .collect::>(); Box::new(CartesianProductJoinIterator { - left_iter: left(from), - right: right_values, + probe_iter: probe(from), + built: build_values, buffered_results: errors, }) }) @@ -412,8 +415,8 @@ impl SimpleEvaluator { // Real hash join Rc::new(move |from| { let mut errors = Vec::default(); - let mut right_values = EncodedTupleSet::new(join_keys.clone()); - right_values.extend(right(from.clone()).filter_map( + let mut built_values = EncodedTupleSet::new(join_keys.clone()); + built_values.extend(build(from.clone()).filter_map( |result| match result { Ok(result) => Some(result), Err(error) => { @@ -423,8 +426,8 @@ impl SimpleEvaluator { }, )); Box::new(HashJoinIterator { - left_iter: left(from), - right: right_values, + probe_iter: probe(from), + built: built_values, buffered_results: errors, }) }) @@ -516,33 +519,17 @@ impl SimpleEvaluator { }) }) } - PlanNode::ForLoopLeftJoin { - left, - right, - possible_problem_vars, - } => { + PlanNode::ForLoopLeftJoin { left, right } => { let (left, left_stats) = self.plan_evaluator(Rc::clone(left)); stat_children.push(left_stats); let (right, right_stats) = self.plan_evaluator(Rc::clone(right)); stat_children.push(right_stats); - let possible_problem_vars = Rc::clone(possible_problem_vars); Rc::new(move |from| { - if possible_problem_vars.is_empty() { - Box::new(ForLoopLeftJoinIterator { - right_evaluator: Rc::clone(&right), - left_iter: left(from), - current_right: Box::new(empty()), - }) - } else { - Box::new(BadForLoopLeftJoinIterator { - from_tuple: from.clone(), - right_evaluator: Rc::clone(&right), - left_iter: left(from), - current_left: EncodedTuple::with_capacity(0), - current_right: Box::new(empty()), - problem_vars: Rc::clone(&possible_problem_vars), - }) - } + Box::new(ForLoopLeftJoinIterator { + right_evaluator: Rc::clone(&right), + left_iter: left(from), + current_right: Box::new(empty()), + }) }) } PlanNode::Filter { child, expression } => { @@ -3887,8 +3874,8 @@ impl PathEvaluator { } struct CartesianProductJoinIterator { - left_iter: EncodedTuplesIterator, - right: Vec, + probe_iter: EncodedTuplesIterator, + built: Vec, buffered_results: Vec>, } @@ -3900,12 +3887,12 @@ impl Iterator for CartesianProductJoinIterator { if let Some(result) = self.buffered_results.pop() { return Some(result); } - let left_tuple = match self.left_iter.next()? { - Ok(left_tuple) => left_tuple, + let probe_tuple = match self.probe_iter.next()? { + Ok(probe_tuple) => probe_tuple, Err(error) => return Some(Err(error)), }; - for right_tuple in &self.right { - if let Some(result_tuple) = left_tuple.combine_with(right_tuple) { + for built_tuple in &self.built { + if let Some(result_tuple) = probe_tuple.combine_with(built_tuple) { self.buffered_results.push(Ok(result_tuple)) } } @@ -3913,17 +3900,17 @@ impl Iterator for CartesianProductJoinIterator { } fn size_hint(&self) -> (usize, Option) { - let (min, max) = self.left_iter.size_hint(); + let (min, max) = self.probe_iter.size_hint(); ( - min.saturating_mul(self.right.len()), - max.map(|v| v.saturating_mul(self.right.len())), + min.saturating_mul(self.built.len()), + max.map(|v| v.saturating_mul(self.built.len())), ) } } struct HashJoinIterator { - left_iter: EncodedTuplesIterator, - right: EncodedTupleSet, + probe_iter: EncodedTuplesIterator, + built: EncodedTupleSet, buffered_results: Vec>, } @@ -3935,15 +3922,15 @@ impl Iterator for HashJoinIterator { if let Some(result) = self.buffered_results.pop() { return Some(result); } - let left_tuple = match self.left_iter.next()? { - Ok(left_tuple) => left_tuple, + let probe_tuple = match self.probe_iter.next()? { + Ok(probe_tuple) => probe_tuple, Err(error) => return Some(Err(error)), }; self.buffered_results.extend( - self.right - .get(&left_tuple) + self.built + .get(&probe_tuple) .iter() - .filter_map(|right_tuple| left_tuple.combine_with(right_tuple).map(Ok)), + .filter_map(|built_tuple| probe_tuple.combine_with(built_tuple).map(Ok)), ) } } @@ -3951,10 +3938,10 @@ impl Iterator for HashJoinIterator { fn size_hint(&self) -> (usize, Option) { ( 0, - self.left_iter + self.probe_iter .size_hint() .1 - .map(|v| v.saturating_mul(self.right.len())), + .map(|v| v.saturating_mul(self.built.len())), ) } } @@ -4034,58 +4021,6 @@ impl Iterator for ForLoopLeftJoinIterator { } } -struct BadForLoopLeftJoinIterator { - from_tuple: EncodedTuple, - right_evaluator: Rc EncodedTuplesIterator>, - left_iter: EncodedTuplesIterator, - current_left: EncodedTuple, - current_right: EncodedTuplesIterator, - problem_vars: Rc<[usize]>, -} - -impl Iterator for BadForLoopLeftJoinIterator { - type Item = Result; - - fn next(&mut self) -> Option> { - for right_tuple in &mut self.current_right { - match right_tuple { - Ok(right_tuple) => { - if let Some(combined) = right_tuple.combine_with(&self.current_left) { - return Some(Ok(combined)); - } - } - Err(error) => return Some(Err(error)), - } - } - match self.left_iter.next()? { - Ok(left_tuple) => { - let mut right_input = self.from_tuple.clone(); - for (var, val) in left_tuple.iter().enumerate() { - if let Some(val) = val { - if !self.problem_vars.contains(&var) { - right_input.set(var, val); - } - } - } - self.current_right = (self.right_evaluator)(right_input); - for right_tuple in &mut self.current_right { - match right_tuple { - Ok(right_tuple) => { - if let Some(combined) = right_tuple.combine_with(&left_tuple) { - self.current_left = left_tuple; - return Some(Ok(combined)); - } - } - Err(error) => return Some(Err(error)), - } - } - Some(Ok(left_tuple)) - } - Err(error) => Some(Err(error)), - } - } -} - struct UnionIterator { plans: Vec EncodedTuplesIterator>>, input: EncodedTuple, diff --git a/lib/src/sparql/mod.rs b/lib/src/sparql/mod.rs index 93fda294..9612d514 100644 --- a/lib/src/sparql/mod.rs +++ b/lib/src/sparql/mod.rs @@ -105,7 +105,6 @@ pub(crate) fn evaluate_query( &template, variables, &options.custom_functions, - options.without_optimizations, ); let planning_duration = start_planning.elapsed(); let (results, explanation) = SimpleEvaluator::new( diff --git a/lib/src/sparql/plan.rs b/lib/src/sparql/plan.rs index 88469433..af51e7a4 100644 --- a/lib/src/sparql/plan.rs +++ b/lib/src/sparql/plan.rs @@ -41,8 +41,8 @@ pub enum PlanNode { }, /// Streams left and materializes right join HashJoin { - left: Rc, - right: Rc, + probe_child: Rc, + build_child: Rc, }, /// Right nested in left loop ForLoopJoin { @@ -71,7 +71,6 @@ pub enum PlanNode { ForLoopLeftJoin { left: Rc, right: Rc, - possible_problem_vars: Rc<[usize]>, //Variables that should not be part of the entry of the left join }, Extend { child: Rc, @@ -160,7 +159,10 @@ impl PlanNode { child.lookup_used_variables(callback); } } - Self::HashJoin { left, right } + Self::HashJoin { + probe_child: left, + build_child: right, + } | Self::ForLoopJoin { left, right, .. } | Self::AntiJoin { left, right } | Self::ForLoopLeftJoin { left, right, .. } => { @@ -296,7 +298,11 @@ impl PlanNode { } } } - Self::HashJoin { left, right } | Self::ForLoopJoin { left, right, .. } => { + Self::HashJoin { + probe_child: left, + build_child: right, + } + | Self::ForLoopJoin { left, right, .. } => { left.lookup_always_bound_variables(callback); right.lookup_always_bound_variables(callback); } @@ -344,16 +350,6 @@ impl PlanNode { } } } - - pub fn is_variable_bound(&self, variable: usize) -> bool { - let mut found = false; - self.lookup_always_bound_variables(&mut |v| { - if v == variable { - found = true; - } - }); - found - } } #[derive(Debug, Clone)] diff --git a/lib/src/sparql/plan_builder.rs b/lib/src/sparql/plan_builder.rs index 7d985504..ec93bb7f 100644 --- a/lib/src/sparql/plan_builder.rs +++ b/lib/src/sparql/plan_builder.rs @@ -5,43 +5,37 @@ use crate::sparql::eval::compile_pattern; use crate::sparql::plan::*; use crate::storage::numeric_encoder::{EncodedTerm, EncodedTriple}; use oxrdf::vocab::xsd; -use oxrdf::TermRef; -use rand::random; +use oxrdf::{BlankNode, Term, TermRef, Triple}; use regex::Regex; -use spargebra::algebra::*; -use spargebra::term::*; -use std::collections::{BTreeSet, HashMap, HashSet}; -use std::mem::swap; +use spargebra::term::{GroundSubject, GroundTriple, TermPattern, TriplePattern}; +use sparopt::algebra::*; +use sparopt::Optimizer; +use std::collections::HashMap; use std::rc::Rc; pub struct PlanBuilder<'a> { dataset: &'a DatasetView, custom_functions: &'a HashMap Option>>, - with_optimizations: bool, } impl<'a> PlanBuilder<'a> { pub fn build( dataset: &'a DatasetView, - pattern: &GraphPattern, + pattern: &spargebra::algebra::GraphPattern, is_cardinality_meaningful: bool, custom_functions: &'a HashMap Option>>, without_optimizations: bool, ) -> Result<(PlanNode, Vec), EvaluationError> { + let mut pattern = GraphPattern::from(pattern); + if !without_optimizations { + pattern = Optimizer::optimize_graph_pattern(pattern); + } let mut variables = Vec::default(); let plan = PlanBuilder { dataset, custom_functions, - with_optimizations: !without_optimizations, } - .build_for_graph_pattern( - pattern, - &mut variables, - &PatternValue::Constant(PlanTerm { - encoded: EncodedTerm::DefaultGraph, - plain: PatternValueConstant::DefaultGraph, - }), - )?; + .build_for_graph_pattern(&pattern, &mut variables)?; let plan = if !without_optimizations && !is_cardinality_meaningful { // let's reduce downstream task. // TODO: avoid if already REDUCED or DISTINCT @@ -59,12 +53,10 @@ impl<'a> PlanBuilder<'a> { template: &[TriplePattern], mut variables: Vec, custom_functions: &'a HashMap Option>>, - without_optimizations: bool, ) -> Vec { PlanBuilder { dataset, custom_functions, - with_optimizations: !without_optimizations, } .build_for_graph_template(template, &mut variables) } @@ -73,112 +65,111 @@ impl<'a> PlanBuilder<'a> { &self, pattern: &GraphPattern, variables: &mut Vec, - graph_name: &PatternValue, ) -> Result { Ok(match pattern { - GraphPattern::Bgp { patterns } => { - if self.with_optimizations { - self.build_for_bgp(sort_bgp(patterns), variables, graph_name) - } else { - self.build_for_bgp(patterns, variables, graph_name) - } - } + GraphPattern::QuadPattern { + subject, + predicate, + object, + graph_name, + } => PlanNode::QuadPattern { + subject: self.pattern_value_from_ground_term_pattern(subject, variables), + predicate: self.pattern_value_from_named_node_or_variable(predicate, variables), + object: self.pattern_value_from_ground_term_pattern(object, variables), + graph_name: graph_name.as_ref().map_or( + PatternValue::Constant(PlanTerm { + encoded: EncodedTerm::DefaultGraph, + plain: PatternValueConstant::DefaultGraph, + }), + |g| self.pattern_value_from_named_node_or_variable(g, variables), + ), + }, GraphPattern::Path { subject, path, object, + graph_name, } => PlanNode::PathPattern { - subject: self.pattern_value_from_term_or_variable(subject, variables), + subject: self.pattern_value_from_ground_term_pattern(subject, variables), path: Rc::new(self.build_for_path(path)), - object: self.pattern_value_from_term_or_variable(object, variables), - graph_name: graph_name.clone(), + object: self.pattern_value_from_ground_term_pattern(object, variables), + graph_name: graph_name.as_ref().map_or( + PatternValue::Constant(PlanTerm { + encoded: EncodedTerm::DefaultGraph, + plain: PatternValueConstant::DefaultGraph, + }), + |g| self.pattern_value_from_named_node_or_variable(g, variables), + ), + }, + GraphPattern::Join { + left, + right, + algorithm, + } => match algorithm { + JoinAlgorithm::HashBuildLeftProbeRight => PlanNode::HashJoin { + build_child: Rc::new(self.build_for_graph_pattern(left, variables)?), + probe_child: Rc::new(self.build_for_graph_pattern(right, variables)?), + }, }, - GraphPattern::Join { left, right } => self.new_join( - self.build_for_graph_pattern(left, variables, graph_name)?, - self.build_for_graph_pattern(right, variables, graph_name)?, - ), GraphPattern::LeftJoin { left, right, expression, - } => { - let left = self.build_for_graph_pattern(left, variables, graph_name)?; - let right = self.build_for_graph_pattern(right, variables, graph_name)?; - - if self.with_optimizations && Self::can_use_for_loop_left_join(&right) { - let mut possible_problem_vars = BTreeSet::new(); - Self::add_left_join_problematic_variables(&right, &mut possible_problem_vars); - - //We add the extra filter if needed - let right = if let Some(expr) = expression { - self.push_filter( - Rc::new(right), - Box::new(self.build_for_expression(expr, variables, graph_name)?), - ) + } => PlanNode::HashLeftJoin { + left: Rc::new(self.build_for_graph_pattern(left, variables)?), + right: Rc::new(self.build_for_graph_pattern(right, variables)?), + expression: Box::new(self.build_for_expression(expression, variables)?), + }, + GraphPattern::Lateral { left, right } => { + if let GraphPattern::LeftJoin { + left: nested_left, + right: nested_right, + expression, + } = right.as_ref() + { + if nested_left.is_empty_singleton() { + // We are in a ForLoopLeftJoin + let right = + GraphPattern::filter(nested_right.as_ref().clone(), expression.clone()); + PlanNode::ForLoopLeftJoin { + left: Rc::new(self.build_for_graph_pattern(left, variables)?), + right: Rc::new(self.build_for_graph_pattern(&right, variables)?), + } } else { - right - }; - PlanNode::ForLoopLeftJoin { - left: Rc::new(left), - right: Rc::new(right), - possible_problem_vars: possible_problem_vars.into_iter().collect(), + PlanNode::ForLoopJoin { + left: Rc::new(self.build_for_graph_pattern(left, variables)?), + right: Rc::new(self.build_for_graph_pattern(right, variables)?), + } } } else { - PlanNode::HashLeftJoin { - left: Rc::new(left), - right: Rc::new(right), - expression: Box::new(expression.as_ref().map_or( - Ok(PlanExpression::Literal(PlanTerm { - encoded: true.into(), - plain: true.into(), - })), - |e| self.build_for_expression(e, variables, graph_name), - )?), + PlanNode::ForLoopJoin { + left: Rc::new(self.build_for_graph_pattern(left, variables)?), + right: Rc::new(self.build_for_graph_pattern(right, variables)?), } } } - GraphPattern::Lateral { left, right } => PlanNode::ForLoopJoin { - left: Rc::new(self.build_for_graph_pattern(left, variables, graph_name)?), - right: Rc::new(self.build_for_graph_pattern(right, variables, graph_name)?), + GraphPattern::Filter { expression, inner } => PlanNode::Filter { + child: Rc::new(self.build_for_graph_pattern(inner, variables)?), + expression: Box::new(self.build_for_expression(expression, variables)?), + }, + GraphPattern::Union { inner } => PlanNode::Union { + children: inner + .iter() + .map(|p| Ok(Rc::new(self.build_for_graph_pattern(p, variables)?))) + .collect::>()?, }, - GraphPattern::Filter { expr, inner } => self.push_filter( - Rc::new(self.build_for_graph_pattern(inner, variables, graph_name)?), - Box::new(self.build_for_expression(expr, variables, graph_name)?), - ), - GraphPattern::Union { left, right } => { - //We flatten the UNION - let mut stack: Vec<&GraphPattern> = vec![left, right]; - let mut children = vec![]; - loop { - match stack.pop() { - None => break, - Some(GraphPattern::Union { left, right }) => { - stack.push(left); - stack.push(right); - } - Some(p) => children.push(Rc::new( - self.build_for_graph_pattern(p, variables, graph_name)?, - )), - } - } - PlanNode::Union { children } - } - GraphPattern::Graph { name, inner } => { - let graph_name = self.pattern_value_from_named_node_or_variable(name, variables); - self.build_for_graph_pattern(inner, variables, &graph_name)? - } GraphPattern::Extend { inner, variable, expression, } => PlanNode::Extend { - child: Rc::new(self.build_for_graph_pattern(inner, variables, graph_name)?), + child: Rc::new(self.build_for_graph_pattern(inner, variables)?), variable: build_plan_variable(variables, variable), - expression: Box::new(self.build_for_expression(expression, variables, graph_name)?), + expression: Box::new(self.build_for_expression(expression, variables)?), }, GraphPattern::Minus { left, right } => PlanNode::AntiJoin { - left: Rc::new(self.build_for_graph_pattern(left, variables, graph_name)?), - right: Rc::new(self.build_for_graph_pattern(right, variables, graph_name)?), + left: Rc::new(self.build_for_graph_pattern(left, variables)?), + right: Rc::new(self.build_for_graph_pattern(right, variables)?), }, GraphPattern::Service { name, @@ -186,13 +177,13 @@ impl<'a> PlanBuilder<'a> { silent, } => { // Child building should be at the begging in order for `variables` to be filled - let child = self.build_for_graph_pattern(inner, variables, graph_name)?; + let child = self.build_for_graph_pattern(inner, variables)?; let service_name = self.pattern_value_from_named_node_or_variable(name, variables); PlanNode::Service { service_name, variables: Rc::from(variables.as_slice()), child: Rc::new(child), - graph_pattern: Rc::new(inner.as_ref().clone()), + graph_pattern: Rc::new(inner.as_ref().into()), silent: *silent, } } @@ -201,7 +192,7 @@ impl<'a> PlanBuilder<'a> { variables: by, aggregates, } => PlanNode::Aggregate { - child: Rc::new(self.build_for_graph_pattern(inner, variables, graph_name)?), + child: Rc::new(self.build_for_graph_pattern(inner, variables)?), key_variables: by .iter() .map(|k| build_plan_variable(variables, k)) @@ -210,7 +201,7 @@ impl<'a> PlanBuilder<'a> { .iter() .map(|(v, a)| { Ok(( - self.build_for_aggregate(a, variables, graph_name)?, + self.build_for_aggregate(a, variables)?, build_plan_variable(variables, v), )) }) @@ -253,16 +244,16 @@ impl<'a> PlanBuilder<'a> { let condition: Result, EvaluationError> = expression .iter() .map(|comp| match comp { - OrderExpression::Asc(e) => Ok(Comparator::Asc( - self.build_for_expression(e, variables, graph_name)?, - )), - OrderExpression::Desc(e) => Ok(Comparator::Desc( - self.build_for_expression(e, variables, graph_name)?, - )), + OrderExpression::Asc(e) => { + Ok(Comparator::Asc(self.build_for_expression(e, variables)?)) + } + OrderExpression::Desc(e) => { + Ok(Comparator::Desc(self.build_for_expression(e, variables)?)) + } }) .collect(); PlanNode::Sort { - child: Rc::new(self.build_for_graph_pattern(inner, variables, graph_name)?), + child: Rc::new(self.build_for_graph_pattern(inner, variables)?), by: condition?, } } @@ -271,14 +262,8 @@ impl<'a> PlanBuilder<'a> { variables: projection, } => { let mut inner_variables = projection.clone(); - let inner_graph_name = - Self::convert_pattern_value_id(graph_name, &mut inner_variables); PlanNode::Project { - child: Rc::new(self.build_for_graph_pattern( - inner, - &mut inner_variables, - &inner_graph_name, - )?), + child: Rc::new(self.build_for_graph_pattern(inner, &mut inner_variables)?), mapping: projection .iter() .enumerate() @@ -295,17 +280,17 @@ impl<'a> PlanBuilder<'a> { } } GraphPattern::Distinct { inner } => PlanNode::HashDeduplicate { - child: Rc::new(self.build_for_graph_pattern(inner, variables, graph_name)?), + child: Rc::new(self.build_for_graph_pattern(inner, variables)?), }, GraphPattern::Reduced { inner } => PlanNode::Reduced { - child: Rc::new(self.build_for_graph_pattern(inner, variables, graph_name)?), + child: Rc::new(self.build_for_graph_pattern(inner, variables)?), }, GraphPattern::Slice { inner, start, length, } => { - let mut plan = self.build_for_graph_pattern(inner, variables, graph_name)?; + let mut plan = self.build_for_graph_pattern(inner, variables)?; if *start > 0 { plan = PlanNode::Skip { child: Rc::new(plan), @@ -323,29 +308,6 @@ impl<'a> PlanBuilder<'a> { }) } - fn build_for_bgp<'b>( - &self, - patterns: impl IntoIterator, - variables: &mut Vec, - graph_name: &PatternValue, - ) -> PlanNode { - patterns - .into_iter() - .map(|triple| PlanNode::QuadPattern { - subject: self.pattern_value_from_term_or_variable(&triple.subject, variables), - predicate: self - .pattern_value_from_named_node_or_variable(&triple.predicate, variables), - object: self.pattern_value_from_term_or_variable(&triple.object, variables), - graph_name: graph_name.clone(), - }) - .reduce(|a, b| self.new_join(a, b)) - .unwrap_or_else(|| PlanNode::StaticBindings { - encoded_tuples: vec![EncodedTuple::with_capacity(variables.len())], - variables: Vec::new(), - plain_bindings: vec![Vec::new()], - }) - } - fn build_for_path(&self, path: &PropertyPathExpression) -> PlanPropertyPath { match path { PropertyPathExpression::NamedNode(p) => PlanPropertyPath::Path(PlanTerm { @@ -387,7 +349,6 @@ impl<'a> PlanBuilder<'a> { &self, expression: &Expression, variables: &mut Vec, - graph_name: &PatternValue, ) -> Result { Ok(match expression { Expression::NamedNode(node) => PlanExpression::NamedNode(PlanTerm { @@ -399,382 +360,270 @@ impl<'a> PlanBuilder<'a> { plain: l.clone(), }), Expression::Variable(v) => PlanExpression::Variable(build_plan_variable(variables, v)), - Expression::Or(a, b) => PlanExpression::Or(vec![ - self.build_for_expression(a, variables, graph_name)?, - self.build_for_expression(b, variables, graph_name)?, - ]), - Expression::And(a, b) => PlanExpression::And(vec![ - self.build_for_expression(a, variables, graph_name)?, - self.build_for_expression(b, variables, graph_name)?, - ]), + Expression::Or(inner) => PlanExpression::Or( + inner + .iter() + .map(|e| self.build_for_expression(e, variables)) + .collect::>()?, + ), + Expression::And(inner) => PlanExpression::And( + inner + .iter() + .map(|e| self.build_for_expression(e, variables)) + .collect::>()?, + ), Expression::Equal(a, b) => PlanExpression::Equal( - Box::new(self.build_for_expression(a, variables, graph_name)?), - Box::new(self.build_for_expression(b, variables, graph_name)?), + Box::new(self.build_for_expression(a, variables)?), + Box::new(self.build_for_expression(b, variables)?), ), Expression::SameTerm(a, b) => PlanExpression::SameTerm( - Box::new(self.build_for_expression(a, variables, graph_name)?), - Box::new(self.build_for_expression(b, variables, graph_name)?), + Box::new(self.build_for_expression(a, variables)?), + Box::new(self.build_for_expression(b, variables)?), ), Expression::Greater(a, b) => PlanExpression::Greater( - Box::new(self.build_for_expression(a, variables, graph_name)?), - Box::new(self.build_for_expression(b, variables, graph_name)?), + Box::new(self.build_for_expression(a, variables)?), + Box::new(self.build_for_expression(b, variables)?), ), Expression::GreaterOrEqual(a, b) => PlanExpression::GreaterOrEqual( - Box::new(self.build_for_expression(a, variables, graph_name)?), - Box::new(self.build_for_expression(b, variables, graph_name)?), + Box::new(self.build_for_expression(a, variables)?), + Box::new(self.build_for_expression(b, variables)?), ), Expression::Less(a, b) => PlanExpression::Less( - Box::new(self.build_for_expression(a, variables, graph_name)?), - Box::new(self.build_for_expression(b, variables, graph_name)?), + Box::new(self.build_for_expression(a, variables)?), + Box::new(self.build_for_expression(b, variables)?), ), Expression::LessOrEqual(a, b) => PlanExpression::LessOrEqual( - Box::new(self.build_for_expression(a, variables, graph_name)?), - Box::new(self.build_for_expression(b, variables, graph_name)?), + Box::new(self.build_for_expression(a, variables)?), + Box::new(self.build_for_expression(b, variables)?), ), - Expression::In(e, l) => { - let e = self.build_for_expression(e, variables, graph_name)?; - if l.is_empty() { - // False except on error - PlanExpression::If( - Box::new(e), - Box::new(PlanExpression::Literal(PlanTerm { - encoded: false.into(), - plain: false.into(), - })), - Box::new(PlanExpression::Literal(PlanTerm { - encoded: false.into(), - plain: false.into(), - })), - ) - } else { - PlanExpression::Or( - l.iter() - .map(|v| { - Ok(PlanExpression::Equal( - Box::new(e.clone()), - Box::new(self.build_for_expression(v, variables, graph_name)?), - )) - }) - .collect::>()?, - ) - } - } Expression::Add(a, b) => PlanExpression::Add( - Box::new(self.build_for_expression(a, variables, graph_name)?), - Box::new(self.build_for_expression(b, variables, graph_name)?), + Box::new(self.build_for_expression(a, variables)?), + Box::new(self.build_for_expression(b, variables)?), ), Expression::Subtract(a, b) => PlanExpression::Subtract( - Box::new(self.build_for_expression(a, variables, graph_name)?), - Box::new(self.build_for_expression(b, variables, graph_name)?), + Box::new(self.build_for_expression(a, variables)?), + Box::new(self.build_for_expression(b, variables)?), ), Expression::Multiply(a, b) => PlanExpression::Multiply( - Box::new(self.build_for_expression(a, variables, graph_name)?), - Box::new(self.build_for_expression(b, variables, graph_name)?), + Box::new(self.build_for_expression(a, variables)?), + Box::new(self.build_for_expression(b, variables)?), ), Expression::Divide(a, b) => PlanExpression::Divide( - Box::new(self.build_for_expression(a, variables, graph_name)?), - Box::new(self.build_for_expression(b, variables, graph_name)?), + Box::new(self.build_for_expression(a, variables)?), + Box::new(self.build_for_expression(b, variables)?), ), - Expression::UnaryPlus(e) => PlanExpression::UnaryPlus(Box::new( - self.build_for_expression(e, variables, graph_name)?, - )), - Expression::UnaryMinus(e) => PlanExpression::UnaryMinus(Box::new( - self.build_for_expression(e, variables, graph_name)?, - )), - Expression::Not(e) => PlanExpression::Not(Box::new( - self.build_for_expression(e, variables, graph_name)?, - )), + Expression::UnaryPlus(e) => { + PlanExpression::UnaryPlus(Box::new(self.build_for_expression(e, variables)?)) + } + Expression::UnaryMinus(e) => { + PlanExpression::UnaryMinus(Box::new(self.build_for_expression(e, variables)?)) + } + Expression::Not(e) => { + PlanExpression::Not(Box::new(self.build_for_expression(e, variables)?)) + } Expression::FunctionCall(function, parameters) => match function { - Function::Str => PlanExpression::Str(Box::new(self.build_for_expression( - ¶meters[0], - variables, - graph_name, - )?)), - Function::Lang => PlanExpression::Lang(Box::new(self.build_for_expression( - ¶meters[0], - variables, - graph_name, - )?)), + Function::Str => PlanExpression::Str(Box::new( + self.build_for_expression(¶meters[0], variables)?, + )), + Function::Lang => PlanExpression::Lang(Box::new( + self.build_for_expression(¶meters[0], variables)?, + )), Function::LangMatches => PlanExpression::LangMatches( - Box::new(self.build_for_expression(¶meters[0], variables, graph_name)?), - Box::new(self.build_for_expression(¶meters[1], variables, graph_name)?), + Box::new(self.build_for_expression(¶meters[0], variables)?), + Box::new(self.build_for_expression(¶meters[1], variables)?), ), Function::Datatype => PlanExpression::Datatype(Box::new( - self.build_for_expression(¶meters[0], variables, graph_name)?, + self.build_for_expression(¶meters[0], variables)?, + )), + Function::Iri => PlanExpression::Iri(Box::new( + self.build_for_expression(¶meters[0], variables)?, )), - Function::Iri => PlanExpression::Iri(Box::new(self.build_for_expression( - ¶meters[0], - variables, - graph_name, - )?)), Function::BNode => PlanExpression::BNode(match parameters.get(0) { - Some(e) => Some(Box::new( - self.build_for_expression(e, variables, graph_name)?, - )), + Some(e) => Some(Box::new(self.build_for_expression(e, variables)?)), None => None, }), Function::Rand => PlanExpression::Rand, - Function::Abs => PlanExpression::Abs(Box::new(self.build_for_expression( - ¶meters[0], - variables, - graph_name, - )?)), - Function::Ceil => PlanExpression::Ceil(Box::new(self.build_for_expression( - ¶meters[0], - variables, - graph_name, - )?)), - Function::Floor => PlanExpression::Floor(Box::new(self.build_for_expression( - ¶meters[0], - variables, - graph_name, - )?)), - Function::Round => PlanExpression::Round(Box::new(self.build_for_expression( - ¶meters[0], - variables, - graph_name, - )?)), + Function::Abs => PlanExpression::Abs(Box::new( + self.build_for_expression(¶meters[0], variables)?, + )), + Function::Ceil => PlanExpression::Ceil(Box::new( + self.build_for_expression(¶meters[0], variables)?, + )), + Function::Floor => PlanExpression::Floor(Box::new( + self.build_for_expression(¶meters[0], variables)?, + )), + Function::Round => PlanExpression::Round(Box::new( + self.build_for_expression(¶meters[0], variables)?, + )), Function::Concat => { - PlanExpression::Concat(self.expression_list(parameters, variables, graph_name)?) + PlanExpression::Concat(self.expression_list(parameters, variables)?) } Function::SubStr => PlanExpression::SubStr( - Box::new(self.build_for_expression(¶meters[0], variables, graph_name)?), - Box::new(self.build_for_expression(¶meters[1], variables, graph_name)?), + Box::new(self.build_for_expression(¶meters[0], variables)?), + Box::new(self.build_for_expression(¶meters[1], variables)?), match parameters.get(2) { - Some(flags) => Some(Box::new( - self.build_for_expression(flags, variables, graph_name)?, - )), + Some(flags) => Some(Box::new(self.build_for_expression(flags, variables)?)), None => None, }, ), - Function::StrLen => PlanExpression::StrLen(Box::new(self.build_for_expression( - ¶meters[0], - variables, - graph_name, - )?)), + Function::StrLen => PlanExpression::StrLen(Box::new( + self.build_for_expression(¶meters[0], variables)?, + )), Function::Replace => { if let Some(static_regex) = compile_static_pattern_if_exists(¶meters[1], parameters.get(3)) { PlanExpression::StaticReplace( - Box::new(self.build_for_expression( - ¶meters[0], - variables, - graph_name, - )?), + Box::new(self.build_for_expression(¶meters[0], variables)?), static_regex, - Box::new(self.build_for_expression( - ¶meters[2], - variables, - graph_name, - )?), + Box::new(self.build_for_expression(¶meters[2], variables)?), ) } else { PlanExpression::DynamicReplace( - Box::new(self.build_for_expression( - ¶meters[0], - variables, - graph_name, - )?), - Box::new(self.build_for_expression( - ¶meters[1], - variables, - graph_name, - )?), - Box::new(self.build_for_expression( - ¶meters[2], - variables, - graph_name, - )?), + Box::new(self.build_for_expression(¶meters[0], variables)?), + Box::new(self.build_for_expression(¶meters[1], variables)?), + Box::new(self.build_for_expression(¶meters[2], variables)?), match parameters.get(3) { - Some(flags) => Some(Box::new( - self.build_for_expression(flags, variables, graph_name)?, - )), + Some(flags) => { + Some(Box::new(self.build_for_expression(flags, variables)?)) + } None => None, }, ) } } - Function::UCase => PlanExpression::UCase(Box::new(self.build_for_expression( - ¶meters[0], - variables, - graph_name, - )?)), - Function::LCase => PlanExpression::LCase(Box::new(self.build_for_expression( - ¶meters[0], - variables, - graph_name, - )?)), + Function::UCase => PlanExpression::UCase(Box::new( + self.build_for_expression(¶meters[0], variables)?, + )), + Function::LCase => PlanExpression::LCase(Box::new( + self.build_for_expression(¶meters[0], variables)?, + )), Function::EncodeForUri => PlanExpression::EncodeForUri(Box::new( - self.build_for_expression(¶meters[0], variables, graph_name)?, + self.build_for_expression(¶meters[0], variables)?, )), Function::Contains => PlanExpression::Contains( - Box::new(self.build_for_expression(¶meters[0], variables, graph_name)?), - Box::new(self.build_for_expression(¶meters[1], variables, graph_name)?), + Box::new(self.build_for_expression(¶meters[0], variables)?), + Box::new(self.build_for_expression(¶meters[1], variables)?), ), Function::StrStarts => PlanExpression::StrStarts( - Box::new(self.build_for_expression(¶meters[0], variables, graph_name)?), - Box::new(self.build_for_expression(¶meters[1], variables, graph_name)?), + Box::new(self.build_for_expression(¶meters[0], variables)?), + Box::new(self.build_for_expression(¶meters[1], variables)?), ), Function::StrEnds => PlanExpression::StrEnds( - Box::new(self.build_for_expression(¶meters[0], variables, graph_name)?), - Box::new(self.build_for_expression(¶meters[1], variables, graph_name)?), + Box::new(self.build_for_expression(¶meters[0], variables)?), + Box::new(self.build_for_expression(¶meters[1], variables)?), ), Function::StrBefore => PlanExpression::StrBefore( - Box::new(self.build_for_expression(¶meters[0], variables, graph_name)?), - Box::new(self.build_for_expression(¶meters[1], variables, graph_name)?), + Box::new(self.build_for_expression(¶meters[0], variables)?), + Box::new(self.build_for_expression(¶meters[1], variables)?), ), Function::StrAfter => PlanExpression::StrAfter( - Box::new(self.build_for_expression(¶meters[0], variables, graph_name)?), - Box::new(self.build_for_expression(¶meters[1], variables, graph_name)?), + Box::new(self.build_for_expression(¶meters[0], variables)?), + Box::new(self.build_for_expression(¶meters[1], variables)?), ), - Function::Year => PlanExpression::Year(Box::new(self.build_for_expression( - ¶meters[0], - variables, - graph_name, - )?)), - Function::Month => PlanExpression::Month(Box::new(self.build_for_expression( - ¶meters[0], - variables, - graph_name, - )?)), - Function::Day => PlanExpression::Day(Box::new(self.build_for_expression( - ¶meters[0], - variables, - graph_name, - )?)), - Function::Hours => PlanExpression::Hours(Box::new(self.build_for_expression( - ¶meters[0], - variables, - graph_name, - )?)), - Function::Minutes => PlanExpression::Minutes(Box::new(self.build_for_expression( - ¶meters[0], - variables, - graph_name, - )?)), - Function::Seconds => PlanExpression::Seconds(Box::new(self.build_for_expression( - ¶meters[0], - variables, - graph_name, - )?)), + Function::Year => PlanExpression::Year(Box::new( + self.build_for_expression(¶meters[0], variables)?, + )), + Function::Month => PlanExpression::Month(Box::new( + self.build_for_expression(¶meters[0], variables)?, + )), + Function::Day => PlanExpression::Day(Box::new( + self.build_for_expression(¶meters[0], variables)?, + )), + Function::Hours => PlanExpression::Hours(Box::new( + self.build_for_expression(¶meters[0], variables)?, + )), + Function::Minutes => PlanExpression::Minutes(Box::new( + self.build_for_expression(¶meters[0], variables)?, + )), + Function::Seconds => PlanExpression::Seconds(Box::new( + self.build_for_expression(¶meters[0], variables)?, + )), Function::Timezone => PlanExpression::Timezone(Box::new( - self.build_for_expression(¶meters[0], variables, graph_name)?, + self.build_for_expression(¶meters[0], variables)?, + )), + Function::Tz => PlanExpression::Tz(Box::new( + self.build_for_expression(¶meters[0], variables)?, )), - Function::Tz => PlanExpression::Tz(Box::new(self.build_for_expression( - ¶meters[0], - variables, - graph_name, - )?)), Function::Now => PlanExpression::Now, Function::Uuid => PlanExpression::Uuid, Function::StrUuid => PlanExpression::StrUuid, - Function::Md5 => PlanExpression::Md5(Box::new(self.build_for_expression( - ¶meters[0], - variables, - graph_name, - )?)), - Function::Sha1 => PlanExpression::Sha1(Box::new(self.build_for_expression( - ¶meters[0], - variables, - graph_name, - )?)), - Function::Sha256 => PlanExpression::Sha256(Box::new(self.build_for_expression( - ¶meters[0], - variables, - graph_name, - )?)), - Function::Sha384 => PlanExpression::Sha384(Box::new(self.build_for_expression( - ¶meters[0], - variables, - graph_name, - )?)), - Function::Sha512 => PlanExpression::Sha512(Box::new(self.build_for_expression( - ¶meters[0], - variables, - graph_name, - )?)), + Function::Md5 => PlanExpression::Md5(Box::new( + self.build_for_expression(¶meters[0], variables)?, + )), + Function::Sha1 => PlanExpression::Sha1(Box::new( + self.build_for_expression(¶meters[0], variables)?, + )), + Function::Sha256 => PlanExpression::Sha256(Box::new( + self.build_for_expression(¶meters[0], variables)?, + )), + Function::Sha384 => PlanExpression::Sha384(Box::new( + self.build_for_expression(¶meters[0], variables)?, + )), + Function::Sha512 => PlanExpression::Sha512(Box::new( + self.build_for_expression(¶meters[0], variables)?, + )), Function::StrLang => PlanExpression::StrLang( - Box::new(self.build_for_expression(¶meters[0], variables, graph_name)?), - Box::new(self.build_for_expression(¶meters[1], variables, graph_name)?), + Box::new(self.build_for_expression(¶meters[0], variables)?), + Box::new(self.build_for_expression(¶meters[1], variables)?), ), Function::StrDt => PlanExpression::StrDt( - Box::new(self.build_for_expression(¶meters[0], variables, graph_name)?), - Box::new(self.build_for_expression(¶meters[1], variables, graph_name)?), + Box::new(self.build_for_expression(¶meters[0], variables)?), + Box::new(self.build_for_expression(¶meters[1], variables)?), ), - Function::IsIri => PlanExpression::IsIri(Box::new(self.build_for_expression( - ¶meters[0], - variables, - graph_name, - )?)), - Function::IsBlank => PlanExpression::IsBlank(Box::new(self.build_for_expression( - ¶meters[0], - variables, - graph_name, - )?)), + Function::IsIri => PlanExpression::IsIri(Box::new( + self.build_for_expression(¶meters[0], variables)?, + )), + Function::IsBlank => PlanExpression::IsBlank(Box::new( + self.build_for_expression(¶meters[0], variables)?, + )), Function::IsLiteral => PlanExpression::IsLiteral(Box::new( - self.build_for_expression(¶meters[0], variables, graph_name)?, + self.build_for_expression(¶meters[0], variables)?, )), Function::IsNumeric => PlanExpression::IsNumeric(Box::new( - self.build_for_expression(¶meters[0], variables, graph_name)?, + self.build_for_expression(¶meters[0], variables)?, )), Function::Regex => { if let Some(static_regex) = compile_static_pattern_if_exists(¶meters[1], parameters.get(2)) { PlanExpression::StaticRegex( - Box::new(self.build_for_expression( - ¶meters[0], - variables, - graph_name, - )?), + Box::new(self.build_for_expression(¶meters[0], variables)?), static_regex, ) } else { PlanExpression::DynamicRegex( - Box::new(self.build_for_expression( - ¶meters[0], - variables, - graph_name, - )?), - Box::new(self.build_for_expression( - ¶meters[1], - variables, - graph_name, - )?), + Box::new(self.build_for_expression(¶meters[0], variables)?), + Box::new(self.build_for_expression(¶meters[1], variables)?), match parameters.get(2) { - Some(flags) => Some(Box::new( - self.build_for_expression(flags, variables, graph_name)?, - )), + Some(flags) => { + Some(Box::new(self.build_for_expression(flags, variables)?)) + } None => None, }, ) } } Function::Triple => PlanExpression::Triple( - Box::new(self.build_for_expression(¶meters[0], variables, graph_name)?), - Box::new(self.build_for_expression(¶meters[1], variables, graph_name)?), - Box::new(self.build_for_expression(¶meters[2], variables, graph_name)?), + Box::new(self.build_for_expression(¶meters[0], variables)?), + Box::new(self.build_for_expression(¶meters[1], variables)?), + Box::new(self.build_for_expression(¶meters[2], variables)?), ), - Function::Subject => PlanExpression::Subject(Box::new(self.build_for_expression( - ¶meters[0], - variables, - graph_name, - )?)), + Function::Subject => PlanExpression::Subject(Box::new( + self.build_for_expression(¶meters[0], variables)?, + )), Function::Predicate => PlanExpression::Predicate(Box::new( - self.build_for_expression(¶meters[0], variables, graph_name)?, + self.build_for_expression(¶meters[0], variables)?, + )), + Function::Object => PlanExpression::Object(Box::new( + self.build_for_expression(¶meters[0], variables)?, )), - Function::Object => PlanExpression::Object(Box::new(self.build_for_expression( - ¶meters[0], - variables, - graph_name, - )?)), Function::IsTriple => PlanExpression::IsTriple(Box::new( - self.build_for_expression(¶meters[0], variables, graph_name)?, + self.build_for_expression(¶meters[0], variables)?, )), Function::Adjust => PlanExpression::Adjust( - Box::new(self.build_for_expression(¶meters[0], variables, graph_name)?), - Box::new(self.build_for_expression(¶meters[1], variables, graph_name)?), + Box::new(self.build_for_expression(¶meters[0], variables)?), + Box::new(self.build_for_expression(¶meters[1], variables)?), ), Function::Custom(name) => { if self.custom_functions.contains_key(name) { @@ -782,7 +631,7 @@ impl<'a> PlanBuilder<'a> { name.clone(), parameters .iter() - .map(|p| self.build_for_expression(p, variables, graph_name)) + .map(|p| self.build_for_expression(p, variables)) .collect::, EvaluationError>>()?, ) } else if name.as_ref() == xsd::BOOLEAN { @@ -790,7 +639,6 @@ impl<'a> PlanBuilder<'a> { parameters, PlanExpression::BooleanCast, variables, - graph_name, "boolean", )? } else if name.as_ref() == xsd::DOUBLE { @@ -798,23 +646,15 @@ impl<'a> PlanBuilder<'a> { parameters, PlanExpression::DoubleCast, variables, - graph_name, "double", )? } else if name.as_ref() == xsd::FLOAT { - self.build_cast( - parameters, - PlanExpression::FloatCast, - variables, - graph_name, - "float", - )? + self.build_cast(parameters, PlanExpression::FloatCast, variables, "float")? } else if name.as_ref() == xsd::DECIMAL { self.build_cast( parameters, PlanExpression::DecimalCast, variables, - graph_name, "decimal", )? } else if name.as_ref() == xsd::INTEGER { @@ -822,31 +662,17 @@ impl<'a> PlanBuilder<'a> { parameters, PlanExpression::IntegerCast, variables, - graph_name, "integer", )? } else if name.as_ref() == xsd::DATE { - self.build_cast( - parameters, - PlanExpression::DateCast, - variables, - graph_name, - "date", - )? + self.build_cast(parameters, PlanExpression::DateCast, variables, "date")? } else if name.as_ref() == xsd::TIME { - self.build_cast( - parameters, - PlanExpression::TimeCast, - variables, - graph_name, - "time", - )? + self.build_cast(parameters, PlanExpression::TimeCast, variables, "time")? } else if name.as_ref() == xsd::DATE_TIME { self.build_cast( parameters, PlanExpression::DateTimeCast, variables, - graph_name, "dateTime", )? } else if name.as_ref() == xsd::DURATION { @@ -854,7 +680,6 @@ impl<'a> PlanBuilder<'a> { parameters, PlanExpression::DurationCast, variables, - graph_name, "duration", )? } else if name.as_ref() == xsd::YEAR_MONTH_DURATION { @@ -862,7 +687,6 @@ impl<'a> PlanBuilder<'a> { parameters, PlanExpression::YearMonthDurationCast, variables, - graph_name, "yearMonthDuration", )? } else if name.as_ref() == xsd::DAY_TIME_DURATION { @@ -870,7 +694,6 @@ impl<'a> PlanBuilder<'a> { parameters, PlanExpression::DayTimeDurationCast, variables, - graph_name, "dayTimeDuration", )? } else if name.as_ref() == xsd::STRING { @@ -878,32 +701,27 @@ impl<'a> PlanBuilder<'a> { parameters, PlanExpression::StringCast, variables, - graph_name, "string", )? } else { return Err(EvaluationError::msg(format!( - "Not supported custom function {expression}" + "Not supported custom function {name}" ))); } } }, Expression::Bound(v) => PlanExpression::Bound(build_plan_variable(variables, v)), Expression::If(a, b, c) => PlanExpression::If( - Box::new(self.build_for_expression(a, variables, graph_name)?), - Box::new(self.build_for_expression(b, variables, graph_name)?), - Box::new(self.build_for_expression(c, variables, graph_name)?), + Box::new(self.build_for_expression(a, variables)?), + Box::new(self.build_for_expression(b, variables)?), + Box::new(self.build_for_expression(c, variables)?), ), Expression::Exists(n) => { let mut variables = variables.clone(); // Do not expose the exists variables outside - PlanExpression::Exists(Rc::new(self.build_for_graph_pattern( - n, - &mut variables, - graph_name, - )?)) + PlanExpression::Exists(Rc::new(self.build_for_graph_pattern(n, &mut variables)?)) } Expression::Coalesce(parameters) => { - PlanExpression::Coalesce(self.expression_list(parameters, variables, graph_name)?) + PlanExpression::Coalesce(self.expression_list(parameters, variables)?) } }) } @@ -913,15 +731,12 @@ impl<'a> PlanBuilder<'a> { parameters: &[Expression], constructor: impl Fn(Box) -> PlanExpression, variables: &mut Vec, - graph_name: &PatternValue, name: &'static str, ) -> Result { if parameters.len() == 1 { - Ok(constructor(Box::new(self.build_for_expression( - ¶meters[0], - variables, - graph_name, - )?))) + Ok(constructor(Box::new( + self.build_for_expression(¶meters[0], variables)?, + ))) } else { Err(EvaluationError::msg(format!( "The xsd:{name} casting takes only one parameter" @@ -933,42 +748,34 @@ impl<'a> PlanBuilder<'a> { &self, l: &[Expression], variables: &mut Vec, - graph_name: &PatternValue, ) -> Result, EvaluationError> { l.iter() - .map(|e| self.build_for_expression(e, variables, graph_name)) + .map(|e| self.build_for_expression(e, variables)) .collect() } - fn pattern_value_from_term_or_variable( + fn pattern_value_from_ground_term_pattern( &self, - term_or_variable: &TermPattern, + term_pattern: &GroundTermPattern, variables: &mut Vec, ) -> PatternValue { - match term_or_variable { - TermPattern::Variable(variable) => { + match term_pattern { + GroundTermPattern::Variable(variable) => { PatternValue::Variable(build_plan_variable(variables, variable)) } - TermPattern::NamedNode(node) => PatternValue::Constant(PlanTerm { + GroundTermPattern::NamedNode(node) => PatternValue::Constant(PlanTerm { encoded: self.build_term(node), plain: PatternValueConstant::NamedNode(node.clone()), }), - TermPattern::BlankNode(bnode) => { - PatternValue::Variable(build_plan_variable( - variables, - &Variable::new_unchecked(bnode.as_str()), - )) - //TODO: very bad hack to convert bnode to variable - } - TermPattern::Literal(literal) => PatternValue::Constant(PlanTerm { + GroundTermPattern::Literal(literal) => PatternValue::Constant(PlanTerm { encoded: self.build_term(literal), plain: PatternValueConstant::Literal(literal.clone()), }), - TermPattern::Triple(triple) => { + GroundTermPattern::Triple(triple) => { match ( - self.pattern_value_from_term_or_variable(&triple.subject, variables), + self.pattern_value_from_ground_term_pattern(&triple.subject, variables), self.pattern_value_from_named_node_or_variable(&triple.predicate, variables), - self.pattern_value_from_term_or_variable(&triple.object, variables), + self.pattern_value_from_ground_term_pattern(&triple.object, variables), ) { ( PatternValue::Constant(PlanTerm { @@ -1043,40 +850,39 @@ impl<'a> PlanBuilder<'a> { &self, aggregate: &AggregateExpression, variables: &mut Vec, - graph_name: &PatternValue, ) -> Result { match aggregate { AggregateExpression::Count { expr, distinct } => Ok(PlanAggregation { function: PlanAggregationFunction::Count, parameter: match expr { - Some(expr) => Some(self.build_for_expression(expr, variables, graph_name)?), + Some(expr) => Some(self.build_for_expression(expr, variables)?), None => None, }, distinct: *distinct, }), AggregateExpression::Sum { expr, distinct } => Ok(PlanAggregation { function: PlanAggregationFunction::Sum, - parameter: Some(self.build_for_expression(expr, variables, graph_name)?), + parameter: Some(self.build_for_expression(expr, variables)?), distinct: *distinct, }), AggregateExpression::Min { expr, distinct } => Ok(PlanAggregation { function: PlanAggregationFunction::Min, - parameter: Some(self.build_for_expression(expr, variables, graph_name)?), + parameter: Some(self.build_for_expression(expr, variables)?), distinct: *distinct, }), AggregateExpression::Max { expr, distinct } => Ok(PlanAggregation { function: PlanAggregationFunction::Max, - parameter: Some(self.build_for_expression(expr, variables, graph_name)?), + parameter: Some(self.build_for_expression(expr, variables)?), distinct: *distinct, }), AggregateExpression::Avg { expr, distinct } => Ok(PlanAggregation { function: PlanAggregationFunction::Avg, - parameter: Some(self.build_for_expression(expr, variables, graph_name)?), + parameter: Some(self.build_for_expression(expr, variables)?), distinct: *distinct, }), AggregateExpression::Sample { expr, distinct } => Ok(PlanAggregation { function: PlanAggregationFunction::Sample, - parameter: Some(self.build_for_expression(expr, variables, graph_name)?), + parameter: Some(self.build_for_expression(expr, variables)?), distinct: *distinct, }), AggregateExpression::GroupConcat { @@ -1087,7 +893,7 @@ impl<'a> PlanBuilder<'a> { function: PlanAggregationFunction::GroupConcat { separator: Rc::from(separator.as_deref().unwrap_or(" ")), }, - parameter: Some(self.build_for_expression(expr, variables, graph_name)?), + parameter: Some(self.build_for_expression(expr, variables)?), distinct: *distinct, }), AggregateExpression::Custom { .. } => Err(EvaluationError::msg( @@ -1203,348 +1009,6 @@ impl<'a> PlanBuilder<'a> { } } - fn convert_pattern_value_id(from_value: &PatternValue, to: &mut Vec) -> PatternValue { - match from_value { - PatternValue::Constant(c) => PatternValue::Constant(c.clone()), - PatternValue::Variable(from_id) => { - PatternValue::Variable(Self::convert_plan_variable(from_id, to)) - } - PatternValue::TriplePattern(triple) => { - PatternValue::TriplePattern(Box::new(TriplePatternValue { - subject: Self::convert_pattern_value_id(&triple.subject, to), - predicate: Self::convert_pattern_value_id(&triple.predicate, to), - object: Self::convert_pattern_value_id(&triple.object, to), - })) - } - } - } - - fn convert_plan_variable(from_variable: &PlanVariable, to: &mut Vec) -> PlanVariable { - let encoded = if let Some(to_id) = to - .iter() - .enumerate() - .find_map(|(to_id, var)| (*var == from_variable.plain).then_some(to_id)) - { - to_id - } else { - to.push(Variable::new_unchecked(format!("{:x}", random::()))); - to.len() - 1 - }; - PlanVariable { - encoded, - plain: from_variable.plain.clone(), - } - } - - fn can_use_for_loop_left_join(node: &PlanNode) -> bool { - // We forbid MINUS, SERVICE and everything that affects cardinality in for loop left joins - match node { - PlanNode::StaticBindings { .. } - | PlanNode::QuadPattern { .. } - | PlanNode::PathPattern { .. } => true, - PlanNode::Filter { child, .. } - | PlanNode::Extend { child, .. } - | PlanNode::Sort { child, .. } - | PlanNode::Project { child, .. } - | PlanNode::Aggregate { child, .. } => Self::can_use_for_loop_left_join(child), - PlanNode::Union { children } => { - children.iter().all(|c| Self::can_use_for_loop_left_join(c)) - } - PlanNode::HashJoin { left, right } - | PlanNode::ForLoopJoin { left, right } - | PlanNode::ForLoopLeftJoin { left, right, .. } - | PlanNode::HashLeftJoin { left, right, .. } => { - Self::can_use_for_loop_left_join(left) && Self::can_use_for_loop_left_join(right) - } - PlanNode::AntiJoin { .. } - | PlanNode::Service { .. } - | PlanNode::HashDeduplicate { .. } - | PlanNode::Reduced { .. } - | PlanNode::Skip { .. } - | PlanNode::Limit { .. } => false, - } - } - - fn add_left_join_problematic_variables(node: &PlanNode, set: &mut BTreeSet) { - match node { - PlanNode::StaticBindings { .. } - | PlanNode::QuadPattern { .. } - | PlanNode::PathPattern { .. } => (), - PlanNode::Filter { child, expression } => { - let always_already_bound = child.always_bound_variables(); - expression.lookup_used_variables(&mut |v| { - if !always_already_bound.contains(&v) { - set.insert(v); - } - }); - Self::add_left_join_problematic_variables(child, set); - } - PlanNode::Union { children } => { - for child in children.iter() { - Self::add_left_join_problematic_variables(child, set); - } - } - PlanNode::HashJoin { left, right } | PlanNode::ForLoopJoin { left, right } => { - Self::add_left_join_problematic_variables(left, set); - Self::add_left_join_problematic_variables(right, set); - } - PlanNode::AntiJoin { left, .. } => { - Self::add_left_join_problematic_variables(left, set); - } - PlanNode::ForLoopLeftJoin { left, right, .. } => { - Self::add_left_join_problematic_variables(left, set); - right.lookup_used_variables(&mut |v| { - set.insert(v); - }); - } - PlanNode::HashLeftJoin { - left, - right, - expression, - } => { - Self::add_left_join_problematic_variables(left, set); - right.lookup_used_variables(&mut |v| { - set.insert(v); - }); - let always_already_bound = left.always_bound_variables(); - expression.lookup_used_variables(&mut |v| { - if !always_already_bound.contains(&v) { - set.insert(v); - } - }); - } - PlanNode::Extend { - child, expression, .. - } => { - let always_already_bound = child.always_bound_variables(); - expression.lookup_used_variables(&mut |v| { - if !always_already_bound.contains(&v) { - set.insert(v); - } - }); - Self::add_left_join_problematic_variables(child, set); - Self::add_left_join_problematic_variables(child, set); - } - PlanNode::Sort { child, .. } - | PlanNode::HashDeduplicate { child } - | PlanNode::Reduced { child } - | PlanNode::Project { child, .. } => { - Self::add_left_join_problematic_variables(child, set); - } - PlanNode::Skip { child, .. } | PlanNode::Limit { child, .. } => { - // Any variable might affect arity - child.lookup_used_variables(&mut |v| { - set.insert(v); - }) - } - PlanNode::Service { child, silent, .. } => { - if *silent { - child.lookup_used_variables(&mut |v| { - set.insert(v); - }); - } else { - Self::add_left_join_problematic_variables(child, set) - } - } - PlanNode::Aggregate { - key_variables, - aggregates, - .. - } => { - set.extend(key_variables.iter().map(|v| v.encoded)); - //TODO: This is too harsh - for (_, var) in aggregates.iter() { - set.insert(var.encoded); - } - } - } - } - - fn new_join(&self, mut left: PlanNode, mut right: PlanNode) -> PlanNode { - // We first use VALUES to filter the following patterns evaluation - if matches!(right, PlanNode::StaticBindings { .. }) { - swap(&mut left, &mut right); - } - - if self.with_optimizations - && Self::is_fit_for_for_loop_join(&right) - && Self::has_some_common_variables(&left, &right) - { - PlanNode::ForLoopJoin { - left: Rc::new(left), - right: Rc::new(right), - } - } else { - // Let's avoid materializing right if left is already materialized - // TODO: be smarter and reuse already existing materialization - if matches!(left, PlanNode::StaticBindings { .. }) { - swap(&mut left, &mut right); - } - PlanNode::HashJoin { - left: Rc::new(left), - right: Rc::new(right), - } - } - } - - fn has_some_common_variables(left: &PlanNode, right: &PlanNode) -> bool { - left.always_bound_variables() - .intersection(&right.always_bound_variables()) - .next() - .is_some() - } - - fn is_fit_for_for_loop_join(node: &PlanNode) -> bool { - //TODO: think more about it - match node { - PlanNode::StaticBindings { .. } - | PlanNode::QuadPattern { .. } - | PlanNode::PathPattern { .. } => true, - PlanNode::ForLoopJoin { left, right } | PlanNode::HashJoin { left, right } => { - Self::is_fit_for_for_loop_join(left) && Self::is_fit_for_for_loop_join(right) - } - PlanNode::Filter { child, .. } | PlanNode::Extend { child, .. } => { - Self::is_fit_for_for_loop_join(child) - } - PlanNode::Union { children } => { - children.iter().all(|c| Self::is_fit_for_for_loop_join(c)) - } - PlanNode::AntiJoin { .. } - | PlanNode::HashLeftJoin { .. } - | PlanNode::ForLoopLeftJoin { .. } - | PlanNode::Service { .. } - | PlanNode::Sort { .. } - | PlanNode::HashDeduplicate { .. } - | PlanNode::Reduced { .. } - | PlanNode::Skip { .. } - | PlanNode::Limit { .. } - | PlanNode::Project { .. } - | PlanNode::Aggregate { .. } => false, - } - } - - fn push_filter(&self, node: Rc, filter: Box) -> PlanNode { - if !self.with_optimizations { - return PlanNode::Filter { - child: node, - expression: filter, - }; - } - if let PlanExpression::And(filters) = *filter { - return filters - .into_iter() - .fold((*node.as_ref()).clone(), |acc, f| { - self.push_filter(Rc::new(acc), Box::new(f)) - }); - } - let mut filter_variables = BTreeSet::new(); - filter.lookup_used_variables(&mut |v| { - filter_variables.insert(v); - }); - match node.as_ref() { - PlanNode::HashJoin { left, right } => { - if filter_variables.iter().all(|v| left.is_variable_bound(*v)) { - if filter_variables.iter().all(|v| right.is_variable_bound(*v)) { - PlanNode::HashJoin { - left: Rc::new(self.push_filter(Rc::clone(left), filter.clone())), - right: Rc::new(self.push_filter(Rc::clone(right), filter)), - } - } else { - PlanNode::HashJoin { - left: Rc::new(self.push_filter(Rc::clone(left), filter)), - right: Rc::clone(right), - } - } - } else if filter_variables.iter().all(|v| right.is_variable_bound(*v)) { - PlanNode::HashJoin { - left: Rc::clone(left), - right: Rc::new(self.push_filter(Rc::clone(right), filter)), - } - } else { - PlanNode::Filter { - child: Rc::new(PlanNode::HashJoin { - left: Rc::clone(left), - right: Rc::clone(right), - }), - expression: filter, - } - } - } - PlanNode::ForLoopJoin { left, right } => { - if filter_variables.iter().all(|v| left.is_variable_bound(*v)) { - PlanNode::ForLoopJoin { - left: Rc::new(self.push_filter(Rc::clone(left), filter)), - right: Rc::clone(right), - } - } else if filter_variables.iter().all(|v| right.is_variable_bound(*v)) { - PlanNode::ForLoopJoin { - //TODO: should we do that always? - left: Rc::clone(left), - right: Rc::new(self.push_filter(Rc::clone(right), filter)), - } - } else { - PlanNode::Filter { - child: Rc::new(PlanNode::HashJoin { - left: Rc::clone(left), - right: Rc::clone(right), - }), - expression: filter, - } - } - } - PlanNode::Extend { - child, - expression, - variable, - } => { - //TODO: handle the case where the filter generates an expression variable - if filter_variables.iter().all(|v| child.is_variable_bound(*v)) { - PlanNode::Extend { - child: Rc::new(self.push_filter(Rc::clone(child), filter)), - expression: expression.clone(), - variable: variable.clone(), - } - } else { - PlanNode::Filter { - child: Rc::new(PlanNode::Extend { - child: Rc::clone(child), - expression: expression.clone(), - variable: variable.clone(), - }), - expression: filter, - } - } - } - PlanNode::Filter { child, expression } => { - if filter_variables.iter().all(|v| child.is_variable_bound(*v)) { - PlanNode::Filter { - child: Rc::new(self.push_filter(Rc::clone(child), filter)), - expression: expression.clone(), - } - } else { - PlanNode::Filter { - child: Rc::clone(child), - expression: Box::new(PlanExpression::And(vec![ - *expression.clone(), - *filter, - ])), - } - } - } - PlanNode::Union { children } => PlanNode::Union { - children: children - .iter() - .map(|c| Rc::new(self.push_filter(Rc::clone(c), filter.clone()))) - .collect(), - }, - _ => PlanNode::Filter { - //TODO: more? - child: node, - expression: filter, - }, - } - } - fn build_term<'b>(&self, term: impl Into>) -> EncodedTerm { self.dataset.encode_term(term) } @@ -1597,101 +1061,6 @@ fn slice_key(slice: &[T], element: &T) -> Option { None } -fn sort_bgp(p: &[TriplePattern]) -> Vec<&TriplePattern> { - let mut assigned_variables = HashSet::default(); - let mut assigned_blank_nodes = HashSet::default(); - let mut new_p: Vec<_> = p.iter().collect(); - - for i in 0..new_p.len() { - new_p[i..].sort_by(|p1, p2| { - estimate_pattern_cost(p1, &assigned_variables, &assigned_blank_nodes).cmp( - &estimate_pattern_cost(p2, &assigned_variables, &assigned_blank_nodes), - ) - }); - add_pattern_variables(new_p[i], &mut assigned_variables, &mut assigned_blank_nodes); - } - - new_p -} - -fn estimate_pattern_cost( - pattern: &TriplePattern, - assigned_variables: &HashSet<&Variable>, - assigned_blank_nodes: &HashSet<&BlankNode>, -) -> u32 { - let mut count = 0; - match &pattern.subject { - TermPattern::NamedNode(_) | TermPattern::Literal(_) => count += 1, - TermPattern::BlankNode(bnode) => { - if !assigned_blank_nodes.contains(bnode) { - count += 4; - } - } - TermPattern::Variable(v) => { - if !assigned_variables.contains(v) { - count += 4; - } - } - TermPattern::Triple(t) => { - count += estimate_pattern_cost(t, assigned_variables, assigned_blank_nodes) - } - } - if let NamedNodePattern::Variable(v) = &pattern.predicate { - if !assigned_variables.contains(v) { - count += 4; - } - } else { - count += 1; - } - match &pattern.object { - TermPattern::NamedNode(_) | TermPattern::Literal(_) => count += 1, - TermPattern::BlankNode(bnode) => { - if !assigned_blank_nodes.contains(bnode) { - count += 4; - } - } - TermPattern::Variable(v) => { - if !assigned_variables.contains(v) { - count += 4; - } - } - TermPattern::Triple(t) => { - count += estimate_pattern_cost(t, assigned_variables, assigned_blank_nodes) - } - } - count -} - -fn add_pattern_variables<'a>( - pattern: &'a TriplePattern, - variables: &mut HashSet<&'a Variable>, - blank_nodes: &mut HashSet<&'a BlankNode>, -) { - match &pattern.subject { - TermPattern::NamedNode(_) | TermPattern::Literal(_) => (), - TermPattern::BlankNode(bnode) => { - blank_nodes.insert(bnode); - } - TermPattern::Variable(v) => { - variables.insert(v); - } - TermPattern::Triple(t) => add_pattern_variables(t, variables, blank_nodes), - } - if let NamedNodePattern::Variable(v) = &pattern.predicate { - variables.insert(v); - } - match &pattern.object { - TermPattern::NamedNode(_) | TermPattern::Literal(_) => (), - TermPattern::BlankNode(bnode) => { - blank_nodes.insert(bnode); - } - TermPattern::Variable(v) => { - variables.insert(v); - } - TermPattern::Triple(t) => add_pattern_variables(t, variables, blank_nodes), - } -} - fn compile_static_pattern_if_exists( pattern: &Expression, options: Option<&Expression>, diff --git a/testsuite/Cargo.toml b/testsuite/Cargo.toml index 92305371..20579b1f 100644 --- a/testsuite/Cargo.toml +++ b/testsuite/Cargo.toml @@ -16,4 +16,6 @@ anyhow = "1" clap = { version = "4", features = ["derive"] } time = { version = "0.3", features = ["formatting"] } oxigraph = { path = "../lib" } +sparopt = { path = "../lib/sparopt" } +spargebra = { path = "../lib/spargebra" } text-diff = "0.4" diff --git a/testsuite/oxigraph-tests/sparql-optimization/bgp_join_reordering_input.rq b/testsuite/oxigraph-tests/sparql-optimization/bgp_join_reordering_input.rq new file mode 100644 index 00000000..c71b46cb --- /dev/null +++ b/testsuite/oxigraph-tests/sparql-optimization/bgp_join_reordering_input.rq @@ -0,0 +1,9 @@ +PREFIX ex: + +SELECT ?s ?o WHERE { + ?m2 ex:p2 ?o . + ?s ex:p1 ?m1 , ?m2 . + ?m1 ex:p2 ?o . + ?s ex:p1prime ?m1 . + ?s a ex:C . +} diff --git a/testsuite/oxigraph-tests/sparql-optimization/bgp_join_reordering_output.rq b/testsuite/oxigraph-tests/sparql-optimization/bgp_join_reordering_output.rq new file mode 100644 index 00000000..040c11e3 --- /dev/null +++ b/testsuite/oxigraph-tests/sparql-optimization/bgp_join_reordering_output.rq @@ -0,0 +1,10 @@ +PREFIX ex: + +SELECT ?s ?o WHERE { + ?s a ex:C . + ?s ex:p1 ?m1 . + ?s ex:p1prime ?m1 . + ?s ex:p1 ?m2 . + ?m2 ex:p2 ?o . + ?m1 ex:p2 ?o . +} diff --git a/testsuite/oxigraph-tests/sparql-optimization/bind_always_false_input.rq b/testsuite/oxigraph-tests/sparql-optimization/bind_always_false_input.rq new file mode 100644 index 00000000..2ebda3b4 --- /dev/null +++ b/testsuite/oxigraph-tests/sparql-optimization/bind_always_false_input.rq @@ -0,0 +1,4 @@ +SELECT ?a ?o WHERE { + ?s ?p ?o . + FILTER(BOUND(?a)) +} diff --git a/testsuite/oxigraph-tests/sparql-optimization/bind_always_false_output.rq b/testsuite/oxigraph-tests/sparql-optimization/bind_always_false_output.rq new file mode 100644 index 00000000..493c0c64 --- /dev/null +++ b/testsuite/oxigraph-tests/sparql-optimization/bind_always_false_output.rq @@ -0,0 +1,3 @@ +SELECT ?a ?o WHERE { + VALUES () {} +} diff --git a/testsuite/oxigraph-tests/sparql-optimization/bind_always_true_input.rq b/testsuite/oxigraph-tests/sparql-optimization/bind_always_true_input.rq new file mode 100644 index 00000000..1230938d --- /dev/null +++ b/testsuite/oxigraph-tests/sparql-optimization/bind_always_true_input.rq @@ -0,0 +1,4 @@ +SELECT ?s WHERE { + ?s ?p ?o . + FILTER(BOUND(?s)) +} diff --git a/testsuite/oxigraph-tests/sparql-optimization/bind_always_true_output.rq b/testsuite/oxigraph-tests/sparql-optimization/bind_always_true_output.rq new file mode 100644 index 00000000..89aeaa3d --- /dev/null +++ b/testsuite/oxigraph-tests/sparql-optimization/bind_always_true_output.rq @@ -0,0 +1,3 @@ +SELECT ?s WHERE { + ?s ?p ?o +} diff --git a/testsuite/oxigraph-tests/sparql-optimization/empty_union_input.rq b/testsuite/oxigraph-tests/sparql-optimization/empty_union_input.rq new file mode 100644 index 00000000..dc8de6f4 --- /dev/null +++ b/testsuite/oxigraph-tests/sparql-optimization/empty_union_input.rq @@ -0,0 +1,3 @@ +SELECT ?o WHERE { + { ?s ?p ?o } UNION { VALUES () {} } +} diff --git a/testsuite/oxigraph-tests/sparql-optimization/empty_union_output.rq b/testsuite/oxigraph-tests/sparql-optimization/empty_union_output.rq new file mode 100644 index 00000000..e0da523f --- /dev/null +++ b/testsuite/oxigraph-tests/sparql-optimization/empty_union_output.rq @@ -0,0 +1,3 @@ +SELECT ?o WHERE { + ?s ?p ?o . +} diff --git a/testsuite/oxigraph-tests/sparql-optimization/equal_to_same_term_input.rq b/testsuite/oxigraph-tests/sparql-optimization/equal_to_same_term_input.rq new file mode 100644 index 00000000..015a2163 --- /dev/null +++ b/testsuite/oxigraph-tests/sparql-optimization/equal_to_same_term_input.rq @@ -0,0 +1,5 @@ +SELECT ?s1 ?s2 ?o1 ?o2 WHERE { + ?s1 ?p1 ?o1 . + ?s2 ?p2 ?o2 . + FILTER(?p1 = ?p2) +} diff --git a/testsuite/oxigraph-tests/sparql-optimization/equal_to_same_term_output.rq b/testsuite/oxigraph-tests/sparql-optimization/equal_to_same_term_output.rq new file mode 100644 index 00000000..8e3c96c7 --- /dev/null +++ b/testsuite/oxigraph-tests/sparql-optimization/equal_to_same_term_output.rq @@ -0,0 +1,5 @@ +SELECT ?s1 ?s2 ?o1 ?o2 WHERE { + ?s1 ?p1 ?o1 . + ?s2 ?p2 ?o2 . + FILTER(sameTerm(?p2, ?p1)) +} diff --git a/testsuite/oxigraph-tests/sparql-optimization/exists_always_false_input.rq b/testsuite/oxigraph-tests/sparql-optimization/exists_always_false_input.rq new file mode 100644 index 00000000..e3615fa7 --- /dev/null +++ b/testsuite/oxigraph-tests/sparql-optimization/exists_always_false_input.rq @@ -0,0 +1,4 @@ +SELECT ?s WHERE { + ?s ?p ?o . + FILTER(EXISTS { VALUES () {}}) +} diff --git a/testsuite/oxigraph-tests/sparql-optimization/exists_always_false_output.rq b/testsuite/oxigraph-tests/sparql-optimization/exists_always_false_output.rq new file mode 100644 index 00000000..13583411 --- /dev/null +++ b/testsuite/oxigraph-tests/sparql-optimization/exists_always_false_output.rq @@ -0,0 +1,3 @@ +SELECT ?s WHERE { + VALUES() {} +} diff --git a/testsuite/oxigraph-tests/sparql-optimization/false_and_something_input.rq b/testsuite/oxigraph-tests/sparql-optimization/false_and_something_input.rq new file mode 100644 index 00000000..2123e97a --- /dev/null +++ b/testsuite/oxigraph-tests/sparql-optimization/false_and_something_input.rq @@ -0,0 +1,4 @@ +SELECT ?o1 ?o2 WHERE { + ?s ?p ?o1 , ?o2 . + FILTER(false && ?o1 = ?o2) +} diff --git a/testsuite/oxigraph-tests/sparql-optimization/false_and_something_output.rq b/testsuite/oxigraph-tests/sparql-optimization/false_and_something_output.rq new file mode 100644 index 00000000..ba012e0a --- /dev/null +++ b/testsuite/oxigraph-tests/sparql-optimization/false_and_something_output.rq @@ -0,0 +1,3 @@ +SELECT ?o1 ?o2 WHERE { + VALUES () {} +} diff --git a/testsuite/oxigraph-tests/sparql-optimization/false_or_something_input.rq b/testsuite/oxigraph-tests/sparql-optimization/false_or_something_input.rq new file mode 100644 index 00000000..3710d7eb --- /dev/null +++ b/testsuite/oxigraph-tests/sparql-optimization/false_or_something_input.rq @@ -0,0 +1,4 @@ +SELECT ?o1 ?o2 WHERE { + ?s ?p ?o1 , ?o2 . + FILTER(false || ?o1 = ?o2) +} diff --git a/testsuite/oxigraph-tests/sparql-optimization/false_or_something_output.rq b/testsuite/oxigraph-tests/sparql-optimization/false_or_something_output.rq new file mode 100644 index 00000000..95741032 --- /dev/null +++ b/testsuite/oxigraph-tests/sparql-optimization/false_or_something_output.rq @@ -0,0 +1,4 @@ +SELECT ?o1 ?o2 WHERE { + ?s ?p ?o1 , ?o2 . + FILTER(?o2 = ?o1) +} diff --git a/testsuite/oxigraph-tests/sparql-optimization/if_always_false_input.rq b/testsuite/oxigraph-tests/sparql-optimization/if_always_false_input.rq new file mode 100644 index 00000000..6c8aa537 --- /dev/null +++ b/testsuite/oxigraph-tests/sparql-optimization/if_always_false_input.rq @@ -0,0 +1,4 @@ +SELECT ?o1 ?o2 WHERE { + ?s ?p ?o1 , ?o2 . + FILTER(IF(false, ?o1 = ?o2, ?o1 != ?o2)) +} diff --git a/testsuite/oxigraph-tests/sparql-optimization/if_always_false_output.rq b/testsuite/oxigraph-tests/sparql-optimization/if_always_false_output.rq new file mode 100644 index 00000000..c64ec6ea --- /dev/null +++ b/testsuite/oxigraph-tests/sparql-optimization/if_always_false_output.rq @@ -0,0 +1,4 @@ +SELECT ?o1 ?o2 WHERE { + ?s ?p ?o1 , ?o2 . + FILTER(?o2 != ?o1) +} diff --git a/testsuite/oxigraph-tests/sparql-optimization/if_always_true_input.rq b/testsuite/oxigraph-tests/sparql-optimization/if_always_true_input.rq new file mode 100644 index 00000000..52a63e4b --- /dev/null +++ b/testsuite/oxigraph-tests/sparql-optimization/if_always_true_input.rq @@ -0,0 +1,4 @@ +SELECT ?o1 ?o2 WHERE { + ?s ?p ?o1 , ?o2 . + FILTER(IF(true, ?o1 = ?o2, ?o1 != ?o2)) +} diff --git a/testsuite/oxigraph-tests/sparql-optimization/if_always_true_output.rq b/testsuite/oxigraph-tests/sparql-optimization/if_always_true_output.rq new file mode 100644 index 00000000..95741032 --- /dev/null +++ b/testsuite/oxigraph-tests/sparql-optimization/if_always_true_output.rq @@ -0,0 +1,4 @@ +SELECT ?o1 ?o2 WHERE { + ?s ?p ?o1 , ?o2 . + FILTER(?o2 = ?o1) +} diff --git a/testsuite/oxigraph-tests/sparql-optimization/manifest.ttl b/testsuite/oxigraph-tests/sparql-optimization/manifest.ttl new file mode 100644 index 00000000..c509fe79 --- /dev/null +++ b/testsuite/oxigraph-tests/sparql-optimization/manifest.ttl @@ -0,0 +1,132 @@ +@prefix rdf: . +@prefix : . +@prefix rdfs: . +@prefix mf: . +@prefix ox: . + +<> rdf:type mf:Manifest ; + rdfs:label "Oxigraph SPARQL optimization tests" ; + mf:entries + ( + :unbound_filter + :unbound_bind + :something_or_true + :true_or_something + :something_or_false + :false_or_something + :something_and_true + :true_and_something + :something_and_false + :false_and_something + :equal_to_same_term + :bind_always_true + :bind_always_false + :if_always_true + :if_always_false + :exists_always_false + :push_filter + :push_optional_filter + :empty_union + :bgp_join_reordering + ) . + + +:unbound_filter rdf:type ox:QueryOptimizationTest ; + mf:name "unbound variable in filter" ; + mf:action ; + mf:result . + +:unbound_bind rdf:type ox:QueryOptimizationTest ; + mf:name "unbound variable in bindr" ; + mf:action ; + mf:result . + +:something_or_true rdf:type ox:QueryOptimizationTest ; + mf:name "something || true" ; + mf:action ; + mf:result . + +:true_or_something rdf:type ox:QueryOptimizationTest ; + mf:name "true || something" ; + mf:action ; + mf:result . + +:something_or_false rdf:type ox:QueryOptimizationTest ; + mf:name "something || false" ; + mf:action ; + mf:result . + +:false_or_something rdf:type ox:QueryOptimizationTest ; + mf:name "false || something" ; + mf:action ; + mf:result . + +:something_and_true rdf:type ox:QueryOptimizationTest ; + mf:name "something && true" ; + mf:action ; + mf:result . + +:true_and_something rdf:type ox:QueryOptimizationTest ; + mf:name "true && something" ; + mf:action ; + mf:result . + +:something_and_false rdf:type ox:QueryOptimizationTest ; + mf:name "something && false" ; + mf:action ; + mf:result . + +:false_and_something rdf:type ox:QueryOptimizationTest ; + mf:name "false && something" ; + mf:action ; + mf:result . + +:equal_to_same_term a ox:QueryOptimizationTest ; + mf:name "equal to same term" ; + mf:action ; + mf:result . + +:bind_always_true rdf:type ox:QueryOptimizationTest ; + mf:name "BIND() always true" ; + mf:action ; + mf:result . + +:bind_always_false rdf:type ox:QueryOptimizationTest ; + mf:name "BIND() always false" ; + mf:action ; + mf:result . + +:if_always_true rdf:type ox:QueryOptimizationTest ; + mf:name "IF() always true" ; + mf:action ; + mf:result . + +:if_always_false rdf:type ox:QueryOptimizationTest ; + mf:name "IF() always false" ; + mf:action ; + mf:result . + +:exists_always_false rdf:type ox:QueryOptimizationTest ; + mf:name "EXISTS {} always false" ; + mf:action ; + mf:result . + +:push_filter rdf:type ox:QueryOptimizationTest ; + mf:name "push filter down" ; + mf:action ; + mf:result . + +:push_optional_filter rdf:type ox:QueryOptimizationTest ; + mf:name "push OPTIONAL filter down" ; + mf:action ; + mf:result . + +:empty_union rdf:type ox:QueryOptimizationTest ; + mf:name "empty UNION" ; + mf:action ; + mf:result . + +:bgp_join_reordering rdf:type ox:QueryOptimizationTest ; + mf:name "BGP join reordering" ; + mf:action ; + mf:result . diff --git a/testsuite/oxigraph-tests/sparql-optimization/push_filter_input.rq b/testsuite/oxigraph-tests/sparql-optimization/push_filter_input.rq new file mode 100644 index 00000000..58e64a8e --- /dev/null +++ b/testsuite/oxigraph-tests/sparql-optimization/push_filter_input.rq @@ -0,0 +1,11 @@ +PREFIX : + +SELECT ?o1 ?o2 ?o4 ?o5 WHERE { + ?s :p1 ?o1 ; :p4 ?o4 ; :p5 ?o5 . + LATERAL { ?s :p2 ?o2 } + MINUS { ?s :p3 ?o3 } + FILTER(?o1 = 1) + FILTER(?o2 = 2) + FILTER(?o4 = 4) + FILTER(?o1 = ?o5) +} diff --git a/testsuite/oxigraph-tests/sparql-optimization/push_filter_output.rq b/testsuite/oxigraph-tests/sparql-optimization/push_filter_output.rq new file mode 100644 index 00000000..3cecb4b3 --- /dev/null +++ b/testsuite/oxigraph-tests/sparql-optimization/push_filter_output.rq @@ -0,0 +1,18 @@ +PREFIX : + +SELECT ?o1 ?o2 ?o4 ?o5 WHERE { + { + { + { + { ?s :p1 ?o1 FILTER(1 = ?o1) } + LATERAL { ?s :p4 ?o4 } + FILTER(?o4 = 4) + } + LATERAL { ?s :p5 ?o5 } + FILTER(?o5 = ?o1) + } + LATERAL { ?s :p2 ?o2 } + FILTER(?o2 = 2) + } + MINUS { ?s :p3 ?o3 } +} diff --git a/testsuite/oxigraph-tests/sparql-optimization/push_optional_filter_input.rq b/testsuite/oxigraph-tests/sparql-optimization/push_optional_filter_input.rq new file mode 100644 index 00000000..a78e08e3 --- /dev/null +++ b/testsuite/oxigraph-tests/sparql-optimization/push_optional_filter_input.rq @@ -0,0 +1,5 @@ +SELECT ?s ?o WHERE { + ?s a ?t . + OPTIONAL { { ?s ?p ?o } UNION { ?s ?p ?o2 } FILTER(?o = 1) } + OPTIONAL { { ?s ?p ?o } UNION { ?s ?p2 ?o2 } FILTER(?o = ?t) } +} \ No newline at end of file diff --git a/testsuite/oxigraph-tests/sparql-optimization/push_optional_filter_output.rq b/testsuite/oxigraph-tests/sparql-optimization/push_optional_filter_output.rq new file mode 100644 index 00000000..067f0d87 --- /dev/null +++ b/testsuite/oxigraph-tests/sparql-optimization/push_optional_filter_output.rq @@ -0,0 +1,5 @@ +SELECT ?s ?o WHERE { + ?s a ?t . + LATERAL { VALUES () {()} OPTIONAL { { ?s ?p ?o FILTER(1 = ?o) } UNION { ?s ?p ?o2 FILTER(1 = ?o) } } } + LATERAL { VALUES () {()} OPTIONAL { { ?s ?p ?o } UNION { ?s ?p2 ?o2 } FILTER(?t = ?o) } } +} \ No newline at end of file diff --git a/testsuite/oxigraph-tests/sparql-optimization/something_and_false_input.rq b/testsuite/oxigraph-tests/sparql-optimization/something_and_false_input.rq new file mode 100644 index 00000000..c99ca675 --- /dev/null +++ b/testsuite/oxigraph-tests/sparql-optimization/something_and_false_input.rq @@ -0,0 +1,4 @@ +SELECT ?o1 ?o2 WHERE { + ?s ?p ?o1 , ?o2 . + FILTER(?o1 = ?o2 && false) +} diff --git a/testsuite/oxigraph-tests/sparql-optimization/something_and_false_output.rq b/testsuite/oxigraph-tests/sparql-optimization/something_and_false_output.rq new file mode 100644 index 00000000..ba012e0a --- /dev/null +++ b/testsuite/oxigraph-tests/sparql-optimization/something_and_false_output.rq @@ -0,0 +1,3 @@ +SELECT ?o1 ?o2 WHERE { + VALUES () {} +} diff --git a/testsuite/oxigraph-tests/sparql-optimization/something_and_true_input.rq b/testsuite/oxigraph-tests/sparql-optimization/something_and_true_input.rq new file mode 100644 index 00000000..de09d711 --- /dev/null +++ b/testsuite/oxigraph-tests/sparql-optimization/something_and_true_input.rq @@ -0,0 +1,4 @@ +SELECT ?o1 ?o2 WHERE { + ?s ?p ?o1 , ?o2 . + FILTER(?o1 = ?o2 && true) +} diff --git a/testsuite/oxigraph-tests/sparql-optimization/something_and_true_output.rq b/testsuite/oxigraph-tests/sparql-optimization/something_and_true_output.rq new file mode 100644 index 00000000..95741032 --- /dev/null +++ b/testsuite/oxigraph-tests/sparql-optimization/something_and_true_output.rq @@ -0,0 +1,4 @@ +SELECT ?o1 ?o2 WHERE { + ?s ?p ?o1 , ?o2 . + FILTER(?o2 = ?o1) +} diff --git a/testsuite/oxigraph-tests/sparql-optimization/something_or_false_input.rq b/testsuite/oxigraph-tests/sparql-optimization/something_or_false_input.rq new file mode 100644 index 00000000..495bbd1b --- /dev/null +++ b/testsuite/oxigraph-tests/sparql-optimization/something_or_false_input.rq @@ -0,0 +1,4 @@ +SELECT ?o1 ?o2 WHERE { + ?s ?p ?o1 , ?o2 . + FILTER(?o1 = ?o2 || false) +} diff --git a/testsuite/oxigraph-tests/sparql-optimization/something_or_false_output.rq b/testsuite/oxigraph-tests/sparql-optimization/something_or_false_output.rq new file mode 100644 index 00000000..95741032 --- /dev/null +++ b/testsuite/oxigraph-tests/sparql-optimization/something_or_false_output.rq @@ -0,0 +1,4 @@ +SELECT ?o1 ?o2 WHERE { + ?s ?p ?o1 , ?o2 . + FILTER(?o2 = ?o1) +} diff --git a/testsuite/oxigraph-tests/sparql-optimization/something_or_true_input.rq b/testsuite/oxigraph-tests/sparql-optimization/something_or_true_input.rq new file mode 100644 index 00000000..1a1c3ca6 --- /dev/null +++ b/testsuite/oxigraph-tests/sparql-optimization/something_or_true_input.rq @@ -0,0 +1,4 @@ +SELECT ?o1 ?o2 WHERE { + ?s ?p ?o1 , ?o2 . + FILTER(?o1 = ?o2 || true) +} diff --git a/testsuite/oxigraph-tests/sparql-optimization/something_or_true_output.rq b/testsuite/oxigraph-tests/sparql-optimization/something_or_true_output.rq new file mode 100644 index 00000000..7af27485 --- /dev/null +++ b/testsuite/oxigraph-tests/sparql-optimization/something_or_true_output.rq @@ -0,0 +1,3 @@ +SELECT ?o1 ?o2 WHERE { + ?s ?p ?o1 , ?o2 . +} diff --git a/testsuite/oxigraph-tests/sparql-optimization/true_and_something_input.rq b/testsuite/oxigraph-tests/sparql-optimization/true_and_something_input.rq new file mode 100644 index 00000000..3e34f239 --- /dev/null +++ b/testsuite/oxigraph-tests/sparql-optimization/true_and_something_input.rq @@ -0,0 +1,4 @@ +SELECT ?o1 ?o2 WHERE { + ?s ?p ?o1 , ?o2 . + FILTER(true && ?o1 = ?o2) +} diff --git a/testsuite/oxigraph-tests/sparql-optimization/true_and_something_output.rq b/testsuite/oxigraph-tests/sparql-optimization/true_and_something_output.rq new file mode 100644 index 00000000..95741032 --- /dev/null +++ b/testsuite/oxigraph-tests/sparql-optimization/true_and_something_output.rq @@ -0,0 +1,4 @@ +SELECT ?o1 ?o2 WHERE { + ?s ?p ?o1 , ?o2 . + FILTER(?o2 = ?o1) +} diff --git a/testsuite/oxigraph-tests/sparql-optimization/true_or_something_input.rq b/testsuite/oxigraph-tests/sparql-optimization/true_or_something_input.rq new file mode 100644 index 00000000..a47c54f7 --- /dev/null +++ b/testsuite/oxigraph-tests/sparql-optimization/true_or_something_input.rq @@ -0,0 +1,4 @@ +SELECT ?o1 ?o2 WHERE { + ?s ?p ?o1 , ?o2 . + FILTER(true || ?o1 = ?o2) +} diff --git a/testsuite/oxigraph-tests/sparql-optimization/true_or_something_output.rq b/testsuite/oxigraph-tests/sparql-optimization/true_or_something_output.rq new file mode 100644 index 00000000..7af27485 --- /dev/null +++ b/testsuite/oxigraph-tests/sparql-optimization/true_or_something_output.rq @@ -0,0 +1,3 @@ +SELECT ?o1 ?o2 WHERE { + ?s ?p ?o1 , ?o2 . +} diff --git a/testsuite/oxigraph-tests/sparql-optimization/unbound_bind_input.rq b/testsuite/oxigraph-tests/sparql-optimization/unbound_bind_input.rq new file mode 100644 index 00000000..25808725 --- /dev/null +++ b/testsuite/oxigraph-tests/sparql-optimization/unbound_bind_input.rq @@ -0,0 +1,3 @@ +SELECT ?o WHERE { + BIND(?a AS ?o) +} diff --git a/testsuite/oxigraph-tests/sparql-optimization/unbound_bind_output.rq b/testsuite/oxigraph-tests/sparql-optimization/unbound_bind_output.rq new file mode 100644 index 00000000..7ef17038 --- /dev/null +++ b/testsuite/oxigraph-tests/sparql-optimization/unbound_bind_output.rq @@ -0,0 +1,3 @@ +SELECT ?o WHERE { + VALUES () { () } +} diff --git a/testsuite/oxigraph-tests/sparql-optimization/unbound_filter_input.rq b/testsuite/oxigraph-tests/sparql-optimization/unbound_filter_input.rq new file mode 100644 index 00000000..273ed694 --- /dev/null +++ b/testsuite/oxigraph-tests/sparql-optimization/unbound_filter_input.rq @@ -0,0 +1,4 @@ +SELECT ?o WHERE { + ?s ?p ?o . + FILTER(?a) +} diff --git a/testsuite/oxigraph-tests/sparql-optimization/unbound_filter_output.rq b/testsuite/oxigraph-tests/sparql-optimization/unbound_filter_output.rq new file mode 100644 index 00000000..36ac8cf8 --- /dev/null +++ b/testsuite/oxigraph-tests/sparql-optimization/unbound_filter_output.rq @@ -0,0 +1,3 @@ +SELECT ?o WHERE { + VALUES () {} +} diff --git a/testsuite/src/sparql_evaluator.rs b/testsuite/src/sparql_evaluator.rs index 4790d175..dc68dff4 100644 --- a/testsuite/src/sparql_evaluator.rs +++ b/testsuite/src/sparql_evaluator.rs @@ -8,6 +8,7 @@ use oxigraph::model::vocab::*; use oxigraph::model::*; use oxigraph::sparql::*; use oxigraph::store::Store; +use sparopt::Optimizer; use std::collections::HashMap; use std::fmt::Write; use std::io::{self, Cursor}; @@ -67,6 +68,10 @@ pub fn register_sparql_tests(evaluator: &mut TestEvaluator) { "https://github.com/oxigraph/oxigraph/tests#NegativeTsvResultsSyntaxTest", evaluate_negative_tsv_result_syntax_test, ); + evaluator.register( + "https://github.com/oxigraph/oxigraph/tests#QueryOptimizationTest", + evaluate_query_optimization_test, + ); } fn evaluate_positive_syntax_test(test: &Test) -> Result<()> { @@ -717,3 +722,54 @@ fn load_dataset_to_store(url: &str, store: &Store) -> Result<()> { }?; Ok(()) } + +fn evaluate_query_optimization_test(test: &Test) -> Result<()> { + let action = test + .action + .as_deref() + .ok_or_else(|| anyhow!("No action found for test {test}"))?; + let actual = (&Optimizer::optimize_graph_pattern( + (&if let spargebra::Query::Select { pattern, .. } = + spargebra::Query::parse(&read_file_to_string(action)?, Some(action))? + { + pattern + } else { + bail!("Only SELECT queries are supported in query sparql-optimization tests") + }) + .into(), + )) + .into(); + let result = test + .result + .as_ref() + .ok_or_else(|| anyhow!("No tests result found"))?; + let expected = if let spargebra::Query::Select { pattern, .. } = + spargebra::Query::parse(&read_file_to_string(result)?, Some(result))? + { + pattern + } else { + bail!("Only SELECT queries are supported in query sparql-optimization tests") + }; + if expected == actual { + Ok(()) + } else { + bail!( + "Failure on {test}.\nDiff:\n{}\n", + format_diff( + &spargebra::Query::Select { + pattern: expected, + dataset: None, + base_iri: None + } + .to_sse(), + &spargebra::Query::Select { + pattern: actual, + dataset: None, + base_iri: None + } + .to_sse(), + "query" + ) + ) + } +} diff --git a/testsuite/tests/oxigraph.rs b/testsuite/tests/oxigraph.rs index 04360a9e..a37bfab2 100644 --- a/testsuite/tests/oxigraph.rs +++ b/testsuite/tests/oxigraph.rs @@ -3,10 +3,10 @@ use oxigraph_testsuite::evaluator::TestEvaluator; use oxigraph_testsuite::manifest::TestManifest; use oxigraph_testsuite::sparql_evaluator::register_sparql_tests; -fn run_testsuite(manifest_urls: Vec<&str>) -> Result<()> { +fn run_testsuite(manifest_url: &str) -> Result<()> { let mut evaluator = TestEvaluator::default(); register_sparql_tests(&mut evaluator); - let manifest = TestManifest::new(manifest_urls); + let manifest = TestManifest::new([manifest_url]); let results = evaluator.evaluate(manifest)?; let mut errors = Vec::default(); @@ -27,14 +27,15 @@ fn run_testsuite(manifest_urls: Vec<&str>) -> Result<()> { #[test] fn oxigraph_sparql_testsuite() -> Result<()> { - run_testsuite(vec![ - "https://github.com/oxigraph/oxigraph/tests/sparql/manifest.ttl", - ]) + run_testsuite("https://github.com/oxigraph/oxigraph/tests/sparql/manifest.ttl") } #[test] fn oxigraph_sparql_results_testsuite() -> Result<()> { - run_testsuite(vec![ - "https://github.com/oxigraph/oxigraph/tests/sparql-results/manifest.ttl", - ]) + run_testsuite("https://github.com/oxigraph/oxigraph/tests/sparql-results/manifest.ttl") +} + +#[test] +fn oxigraph_optimizer_testsuite() -> Result<()> { + run_testsuite("https://github.com/oxigraph/oxigraph/tests/sparql-optimization/manifest.ttl") } diff --git a/testsuite/tests/sparql.rs b/testsuite/tests/sparql.rs index ca59e397..d058bd71 100644 --- a/testsuite/tests/sparql.rs +++ b/testsuite/tests/sparql.rs @@ -67,6 +67,7 @@ fn sparql10_w3c_query_evaluation_testsuite() -> Result<()> { // We choose to simplify first the nested group patterns in OPTIONAL "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/optional-filter/manifest#dawg-optional-filter-005-not-simplified", // This test relies on naive iteration on the input file + "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/reduced/manifest#reduced-1", "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/reduced/manifest#reduced-2" ]) } From 81895cb6bc4024e58fc9905173f30db30955cea9 Mon Sep 17 00:00:00 2001 From: Tpt Date: Tue, 13 Jun 2023 20:46:06 +0200 Subject: [PATCH 007/217] xsd:integer: checked_div is integer division and not regular division according to XPath --- lib/oxsdatatypes/src/integer.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/oxsdatatypes/src/integer.rs b/lib/oxsdatatypes/src/integer.rs index 87d2dbd7..50f59d2b 100644 --- a/lib/oxsdatatypes/src/integer.rs +++ b/lib/oxsdatatypes/src/integer.rs @@ -50,7 +50,7 @@ impl Integer { }) } - /// [op:numeric-divide](https://www.w3.org/TR/xpath-functions-31/#func-numeric-divide) + /// [op:numeric-integer-divide](https://www.w3.org/TR/xpath-functions-31/#func-numeric-integer-divide) #[inline] pub fn checked_div(self, rhs: impl Into) -> Option { Some(Self { From 01caaa5d70bf84090a0b4d24c24e1ca13a0e6941 Mon Sep 17 00:00:00 2001 From: Tpt Date: Tue, 13 Jun 2023 20:53:48 +0200 Subject: [PATCH 008/217] calendar subtraction: it should return xsd:dayTimeDuration following XPath --- lib/src/sparql/eval.rs | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/lib/src/sparql/eval.rs b/lib/src/sparql/eval.rs index f9a8690b..8821fb1e 100644 --- a/lib/src/sparql/eval.rs +++ b/lib/src/sparql/eval.rs @@ -1089,15 +1089,9 @@ impl SimpleEvaluator { NumericBinaryOperands::Double(v1, v2) => (v1 - v2).into(), NumericBinaryOperands::Integer(v1, v2) => v1.checked_sub(v2)?.into(), NumericBinaryOperands::Decimal(v1, v2) => v1.checked_sub(v2)?.into(), - NumericBinaryOperands::DateTime(v1, v2) => { - Duration::from(v1.checked_sub(v2)?).into() - } - NumericBinaryOperands::Date(v1, v2) => { - Duration::from(v1.checked_sub(v2)?).into() - } - NumericBinaryOperands::Time(v1, v2) => { - Duration::from(v1.checked_sub(v2)?).into() - } + NumericBinaryOperands::DateTime(v1, v2) => v1.checked_sub(v2)?.into(), + NumericBinaryOperands::Date(v1, v2) => v1.checked_sub(v2)?.into(), + NumericBinaryOperands::Time(v1, v2) => v1.checked_sub(v2)?.into(), NumericBinaryOperands::Duration(v1, v2) => v1.checked_sub(v2)?.into(), NumericBinaryOperands::YearMonthDuration(v1, v2) => { v1.checked_sub(v2)?.into() From 5af06e926aa5df11a23141a4a61cbce22c3745fa Mon Sep 17 00:00:00 2001 From: Tpt Date: Tue, 13 Jun 2023 10:04:38 +0200 Subject: [PATCH 009/217] Removes deprecated methods from oxsdatatypes --- lib/oxsdatatypes/src/decimal.rs | 12 ----------- lib/oxsdatatypes/src/double.rs | 6 ------ lib/oxsdatatypes/src/duration.rs | 35 -------------------------------- lib/oxsdatatypes/src/float.rs | 6 ------ lib/oxsdatatypes/src/integer.rs | 10 --------- 5 files changed, 69 deletions(-) diff --git a/lib/oxsdatatypes/src/decimal.rs b/lib/oxsdatatypes/src/decimal.rs index 0fd4d3db..412ca2b8 100644 --- a/lib/oxsdatatypes/src/decimal.rs +++ b/lib/oxsdatatypes/src/decimal.rs @@ -2,7 +2,6 @@ use crate::{Boolean, Double, Float, Integer}; use std::error::Error; use std::fmt; use std::fmt::Write; -use std::ops::Neg; use std::str::FromStr; const DECIMAL_PART_DIGITS: u32 = 18; @@ -564,17 +563,6 @@ impl fmt::Display for Decimal { } } -impl Neg for Decimal { - type Output = Self; - - #[inline] - fn neg(self) -> Self { - Self { - value: self.value.neg(), - } - } -} - /// An error when parsing a [`Decimal`]. #[derive(Debug, Clone)] pub struct ParseDecimalError { diff --git a/lib/oxsdatatypes/src/double.rs b/lib/oxsdatatypes/src/double.rs index 64d3c786..6768c772 100644 --- a/lib/oxsdatatypes/src/double.rs +++ b/lib/oxsdatatypes/src/double.rs @@ -58,12 +58,6 @@ impl Double { self.value.is_nan() } - #[deprecated(note = "Use .is_nan()")] - #[inline] - pub fn is_naan(self) -> bool { - self.value.is_nan() - } - #[inline] pub fn is_finite(self) -> bool { self.value.is_finite() diff --git a/lib/oxsdatatypes/src/duration.rs b/lib/oxsdatatypes/src/duration.rs index 14ea6aac..29167121 100644 --- a/lib/oxsdatatypes/src/duration.rs +++ b/lib/oxsdatatypes/src/duration.rs @@ -3,7 +3,6 @@ use super::parser::*; use super::*; use std::cmp::Ordering; use std::fmt; -use std::ops::Neg; use std::str::FromStr; use std::time::Duration as StdDuration; @@ -232,18 +231,6 @@ impl PartialOrd for Duration { } } -impl Neg for Duration { - type Output = Self; - - #[inline] - fn neg(self) -> Self { - Self { - year_month: self.year_month.neg(), - day_time: self.day_time.neg(), - } - } -} - /// [XML Schema `yearMonthDuration` datatype](https://www.w3.org/TR/xmlschema11-2/#yearMonthDuration) /// /// It stores the duration as a number of months encoded using a [`i64`]. @@ -391,17 +378,6 @@ impl PartialOrd for Duration { } } -impl Neg for YearMonthDuration { - type Output = Self; - - #[inline] - fn neg(self) -> Self { - Self { - months: self.months.neg(), - } - } -} - /// [XML Schema `dayTimeDuration` datatype](https://www.w3.org/TR/xmlschema11-2/#dayTimeDuration) /// /// It stores the duration as a number of seconds encoded using a [`Decimal`]. @@ -602,17 +578,6 @@ impl PartialOrd for YearMonthDuration { } } -impl Neg for DayTimeDuration { - type Output = Self; - - #[inline] - fn neg(self) -> Self { - Self { - seconds: self.seconds.neg(), - } - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/lib/oxsdatatypes/src/float.rs b/lib/oxsdatatypes/src/float.rs index 86616a24..8c8602d3 100644 --- a/lib/oxsdatatypes/src/float.rs +++ b/lib/oxsdatatypes/src/float.rs @@ -53,12 +53,6 @@ impl Float { self.value.round().into() } - #[deprecated(note = "Use .is_nan()")] - #[inline] - pub fn is_naan(self) -> bool { - self.value.is_nan() - } - #[inline] pub fn is_nan(self) -> bool { self.value.is_nan() diff --git a/lib/oxsdatatypes/src/integer.rs b/lib/oxsdatatypes/src/integer.rs index 50f59d2b..957a4b24 100644 --- a/lib/oxsdatatypes/src/integer.rs +++ b/lib/oxsdatatypes/src/integer.rs @@ -1,7 +1,6 @@ use crate::{Boolean, Decimal, DecimalOverflowError, Double, Float}; use std::fmt; use std::num::ParseIntError; -use std::ops::Neg; use std::str::FromStr; /// [XML Schema `integer` datatype](https://www.w3.org/TR/xmlschema11-2/#integer) @@ -210,15 +209,6 @@ impl fmt::Display for Integer { } } -impl Neg for Integer { - type Output = Self; - - #[inline] - fn neg(self) -> Self { - (-self.value).into() - } -} - impl TryFrom for Integer { type Error = DecimalOverflowError; From 2281575c146f3065af59c2fc9324d7482a4d2294 Mon Sep 17 00:00:00 2001 From: Tpt Date: Tue, 13 Jun 2023 21:16:23 +0200 Subject: [PATCH 010/217] GraphName: implements Default --- lib/oxrdf/src/triple.rs | 6 ++++-- lib/spargebra/src/term.rs | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/oxrdf/src/triple.rs b/lib/oxrdf/src/triple.rs index c0328f46..850b1375 100644 --- a/lib/oxrdf/src/triple.rs +++ b/lib/oxrdf/src/triple.rs @@ -853,10 +853,11 @@ impl<'a> From> for Triple { /// A possible owned graph name. /// It is the union of [IRIs](https://www.w3.org/TR/rdf11-concepts/#dfn-iri), [blank nodes](https://www.w3.org/TR/rdf11-concepts/#dfn-blank-node), and the [default graph name](https://www.w3.org/TR/rdf11-concepts/#dfn-default-graph). -#[derive(Eq, PartialEq, Debug, Clone, Hash)] +#[derive(Eq, PartialEq, Debug, Clone, Hash, Default)] pub enum GraphName { NamedNode(NamedNode), BlankNode(BlankNode), + #[default] DefaultGraph, } @@ -940,10 +941,11 @@ impl From> for GraphName { /// A possible borrowed graph name. /// It is the union of [IRIs](https://www.w3.org/TR/rdf11-concepts/#dfn-iri), [blank nodes](https://www.w3.org/TR/rdf11-concepts/#dfn-blank-node), and the [default graph name](https://www.w3.org/TR/rdf11-concepts/#dfn-default-graph). -#[derive(Eq, PartialEq, Debug, Clone, Copy, Hash)] +#[derive(Eq, PartialEq, Debug, Clone, Copy, Hash, Default)] pub enum GraphNameRef<'a> { NamedNode(NamedNodeRef<'a>), BlankNode(BlankNodeRef<'a>), + #[default] DefaultGraph, } diff --git a/lib/spargebra/src/term.rs b/lib/spargebra/src/term.rs index 8c41fc1b..6a33294c 100644 --- a/lib/spargebra/src/term.rs +++ b/lib/spargebra/src/term.rs @@ -183,9 +183,10 @@ impl TryFrom for GroundTriple { /// A possible graph name. /// /// It is the union of [IRIs](https://www.w3.org/TR/rdf11-concepts/#dfn-iri) and the [default graph name](https://www.w3.org/TR/rdf11-concepts/#dfn-default-graph). -#[derive(Eq, PartialEq, Debug, Clone, Hash)] +#[derive(Eq, PartialEq, Debug, Clone, Hash, Default)] pub enum GraphName { NamedNode(NamedNode), + #[default] DefaultGraph, } From 76deca135c638d494cc593aab7f7cdf3fa6145d5 Mon Sep 17 00:00:00 2001 From: Tpt Date: Tue, 13 Jun 2023 21:33:19 +0200 Subject: [PATCH 011/217] BulkLoader: Uses thread::scope --- lib/src/storage/mod.rs | 92 +++++++++++++++++++++--------------------- 1 file changed, 46 insertions(+), 46 deletions(-) diff --git a/lib/src/storage/mod.rs b/lib/src/storage/mod.rs index 8a92e77f..f1c81414 100644 --- a/lib/src/storage/mod.rs +++ b/lib/src/storage/mod.rs @@ -28,11 +28,7 @@ use std::path::{Path, PathBuf}; #[cfg(not(target_family = "wasm"))] use std::sync::atomic::{AtomicU64, Ordering}; #[cfg(not(target_family = "wasm"))] -use std::sync::Arc; -#[cfg(not(target_family = "wasm"))] -use std::thread::spawn; -#[cfg(not(target_family = "wasm"))] -use std::thread::JoinHandle; +use std::thread; mod backend; mod binary_encoder; @@ -1253,44 +1249,49 @@ impl StorageBulkLoader { ) .into()); } - let mut threads = VecDeque::with_capacity(num_threads - 1); - let mut buffer = Vec::with_capacity(batch_size); - let done_counter = Arc::new(AtomicU64::new(0)); + let done_counter = AtomicU64::new(0); let mut done_and_displayed_counter = 0; - for quad in quads { - let quad = quad?; - buffer.push(quad); - if buffer.len() >= batch_size { - self.spawn_load_thread( - &mut buffer, - &mut threads, - &done_counter, - &mut done_and_displayed_counter, - num_threads, - batch_size, - )?; + thread::scope(|thread_scope| { + let mut threads = VecDeque::with_capacity(num_threads - 1); + let mut buffer = Vec::with_capacity(batch_size); + for quad in quads { + let quad = quad?; + buffer.push(quad); + if buffer.len() >= batch_size { + self.spawn_load_thread( + &mut buffer, + &mut threads, + thread_scope, + &done_counter, + &mut done_and_displayed_counter, + num_threads, + batch_size, + )?; + } } - } - self.spawn_load_thread( - &mut buffer, - &mut threads, - &done_counter, - &mut done_and_displayed_counter, - num_threads, - batch_size, - )?; - for thread in threads { - thread.join().unwrap()?; - self.on_possible_progress(&done_counter, &mut done_and_displayed_counter); - } - Ok(()) + self.spawn_load_thread( + &mut buffer, + &mut threads, + thread_scope, + &done_counter, + &mut done_and_displayed_counter, + num_threads, + batch_size, + )?; + for thread in threads { + thread.join().unwrap()?; + self.on_possible_progress(&done_counter, &mut done_and_displayed_counter); + } + Ok(()) + }) } - fn spawn_load_thread( - &self, + fn spawn_load_thread<'scope>( + &'scope self, buffer: &mut Vec, - threads: &mut VecDeque>>, - done_counter: &Arc, + threads: &mut VecDeque>>, + thread_scope: &'scope thread::Scope<'scope, '_>, + done_counter: &'scope AtomicU64, done_and_displayed_counter: &mut u64, num_threads: usize, batch_size: usize, @@ -1305,10 +1306,9 @@ impl StorageBulkLoader { } let mut buffer_to_load = Vec::with_capacity(batch_size); swap(buffer, &mut buffer_to_load); - let storage = self.storage.clone(); - let done_counter_clone = Arc::clone(done_counter); - threads.push_back(spawn(move || { - FileBulkLoader::new(storage, batch_size).load(buffer_to_load, &done_counter_clone) + let storage = &self.storage; + threads.push_back(thread_scope.spawn(move || { + FileBulkLoader::new(storage, batch_size).load(buffer_to_load, done_counter) })); Ok(()) } @@ -1326,8 +1326,8 @@ impl StorageBulkLoader { } #[cfg(not(target_family = "wasm"))] -struct FileBulkLoader { - storage: Storage, +struct FileBulkLoader<'a> { + storage: &'a Storage, id2str: HashMap>, quads: HashSet, triples: HashSet, @@ -1335,8 +1335,8 @@ struct FileBulkLoader { } #[cfg(not(target_family = "wasm"))] -impl FileBulkLoader { - fn new(storage: Storage, batch_size: usize) -> Self { +impl<'a> FileBulkLoader<'a> { + fn new(storage: &'a Storage, batch_size: usize) -> Self { Self { storage, id2str: HashMap::with_capacity(3 * batch_size), From 785df9b00b441241c9836634e240a4e2703fa0c5 Mon Sep 17 00:00:00 2001 From: Tpt Date: Tue, 13 Jun 2023 21:39:24 +0200 Subject: [PATCH 012/217] Makes use of io::read_to_string --- server/src/main.rs | 39 ++++++++++++--------------------------- 1 file changed, 12 insertions(+), 27 deletions(-) diff --git a/server/src/main.rs b/server/src/main.rs index f57ada51..afae10f7 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -471,10 +471,7 @@ pub fn main() -> anyhow::Result<()> { format!("Not able to read query file {}", query_file.display()) })? } else { - // TODO: use io::read_to_string - let mut query = String::new(); - stdin().lock().read_to_string(&mut query)?; - query + io::read_to_string(stdin().lock())? }; let query = Query::parse(&query, query_base.as_deref())?; let store = Store::open_read_only( @@ -621,10 +618,7 @@ pub fn main() -> anyhow::Result<()> { format!("Not able to read update file {}", update_file.display()) })? } else { - // TODO: use io::read_to_string - let mut update = String::new(); - stdin().lock().read_to_string(&mut update)?; - update + io::read_to_string(stdin().lock())? }; let update = Update::parse(&update, update_base.as_deref())?; let store = Store::open( @@ -862,16 +856,12 @@ fn handle_request( let content_type = content_type(request).ok_or_else(|| bad_request("No Content-Type given"))?; if content_type == "application/sparql-query" { - let mut buffer = String::new(); - request - .body_mut() - .take(MAX_SPARQL_BODY_SIZE) - .read_to_string(&mut buffer) + let query = io::read_to_string(request.body_mut().take(MAX_SPARQL_BODY_SIZE)) .map_err(bad_request)?; configure_and_evaluate_sparql_query( &store, &[url_query(request)], - Some(buffer), + Some(query), request, ) } else if content_type == "application/x-www-form-urlencoded" { @@ -898,16 +888,12 @@ fn handle_request( let content_type = content_type(request).ok_or_else(|| bad_request("No Content-Type given"))?; if content_type == "application/sparql-update" { - let mut buffer = String::new(); - request - .body_mut() - .take(MAX_SPARQL_BODY_SIZE) - .read_to_string(&mut buffer) + let update = io::read_to_string(request.body_mut().take(MAX_SPARQL_BODY_SIZE)) .map_err(bad_request)?; configure_and_evaluate_sparql_update( &store, &[url_query(request)], - Some(buffer), + Some(update), request, ) } else if content_type == "application/x-www-form-urlencoded" { @@ -1728,6 +1714,7 @@ mod tests { use oxhttp::model::Method; use predicates::prelude::*; use std::fs::remove_dir_all; + use std::io::read_to_string; fn cli_command() -> Result { Ok(Command::from_std( @@ -2748,18 +2735,16 @@ mod tests { } fn check_status(mut response: Response, expected_status: Status) -> Result<()> { - let mut buf = String::new(); - response.body_mut().read_to_string(&mut buf)?; - assert_eq!(response.status(), expected_status, "Error message: {buf}"); + let body = read_to_string(response.body_mut())?; + assert_eq!(response.status(), expected_status, "Error message: {body}"); Ok(()) } fn test_body(&self, request: Request, expected_body: &str) -> Result<()> { let mut response = self.exec(request); - let mut buf = String::new(); - response.body_mut().read_to_string(&mut buf)?; - assert_eq!(response.status(), Status::OK, "Error message: {buf}"); - assert_eq!(&buf, expected_body); + let body = read_to_string(response.body_mut())?; + assert_eq!(response.status(), Status::OK, "Error message: {body}"); + assert_eq!(&body, expected_body); Ok(()) } } From a27f31b84ec1eaa9e8aae17af4160180c425f1da Mon Sep 17 00:00:00 2001 From: Tpt Date: Tue, 13 Jun 2023 21:42:25 +0200 Subject: [PATCH 013/217] Python: Removes DefaultGraph.value Not useful, always returns the empty string --- python/src/model.rs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/python/src/model.rs b/python/src/model.rs index 26791351..9bebf1db 100644 --- a/python/src/model.rs +++ b/python/src/model.rs @@ -441,13 +441,6 @@ impl PyDefaultGraph { Self {} } - /// :return: the empty string. - /// :rtype: str - #[getter] - fn value(&self) -> &str { - "" - } - fn __str__(&self) -> &str { "DEFAULT" } From 6cc7488905289f4e1c4aed74cf97342b936844a3 Mon Sep 17 00:00:00 2001 From: Tpt Date: Tue, 13 Jun 2023 21:46:58 +0200 Subject: [PATCH 014/217] SPARQL: requires decimal digits for DECIMAL serialization Follows SPARQL 1.1 grammar --- lib/spargebra/src/parser.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spargebra/src/parser.rs b/lib/spargebra/src/parser.rs index fbe7af91..daeddd55 100644 --- a/lib/spargebra/src/parser.rs +++ b/lib/spargebra/src/parser.rs @@ -2174,7 +2174,7 @@ parser! { rule INTEGER() = ['0'..='9']+ - rule DECIMAL() = ['0'..='9']+ "." ['0'..='9']* / ['0'..='9']* "." ['0'..='9']+ + rule DECIMAL() = ['0'..='9']* "." ['0'..='9']+ rule DOUBLE() = (['0'..='9']+ "." ['0'..='9']* / "." ['0'..='9']+ / ['0'..='9']+) EXPONENT() From a1cbfdf67d304758e50a6ac419c75088ac37a2fa Mon Sep 17 00:00:00 2001 From: Tpt Date: Wed, 14 Jun 2023 17:31:18 +0200 Subject: [PATCH 015/217] Improves some code style details --- js/src/model.rs | 9 ++++----- js/src/store.rs | 9 ++++----- lib/oxrdf/src/blank_node.rs | 8 ++++---- lib/oxrdf/src/variable.rs | 6 +++--- lib/sparesults/src/csv.rs | 5 ++--- lib/src/sparql/http/dummy.rs | 2 +- lib/src/sparql/model.rs | 7 +++---- lib/src/storage/binary_encoder.rs | 29 ++++++++++++++--------------- lib/src/storage/numeric_encoder.rs | 12 ++++++------ lib/tests/store.rs | 13 ++++++------- 10 files changed, 47 insertions(+), 53 deletions(-) diff --git a/js/src/model.rs b/js/src/model.rs index 753ab66e..323b5978 100644 --- a/js/src/model.rs +++ b/js/src/model.rs @@ -56,7 +56,7 @@ pub fn literal( #[wasm_bindgen(js_name = defaultGraph)] pub fn default_graph() -> JsDefaultGraph { - JsDefaultGraph {} + JsDefaultGraph } #[wasm_bindgen(js_name = variable)] @@ -532,7 +532,7 @@ impl From for JsTerm { match name { GraphName::NamedNode(node) => node.into(), GraphName::BlankNode(node) => node.into(), - GraphName::DefaultGraph => Self::DefaultGraph(JsDefaultGraph {}), + GraphName::DefaultGraph => Self::DefaultGraph(JsDefaultGraph), } } } @@ -744,7 +744,7 @@ impl FromJsConverter { )) } } - "DefaultGraph" => Ok(JsTerm::DefaultGraph(JsDefaultGraph {})), + "DefaultGraph" => Ok(JsTerm::DefaultGraph(JsDefaultGraph)), "Variable" => Ok(Variable::new( Reflect::get(value, &self.value)? .as_string() @@ -754,8 +754,7 @@ impl FromJsConverter { .into()), "Quad" => Ok(self.to_quad(value)?.into()), _ => Err(format_err!( - "The termType {} is not supported by Oxigraph", - term_type + "The termType {term_type} is not supported by Oxigraph" )), } } else if term_type.is_undefined() { diff --git a/js/src/store.rs b/js/src/store.rs index adee6eef..12d4a039 100644 --- a/js/src/store.rs +++ b/js/src/store.rs @@ -8,7 +8,6 @@ use oxigraph::io::{DatasetFormat, GraphFormat}; use oxigraph::model::*; use oxigraph::sparql::QueryResults; use oxigraph::store::Store; -use std::io::Cursor; use wasm_bindgen::prelude::*; #[wasm_bindgen(js_name = Store)] @@ -171,7 +170,7 @@ impl JsStore { if let Some(graph_format) = GraphFormat::from_media_type(mime_type) { self.store .load_graph( - Cursor::new(data), + data.as_bytes(), graph_format, &to_graph_name.unwrap_or(GraphName::DefaultGraph), base_iri.as_deref(), @@ -184,10 +183,10 @@ impl JsStore { )); } self.store - .load_dataset(Cursor::new(data), dataset_format, base_iri.as_deref()) + .load_dataset(data.as_bytes(), dataset_format, base_iri.as_deref()) .map_err(to_err) } else { - Err(format_err!("Not supported MIME type: {}", mime_type)) + Err(format_err!("Not supported MIME type: {mime_type}")) } } @@ -218,7 +217,7 @@ impl JsStore { .dump_dataset(&mut buffer, dataset_format) .map_err(to_err)?; } else { - return Err(format_err!("Not supported MIME type: {}", mime_type)); + return Err(format_err!("Not supported MIME type: {mime_type}")); } String::from_utf8(buffer).map_err(to_err) } diff --git a/lib/oxrdf/src/blank_node.rs b/lib/oxrdf/src/blank_node.rs index 5b9172c2..0b485beb 100644 --- a/lib/oxrdf/src/blank_node.rs +++ b/lib/oxrdf/src/blank_node.rs @@ -264,7 +264,7 @@ impl IdStr { fn validate_blank_node_identifier(id: &str) -> Result<(), BlankNodeIdParseError> { let mut chars = id.chars(); - let front = chars.next().ok_or(BlankNodeIdParseError {})?; + let front = chars.next().ok_or(BlankNodeIdParseError)?; match front { '0'..='9' | '_' @@ -283,7 +283,7 @@ fn validate_blank_node_identifier(id: &str) -> Result<(), BlankNodeIdParseError> | '\u{F900}'..='\u{FDCF}' | '\u{FDF0}'..='\u{FFFD}' | '\u{10000}'..='\u{EFFFF}' => (), - _ => return Err(BlankNodeIdParseError {}), + _ => return Err(BlankNodeIdParseError), } for c in chars { match c { @@ -309,13 +309,13 @@ fn validate_blank_node_identifier(id: &str) -> Result<(), BlankNodeIdParseError> | '\u{F900}'..='\u{FDCF}' | '\u{FDF0}'..='\u{FFFD}' | '\u{10000}'..='\u{EFFFF}' => (), - _ => return Err(BlankNodeIdParseError {}), + _ => return Err(BlankNodeIdParseError), } } // Could not end with a dot if id.ends_with('.') { - Err(BlankNodeIdParseError {}) + Err(BlankNodeIdParseError) } else { Ok(()) } diff --git a/lib/oxrdf/src/variable.rs b/lib/oxrdf/src/variable.rs index af055bb9..7b9cd732 100644 --- a/lib/oxrdf/src/variable.rs +++ b/lib/oxrdf/src/variable.rs @@ -169,7 +169,7 @@ impl PartialOrd> for Variable { fn validate_variable_identifier(id: &str) -> Result<(), VariableNameParseError> { let mut chars = id.chars(); - let front = chars.next().ok_or(VariableNameParseError {})?; + let front = chars.next().ok_or(VariableNameParseError)?; match front { '0'..='9' | '_' @@ -188,7 +188,7 @@ fn validate_variable_identifier(id: &str) -> Result<(), VariableNameParseError> | '\u{F900}'..='\u{FDCF}' | '\u{FDF0}'..='\u{FFFD}' | '\u{10000}'..='\u{EFFFF}' => (), - _ => return Err(VariableNameParseError {}), + _ => return Err(VariableNameParseError), } for c in chars { match c { @@ -211,7 +211,7 @@ fn validate_variable_identifier(id: &str) -> Result<(), VariableNameParseError> | '\u{F900}'..='\u{FDCF}' | '\u{FDF0}'..='\u{FFFD}' | '\u{10000}'..='\u{EFFFF}' => (), - _ => return Err(VariableNameParseError {}), + _ => return Err(VariableNameParseError), } } Ok(()) diff --git a/lib/sparesults/src/csv.rs b/lib/sparesults/src/csv.rs index b365c4ac..7d737e5b 100644 --- a/lib/sparesults/src/csv.rs +++ b/lib/sparesults/src/csv.rs @@ -375,7 +375,6 @@ impl TsvSolutionsReader { mod tests { use super::*; use std::error::Error; - use std::io::Cursor; use std::rc::Rc; use std::str; @@ -466,7 +465,7 @@ mod tests { if let TsvQueryResultsReader::Solutions { solutions: mut solutions_iter, variables: actual_variables, - } = TsvQueryResultsReader::read(Cursor::new(result))? + } = TsvQueryResultsReader::read(result.as_slice())? { assert_eq!(actual_variables.as_slice(), variables.as_slice()); let mut rows = Vec::new(); @@ -499,7 +498,7 @@ mod tests { bad_tsvs.push(&a_lot_of_strings); for bad_tsv in bad_tsvs { if let Ok(TsvQueryResultsReader::Solutions { mut solutions, .. }) = - TsvQueryResultsReader::read(Cursor::new(bad_tsv)) + TsvQueryResultsReader::read(bad_tsv.as_bytes()) { while let Ok(Some(_)) = solutions.read_next() {} } diff --git a/lib/src/sparql/http/dummy.rs b/lib/src/sparql/http/dummy.rs index 3eb47e62..dc8516a6 100644 --- a/lib/src/sparql/http/dummy.rs +++ b/lib/src/sparql/http/dummy.rs @@ -7,7 +7,7 @@ pub struct Client; impl Client { pub fn new(_timeout: Option, _redirection_limit: usize) -> Self { - Self {} + Self } #[allow(clippy::unused_self)] diff --git a/lib/src/sparql/model.rs b/lib/src/sparql/model.rs index 9525a260..b5b1a650 100644 --- a/lib/src/sparql/model.rs +++ b/lib/src/sparql/model.rs @@ -98,16 +98,15 @@ impl QueryResults { /// use oxigraph::store::Store; /// use oxigraph::io::GraphFormat; /// use oxigraph::model::*; - /// use std::io::Cursor; /// - /// let graph = " .\n".as_bytes(); + /// let graph = " .\n"; /// /// let store = Store::new()?; - /// store.load_graph(Cursor::new(graph), GraphFormat::NTriples, GraphNameRef::DefaultGraph, None)?; + /// store.load_graph(graph.as_bytes(), GraphFormat::NTriples, GraphNameRef::DefaultGraph, None)?; /// /// let mut results = Vec::new(); /// store.query("CONSTRUCT WHERE { ?s ?p ?o }")?.write_graph(&mut results, GraphFormat::NTriples)?; - /// assert_eq!(results, graph); + /// assert_eq!(results, graph.as_bytes()); /// # Result::<_,Box>::Ok(()) /// ``` pub fn write_graph( diff --git a/lib/src/storage/binary_encoder.rs b/lib/src/storage/binary_encoder.rs index 4e888c2f..cd2272dd 100644 --- a/lib/src/storage/binary_encoder.rs +++ b/lib/src/storage/binary_encoder.rs @@ -2,7 +2,7 @@ use crate::storage::error::{CorruptionError, StorageError}; use crate::storage::numeric_encoder::{EncodedQuad, EncodedTerm, EncodedTriple, StrHash}; use crate::storage::small_string::SmallString; use oxsdatatypes::*; -use std::io::{Cursor, Read}; +use std::io::Read; use std::mem::size_of; #[cfg(not(target_family = "wasm"))] @@ -62,24 +62,23 @@ pub enum QuadEncoding { } impl QuadEncoding { - pub fn decode(self, buffer: &[u8]) -> Result { - let mut cursor = Cursor::new(&buffer); + pub fn decode(self, mut buffer: &[u8]) -> Result { match self { - Self::Spog => cursor.read_spog_quad(), - Self::Posg => cursor.read_posg_quad(), - Self::Ospg => cursor.read_ospg_quad(), - Self::Gspo => cursor.read_gspo_quad(), - Self::Gpos => cursor.read_gpos_quad(), - Self::Gosp => cursor.read_gosp_quad(), - Self::Dspo => cursor.read_dspo_quad(), - Self::Dpos => cursor.read_dpos_quad(), - Self::Dosp => cursor.read_dosp_quad(), + Self::Spog => buffer.read_spog_quad(), + Self::Posg => buffer.read_posg_quad(), + Self::Ospg => buffer.read_ospg_quad(), + Self::Gspo => buffer.read_gspo_quad(), + Self::Gpos => buffer.read_gpos_quad(), + Self::Gosp => buffer.read_gosp_quad(), + Self::Dspo => buffer.read_dspo_quad(), + Self::Dpos => buffer.read_dpos_quad(), + Self::Dosp => buffer.read_dosp_quad(), } } } -pub fn decode_term(buffer: &[u8]) -> Result { - Cursor::new(&buffer).read_term() +pub fn decode_term(mut buffer: &[u8]) -> Result { + buffer.read_term() } pub trait TermReader { @@ -740,7 +739,7 @@ mod tests { let mut buffer = Vec::new(); write_term(&mut buffer, &encoded); - assert_eq!(encoded, Cursor::new(&buffer).read_term().unwrap()); + assert_eq!(encoded, buffer.as_slice().read_term().unwrap()); } } } diff --git a/lib/src/storage/numeric_encoder.rs b/lib/src/storage/numeric_encoder.rs index 92fc1b5b..19632624 100644 --- a/lib/src/storage/numeric_encoder.rs +++ b/lib/src/storage/numeric_encoder.rs @@ -717,13 +717,13 @@ pub fn insert_term Result<(), StorageError>>( if let EncodedTerm::NamedNode { iri_id } = encoded { insert_str(iri_id, node.as_str()) } else { - unreachable!("Invalid term encoding {:?} for {}", encoded, term) + unreachable!("Invalid term encoding {encoded:?} for {term}") } } TermRef::BlankNode(node) => match encoded { EncodedTerm::BigBlankNode { id_id } => insert_str(id_id, node.as_str()), EncodedTerm::SmallBlankNode(..) | EncodedTerm::NumericalBlankNode { .. } => Ok(()), - _ => unreachable!("Invalid term encoding {:?} for {}", encoded, term), + _ => unreachable!("Invalid term encoding {encoded:?} for {term}"), }, TermRef::Literal(literal) => match encoded { EncodedTerm::BigStringLiteral { value_id } @@ -734,7 +734,7 @@ pub fn insert_term Result<(), StorageError>>( if let Some(language) = literal.language() { insert_str(language_id, language) } else { - unreachable!("Invalid term encoding {:?} for {}", encoded, term) + unreachable!("Invalid term encoding {encoded:?} for {term}") } } EncodedTerm::BigBigLangStringLiteral { @@ -745,7 +745,7 @@ pub fn insert_term Result<(), StorageError>>( if let Some(language) = literal.language() { insert_str(language_id, language) } else { - unreachable!("Invalid term encoding {:?} for {}", encoded, term) + unreachable!("Invalid term encoding {encoded:?} for {term}") } } EncodedTerm::SmallTypedLiteral { datatype_id, .. } => { @@ -776,7 +776,7 @@ pub fn insert_term Result<(), StorageError>>( | EncodedTerm::DurationLiteral(..) | EncodedTerm::YearMonthDurationLiteral(..) | EncodedTerm::DayTimeDurationLiteral(..) => Ok(()), - _ => unreachable!("Invalid term encoding {:?} for {}", encoded, term), + _ => unreachable!("Invalid term encoding {encoded:?} for {term}"), }, TermRef::Triple(triple) => { if let EncodedTerm::Triple(encoded) = encoded { @@ -788,7 +788,7 @@ pub fn insert_term Result<(), StorageError>>( )?; insert_term(triple.object.as_ref(), &encoded.object, insert_str) } else { - unreachable!("Invalid term encoding {:?} for {}", encoded, term) + unreachable!("Invalid term encoding {encoded:?} for {term}") } } } diff --git a/lib/tests/store.rs b/lib/tests/store.rs index 5f8a6809..7328ef4a 100644 --- a/lib/tests/store.rs +++ b/lib/tests/store.rs @@ -9,7 +9,6 @@ use std::env::temp_dir; use std::error::Error; #[cfg(not(target_family = "wasm"))] use std::fs::{create_dir, remove_dir_all, File}; -use std::io::Cursor; #[cfg(not(target_family = "wasm"))] use std::io::Write; #[cfg(target_os = "linux")] @@ -109,7 +108,7 @@ fn quads(graph_name: impl Into>) -> Vec> fn test_load_graph() -> Result<(), Box> { let store = Store::new()?; store.load_graph( - Cursor::new(DATA), + DATA.as_bytes(), GraphFormat::Turtle, GraphNameRef::DefaultGraph, None, @@ -126,7 +125,7 @@ fn test_load_graph() -> Result<(), Box> { fn test_bulk_load_graph() -> Result<(), Box> { let store = Store::new()?; store.bulk_loader().load_graph( - Cursor::new(DATA), + DATA.as_bytes(), GraphFormat::Turtle, GraphNameRef::DefaultGraph, None, @@ -143,7 +142,7 @@ fn test_bulk_load_graph() -> Result<(), Box> { fn test_bulk_load_graph_lenient() -> Result<(), Box> { let store = Store::new()?; store.bulk_loader().on_parse_error(|_| Ok(())).load_graph( - Cursor::new(b" .\n ."), + b" .\n .".as_slice(), GraphFormat::NTriples, GraphNameRef::DefaultGraph, None, @@ -162,7 +161,7 @@ fn test_bulk_load_graph_lenient() -> Result<(), Box> { #[test] fn test_load_dataset() -> Result<(), Box> { let store = Store::new()?; - store.load_dataset(Cursor::new(GRAPH_DATA), DatasetFormat::TriG, None)?; + store.load_dataset(GRAPH_DATA.as_bytes(), DatasetFormat::TriG, None)?; for q in quads(NamedNodeRef::new_unchecked( "http://www.wikidata.org/wiki/Special:EntityData/Q90", )) { @@ -178,7 +177,7 @@ fn test_bulk_load_dataset() -> Result<(), Box> { let store = Store::new()?; store .bulk_loader() - .load_dataset(Cursor::new(GRAPH_DATA), DatasetFormat::TriG, None)?; + .load_dataset(GRAPH_DATA.as_bytes(), DatasetFormat::TriG, None)?; let graph_name = NamedNodeRef::new_unchecked("http://www.wikidata.org/wiki/Special:EntityData/Q90"); for q in quads(graph_name) { @@ -194,7 +193,7 @@ fn test_load_graph_generates_new_blank_nodes() -> Result<(), Box> { let store = Store::new()?; for _ in 0..2 { store.load_graph( - Cursor::new("_:a ."), + "_:a .".as_bytes(), GraphFormat::NTriples, GraphNameRef::DefaultGraph, None, From 71b1768d282d13825b6fb66f77f60dba0fa1ad4d Mon Sep 17 00:00:00 2001 From: Tpt Date: Sun, 14 May 2023 16:25:44 +0200 Subject: [PATCH 016/217] New N3/Turtle/TriG/N-Triple/N-Quad parsers and serializers - Compatible with async IO - Turtle/TriG parser recovery on simple errors --- .clusterfuzzlite/build.sh | 6 +- .github/workflows/tests.yml | 6 +- .gitmodules | 4 + Cargo.lock | 16 +- Cargo.toml | 1 + fuzz/Cargo.toml | 13 + fuzz/fuzz_targets/n3.rs | 28 + fuzz/fuzz_targets/nquads.rs | 49 + fuzz/fuzz_targets/trig.rs | 53 + lib/Cargo.toml | 8 +- lib/benches/store.rs | 6 +- lib/oxttl/Cargo.toml | 27 + lib/oxttl/src/lexer.rs | 938 ++ lib/oxttl/src/lib.rs | 19 + lib/oxttl/src/line_formats.rs | 305 + lib/oxttl/src/n3.rs | 1035 ++ lib/oxttl/src/nquads.rs | 393 + lib/oxttl/src/ntriples.rs | 389 + lib/oxttl/src/terse.rs | 932 ++ lib/oxttl/src/toolkit/lexer.rs | 280 + lib/oxttl/src/toolkit/mod.rs | 11 + lib/oxttl/src/toolkit/parser.rs | 244 + lib/oxttl/src/trig.rs | 666 + lib/oxttl/src/turtle.rs | 462 + lib/src/io/error.rs | 30 +- lib/src/io/mod.rs | 9 +- lib/src/io/read.rs | 318 +- lib/src/io/write.rs | 151 +- lib/src/sparql/update.rs | 2 +- lib/src/store.rs | 16 +- python/src/io.rs | 8 +- python/tests/test_io.py | 2 +- python/tests/test_store.py | 3 +- server/src/main.rs | 2 +- testsuite/Cargo.toml | 10 + testsuite/N3 | 1 + testsuite/benches/parser.rs | 194 + .../parser-recovery/invalid_bnode.nt | 2 + .../parser-recovery/invalid_iri.nt | 2 + .../parser-recovery/invalid_string.nt | 2 + .../parser-recovery/iri2_spo.nt | 2 + .../parser-recovery/iri2_string_spo.nt | 2 + .../oxigraph-tests/parser-recovery/iri_spo.nt | 1 + .../parser-recovery/iri_string_spo.nt | 1 + .../parser-recovery/manifest.ttl | 129 + ...ssing_dot_at_end_of_triple_with_iri_end.nt | 1 + ...ng_dot_at_end_of_triple_with_iri_middle.nt | 2 + ...ng_dot_at_end_of_triple_with_string_end.nt | 1 + ...dot_at_end_of_triple_with_string_middle.nt | 2 + .../parser/at_keywords_as_lang_tag.nt | 2 + .../parser/at_keywords_as_lang_tag.ttl | 3 + testsuite/oxigraph-tests/parser/bad_lang.ttl | 1 + .../oxigraph-tests/parser/bad_parentheses.ttl | 2 + .../parser/blank_node_with_linebreak.nt | 2 + .../parser/blank_node_with_linebreak.ttl | 6 + .../parser/keyword_vs_prefix.nq | 3 + .../parser/keyword_vs_prefix.nt | 2 + .../parser/keyword_vs_prefix.trig | 10 + .../parser/keyword_vs_prefix.ttl | 8 + .../parser/language_normalization.nt | 1 + .../parser/language_normalization.rdf | 4 + .../parser/language_normalization.ttl | 1 + .../parser/literal_value_space.nt | 1 + .../parser/literal_value_space.rdf | 7 + testsuite/oxigraph-tests/parser/manifest.ttl | 90 + .../oxigraph-tests/parser/no_end_line_jump.nt | 1 + .../oxigraph-tests/parser/xml_entities.nt | 1 + .../oxigraph-tests/parser/xml_entities.rdf | 10 + .../parser/xml_nested_entities.nt | 1 + .../parser/xml_nested_entities.rdf | 15 + testsuite/serd-tests/LICENSE | 13 + testsuite/serd-tests/README.md | 1 + testsuite/serd-tests/bad/bad-00.ttl | 2 + testsuite/serd-tests/bad/bad-01.ttl | 3 + testsuite/serd-tests/bad/bad-02.ttl | 3 + testsuite/serd-tests/bad/bad-03.ttl | 3 + testsuite/serd-tests/bad/bad-04.ttl | 3 + testsuite/serd-tests/bad/bad-05.ttl | 4 + testsuite/serd-tests/bad/bad-06.ttl | 3 + testsuite/serd-tests/bad/bad-07.ttl | 4 + testsuite/serd-tests/bad/bad-08.ttl | 2 + testsuite/serd-tests/bad/bad-09.ttl | 3 + testsuite/serd-tests/bad/bad-10.ttl | 3 + testsuite/serd-tests/bad/bad-11.ttl | 3 + testsuite/serd-tests/bad/bad-12.ttl | 3 + testsuite/serd-tests/bad/bad-13.ttl | 3 + testsuite/serd-tests/bad/bad-14.ttl | 6 + testsuite/serd-tests/bad/bad-base.ttl | 1 + testsuite/serd-tests/bad/bad-blank-syntax.ttl | 1 + testsuite/serd-tests/bad/bad-blank.ttl | 3 + testsuite/serd-tests/bad/bad-bom.ttl | 3 + .../serd-tests/bad/bad-char-in-local.ttl | 3 + .../serd-tests/bad/bad-char-in-prefix.ttl | 1 + testsuite/serd-tests/bad/bad-char-in-uri.ttl | 1 + .../serd-tests/bad/bad-datatype-syntax.ttl | 1 + testsuite/serd-tests/bad/bad-datatype.ttl | 1 + .../serd-tests/bad/bad-dot-after-subject.ttl | 1 + .../serd-tests/bad/bad-dot-in-collection.ttl | 1 + .../serd-tests/bad/bad-eof-after-quotes.ttl | 3 + .../bad/bad-eof-at-string-start.ttl | 3 + testsuite/serd-tests/bad/bad-eof-in-blank.ttl | 3 + .../serd-tests/bad/bad-eof-in-escape.ttl | 3 + .../serd-tests/bad/bad-eof-in-lang-suffix.ttl | 3 + testsuite/serd-tests/bad/bad-eof-in-lang.ttl | 3 + testsuite/serd-tests/bad/bad-eof-in-list.ttl | 3 + .../serd-tests/bad/bad-eof-in-long-string.ttl | 3 + .../serd-tests/bad/bad-eof-in-object-list.ttl | 2 + .../bad/bad-eof-in-object-list2.ttl | 2 + .../bad/bad-eof-in-predicate-list.ttl | 2 + .../serd-tests/bad/bad-eof-in-string.ttl | 3 + .../bad/bad-eof-in-triple-quote.ttl | 3 + .../serd-tests/bad/bad-eof-in-uri-scheme.nt | 1 + testsuite/serd-tests/bad/bad-eof-in-uri.ttl | 3 + testsuite/serd-tests/bad/bad-escape.ttl | 1 + .../serd-tests/bad/bad-ext-namedblank-op.ttl | 3 + .../serd-tests/bad/bad-graph-blank-label.trig | 3 + testsuite/serd-tests/bad/bad-hex-digit.ttl | 1 + testsuite/serd-tests/bad/bad-id-clash.ttl | 2 + testsuite/serd-tests/bad/bad-lang.ttl | 1 + testsuite/serd-tests/bad/bad-list.ttl | 1 + testsuite/serd-tests/bad/bad-list2.ttl | 3 + .../bad/bad-long-literal-in-list.ttl | 1 + testsuite/serd-tests/bad/bad-missing-semi.ttl | 3 + .../serd-tests/bad/bad-missing-uri-scheme.nt | 1 + .../serd-tests/bad/bad-misspelled-base.ttl | 1 + .../serd-tests/bad/bad-misspelled-prefix.ttl | 1 + testsuite/serd-tests/bad/bad-namespace.ttl | 1 + testsuite/serd-tests/bad/bad-ns.ttl | 1 + testsuite/serd-tests/bad/bad-null-byte.ttl | Bin 0 -> 27 bytes testsuite/serd-tests/bad/bad-num.ttl | 1 + testsuite/serd-tests/bad/bad-object.ttl | 3 + testsuite/serd-tests/bad/bad-object2.ttl | 3 + testsuite/serd-tests/bad/bad-pn-escape.ttl | 2 + testsuite/serd-tests/bad/bad-prefix.ttl | 1 + testsuite/serd-tests/bad/bad-quote-in-uri.ttl | 1 + .../bad/bad-semicolon-after-subject.ttl | 1 + testsuite/serd-tests/bad/bad-string.ttl | 1 + testsuite/serd-tests/bad/bad-subject.ttl | 1 + testsuite/serd-tests/bad/bad-uri-escape.ttl | 1 + .../serd-tests/bad/bad-uri-scheme-start.nt | 1 + testsuite/serd-tests/bad/bad-uri-scheme.nt | 1 + testsuite/serd-tests/bad/bad-uri-truncated.nt | 1 + testsuite/serd-tests/bad/bad-verb.ttl | 2 + .../serd-tests/bad/invalid-char-in-local.ttl | 3 + .../serd-tests/bad/invalid-char-in-prefix.ttl | 1 + testsuite/serd-tests/bad/manifest.ttl | 449 + testsuite/serd-tests/good/README.txt | 20 + testsuite/serd-tests/good/UTF-8.nt | 2 + testsuite/serd-tests/good/UTF-8.ttl | 219 + testsuite/serd-tests/good/base.nt | 1 + testsuite/serd-tests/good/base.ttl | 3 + testsuite/serd-tests/good/manifest.ttl | 319 + testsuite/serd-tests/good/qualify-in.ttl | 3 + testsuite/serd-tests/good/qualify-out.ttl | 5 + testsuite/serd-tests/good/test-14.nt | 10000 +++++++++++++++ testsuite/serd-tests/good/test-14.ttl | 10002 ++++++++++++++++ testsuite/serd-tests/good/test-15.nt | 10000 +++++++++++++++ testsuite/serd-tests/good/test-15.ttl | 3 + testsuite/serd-tests/good/test-16.nt | 10000 +++++++++++++++ testsuite/serd-tests/good/test-16.ttl | 10002 ++++++++++++++++ testsuite/serd-tests/good/test-18.nt | 2 + testsuite/serd-tests/good/test-18.ttl | 9 + testsuite/serd-tests/good/test-30.nt | 5 + testsuite/serd-tests/good/test-30.ttl | 12 + .../good/test-a-without-whitespace.nt | 6 + .../good/test-a-without-whitespace.ttl | 6 + testsuite/serd-tests/good/test-backspace.nt | 3 + testsuite/serd-tests/good/test-backspace.ttl | 5 + testsuite/serd-tests/good/test-base-nopath.nt | 1 + .../serd-tests/good/test-base-nopath.ttl | 3 + testsuite/serd-tests/good/test-base-query.nt | 1 + testsuite/serd-tests/good/test-base-query.ttl | 3 + testsuite/serd-tests/good/test-blank-cont.nt | 4 + testsuite/serd-tests/good/test-blank-cont.ttl | 4 + .../serd-tests/good/test-blank-in-list.nt | 4 + .../serd-tests/good/test-blank-in-list.ttl | 2 + .../good/test-blank-node-statement.nt | 1 + .../good/test-blank-node-statement.ttl | 3 + testsuite/serd-tests/good/test-blankdot.nt | 1 + testsuite/serd-tests/good/test-blankdot.ttl | 1 + testsuite/serd-tests/good/test-bom.nt | 1 + testsuite/serd-tests/good/test-bom.ttl | 3 + testsuite/serd-tests/good/test-cr.nt | 1 + testsuite/serd-tests/good/test-cr.ttl | 2 + testsuite/serd-tests/good/test-delete.nt | 2 + testsuite/serd-tests/good/test-delete.ttl | 2 + .../serd-tests/good/test-digit-start-pname.nt | 1 + .../good/test-digit-start-pname.ttl | 3 + .../serd-tests/good/test-empty-path-base.nt | 1 + .../serd-tests/good/test-empty-path-base.ttl | 3 + testsuite/serd-tests/good/test-empty.nt | 0 testsuite/serd-tests/good/test-empty.ttl | 0 .../serd-tests/good/test-eof-at-page-end.nt | 1 + .../serd-tests/good/test-eof-at-page-end.ttl | 85 + testsuite/serd-tests/good/test-escapes.nt | 2 + testsuite/serd-tests/good/test-escapes.ttl | 2 + .../good/test-ext-namedblank-iri.nt | 2 + .../good/test-ext-namedblank-iri.ttl | 3 + .../good/test-ext-namedblank-prefix.nt | 2 + .../good/test-ext-namedblank-prefix.ttl | 3 + testsuite/serd-tests/good/test-form-feed.nt | 2 + testsuite/serd-tests/good/test-form-feed.ttl | 2 + testsuite/serd-tests/good/test-id.nt | 2 + testsuite/serd-tests/good/test-id.ttl | 4 + testsuite/serd-tests/good/test-lang.nt | 4 + testsuite/serd-tests/good/test-lang.ttl | 6 + .../serd-tests/good/test-list-in-blank.nt | 6 + .../serd-tests/good/test-list-in-blank.ttl | 2 + .../serd-tests/good/test-list-subject.nt | 6 + .../serd-tests/good/test-list-subject.ttl | 2 + testsuite/serd-tests/good/test-list.nt | 1 + testsuite/serd-tests/good/test-list.ttl | 1 + .../good/test-local-name-ends-with-dot.ttl | 3 + testsuite/serd-tests/good/test-long-string.nt | 1 + .../serd-tests/good/test-long-string.ttl | 7 + testsuite/serd-tests/good/test-no-spaces.nt | 4 + testsuite/serd-tests/good/test-no-spaces.ttl | 3 + .../serd-tests/good/test-non-curie-uri.nt | 1 + .../serd-tests/good/test-non-curie-uri.ttl | 3 + testsuite/serd-tests/good/test-num.nt | 10 + testsuite/serd-tests/good/test-num.ttl | 12 + .../good/test-out-of-range-unicode.nt | 1 + .../good/test-out-of-range-unicode.ttl | 1 + testsuite/serd-tests/good/test-prefix.nt | 6 + testsuite/serd-tests/good/test-prefix.ttl | 8 + testsuite/serd-tests/good/test-pretty.nt | 46 + testsuite/serd-tests/good/test-pretty.ttl | 44 + testsuite/serd-tests/good/test-rel.nt | 6 + testsuite/serd-tests/good/test-rel.ttl | 6 + testsuite/serd-tests/good/test-semi-dot.nt | 1 + testsuite/serd-tests/good/test-semi-dot.ttl | 1 + .../good/test-several-eaten-dots.nq | 3 + .../good/test-several-eaten-dots.trig | 6 + testsuite/serd-tests/good/test-uri-escape.nt | 1 + testsuite/serd-tests/good/test-uri-escape.ttl | 1 + testsuite/serd-tests/good/test-uri.nt | 48 + testsuite/serd-tests/good/test-uri.ttl | 71 + testsuite/serd-tests/good/test-utf8-uri.nt | 1 + testsuite/serd-tests/good/test-utf8-uri.ttl | 1 + testsuite/src/files.rs | 77 +- testsuite/src/lib.rs | 32 + testsuite/src/manifest.rs | 2 +- testsuite/src/parser_evaluator.rs | 137 +- testsuite/src/sparql_evaluator.rs | 2 +- testsuite/tests/oxigraph.rs | 52 +- testsuite/tests/parser.rs | 89 +- testsuite/tests/serd.rs | 13 + testsuite/tests/sparql.rs | 46 +- 248 files changed, 58728 insertions(+), 401 deletions(-) create mode 100644 fuzz/fuzz_targets/n3.rs create mode 100644 fuzz/fuzz_targets/nquads.rs create mode 100644 fuzz/fuzz_targets/trig.rs create mode 100644 lib/oxttl/Cargo.toml create mode 100644 lib/oxttl/src/lexer.rs create mode 100644 lib/oxttl/src/lib.rs create mode 100644 lib/oxttl/src/line_formats.rs create mode 100644 lib/oxttl/src/n3.rs create mode 100644 lib/oxttl/src/nquads.rs create mode 100644 lib/oxttl/src/ntriples.rs create mode 100644 lib/oxttl/src/terse.rs create mode 100644 lib/oxttl/src/toolkit/lexer.rs create mode 100644 lib/oxttl/src/toolkit/mod.rs create mode 100644 lib/oxttl/src/toolkit/parser.rs create mode 100644 lib/oxttl/src/trig.rs create mode 100644 lib/oxttl/src/turtle.rs create mode 160000 testsuite/N3 create mode 100644 testsuite/benches/parser.rs create mode 100644 testsuite/oxigraph-tests/parser-recovery/invalid_bnode.nt create mode 100644 testsuite/oxigraph-tests/parser-recovery/invalid_iri.nt create mode 100644 testsuite/oxigraph-tests/parser-recovery/invalid_string.nt create mode 100644 testsuite/oxigraph-tests/parser-recovery/iri2_spo.nt create mode 100644 testsuite/oxigraph-tests/parser-recovery/iri2_string_spo.nt create mode 100644 testsuite/oxigraph-tests/parser-recovery/iri_spo.nt create mode 100644 testsuite/oxigraph-tests/parser-recovery/iri_string_spo.nt create mode 100644 testsuite/oxigraph-tests/parser-recovery/manifest.ttl create mode 100644 testsuite/oxigraph-tests/parser-recovery/missing_dot_at_end_of_triple_with_iri_end.nt create mode 100644 testsuite/oxigraph-tests/parser-recovery/missing_dot_at_end_of_triple_with_iri_middle.nt create mode 100644 testsuite/oxigraph-tests/parser-recovery/missing_dot_at_end_of_triple_with_string_end.nt create mode 100644 testsuite/oxigraph-tests/parser-recovery/missing_dot_at_end_of_triple_with_string_middle.nt create mode 100644 testsuite/oxigraph-tests/parser/at_keywords_as_lang_tag.nt create mode 100644 testsuite/oxigraph-tests/parser/at_keywords_as_lang_tag.ttl create mode 100644 testsuite/oxigraph-tests/parser/bad_lang.ttl create mode 100644 testsuite/oxigraph-tests/parser/bad_parentheses.ttl create mode 100644 testsuite/oxigraph-tests/parser/blank_node_with_linebreak.nt create mode 100644 testsuite/oxigraph-tests/parser/blank_node_with_linebreak.ttl create mode 100644 testsuite/oxigraph-tests/parser/keyword_vs_prefix.nq create mode 100644 testsuite/oxigraph-tests/parser/keyword_vs_prefix.nt create mode 100644 testsuite/oxigraph-tests/parser/keyword_vs_prefix.trig create mode 100644 testsuite/oxigraph-tests/parser/keyword_vs_prefix.ttl create mode 100644 testsuite/oxigraph-tests/parser/language_normalization.nt create mode 100644 testsuite/oxigraph-tests/parser/language_normalization.rdf create mode 100644 testsuite/oxigraph-tests/parser/language_normalization.ttl create mode 100644 testsuite/oxigraph-tests/parser/literal_value_space.nt create mode 100644 testsuite/oxigraph-tests/parser/literal_value_space.rdf create mode 100644 testsuite/oxigraph-tests/parser/manifest.ttl create mode 100644 testsuite/oxigraph-tests/parser/no_end_line_jump.nt create mode 100644 testsuite/oxigraph-tests/parser/xml_entities.nt create mode 100644 testsuite/oxigraph-tests/parser/xml_entities.rdf create mode 100644 testsuite/oxigraph-tests/parser/xml_nested_entities.nt create mode 100644 testsuite/oxigraph-tests/parser/xml_nested_entities.rdf create mode 100644 testsuite/serd-tests/LICENSE create mode 100644 testsuite/serd-tests/README.md create mode 100644 testsuite/serd-tests/bad/bad-00.ttl create mode 100644 testsuite/serd-tests/bad/bad-01.ttl create mode 100644 testsuite/serd-tests/bad/bad-02.ttl create mode 100644 testsuite/serd-tests/bad/bad-03.ttl create mode 100644 testsuite/serd-tests/bad/bad-04.ttl create mode 100644 testsuite/serd-tests/bad/bad-05.ttl create mode 100644 testsuite/serd-tests/bad/bad-06.ttl create mode 100644 testsuite/serd-tests/bad/bad-07.ttl create mode 100644 testsuite/serd-tests/bad/bad-08.ttl create mode 100644 testsuite/serd-tests/bad/bad-09.ttl create mode 100644 testsuite/serd-tests/bad/bad-10.ttl create mode 100644 testsuite/serd-tests/bad/bad-11.ttl create mode 100644 testsuite/serd-tests/bad/bad-12.ttl create mode 100644 testsuite/serd-tests/bad/bad-13.ttl create mode 100644 testsuite/serd-tests/bad/bad-14.ttl create mode 100644 testsuite/serd-tests/bad/bad-base.ttl create mode 100644 testsuite/serd-tests/bad/bad-blank-syntax.ttl create mode 100644 testsuite/serd-tests/bad/bad-blank.ttl create mode 100644 testsuite/serd-tests/bad/bad-bom.ttl create mode 100644 testsuite/serd-tests/bad/bad-char-in-local.ttl create mode 100644 testsuite/serd-tests/bad/bad-char-in-prefix.ttl create mode 100644 testsuite/serd-tests/bad/bad-char-in-uri.ttl create mode 100644 testsuite/serd-tests/bad/bad-datatype-syntax.ttl create mode 100644 testsuite/serd-tests/bad/bad-datatype.ttl create mode 100644 testsuite/serd-tests/bad/bad-dot-after-subject.ttl create mode 100644 testsuite/serd-tests/bad/bad-dot-in-collection.ttl create mode 100644 testsuite/serd-tests/bad/bad-eof-after-quotes.ttl create mode 100644 testsuite/serd-tests/bad/bad-eof-at-string-start.ttl create mode 100644 testsuite/serd-tests/bad/bad-eof-in-blank.ttl create mode 100644 testsuite/serd-tests/bad/bad-eof-in-escape.ttl create mode 100644 testsuite/serd-tests/bad/bad-eof-in-lang-suffix.ttl create mode 100644 testsuite/serd-tests/bad/bad-eof-in-lang.ttl create mode 100644 testsuite/serd-tests/bad/bad-eof-in-list.ttl create mode 100644 testsuite/serd-tests/bad/bad-eof-in-long-string.ttl create mode 100644 testsuite/serd-tests/bad/bad-eof-in-object-list.ttl create mode 100644 testsuite/serd-tests/bad/bad-eof-in-object-list2.ttl create mode 100644 testsuite/serd-tests/bad/bad-eof-in-predicate-list.ttl create mode 100644 testsuite/serd-tests/bad/bad-eof-in-string.ttl create mode 100644 testsuite/serd-tests/bad/bad-eof-in-triple-quote.ttl create mode 100644 testsuite/serd-tests/bad/bad-eof-in-uri-scheme.nt create mode 100644 testsuite/serd-tests/bad/bad-eof-in-uri.ttl create mode 100644 testsuite/serd-tests/bad/bad-escape.ttl create mode 100644 testsuite/serd-tests/bad/bad-ext-namedblank-op.ttl create mode 100644 testsuite/serd-tests/bad/bad-graph-blank-label.trig create mode 100644 testsuite/serd-tests/bad/bad-hex-digit.ttl create mode 100644 testsuite/serd-tests/bad/bad-id-clash.ttl create mode 100644 testsuite/serd-tests/bad/bad-lang.ttl create mode 100644 testsuite/serd-tests/bad/bad-list.ttl create mode 100644 testsuite/serd-tests/bad/bad-list2.ttl create mode 100644 testsuite/serd-tests/bad/bad-long-literal-in-list.ttl create mode 100644 testsuite/serd-tests/bad/bad-missing-semi.ttl create mode 100644 testsuite/serd-tests/bad/bad-missing-uri-scheme.nt create mode 100644 testsuite/serd-tests/bad/bad-misspelled-base.ttl create mode 100644 testsuite/serd-tests/bad/bad-misspelled-prefix.ttl create mode 100644 testsuite/serd-tests/bad/bad-namespace.ttl create mode 100644 testsuite/serd-tests/bad/bad-ns.ttl create mode 100644 testsuite/serd-tests/bad/bad-null-byte.ttl create mode 100644 testsuite/serd-tests/bad/bad-num.ttl create mode 100644 testsuite/serd-tests/bad/bad-object.ttl create mode 100644 testsuite/serd-tests/bad/bad-object2.ttl create mode 100644 testsuite/serd-tests/bad/bad-pn-escape.ttl create mode 100644 testsuite/serd-tests/bad/bad-prefix.ttl create mode 100644 testsuite/serd-tests/bad/bad-quote-in-uri.ttl create mode 100644 testsuite/serd-tests/bad/bad-semicolon-after-subject.ttl create mode 100644 testsuite/serd-tests/bad/bad-string.ttl create mode 100644 testsuite/serd-tests/bad/bad-subject.ttl create mode 100644 testsuite/serd-tests/bad/bad-uri-escape.ttl create mode 100644 testsuite/serd-tests/bad/bad-uri-scheme-start.nt create mode 100644 testsuite/serd-tests/bad/bad-uri-scheme.nt create mode 100644 testsuite/serd-tests/bad/bad-uri-truncated.nt create mode 100644 testsuite/serd-tests/bad/bad-verb.ttl create mode 100644 testsuite/serd-tests/bad/invalid-char-in-local.ttl create mode 100644 testsuite/serd-tests/bad/invalid-char-in-prefix.ttl create mode 100644 testsuite/serd-tests/bad/manifest.ttl create mode 100644 testsuite/serd-tests/good/README.txt create mode 100644 testsuite/serd-tests/good/UTF-8.nt create mode 100644 testsuite/serd-tests/good/UTF-8.ttl create mode 100644 testsuite/serd-tests/good/base.nt create mode 100644 testsuite/serd-tests/good/base.ttl create mode 100644 testsuite/serd-tests/good/manifest.ttl create mode 100644 testsuite/serd-tests/good/qualify-in.ttl create mode 100644 testsuite/serd-tests/good/qualify-out.ttl create mode 100644 testsuite/serd-tests/good/test-14.nt create mode 100644 testsuite/serd-tests/good/test-14.ttl create mode 100644 testsuite/serd-tests/good/test-15.nt create mode 100644 testsuite/serd-tests/good/test-15.ttl create mode 100644 testsuite/serd-tests/good/test-16.nt create mode 100644 testsuite/serd-tests/good/test-16.ttl create mode 100644 testsuite/serd-tests/good/test-18.nt create mode 100644 testsuite/serd-tests/good/test-18.ttl create mode 100644 testsuite/serd-tests/good/test-30.nt create mode 100644 testsuite/serd-tests/good/test-30.ttl create mode 100644 testsuite/serd-tests/good/test-a-without-whitespace.nt create mode 100644 testsuite/serd-tests/good/test-a-without-whitespace.ttl create mode 100644 testsuite/serd-tests/good/test-backspace.nt create mode 100644 testsuite/serd-tests/good/test-backspace.ttl create mode 100644 testsuite/serd-tests/good/test-base-nopath.nt create mode 100644 testsuite/serd-tests/good/test-base-nopath.ttl create mode 100644 testsuite/serd-tests/good/test-base-query.nt create mode 100644 testsuite/serd-tests/good/test-base-query.ttl create mode 100644 testsuite/serd-tests/good/test-blank-cont.nt create mode 100644 testsuite/serd-tests/good/test-blank-cont.ttl create mode 100644 testsuite/serd-tests/good/test-blank-in-list.nt create mode 100644 testsuite/serd-tests/good/test-blank-in-list.ttl create mode 100644 testsuite/serd-tests/good/test-blank-node-statement.nt create mode 100644 testsuite/serd-tests/good/test-blank-node-statement.ttl create mode 100644 testsuite/serd-tests/good/test-blankdot.nt create mode 100644 testsuite/serd-tests/good/test-blankdot.ttl create mode 100644 testsuite/serd-tests/good/test-bom.nt create mode 100644 testsuite/serd-tests/good/test-bom.ttl create mode 100644 testsuite/serd-tests/good/test-cr.nt create mode 100644 testsuite/serd-tests/good/test-cr.ttl create mode 100644 testsuite/serd-tests/good/test-delete.nt create mode 100644 testsuite/serd-tests/good/test-delete.ttl create mode 100644 testsuite/serd-tests/good/test-digit-start-pname.nt create mode 100644 testsuite/serd-tests/good/test-digit-start-pname.ttl create mode 100644 testsuite/serd-tests/good/test-empty-path-base.nt create mode 100644 testsuite/serd-tests/good/test-empty-path-base.ttl create mode 100644 testsuite/serd-tests/good/test-empty.nt create mode 100644 testsuite/serd-tests/good/test-empty.ttl create mode 100644 testsuite/serd-tests/good/test-eof-at-page-end.nt create mode 100644 testsuite/serd-tests/good/test-eof-at-page-end.ttl create mode 100644 testsuite/serd-tests/good/test-escapes.nt create mode 100644 testsuite/serd-tests/good/test-escapes.ttl create mode 100644 testsuite/serd-tests/good/test-ext-namedblank-iri.nt create mode 100644 testsuite/serd-tests/good/test-ext-namedblank-iri.ttl create mode 100644 testsuite/serd-tests/good/test-ext-namedblank-prefix.nt create mode 100644 testsuite/serd-tests/good/test-ext-namedblank-prefix.ttl create mode 100644 testsuite/serd-tests/good/test-form-feed.nt create mode 100644 testsuite/serd-tests/good/test-form-feed.ttl create mode 100644 testsuite/serd-tests/good/test-id.nt create mode 100644 testsuite/serd-tests/good/test-id.ttl create mode 100644 testsuite/serd-tests/good/test-lang.nt create mode 100644 testsuite/serd-tests/good/test-lang.ttl create mode 100644 testsuite/serd-tests/good/test-list-in-blank.nt create mode 100644 testsuite/serd-tests/good/test-list-in-blank.ttl create mode 100644 testsuite/serd-tests/good/test-list-subject.nt create mode 100644 testsuite/serd-tests/good/test-list-subject.ttl create mode 100644 testsuite/serd-tests/good/test-list.nt create mode 100644 testsuite/serd-tests/good/test-list.ttl create mode 100644 testsuite/serd-tests/good/test-local-name-ends-with-dot.ttl create mode 100644 testsuite/serd-tests/good/test-long-string.nt create mode 100644 testsuite/serd-tests/good/test-long-string.ttl create mode 100644 testsuite/serd-tests/good/test-no-spaces.nt create mode 100644 testsuite/serd-tests/good/test-no-spaces.ttl create mode 100644 testsuite/serd-tests/good/test-non-curie-uri.nt create mode 100644 testsuite/serd-tests/good/test-non-curie-uri.ttl create mode 100644 testsuite/serd-tests/good/test-num.nt create mode 100644 testsuite/serd-tests/good/test-num.ttl create mode 100644 testsuite/serd-tests/good/test-out-of-range-unicode.nt create mode 100644 testsuite/serd-tests/good/test-out-of-range-unicode.ttl create mode 100644 testsuite/serd-tests/good/test-prefix.nt create mode 100644 testsuite/serd-tests/good/test-prefix.ttl create mode 100644 testsuite/serd-tests/good/test-pretty.nt create mode 100644 testsuite/serd-tests/good/test-pretty.ttl create mode 100644 testsuite/serd-tests/good/test-rel.nt create mode 100644 testsuite/serd-tests/good/test-rel.ttl create mode 100644 testsuite/serd-tests/good/test-semi-dot.nt create mode 100644 testsuite/serd-tests/good/test-semi-dot.ttl create mode 100644 testsuite/serd-tests/good/test-several-eaten-dots.nq create mode 100644 testsuite/serd-tests/good/test-several-eaten-dots.trig create mode 100644 testsuite/serd-tests/good/test-uri-escape.nt create mode 100644 testsuite/serd-tests/good/test-uri-escape.ttl create mode 100644 testsuite/serd-tests/good/test-uri.nt create mode 100644 testsuite/serd-tests/good/test-uri.ttl create mode 100644 testsuite/serd-tests/good/test-utf8-uri.nt create mode 100644 testsuite/serd-tests/good/test-utf8-uri.ttl create mode 100644 testsuite/tests/serd.rs diff --git a/.clusterfuzzlite/build.sh b/.clusterfuzzlite/build.sh index 7782c767..698dd187 100755 --- a/.clusterfuzzlite/build.sh +++ b/.clusterfuzzlite/build.sh @@ -15,10 +15,14 @@ function build_seed_corpus() { cd "$SRC"/oxigraph cargo fuzz build -O --debug-assertions -for TARGET in sparql_eval sparql_results_json sparql_results_tsv # sparql_results_xml https://github.com/tafia/quick-xml/issues/608 +for TARGET in sparql_eval sparql_results_json sparql_results_tsv n3 nquads trig # sparql_results_xml https://github.com/tafia/quick-xml/issues/608 do cp fuzz/target/x86_64-unknown-linux-gnu/release/$TARGET "$OUT"/ done build_seed_corpus sparql_results_json srj build_seed_corpus sparql_results_tsv tsv build_seed_corpus sparql_results_xml srx +build_seed_corpus n3 n3 +build_seed_corpus nquads nq +build_seed_corpus trig trig + diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5add21ce..106fba84 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -32,6 +32,8 @@ jobs: working-directory: ./lib/oxsdatatypes - run: cargo clippy working-directory: ./lib/oxrdf + - run: cargo clippy + working-directory: ./lib/oxttl - run: cargo clippy working-directory: ./lib/sparesults - run: cargo clippy @@ -74,6 +76,8 @@ jobs: working-directory: ./lib/oxsdatatypes - run: cargo clippy -- -D warnings -D clippy::all working-directory: ./lib/oxrdf + - run: cargo clippy -- -D warnings -D clippy::all + working-directory: ./lib/oxttl - run: cargo clippy -- -D warnings -D clippy::all working-directory: ./lib/sparesults - run: cargo clippy -- -D warnings -D clippy::all @@ -123,7 +127,7 @@ jobs: - run: rustup update - uses: Swatinem/rust-cache@v2 - run: cargo install cargo-semver-checks || true - - run: cargo semver-checks check-release --exclude oxrocksdb-sys --exclude oxigraph_js --exclude pyoxigraph --exclude oxigraph_testsuite --exclude oxigraph_server --exclude sparopt + - run: cargo semver-checks check-release --exclude oxrocksdb-sys --exclude oxigraph_js --exclude pyoxigraph --exclude oxigraph_testsuite --exclude oxigraph_server --exclude oxttl --exclude sparopt test_linux: runs-on: ubuntu-latest diff --git a/.gitmodules b/.gitmodules index d96fd0dc..0c6310f6 100644 --- a/.gitmodules +++ b/.gitmodules @@ -13,3 +13,7 @@ [submodule "oxrocksdb-sys/lz4"] path = oxrocksdb-sys/lz4 url = https://github.com/lz4/lz4.git +[submodule "testsuite/N3"] + path = testsuite/N3 + url = https://github.com/w3c/N3.git + branch = master diff --git a/Cargo.lock b/Cargo.lock index 649bd0aa..3ab7ed54 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -948,10 +948,10 @@ dependencies = [ "oxrdf", "oxrocksdb-sys", "oxsdatatypes", + "oxttl", "rand", "regex", "rio_api", - "rio_turtle", "rio_xml", "sha-1", "sha2", @@ -998,7 +998,11 @@ version = "0.0.0" dependencies = [ "anyhow", "clap", + "criterion", "oxigraph", + "oxttl", + "rio_api", + "rio_turtle", "spargebra", "sparopt", "text-diff", @@ -1043,6 +1047,16 @@ dependencies = [ "js-sys", ] +[[package]] +name = "oxttl" +version = "0.1.0" +dependencies = [ + "memchr", + "oxilangtag", + "oxiri", + "oxrdf", +] + [[package]] name = "parking_lot" version = "0.12.1" diff --git a/Cargo.toml b/Cargo.toml index 041afb39..0da9b1c9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,6 +4,7 @@ members = [ "lib", "lib/oxrdf", "lib/oxsdatatypes", + "lib/oxttl", "lib/spargebra", "lib/sparesults", "lib/sparopt", diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index 0c8a0b72..216ba902 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -12,6 +12,7 @@ cargo-fuzz = true anyhow = "1" lazy_static = "1" libfuzzer-sys = "0.4" +oxttl = { path = "../lib/oxttl", features = ["rdf-star"] } spargebra = { path = "../lib/spargebra", features = ["rdf-star", "sep-0006"] } sparesults = { path = "../lib/sparesults", features = ["rdf-star"] } sparql-smith = { path = "../lib/sparql-smith", features = ["sep-0006"] } @@ -23,6 +24,14 @@ debug = true [workspace] +[[bin]] +name = "nquads" +path = "fuzz_targets/nquads.rs" + +[[bin]] +name = "n3" +path = "fuzz_targets/n3.rs" + [[bin]] name = "sparql_eval" path = "fuzz_targets/sparql_eval.rs" @@ -46,3 +55,7 @@ path = "fuzz_targets/sparql_results_xml.rs" [[bin]] name = "sparql_results_tsv" path = "fuzz_targets/sparql_results_tsv.rs" + +[[bin]] +name = "trig" +path = "fuzz_targets/trig.rs" diff --git a/fuzz/fuzz_targets/n3.rs b/fuzz/fuzz_targets/n3.rs new file mode 100644 index 00000000..29e63458 --- /dev/null +++ b/fuzz/fuzz_targets/n3.rs @@ -0,0 +1,28 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; +use oxttl::N3Parser; + +fuzz_target!(|data: &[u8]| { + let mut quads = Vec::new(); + let mut parser = N3Parser::new() + .with_base_iri("http://example.com/") + .unwrap() + .parse(); + for chunk in data.split(|c| *c == 0xFF) { + parser.extend_from_slice(chunk); + while let Some(result) = parser.read_next() { + if let Ok(quad) = result { + quads.push(quad); + } + } + } + parser.end(); + while let Some(result) = parser.read_next() { + if let Ok(quad) = result { + quads.push(quad); + } + } + assert!(parser.is_end()); + //TODO: serialize +}); diff --git a/fuzz/fuzz_targets/nquads.rs b/fuzz/fuzz_targets/nquads.rs new file mode 100644 index 00000000..80cf9160 --- /dev/null +++ b/fuzz/fuzz_targets/nquads.rs @@ -0,0 +1,49 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; +use oxttl::{NQuadsParser, NQuadsSerializer}; + +fuzz_target!(|data: &[u8]| { + // We parse + let mut quads = Vec::new(); + let mut parser = NQuadsParser::new().with_quoted_triples().parse(); + for chunk in data.split(|c| *c == 0xFF) { + parser.extend_from_slice(chunk); + while let Some(result) = parser.read_next() { + if let Ok(quad) = result { + quads.push(quad); + } + } + } + parser.end(); + while let Some(result) = parser.read_next() { + if let Ok(quad) = result { + quads.push(quad); + } + } + assert!(parser.is_end()); + + // We serialize + let mut writer = NQuadsSerializer::new().serialize_to_write(Vec::new()); + for quad in &quads { + writer.write_quad(quad).unwrap(); + } + let new_serialization = writer.finish(); + + // We parse the serialization + let new_quads = NQuadsParser::new() + .with_quoted_triples() + .parse_from_read(new_serialization.as_slice()) + .collect::, _>>() + .map_err(|e| { + format!( + "Error on {:?} from {quads:?} based on {:?}: {e}", + String::from_utf8_lossy(&new_serialization), + String::from_utf8_lossy(data) + ) + }) + .unwrap(); + + // We check the roundtrip has not changed anything + assert_eq!(new_quads, quads); +}); diff --git a/fuzz/fuzz_targets/trig.rs b/fuzz/fuzz_targets/trig.rs new file mode 100644 index 00000000..9c2385df --- /dev/null +++ b/fuzz/fuzz_targets/trig.rs @@ -0,0 +1,53 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; +use oxttl::{TriGParser, TriGSerializer}; + +fuzz_target!(|data: &[u8]| { + // We parse + let mut quads = Vec::new(); + let mut parser = TriGParser::new() + .with_quoted_triples() + .with_base_iri("http://example.com/") + .unwrap() + .parse(); + for chunk in data.split(|c| *c == 0xFF) { + parser.extend_from_slice(chunk); + while let Some(result) = parser.read_next() { + if let Ok(quad) = result { + quads.push(quad); + } + } + } + parser.end(); + while let Some(result) = parser.read_next() { + if let Ok(quad) = result { + quads.push(quad); + } + } + assert!(parser.is_end()); + + // We serialize + let mut writer = TriGSerializer::new().serialize_to_write(Vec::new()); + for quad in &quads { + writer.write_quad(quad).unwrap(); + } + let new_serialization = writer.finish().unwrap(); + + // We parse the serialization + let new_quads = TriGParser::new() + .with_quoted_triples() + .parse_from_read(new_serialization.as_slice()) + .collect::, _>>() + .map_err(|e| { + format!( + "Error on {:?} from {quads:?} based on {:?}: {e}", + String::from_utf8_lossy(&new_serialization), + String::from_utf8_lossy(data) + ) + }) + .unwrap(); + + // We check the roundtrip has not changed anything + assert_eq!(new_quads, quads); +}); diff --git a/lib/Cargo.toml b/lib/Cargo.toml index 100be488..224cf7b4 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -32,17 +32,17 @@ regex = "1" oxilangtag = "0.1" oxiri = "0.2" rio_api = "0.8" -rio_turtle = "0.8" rio_xml = "0.8" hex = "0.4" siphasher = "0.3" lazy_static = "1" json-event-parser = "0.1" -oxrdf = { version = "0.2.0-alpha.1-dev", path="oxrdf", features = ["rdf-star", "oxsdatatypes"] } +oxrdf = { version = "0.2.0-alpha.1-dev", path = "oxrdf", features = ["rdf-star", "oxsdatatypes"] } oxsdatatypes = { version = "0.2.0-alpha.1-dev", path="oxsdatatypes" } -spargebra = { version = "0.3.0-alpha.1-dev", path="spargebra", features = ["rdf-star", "sep-0002", "sep-0006"] } +oxttl = { version = "0.1.0" , path = "oxttl", features = ["rdf-star"] } +spargebra = { version = "0.3.0-alpha.1-dev", path = "spargebra", features = ["rdf-star", "sep-0002", "sep-0006"] } sparopt = { version = "0.1.0-alpha.1-dev", path="sparopt", features = ["rdf-star", "sep-0002", "sep-0006"] } -sparesults = { version = "0.2.0-alpha.1-dev", path="sparesults", features = ["rdf-star"] } +sparesults = { version = "0.2.0-alpha.1-dev", path = "sparesults", features = ["rdf-star"] } [target.'cfg(not(target_family = "wasm"))'.dependencies] libc = "0.2" diff --git a/lib/benches/store.rs b/lib/benches/store.rs index c562684c..0f35a623 100644 --- a/lib/benches/store.rs +++ b/lib/benches/store.rs @@ -7,7 +7,7 @@ use oxigraph::store::Store; use rand::random; use std::env::temp_dir; use std::fs::{remove_dir_all, File}; -use std::io::{BufRead, BufReader, Cursor, Read}; +use std::io::{BufRead, BufReader, Read}; use std::path::{Path, PathBuf}; fn store_load(c: &mut Criterion) { @@ -64,7 +64,7 @@ fn store_load(c: &mut Criterion) { fn do_load(store: &Store, data: &[u8]) { store .load_graph( - Cursor::new(&data), + data, GraphFormat::NTriples, GraphNameRef::DefaultGraph, None, @@ -77,7 +77,7 @@ fn do_bulk_load(store: &Store, data: &[u8]) { store .bulk_loader() .load_graph( - Cursor::new(&data), + data, GraphFormat::NTriples, GraphNameRef::DefaultGraph, None, diff --git a/lib/oxttl/Cargo.toml b/lib/oxttl/Cargo.toml new file mode 100644 index 00000000..bc8105c3 --- /dev/null +++ b/lib/oxttl/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "oxttl" +version = "0.1.0" +authors = ["Tpt "] +license = "MIT OR Apache-2.0" +readme = "README.md" +keywords = ["SPARQL"] +repository = "https://github.com/oxigraph/oxigraph/tree/master/lib/oxttl" +homepage = "https://oxigraph.org/" +description = """ +N-Triples parser +""" +edition = "2021" +rust-version = "1.65" + +[features] +default = [] +rdf-star = ["oxrdf/rdf-star"] + +[dependencies] +memchr = "2" +oxrdf = { version = "0.2.0-alpha.1-dev", path = "../oxrdf" } +oxiri = "0.2" +oxilangtag = "0.1" + +[package.metadata.docs.rs] +all-features = true diff --git a/lib/oxttl/src/lexer.rs b/lib/oxttl/src/lexer.rs new file mode 100644 index 00000000..7480908b --- /dev/null +++ b/lib/oxttl/src/lexer.rs @@ -0,0 +1,938 @@ +use crate::toolkit::{TokenRecognizer, TokenRecognizerError}; +use memchr::{memchr, memchr2}; +use oxilangtag::LanguageTag; +use oxiri::Iri; +use oxrdf::NamedNode; +use std::borrow::Cow; +use std::collections::HashMap; +use std::ops::{Range, RangeInclusive}; +use std::str; + +#[derive(Debug, PartialEq, Eq)] +pub enum N3Token<'a> { + IriRef(Iri), + PrefixedName { + prefix: &'a str, + local: Cow<'a, str>, + might_be_invalid_iri: bool, + }, + Variable(Cow<'a, str>), + BlankNodeLabel(&'a str), + String(String), + Integer(&'a str), + Decimal(&'a str), + Double(&'a str), + LangTag(&'a str), + Punctuation(&'a str), + PlainKeyword(&'a str), +} + +#[derive(Eq, PartialEq)] +pub enum N3LexerMode { + NTriples, + Turtle, + N3, +} + +#[derive(Default)] +pub struct N3LexerOptions { + pub base_iri: Option>, +} + +pub struct N3Lexer { + mode: N3LexerMode, +} + +// TODO: there are a lot of 'None' (missing data) returned even if the stream is ending!!! +// TODO: simplify by not giving is_end and fail with an "unexpected eof" is none is returned when is_end=true? + +impl TokenRecognizer for N3Lexer { + type Token<'a> = N3Token<'a>; + type Options = N3LexerOptions; + + fn recognize_next_token<'a>( + &mut self, + data: &'a [u8], + is_ending: bool, + options: &N3LexerOptions, + ) -> Option<(usize, Result, TokenRecognizerError>)> { + match *data.first()? { + b'<' => match *data.get(1)? { + b'<' => Some((2, Ok(N3Token::Punctuation("<<")))), + b'=' if self.mode == N3LexerMode::N3 => { + if let Some((consumed, result)) = Self::recognize_iri(data, options) { + Some(if let Ok(result) = result { + (consumed, Ok(result)) + } else { + (2, Ok(N3Token::Punctuation("<="))) + }) + } else if is_ending { + Some((2, Ok(N3Token::Punctuation("<=")))) + } else { + None + } + } + b'-' if self.mode == N3LexerMode::N3 => { + if let Some((consumed, result)) = Self::recognize_iri(data, options) { + Some(if let Ok(result) = result { + (consumed, Ok(result)) + } else { + (2, Ok(N3Token::Punctuation("<-"))) + }) + } else if is_ending { + Some((2, Ok(N3Token::Punctuation("<-")))) + } else { + None + } + } + _ => Self::recognize_iri(data, options), + }, + b'>' => { + if *data.get(1)? == b'>' { + Some((2, Ok(N3Token::Punctuation(">>")))) + } else { + Some((1, Ok(N3Token::Punctuation(">")))) + } + } + b'_' => match data.get(1)? { + b':' => Self::recognize_blank_node_label(data), + c => Some(( + 1, + Err((0, format!("Unexpected character '{}'", char::from(*c))).into()), + )), + }, + b'"' => { + if self.mode != N3LexerMode::NTriples + && *data.get(1)? == b'"' + && *data.get(2)? == b'"' + { + Self::recognize_long_string(data, b'"') + } else { + Self::recognize_string(data, b'"') + } + } + b'\'' if self.mode != N3LexerMode::NTriples => { + if *data.get(1)? == b'\'' && *data.get(2)? == b'\'' { + Self::recognize_long_string(data, b'\'') + } else { + Self::recognize_string(data, b'\'') + } + } + b'@' => Self::recognize_lang_tag(data), + b'.' => match data.get(1) { + Some(b'0'..=b'9') => Self::recognize_number(data), + Some(_) => Some((1, Ok(N3Token::Punctuation(".")))), + None => is_ending.then_some((1, Ok(N3Token::Punctuation(".")))), + }, + b'^' => { + if *data.get(1)? == b'^' { + Some((2, Ok(N3Token::Punctuation("^^")))) + } else { + Some((1, Ok(N3Token::Punctuation("^")))) + } + } + b'(' => Some((1, Ok(N3Token::Punctuation("(")))), + b')' => Some((1, Ok(N3Token::Punctuation(")")))), + b'[' => Some((1, Ok(N3Token::Punctuation("[")))), + b']' => Some((1, Ok(N3Token::Punctuation("]")))), + b'{' => { + if *data.get(1)? == b'|' { + Some((2, Ok(N3Token::Punctuation("{|")))) + } else { + Some((1, Ok(N3Token::Punctuation("{")))) + } + } + b'}' => Some((1, Ok(N3Token::Punctuation("}")))), + b',' => Some((1, Ok(N3Token::Punctuation(",")))), + b';' => Some((1, Ok(N3Token::Punctuation(";")))), + b'!' => Some((1, Ok(N3Token::Punctuation("!")))), + b'|' => { + if *data.get(1)? == b'}' { + Some((2, Ok(N3Token::Punctuation("|}")))) + } else { + Some((1, Ok(N3Token::Punctuation("|")))) + } + } + b'=' => { + if *data.get(1)? == b'>' { + Some((2, Ok(N3Token::Punctuation("=>")))) + } else { + Some((1, Ok(N3Token::Punctuation("=")))) + } + } + b'0'..=b'9' | b'+' | b'-' => Self::recognize_number(data), + b'?' => Self::recognize_variable(data, is_ending), + _ => Self::recognize_pname_or_keyword(data, is_ending), + } + } +} + +impl N3Lexer { + pub fn new(mode: N3LexerMode) -> Self { + Self { mode } + } + + fn recognize_iri( + data: &[u8], + options: &N3LexerOptions, + ) -> Option<(usize, Result, TokenRecognizerError>)> { + // [18] IRIREF ::= '<' ([^#x00-#x20<>"{}|^`\] | UCHAR)* '>' /* #x00=NULL #01-#x1F=control codes #x20=space */ + let mut string = Vec::new(); + let mut i = 1; + loop { + let end = memchr2(b'>', b'\\', &data[i..])?; + string.extend_from_slice(&data[i..i + end]); + i += end; + match data[i] { + b'>' => { + return Some((i + 1, Self::parse_iri(string, 0..=i, options))); + } + b'\\' => { + let (additional, c) = Self::recognize_escape(&data[i..], i, false)?; + i += additional + 1; + match c { + Ok(c) => { + let mut buf = [0; 4]; + string.extend_from_slice(c.encode_utf8(&mut buf).as_bytes()); + } + Err(e) => return Some((i, Err(e))), + } + } + _ => unreachable!(), + } + } + } + + fn parse_iri( + iri: Vec, + position: RangeInclusive, + options: &N3LexerOptions, + ) -> Result, TokenRecognizerError> { + let iri = String::from_utf8(iri).map_err(|e| { + ( + position.clone(), + format!("The IRI contains invalid UTF-8 characters: {e}"), + ) + })?; + let iri = if let Some(base_iri) = options.base_iri.as_ref() { + base_iri.resolve(&iri) + } else { + Iri::parse(iri) + } + .map_err(|e| (position, e.to_string()))?; + Ok(N3Token::IriRef(iri)) + } + + fn recognize_pname_or_keyword( + data: &[u8], + is_ending: bool, + ) -> Option<(usize, Result, TokenRecognizerError>)> { + // [139s] PNAME_NS ::= PN_PREFIX? ':' + // [140s] PNAME_LN ::= PNAME_NS PN_LOCAL + + // [167s] PN_PREFIX ::= PN_CHARS_BASE ((PN_CHARS | '.')* PN_CHARS)? + let mut i = 0; + loop { + if let Some(r) = Self::recognize_unicode_char(&data[i..], i) { + match r { + Ok((c, consumed)) => { + if c == ':' { + i += consumed; + break; + } else if i == 0 { + if !Self::is_possible_pn_chars_base(c) { + return Some(( + consumed, + Err(( + 0..consumed, + format!( + "'{c}' is not allowed at the beginning of a prefix name" + ), + ) + .into()), + )); + } + i += consumed; + } else if Self::is_possible_pn_chars(c) || c == '.' { + i += consumed; + } else { + while data[..i].ends_with(b".") { + i -= 1; + } + return Some(( + i, + Ok(N3Token::PlainKeyword(str::from_utf8(&data[..i]).unwrap())), + )); + } + } + Err(e) => return Some((e.position.end, Err(e))), + } + } else if is_ending { + while data[..i].ends_with(b".") { + i -= 1; + } + return Some(if i == 0 { + ( + 1, + Err((0..1, format!("Unexpected byte {}", data[0])).into()), + ) + } else { + ( + i, + Ok(N3Token::PlainKeyword(str::from_utf8(&data[..i]).unwrap())), + ) + }); + } else { + return None; + } + } + let pn_prefix = str::from_utf8(&data[..i - 1]).unwrap(); + if pn_prefix.ends_with('.') { + return Some(( + i, + Err(( + 0..i, + format!( + "'{pn_prefix}' is not a valid prefix: prefixes are not allowed to end with '.'"), + ) + .into()), + )); + } + + let (consumed, pn_local_result) = Self::recognize_optional_pn_local(&data[i..], is_ending)?; + Some(( + consumed + i, + pn_local_result.map(|(local, might_be_invalid_iri)| N3Token::PrefixedName { + prefix: pn_prefix, + local, + might_be_invalid_iri, + }), + )) + } + + fn recognize_variable( + data: &[u8], + is_ending: bool, + ) -> Option<(usize, Result, TokenRecognizerError>)> { + // [36] QUICK_VAR_NAME ::= "?" PN_LOCAL + let (consumed, result) = Self::recognize_optional_pn_local(&data[1..], is_ending)?; + Some(( + consumed + 1, + result.and_then(|(name, _)| { + if name.is_empty() { + Err((0..consumed, "A variable name is not allowed to be empty").into()) + } else { + Ok(N3Token::Variable(name)) + } + }), + )) + } + + fn recognize_optional_pn_local( + data: &[u8], + is_ending: bool, + ) -> Option<(usize, Result<(Cow<'_, str>, bool), TokenRecognizerError>)> { + // [168s] PN_LOCAL ::= (PN_CHARS_U | ':' | [0-9] | PLX) ((PN_CHARS | '.' | ':' | PLX)* (PN_CHARS | ':' | PLX))? + let mut i = 0; + let mut buffer = None; // Buffer if there are some escaped characters + let mut position_that_is_already_in_buffer = 0; + let mut might_be_invalid_iri = false; + loop { + if let Some(r) = Self::recognize_unicode_char(&data[i..], i) { + match r { + Ok((c, consumed)) => { + if c == '%' { + i += 1; + let a = char::from(*data.get(i)?); + i += 1; + let b = char::from(*data.get(i)?); + if !matches!(a, '0'..='9' | 'A'..='F' | 'a'..='f') + || !matches!(b, '0'..='9' | 'A'..='F' | 'a'..='f') + { + return Some((i + 1, Err(( + i - 2..=i, format!("escapes in IRIs should be % followed by two hexadecimal characters, found '%{a}{b}'") + ).into()))); + } + i += 1; + } else if c == '\\' { + i += 1; + let a = char::from(*data.get(i)?); + if matches!( + a, + '_' | '~' + | '.' + | '-' + | '!' + | '$' + | '&' + | '\'' + | '(' + | ')' + | '*' + | '+' + | ',' + | ';' + | '=' + ) { + // ok to escape + } else if matches!(a, '/' | '?' | '#' | '@' | '%') { + // ok to escape but requires IRI validation + might_be_invalid_iri = true; + } else { + return Some((i + 1, Err(( + i..=i, format!("The character that are allowed to be escaped in IRIs are _~.-!$&'()*+,;=/?#@%, found '{a}'") + ).into()))); + } + let buffer = buffer.get_or_insert_with(String::new); + // We add the missing bytes + if i - position_that_is_already_in_buffer > 1 { + buffer.push_str( + str::from_utf8( + &data[position_that_is_already_in_buffer..i - 1], + ) + .unwrap(), + ) + } + buffer.push(a); + i += 1; + position_that_is_already_in_buffer = i; + } else if i == 0 { + if !(Self::is_possible_pn_chars_u(c) || c == ':' || c.is_ascii_digit()) + { + return Some((0, Ok((Cow::Borrowed(""), false)))); + } + might_be_invalid_iri |= + Self::is_possible_pn_chars_base_but_not_valid_iri(c) || c == ':'; + i += consumed; + } else if Self::is_possible_pn_chars(c) || c == ':' || c == '.' { + might_be_invalid_iri |= + Self::is_possible_pn_chars_base_but_not_valid_iri(c) || c == ':'; + i += consumed; + } else { + let buffer = if let Some(mut buffer) = buffer { + buffer.push_str( + str::from_utf8(&data[position_that_is_already_in_buffer..i]) + .unwrap(), + ); + // We do not include the last dot + while buffer.ends_with('.') { + buffer.pop(); + i -= 1; + } + Cow::Owned(buffer) + } else { + let mut data = str::from_utf8(&data[..i]).unwrap(); + // We do not include the last dot + while let Some(d) = data.strip_suffix('.') { + data = d; + i -= 1; + } + Cow::Borrowed(data) + }; + return Some((i, Ok((buffer, might_be_invalid_iri)))); + } + } + Err(e) => return Some((e.position.end, Err(e))), + } + } else if is_ending { + let buffer = if let Some(mut buffer) = buffer { + // We do not include the last dot + while buffer.ends_with('.') { + buffer.pop(); + i -= 1; + } + Cow::Owned(buffer) + } else { + let mut data = str::from_utf8(&data[..i]).unwrap(); + // We do not include the last dot + while let Some(d) = data.strip_suffix('.') { + data = d; + i -= 1; + } + Cow::Borrowed(data) + }; + return Some((i, Ok((buffer, might_be_invalid_iri)))); + } else { + return None; + } + } + } + + fn recognize_blank_node_label( + data: &[u8], + ) -> Option<(usize, Result, TokenRecognizerError>)> { + // [141s] BLANK_NODE_LABEL ::= '_:' (PN_CHARS_U | [0-9]) ((PN_CHARS | '.')* PN_CHARS)? + let mut i = 2; + loop { + match Self::recognize_unicode_char(&data[i..], i)? { + Ok((c, consumed)) => { + if (i == 2 && (Self::is_possible_pn_chars_u(c) || c.is_ascii_digit())) + || (i > 2 && Self::is_possible_pn_chars(c)) + { + // Ok + } else if i > 2 && c == '.' { + if data[i - 1] == b'.' { + i -= 1; + return Some(( + i, + Ok(N3Token::BlankNodeLabel( + str::from_utf8(&data[2..i]).unwrap(), + )), + )); + } + } else if i == 0 { + return Some(( + i, + Err((0..i, "A blank node ID should not be empty").into()), + )); + } else if data[i - 1] == b'.' { + i -= 1; + return Some(( + i, + Ok(N3Token::BlankNodeLabel( + str::from_utf8(&data[2..i]).unwrap(), + )), + )); + } else { + return Some(( + i, + Ok(N3Token::BlankNodeLabel( + str::from_utf8(&data[2..i]).unwrap(), + )), + )); + } + i += consumed; + } + Err(e) => return Some((e.position.end, Err(e))), + } + } + } + + fn recognize_lang_tag( + data: &[u8], + ) -> Option<(usize, Result, TokenRecognizerError>)> { + // [144s] LANGTAG ::= '@' [a-zA-Z]+ ('-' [a-zA-Z0-9]+)* + let mut is_last_block_empty = true; + for (i, c) in data[1..].iter().enumerate() { + if c.is_ascii_alphabetic() { + is_last_block_empty = false; + } else if i == 0 { + return Some(( + 1, + Err((1..2, "A language code should always start with a letter").into()), + )); + } else if is_last_block_empty { + return Some((i, Self::parse_lang_tag(&data[1..i], 1..i - 1))); + } else if *c == b'-' { + is_last_block_empty = true; + } else { + return Some((i + 1, Self::parse_lang_tag(&data[1..=i], 1..i))); + } + } + None + } + + fn parse_lang_tag( + lang_tag: &[u8], + position: Range, + ) -> Result, TokenRecognizerError> { + Ok(N3Token::LangTag( + LanguageTag::parse(str::from_utf8(lang_tag).unwrap()) + .map_err(|e| (position.clone(), e.to_string()))? + .into_inner(), + )) + } + + fn recognize_string( + data: &[u8], + delimiter: u8, + ) -> Option<(usize, Result, TokenRecognizerError>)> { + // [22] STRING_LITERAL_QUOTE ::= '"' ([^#x22#x5C#xA#xD] | ECHAR | UCHAR)* '"' /* #x22=" #x5C=\ #xA=new line #xD=carriage return */ + // [23] STRING_LITERAL_SINGLE_QUOTE ::= "'" ([^#x27#x5C#xA#xD] | ECHAR | UCHAR)* "'" /* #x27=' #x5C=\ #xA=new line #xD=carriage return */ + let mut string = String::new(); + let mut i = 1; + loop { + let end = memchr2(delimiter, b'\\', &data[i..])?; + match str::from_utf8(&data[i..i + end]) { + Ok(a) => string.push_str(a), + Err(e) => { + return Some(( + end, + Err(( + i..i + end, + format!("The string contains invalid UTF-8 characters: {e}"), + ) + .into()), + )) + } + }; + i += end; + match data[i] { + c if c == delimiter => { + return Some((i + 1, Ok(N3Token::String(string)))); + } + b'\\' => { + let (additional, c) = Self::recognize_escape(&data[i..], i, true)?; + i += additional + 1; + match c { + Ok(c) => { + string.push(c); + } + Err(e) => { + // We read until the end of string char + let end = memchr(delimiter, &data[i..])?; + return Some((i + end + 1, Err(e))); + } + } + } + _ => unreachable!(), + } + } + } + + fn recognize_long_string( + data: &[u8], + delimiter: u8, + ) -> Option<(usize, Result, TokenRecognizerError>)> { + // [24] STRING_LITERAL_LONG_SINGLE_QUOTE ::= "'''" (("'" | "''")? ([^'\] | ECHAR | UCHAR))* "'''" + // [25] STRING_LITERAL_LONG_QUOTE ::= '"""' (('"' | '""')? ([^"\] | ECHAR | UCHAR))* '"""' + let mut string = String::new(); + let mut i = 3; + loop { + let end = memchr2(delimiter, b'\\', &data[i..])?; + match str::from_utf8(&data[i..i + end]) { + Ok(a) => string.push_str(a), + Err(e) => { + return Some(( + end, + Err(( + i..i + end, + format!("The string contains invalid UTF-8 characters: {e}"), + ) + .into()), + )) + } + }; + i += end; + match data[i] { + c if c == delimiter => { + if *data.get(i + 1)? == delimiter && *data.get(i + 2)? == delimiter { + return Some((i + 3, Ok(N3Token::String(string)))); + } + i += 1; + string.push(char::from(delimiter)); + } + b'\\' => { + let (additional, c) = Self::recognize_escape(&data[i..], i, true)?; + i += additional + 1; + match c { + Ok(c) => { + string.push(c); + } + Err(e) => return Some((i, Err(e))), + } + } + _ => unreachable!(), + } + } + } + + fn recognize_number(data: &[u8]) -> Option<(usize, Result, TokenRecognizerError>)> { + // [19] INTEGER ::= [+-]? [0-9]+ + // [20] DECIMAL ::= [+-]? [0-9]* '.' [0-9]+ + // [21] DOUBLE ::= [+-]? ([0-9]+ '.' [0-9]* EXPONENT | '.' [0-9]+ EXPONENT | [0-9]+ EXPONENT) + // [154s] EXPONENT ::= [eE] [+-]? [0-9]+ + let mut i = 0; + let c = *data.first()?; + if matches!(c, b'+' | b'-') { + i += 1; + } + // We read the digits before . + let mut count_before: usize = 0; + loop { + let c = *data.get(i)?; + if c.is_ascii_digit() { + i += 1; + count_before += 1; + } else { + break; + } + } + + // We read the digits after . + #[allow(clippy::if_then_some_else_none)] + let count_after = if *data.get(i)? == b'.' { + i += 1; + + let mut count_after = 0; + loop { + let c = *data.get(i)?; + if c.is_ascii_digit() { + i += 1; + count_after += 1; + } else { + break; + } + } + Some(count_after) + } else { + None + }; + + // End + let c = *data.get(i)?; + if matches!(c, b'e' | b'E') { + i += 1; + + let c = *data.get(i)?; + if matches!(c, b'+' | b'-') { + i += 1; + } + + let mut found = false; + loop { + let c = *data.get(i)?; + if c.is_ascii_digit() { + i += 1; + found = true; + } else { + break; + } + } + Some(( + i, + if !found { + Err((0..i, "A double exponent cannot be empty").into()) + } else if count_before == 0 && count_after.unwrap_or(0) == 0 { + Err((0..i, "A double should not be empty").into()) + } else { + Ok(N3Token::Double(str::from_utf8(&data[..i]).unwrap())) + }, + )) + } else if let Some(count_after) = count_after { + if count_after == 0 { + // We do not consume the '.' after all + i -= 1; + Some(( + i, + if count_before == 0 { + Err((0..i, "An integer should not be empty").into()) + } else { + Ok(N3Token::Integer(str::from_utf8(&data[..i]).unwrap())) + }, + )) + } else { + Some((i, Ok(N3Token::Decimal(str::from_utf8(&data[..i]).unwrap())))) + } + } else { + Some(( + i, + if count_before == 0 { + Err((0..i, "An integer should not be empty").into()) + } else { + Ok(N3Token::Integer(str::from_utf8(&data[..i]).unwrap())) + }, + )) + } + } + + fn recognize_escape( + data: &[u8], + position: usize, + with_echar: bool, + ) -> Option<(usize, Result)> { + // [26] UCHAR ::= '\u' HEX HEX HEX HEX | '\U' HEX HEX HEX HEX HEX HEX HEX HEX + // [159s] ECHAR ::= '\' [tbnrf"'\] + match *data.get(1)? { + b'u' => match Self::recognize_hex_char(&data[2..], 4, 'u', position) { + Ok(c) => Some((5, Ok(c?))), + Err(e) => Some((5, Err(e))), + }, + b'U' => match Self::recognize_hex_char(&data[2..], 8, 'u', position) { + Ok(c) => Some((9, Ok(c?))), + Err(e) => Some((9, Err(e))), + }, + b't' if with_echar => Some((1, Ok('\t'))), + b'b' if with_echar => Some((1, Ok('\x08'))), + b'n' if with_echar => Some((1, Ok('\n'))), + b'r' if with_echar => Some((1, Ok('\r'))), + b'f' if with_echar => Some((1, Ok('\x0C'))), + b'"' if with_echar => Some((1, Ok('"'))), + b'\'' if with_echar => Some((1, Ok('\''))), + b'\\' if with_echar => Some((1, Ok('\\'))), + c => Some(( + 1, + Err(( + position..position + 2, + format!("Unexpected escape character '\\{}'", char::from(c)), + ) + .into()), + )), //TODO: read until end of string + } + } + + fn recognize_hex_char( + data: &[u8], + len: usize, + escape_char: char, + position: usize, + ) -> Result, TokenRecognizerError> { + if data.len() < len { + return Ok(None); + } + let val = str::from_utf8(&data[..len]).map_err(|e| { + ( + position..position + len + 2, + format!("The escape sequence contains invalid UTF-8 characters: {e}"), + ) + })?; + let codepoint = u32::from_str_radix(val, 16).map_err(|e| { + ( + position..position + len + 2, + format!( + "The escape sequence '\\{escape_char}{val}' is not a valid hexadecimal string: {e}" + ), + ) + })?; + let c = char::from_u32(codepoint).ok_or_else(|| { + ( + position..position + len +2, + format!( + "The escape sequence '\\{escape_char}{val}' is encoding {codepoint:X} that is not a valid unicode character", + ), + ) + })?; + Ok(Some(c)) + } + + fn recognize_unicode_char( + data: &[u8], + position: usize, + ) -> Option> { + let mut code_point: u32; + let bytes_needed: usize; + let mut lower_boundary = 0x80; + let mut upper_boundary = 0xBF; + + let byte = *data.first()?; + match byte { + 0x00..=0x7F => return Some(Ok((char::from(byte), 1))), + 0xC2..=0xDF => { + bytes_needed = 1; + code_point = u32::from(byte) & 0x1F; + } + 0xE0..=0xEF => { + if byte == 0xE0 { + lower_boundary = 0xA0; + } + if byte == 0xED { + upper_boundary = 0x9F; + } + bytes_needed = 2; + code_point = u32::from(byte) & 0xF; + } + 0xF0..=0xF4 => { + if byte == 0xF0 { + lower_boundary = 0x90; + } + if byte == 0xF4 { + upper_boundary = 0x8F; + } + bytes_needed = 3; + code_point = u32::from(byte) & 0x7; + } + _ => { + return Some(Err(( + position..=position, + "Invalid UTF-8 character encoding", + ) + .into())) + } + } + + for i in 1..=bytes_needed { + let byte = *data.get(i)?; + if byte < lower_boundary || upper_boundary < byte { + return Some(Err(( + position..=position + i, + "Invalid UTF-8 character encoding", + ) + .into())); + } + lower_boundary = 0x80; + upper_boundary = 0xBF; + code_point = (code_point << 6) | (u32::from(byte) & 0x3F); + } + + Some( + char::from_u32(code_point) + .map(|c| (c, bytes_needed + 1)) + .ok_or_else(|| { + ( + position..=position + bytes_needed, + format!("The codepoint {code_point:X} is not a valid unicode character"), + ) + .into() + }), + ) + } + + // [157s] PN_CHARS_BASE ::= [A-Z] | [a-z] | [#x00C0-#x00D6] | [#x00D8-#x00F6] | [#x00F8-#x02FF] | [#x0370-#x037D] | [#x037F-#x1FFF] | [#x200C-#x200D] | [#x2070-#x218F] | [#x2C00-#x2FEF] | [#x3001-#xD7FF] | [#xF900-#xFDCF] | [#xFDF0-#xFFFD] | [#x10000-#xEFFFF] + fn is_possible_pn_chars_base(c: char) -> bool { + matches!(c, + 'A'..='Z' + | 'a'..='z' + | '\u{00C0}'..='\u{00D6}' + | '\u{00D8}'..='\u{00F6}' + | '\u{00F8}'..='\u{02FF}' + | '\u{0370}'..='\u{037D}' + | '\u{037F}'..='\u{1FFF}' + | '\u{200C}'..='\u{200D}' + | '\u{2070}'..='\u{218F}' + | '\u{2C00}'..='\u{2FEF}' + | '\u{3001}'..='\u{D7FF}' + | '\u{F900}'..='\u{FDCF}' + | '\u{FDF0}'..='\u{FFFD}' + | '\u{10000}'..='\u{EFFFF}') + } + + // [158s] PN_CHARS_U ::= PN_CHARS_BASE | '_' | ':' + fn is_possible_pn_chars_u(c: char) -> bool { + Self::is_possible_pn_chars_base(c) || c == '_' + } + + // [160s] PN_CHARS ::= PN_CHARS_U | '-' | [0-9] | #x00B7 | [#x0300-#x036F] | [#x203F-#x2040] + fn is_possible_pn_chars(c: char) -> bool { + Self::is_possible_pn_chars_u(c) + || matches!(c, + '-' | '0'..='9' | '\u{00B7}' | '\u{0300}'..='\u{036F}' | '\u{203F}'..='\u{2040}') + } + + fn is_possible_pn_chars_base_but_not_valid_iri(c: char) -> bool { + matches!(c, '\u{FFF0}'..='\u{FFFD}') + || u32::from(c) % u32::from('\u{FFFE}') == 0 + || u32::from(c) % u32::from('\u{FFFF}') == 0 + } +} + +pub fn resolve_local_name( + prefix: &str, + local: &str, + might_be_invalid_iri: bool, + prefixes: &HashMap>, +) -> Result { + if let Some(start) = prefixes.get(prefix) { + let iri = format!("{start}{local}"); + if might_be_invalid_iri || start.path().is_empty() { + // We validate again. We always validate if the local part might be the IRI authority. + if let Err(e) = Iri::parse(iri.as_str()) { + return Err(format!( + "The prefixed name {prefix}:{local} builds IRI {iri} that is invalid: {e}" + )); + } + } + Ok(NamedNode::new_unchecked(iri)) + } else { + Err(format!("The prefix {prefix}: has not been declared")) + } +} diff --git a/lib/oxttl/src/lib.rs b/lib/oxttl/src/lib.rs new file mode 100644 index 00000000..e0d1ce90 --- /dev/null +++ b/lib/oxttl/src/lib.rs @@ -0,0 +1,19 @@ +mod lexer; +mod line_formats; +pub mod n3; +pub mod nquads; +pub mod ntriples; +mod terse; +mod toolkit; +pub mod trig; +pub mod turtle; + +pub use crate::n3::N3Parser; +pub use crate::nquads::{NQuadsParser, NQuadsSerializer}; +pub use crate::ntriples::{NTriplesParser, NTriplesSerializer}; +pub use crate::toolkit::{ParseError, ParseOrIoError}; +pub use crate::trig::{TriGParser, TriGSerializer}; +pub use crate::turtle::{TurtleParser, TurtleSerializer}; + +pub(crate) const MIN_BUFFER_SIZE: usize = 4096; +pub(crate) const MAX_BUFFER_SIZE: usize = 4096 * 4096; diff --git a/lib/oxttl/src/line_formats.rs b/lib/oxttl/src/line_formats.rs new file mode 100644 index 00000000..6a253658 --- /dev/null +++ b/lib/oxttl/src/line_formats.rs @@ -0,0 +1,305 @@ +//! Shared parser implementation for N-Triples and N-Quads. + +use crate::lexer::{N3Lexer, N3LexerMode, N3LexerOptions, N3Token}; +use crate::toolkit::{Lexer, Parser, RuleRecognizer, RuleRecognizerError}; +use crate::{MAX_BUFFER_SIZE, MIN_BUFFER_SIZE}; +#[cfg(feature = "rdf-star")] +use oxrdf::Triple; +use oxrdf::{BlankNode, GraphName, Literal, NamedNode, Quad, Subject, Term}; + +pub struct NQuadsRecognizer { + stack: Vec, + with_graph_name: bool, + #[cfg(feature = "rdf-star")] + with_quoted_triples: bool, + lexer_options: N3LexerOptions, + subjects: Vec, + predicates: Vec, + objects: Vec, +} + +enum NQuadsState { + ExpectSubject, + ExpectPredicate, + ExpectedObject, + ExpectPossibleGraphOrEndOfQuotedTriple, + ExpectDot, + ExpectLiteralAnnotationOrGraphNameOrDot { + value: String, + }, + ExpectLiteralDatatype { + value: String, + }, + #[cfg(feature = "rdf-star")] + AfterQuotedSubject, + #[cfg(feature = "rdf-star")] + AfterQuotedObject, +} + +impl RuleRecognizer for NQuadsRecognizer { + type TokenRecognizer = N3Lexer; + type Output = Quad; + + fn error_recovery_state(mut self) -> Self { + self.stack.clear(); + self.subjects.clear(); + self.predicates.clear(); + self.objects.clear(); + self + } + + fn recognize_next( + mut self, + token: N3Token, + results: &mut Vec, + errors: &mut Vec, + ) -> Self { + if let Some(state) = self.stack.pop() { + match state { + NQuadsState::ExpectSubject => match token { + N3Token::IriRef(s) => { + self.subjects + .push(NamedNode::new_unchecked(s.into_inner()).into()); + self.stack.push(NQuadsState::ExpectPredicate); + self + } + N3Token::BlankNodeLabel(s) => { + self.subjects.push(BlankNode::new_unchecked(s).into()); + self.stack.push(NQuadsState::ExpectPredicate); + self + } + #[cfg(feature = "rdf-star")] + N3Token::Punctuation("<<") if self.with_quoted_triples => { + self.stack.push(NQuadsState::AfterQuotedSubject); + self.stack.push(NQuadsState::ExpectSubject); + self + } + token => self.error( + errors, + format!("The subject of a triple should be an IRI or a blank node, {token:?} found"), + ), + }, + NQuadsState::ExpectPredicate => match token { + N3Token::IriRef(p) => { + self.predicates + .push(NamedNode::new_unchecked(p.into_inner())); + self.stack.push(NQuadsState::ExpectedObject); + self + } + token => self.error( + errors, + format!("The predicate of a triple should be an IRI, {token:?} found"), + ), + }, + NQuadsState::ExpectedObject => match token { + N3Token::IriRef(o) => { + self.objects + .push(NamedNode::new_unchecked(o.into_inner()).into()); + self.stack + .push(NQuadsState::ExpectPossibleGraphOrEndOfQuotedTriple); + self + } + N3Token::BlankNodeLabel(o) => { + self.objects.push(BlankNode::new_unchecked(o).into()); + self.stack + .push(NQuadsState::ExpectPossibleGraphOrEndOfQuotedTriple); + self + } + N3Token::String(value) => { + self.stack + .push(NQuadsState::ExpectLiteralAnnotationOrGraphNameOrDot { value }); + self + } + #[cfg(feature = "rdf-star")] + N3Token::Punctuation("<<") if self.with_quoted_triples => { + self.stack.push(NQuadsState::AfterQuotedObject); + self.stack.push(NQuadsState::ExpectSubject); + self + } + token => self.error( + errors, + format!("The object of a triple should be an IRI, a blank node or a literal, {token:?} found"), + ), + }, + NQuadsState::ExpectLiteralAnnotationOrGraphNameOrDot { value } => match token { + N3Token::LangTag(lang_tag) => { + self.objects.push( + Literal::new_language_tagged_literal_unchecked( + value, + lang_tag.to_ascii_lowercase(), + ) + .into(), + ); + self.stack + .push(NQuadsState::ExpectPossibleGraphOrEndOfQuotedTriple); + self + } + N3Token::Punctuation("^^") => { + self.stack + .push(NQuadsState::ExpectLiteralDatatype { value }); + self + } + token => { + self.objects.push(Literal::new_simple_literal(value).into()); + self.stack + .push(NQuadsState::ExpectPossibleGraphOrEndOfQuotedTriple); + self.recognize_next(token, results, errors) + } + }, + NQuadsState::ExpectLiteralDatatype { value } => match token { + N3Token::IriRef(d) => { + self.objects.push( + Literal::new_typed_literal( + value, + NamedNode::new_unchecked(d.into_inner()), + ) + .into(), + ); + self.stack + .push(NQuadsState::ExpectPossibleGraphOrEndOfQuotedTriple); + self + } + token => self.error(errors, format!("A literal datatype must be an IRI, found {token:?}")), + }, + NQuadsState::ExpectPossibleGraphOrEndOfQuotedTriple => { + if self.stack.is_empty() { + match token { + N3Token::IriRef(g) if self.with_graph_name => { + self.emit_quad( + results, + NamedNode::new_unchecked(g.into_inner()).into(), + ); + self.stack.push(NQuadsState::ExpectDot); + self + } + N3Token::BlankNodeLabel(g) if self.with_graph_name => { + self.emit_quad(results, BlankNode::new_unchecked(g).into()); + self.stack.push(NQuadsState::ExpectDot); + self + } + token => { + self.emit_quad(results, GraphName::DefaultGraph); + self.stack.push(NQuadsState::ExpectDot); + self.recognize_next(token, results, errors) + } + } + } else if token == N3Token::Punctuation(">>") { + self + } else { + self.error(errors, "Expecting the end of a quoted triple '>>'") + } + } + NQuadsState::ExpectDot => match token { + N3Token::Punctuation(".") => { + self.stack.push(NQuadsState::ExpectSubject); + self + } + token => { + errors.push("Quads should be followed by a dot".into()); + self.stack.push(NQuadsState::ExpectSubject); + self.recognize_next(token, results, errors) + } + }, + #[cfg(feature = "rdf-star")] + NQuadsState::AfterQuotedSubject => { + let triple = Triple { + subject: self.subjects.pop().unwrap(), + predicate: self.predicates.pop().unwrap(), + object: self.objects.pop().unwrap(), + }; + self.subjects.push(triple.into()); + self.stack.push(NQuadsState::ExpectPredicate); + self.recognize_next(token, results, errors) + } + #[cfg(feature = "rdf-star")] + NQuadsState::AfterQuotedObject => { + let triple = Triple { + subject: self.subjects.pop().unwrap(), + predicate: self.predicates.pop().unwrap(), + object: self.objects.pop().unwrap(), + }; + self.objects.push(triple.into()); + self.stack + .push(NQuadsState::ExpectPossibleGraphOrEndOfQuotedTriple); + self.recognize_next(token, results, errors) + } + } + } else if token == N3Token::Punctuation(".") { + self.stack.push(NQuadsState::ExpectSubject); + self + } else { + self + } + } + + fn recognize_end(mut self, results: &mut Vec, errors: &mut Vec) { + match &*self.stack { + [NQuadsState::ExpectSubject] | [] => (), + [NQuadsState::ExpectDot] => errors.push("Triples should be followed by a dot".into()), + [NQuadsState::ExpectPossibleGraphOrEndOfQuotedTriple] => { + self.emit_quad(results, GraphName::DefaultGraph); + errors.push("Triples should be followed by a dot".into()) + } + [NQuadsState::ExpectLiteralAnnotationOrGraphNameOrDot { ref value }] => { + self.objects.push(Literal::new_simple_literal(value).into()); + self.emit_quad(results, GraphName::DefaultGraph); + errors.push("Triples should be followed by a dot".into()) + } + _ => errors.push("Unexpected end".into()), //TODO + } + } + + fn lexer_options(&self) -> &N3LexerOptions { + &self.lexer_options + } +} + +impl NQuadsRecognizer { + pub fn new_parser( + with_graph_name: bool, + #[cfg(feature = "rdf-star")] with_quoted_triples: bool, + ) -> Parser { + Parser::new( + Lexer::new( + N3Lexer::new(N3LexerMode::NTriples), + MIN_BUFFER_SIZE, + MAX_BUFFER_SIZE, + true, + Some(b"#"), + ), + NQuadsRecognizer { + stack: vec![NQuadsState::ExpectSubject], + with_graph_name, + #[cfg(feature = "rdf-star")] + with_quoted_triples, + lexer_options: N3LexerOptions::default(), + subjects: Vec::new(), + predicates: Vec::new(), + objects: Vec::new(), + }, + ) + } + + #[must_use] + fn error( + mut self, + errors: &mut Vec, + msg: impl Into, + ) -> Self { + errors.push(msg.into()); + self.stack.clear(); + self.subjects.clear(); + self.predicates.clear(); + self.objects.clear(); + self + } + + fn emit_quad(&mut self, results: &mut Vec, graph_name: GraphName) { + results.push(Quad { + subject: self.subjects.pop().unwrap(), + predicate: self.predicates.pop().unwrap(), + object: self.objects.pop().unwrap(), + graph_name, + }) + } +} diff --git a/lib/oxttl/src/n3.rs b/lib/oxttl/src/n3.rs new file mode 100644 index 00000000..9722e500 --- /dev/null +++ b/lib/oxttl/src/n3.rs @@ -0,0 +1,1035 @@ +//! A [N3](https://w3c.github.io/N3/spec/) streaming parser implemented by [`N3Parser`]. + +use crate::lexer::{resolve_local_name, N3Lexer, N3LexerMode, N3LexerOptions, N3Token}; +use crate::toolkit::{ + FromReadIterator, Lexer, ParseError, Parser, RuleRecognizer, RuleRecognizerError, +}; +use crate::{ParseOrIoError, MAX_BUFFER_SIZE, MIN_BUFFER_SIZE}; +use oxiri::{Iri, IriParseError}; +use oxrdf::vocab::{rdf, xsd}; +#[cfg(feature = "rdf-star")] +use oxrdf::Triple; +use oxrdf::{ + BlankNode, GraphName, Literal, NamedNode, NamedNodeRef, NamedOrBlankNode, Quad, Subject, Term, + Variable, +}; +use std::collections::HashMap; +use std::fmt; +use std::io::Read; + +/// A N3 term i.e. a RDF `Term` or a `Variable`. +#[derive(Eq, PartialEq, Debug, Clone, Hash)] +pub enum N3Term { + NamedNode(NamedNode), + BlankNode(BlankNode), + Literal(Literal), + #[cfg(feature = "rdf-star")] + Triple(Box), + Variable(Variable), +} + +impl fmt::Display for N3Term { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::NamedNode(term) => term.fmt(f), + Self::BlankNode(term) => term.fmt(f), + Self::Literal(term) => term.fmt(f), + #[cfg(feature = "rdf-star")] + Self::Triple(term) => term.fmt(f), + Self::Variable(term) => term.fmt(f), + } + } +} + +impl From for N3Term { + #[inline] + fn from(node: NamedNode) -> Self { + Self::NamedNode(node) + } +} + +impl From> for N3Term { + #[inline] + fn from(node: NamedNodeRef<'_>) -> Self { + Self::NamedNode(node.into_owned()) + } +} + +impl From for N3Term { + #[inline] + fn from(node: BlankNode) -> Self { + Self::BlankNode(node) + } +} + +impl From for N3Term { + #[inline] + fn from(literal: Literal) -> Self { + Self::Literal(literal) + } +} + +#[cfg(feature = "rdf-star")] +impl From for N3Term { + #[inline] + fn from(triple: Triple) -> Self { + Self::Triple(Box::new(triple)) + } +} + +#[cfg(feature = "rdf-star")] +impl From> for N3Term { + #[inline] + fn from(node: Box) -> Self { + Self::Triple(node) + } +} + +impl From for N3Term { + #[inline] + fn from(node: NamedOrBlankNode) -> Self { + match node { + NamedOrBlankNode::NamedNode(node) => node.into(), + NamedOrBlankNode::BlankNode(node) => node.into(), + } + } +} + +impl From for N3Term { + #[inline] + fn from(node: Subject) -> Self { + match node { + Subject::NamedNode(node) => node.into(), + Subject::BlankNode(node) => node.into(), + #[cfg(feature = "rdf-star")] + Subject::Triple(triple) => Self::Triple(triple), + } + } +} + +impl From for N3Term { + #[inline] + fn from(node: Term) -> Self { + match node { + Term::NamedNode(node) => node.into(), + Term::BlankNode(node) => node.into(), + Term::Literal(node) => node.into(), + #[cfg(feature = "rdf-star")] + Term::Triple(triple) => Self::Triple(triple), + } + } +} + +impl From for N3Term { + #[inline] + fn from(variable: Variable) -> Self { + Self::Variable(variable) + } +} + +/// A N3 quad i.e. a quad composed of [`N3Term`]. +/// +/// The `graph_name` is used to encode the formula where the triple is in. +/// In this case the formula is encoded by a blank node. +#[derive(Eq, PartialEq, Debug, Clone, Hash)] +pub struct N3Quad { + /// The [subject](https://www.w3.org/TR/rdf11-concepts/#dfn-subject) of this triple. + pub subject: N3Term, + + /// The [predicate](https://www.w3.org/TR/rdf11-concepts/#dfn-predicate) of this triple. + pub predicate: N3Term, + + /// The [object](https://www.w3.org/TR/rdf11-concepts/#dfn-object) of this triple. + pub object: N3Term, + + /// The name of the RDF [graph](https://www.w3.org/TR/rdf11-concepts/#dfn-rdf-graph) in which the triple is. + pub graph_name: GraphName, +} + +impl fmt::Display for N3Quad { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if self.graph_name == GraphName::DefaultGraph { + write!(f, "{} {} {}", self.subject, self.predicate, self.object) + } else { + write!( + f, + "{} {} {} {}", + self.subject, self.predicate, self.object, self.graph_name + ) + } + } +} + +impl From for N3Quad { + fn from(quad: Quad) -> Self { + Self { + subject: quad.subject.into(), + predicate: quad.predicate.into(), + object: quad.object.into(), + graph_name: quad.graph_name, + } + } +} + +/// A [N3](https://w3c.github.io/N3/spec/) streaming parser. +/// +/// Count the number of people: +/// ``` +/// use oxrdf::NamedNode; +/// use oxttl::ParseError; +/// use oxttl::n3::{N3Parser, N3Term}; +/// +/// let file = b"@base . +/// @prefix schema: . +/// a schema:Person ; +/// schema:name \"Foo\" . +/// a schema:Person ; +/// schema:name \"Bar\" ."; +/// +/// let rdf_type = N3Term::NamedNode(NamedNode::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?); +/// let schema_person = N3Term::NamedNode(NamedNode::new("http://schema.org/Person")?); +/// let mut count = 0; +/// for triple in N3Parser::new().parse_from_read(file.as_ref()) { +/// let triple = triple?; +/// if triple.predicate == rdf_type && triple.object == schema_person { +/// count += 1; +/// } +/// } +/// assert_eq!(2, count); +/// # Result::<_,Box>::Ok(()) +/// ``` +#[derive(Default)] +pub struct N3Parser { + base: Option>, + prefixes: HashMap>, +} + +impl N3Parser { + /// Builds a new [`N3Parser`]. + #[inline] + pub fn new() -> Self { + Self::default() + } + + #[inline] + pub fn with_base_iri(mut self, base_iri: impl Into) -> Result { + self.base = Some(Iri::parse(base_iri.into())?); + Ok(self) + } + + #[inline] + pub fn with_prefix( + mut self, + prefix_name: impl Into, + prefix_iri: impl Into, + ) -> Result { + self.prefixes + .insert(prefix_name.into(), Iri::parse(prefix_iri.into())?); + Ok(self) + } + + /// Parses a N3 file from a [`Read`] implementation. + /// + /// Count the number of people: + /// ``` + /// use oxrdf::NamedNode; + /// use oxttl::ParseError; + /// use oxttl::n3::{N3Parser, N3Term}; + /// + /// let file = b"@base . + /// @prefix schema: . + /// a schema:Person ; + /// schema:name \"Foo\" . + /// a schema:Person ; + /// schema:name \"Bar\" ."; + /// + /// let rdf_type = N3Term::NamedNode(NamedNode::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?); + /// let schema_person = N3Term::NamedNode(NamedNode::new("http://schema.org/Person")?); + /// let mut count = 0; + /// for triple in N3Parser::new().parse_from_read(file.as_ref()) { + /// let triple = triple?; + /// if triple.predicate == rdf_type && triple.object == schema_person { + /// count += 1; + /// } + /// } + /// assert_eq!(2, count); + /// # Result::<_,Box>::Ok(()) + /// ``` + pub fn parse_from_read(&self, read: R) -> FromReadN3Reader { + FromReadN3Reader { + inner: self.parse().parser.parse_from_read(read), + } + } + + /// Allows to parse a N3 file by using a low-level API. + /// + /// Count the number of people: + /// ``` + /// use oxrdf::NamedNode; + /// use oxttl::ParseError; + /// use oxttl::n3::{N3Parser, N3Term}; + /// + /// let file: [&[u8]; 5] = [b"@base ", + /// b". @prefix schema: .", + /// b" a schema:Person", + /// b" ; schema:name \"Foo\" . ", + /// b" a schema:Person ; schema:name \"Bar\" ." + /// ]; + /// + /// let rdf_type = N3Term::NamedNode(NamedNode::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?); + /// let schema_person = N3Term::NamedNode(NamedNode::new("http://schema.org/Person")?); + /// let mut count = 0; + /// let mut parser = N3Parser::new().parse(); + /// let mut file_chunks = file.iter(); + /// while !parser.is_end() { + /// // We feed more data to the parser + /// if let Some(chunk) = file_chunks.next() { + /// parser.extend_from_slice(chunk); + /// } else { + /// parser.end(); // It's finished + /// } + /// // We read as many triples from the parser as possible + /// while let Some(triple) = parser.read_next() { + /// let triple = triple?; + /// if triple.predicate == rdf_type && triple.object == schema_person { + /// count += 1; + /// } + /// } + /// } + /// assert_eq!(2, count); + /// # Result::<_,Box>::Ok(()) + /// ``` + pub fn parse(&self) -> LowLevelN3Reader { + LowLevelN3Reader { + parser: N3Recognizer::new_parser(self.base.clone(), self.prefixes.clone()), + } + } +} + +/// Parses a N3 file from a [`Read`] implementation. Can be built using [`N3Parser::parse_from_read`]. +/// +/// Count the number of people: +/// ``` +/// use oxrdf::NamedNode; +/// use oxttl::ParseError; +/// use oxttl::n3::{N3Parser, N3Term}; +/// +/// let file = b"@base . +/// @prefix schema: . +/// a schema:Person ; +/// schema:name \"Foo\" . +/// a schema:Person ; +/// schema:name \"Bar\" ."; +/// +/// let rdf_type = N3Term::NamedNode(NamedNode::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?); +/// let schema_person = N3Term::NamedNode(NamedNode::new("http://schema.org/Person")?); +/// let mut count = 0; +/// for triple in N3Parser::new().parse_from_read(file.as_ref()) { +/// let triple = triple?; +/// if triple.predicate == rdf_type && triple.object == schema_person { +/// count += 1; +/// } +/// } +/// assert_eq!(2, count); +/// # Result::<_,Box>::Ok(()) +/// ``` +pub struct FromReadN3Reader { + inner: FromReadIterator, +} + +impl Iterator for FromReadN3Reader { + type Item = Result; + + fn next(&mut self) -> Option> { + self.inner.next() + } +} + +/// Parses a N3 file by using a low-level API. Can be built using [`N3Parser::parse`]. +/// +/// Count the number of people: +/// ``` +/// use oxrdf::NamedNode; +/// use oxttl::ParseError; +/// use oxttl::n3::{N3Parser, N3Term}; +/// +/// let file: [&[u8]; 5] = [b"@base ", +/// b". @prefix schema: .", +/// b" a schema:Person", +/// b" ; schema:name \"Foo\" . ", +/// b" a schema:Person ; schema:name \"Bar\" ." +/// ]; +/// +/// let rdf_type = N3Term::NamedNode(NamedNode::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?); +/// let schema_person = N3Term::NamedNode(NamedNode::new("http://schema.org/Person")?); +/// let mut count = 0; +/// let mut parser = N3Parser::new().parse(); +/// let mut file_chunks = file.iter(); +/// while !parser.is_end() { +/// // We feed more data to the parser +/// if let Some(chunk) = file_chunks.next() { +/// parser.extend_from_slice(chunk); +/// } else { +/// parser.end(); // It's finished +/// } +/// // We read as many triples from the parser as possible +/// while let Some(triple) = parser.read_next() { +/// let triple = triple?; +/// if triple.predicate == rdf_type && triple.object == schema_person { +/// count += 1; +/// } +/// } +/// } +/// assert_eq!(2, count); +/// # Result::<_,Box>::Ok(()) +/// ``` +pub struct LowLevelN3Reader { + parser: Parser, +} + +impl LowLevelN3Reader { + /// Adds some extra bytes to the parser. Should be called when [`read_next`](Self::read_next) returns [`None`] and there is still unread data. + pub fn extend_from_slice(&mut self, other: &[u8]) { + self.parser.extend_from_slice(other) + } + + /// Tell the parser that the file is finished. + /// + /// This triggers the parsing of the final bytes and might lead [`read_next`](Self::read_next) to return some extra values. + pub fn end(&mut self) { + self.parser.end() + } + + /// Returns if the parsing is finished i.e. [`end`](Self::end) has been called and [`read_next`](Self::read_next) is always going to return `None`. + pub fn is_end(&self) -> bool { + self.parser.is_end() + } + + /// Attempt to parse a new quad from the already provided data. + /// + /// Returns [`None`] if the parsing is finished or more data is required. + /// If it is the case more data should be fed using [`extend_from_slice`](Self::extend_from_slice). + pub fn read_next(&mut self) -> Option> { + self.parser.read_next() + } +} + +#[derive(Clone)] +enum Predicate { + Regular(N3Term), + Inverted(N3Term), +} + +struct N3Recognizer { + stack: Vec, + lexer_options: N3LexerOptions, + prefixes: HashMap>, + terms: Vec, + predicates: Vec, + contexts: Vec, +} + +impl RuleRecognizer for N3Recognizer { + type TokenRecognizer = N3Lexer; + type Output = N3Quad; + + fn error_recovery_state(mut self) -> Self { + self.stack.clear(); + self.terms.clear(); + self.predicates.clear(); + self.contexts.clear(); + self + } + + fn recognize_next( + mut self, + token: N3Token, + results: &mut Vec, + errors: &mut Vec, + ) -> Self { + if let Some(rule) = self.stack.pop() { + match rule { + // [1] n3Doc ::= ( ( n3Statement ".") | sparqlDirective) * + // [2] n3Statement ::= n3Directive | triples + // [3] n3Directive ::= prefixID | base + // [4] sparqlDirective ::= sparqlBase | sparqlPrefix + // [5] sparqlBase ::= BASE IRIREF + // [6] sparqlPrefix ::= PREFIX PNAME_NS IRIREF + // [7] prefixID ::= "@prefix" PNAME_NS IRIREF + // [8] base ::= "@base" IRIREF + N3State::N3Doc => { + self.stack.push(N3State::N3Doc); + match token { + N3Token::PlainKeyword(k) if k.eq_ignore_ascii_case("base") => { + self.stack.push(N3State::BaseExpectIri); + self + } + N3Token::PlainKeyword(k) if k.eq_ignore_ascii_case("prefix") => { + self.stack.push(N3State::PrefixExpectPrefix); + self + } + N3Token::LangTag("prefix") => { + self.stack.push(N3State::N3DocExpectDot); + self.stack.push(N3State::PrefixExpectPrefix); + self + } + N3Token::LangTag("base") => { + self.stack.push(N3State::N3DocExpectDot); + self.stack.push(N3State::BaseExpectIri); + self + } + token => { + self.stack.push(N3State::N3DocExpectDot); + self.stack.push(N3State::Triples); + self.recognize_next(token, results, errors) + } + } + }, + N3State::N3DocExpectDot => { + if token == N3Token::Punctuation(".") { + self + } else { + errors.push("A dot is expected at the end of N3 statements".into()); + self.recognize_next(token, results, errors) + } + }, + N3State::BaseExpectIri => match token { + N3Token::IriRef(iri) => { + self.lexer_options.base_iri = Some(iri); + self + } + _ => self.error(errors, "The BASE keyword should be followed by an IRI"), + }, + N3State::PrefixExpectPrefix => match token { + N3Token::PrefixedName { prefix, local, .. } if local.is_empty() => { + self.stack.push(N3State::PrefixExpectIri { name: prefix.to_owned() }); + self + } + _ => { + self.error(errors, "The PREFIX keyword should be followed by a prefix like 'ex:'") + } + }, + N3State::PrefixExpectIri { name } => match token { + N3Token::IriRef(iri) => { + self.prefixes.insert(name, iri); + self + } + _ => self.error(errors, "The PREFIX declaration should be followed by a prefix and its value as an IRI"), + }, + // [9] triples ::= subject predicateObjectList? + N3State::Triples => { + self.stack.push(N3State::TriplesMiddle); + self.stack.push(N3State::Path); + self.recognize_next(token, results, errors) + }, + N3State::TriplesMiddle => if matches!(token, N3Token::Punctuation("." | "]" | "}" | ")")) { + self.recognize_next(token, results, errors) + } else { + self.stack.push(N3State::TriplesEnd); + self.stack.push(N3State::PredicateObjectList); + self.recognize_next(token, results, errors) + }, + N3State::TriplesEnd => { + self.terms.pop(); + self.recognize_next(token, results, errors) + }, + // [10] predicateObjectList ::= verb objectList ( ";" ( verb objectList) ? ) * + N3State::PredicateObjectList => { + self.stack.push(N3State::PredicateObjectListEnd); + self.stack.push(N3State::ObjectsList); + self.stack.push(N3State::Verb); + self.recognize_next(token, results, errors) + }, + N3State::PredicateObjectListEnd => { + self.predicates.pop(); + if token == N3Token::Punctuation(";") { + self.stack.push(N3State::PredicateObjectListPossibleContinuation); + self + } else { + self.recognize_next(token, results, errors) + } + }, + N3State::PredicateObjectListPossibleContinuation => if token == N3Token::Punctuation(";") { + self.stack.push(N3State::PredicateObjectListPossibleContinuation); + self + } else if matches!(token, N3Token::Punctuation(";" | "." | "}" | "]" | ")")) { + self.recognize_next(token, results, errors) + } else { + self.stack.push(N3State::PredicateObjectListEnd); + self.stack.push(N3State::ObjectsList); + self.stack.push(N3State::Verb); + self.recognize_next(token, results, errors) + }, + // [11] objectList ::= object ( "," object) * + N3State::ObjectsList => { + self.stack.push(N3State::ObjectsListEnd); + self.stack.push(N3State::Path); + self.recognize_next(token, results, errors) + } + N3State::ObjectsListEnd => { + let object = self.terms.pop().unwrap(); + let subject = self.terms.last().unwrap().clone(); + results.push(match self.predicates.last().unwrap().clone() { + Predicate::Regular(predicate) => self.quad( + subject, + predicate, + object, + ), + Predicate::Inverted(predicate) => self.quad( + object, + predicate, + subject, + ) + }); + if token == N3Token::Punctuation(",") { + self.stack.push(N3State::ObjectsListEnd); + self.stack.push(N3State::Path); + self + } else { + self.recognize_next(token, results, errors) + } + }, + // [12] verb ::= predicate | "a" | ( "has" expression) | ( "is" expression "of") | "=" | "<=" | "=>" + // [14] predicate ::= expression | ( "<-" expression) + N3State::Verb => match token { + N3Token::PlainKeyword("a") => { + self.predicates.push(Predicate::Regular(rdf::TYPE.into())); + self + } + N3Token::PlainKeyword("has") => { + self.stack.push(N3State::AfterRegularVerb); + self.stack.push(N3State::Path); + self + } + N3Token::PlainKeyword("is") => { + self.stack.push(N3State::AfterVerbIs); + self.stack.push(N3State::Path); + self + } + N3Token::Punctuation("=") => { + self.predicates.push(Predicate::Regular(NamedNode::new_unchecked("http://www.w3.org/2002/07/owl#sameAs").into())); + self + } + N3Token::Punctuation("=>") => { + self.predicates.push(Predicate::Regular(NamedNode::new_unchecked("http://www.w3.org/2000/10/swap/log#implies").into())); + self + } + N3Token::Punctuation("<=") => { + self.predicates.push(Predicate::Inverted(NamedNode::new_unchecked("http://www.w3.org/2000/10/swap/log#implies").into())); + self + } + N3Token::Punctuation("<-") => { + self.stack.push(N3State::AfterInvertedVerb); + self.stack.push(N3State::Path); + self + } + token => { + self.stack.push(N3State::AfterRegularVerb); + self.stack.push(N3State::Path); + self.recognize_next(token, results, errors) + } + } + N3State::AfterRegularVerb => { + self.predicates.push(Predicate::Regular(self.terms.pop().unwrap())); + self.recognize_next(token, results, errors) + } + N3State::AfterInvertedVerb => { + self.predicates.push(Predicate::Inverted(self.terms.pop().unwrap())); + self.recognize_next(token, results, errors) + } + N3State::AfterVerbIs => match token { + N3Token::PlainKeyword("of") => { + self.predicates.push(Predicate::Inverted(self.terms.pop().unwrap())); + self + }, + _ => { + self.error(errors, "The keyword 'is' should be followed by a predicate then by the keyword 'of'") + } + } + // [13] subject ::= expression + // [15] object ::= expression + // [16] expression ::= path + // [17] path ::= pathItem ( ( "!" path) | ( "^" path) ) ? + N3State::Path => { + self.stack.push(N3State::PathFollowUp); + self.stack.push(N3State::PathItem); + self.recognize_next(token, results, errors) + } + N3State::PathFollowUp => match token { + N3Token::Punctuation("!") => { + self.stack.push(N3State::PathAfterIndicator { is_inverse: false }); + self.stack.push(N3State::PathItem); + self + } + N3Token::Punctuation("^") => { + self.stack.push(N3State::PathAfterIndicator { is_inverse: true }); + self.stack.push(N3State::PathItem); + self + } + token => self.recognize_next(token, results, errors) + }, + N3State::PathAfterIndicator { is_inverse } => { + let predicate = self.terms.pop().unwrap(); + let previous = self.terms.pop().unwrap(); + let current = BlankNode::default(); + results.push(if is_inverse { self.quad(current.clone(), predicate, previous) } else { self.quad(previous, predicate, current.clone())}); + self.terms.push(current.into()); + self.stack.push(N3State::PathFollowUp); + self.recognize_next(token, results, errors) + }, + // [18] pathItem ::= iri | blankNode | quickVar | collection | blankNodePropertyList | iriPropertyList | literal | formula + // [19] literal ::= rdfLiteral | numericLiteral | BOOLEAN_LITERAL + // [20] blankNodePropertyList ::= "[" predicateObjectList "]" + // [21] iriPropertyList ::= IPLSTART iri predicateObjectList "]" + // [22] collection ::= "(" object* ")" + // [23] formula ::= "{" formulaContent? "}" + // [25] numericLiteral ::= DOUBLE | DECIMAL | INTEGER + // [26] rdfLiteral ::= STRING ( LANGTAG | ( "^^" iri) ) ? + // [27] iri ::= IRIREF | prefixedName + // [28] prefixedName ::= PNAME_LN | PNAME_NS + // [29] blankNode ::= BLANK_NODE_LABEL | ANON + // [30] quickVar ::= QUICK_VAR_NAME + N3State::PathItem => { + match token { + N3Token::IriRef(iri) => { + self.terms.push(NamedNode::new_unchecked(iri.into_inner()).into()); + self + } + N3Token::PrefixedName { prefix, local, might_be_invalid_iri } => match resolve_local_name(prefix, &local, might_be_invalid_iri, &self.prefixes) { + Ok(t) => { + self.terms.push(t.into()); + self + }, + Err(e) => self.error(errors, e) + } + N3Token::BlankNodeLabel(bnode) => { + self.terms.push(BlankNode::new_unchecked(bnode).into()); + self + } + N3Token::Variable(name) => { + self.terms.push(Variable::new_unchecked(name).into()); + self + } + N3Token::Punctuation("[") => { + self.stack.push(N3State::PropertyListMiddle); + self + } + N3Token::Punctuation("(") => { + self.stack.push(N3State::CollectionBeginning); + self + } + N3Token::String(value) => { + self.stack.push(N3State::LiteralPossibleSuffix { value }); + self + } + N3Token::Integer(v) => { + self.terms.push(Literal::new_typed_literal(v, xsd::INTEGER).into()); + self + } + N3Token::Decimal(v) => { + self.terms.push(Literal::new_typed_literal(v, xsd::DECIMAL).into()); + self + } + N3Token::Double(v) => { + self.terms.push(Literal::new_typed_literal(v, xsd::DOUBLE).into()); + self + } + N3Token::PlainKeyword("true") => { + self.terms.push(Literal::new_typed_literal("true", xsd::BOOLEAN).into()); + self + } + N3Token::PlainKeyword("false") => { + self.terms.push(Literal::new_typed_literal("false", xsd::BOOLEAN).into()); + self + } + N3Token::Punctuation("{") => { + self.contexts.push(BlankNode::default()); + self.stack.push(N3State::FormulaContent); + self + } + token => self.error(errors, format!("This is not a valid RDF value: {token:?}")) + } + } + N3State::PropertyListMiddle => match token { + N3Token::Punctuation("]") => { + self.terms.push(BlankNode::default().into()); + self + }, + N3Token::PlainKeyword("id") => { + self.stack.push(N3State::IriPropertyList); + self + }, + token => { + self.terms.push(BlankNode::default().into()); + self.stack.push(N3State::PropertyListEnd); + self.stack.push(N3State::PredicateObjectList); + self.recognize_next(token, results, errors) + } + } + N3State::PropertyListEnd => if token == N3Token::Punctuation("]") { + self + } else { + errors.push("blank node property lists should end with a ']'".into()); + self.recognize_next(token, results, errors) + } + N3State::IriPropertyList => match token { + N3Token::IriRef(id) => { + self.terms.push(NamedNode::new_unchecked(id.into_inner()).into()); + self.stack.push(N3State::PropertyListEnd); + self.stack.push(N3State::PredicateObjectList); + self + } + N3Token::PrefixedName { prefix, local, might_be_invalid_iri } => match resolve_local_name(prefix, &local, might_be_invalid_iri, &self.prefixes) { + Ok(t) => { + self.terms.push(t.into()); + self.stack.push(N3State::PropertyListEnd); + self.stack.push(N3State::PredicateObjectList); + self + }, + Err(e) => self.error(errors, e) + } + _ => { + self.error(errors, "The '[ id' construction should be followed by an IRI") + } + } + N3State::CollectionBeginning => match token { + N3Token::Punctuation(")") => { + self.terms.push(rdf::NIL.into()); + self + } + token => { + let root = BlankNode::default(); + self.terms.push(root.clone().into()); + self.terms.push(root.into()); + self.stack.push(N3State::CollectionPossibleEnd); + self.stack.push(N3State::Path); + self.recognize_next(token, results, errors) + } + }, + N3State::CollectionPossibleEnd => { + let value = self.terms.pop().unwrap(); + let old = self.terms.pop().unwrap(); + results.push(self.quad( + old.clone(), + rdf::FIRST, + value, + )); + match token { + N3Token::Punctuation(")") => { + results.push(self.quad(old, + rdf::REST, + rdf::NIL + )); + self + } + token => { + let new = BlankNode::default(); + results.push(self.quad( old, + rdf::REST, + new.clone() + )); + self.terms.push(new.into()); + self.stack.push(N3State::CollectionPossibleEnd); + self.stack.push(N3State::Path); + self.recognize_next(token, results, errors) + } + } + } + N3State::LiteralPossibleSuffix { value } => { + match token { + N3Token::LangTag(lang) => { + self.terms.push(Literal::new_language_tagged_literal_unchecked(value, lang.to_ascii_lowercase()).into()); + self + }, + N3Token::Punctuation("^^") => { + self.stack.push(N3State::LiteralExpectDatatype { value }); + self + } + token => { + self.terms.push(Literal::new_simple_literal(value).into()); + self.recognize_next(token, results, errors) + } + } + } + N3State::LiteralExpectDatatype { value } => { + match token { + N3Token::IriRef(datatype) => { + self.terms.push(Literal::new_typed_literal(value, NamedNode::new_unchecked(datatype.into_inner())).into()); + self + }, + N3Token::PrefixedName { prefix, local, might_be_invalid_iri } => match resolve_local_name(prefix, &local, might_be_invalid_iri, &self.prefixes) { + Ok(datatype) =>{ + self.terms.push(Literal::new_typed_literal(value, datatype).into()); + self + }, + Err(e) => self.error(errors, e) + } + token => { + self.error(errors, format!("Expecting a datatype IRI after '^^, found {token:?}")).recognize_next(token, results, errors) + } + } + } + // [24] formulaContent ::= ( n3Statement ( "." formulaContent? ) ? ) | ( sparqlDirective formulaContent? ) + N3State::FormulaContent => { + match token { + N3Token::Punctuation("}") => { + self.terms.push(self.contexts.pop().unwrap().into()); + self + } + N3Token::PlainKeyword(k)if k.eq_ignore_ascii_case("base") => { + self.stack.push(N3State::FormulaContent); + self.stack.push(N3State::BaseExpectIri); + self + } + N3Token::PlainKeyword(k)if k.eq_ignore_ascii_case("prefix") => { + self.stack.push(N3State::FormulaContent); + self.stack.push(N3State::PrefixExpectPrefix); + self + } + N3Token::LangTag("prefix") => { + self.stack.push(N3State::FormulaContentExpectDot); + self.stack.push(N3State::PrefixExpectPrefix); + self + } + N3Token::LangTag("base") => { + self.stack.push(N3State::FormulaContentExpectDot); + self.stack.push(N3State::BaseExpectIri); + self + } + token => { + self.stack.push(N3State::FormulaContentExpectDot); + self.stack.push(N3State::Triples); + self.recognize_next(token, results, errors) + } + } + } + N3State::FormulaContentExpectDot => { + match token { + N3Token::Punctuation("}") => { + self.terms.push(self.contexts.pop().unwrap().into()); + self + } + N3Token::Punctuation(".") => { + self.stack.push(N3State::FormulaContent); + self + } + token => { + errors.push("A dot is expected at the end of N3 statements".into()); + self.stack.push(N3State::FormulaContent); + self.recognize_next(token, results, errors) + } + } + } + } + } else if token == N3Token::Punctuation(".") { + self.stack.push(N3State::N3Doc); + self + } else { + self + } + } + + fn recognize_end( + self, + _results: &mut Vec, + errors: &mut Vec, + ) { + match &*self.stack { + [] | [N3State::N3Doc] => (), + _ => errors.push("Unexpected end".into()), //TODO + } + } + + fn lexer_options(&self) -> &N3LexerOptions { + &self.lexer_options + } +} + +impl N3Recognizer { + pub fn new_parser( + base_iri: Option>, + prefixes: HashMap>, + ) -> Parser { + Parser::new( + Lexer::new( + N3Lexer::new(N3LexerMode::N3), + MIN_BUFFER_SIZE, + MAX_BUFFER_SIZE, + true, + Some(b"#"), + ), + N3Recognizer { + stack: vec![N3State::N3Doc], + lexer_options: N3LexerOptions { base_iri }, + prefixes, + terms: Vec::new(), + predicates: Vec::new(), + contexts: Vec::new(), + }, + ) + } + + #[must_use] + fn error( + mut self, + errors: &mut Vec, + msg: impl Into, + ) -> Self { + errors.push(msg.into()); + self.stack.clear(); + self + } + + fn quad( + &self, + subject: impl Into, + predicate: impl Into, + object: impl Into, + ) -> N3Quad { + N3Quad { + subject: subject.into(), + predicate: predicate.into(), + object: object.into(), + graph_name: self + .contexts + .last() + .map_or(GraphName::DefaultGraph, |g| g.clone().into()), + } + } +} + +#[derive(Debug)] +enum N3State { + N3Doc, + N3DocExpectDot, + BaseExpectIri, + PrefixExpectPrefix, + PrefixExpectIri { name: String }, + Triples, + TriplesMiddle, + TriplesEnd, + PredicateObjectList, + PredicateObjectListEnd, + PredicateObjectListPossibleContinuation, + ObjectsList, + ObjectsListEnd, + Verb, + AfterRegularVerb, + AfterInvertedVerb, + AfterVerbIs, + Path, + PathFollowUp, + PathAfterIndicator { is_inverse: bool }, + PathItem, + PropertyListMiddle, + PropertyListEnd, + IriPropertyList, + CollectionBeginning, + CollectionPossibleEnd, + LiteralPossibleSuffix { value: String }, + LiteralExpectDatatype { value: String }, + FormulaContent, + FormulaContentExpectDot, +} diff --git a/lib/oxttl/src/nquads.rs b/lib/oxttl/src/nquads.rs new file mode 100644 index 00000000..c3357a38 --- /dev/null +++ b/lib/oxttl/src/nquads.rs @@ -0,0 +1,393 @@ +//! A [N-Quads](https://www.w3.org/TR/n-quads/) streaming parser implemented by [`NQuadsParser`]. + +use crate::line_formats::NQuadsRecognizer; +use crate::toolkit::{FromReadIterator, ParseError, ParseOrIoError, Parser}; +use oxrdf::{Quad, QuadRef}; +use std::io::{self, Read, Write}; + +/// A [N-Quads](https://www.w3.org/TR/n-quads/) streaming parser. +/// +/// Support for [N-Quads-star](https://w3c.github.io/rdf-star/cg-spec/2021-12-17.html#n-quads-star) is available behind the `rdf-star` feature and the [`NQuadsParser::with_quoted_triples`] option. +/// +/// Count the number of people: +/// ``` +/// use oxrdf::NamedNodeRef; +/// use oxttl::{NQuadsParser, ParseError}; +/// +/// let file = b" . +/// \"Foo\" . +/// . +/// \"Bar\" ."; +/// +/// let rdf_type = NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?; +/// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; +/// let mut count = 0; +/// for quad in NQuadsParser::new().parse_from_read(file.as_ref()) { +/// let quad = quad?; +/// if quad.predicate == rdf_type && quad.object == schema_person.into() { +/// count += 1; +/// } +/// } +/// assert_eq!(2, count); +/// # Result::<_,Box>::Ok(()) +/// ``` +#[derive(Default)] +pub struct NQuadsParser { + #[cfg(feature = "rdf-star")] + with_quoted_triples: bool, +} + +impl NQuadsParser { + /// Builds a new [`NQuadsParser`]. + #[inline] + pub fn new() -> Self { + Self::default() + } + + /// Enables [N-Quads-star](https://w3c.github.io/rdf-star/cg-spec/2021-12-17.html#n-quads-star). + #[cfg(feature = "rdf-star")] + #[inline] + #[must_use] + pub fn with_quoted_triples(mut self) -> Self { + self.with_quoted_triples = true; + self + } + + /// Parses a N-Quads file from a [`Read`] implementation. + /// + /// Count the number of people: + /// ``` + /// use oxrdf::NamedNodeRef; + /// use oxttl::{NQuadsParser, ParseError}; + /// + /// let file = b" . + /// \"Foo\" . + /// . + /// \"Bar\" ."; + /// + /// let rdf_type = NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?; + /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; + /// let mut count = 0; + /// for quad in NQuadsParser::new().parse_from_read(file.as_ref()) { + /// let quad = quad?; + /// if quad.predicate == rdf_type && quad.object == schema_person.into() { + /// count += 1; + /// } + /// } + /// assert_eq!(2, count); + /// # Result::<_,Box>::Ok(()) + /// ``` + pub fn parse_from_read(&self, read: R) -> FromReadNQuadsReader { + FromReadNQuadsReader { + inner: self.parse().parser.parse_from_read(read), + } + } + + /// Allows to parse a N-Quads file by using a low-level API. + /// + /// Count the number of people: + /// ``` + /// use oxrdf::NamedNodeRef; + /// use oxttl::{NQuadsParser, ParseError}; + /// + /// let file: [&[u8]; 4] = [ + /// b" .\n", + /// b" \"Foo\" .\n", + /// b" .\n", + /// b" \"Bar\" .\n" + /// ]; + /// + /// let rdf_type = NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?; + /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; + /// let mut count = 0; + /// let mut parser = NQuadsParser::new().parse(); + /// let mut file_chunks = file.iter(); + /// while !parser.is_end() { + /// // We feed more data to the parser + /// if let Some(chunk) = file_chunks.next() { + /// parser.extend_from_slice(chunk); + /// } else { + /// parser.end(); // It's finished + /// } + /// // We read as many quads from the parser as possible + /// while let Some(quad) = parser.read_next() { + /// let quad = quad?; + /// if quad.predicate == rdf_type && quad.object == schema_person.into() { + /// count += 1; + /// } + /// } + /// } + /// assert_eq!(2, count); + /// # Result::<_,Box>::Ok(()) + /// ``` + #[allow(clippy::unused_self)] + pub fn parse(&self) -> LowLevelNQuadsReader { + LowLevelNQuadsReader { + parser: NQuadsRecognizer::new_parser( + true, + #[cfg(feature = "rdf-star")] + self.with_quoted_triples, + ), + } + } +} + +/// Parses a N-Quads file from a [`Read`] implementation. Can be built using [`NQuadsParser::parse_from_read`]. +/// +/// Count the number of people: +/// ``` +/// use oxrdf::NamedNodeRef; +/// use oxttl::{NQuadsParser, ParseError}; +/// +/// let file = b" . +/// \"Foo\" . +/// . +/// \"Bar\" ."; +/// +/// let rdf_type = NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?; +/// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; +/// let mut count = 0; +/// for quad in NQuadsParser::new().parse_from_read(file.as_ref()) { +/// let quad = quad?; +/// if quad.predicate == rdf_type && quad.object == schema_person.into() { +/// count += 1; +/// } +/// } +/// assert_eq!(2, count); +/// # Result::<_,Box>::Ok(()) +/// ``` +pub struct FromReadNQuadsReader { + inner: FromReadIterator, +} + +impl Iterator for FromReadNQuadsReader { + type Item = Result; + + fn next(&mut self) -> Option> { + self.inner.next() + } +} + +/// Parses a N-Quads file by using a low-level API. Can be built using [`NQuadsParser::parse`]. +/// +/// Count the number of people: +/// ``` +/// use oxrdf::NamedNodeRef; +/// use oxttl::{NQuadsParser, ParseError}; +/// +/// let file: [&[u8]; 4] = [ +/// b" .\n", +/// b" \"Foo\" .\n", +/// b" .\n", +/// b" \"Bar\" .\n" +/// ]; +/// +/// let rdf_type = NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?; +/// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; +/// let mut count = 0; +/// let mut parser = NQuadsParser::new().parse(); +/// let mut file_chunks = file.iter(); +/// while !parser.is_end() { +/// // We feed more data to the parser +/// if let Some(chunk) = file_chunks.next() { +/// parser.extend_from_slice(chunk); +/// } else { +/// parser.end(); // It's finished +/// } +/// // We read as many quads from the parser as possible +/// while let Some(quad) = parser.read_next() { +/// let quad = quad?; +/// if quad.predicate == rdf_type && quad.object == schema_person.into() { +/// count += 1; +/// } +/// } +/// } +/// assert_eq!(2, count); +/// # Result::<_,Box>::Ok(()) +/// ``` +pub struct LowLevelNQuadsReader { + parser: Parser, +} + +impl LowLevelNQuadsReader { + /// Adds some extra bytes to the parser. Should be called when [`read_next`](Self::read_next) returns [`None`] and there is still unread data. + pub fn extend_from_slice(&mut self, other: &[u8]) { + self.parser.extend_from_slice(other) + } + + /// Tell the parser that the file is finished. + /// + /// This triggers the parsing of the final bytes and might lead [`read_next`](Self::read_next) to return some extra values. + pub fn end(&mut self) { + self.parser.end() + } + + /// Returns if the parsing is finished i.e. [`end`](Self::end) has been called and [`read_next`](Self::read_next) is always going to return `None`. + pub fn is_end(&self) -> bool { + self.parser.is_end() + } + + /// Attempt to parse a new quad from the already provided data. + /// + /// Returns [`None`] if the parsing is finished or more data is required. + /// If it is the case more data should be fed using [`extend_from_slice`](Self::extend_from_slice). + pub fn read_next(&mut self) -> Option> { + self.parser.read_next() + } +} + +/// A [N-Quads](https://www.w3.org/TR/n-quads/) serializer. +/// +/// Support for [N-Quads-star](https://w3c.github.io/rdf-star/cg-spec/2021-12-17.html#n-quads-star) is available behind the `rdf-star` feature. +/// +/// ``` +/// use oxrdf::{NamedNodeRef, QuadRef}; +/// use oxttl::NQuadsSerializer; +/// +/// let mut buf = Vec::new(); +/// let mut writer = NQuadsSerializer::new().serialize_to_write(buf); +/// writer.write_quad(QuadRef::new( +/// NamedNodeRef::new("http://example.com#me")?, +/// NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?, +/// NamedNodeRef::new("http://schema.org/Person")?, +/// NamedNodeRef::new("http://example.com")?, +/// ))?; +/// assert_eq!( +/// b" .\n", +/// writer.finish().as_slice() +/// ); +/// # Result::<_,Box>::Ok(()) +/// ``` +#[derive(Default)] +pub struct NQuadsSerializer; + +impl NQuadsSerializer { + /// Builds a new [`NQuadsSerializer`]. + #[inline] + pub fn new() -> Self { + Self + } + + /// Writes a N-Quads file to a [`Write`] implementation. + /// + /// ``` + /// use oxrdf::{NamedNodeRef, QuadRef}; + /// use oxttl::NQuadsSerializer; + /// + /// let mut buf = Vec::new(); + /// let mut writer = NQuadsSerializer::new().serialize_to_write(buf); + /// writer.write_quad(QuadRef::new( + /// NamedNodeRef::new("http://example.com#me")?, + /// NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?, + /// NamedNodeRef::new("http://schema.org/Person")?, + /// NamedNodeRef::new("http://example.com")?, + /// ))?; + /// assert_eq!( + /// b" .\n", + /// writer.finish().as_slice() + /// ); + /// # Result::<_,Box>::Ok(()) + /// ``` + pub fn serialize_to_write(&self, write: W) -> ToWriteNQuadsWriter { + ToWriteNQuadsWriter { + write, + writer: self.serialize(), + } + } + + /// Builds a low-level N-Quads writer. + /// + /// ``` + /// use oxrdf::{NamedNodeRef, QuadRef}; + /// use oxttl::NQuadsSerializer; + /// + /// let mut buf = Vec::new(); + /// let mut writer = NQuadsSerializer::new().serialize(); + /// writer.write_quad(QuadRef::new( + /// NamedNodeRef::new("http://example.com#me")?, + /// NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?, + /// NamedNodeRef::new("http://schema.org/Person")?, + /// NamedNodeRef::new("http://example.com")?, + /// ), &mut buf)?; + /// assert_eq!( + /// b" .\n", + /// buf.as_slice() + /// ); + /// # Result::<_,Box>::Ok(()) + /// ``` + #[allow(clippy::unused_self)] + pub fn serialize(&self) -> LowLevelNQuadsWriter { + LowLevelNQuadsWriter + } +} + +/// Writes a N-Quads file to a [`Write`] implementation. Can be built using [`NQuadsSerializer::serialize_to_write`]. +/// +/// ``` +/// use oxrdf::{NamedNodeRef, QuadRef}; +/// use oxttl::NQuadsSerializer; +/// +/// let mut buf = Vec::new(); +/// let mut writer = NQuadsSerializer::new().serialize_to_write(buf); +/// writer.write_quad(QuadRef::new( +/// NamedNodeRef::new("http://example.com#me")?, +/// NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?, +/// NamedNodeRef::new("http://schema.org/Person")?, +/// NamedNodeRef::new("http://example.com")?, +/// ))?; +/// assert_eq!( +/// b" .\n", +/// writer.finish().as_slice() +/// ); +/// # Result::<_,Box>::Ok(()) +/// ``` +pub struct ToWriteNQuadsWriter { + write: W, + writer: LowLevelNQuadsWriter, +} + +impl ToWriteNQuadsWriter { + /// Writes an extra quad. + pub fn write_quad<'a>(&mut self, q: impl Into>) -> io::Result<()> { + self.writer.write_quad(q, &mut self.write) + } + + /// Ends the write process and returns the underlying [`Write`]. + pub fn finish(self) -> W { + self.write + } +} + +/// Writes a N-Quads file by using a low-level API. Can be built using [`NQuadsSerializer::serialize`]. +/// +/// ``` +/// use oxrdf::{NamedNodeRef, QuadRef}; +/// use oxttl::NQuadsSerializer; +/// +/// let mut buf = Vec::new(); +/// let mut writer = NQuadsSerializer::new().serialize(); +/// writer.write_quad(QuadRef::new( +/// NamedNodeRef::new("http://example.com#me")?, +/// NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?, +/// NamedNodeRef::new("http://schema.org/Person")?, +/// NamedNodeRef::new("http://example.com")?, +/// ), &mut buf)?; +/// assert_eq!( +/// b" .\n", +/// buf.as_slice() +/// ); +/// # Result::<_,Box>::Ok(()) +/// ``` +pub struct LowLevelNQuadsWriter; + +impl LowLevelNQuadsWriter { + /// Writes an extra quad. + #[allow(clippy::unused_self)] + pub fn write_quad<'a>( + &mut self, + q: impl Into>, + mut write: impl Write, + ) -> io::Result<()> { + writeln!(write, "{} .", q.into()) + } +} diff --git a/lib/oxttl/src/ntriples.rs b/lib/oxttl/src/ntriples.rs new file mode 100644 index 00000000..fb6692fc --- /dev/null +++ b/lib/oxttl/src/ntriples.rs @@ -0,0 +1,389 @@ +//! A [N-Triples](https://www.w3.org/TR/n-triples/) streaming parser implemented by [`NTriplesParser`] +//! and a serializer implemented by [`NTriplesSerializer`]. + +use crate::line_formats::NQuadsRecognizer; +use crate::toolkit::{FromReadIterator, ParseError, ParseOrIoError, Parser}; +use oxrdf::{Triple, TripleRef}; +use std::io::{self, Read, Write}; + +/// A [N-Triples](https://www.w3.org/TR/n-triples/) streaming parser. +/// +/// Support for [N-Triples-star](https://w3c.github.io/rdf-star/cg-spec/2021-12-17.html#n-triples-star) is available behind the `rdf-star` feature and the [`NTriplesParser::with_quoted_triples`] option. +/// +/// Count the number of people: +/// ``` +/// use oxrdf::NamedNodeRef; +/// use oxttl::{NTriplesParser, ParseError}; +/// +/// let file = b" . +/// \"Foo\" . +/// . +/// \"Bar\" ."; +/// +/// let rdf_type = NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?; +/// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; +/// let mut count = 0; +/// for triple in NTriplesParser::new().parse_from_read(file.as_ref()) { +/// let triple = triple?; +/// if triple.predicate == rdf_type && triple.object == schema_person.into() { +/// count += 1; +/// } +/// } +/// assert_eq!(2, count); +/// # Result::<_,Box>::Ok(()) +/// ``` +#[derive(Default)] +pub struct NTriplesParser { + #[cfg(feature = "rdf-star")] + with_quoted_triples: bool, +} + +impl NTriplesParser { + /// Builds a new [`NTriplesParser`]. + #[inline] + pub fn new() -> Self { + Self::default() + } + + /// Enables [N-Triples-star](https://w3c.github.io/rdf-star/cg-spec/2021-12-17.html#n-triples-star). + #[cfg(feature = "rdf-star")] + #[inline] + #[must_use] + pub fn with_quoted_triples(mut self) -> Self { + self.with_quoted_triples = true; + self + } + + /// Parses a N-Triples file from a [`Read`] implementation. + /// + /// Count the number of people: + /// ``` + /// use oxrdf::NamedNodeRef; + /// use oxttl::{NTriplesParser, ParseError}; + /// + /// let file = b" . + /// \"Foo\" . + /// . + /// \"Bar\" ."; + /// + /// let rdf_type = NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?; + /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; + /// let mut count = 0; + /// for triple in NTriplesParser::new().parse_from_read(file.as_ref()) { + /// let triple = triple?; + /// if triple.predicate == rdf_type && triple.object == schema_person.into() { + /// count += 1; + /// } + /// } + /// assert_eq!(2, count); + /// # Result::<_,Box>::Ok(()) + /// ``` + pub fn parse_from_read(&self, read: R) -> FromReadNTriplesReader { + FromReadNTriplesReader { + inner: self.parse().parser.parse_from_read(read), + } + } + + /// Allows to parse a N-Triples file by using a low-level API. + /// + /// Count the number of people: + /// ``` + /// use oxrdf::NamedNodeRef; + /// use oxttl::{NTriplesParser, ParseError}; + /// + /// let file: [&[u8]; 4] = [ + /// b" .\n", + /// b" \"Foo\" .\n", + /// b" .\n", + /// b" \"Bar\" .\n" + /// ]; + /// + /// let rdf_type = NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?; + /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; + /// let mut count = 0; + /// let mut parser = NTriplesParser::new().parse(); + /// let mut file_chunks = file.iter(); + /// while !parser.is_end() { + /// // We feed more data to the parser + /// if let Some(chunk) = file_chunks.next() { + /// parser.extend_from_slice(chunk); + /// } else { + /// parser.end(); // It's finished + /// } + /// // We read as many triples from the parser as possible + /// while let Some(triple) = parser.read_next() { + /// let triple = triple?; + /// if triple.predicate == rdf_type && triple.object == schema_person.into() { + /// count += 1; + /// } + /// } + /// } + /// assert_eq!(2, count); + /// # Result::<_,Box>::Ok(()) + /// ``` + #[allow(clippy::unused_self)] + pub fn parse(&self) -> LowLevelNTriplesReader { + LowLevelNTriplesReader { + parser: NQuadsRecognizer::new_parser( + false, + #[cfg(feature = "rdf-star")] + self.with_quoted_triples, + ), + } + } +} + +/// Parses a N-Triples file from a [`Read`] implementation. Can be built using [`NTriplesParser::parse_from_read`]. +/// +/// Count the number of people: +/// ``` +/// use oxrdf::NamedNodeRef; +/// use oxttl::{NTriplesParser, ParseError}; +/// +/// let file = b" . +/// \"Foo\" . +/// . +/// \"Bar\" ."; +/// +/// let rdf_type = NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?; +/// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; +/// let mut count = 0; +/// for triple in NTriplesParser::new().parse_from_read(file.as_ref()) { +/// let triple = triple?; +/// if triple.predicate == rdf_type && triple.object == schema_person.into() { +/// count += 1; +/// } +/// } +/// assert_eq!(2, count); +/// # Result::<_,Box>::Ok(()) +/// ``` +pub struct FromReadNTriplesReader { + inner: FromReadIterator, +} + +impl Iterator for FromReadNTriplesReader { + type Item = Result; + + fn next(&mut self) -> Option> { + Some(self.inner.next()?.map(Into::into)) + } +} + +/// Parses a N-Triples file by using a low-level API. Can be built using [`NTriplesParser::parse`]. +/// +/// Count the number of people: +/// ``` +/// use oxrdf::NamedNodeRef; +/// use oxttl::{NTriplesParser, ParseError}; +/// +/// let file: [&[u8]; 4] = [ +/// b" .\n", +/// b" \"Foo\" .\n", +/// b" .\n", +/// b" \"Bar\" .\n" +/// ]; +/// +/// let rdf_type = NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?; +/// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; +/// let mut count = 0; +/// let mut parser = NTriplesParser::new().parse(); +/// let mut file_chunks = file.iter(); +/// while !parser.is_end() { +/// // We feed more data to the parser +/// if let Some(chunk) = file_chunks.next() { +/// parser.extend_from_slice(chunk); +/// } else { +/// parser.end(); // It's finished +/// } +/// // We read as many triples from the parser as possible +/// while let Some(triple) = parser.read_next() { +/// let triple = triple?; +/// if triple.predicate == rdf_type && triple.object == schema_person.into() { +/// count += 1; +/// } +/// } +/// } +/// assert_eq!(2, count); +/// # Result::<_,Box>::Ok(()) +/// ``` +pub struct LowLevelNTriplesReader { + parser: Parser, +} + +impl LowLevelNTriplesReader { + /// Adds some extra bytes to the parser. Should be called when [`read_next`](Self::read_next) returns [`None`] and there is still unread data. + pub fn extend_from_slice(&mut self, other: &[u8]) { + self.parser.extend_from_slice(other) + } + + /// Tell the parser that the file is finished. + /// + /// This triggers the parsing of the final bytes and might lead [`read_next`](Self::read_next) to return some extra values. + pub fn end(&mut self) { + self.parser.end() + } + + /// Returns if the parsing is finished i.e. [`end`](Self::end) has been called and [`read_next`](Self::read_next) is always going to return `None`. + pub fn is_end(&self) -> bool { + self.parser.is_end() + } + + /// Attempt to parse a new triple from the already provided data. + /// + /// Returns [`None`] if the parsing is finished or more data is required. + /// If it is the case more data should be fed using [`extend_from_slice`](Self::extend_from_slice). + pub fn read_next(&mut self) -> Option> { + Some(self.parser.read_next()?.map(Into::into)) + } +} + +/// A [N-Triples](https://www.w3.org/TR/n-triples/) serializer. +/// +/// Support for [N-Triples-star](https://w3c.github.io/rdf-star/cg-spec/2021-12-17.html#n-triples-star) is available behind the `rdf-star` feature. +/// +/// ``` +/// use oxrdf::{NamedNodeRef, TripleRef}; +/// use oxttl::NTriplesSerializer; +/// +/// let mut buf = Vec::new(); +/// let mut writer = NTriplesSerializer::new().serialize_to_write(buf); +/// writer.write_triple(TripleRef::new( +/// NamedNodeRef::new("http://example.com#me")?, +/// NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?, +/// NamedNodeRef::new("http://schema.org/Person")?, +/// ))?; +/// assert_eq!( +/// b" .\n", +/// writer.finish().as_slice() +/// ); +/// # Result::<_,Box>::Ok(()) +/// ``` +#[derive(Default)] +pub struct NTriplesSerializer; + +impl NTriplesSerializer { + /// Builds a new [`NTriplesSerializer`]. + #[inline] + pub fn new() -> Self { + Self + } + + /// Writes a N-Triples file to a [`Write`] implementation. + /// + /// ``` + /// use oxrdf::{NamedNodeRef, TripleRef}; + /// use oxttl::NTriplesSerializer; + /// + /// let mut buf = Vec::new(); + /// let mut writer = NTriplesSerializer::new().serialize_to_write(buf); + /// writer.write_triple(TripleRef::new( + /// NamedNodeRef::new("http://example.com#me")?, + /// NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?, + /// NamedNodeRef::new("http://schema.org/Person")?, + /// ))?; + /// assert_eq!( + /// b" .\n", + /// writer.finish().as_slice() + /// ); + /// # Result::<_,Box>::Ok(()) + /// ``` + pub fn serialize_to_write(&self, write: W) -> ToWriteNTriplesWriter { + ToWriteNTriplesWriter { + write, + writer: self.serialize(), + } + } + + /// Builds a low-level N-Triples writer. + /// + /// ``` + /// use oxrdf::{NamedNodeRef, TripleRef}; + /// use oxttl::NTriplesSerializer; + /// + /// let mut buf = Vec::new(); + /// let mut writer = NTriplesSerializer::new().serialize(); + /// writer.write_triple(TripleRef::new( + /// NamedNodeRef::new("http://example.com#me")?, + /// NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?, + /// NamedNodeRef::new("http://schema.org/Person")?, + /// ), &mut buf)?; + /// assert_eq!( + /// b" .\n", + /// buf.as_slice() + /// ); + /// # Result::<_,Box>::Ok(()) + /// ``` + #[allow(clippy::unused_self)] + pub fn serialize(&self) -> LowLevelNTriplesWriter { + LowLevelNTriplesWriter + } +} + +/// Writes a N-Triples file to a [`Write`] implementation. Can be built using [`NTriplesSerializer::serialize_to_write`]. +/// +/// ``` +/// use oxrdf::{NamedNodeRef, TripleRef}; +/// use oxttl::NTriplesSerializer; +/// +/// let mut buf = Vec::new(); +/// let mut writer = NTriplesSerializer::new().serialize_to_write(buf); +/// writer.write_triple(TripleRef::new( +/// NamedNodeRef::new("http://example.com#me")?, +/// NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?, +/// NamedNodeRef::new("http://schema.org/Person")?, +/// ))?; +/// assert_eq!( +/// b" .\n", +/// writer.finish().as_slice() +/// ); +/// # Result::<_,Box>::Ok(()) +/// ``` +pub struct ToWriteNTriplesWriter { + write: W, + writer: LowLevelNTriplesWriter, +} + +impl ToWriteNTriplesWriter { + /// Writes an extra triple. + pub fn write_triple<'a>(&mut self, t: impl Into>) -> io::Result<()> { + self.writer.write_triple(t, &mut self.write) + } + + /// Ends the write process and returns the underlying [`Write`]. + pub fn finish(self) -> W { + self.write + } +} + +/// Writes a N-Triples file by using a low-level API. Can be built using [`NTriplesSerializer::serialize`]. +/// +/// ``` +/// use oxrdf::{NamedNodeRef, TripleRef}; +/// use oxttl::NTriplesSerializer; +/// +/// let mut buf = Vec::new(); +/// let mut writer = NTriplesSerializer::new().serialize(); +/// writer.write_triple(TripleRef::new( +/// NamedNodeRef::new("http://example.com#me")?, +/// NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?, +/// NamedNodeRef::new("http://schema.org/Person")?, +/// ), &mut buf)?; +/// assert_eq!( +/// b" .\n", +/// buf.as_slice() +/// ); +/// # Result::<_,Box>::Ok(()) +/// ``` +pub struct LowLevelNTriplesWriter; + +impl LowLevelNTriplesWriter { + /// Writes an extra triple. + #[allow(clippy::unused_self)] + pub fn write_triple<'a>( + &mut self, + t: impl Into>, + mut write: impl Write, + ) -> io::Result<()> { + writeln!(write, "{} .", t.into()) + } +} diff --git a/lib/oxttl/src/terse.rs b/lib/oxttl/src/terse.rs new file mode 100644 index 00000000..151ee062 --- /dev/null +++ b/lib/oxttl/src/terse.rs @@ -0,0 +1,932 @@ +//! Shared parser implementation for Turtle and TriG. + +use crate::lexer::{resolve_local_name, N3Lexer, N3LexerMode, N3LexerOptions, N3Token}; +use crate::toolkit::{Lexer, Parser, RuleRecognizer, RuleRecognizerError}; +use crate::{MAX_BUFFER_SIZE, MIN_BUFFER_SIZE}; +use oxiri::Iri; +#[cfg(feature = "rdf-star")] +use oxrdf::Triple; +use oxrdf::{ + vocab::{rdf, xsd}, + BlankNode, GraphName, Literal, NamedNode, NamedOrBlankNode, Quad, Subject, Term, +}; +use std::collections::HashMap; + +pub struct TriGRecognizer { + stack: Vec, + with_graph_name: bool, + #[cfg(feature = "rdf-star")] + with_quoted_triples: bool, + lexer_options: N3LexerOptions, + prefixes: HashMap>, + cur_subject: Vec, + cur_predicate: Vec, + cur_object: Vec, + cur_graph: GraphName, +} + +impl RuleRecognizer for TriGRecognizer { + type TokenRecognizer = N3Lexer; + type Output = Quad; + + fn error_recovery_state(mut self) -> Self { + self.stack.clear(); + self.cur_subject.clear(); + self.cur_predicate.clear(); + self.cur_object.clear(); + self.cur_graph = GraphName::DefaultGraph; + self + } + + fn recognize_next( + mut self, + token: N3Token, + results: &mut Vec, + errors: &mut Vec, + ) -> Self { + if let Some(rule) = self.stack.pop() { + match rule { + // [1g] trigDoc ::= (directive | block)* + // [2g] block ::= triplesOrGraph | wrappedGraph | triples2 | "GRAPH" labelOrSubject wrappedGraph + // [3] directive ::= prefixID | base | sparqlPrefix | sparqlBase + // [4] prefixID ::= '@prefix' PNAME_NS IRIREF '.' + // [5] base ::= '@base' IRIREF '.' + // [5s] sparqlPrefix ::= "PREFIX" PNAME_NS IRIREF + // [6s] sparqlBase ::= "BASE" IRIREF + TriGState::TriGDoc => { + self.cur_graph = GraphName::DefaultGraph; + self.stack.push(TriGState::TriGDoc); + match token { + N3Token::PlainKeyword(k) if k.eq_ignore_ascii_case("base") => { + self.stack.push(TriGState::BaseExpectIri); + self + } + N3Token::PlainKeyword(k) if k.eq_ignore_ascii_case("prefix") => { + self.stack.push(TriGState::PrefixExpectPrefix); + self + } + N3Token::LangTag("prefix") => { + self.stack.push(TriGState::ExpectDot); + self.stack.push(TriGState::PrefixExpectPrefix); + self + } + N3Token::LangTag("base") => { + self.stack.push(TriGState::ExpectDot); + self.stack.push(TriGState::BaseExpectIri); + self + } + N3Token::PlainKeyword(k) if k.eq_ignore_ascii_case("graph") && self.with_graph_name => { + self.stack.push(TriGState::WrappedGraph); + self.stack.push(TriGState::GraphName); + self + } + token @ N3Token::Punctuation("{") if self.with_graph_name => { + self.stack.push(TriGState::WrappedGraph); + self.recognize_next(token, results, errors) + } + token => { + self.stack.push(TriGState::TriplesOrGraph); + self.recognize_next(token, results, errors) + } + } + }, + TriGState::ExpectDot => { + self.cur_subject.pop(); + if token == N3Token::Punctuation(".") { + self + } else { + errors.push("A dot is expected at the end of statements".into()); + self.recognize_next(token, results, errors) + } + }, + TriGState::BaseExpectIri => match token { + N3Token::IriRef(iri) => { + self.lexer_options.base_iri = Some(iri); + self + } + _ => self.error(errors, "The BASE keyword should be followed by an IRI"), + }, + TriGState::PrefixExpectPrefix => match token { + N3Token::PrefixedName { prefix, local, .. } if local.is_empty() => { + self.stack.push(TriGState::PrefixExpectIri { name: prefix.to_owned() }); + self + } + _ => { + self.error(errors, "The PREFIX keyword should be followed by a prefix like 'ex:'") + } + }, + TriGState::PrefixExpectIri { name } => match token { + N3Token::IriRef(iri) => { + self.prefixes.insert(name, iri); + self + } + _ => self.error(errors, "The PREFIX declaration should be followed by a prefix and its value as an IRI"), + }, + // [3g] triplesOrGraph ::= labelOrSubject ( wrappedGraph | predicateObjectList '.' ) | quotedTriple predicateObjectList '.' + // [4g] triples2 ::= blankNodePropertyList predicateObjectList? '.' | collection predicateObjectList '.' + TriGState::TriplesOrGraph => match token { + N3Token::IriRef(iri) => { + self.stack.push(TriGState::WrappedGraphOrPredicateObjectList { + term: NamedNode::new_unchecked(iri.into_inner()).into() + }); + self + } + N3Token::PrefixedName { prefix, local, might_be_invalid_iri } => match resolve_local_name(prefix, &local, might_be_invalid_iri, &self.prefixes) { + Ok(t) => { + self.stack.push(TriGState::WrappedGraphOrPredicateObjectList { + term: t.into() + }); + self + }, + Err(e) => self.error(errors, e) + } + N3Token::BlankNodeLabel(label) => { + self.stack.push(TriGState::WrappedGraphOrPredicateObjectList { + term: BlankNode::new_unchecked(label).into() + }); + self + } + N3Token::Punctuation("[") => { + self.stack.push(TriGState::WrappedGraphBlankNodePropertyListCurrent); + self + } + N3Token::Punctuation("(") => { + self.stack.push(TriGState::ExpectDot); + self.stack.push(TriGState::PredicateObjectList); + self.stack.push(TriGState::SubjectCollectionBeginning); + self + } + #[cfg(feature = "rdf-star")] + N3Token::Punctuation("<<") if self.with_quoted_triples => { + self.stack.push(TriGState::ExpectDot); + self.stack.push(TriGState::PredicateObjectList); + self.stack.push(TriGState::SubjectQuotedTripleEnd); + self.stack.push(TriGState::QuotedObject); + self.stack.push(TriGState::Verb); + self.stack.push(TriGState::QuotedSubject); + self + } + token => { + self.error(errors, format!("The token {token:?} is not a valid subject or graph name")) + } + } + TriGState::WrappedGraphOrPredicateObjectList { term } => { + if token == N3Token::Punctuation("{") && self.with_graph_name { + self.cur_graph = term.into(); + self.stack.push(TriGState::WrappedGraph); + } else { + self.cur_subject.push(term.into()); + self.stack.push(TriGState::ExpectDot); + self.stack.push(TriGState::PredicateObjectList); + } + self.recognize_next(token, results, errors) + } + TriGState::WrappedGraphBlankNodePropertyListCurrent => if token == N3Token::Punctuation("]") { + self.stack.push(TriGState::WrappedGraphOrPredicateObjectList { + term: BlankNode::default().into() + }); + self + } else { + self.cur_subject.push(BlankNode::default().into()); + self.stack.push(TriGState::ExpectDot); + self.stack.push(TriGState::SubjectBlankNodePropertyListEnd); + self.stack.push(TriGState::PredicateObjectList); + self.recognize_next(token, results, errors) + } + TriGState::SubjectBlankNodePropertyListEnd => if token == N3Token::Punctuation("]") { + self.stack.push(TriGState::SubjectBlankNodePropertyListAfter ); + self + } else { + errors.push("blank node property lists should end with a ']'".into()); + self.stack.push(TriGState::SubjectBlankNodePropertyListAfter ); + self.recognize_next(token, results, errors) + } + TriGState::SubjectBlankNodePropertyListAfter => if matches!(token, N3Token::Punctuation("." | "}")) { + self.recognize_next(token, results, errors) + } else { + self.stack.push(TriGState::PredicateObjectList); + self.recognize_next(token, results, errors) + } + TriGState::SubjectCollectionBeginning => { + match token { + N3Token::Punctuation(")") => { + self.cur_subject.push(rdf::NIL.into()); + self + } + token => { + let root = BlankNode::default(); + self.cur_subject.push(root.clone().into()); + self.cur_subject.push(root.into()); + self.cur_predicate.push(rdf::FIRST.into()); + self.stack.push(TriGState::SubjectCollectionPossibleEnd); + self.stack.push(TriGState::Object); + self.recognize_next(token, results, errors) + } + } + }, + TriGState::SubjectCollectionPossibleEnd => { + let old = self.cur_subject.pop().unwrap(); + self.cur_object.pop(); + match token { + N3Token::Punctuation(")") => { + self.cur_predicate.pop(); + results.push(Quad::new( + old, + rdf::REST, + rdf::NIL, + self.cur_graph.clone() + )); + self + } + token => { + let new = BlankNode::default(); + results.push(Quad::new( + old, + rdf::REST, + new.clone(), + self.cur_graph.clone() + )); + self.cur_subject.push(new.into()); + self.stack.push(TriGState::ObjectCollectionPossibleEnd); + self.stack.push(TriGState::Object); + self.recognize_next(token, results, errors) + } + } + } + // [5g] wrappedGraph ::= '{' triplesBlock? '}' + // [6g] triplesBlock ::= triples ('.' triplesBlock?)? + TriGState::WrappedGraph => if token == N3Token::Punctuation("{") { + self.stack.push(TriGState::WrappedGraphPossibleEnd); + self.stack.push(TriGState::Triples); + self + } else { + self.error(errors, "The GRAPH keyword should be followed by a graph name and a value in '{'") + }, + TriGState::WrappedGraphPossibleEnd => { + self.cur_subject.pop(); + match token { + N3Token::Punctuation("}") => { + self + } + N3Token::Punctuation(".") => { + self.stack.push(TriGState::WrappedGraphPossibleEnd); + self.stack.push(TriGState::Triples); + self + } + token => { + errors.push("A '}' or a '.' is expected at the end of a graph block".into()); + self.recognize_next(token, results, errors) + } + } + } + // [6] triples ::= subject predicateObjectList | blankNodePropertyList predicateObjectList? + // [10] subject ::= iri | BlankNode | collection | quotedTriple + TriGState::Triples => match token { + N3Token::Punctuation("}") => { + self.recognize_next(token, results, errors) // Early end + }, + N3Token::Punctuation("[") => { + self.cur_subject.push(BlankNode::default().into()); + self.stack.push(TriGState::TriplesBlankNodePropertyListCurrent); + self + } + N3Token::IriRef(iri) => { + self.cur_subject.push(NamedNode::new_unchecked(iri.into_inner()).into()); + self.stack.push(TriGState::PredicateObjectList); + self + } + N3Token::PrefixedName { prefix, local, might_be_invalid_iri } => match resolve_local_name(prefix, &local, might_be_invalid_iri, &self.prefixes) { + Ok(t) => { + self.cur_subject.push(t.into()); + self.stack.push(TriGState::PredicateObjectList); + self + }, + Err(e) => self.error(errors, e) + } + N3Token::BlankNodeLabel(label) => { + self.cur_subject.push(BlankNode::new_unchecked(label).into()); + self.stack.push(TriGState::PredicateObjectList); + self + } + N3Token::Punctuation("(") => { + self.stack.push(TriGState::PredicateObjectList); + self.stack.push(TriGState::SubjectCollectionBeginning); + self + } + #[cfg(feature = "rdf-star")] + N3Token::Punctuation("<<") if self.with_quoted_triples => { + self.stack.push(TriGState::PredicateObjectList); + self.stack.push(TriGState::SubjectQuotedTripleEnd); + self.stack.push(TriGState::QuotedObject); + self.stack.push(TriGState::Verb); + self.stack.push(TriGState::QuotedSubject); + self + } + token => { + self.error(errors, format!("The token {token:?} is not a valid RDF subject")) + } + }, + TriGState::TriplesBlankNodePropertyListCurrent => if token == N3Token::Punctuation("]") { + self.stack.push(TriGState::PredicateObjectList); + self + } else { + self.stack.push(TriGState::SubjectBlankNodePropertyListEnd); + self.stack.push(TriGState::PredicateObjectList); + self.recognize_next(token, results, errors) + } + // [7g] labelOrSubject ::= iri | BlankNode + TriGState::GraphName => match token { + N3Token::IriRef(iri) => { + self.cur_graph = NamedNode::new_unchecked(iri.into_inner()).into(); + self + } + N3Token::PrefixedName { prefix, local, might_be_invalid_iri } => match resolve_local_name(prefix, &local, might_be_invalid_iri, &self.prefixes) { + Ok(t) => { + self.cur_graph = t.into(); + self + }, + Err(e) => self.error(errors, e) + } + N3Token::BlankNodeLabel(label) => { + self.cur_graph = BlankNode::new_unchecked(label).into(); + self + } + N3Token::Punctuation("[") => { + self.stack.push(TriGState::GraphNameAnonEnd); + self + } + token => { + self.error(errors, format!("The token {token:?} is not a valid graph name")) + } + } + TriGState::GraphNameAnonEnd => if token == N3Token::Punctuation("]") { + self.cur_graph = BlankNode::default().into(); + self + } else { + self.error(errors, "Anonymous blank node with a property list are not allowed as graph name") + } + // [7] predicateObjectList ::= verb objectList (';' (verb objectList)?)* + TriGState::PredicateObjectList => { + self.stack.push(TriGState::PredicateObjectListEnd); + self.stack.push(TriGState::ObjectsList); + self.stack.push(TriGState::Verb); + self.recognize_next(token, results, errors) + }, + TriGState::PredicateObjectListEnd => { + self.cur_predicate.pop(); + if token == N3Token::Punctuation(";") { + self.stack.push(TriGState::PredicateObjectListPossibleContinuation); + self + } else { + self.recognize_next(token, results, errors) + } + }, + TriGState::PredicateObjectListPossibleContinuation => if token == N3Token::Punctuation(";") { + self.stack.push(TriGState::PredicateObjectListPossibleContinuation); + self + } else if matches!(token, N3Token::Punctuation("." | "}" | "]")) { + self.recognize_next(token, results, errors) + } else { + self.stack.push(TriGState::PredicateObjectListEnd); + self.stack.push(TriGState::ObjectsList); + self.stack.push(TriGState::Verb); + self.recognize_next(token, results, errors) + }, + // [8] objectList ::= object annotation? ( ',' object annotation? )* + // [30t] annotation ::= '{|' predicateObjectList '|}' + TriGState::ObjectsList => { + self.stack.push(TriGState::ObjectsListEnd); + self.stack.push(TriGState::Object); + self.recognize_next(token, results, errors) + } + TriGState::ObjectsListEnd => { + match token { + N3Token::Punctuation(",") => { + self.cur_object.pop(); + self.stack.push(TriGState::ObjectsListEnd); + self.stack.push(TriGState::Object); + self + }, + #[cfg(feature = "rdf-star")] + N3Token::Punctuation("{|") => { + let triple = Triple::new( + self.cur_subject.last().unwrap().clone(), + self.cur_predicate.last().unwrap().clone(), + self.cur_object.pop().unwrap() + ); + self.cur_subject.push(triple.into()); + self.stack.push(TriGState::AnnotationEnd); + self.stack.push(TriGState::PredicateObjectList); + self + } + token => { + self.cur_object.pop(); + self.recognize_next(token, results, errors) + } + } + }, + #[cfg(feature = "rdf-star")] + TriGState::AnnotationEnd => { + self.cur_subject.pop(); + self.stack.push(TriGState::ObjectsListAfterAnnotation); + if token == N3Token::Punctuation("|}") { + self + } else { + self.error(errors, "Annotations should end with '|}'") + } + }, + #[cfg(feature = "rdf-star")] + TriGState::ObjectsListAfterAnnotation => if token == N3Token::Punctuation(",") { + self.stack.push(TriGState::ObjectsListEnd); + self.stack.push(TriGState::Object); + self + } else { + self.recognize_next(token, results, errors) + }, + // [9] verb ::= predicate | 'a' + // [11] predicate ::= iri + TriGState::Verb => match token { + N3Token::PlainKeyword("a") => { + self.cur_predicate.push(rdf::TYPE.into()); + self + } + N3Token::IriRef(iri) => { + self.cur_predicate.push(NamedNode::new_unchecked(iri.into_inner())); + self + } + N3Token::PrefixedName { prefix, local, might_be_invalid_iri } => match resolve_local_name(prefix, &local, might_be_invalid_iri, &self.prefixes) { + Ok(t) => { + self.cur_predicate.push(t); + self + }, + Err(e) => self.error(errors, e) + } + token => { + self.error(errors, format!("The token {token:?} is not a valid predicate")) + } + } + // [12] object ::= iri | BlankNode | collection | blankNodePropertyList | literal | quotedTriple + // [13] literal ::= RDFLiteral | NumericLiteral | BooleanLiteral + // [14] blank ::= BlankNode | collection + // [15] blankNodePropertyList ::= '[' predicateObjectList ']' + // [16] collection ::= '(' object* ')' + // [17] NumericLiteral ::= INTEGER | DECIMAL | DOUBLE + // [128s] RDFLiteral ::= String (LANGTAG | '^^' iri)? + // [133s] BooleanLiteral ::= 'true' | 'false' + // [18] String ::= STRING_LITERAL_QUOTE | STRING_LITERAL_SINGLE_QUOTE | STRING_LITERAL_LONG_SINGLE_QUOTE | STRING_LITERAL_LONG_QUOTE + // [135s] iri ::= IRIREF | PrefixedName + // [136s] PrefixedName ::= PNAME_LN | PNAME_NS + // [137s] BlankNode ::= BLANK_NODE_LABEL | ANON + TriGState::Object => match token { + N3Token::IriRef(iri) => { + self.cur_object.push(NamedNode::new_unchecked(iri.into_inner()).into()); + self.emit_quad(results); + self + } + N3Token::PrefixedName { prefix, local, might_be_invalid_iri } => match resolve_local_name(prefix, &local, might_be_invalid_iri, &self.prefixes) { + Ok(t) => { + self.cur_object.push(t.into()); + self.emit_quad(results); + self + }, + Err(e) => self.error(errors, e) + } + N3Token::BlankNodeLabel(label) => { + self.cur_object.push(BlankNode::new_unchecked(label).into()); + self.emit_quad(results); + self + } + N3Token::Punctuation("[") => { + self.stack.push(TriGState::ObjectBlankNodePropertyListCurrent); + self + } + N3Token::Punctuation("(") => { + self.stack.push(TriGState::ObjectCollectionBeginning); + self + } + N3Token::String(value) => { + self.stack.push(TriGState::LiteralPossibleSuffix { value, emit: true }); + self + } + N3Token::Integer(v) => { + self.cur_object.push(Literal::new_typed_literal(v, xsd::INTEGER).into()); + self.emit_quad(results); + self + } + N3Token::Decimal(v) => { + self.cur_object.push(Literal::new_typed_literal(v, xsd::DECIMAL).into()); + self.emit_quad(results); + self + } + N3Token::Double(v) => { + self.cur_object.push(Literal::new_typed_literal(v, xsd::DOUBLE).into()); + self.emit_quad(results); + self + } + N3Token::PlainKeyword("true") => { + self.cur_object.push(Literal::new_typed_literal("true", xsd::BOOLEAN).into()); + self.emit_quad(results); + self + } + N3Token::PlainKeyword("false") => { + self.cur_object.push(Literal::new_typed_literal("false", xsd::BOOLEAN).into()); + self.emit_quad(results); + self + } + #[cfg(feature = "rdf-star")] + N3Token::Punctuation("<<") if self.with_quoted_triples => { + self.stack.push(TriGState::ObjectQuotedTripleEnd { emit: true }); + self.stack.push(TriGState::QuotedObject); + self.stack.push(TriGState::Verb); + self.stack.push(TriGState::QuotedSubject); + self + } + token => { + self.error(errors, format!("This is not a valid RDF object: {token:?}")) + } + + } + TriGState::ObjectBlankNodePropertyListCurrent => if token == N3Token::Punctuation("]") { + self.cur_object.push(BlankNode::default().into()); + self.emit_quad(results); + self + } else { + self.cur_subject.push(BlankNode::default().into()); + self.stack.push(TriGState::ObjectBlankNodePropertyListEnd); + self.stack.push(TriGState::PredicateObjectList); + self.recognize_next(token, results, errors) + } + TriGState::ObjectBlankNodePropertyListEnd => if token == N3Token::Punctuation("]") { + self.cur_object.push(self.cur_subject.pop().unwrap().into()); + self.emit_quad(results); + self + } else { + self.error(errors, "blank node property lists should end with a ']'") + } + TriGState::ObjectCollectionBeginning => match token { + N3Token::Punctuation(")") => { + self.cur_object.push(rdf::NIL.into()); + self.emit_quad(results); + self + } + token => { + let root = BlankNode::default(); + self.cur_object.push(root.clone().into()); + self.emit_quad(results); + self.cur_subject.push(root.into()); + self.cur_predicate.push(rdf::FIRST.into()); + self.stack.push(TriGState::ObjectCollectionPossibleEnd); + self.stack.push(TriGState::Object); + self.recognize_next(token, results, errors) + } + }, + TriGState::ObjectCollectionPossibleEnd => { + let old = self.cur_subject.pop().unwrap(); + self.cur_object.pop(); + match token { + N3Token::Punctuation(")") => { + self.cur_predicate.pop(); + results.push(Quad::new(old, + rdf::REST, + rdf::NIL, + self.cur_graph.clone() + )); + self + } + token => { + let new = BlankNode::default(); + results.push(Quad::new(old, + rdf::REST, + new.clone(), + self.cur_graph.clone() + )); + self.cur_subject.push(new.into()); + self.stack.push(TriGState::ObjectCollectionPossibleEnd); + self.stack.push(TriGState::Object); + self.recognize_next(token, results, errors) + } + } + } + TriGState::LiteralPossibleSuffix { value, emit } => { + match token { + N3Token::LangTag(lang) => { + self.cur_object.push(Literal::new_language_tagged_literal_unchecked(value, lang.to_ascii_lowercase()).into()); + if emit { + self.emit_quad(results); + } + self + }, + N3Token::Punctuation("^^") => { + self.stack.push(TriGState::LiteralExpectDatatype { value, emit }); + self + } + token => { + self.cur_object.push(Literal::new_simple_literal(value).into()); + if emit { + self.emit_quad(results); + } + self.recognize_next(token, results, errors) + } + } + } + TriGState::LiteralExpectDatatype { value, emit } => { + match token { + N3Token::IriRef(datatype) => { + self.cur_object.push(Literal::new_typed_literal(value, NamedNode::new_unchecked(datatype.into_inner())).into()); + if emit { + self.emit_quad(results); + } + self + }, + N3Token::PrefixedName { prefix, local, might_be_invalid_iri } => match resolve_local_name(prefix, &local, might_be_invalid_iri, &self.prefixes) { + Ok(t) => { + self.cur_object.push(Literal::new_typed_literal(value, t).into()); + if emit { + self.emit_quad(results); + } + self + }, + Err(e) => self.error(errors, e) + } + token => { + self.error(errors, format!("Expecting a datatype IRI after '^^, found {token:?}")).recognize_next(token, results, errors) + } + } + } + // [27t] quotedTriple ::= '<<' qtSubject verb qtObject '>>' + #[cfg(feature = "rdf-star")] + TriGState::SubjectQuotedTripleEnd => { + let triple = Triple::new( + self.cur_subject.pop().unwrap(), + self.cur_predicate.pop().unwrap(), + self.cur_object.pop().unwrap() + ); + self.cur_subject.push(triple.into()); + if token == N3Token::Punctuation(">>") { + self + } else { + self.error(errors, format!("Expecting '>>' to close a quoted triple, found {token:?}")) + } + } + #[cfg(feature = "rdf-star")] + TriGState::ObjectQuotedTripleEnd { emit } => { + let triple = Triple::new( + self.cur_subject.pop().unwrap(), + self.cur_predicate.pop().unwrap(), + self.cur_object.pop().unwrap() + ); + self.cur_object.push(triple.into()); + if emit { + self.emit_quad(results); + } + if token == N3Token::Punctuation(">>") { + self + } else { + self.error(errors, format!("Expecting '>>' to close a quoted triple, found {token:?}")) + } + } + // [28t] qtSubject ::= iri | BlankNode | quotedTriple + #[cfg(feature = "rdf-star")] + TriGState::QuotedSubject => match token { + N3Token::Punctuation("[") => { + self.cur_subject.push(BlankNode::default().into()); + self.stack.push(TriGState::QuotedAnonEnd); + self + } + N3Token::IriRef(iri) => { + self.cur_subject.push(NamedNode::new_unchecked(iri.into_inner()).into()); + self + } + N3Token::PrefixedName { prefix, local, might_be_invalid_iri } => match resolve_local_name(prefix, &local, might_be_invalid_iri, &self.prefixes) { + Ok(t) => { + self.cur_subject.push(t.into()); + self + }, + Err(e) => self.error(errors, e) + } + N3Token::BlankNodeLabel(label) => { + self.cur_subject.push(BlankNode::new_unchecked(label).into()); + self + } + N3Token::Punctuation("<<") => { + self.stack.push(TriGState::SubjectQuotedTripleEnd); + self.stack.push(TriGState::QuotedObject); + self.stack.push(TriGState::Verb); + self.stack.push(TriGState::QuotedSubject); + self + } + token => self.error(errors, format!("This is not a valid RDF quoted triple subject: {token:?}")) + } + // [29t] qtObject ::= iri | BlankNode | literal | quotedTriple + #[cfg(feature = "rdf-star")] + TriGState::QuotedObject => match token { + N3Token::Punctuation("[") => { + self.cur_object.push(BlankNode::default().into()); + self.stack.push(TriGState::QuotedAnonEnd); + self + } + N3Token::IriRef(iri) => { + self.cur_object.push(NamedNode::new_unchecked(iri.into_inner()).into()); + self + } + N3Token::PrefixedName { prefix, local, might_be_invalid_iri } => match resolve_local_name(prefix, &local, might_be_invalid_iri, &self.prefixes) { + Ok(t) => { + self.cur_object.push(t.into()); + self + }, + Err(e) => self.error(errors, e) + } + N3Token::BlankNodeLabel(label) => { + self.cur_object.push(BlankNode::new_unchecked(label).into()); + self + } + N3Token::String(value) => { + self.stack.push(TriGState::LiteralPossibleSuffix { value, emit: false }); + self + } + N3Token::Integer(v) => { + self.cur_object.push(Literal::new_typed_literal(v, xsd::INTEGER).into()); + self + } + N3Token::Decimal(v) => { + self.cur_object.push(Literal::new_typed_literal(v, xsd::DECIMAL).into()); + self + } + N3Token::Double(v) => { + self.cur_object.push(Literal::new_typed_literal(v, xsd::DOUBLE).into()); + self + } + N3Token::PlainKeyword("true") => { + self.cur_object.push(Literal::new_typed_literal("true", xsd::BOOLEAN).into()); + self + } + N3Token::PlainKeyword("false") => { + self.cur_object.push(Literal::new_typed_literal("false", xsd::BOOLEAN).into()); + self + } + N3Token::Punctuation("<<") => { + self.stack.push(TriGState::ObjectQuotedTripleEnd { emit: false }); + self.stack.push(TriGState::QuotedObject); + self.stack.push(TriGState::Verb); + self.stack.push(TriGState::QuotedSubject); + self + } + token => self.error(errors, format!("This is not a valid RDF quoted triple object: {token:?}")) + } + #[cfg(feature = "rdf-star")] + TriGState::QuotedAnonEnd => if token == N3Token::Punctuation("]") { + self + } else { + self.error(errors, "Anonymous blank node with a property list are not allowed in quoted triples") + } + } + } else if token == N3Token::Punctuation(".") || token == N3Token::Punctuation("}") { + //TODO: be smarter depending if we are in '{' or not + self.stack.push(TriGState::TriGDoc); + self + } else { + self + } + } + + fn recognize_end( + mut self, + results: &mut Vec, + errors: &mut Vec, + ) { + match &*self.stack { + [] | [TriGState::TriGDoc] => { + debug_assert!(self.cur_subject.is_empty()); + debug_assert!(self.cur_predicate.is_empty()); + debug_assert!(self.cur_object.is_empty()); + } + [.., TriGState::LiteralPossibleSuffix { value, emit: true }] => { + self.cur_object + .push(Literal::new_simple_literal(value).into()); + self.emit_quad(results); + errors.push("Triples should be followed by a dot".into()) + } + _ => errors.push("Unexpected end".into()), //TODO + } + } + + fn lexer_options(&self) -> &N3LexerOptions { + &self.lexer_options + } +} + +impl TriGRecognizer { + pub fn new_parser( + with_graph_name: bool, + #[cfg(feature = "rdf-star")] with_quoted_triples: bool, + base_iri: Option>, + prefixes: HashMap>, + ) -> Parser { + Parser::new( + Lexer::new( + N3Lexer::new(N3LexerMode::Turtle), + MIN_BUFFER_SIZE, + MAX_BUFFER_SIZE, + true, + Some(b"#"), + ), + TriGRecognizer { + stack: vec![TriGState::TriGDoc], + with_graph_name, + #[cfg(feature = "rdf-star")] + with_quoted_triples, + lexer_options: N3LexerOptions { base_iri }, + prefixes, + cur_subject: Vec::new(), + cur_predicate: Vec::new(), + cur_object: Vec::new(), + cur_graph: GraphName::DefaultGraph, + }, + ) + } + + #[must_use] + fn error( + mut self, + errors: &mut Vec, + msg: impl Into, + ) -> Self { + errors.push(msg.into()); + self.stack.clear(); + self.cur_subject.clear(); + self.cur_predicate.clear(); + self.cur_object.clear(); + self.cur_graph = GraphName::DefaultGraph; + self + } + + fn emit_quad(&mut self, results: &mut Vec) { + results.push(Quad::new( + self.cur_subject.last().unwrap().clone(), + self.cur_predicate.last().unwrap().clone(), + self.cur_object.last().unwrap().clone(), + self.cur_graph.clone(), + )); + } +} + +#[derive(Debug)] +enum TriGState { + TriGDoc, + ExpectDot, + BaseExpectIri, + PrefixExpectPrefix, + PrefixExpectIri { + name: String, + }, + TriplesOrGraph, + WrappedGraphBlankNodePropertyListCurrent, + SubjectBlankNodePropertyListEnd, + SubjectBlankNodePropertyListAfter, + SubjectCollectionBeginning, + SubjectCollectionPossibleEnd, + WrappedGraphOrPredicateObjectList { + term: NamedOrBlankNode, + }, + WrappedGraph, + WrappedGraphPossibleEnd, + GraphName, + GraphNameAnonEnd, + Triples, + TriplesBlankNodePropertyListCurrent, + PredicateObjectList, + PredicateObjectListEnd, + PredicateObjectListPossibleContinuation, + ObjectsList, + ObjectsListEnd, + #[cfg(feature = "rdf-star")] + AnnotationEnd, + #[cfg(feature = "rdf-star")] + ObjectsListAfterAnnotation, + Verb, + Object, + ObjectBlankNodePropertyListCurrent, + ObjectBlankNodePropertyListEnd, + ObjectCollectionBeginning, + ObjectCollectionPossibleEnd, + LiteralPossibleSuffix { + value: String, + emit: bool, + }, + LiteralExpectDatatype { + value: String, + emit: bool, + }, + #[cfg(feature = "rdf-star")] + SubjectQuotedTripleEnd, + #[cfg(feature = "rdf-star")] + ObjectQuotedTripleEnd { + emit: bool, + }, + #[cfg(feature = "rdf-star")] + QuotedSubject, + #[cfg(feature = "rdf-star")] + QuotedObject, + #[cfg(feature = "rdf-star")] + QuotedAnonEnd, +} diff --git a/lib/oxttl/src/toolkit/lexer.rs b/lib/oxttl/src/toolkit/lexer.rs new file mode 100644 index 00000000..26906386 --- /dev/null +++ b/lib/oxttl/src/toolkit/lexer.rs @@ -0,0 +1,280 @@ +use memchr::memchr2; +use std::error::Error; +use std::fmt; +use std::io::{self, Read}; +use std::ops::{Range, RangeInclusive}; + +pub trait TokenRecognizer { + type Token<'a> + where + Self: 'a; + type Options: Default; + + fn recognize_next_token<'a>( + &mut self, + data: &'a [u8], + is_ending: bool, + config: &Self::Options, + ) -> Option<(usize, Result, TokenRecognizerError>)>; +} + +pub struct TokenRecognizerError { + pub position: Range, + pub message: String, +} + +impl> From<(Range, S)> for TokenRecognizerError { + fn from((position, message): (Range, S)) -> Self { + Self { + position, + message: message.into(), + } + } +} + +#[allow(clippy::range_plus_one)] +impl> From<(RangeInclusive, S)> for TokenRecognizerError { + fn from((position, message): (RangeInclusive, S)) -> Self { + (*position.start()..*position.end() + 1, message).into() + } +} + +impl> From<(usize, S)> for TokenRecognizerError { + fn from((position, message): (usize, S)) -> Self { + (position..=position, message).into() + } +} + +pub struct TokenWithPosition { + pub token: T, + pub position: Range, +} + +pub struct Lexer { + parser: R, + data: Vec, + start: usize, + end: usize, + is_ending: bool, + position: usize, + min_buffer_size: usize, + max_buffer_size: usize, + is_line_jump_whitespace: bool, + line_comment_start: Option<&'static [u8]>, +} + +impl Lexer { + pub fn new( + parser: R, + min_buffer_size: usize, + max_buffer_size: usize, + is_line_jump_whitespace: bool, + line_comment_start: Option<&'static [u8]>, + ) -> Self { + Self { + parser, + data: Vec::new(), + start: 0, + end: 0, + is_ending: false, + position: 0, + min_buffer_size, + max_buffer_size, + is_line_jump_whitespace, + line_comment_start, + } + } + + pub fn extend_from_slice(&mut self, other: &[u8]) { + self.shrink_if_useful(); + self.data.truncate(self.end); + self.data.extend_from_slice(other); + self.end = self.data.len(); + } + + pub fn end(&mut self) { + self.is_ending = true; + } + + pub fn extend_from_read(&mut self, read: &mut impl Read) -> io::Result<()> { + self.shrink_if_useful(); + let min_end = self.end + self.min_buffer_size; + if min_end > self.max_buffer_size { + return Err(io::Error::new( + io::ErrorKind::OutOfMemory, + format!( + "The buffer maximal size is {} < {min_end}", + self.max_buffer_size + ), + )); + } + if self.data.len() < min_end { + self.data.resize(min_end, 0); + } + if self.data.len() < self.data.capacity() { + // We keep extending to have as much space as available without reallocation + self.data.resize(self.data.capacity(), 0); + } + let read = read.read(&mut self.data[self.end..])?; + self.end += read; + self.is_ending = read == 0; + Ok(()) + } + + pub fn read_next( + &mut self, + options: &R::Options, + ) -> Option>, LexerError>> { + self.skip_whitespaces_and_comments()?; + let (consumed, result) = if let Some(r) = self.parser.recognize_next_token( + &self.data[self.start..self.end], + self.is_ending, + options, + ) { + r + } else { + return if self.is_ending { + if self.start == self.end { + None // We have finished + } else { + let error = LexerError { + position: self.position..self.position + (self.end - self.start), + message: "Unexpected end of file".into(), + }; + self.end = self.start; // We consume everything + Some(Err(error)) + } + } else { + None + }; + }; + debug_assert!( + consumed > 0, + "The lexer must consume at least one byte each time" + ); + debug_assert!( + self.start + consumed <= self.end, + "The lexer tried to consumed {consumed} bytes but only {} bytes are readable", + self.end - self.start + ); + let old_position = self.position; + self.start += consumed; + self.position += consumed; + Some(match result { + Ok(token) => Ok(TokenWithPosition { + token, + position: old_position..self.position, + }), + Err(e) => Err(LexerError { + position: e.position.start + self.position..e.position.end + self.position, + message: e.message, + }), + }) + } + + pub fn is_end(&self) -> bool { + self.is_ending && self.end == self.start + } + + fn skip_whitespaces_and_comments(&mut self) -> Option<()> { + loop { + self.skip_whitespaces(); + + let buf = &self.data[self.start..self.end]; + if let Some(line_comment_start) = self.line_comment_start { + if buf.starts_with(line_comment_start) { + // Comment + if let Some(end) = memchr2(b'\r', b'\n', &buf[line_comment_start.len()..]) { + self.start += end + line_comment_start.len(); + self.position += end + line_comment_start.len(); + continue; + } + if self.is_ending { + self.end = self.start; // EOF + return Some(()); + } + return None; // We need more data + } + } + return Some(()); + } + } + + fn skip_whitespaces(&mut self) { + if self.is_line_jump_whitespace { + for (i, c) in self.data[self.start..self.end].iter().enumerate() { + if !matches!(c, b' ' | b'\t' | b'\r' | b'\n') { + self.start += i; + self.position += i; + return; + } + //TODO: SIMD + } + } else { + for (i, c) in self.data[self.start..self.end].iter().enumerate() { + if !matches!(c, b' ' | b'\t') { + self.start += i; + self.position += i; + return; + } + //TODO: SIMD + } + } + // We only have whitespaces + self.position += self.end - self.start; + self.end = self.start; + } + + fn shrink_if_useful(&mut self) { + if self.start * 2 > self.data.len() { + // We have read more than half of the buffer, let's move the data to the beginning + self.data.copy_within(self.start..self.end, 0); + self.end -= self.start; + self.start = 0; + } + } +} + +#[derive(Debug)] +pub struct LexerError { + position: Range, + message: String, +} + +impl LexerError { + pub fn position(&self) -> Range { + self.position.clone() + } + + pub fn message(&self) -> &str { + &self.message + } + + pub fn into_message(self) -> String { + self.message + } +} + +impl fmt::Display for LexerError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if self.position.start + 1 == self.position.end { + write!( + f, + "Lexer error at byte {}: {}", + self.position.start, self.message + ) + } else { + write!( + f, + "Lexer error between bytes {} and {}: {}", + self.position.start, self.position.end, self.message + ) + } + } +} + +impl Error for LexerError { + fn description(&self) -> &str { + self.message() + } +} diff --git a/lib/oxttl/src/toolkit/mod.rs b/lib/oxttl/src/toolkit/mod.rs new file mode 100644 index 00000000..986504f9 --- /dev/null +++ b/lib/oxttl/src/toolkit/mod.rs @@ -0,0 +1,11 @@ +//! oxttl parsing toolkit. +//! +//! Provides the basic code to write plain Rust lexers and parsers able to read files chunk by chunk. + +mod lexer; +mod parser; + +pub use self::lexer::{Lexer, LexerError, TokenRecognizer, TokenRecognizerError}; +pub use self::parser::{ + FromReadIterator, ParseError, ParseOrIoError, Parser, RuleRecognizer, RuleRecognizerError, +}; diff --git a/lib/oxttl/src/toolkit/parser.rs b/lib/oxttl/src/toolkit/parser.rs new file mode 100644 index 00000000..10965818 --- /dev/null +++ b/lib/oxttl/src/toolkit/parser.rs @@ -0,0 +1,244 @@ +use crate::toolkit::lexer::TokenWithPosition; +use crate::toolkit::{Lexer, LexerError, TokenRecognizer}; +use std::error::Error; +use std::io::Read; +use std::ops::Range; +use std::{fmt, io}; + +pub trait RuleRecognizer: Sized { + type TokenRecognizer: TokenRecognizer; + type Output; + + fn error_recovery_state(self) -> Self; + + fn recognize_next( + self, + token: ::Token<'_>, + results: &mut Vec, + errors: &mut Vec, + ) -> Self; + + fn recognize_end(self, results: &mut Vec, errors: &mut Vec); + + fn lexer_options(&self) -> &::Options; +} + +pub struct RuleRecognizerError { + pub message: String, +} + +impl> From for RuleRecognizerError { + fn from(message: S) -> Self { + Self { + message: message.into(), + } + } +} + +pub struct Parser { + lexer: Lexer, + state: Option, + results: Vec, + errors: Vec, + position: Range, + default_lexer_options: ::Options, +} + +impl Parser { + pub fn new(lexer: Lexer, recognizer: RR) -> Self { + Self { + lexer, + state: Some(recognizer), + results: vec![], + errors: vec![], + position: 0..0, + default_lexer_options: ::Options::default(), + } + } + + pub fn extend_from_slice(&mut self, other: &[u8]) { + self.lexer.extend_from_slice(other) + } + + pub fn end(&mut self) { + self.lexer.end() + } + + pub fn is_end(&self) -> bool { + self.state.is_none() && self.results.is_empty() && self.errors.is_empty() + } + + pub fn read_next(&mut self) -> Option> { + loop { + if let Some(error) = self.errors.pop() { + return Some(Err(ParseError { + position: self.position.clone(), + message: error.message, + })); + } + if let Some(result) = self.results.pop() { + return Some(Ok(result)); + } + if let Some(result) = self.lexer.read_next( + self.state + .as_ref() + .map_or(&self.default_lexer_options, |p| p.lexer_options()), + ) { + match result { + Ok(TokenWithPosition { token, position }) => { + self.position = position; + self.state = self.state.take().map(|state| { + state.recognize_next(token, &mut self.results, &mut self.errors) + }); + continue; + } + Err(e) => { + self.state = self.state.take().map(RR::error_recovery_state); + return Some(Err(e.into())); + } + } + } + if self.lexer.is_end() { + if let Some(state) = self.state.take() { + state.recognize_end(&mut self.results, &mut self.errors) + } else { + return None; + } + } else { + return None; + } + } + } + + pub fn parse_from_read(self, read: R) -> FromReadIterator { + FromReadIterator { read, parser: self } + } +} + +/// An error from parsing. +/// +/// It is composed of a message and a byte range in the input. +#[derive(Debug)] +pub struct ParseError { + position: Range, + message: String, +} + +impl ParseError { + /// The invalid byte range in the input. + pub fn position(&self) -> Range { + self.position.clone() + } + + /// The error message. + pub fn message(&self) -> &str { + &self.message + } + + /// Converts this error to an error message. + pub fn into_message(self) -> String { + self.message + } +} + +impl fmt::Display for ParseError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if self.position.start + 1 == self.position.end { + write!( + f, + "Parser error at byte {}: {}", + self.position.start, self.message + ) + } else { + write!( + f, + "Parser error between bytes {} and {}: {}", + self.position.start, self.position.end, self.message + ) + } + } +} + +impl Error for ParseError {} + +impl From for io::Error { + fn from(error: ParseError) -> Self { + io::Error::new(io::ErrorKind::InvalidData, error) + } +} + +impl From for ParseError { + fn from(e: LexerError) -> Self { + Self { + position: e.position(), + message: e.into_message(), + } + } +} + +/// The union of [`ParseError`] and [`std::io::Error`]. +#[derive(Debug)] +pub enum ParseOrIoError { + Parse(ParseError), + Io(io::Error), +} + +impl fmt::Display for ParseOrIoError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Parse(e) => e.fmt(f), + Self::Io(e) => e.fmt(f), + } + } +} + +impl Error for ParseOrIoError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + Some(match self { + Self::Parse(e) => e, + Self::Io(e) => e, + }) + } +} + +impl From for ParseOrIoError { + fn from(error: ParseError) -> Self { + Self::Parse(error) + } +} + +impl From for ParseOrIoError { + fn from(error: io::Error) -> Self { + Self::Io(error) + } +} + +impl From for io::Error { + fn from(error: ParseOrIoError) -> Self { + match error { + ParseOrIoError::Parse(e) => e.into(), + ParseOrIoError::Io(e) => e, + } + } +} + +pub struct FromReadIterator { + read: R, + parser: Parser, +} + +impl Iterator for FromReadIterator { + type Item = Result; + + fn next(&mut self) -> Option { + while !self.parser.is_end() { + if let Some(result) = self.parser.read_next() { + return Some(result.map_err(ParseOrIoError::Parse)); + } + if let Err(e) = self.parser.lexer.extend_from_read(&mut self.read) { + return Some(Err(e.into())); + } + } + None + } +} diff --git a/lib/oxttl/src/trig.rs b/lib/oxttl/src/trig.rs new file mode 100644 index 00000000..692d6543 --- /dev/null +++ b/lib/oxttl/src/trig.rs @@ -0,0 +1,666 @@ +//! A [TriG](https://www.w3.org/TR/trig/) streaming parser implemented by [`TriGParser`]. + +use crate::terse::TriGRecognizer; +use crate::toolkit::{FromReadIterator, ParseError, ParseOrIoError, Parser}; +use oxiri::{Iri, IriParseError}; +use oxrdf::{vocab::xsd, GraphName, NamedNode, Quad, QuadRef, Subject, TermRef}; +use std::collections::HashMap; +use std::fmt; +use std::io::{self, Read, Write}; + +/// A [TriG](https://www.w3.org/TR/trig/) streaming parser. +/// +/// Support for [TriG-star](https://w3c.github.io/rdf-star/cg-spec/2021-12-17.html#trig-star) is available behind the `rdf-star` feature and the [`TriGParser::with_quoted_triples`] option. +/// +/// Count the number of people: +/// ``` +/// use oxrdf::NamedNodeRef; +/// use oxttl::{TriGParser, ParseError}; +/// +/// let file = b"@base . +/// @prefix schema: . +/// a schema:Person ; +/// schema:name \"Foo\" . +/// a schema:Person ; +/// schema:name \"Bar\" ."; +/// +/// let rdf_type = NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?; +/// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; +/// let mut count = 0; +/// for quad in TriGParser::new().parse_from_read(file.as_ref()) { +/// let quad = quad?; +/// if quad.predicate == rdf_type && quad.object == schema_person.into() { +/// count += 1; +/// } +/// } +/// assert_eq!(2, count); +/// # Result::<_,Box>::Ok(()) +/// ``` +#[derive(Default)] +pub struct TriGParser { + base: Option>, + prefixes: HashMap>, + #[cfg(feature = "rdf-star")] + with_quoted_triples: bool, +} + +impl TriGParser { + /// Builds a new [`TriGParser`]. + #[inline] + pub fn new() -> Self { + Self::default() + } + + #[inline] + pub fn with_base_iri(mut self, base_iri: impl Into) -> Result { + self.base = Some(Iri::parse(base_iri.into())?); + Ok(self) + } + + #[inline] + pub fn with_prefix( + mut self, + prefix_name: impl Into, + prefix_iri: impl Into, + ) -> Result { + self.prefixes + .insert(prefix_name.into(), Iri::parse(prefix_iri.into())?); + Ok(self) + } + + /// Enables [TriG-star](https://w3c.github.io/rdf-star/cg-spec/2021-12-17.html#trig-star). + #[cfg(feature = "rdf-star")] + #[inline] + #[must_use] + pub fn with_quoted_triples(mut self) -> Self { + self.with_quoted_triples = true; + self + } + + /// Parses a TriG file from a [`Read`] implementation. + /// + /// Count the number of people: + /// ``` + /// use oxrdf::NamedNodeRef; + /// use oxttl::{TriGParser, ParseError}; + /// + /// let file = b"@base . + /// @prefix schema: . + /// a schema:Person ; + /// schema:name \"Foo\" . + /// a schema:Person ; + /// schema:name \"Bar\" ."; + /// + /// let rdf_type = NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?; + /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; + /// let mut count = 0; + /// for quad in TriGParser::new().parse_from_read(file.as_ref()) { + /// let quad = quad?; + /// if quad.predicate == rdf_type && quad.object == schema_person.into() { + /// count += 1; + /// } + /// } + /// assert_eq!(2, count); + /// # Result::<_,Box>::Ok(()) + /// ``` + pub fn parse_from_read(&self, read: R) -> FromReadTriGReader { + FromReadTriGReader { + inner: self.parse().parser.parse_from_read(read), + } + } + + /// Allows to parse a TriG file by using a low-level API. + /// + /// Count the number of people: + /// ``` + /// use oxrdf::NamedNodeRef; + /// use oxttl::{TriGParser, ParseError}; + /// + /// let file: [&[u8]; 5] = [b"@base ", + /// b". @prefix schema: .", + /// b" a schema:Person", + /// b" ; schema:name \"Foo\" . ", + /// b" a schema:Person ; schema:name \"Bar\" ." + /// ]; + /// + /// let rdf_type = NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?; + /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; + /// let mut count = 0; + /// let mut parser = TriGParser::new().parse(); + /// let mut file_chunks = file.iter(); + /// while !parser.is_end() { + /// // We feed more data to the parser + /// if let Some(chunk) = file_chunks.next() { + /// parser.extend_from_slice(chunk); + /// } else { + /// parser.end(); // It's finished + /// } + /// // We read as many quads from the parser as possible + /// while let Some(quad) = parser.read_next() { + /// let quad = quad?; + /// if quad.predicate == rdf_type && quad.object == schema_person.into() { + /// count += 1; + /// } + /// } + /// } + /// assert_eq!(2, count); + /// # Result::<_,Box>::Ok(()) + /// ``` + pub fn parse(&self) -> LowLevelTriGReader { + LowLevelTriGReader { + parser: TriGRecognizer::new_parser( + true, + #[cfg(feature = "rdf-star")] + self.with_quoted_triples, + self.base.clone(), + self.prefixes.clone(), + ), + } + } +} + +/// Parses a TriG file from a [`Read`] implementation. Can be built using [`TriGParser::parse_from_read`]. +/// +/// Count the number of people: +/// ``` +/// use oxrdf::NamedNodeRef; +/// use oxttl::{TriGParser, ParseError}; +/// +/// let file = b"@base . +/// @prefix schema: . +/// a schema:Person ; +/// schema:name \"Foo\" . +/// a schema:Person ; +/// schema:name \"Bar\" ."; +/// +/// let rdf_type = NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?; +/// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; +/// let mut count = 0; +/// for quad in TriGParser::new().parse_from_read(file.as_ref()) { +/// let quad = quad?; +/// if quad.predicate == rdf_type && quad.object == schema_person.into() { +/// count += 1; +/// } +/// } +/// assert_eq!(2, count); +/// # Result::<_,Box>::Ok(()) +/// ``` +pub struct FromReadTriGReader { + inner: FromReadIterator, +} + +impl Iterator for FromReadTriGReader { + type Item = Result; + + fn next(&mut self) -> Option> { + self.inner.next() + } +} + +/// Parses a TriG file by using a low-level API. Can be built using [`TriGParser::parse`]. +/// +/// Count the number of people: +/// ``` +/// use oxrdf::NamedNodeRef; +/// use oxttl::{TriGParser, ParseError}; +/// +/// let file: [&[u8]; 5] = [b"@base ", +/// b". @prefix schema: .", +/// b" a schema:Person", +/// b" ; schema:name \"Foo\" . ", +/// b" a schema:Person ; schema:name \"Bar\" ." +/// ]; +/// +/// let rdf_type = NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?; +/// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; +/// let mut count = 0; +/// let mut parser = TriGParser::new().parse(); +/// let mut file_chunks = file.iter(); +/// while !parser.is_end() { +/// // We feed more data to the parser +/// if let Some(chunk) = file_chunks.next() { +/// parser.extend_from_slice(chunk); +/// } else { +/// parser.end(); // It's finished +/// } +/// // We read as many quads from the parser as possible +/// while let Some(quad) = parser.read_next() { +/// let quad = quad?; +/// if quad.predicate == rdf_type && quad.object == schema_person.into() { +/// count += 1; +/// } +/// } +/// } +/// assert_eq!(2, count); +/// # Result::<_,Box>::Ok(()) +/// ``` +pub struct LowLevelTriGReader { + parser: Parser, +} + +impl LowLevelTriGReader { + /// Adds some extra bytes to the parser. Should be called when [`read_next`](Self::read_next) returns [`None`] and there is still unread data. + pub fn extend_from_slice(&mut self, other: &[u8]) { + self.parser.extend_from_slice(other) + } + + /// Tell the parser that the file is finished. + /// + /// This triggers the parsing of the final bytes and might lead [`read_next`](Self::read_next) to return some extra values. + pub fn end(&mut self) { + self.parser.end() + } + + /// Returns if the parsing is finished i.e. [`end`](Self::end) has been called and [`read_next`](Self::read_next) is always going to return `None`. + pub fn is_end(&self) -> bool { + self.parser.is_end() + } + + /// Attempt to parse a new quad from the already provided data. + /// + /// Returns [`None`] if the parsing is finished or more data is required. + /// If it is the case more data should be fed using [`extend_from_slice`](Self::extend_from_slice). + pub fn read_next(&mut self) -> Option> { + self.parser.read_next() + } +} + +/// A [TriG](https://www.w3.org/TR/trig/) serializer. +/// +/// Support for [TriG-star](https://w3c.github.io/rdf-star/cg-spec/2021-12-17.html#trig-star) is available behind the `rdf-star` feature. +/// +/// ``` +/// use oxrdf::{NamedNodeRef, QuadRef}; +/// use oxttl::TriGSerializer; +/// +/// let mut buf = Vec::new(); +/// let mut writer = TriGSerializer::new().serialize_to_write(buf); +/// writer.write_quad(QuadRef::new( +/// NamedNodeRef::new("http://example.com#me")?, +/// NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?, +/// NamedNodeRef::new("http://schema.org/Person")?, +/// NamedNodeRef::new("http://example.com")?, +/// ))?; +/// assert_eq!( +/// b" {\n\t .\n}\n", +/// writer.finish()?.as_slice() +/// ); +/// # Result::<_,Box>::Ok(()) +/// ``` +#[derive(Default)] +pub struct TriGSerializer; + +impl TriGSerializer { + /// Builds a new [`TriGSerializer`]. + #[inline] + pub fn new() -> Self { + Self + } + + /// Writes a TriG file to a [`Write`] implementation. + /// + /// ``` + /// use oxrdf::{NamedNodeRef, QuadRef}; + /// use oxttl::TriGSerializer; + /// + /// let mut buf = Vec::new(); + /// let mut writer = TriGSerializer::new().serialize_to_write(buf); + /// writer.write_quad(QuadRef::new( + /// NamedNodeRef::new("http://example.com#me")?, + /// NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?, + /// NamedNodeRef::new("http://schema.org/Person")?, + /// NamedNodeRef::new("http://example.com")?, + /// ))?; + /// assert_eq!( + /// b" {\n\t .\n}\n", + /// writer.finish()?.as_slice() + /// ); + /// # Result::<_,Box>::Ok(()) + /// ``` + pub fn serialize_to_write(&self, write: W) -> ToWriteTriGWriter { + ToWriteTriGWriter { + write, + writer: self.serialize(), + } + } + + /// Builds a low-level TriG writer. + /// + /// ``` + /// use oxrdf::{NamedNodeRef, QuadRef}; + /// use oxttl::TriGSerializer; + /// + /// let mut buf = Vec::new(); + /// let mut writer = TriGSerializer::new().serialize(); + /// writer.write_quad(QuadRef::new( + /// NamedNodeRef::new("http://example.com#me")?, + /// NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?, + /// NamedNodeRef::new("http://schema.org/Person")?, + /// NamedNodeRef::new("http://example.com")?, + /// ), &mut buf)?; + /// writer.finish(&mut buf)?; + /// assert_eq!( + /// b" {\n\t .\n}\n", + /// buf.as_slice() + /// ); + /// # Result::<_,Box>::Ok(()) + /// ``` + #[allow(clippy::unused_self)] + pub fn serialize(&self) -> LowLevelTriGWriter { + LowLevelTriGWriter { + current_graph_name: GraphName::DefaultGraph, + current_subject_predicate: None, + } + } +} + +/// Writes a TriG file to a [`Write`] implementation. Can be built using [`TriGSerializer::serialize_to_write`]. +/// +/// ``` +/// use oxrdf::{NamedNodeRef, QuadRef}; +/// use oxttl::TriGSerializer; +/// +/// let mut buf = Vec::new(); +/// let mut writer = TriGSerializer::new().serialize_to_write(buf); +/// writer.write_quad(QuadRef::new( +/// NamedNodeRef::new("http://example.com#me")?, +/// NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?, +/// NamedNodeRef::new("http://schema.org/Person")?, +/// NamedNodeRef::new("http://example.com")?, +/// ))?; +/// assert_eq!( +/// b" {\n\t .\n}\n", +/// writer.finish()?.as_slice() +/// ); +/// # Result::<_,Box>::Ok(()) +/// ``` +pub struct ToWriteTriGWriter { + write: W, + writer: LowLevelTriGWriter, +} + +impl ToWriteTriGWriter { + /// Writes an extra quad. + pub fn write_quad<'a>(&mut self, q: impl Into>) -> io::Result<()> { + self.writer.write_quad(q, &mut self.write) + } + + /// Ends the write process and returns the underlying [`Write`]. + pub fn finish(mut self) -> io::Result { + self.writer.finish(&mut self.write)?; + Ok(self.write) + } +} + +/// Writes a TriG file by using a low-level API. Can be built using [`TriGSerializer::serialize`]. +/// +/// ``` +/// use oxrdf::{NamedNodeRef, QuadRef}; +/// use oxttl::TriGSerializer; +/// +/// let mut buf = Vec::new(); +/// let mut writer = TriGSerializer::new().serialize(); +/// writer.write_quad(QuadRef::new( +/// NamedNodeRef::new("http://example.com#me")?, +/// NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?, +/// NamedNodeRef::new("http://schema.org/Person")?, +/// NamedNodeRef::new("http://example.com")?, +/// ), &mut buf)?; +/// writer.finish(&mut buf)?; +/// assert_eq!( +/// b" {\n\t .\n}\n", +/// buf.as_slice() +/// ); +/// # Result::<_,Box>::Ok(()) +/// ``` +pub struct LowLevelTriGWriter { + current_graph_name: GraphName, + current_subject_predicate: Option<(Subject, NamedNode)>, +} + +impl LowLevelTriGWriter { + /// Writes an extra quad. + pub fn write_quad<'a>( + &mut self, + q: impl Into>, + mut write: impl Write, + ) -> io::Result<()> { + let q = q.into(); + if q.graph_name == self.current_graph_name.as_ref() { + if let Some((current_subject, current_predicate)) = + self.current_subject_predicate.take() + { + if q.subject == current_subject.as_ref() { + if q.predicate == current_predicate { + self.current_subject_predicate = Some((current_subject, current_predicate)); + write!(write, " , {}", TurtleTerm(q.object)) + } else { + self.current_subject_predicate = + Some((current_subject, q.predicate.into_owned())); + writeln!(write, " ;")?; + if !self.current_graph_name.is_default_graph() { + write!(write, "\t")?; + } + write!(write, "\t{} {}", q.predicate, TurtleTerm(q.object)) + } + } else { + self.current_subject_predicate = + Some((q.subject.into_owned(), q.predicate.into_owned())); + writeln!(write, " .")?; + if !self.current_graph_name.is_default_graph() { + write!(write, "\t")?; + } + write!( + write, + "{} {} {}", + TurtleTerm(q.subject.into()), + q.predicate, + TurtleTerm(q.object) + ) + } + } else { + self.current_subject_predicate = + Some((q.subject.into_owned(), q.predicate.into_owned())); + if !self.current_graph_name.is_default_graph() { + write!(write, "\t")?; + } + write!( + write, + "{} {} {}", + TurtleTerm(q.subject.into()), + q.predicate, + TurtleTerm(q.object) + ) + } + } else { + if self.current_subject_predicate.is_some() { + writeln!(write, " .")?; + } + if !self.current_graph_name.is_default_graph() { + writeln!(write, "}}")?; + } + self.current_graph_name = q.graph_name.into_owned(); + self.current_subject_predicate = + Some((q.subject.into_owned(), q.predicate.into_owned())); + if !self.current_graph_name.is_default_graph() { + writeln!(write, "{} {{", q.graph_name)?; + write!(write, "\t")?; + } + write!( + write, + "{} {} {}", + TurtleTerm(q.subject.into()), + q.predicate, + TurtleTerm(q.object) + ) + } + } + + /// Finishes to write the file. + pub fn finish(&mut self, mut write: impl Write) -> io::Result<()> { + if self.current_subject_predicate.is_some() { + writeln!(write, " .")?; + } + if !self.current_graph_name.is_default_graph() { + writeln!(write, "}}")?; + } + Ok(()) + } +} + +struct TurtleTerm<'a>(TermRef<'a>); + +impl<'a> fmt::Display for TurtleTerm<'a> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self.0 { + TermRef::NamedNode(v) => write!(f, "{v}"), + TermRef::BlankNode(v) => write!(f, "{v}"), + TermRef::Literal(v) => { + let value = v.value(); + let inline = match v.datatype() { + xsd::BOOLEAN => is_turtle_boolean(value), + xsd::INTEGER => is_turtle_integer(value), + xsd::DECIMAL => is_turtle_decimal(value), + xsd::DOUBLE => is_turtle_double(value), + _ => false, + }; + if inline { + write!(f, "{value}") + } else { + write!(f, "{v}") + } + } + #[cfg(feature = "rdf-star")] + TermRef::Triple(t) => { + write!( + f, + "<< {} {} {} >>", + TurtleTerm(t.subject.as_ref().into()), + t.predicate, + TurtleTerm(t.object.as_ref()) + ) + } + } + } +} + +fn is_turtle_boolean(value: &str) -> bool { + matches!(value, "true" | "false") +} + +fn is_turtle_integer(value: &str) -> bool { + // [19] INTEGER ::= [+-]? [0-9]+ + let mut value = value.as_bytes(); + if let Some(v) = value.strip_prefix(b"+") { + value = v; + } else if let Some(v) = value.strip_prefix(b"-") { + value = v; + } + !value.is_empty() && value.iter().all(u8::is_ascii_digit) +} + +fn is_turtle_decimal(value: &str) -> bool { + // [20] DECIMAL ::= [+-]? [0-9]* '.' [0-9]+ + let mut value = value.as_bytes(); + if let Some(v) = value.strip_prefix(b"+") { + value = v; + } else if let Some(v) = value.strip_prefix(b"-") { + value = v; + } + while value.first().map_or(false, u8::is_ascii_digit) { + value = &value[1..]; + } + if let Some(v) = value.strip_prefix(b".") { + value = v; + } else { + return false; + } + !value.is_empty() && value.iter().all(u8::is_ascii_digit) +} + +fn is_turtle_double(value: &str) -> bool { + // [21] DOUBLE ::= [+-]? ([0-9]+ '.' [0-9]* EXPONENT | '.' [0-9]+ EXPONENT | [0-9]+ EXPONENT) + // [154s] EXPONENT ::= [eE] [+-]? [0-9]+ + let mut value = value.as_bytes(); + if let Some(v) = value.strip_prefix(b"+") { + value = v; + } else if let Some(v) = value.strip_prefix(b"-") { + value = v; + } + let mut with_before = false; + while value.first().map_or(false, u8::is_ascii_digit) { + value = &value[1..]; + with_before = true; + } + let mut with_after = false; + if let Some(v) = value.strip_prefix(b".") { + value = v; + while value.first().map_or(false, u8::is_ascii_digit) { + value = &value[1..]; + with_after = true; + } + } + if let Some(v) = value.strip_prefix(b"e") { + value = v; + } else if let Some(v) = value.strip_prefix(b"E") { + value = v; + } else { + return false; + } + if let Some(v) = value.strip_prefix(b"+") { + value = v; + } else if let Some(v) = value.strip_prefix(b"-") { + value = v; + } + (with_before || with_after) && !value.is_empty() && value.iter().all(u8::is_ascii_digit) +} + +#[cfg(test)] +mod tests { + use super::*; + use oxrdf::vocab::xsd; + use oxrdf::{BlankNodeRef, GraphNameRef, LiteralRef, NamedNodeRef}; + + #[test] + fn test_write() -> io::Result<()> { + let mut writer = TriGSerializer::new().serialize_to_write(Vec::new()); + writer.write_quad(QuadRef::new( + NamedNodeRef::new_unchecked("http://example.com/s"), + NamedNodeRef::new_unchecked("http://example.com/p"), + NamedNodeRef::new_unchecked("http://example.com/o"), + NamedNodeRef::new_unchecked("http://example.com/g"), + ))?; + writer.write_quad(QuadRef::new( + NamedNodeRef::new_unchecked("http://example.com/s"), + NamedNodeRef::new_unchecked("http://example.com/p"), + LiteralRef::new_simple_literal("foo"), + NamedNodeRef::new_unchecked("http://example.com/g"), + ))?; + writer.write_quad(QuadRef::new( + NamedNodeRef::new_unchecked("http://example.com/s"), + NamedNodeRef::new_unchecked("http://example.com/p2"), + LiteralRef::new_language_tagged_literal_unchecked("foo", "en"), + NamedNodeRef::new_unchecked("http://example.com/g"), + ))?; + writer.write_quad(QuadRef::new( + BlankNodeRef::new_unchecked("b"), + NamedNodeRef::new_unchecked("http://example.com/p2"), + BlankNodeRef::new_unchecked("b2"), + NamedNodeRef::new_unchecked("http://example.com/g"), + ))?; + writer.write_quad(QuadRef::new( + BlankNodeRef::new_unchecked("b"), + NamedNodeRef::new_unchecked("http://example.com/p2"), + LiteralRef::new_typed_literal("true", xsd::BOOLEAN), + GraphNameRef::DefaultGraph, + ))?; + writer.write_quad(QuadRef::new( + BlankNodeRef::new_unchecked("b"), + NamedNodeRef::new_unchecked("http://example.com/p2"), + LiteralRef::new_typed_literal("false", xsd::BOOLEAN), + NamedNodeRef::new_unchecked("http://example.com/g2"), + ))?; + assert_eq!(String::from_utf8(writer.finish()?).unwrap(), " {\n\t , \"foo\" ;\n\t\t \"foo\"@en .\n\t_:b _:b2 .\n}\n_:b true .\n {\n\t_:b false .\n}\n"); + Ok(()) + } +} diff --git a/lib/oxttl/src/turtle.rs b/lib/oxttl/src/turtle.rs new file mode 100644 index 00000000..275cec28 --- /dev/null +++ b/lib/oxttl/src/turtle.rs @@ -0,0 +1,462 @@ +//! A [Turtle](https://www.w3.org/TR/turtle/) streaming parser implemented by [`TurtleParser`]. + +use crate::terse::TriGRecognizer; +use crate::toolkit::{FromReadIterator, ParseError, ParseOrIoError, Parser}; +use crate::trig::{LowLevelTriGWriter, ToWriteTriGWriter}; +use crate::TriGSerializer; +use oxiri::{Iri, IriParseError}; +use oxrdf::{GraphNameRef, Triple, TripleRef}; +use std::collections::HashMap; +use std::io::{self, Read, Write}; + +/// A [Turtle](https://www.w3.org/TR/turtle/) streaming parser. +/// +/// Support for [Turtle-star](https://w3c.github.io/rdf-star/cg-spec/2021-12-17.html#turtle-star) is available behind the `rdf-star` feature and the [`TurtleParser::with_quoted_triples`] option. +/// +/// Count the number of people: +/// ``` +/// use oxrdf::NamedNodeRef; +/// use oxttl::{TurtleParser, ParseError}; +/// +/// let file = b"@base . +/// @prefix schema: . +/// a schema:Person ; +/// schema:name \"Foo\" . +/// a schema:Person ; +/// schema:name \"Bar\" ."; +/// +/// let rdf_type = NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?; +/// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; +/// let mut count = 0; +/// for triple in TurtleParser::new().parse_from_read(file.as_ref()) { +/// let triple = triple?; +/// if triple.predicate == rdf_type && triple.object == schema_person.into() { +/// count += 1; +/// } +/// } +/// assert_eq!(2, count); +/// # Result::<_,Box>::Ok(()) +/// ``` +#[derive(Default)] +pub struct TurtleParser { + base: Option>, + prefixes: HashMap>, + #[cfg(feature = "rdf-star")] + with_quoted_triples: bool, +} + +impl TurtleParser { + /// Builds a new [`TurtleParser`]. + #[inline] + pub fn new() -> Self { + Self::default() + } + + #[inline] + pub fn with_base_iri(mut self, base_iri: impl Into) -> Result { + self.base = Some(Iri::parse(base_iri.into())?); + Ok(self) + } + + #[inline] + pub fn with_prefix( + mut self, + prefix_name: impl Into, + prefix_iri: impl Into, + ) -> Result { + self.prefixes + .insert(prefix_name.into(), Iri::parse(prefix_iri.into())?); + Ok(self) + } + + /// Enables [Turtle-star](https://w3c.github.io/rdf-star/cg-spec/2021-12-17.html#turtle-star). + #[cfg(feature = "rdf-star")] + #[inline] + #[must_use] + pub fn with_quoted_triples(mut self) -> Self { + self.with_quoted_triples = true; + self + } + + /// Parses a Turtle file from a [`Read`] implementation. + /// + /// Count the number of people: + /// ``` + /// use oxrdf::NamedNodeRef; + /// use oxttl::{TurtleParser, ParseError}; + /// + /// let file = b"@base . + /// @prefix schema: . + /// a schema:Person ; + /// schema:name \"Foo\" . + /// a schema:Person ; + /// schema:name \"Bar\" ."; + /// + /// let rdf_type = NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?; + /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; + /// let mut count = 0; + /// for triple in TurtleParser::new().parse_from_read(file.as_ref()) { + /// let triple = triple?; + /// if triple.predicate == rdf_type && triple.object == schema_person.into() { + /// count += 1; + /// } + /// } + /// assert_eq!(2, count); + /// # Result::<_,Box>::Ok(()) + /// ``` + pub fn parse_from_read(&self, read: R) -> FromReadTurtleReader { + FromReadTurtleReader { + inner: self.parse().parser.parse_from_read(read), + } + } + + /// Allows to parse a Turtle file by using a low-level API. + /// + /// Count the number of people: + /// ``` + /// use oxrdf::NamedNodeRef; + /// use oxttl::{TurtleParser, ParseError}; + /// + /// let file: [&[u8]; 5] = [b"@base ", + /// b". @prefix schema: .", + /// b" a schema:Person", + /// b" ; schema:name \"Foo\" . ", + /// b" a schema:Person ; schema:name \"Bar\" ." + /// ]; + /// + /// let rdf_type = NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?; + /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; + /// let mut count = 0; + /// let mut parser = TurtleParser::new().parse(); + /// let mut file_chunks = file.iter(); + /// while !parser.is_end() { + /// // We feed more data to the parser + /// if let Some(chunk) = file_chunks.next() { + /// parser.extend_from_slice(chunk); + /// } else { + /// parser.end(); // It's finished + /// } + /// // We read as many triples from the parser as possible + /// while let Some(triple) = parser.read_next() { + /// let triple = triple?; + /// if triple.predicate == rdf_type && triple.object == schema_person.into() { + /// count += 1; + /// } + /// } + /// } + /// assert_eq!(2, count); + /// # Result::<_,Box>::Ok(()) + /// ``` + pub fn parse(&self) -> LowLevelTurtleReader { + LowLevelTurtleReader { + parser: TriGRecognizer::new_parser( + false, + #[cfg(feature = "rdf-star")] + self.with_quoted_triples, + self.base.clone(), + self.prefixes.clone(), + ), + } + } +} + +/// Parses a Turtle file from a [`Read`] implementation. Can be built using [`TurtleParser::parse_from_read`]. +/// +/// Count the number of people: +/// ``` +/// use oxrdf::NamedNodeRef; +/// use oxttl::{TurtleParser, ParseError}; +/// +/// let file = b"@base . +/// @prefix schema: . +/// a schema:Person ; +/// schema:name \"Foo\" . +/// a schema:Person ; +/// schema:name \"Bar\" ."; +/// +/// let rdf_type = NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?; +/// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; +/// let mut count = 0; +/// for triple in TurtleParser::new().parse_from_read(file.as_ref()) { +/// let triple = triple?; +/// if triple.predicate == rdf_type && triple.object == schema_person.into() { +/// count += 1; +/// } +/// } +/// assert_eq!(2, count); +/// # Result::<_,Box>::Ok(()) +/// ``` +pub struct FromReadTurtleReader { + inner: FromReadIterator, +} + +impl Iterator for FromReadTurtleReader { + type Item = Result; + + fn next(&mut self) -> Option> { + Some(self.inner.next()?.map(Into::into)) + } +} + +/// Parses a Turtle file by using a low-level API. Can be built using [`TurtleParser::parse`]. +/// +/// Count the number of people: +/// ``` +/// use oxrdf::NamedNodeRef; +/// use oxttl::{TurtleParser, ParseError}; +/// +/// let file: [&[u8]; 5] = [b"@base ", +/// b". @prefix schema: .", +/// b" a schema:Person", +/// b" ; schema:name \"Foo\" . ", +/// b" a schema:Person ; schema:name \"Bar\" ." +/// ]; +/// +/// let rdf_type = NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?; +/// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; +/// let mut count = 0; +/// let mut parser = TurtleParser::new().parse(); +/// let mut file_chunks = file.iter(); +/// while !parser.is_end() { +/// // We feed more data to the parser +/// if let Some(chunk) = file_chunks.next() { +/// parser.extend_from_slice(chunk); +/// } else { +/// parser.end(); // It's finished +/// } +/// // We read as many triples from the parser as possible +/// while let Some(triple) = parser.read_next() { +/// let triple = triple?; +/// if triple.predicate == rdf_type && triple.object == schema_person.into() { +/// count += 1; +/// } +/// } +/// } +/// assert_eq!(2, count); +/// # Result::<_,Box>::Ok(()) +/// ``` +pub struct LowLevelTurtleReader { + parser: Parser, +} + +impl LowLevelTurtleReader { + /// Adds some extra bytes to the parser. Should be called when [`read_next`](Self::read_next) returns [`None`] and there is still unread data. + pub fn extend_from_slice(&mut self, other: &[u8]) { + self.parser.extend_from_slice(other) + } + + /// Tell the parser that the file is finished. + /// + /// This triggers the parsing of the final bytes and might lead [`read_next`](Self::read_next) to return some extra values. + pub fn end(&mut self) { + self.parser.end() + } + + /// Returns if the parsing is finished i.e. [`end`](Self::end) has been called and [`read_next`](Self::read_next) is always going to return `None`. + pub fn is_end(&self) -> bool { + self.parser.is_end() + } + + /// Attempt to parse a new triple from the already provided data. + /// + /// Returns [`None`] if the parsing is finished or more data is required. + /// If it is the case more data should be fed using [`extend_from_slice`](Self::extend_from_slice). + pub fn read_next(&mut self) -> Option> { + Some(self.parser.read_next()?.map(Into::into)) + } +} + +/// A [Turtle](https://www.w3.org/TR/turtle/) serializer. +/// +/// Support for [Turtle-star](https://w3c.github.io/rdf-star/cg-spec/2021-12-17.html#turtle-star) is available behind the `rdf-star` feature. +/// +/// ``` +/// use oxrdf::{NamedNodeRef, TripleRef}; +/// use oxttl::TurtleSerializer; +/// +/// let mut buf = Vec::new(); +/// let mut writer = TurtleSerializer::new().serialize_to_write(buf); +/// writer.write_triple(TripleRef::new( +/// NamedNodeRef::new("http://example.com#me")?, +/// NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?, +/// NamedNodeRef::new("http://schema.org/Person")?, +/// ))?; +/// assert_eq!( +/// b" .\n", +/// writer.finish()?.as_slice() +/// ); +/// # Result::<_,Box>::Ok(()) +/// ``` +#[derive(Default)] +pub struct TurtleSerializer { + inner: TriGSerializer, +} + +impl TurtleSerializer { + /// Builds a new [`TurtleSerializer`]. + #[inline] + pub fn new() -> Self { + Self::default() + } + + /// Writes a Turtle file to a [`Write`] implementation. + /// + /// ``` + /// use oxrdf::{NamedNodeRef, TripleRef}; + /// use oxttl::TurtleSerializer; + /// + /// let mut buf = Vec::new(); + /// let mut writer = TurtleSerializer::new().serialize_to_write(buf); + /// writer.write_triple(TripleRef::new( + /// NamedNodeRef::new("http://example.com#me")?, + /// NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?, + /// NamedNodeRef::new("http://schema.org/Person")?, + /// ))?; + /// assert_eq!( + /// b" .\n", + /// writer.finish()?.as_slice() + /// ); + /// # Result::<_,Box>::Ok(()) + /// ``` + pub fn serialize_to_write(&self, write: W) -> ToWriteTurtleWriter { + ToWriteTurtleWriter { + inner: self.inner.serialize_to_write(write), + } + } + + /// Builds a low-level Turtle writer. + /// + /// ``` + /// use oxrdf::{NamedNodeRef, TripleRef}; + /// use oxttl::TurtleSerializer; + /// + /// let mut buf = Vec::new(); + /// let mut writer = TurtleSerializer::new().serialize(); + /// writer.write_triple(TripleRef::new( + /// NamedNodeRef::new("http://example.com#me")?, + /// NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?, + /// NamedNodeRef::new("http://schema.org/Person")?, + /// ), &mut buf)?; + /// writer.finish(&mut buf)?; + /// assert_eq!( + /// b" .\n", + /// buf.as_slice() + /// ); + /// # Result::<_,Box>::Ok(()) + /// ``` + pub fn serialize(&self) -> LowLevelTurtleWriter { + LowLevelTurtleWriter { + inner: self.inner.serialize(), + } + } +} + +/// Writes a Turtle file to a [`Write`] implementation. Can be built using [`TurtleSerializer::serialize_to_write`]. +/// +/// ``` +/// use oxrdf::{NamedNodeRef, TripleRef}; +/// use oxttl::TurtleSerializer; +/// +/// let mut buf = Vec::new(); +/// let mut writer = TurtleSerializer::new().serialize_to_write(buf); +/// writer.write_triple(TripleRef::new( +/// NamedNodeRef::new("http://example.com#me")?, +/// NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?, +/// NamedNodeRef::new("http://schema.org/Person")?, +/// ))?; +/// assert_eq!( +/// b" .\n", +/// writer.finish()?.as_slice() +/// ); +/// # Result::<_,Box>::Ok(()) +/// ``` +pub struct ToWriteTurtleWriter { + inner: ToWriteTriGWriter, +} + +impl ToWriteTurtleWriter { + /// Writes an extra triple. + pub fn write_triple<'a>(&mut self, t: impl Into>) -> io::Result<()> { + self.inner + .write_quad(t.into().in_graph(GraphNameRef::DefaultGraph)) + } + + /// Ends the write process and returns the underlying [`Write`]. + pub fn finish(self) -> io::Result { + self.inner.finish() + } +} + +/// Writes a Turtle file by using a low-level API. Can be built using [`TurtleSerializer::serialize`]. +/// +/// ``` +/// use oxrdf::{NamedNodeRef, TripleRef}; +/// use oxttl::TurtleSerializer; +/// +/// let mut buf = Vec::new(); +/// let mut writer = TurtleSerializer::new().serialize(); +/// writer.write_triple(TripleRef::new( +/// NamedNodeRef::new("http://example.com#me")?, +/// NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?, +/// NamedNodeRef::new("http://schema.org/Person")?, +/// ), &mut buf)?; +/// writer.finish(&mut buf)?; +/// assert_eq!( +/// b" .\n", +/// buf.as_slice() +/// ); +/// # Result::<_,Box>::Ok(()) +/// ``` +pub struct LowLevelTurtleWriter { + inner: LowLevelTriGWriter, +} + +impl LowLevelTurtleWriter { + /// Writes an extra triple. + pub fn write_triple<'a>( + &mut self, + t: impl Into>, + write: impl Write, + ) -> io::Result<()> { + self.inner + .write_quad(t.into().in_graph(GraphNameRef::DefaultGraph), write) + } + + /// Finishes to write the file. + pub fn finish(&mut self, write: impl Write) -> io::Result<()> { + self.inner.finish(write) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use oxrdf::{BlankNodeRef, LiteralRef, NamedNodeRef}; + + #[test] + fn test_write() -> io::Result<()> { + let mut writer = TurtleSerializer::new().serialize_to_write(Vec::new()); + writer.write_triple(TripleRef::new( + NamedNodeRef::new_unchecked("http://example.com/s"), + NamedNodeRef::new_unchecked("http://example.com/p"), + NamedNodeRef::new_unchecked("http://example.com/o"), + ))?; + writer.write_triple(TripleRef::new( + NamedNodeRef::new_unchecked("http://example.com/s"), + NamedNodeRef::new_unchecked("http://example.com/p"), + LiteralRef::new_simple_literal("foo"), + ))?; + writer.write_triple(TripleRef::new( + NamedNodeRef::new_unchecked("http://example.com/s"), + NamedNodeRef::new_unchecked("http://example.com/p2"), + LiteralRef::new_language_tagged_literal_unchecked("foo", "en"), + ))?; + writer.write_triple(TripleRef::new( + BlankNodeRef::new_unchecked("b"), + NamedNodeRef::new_unchecked("http://example.com/p2"), + BlankNodeRef::new_unchecked("b2"), + ))?; + assert_eq!(String::from_utf8(writer.finish()?).unwrap(), " , \"foo\" ;\n\t \"foo\"@en .\n_:b _:b2 .\n"); + Ok(()) + } +} diff --git a/lib/src/io/error.rs b/lib/src/io/error.rs index 6a90404b..54696363 100644 --- a/lib/src/io/error.rs +++ b/lib/src/io/error.rs @@ -1,5 +1,4 @@ use oxiri::IriParseError; -use rio_turtle::TurtleError; use rio_xml::RdfXmlError; use std::error::Error; use std::{fmt, io}; @@ -45,23 +44,26 @@ impl Error for ParseError { } } -impl From for ParseError { +impl From for ParseError { #[inline] - fn from(error: TurtleError) -> Self { - let error = io::Error::from(error); - if error.get_ref().map_or( - false, - <(dyn Error + Send + Sync + 'static)>::is::, - ) { - Self::Syntax(SyntaxError { - inner: SyntaxErrorKind::Turtle(*error.into_inner().unwrap().downcast().unwrap()), - }) - } else { - Self::Io(error) + fn from(error: oxttl::ParseError) -> Self { + Self::Syntax(SyntaxError { + inner: SyntaxErrorKind::Turtle(error), + }) + } +} + +impl From for ParseError { + #[inline] + fn from(error: oxttl::ParseOrIoError) -> Self { + match error { + oxttl::ParseOrIoError::Parse(e) => e.into(), + oxttl::ParseOrIoError::Io(e) => e.into(), } } } +#[allow(clippy::fallible_impl_from)] impl From for ParseError { #[inline] fn from(error: RdfXmlError) -> Self { @@ -111,7 +113,7 @@ pub struct SyntaxError { #[derive(Debug)] enum SyntaxErrorKind { - Turtle(TurtleError), + Turtle(oxttl::ParseError), RdfXml(RdfXmlError), InvalidBaseIri { iri: String, error: IriParseError }, } diff --git a/lib/src/io/mod.rs b/lib/src/io/mod.rs index 6c2c49b4..2e5269de 100644 --- a/lib/src/io/mod.rs +++ b/lib/src/io/mod.rs @@ -5,9 +5,6 @@ mod format; pub mod read; pub mod write; -pub use self::format::DatasetFormat; -pub use self::format::GraphFormat; -pub use self::read::DatasetParser; -pub use self::read::GraphParser; -pub use self::write::DatasetSerializer; -pub use self::write::GraphSerializer; +pub use self::format::{DatasetFormat, GraphFormat}; +pub use self::read::{DatasetParser, GraphParser}; +pub use self::write::{DatasetSerializer, GraphSerializer}; diff --git a/lib/src/io/read.rs b/lib/src/io/read.rs index 5d5ddeb2..3b2a107b 100644 --- a/lib/src/io/read.rs +++ b/lib/src/io/read.rs @@ -4,9 +4,12 @@ pub use crate::io::error::{ParseError, SyntaxError}; use crate::io::{DatasetFormat, GraphFormat}; use crate::model::*; use oxiri::{Iri, IriParseError}; +use oxttl::nquads::{FromReadNQuadsReader, NQuadsParser}; +use oxttl::ntriples::{FromReadNTriplesReader, NTriplesParser}; +use oxttl::trig::{FromReadTriGReader, TriGParser}; +use oxttl::turtle::{FromReadTurtleReader, TurtleParser}; use rio_api::model as rio; -use rio_api::parser::{QuadsParser, TriplesParser}; -use rio_turtle::{NQuadsParser, NTriplesParser, TriGParser, TurtleParser}; +use rio_api::parser::TriplesParser; use rio_xml::RdfXmlParser; use std::collections::HashMap; use std::io::BufRead; @@ -20,20 +23,24 @@ use std::io::BufRead; /// /// ``` /// use oxigraph::io::{GraphFormat, GraphParser}; -/// use std::io::Cursor; /// /// let file = " ."; /// /// let parser = GraphParser::from_format(GraphFormat::NTriples); -/// let triples = parser.read_triples(Cursor::new(file))?.collect::,_>>()?; +/// let triples = parser.read_triples(file.as_bytes()).collect::,_>>()?; /// ///assert_eq!(triples.len(), 1); ///assert_eq!(triples[0].subject.to_string(), ""); /// # std::io::Result::Ok(()) /// ``` pub struct GraphParser { - format: GraphFormat, - base_iri: Option>, + inner: GraphParserKind, +} + +enum GraphParserKind { + NTriples(NTriplesParser), + Turtle(TurtleParser), + RdfXml { base_iri: Option> }, } impl GraphParser { @@ -41,8 +48,15 @@ impl GraphParser { #[inline] pub fn from_format(format: GraphFormat) -> Self { Self { - format, - base_iri: None, + inner: match format { + GraphFormat::NTriples => { + GraphParserKind::NTriples(NTriplesParser::new().with_quoted_triples()) + } + GraphFormat::Turtle => { + GraphParserKind::Turtle(TurtleParser::new().with_quoted_triples()) + } + GraphFormat::RdfXml => GraphParserKind::RdfXml { base_iri: None }, + }, } } @@ -50,39 +64,44 @@ impl GraphParser { /// /// ``` /// use oxigraph::io::{GraphFormat, GraphParser}; - /// use std::io::Cursor; /// /// let file = "

."; /// /// let parser = GraphParser::from_format(GraphFormat::Turtle).with_base_iri("http://example.com")?; - /// let triples = parser.read_triples(Cursor::new(file))?.collect::,_>>()?; + /// let triples = parser.read_triples(file.as_bytes()).collect::,_>>()?; /// ///assert_eq!(triples.len(), 1); ///assert_eq!(triples[0].subject.to_string(), ""); /// # Result::<_,Box>::Ok(()) /// ``` #[inline] - pub fn with_base_iri(mut self, base_iri: impl Into) -> Result { - self.base_iri = Some(Iri::parse(base_iri.into())?); - Ok(self) + pub fn with_base_iri(self, base_iri: impl Into) -> Result { + Ok(Self { + inner: match self.inner { + GraphParserKind::NTriples(p) => GraphParserKind::NTriples(p), + GraphParserKind::Turtle(p) => GraphParserKind::Turtle(p.with_base_iri(base_iri)?), + GraphParserKind::RdfXml { .. } => GraphParserKind::RdfXml { + base_iri: Some(Iri::parse(base_iri.into())?), + }, + }, + }) } /// Executes the parsing itself on a [`BufRead`] implementation and returns an iterator of triples. - #[allow(clippy::unnecessary_wraps)] - pub fn read_triples(&self, reader: R) -> Result, ParseError> { - Ok(TripleReader { - mapper: RioMapper::default(), - parser: match self.format { - GraphFormat::NTriples => TripleReaderKind::NTriples(NTriplesParser::new(reader)), - GraphFormat::Turtle => { - TripleReaderKind::Turtle(TurtleParser::new(reader, self.base_iri.clone())) + pub fn read_triples(&self, reader: R) -> TripleReader { + TripleReader { + mapper: BlankNodeMapper::default(), + parser: match &self.inner { + GraphParserKind::NTriples(p) => { + TripleReaderKind::NTriples(p.parse_from_read(reader)) } - GraphFormat::RdfXml => { - TripleReaderKind::RdfXml(RdfXmlParser::new(reader, self.base_iri.clone())) + GraphParserKind::Turtle(p) => TripleReaderKind::Turtle(p.parse_from_read(reader)), + GraphParserKind::RdfXml { base_iri } => { + TripleReaderKind::RdfXml(RdfXmlParser::new(reader, base_iri.clone())) } }, buffer: Vec::new(), - }) + } } } @@ -91,12 +110,11 @@ impl GraphParser { /// /// ``` /// use oxigraph::io::{GraphFormat, GraphParser}; -/// use std::io::Cursor; /// /// let file = " ."; /// /// let parser = GraphParser::from_format(GraphFormat::NTriples); -/// let triples = parser.read_triples(Cursor::new(file))?.collect::,_>>()?; +/// let triples = parser.read_triples(file.as_bytes()).collect::,_>>()?; /// ///assert_eq!(triples.len(), 1); ///assert_eq!(triples[0].subject.to_string(), ""); @@ -104,15 +122,15 @@ impl GraphParser { /// ``` #[must_use] pub struct TripleReader { - mapper: RioMapper, + mapper: BlankNodeMapper, parser: TripleReaderKind, buffer: Vec, } #[allow(clippy::large_enum_variant)] enum TripleReaderKind { - NTriples(NTriplesParser), - Turtle(TurtleParser), + NTriples(FromReadNTriplesReader), + Turtle(FromReadTurtleReader), RdfXml(RdfXmlParser), } @@ -125,41 +143,28 @@ impl Iterator for TripleReader { return Some(Ok(r)); } - if let Err(error) = match &mut self.parser { - TripleReaderKind::NTriples(parser) => { - Self::read(parser, &mut self.buffer, &mut self.mapper) - } - TripleReaderKind::Turtle(parser) => { - Self::read(parser, &mut self.buffer, &mut self.mapper) - } + return Some(match &mut self.parser { + TripleReaderKind::NTriples(parser) => match parser.next()? { + Ok(triple) => Ok(self.mapper.triple(triple)), + Err(e) => Err(e.into()), + }, + TripleReaderKind::Turtle(parser) => match parser.next()? { + Ok(triple) => Ok(self.mapper.triple(triple)), + Err(e) => Err(e.into()), + }, TripleReaderKind::RdfXml(parser) => { - Self::read(parser, &mut self.buffer, &mut self.mapper) + if parser.is_end() { + return None; + } else if let Err(e) = parser.parse_step(&mut |t| { + self.buffer.push(self.mapper.triple(RioMapper::triple(&t))); + Ok(()) + }) { + Err(e) + } else { + continue; + } } - }? { - return Some(Err(error)); - } - } - } -} - -impl TripleReader { - fn read( - parser: &mut P, - buffer: &mut Vec, - mapper: &mut RioMapper, - ) -> Option> - where - ParseError: From, - { - if parser.is_end() { - None - } else if let Err(e) = parser.parse_step(&mut |t| { - buffer.push(mapper.triple(&t)); - Ok(()) - }) { - Some(Err(e)) - } else { - Some(Ok(())) + }); } } } @@ -172,20 +177,23 @@ impl TripleReader { /// /// ``` /// use oxigraph::io::{DatasetFormat, DatasetParser}; -/// use std::io::Cursor; /// /// let file = " ."; /// /// let parser = DatasetParser::from_format(DatasetFormat::NQuads); -/// let quads = parser.read_quads(Cursor::new(file))?.collect::,_>>()?; +/// let quads = parser.read_quads(file.as_bytes()).collect::,_>>()?; /// ///assert_eq!(quads.len(), 1); ///assert_eq!(quads[0].subject.to_string(), ""); /// # std::io::Result::Ok(()) /// ``` pub struct DatasetParser { - format: DatasetFormat, - base_iri: Option>, + inner: DatasetParserKind, +} + +enum DatasetParserKind { + NQuads(NQuadsParser), + TriG(TriGParser), } impl DatasetParser { @@ -193,8 +201,14 @@ impl DatasetParser { #[inline] pub fn from_format(format: DatasetFormat) -> Self { Self { - format, - base_iri: None, + inner: match format { + DatasetFormat::NQuads => { + DatasetParserKind::NQuads(NQuadsParser::new().with_quoted_triples()) + } + DatasetFormat::TriG => { + DatasetParserKind::TriG(TriGParser::new().with_quoted_triples()) + } + }, } } @@ -202,36 +216,35 @@ impl DatasetParser { /// /// ``` /// use oxigraph::io::{DatasetFormat, DatasetParser}; - /// use std::io::Cursor; /// /// let file = " {

}"; /// /// let parser = DatasetParser::from_format(DatasetFormat::TriG).with_base_iri("http://example.com")?; - /// let triples = parser.read_quads(Cursor::new(file))?.collect::,_>>()?; + /// let triples = parser.read_quads(file.as_bytes()).collect::,_>>()?; /// ///assert_eq!(triples.len(), 1); ///assert_eq!(triples[0].subject.to_string(), ""); /// # Result::<_,Box>::Ok(()) /// ``` #[inline] - pub fn with_base_iri(mut self, base_iri: impl Into) -> Result { - self.base_iri = Some(Iri::parse(base_iri.into())?); - Ok(self) + pub fn with_base_iri(self, base_iri: impl Into) -> Result { + Ok(Self { + inner: match self.inner { + DatasetParserKind::NQuads(p) => DatasetParserKind::NQuads(p), + DatasetParserKind::TriG(p) => DatasetParserKind::TriG(p.with_base_iri(base_iri)?), + }, + }) } /// Executes the parsing itself on a [`BufRead`] implementation and returns an iterator of quads. - #[allow(clippy::unnecessary_wraps)] - pub fn read_quads(&self, reader: R) -> Result, ParseError> { - Ok(QuadReader { - mapper: RioMapper::default(), - parser: match self.format { - DatasetFormat::NQuads => QuadReaderKind::NQuads(NQuadsParser::new(reader)), - DatasetFormat::TriG => { - QuadReaderKind::TriG(TriGParser::new(reader, self.base_iri.clone())) - } + pub fn read_quads(&self, reader: R) -> QuadReader { + QuadReader { + mapper: BlankNodeMapper::default(), + parser: match &self.inner { + DatasetParserKind::NQuads(p) => QuadReaderKind::NQuads(p.parse_from_read(reader)), + DatasetParserKind::TriG(p) => QuadReaderKind::TriG(p.parse_from_read(reader)), }, - buffer: Vec::new(), - }) + } } } @@ -240,12 +253,11 @@ impl DatasetParser { /// /// ``` /// use oxigraph::io::{DatasetFormat, DatasetParser}; -/// use std::io::Cursor; /// /// let file = " ."; /// /// let parser = DatasetParser::from_format(DatasetFormat::NQuads); -/// let quads = parser.read_quads(Cursor::new(file))?.collect::,_>>()?; +/// let quads = parser.read_quads(file.as_bytes()).collect::,_>>()?; /// ///assert_eq!(quads.len(), 1); ///assert_eq!(quads[0].subject.to_string(), ""); @@ -253,76 +265,41 @@ impl DatasetParser { /// ``` #[must_use] pub struct QuadReader { - mapper: RioMapper, + mapper: BlankNodeMapper, parser: QuadReaderKind, - buffer: Vec, } enum QuadReaderKind { - NQuads(NQuadsParser), - TriG(TriGParser), + NQuads(FromReadNQuadsReader), + TriG(FromReadTriGReader), } impl Iterator for QuadReader { type Item = Result; fn next(&mut self) -> Option> { - loop { - if let Some(r) = self.buffer.pop() { - return Some(Ok(r)); - } - - if let Err(error) = match &mut self.parser { - QuadReaderKind::NQuads(parser) => { - Self::read(parser, &mut self.buffer, &mut self.mapper) - } - QuadReaderKind::TriG(parser) => { - Self::read(parser, &mut self.buffer, &mut self.mapper) - } - }? { - return Some(Err(error)); - } - } + Some(match &mut self.parser { + QuadReaderKind::NQuads(parser) => match parser.next()? { + Ok(quad) => Ok(self.mapper.quad(quad)), + Err(e) => Err(e.into()), + }, + QuadReaderKind::TriG(parser) => match parser.next()? { + Ok(quad) => Ok(self.mapper.quad(quad)), + Err(e) => Err(e.into()), + }, + }) } } -impl QuadReader { - fn read( - parser: &mut P, - buffer: &mut Vec, - mapper: &mut RioMapper, - ) -> Option> - where - ParseError: From, - { - if parser.is_end() { - None - } else if let Err(e) = parser.parse_step(&mut |t| { - buffer.push(mapper.quad(&t)); - Ok(()) - }) { - Some(Err(e)) - } else { - Some(Ok(())) - } - } -} - -#[derive(Default)] -struct RioMapper { - bnode_map: HashMap, -} +struct RioMapper; impl<'a> RioMapper { fn named_node(node: rio::NamedNode<'a>) -> NamedNode { NamedNode::new_unchecked(node.iri) } - fn blank_node(&mut self, node: rio::BlankNode<'a>) -> BlankNode { - self.bnode_map - .entry(node.id.to_owned()) - .or_insert_with(BlankNode::default) - .clone() + fn blank_node(node: rio::BlankNode<'a>) -> BlankNode { + BlankNode::new_unchecked(node.id) } fn literal(literal: rio::Literal<'a>) -> Literal { @@ -337,43 +314,82 @@ impl<'a> RioMapper { } } - fn subject(&mut self, node: rio::Subject<'a>) -> Subject { + fn subject(node: rio::Subject<'a>) -> Subject { match node { rio::Subject::NamedNode(node) => Self::named_node(node).into(), - rio::Subject::BlankNode(node) => self.blank_node(node).into(), - rio::Subject::Triple(triple) => self.triple(triple).into(), + rio::Subject::BlankNode(node) => Self::blank_node(node).into(), + rio::Subject::Triple(triple) => Self::triple(triple).into(), } } - fn term(&mut self, node: rio::Term<'a>) -> Term { + fn term(node: rio::Term<'a>) -> Term { match node { rio::Term::NamedNode(node) => Self::named_node(node).into(), - rio::Term::BlankNode(node) => self.blank_node(node).into(), + rio::Term::BlankNode(node) => Self::blank_node(node).into(), rio::Term::Literal(literal) => Self::literal(literal).into(), - rio::Term::Triple(triple) => self.triple(triple).into(), + rio::Term::Triple(triple) => Self::triple(triple).into(), } } - fn triple(&mut self, triple: &rio::Triple<'a>) -> Triple { + fn triple(triple: &rio::Triple<'a>) -> Triple { Triple { - subject: self.subject(triple.subject), + subject: Self::subject(triple.subject), predicate: Self::named_node(triple.predicate), + object: Self::term(triple.object), + } + } +} + +#[derive(Default)] +struct BlankNodeMapper { + bnode_map: HashMap, +} + +impl BlankNodeMapper { + fn blank_node(&mut self, node: BlankNode) -> BlankNode { + self.bnode_map + .entry(node) + .or_insert_with(BlankNode::default) + .clone() + } + + fn subject(&mut self, node: Subject) -> Subject { + match node { + Subject::NamedNode(node) => node.into(), + Subject::BlankNode(node) => self.blank_node(node).into(), + Subject::Triple(triple) => self.triple(*triple).into(), + } + } + + fn term(&mut self, node: Term) -> Term { + match node { + Term::NamedNode(node) => node.into(), + Term::BlankNode(node) => self.blank_node(node).into(), + Term::Literal(literal) => literal.into(), + Term::Triple(triple) => self.triple(*triple).into(), + } + } + + fn triple(&mut self, triple: Triple) -> Triple { + Triple { + subject: self.subject(triple.subject), + predicate: triple.predicate, object: self.term(triple.object), } } - fn graph_name(&mut self, graph_name: Option>) -> GraphName { + fn graph_name(&mut self, graph_name: GraphName) -> GraphName { match graph_name { - Some(rio::GraphName::NamedNode(node)) => Self::named_node(node).into(), - Some(rio::GraphName::BlankNode(node)) => self.blank_node(node).into(), - None => GraphName::DefaultGraph, + GraphName::NamedNode(node) => node.into(), + GraphName::BlankNode(node) => self.blank_node(node).into(), + GraphName::DefaultGraph => GraphName::DefaultGraph, } } - fn quad(&mut self, quad: &rio::Quad<'a>) -> Quad { + fn quad(&mut self, quad: Quad) -> Quad { Quad { subject: self.subject(quad.subject), - predicate: Self::named_node(quad.predicate), + predicate: quad.predicate, object: self.term(quad.object), graph_name: self.graph_name(quad.graph_name), } diff --git a/lib/src/io/write.rs b/lib/src/io/write.rs index a2bddc9c..051ea202 100644 --- a/lib/src/io/write.rs +++ b/lib/src/io/write.rs @@ -2,6 +2,10 @@ use crate::io::{DatasetFormat, GraphFormat}; use crate::model::*; +use oxttl::nquads::{NQuadsSerializer, ToWriteNQuadsWriter}; +use oxttl::ntriples::{NTriplesSerializer, ToWriteNTriplesWriter}; +use oxttl::trig::{ToWriteTriGWriter, TriGSerializer}; +use oxttl::turtle::{ToWriteTurtleWriter, TurtleSerializer}; use rio_api::formatter::TriplesFormatter; use rio_api::model as rio; use rio_xml::RdfXmlFormatter; @@ -45,7 +49,12 @@ impl GraphSerializer { pub fn triple_writer(&self, writer: W) -> io::Result> { Ok(TripleWriter { formatter: match self.format { - GraphFormat::NTriples | GraphFormat::Turtle => TripleWriterKind::NTriples(writer), + GraphFormat::NTriples => { + TripleWriterKind::NTriples(NTriplesSerializer::new().serialize_to_write(writer)) + } + GraphFormat::Turtle => { + TripleWriterKind::Turtle(TurtleSerializer::new().serialize_to_write(writer)) + } GraphFormat::RdfXml => TripleWriterKind::RdfXml(RdfXmlFormatter::new(writer)?), }, }) @@ -79,71 +88,73 @@ pub struct TripleWriter { } enum TripleWriterKind { - NTriples(W), + NTriples(ToWriteNTriplesWriter), + Turtle(ToWriteTurtleWriter), RdfXml(RdfXmlFormatter), } impl TripleWriter { /// Writes a triple pub fn write<'a>(&mut self, triple: impl Into>) -> io::Result<()> { - let triple = triple.into(); match &mut self.formatter { - TripleWriterKind::NTriples(writer) => { - writeln!(writer, "{triple} .")?; - } - TripleWriterKind::RdfXml(formatter) => formatter.format(&rio::Triple { - subject: match triple.subject { - SubjectRef::NamedNode(node) => rio::NamedNode { iri: node.as_str() }.into(), - SubjectRef::BlankNode(node) => rio::BlankNode { id: node.as_str() }.into(), - SubjectRef::Triple(_) => { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "RDF/XML does not support RDF-star yet", - )) - } - }, - predicate: rio::NamedNode { - iri: triple.predicate.as_str(), - }, - object: match triple.object { - TermRef::NamedNode(node) => rio::NamedNode { iri: node.as_str() }.into(), - TermRef::BlankNode(node) => rio::BlankNode { id: node.as_str() }.into(), - TermRef::Literal(literal) => if literal.is_plain() { - if let Some(language) = literal.language() { - rio::Literal::LanguageTaggedString { - value: literal.value(), - language, + TripleWriterKind::NTriples(writer) => writer.write_triple(triple), + TripleWriterKind::Turtle(writer) => writer.write_triple(triple), + TripleWriterKind::RdfXml(formatter) => { + let triple = triple.into(); + formatter.format(&rio::Triple { + subject: match triple.subject { + SubjectRef::NamedNode(node) => rio::NamedNode { iri: node.as_str() }.into(), + SubjectRef::BlankNode(node) => rio::BlankNode { id: node.as_str() }.into(), + SubjectRef::Triple(_) => { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "RDF/XML does not support RDF-star yet", + )) + } + }, + predicate: rio::NamedNode { + iri: triple.predicate.as_str(), + }, + object: match triple.object { + TermRef::NamedNode(node) => rio::NamedNode { iri: node.as_str() }.into(), + TermRef::BlankNode(node) => rio::BlankNode { id: node.as_str() }.into(), + TermRef::Literal(literal) => if literal.is_plain() { + if let Some(language) = literal.language() { + rio::Literal::LanguageTaggedString { + value: literal.value(), + language, + } + } else { + rio::Literal::Simple { + value: literal.value(), + } } } else { - rio::Literal::Simple { + rio::Literal::Typed { value: literal.value(), + datatype: rio::NamedNode { + iri: literal.datatype().as_str(), + }, } } - } else { - rio::Literal::Typed { - value: literal.value(), - datatype: rio::NamedNode { - iri: literal.datatype().as_str(), - }, + .into(), + TermRef::Triple(_) => { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "RDF/XML does not support RDF-star yet", + )) } - } - .into(), - TermRef::Triple(_) => { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "RDF/XML does not support RDF-star yet", - )) - } - }, - })?, + }, + }) + } } - Ok(()) } /// Writes the last bytes of the file pub fn finish(self) -> io::Result<()> { match self.formatter { - TripleWriterKind::NTriples(mut writer) => writer.flush(), + TripleWriterKind::NTriples(writer) => writer.finish().flush(), + TripleWriterKind::Turtle(writer) => writer.finish()?.flush(), TripleWriterKind::RdfXml(formatter) => formatter.finish()?.flush(), //TODO: remove flush when the next version of Rio is going to be released } } @@ -160,7 +171,7 @@ impl TripleWriter { /// use oxigraph::model::*; /// /// let mut buffer = Vec::new(); -/// let mut writer = DatasetSerializer::from_format(DatasetFormat::NQuads).quad_writer(&mut buffer)?; +/// let mut writer = DatasetSerializer::from_format(DatasetFormat::NQuads).quad_writer(&mut buffer); /// writer.write(&Quad { /// subject: NamedNode::new("http://example.com/s")?.into(), /// predicate: NamedNode::new("http://example.com/p")?, @@ -184,14 +195,17 @@ impl DatasetSerializer { } /// Returns a [`QuadWriter`] allowing writing triples into the given [`Write`] implementation - #[allow(clippy::unnecessary_wraps)] - pub fn quad_writer(&self, writer: W) -> io::Result> { - Ok(QuadWriter { + pub fn quad_writer(&self, writer: W) -> QuadWriter { + QuadWriter { formatter: match self.format { - DatasetFormat::NQuads => QuadWriterKind::NQuads(writer), - DatasetFormat::TriG => QuadWriterKind::TriG(writer), + DatasetFormat::NQuads => { + QuadWriterKind::NQuads(NQuadsSerializer::new().serialize_to_write(writer)) + } + DatasetFormat::TriG => { + QuadWriterKind::TriG(TriGSerializer::new().serialize_to_write(writer)) + } }, - }) + } } } @@ -205,7 +219,7 @@ impl DatasetSerializer { /// use oxigraph::model::*; /// /// let mut buffer = Vec::new(); -/// let mut writer = DatasetSerializer::from_format(DatasetFormat::NQuads).quad_writer(&mut buffer)?; +/// let mut writer = DatasetSerializer::from_format(DatasetFormat::NQuads).quad_writer(&mut buffer); /// writer.write(&Quad { /// subject: NamedNode::new("http://example.com/s")?.into(), /// predicate: NamedNode::new("http://example.com/p")?, @@ -223,39 +237,24 @@ pub struct QuadWriter { } enum QuadWriterKind { - NQuads(W), - TriG(W), + NQuads(ToWriteNQuadsWriter), + TriG(ToWriteTriGWriter), } impl QuadWriter { /// Writes a quad pub fn write<'a>(&mut self, quad: impl Into>) -> io::Result<()> { - let quad = quad.into(); match &mut self.formatter { - QuadWriterKind::NQuads(writer) => { - writeln!(writer, "{quad} .")?; - } - QuadWriterKind::TriG(writer) => { - if quad.graph_name.is_default_graph() { - writeln!(writer, "{} .", TripleRef::from(quad)) - } else { - writeln!( - writer, - "{} {{ {} }}", - quad.graph_name, - TripleRef::from(quad) - ) - }?; - } + QuadWriterKind::NQuads(writer) => writer.write_quad(quad), + QuadWriterKind::TriG(writer) => writer.write_quad(quad), } - Ok(()) } /// Writes the last bytes of the file - #[allow(clippy::unused_self, clippy::unnecessary_wraps)] pub fn finish(self) -> io::Result<()> { match self.formatter { - QuadWriterKind::NQuads(mut writer) | QuadWriterKind::TriG(mut writer) => writer.flush(), + QuadWriterKind::NQuads(writer) => writer.finish().flush(), + QuadWriterKind::TriG(writer) => writer.finish()?.flush(), } } } diff --git a/lib/src/sparql/update.rs b/lib/src/sparql/update.rs index 2da5d08c..3c05c8f5 100644 --- a/lib/src/sparql/update.rs +++ b/lib/src/sparql/update.rs @@ -183,7 +183,7 @@ impl<'a, 'b: 'a> SimpleUpdateEvaluator<'a, 'b> { .with_base_iri(base_iri.as_str()) .map_err(|e| ParseError::invalid_base_iri(base_iri, e))?; } - for t in parser.read_triples(BufReader::new(body))? { + for t in parser.read_triples(BufReader::new(body)) { self.transaction .insert(t?.as_ref().in_graph(to_graph_name))?; } diff --git a/lib/src/store.rs b/lib/src/store.rs index 3d854a4a..fc6a83f1 100644 --- a/lib/src/store.rs +++ b/lib/src/store.rs @@ -480,9 +480,7 @@ impl Store { .with_base_iri(base_iri) .map_err(|e| ParseError::invalid_base_iri(base_iri, e))?; } - let quads = parser - .read_triples(reader)? - .collect::, _>>()?; + let quads = parser.read_triples(reader).collect::, _>>()?; let to_graph_name = to_graph_name.into(); self.storage.transaction(move |mut t| { for quad in &quads { @@ -525,7 +523,7 @@ impl Store { .with_base_iri(base_iri) .map_err(|e| ParseError::invalid_base_iri(base_iri, e))?; } - let quads = parser.read_quads(reader)?.collect::, _>>()?; + let quads = parser.read_quads(reader).collect::, _>>()?; self.storage.transaction(move |mut t| { for quad in &quads { t.insert(quad.into())?; @@ -647,7 +645,7 @@ impl Store { writer: impl Write, format: DatasetFormat, ) -> Result<(), SerializerError> { - let mut writer = DatasetSerializer::from_format(format).quad_writer(writer)?; + let mut writer = DatasetSerializer::from_format(format).quad_writer(writer); for quad in self.iter() { writer.write(&quad?)?; } @@ -1091,7 +1089,7 @@ impl<'a> Transaction<'a> { .map_err(|e| ParseError::invalid_base_iri(base_iri, e))?; } let to_graph_name = to_graph_name.into(); - for triple in parser.read_triples(reader)? { + for triple in parser.read_triples(reader) { self.writer .insert(triple?.as_ref().in_graph(to_graph_name))?; } @@ -1131,7 +1129,7 @@ impl<'a> Transaction<'a> { .with_base_iri(base_iri) .map_err(|e| ParseError::invalid_base_iri(base_iri, e))?; } - for quad in parser.read_quads(reader)? { + for quad in parser.read_quads(reader) { self.writer.insert(quad?.as_ref())?; } Ok(()) @@ -1470,7 +1468,7 @@ impl BulkLoader { .with_base_iri(base_iri) .map_err(|e| ParseError::invalid_base_iri(base_iri, e))?; } - self.load_ok_quads(parser.read_quads(reader)?.filter_map(|r| match r { + self.load_ok_quads(parser.read_quads(reader).filter_map(|r| match r { Ok(q) => Some(Ok(q)), Err(e) => { if let Some(callback) = &self.on_parse_error { @@ -1527,7 +1525,7 @@ impl BulkLoader { .map_err(|e| ParseError::invalid_base_iri(base_iri, e))?; } let to_graph_name = to_graph_name.into(); - self.load_ok_quads(parser.read_triples(reader)?.filter_map(|r| match r { + self.load_ok_quads(parser.read_triples(reader).filter_map(|r| match r { Ok(q) => Some(Ok(q.in_graph(to_graph_name.into_owned()))), Err(e) => { if let Some(callback) = &self.on_parse_error { diff --git a/python/src/io.rs b/python/src/io.rs index 53e53af4..35bb00eb 100644 --- a/python/src/io.rs +++ b/python/src/io.rs @@ -68,7 +68,7 @@ pub fn parse( .map_err(|e| PyValueError::new_err(e.to_string()))?; } Ok(PyTripleReader { - inner: py.allow_threads(|| parser.read_triples(input).map_err(map_parse_error))?, + inner: parser.read_triples(input), } .into_py(py)) } else if let Some(dataset_format) = DatasetFormat::from_media_type(mime_type) { @@ -79,7 +79,7 @@ pub fn parse( .map_err(|e| PyValueError::new_err(e.to_string()))?; } Ok(PyQuadReader { - inner: py.allow_threads(|| parser.read_quads(input).map_err(map_parse_error))?, + inner: parser.read_quads(input), } .into_py(py)) } else { @@ -136,9 +136,7 @@ pub fn serialize(input: &PyAny, output: PyObject, mime_type: &str, py: Python<'_ writer.finish().map_err(map_io_err)?; Ok(()) } else if let Some(dataset_format) = DatasetFormat::from_media_type(mime_type) { - let mut writer = DatasetSerializer::from_format(dataset_format) - .quad_writer(output) - .map_err(map_io_err)?; + let mut writer = DatasetSerializer::from_format(dataset_format).quad_writer(output); for i in input.iter()? { writer .write(&*i?.extract::>()?) diff --git a/python/tests/test_io.py b/python/tests/test_io.py index e7519f5d..2d291bc8 100644 --- a/python/tests/test_io.py +++ b/python/tests/test_io.py @@ -110,5 +110,5 @@ class TestSerialize(unittest.TestCase): serialize([EXAMPLE_QUAD], output, "application/trig") self.assertEqual( output.getvalue(), - b' { "1" }\n', + b' {\n\t "1" .\n}\n', ) diff --git a/python/tests/test_store.py b/python/tests/test_store.py index 8883b7ef..b29447c8 100644 --- a/python/tests/test_store.py +++ b/python/tests/test_store.py @@ -305,7 +305,8 @@ class TestStore(unittest.TestCase): store.dump(output, "application/trig") self.assertEqual( output.getvalue(), - b" .\n { }\n", + b" .\n" + b" {\n\t .\n}\n", ) def test_dump_file(self) -> None: diff --git a/server/src/main.rs b/server/src/main.rs index afae10f7..41c77ba8 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -946,7 +946,7 @@ fn handle_request( ReadForWrite::build_response( move |w| { Ok(( - DatasetSerializer::from_format(format).quad_writer(w)?, + DatasetSerializer::from_format(format).quad_writer(w), store.iter(), )) }, diff --git a/testsuite/Cargo.toml b/testsuite/Cargo.toml index 20579b1f..23bfb246 100644 --- a/testsuite/Cargo.toml +++ b/testsuite/Cargo.toml @@ -16,6 +16,16 @@ anyhow = "1" clap = { version = "4", features = ["derive"] } time = { version = "0.3", features = ["formatting"] } oxigraph = { path = "../lib" } +oxttl = { path= "../lib/oxttl" } sparopt = { path = "../lib/sparopt" } spargebra = { path = "../lib/spargebra" } text-diff = "0.4" +rio_api = "0.8" +rio_turtle = "0.8" + +[dev-dependencies] +criterion = "0.5" + +[[bench]] +name = "parser" +harness = false \ No newline at end of file diff --git a/testsuite/N3 b/testsuite/N3 new file mode 160000 index 00000000..5fa35bf6 --- /dev/null +++ b/testsuite/N3 @@ -0,0 +1 @@ +Subproject commit 5fa35bf602669a467cfd0ab24cc732fe49f2b927 diff --git a/testsuite/benches/parser.rs b/testsuite/benches/parser.rs new file mode 100644 index 00000000..63638647 --- /dev/null +++ b/testsuite/benches/parser.rs @@ -0,0 +1,194 @@ +use anyhow::Result; +use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; +use oxigraph_testsuite::files::read_file; +use oxigraph_testsuite::manifest::TestManifest; +use rio_api::parser::*; +use rio_turtle::*; +use std::io::Read; + +fn test_data_from_testsuite(manifest_uri: String, include_tests_types: &[&str]) -> Result> { + let manifest = TestManifest::new([manifest_uri]); + let mut data = Vec::default(); + for test in manifest { + let test = test?; + if include_tests_types.contains(&test.kind.as_str()) { + read_file(&test.action.unwrap())?.read_to_end(&mut data)?; + data.push(b'\n'); + } + } + Ok(data) +} + +fn ntriples_test_data() -> Result> { + test_data_from_testsuite( + "http://w3c.github.io/rdf-tests/ntriples/manifest.ttl".to_owned(), + &["http://www.w3.org/ns/rdftest#TestNTriplesPositiveSyntax"], + ) +} + +fn turtle_test_data() -> Result> { + test_data_from_testsuite( + "http://w3c.github.io/rdf-tests/turtle/manifest.ttl".to_owned(), + &[ + "http://www.w3.org/ns/rdftest#TestTurtlePositiveSyntax", + "http://www.w3.org/ns/rdftest#TestTurtleEval", + ], + ) +} + +fn parse_bench( + c: &mut Criterion, + parser_name: &str, + data_name: &str, + data: Vec, + bench: impl Fn(&[u8]), +) { + let mut group = c.benchmark_group(parser_name); + group.throughput(Throughput::Bytes(data.len() as u64)); + group.bench_with_input(BenchmarkId::from_parameter(data_name), &data, |b, data| { + b.iter(|| bench(data)) + }); + group.finish(); +} + +fn parse_oxttl_ntriples(c: &mut Criterion, name: &str, data: Vec) { + parse_bench(c, "oxttl ntriples", name, data, |data| { + let mut parser = oxttl::NTriplesParser::new().parse(); + parser.extend_from_slice(data); + parser.end(); + while let Some(result) = parser.read_next() { + result.unwrap(); + } + }); +} + +fn parse_oxttl_turtle(c: &mut Criterion, name: &str, data: Vec) { + parse_bench(c, "oxttl turtle", name, data, |data| { + let mut parser = oxttl::TurtleParser::new().parse(); + parser.extend_from_slice(data); + parser.end(); + while let Some(result) = parser.read_next() { + result.unwrap(); + } + }); +} + +fn parse_rio_ntriples(c: &mut Criterion, name: &str, data: Vec) { + parse_bench(c, "rio ntriples", name, data, |data| { + let mut count: u64 = 0; + NTriplesParser::new(data) + .parse_all(&mut |_| { + count += 1; + Ok(()) as Result<(), TurtleError> + }) + .unwrap(); + }); +} + +fn parse_rio_turtle(c: &mut Criterion, name: &str, data: Vec) { + parse_bench(c, "rio turtle", name, data, |data| { + let mut count: u64 = 0; + TurtleParser::new(data, None) + .parse_all(&mut |_| { + count += 1; + Ok(()) as Result<(), TurtleError> + }) + .unwrap(); + }); +} + +fn bench_parse_oxttl_ntriples_with_ntriples(c: &mut Criterion) { + parse_oxttl_ntriples( + c, + "ntriples", + match ntriples_test_data() { + Ok(d) => d, + Err(e) => { + eprintln!("{e}"); + return; + } + }, + ) +} + +fn bench_parse_oxttl_ntriples_with_turtle(c: &mut Criterion) { + parse_oxttl_turtle( + c, + "ntriples", + match ntriples_test_data() { + Ok(d) => d, + Err(e) => { + eprintln!("{e}"); + return; + } + }, + ) +} + +fn bench_parse_oxttl_turtle_with_turtle(c: &mut Criterion) { + parse_oxttl_turtle( + c, + "turtle", + match turtle_test_data() { + Ok(d) => d, + Err(e) => { + eprintln!("{e}"); + return; + } + }, + ) +} + +fn bench_parse_rio_ntriples_with_ntriples(c: &mut Criterion) { + parse_rio_ntriples( + c, + "ntriples", + match ntriples_test_data() { + Ok(d) => d, + Err(e) => { + eprintln!("{e}"); + return; + } + }, + ) +} + +fn bench_parse_rio_ntriples_with_turtle(c: &mut Criterion) { + parse_rio_turtle( + c, + "ntriples", + match ntriples_test_data() { + Ok(d) => d, + Err(e) => { + eprintln!("{e}"); + return; + } + }, + ) +} + +fn bench_parse_rio_turtle_with_turtle(c: &mut Criterion) { + parse_rio_turtle( + c, + "turtle", + match turtle_test_data() { + Ok(d) => d, + Err(e) => { + eprintln!("{e}"); + return; + } + }, + ) +} + +criterion_group!( + w3c_testsuite, + bench_parse_rio_ntriples_with_ntriples, + bench_parse_rio_ntriples_with_turtle, + bench_parse_rio_turtle_with_turtle, + bench_parse_oxttl_ntriples_with_ntriples, + bench_parse_oxttl_ntriples_with_turtle, + bench_parse_oxttl_turtle_with_turtle +); + +criterion_main!(w3c_testsuite); diff --git a/testsuite/oxigraph-tests/parser-recovery/invalid_bnode.nt b/testsuite/oxigraph-tests/parser-recovery/invalid_bnode.nt new file mode 100644 index 00000000..6fd51f08 --- /dev/null +++ b/testsuite/oxigraph-tests/parser-recovery/invalid_bnode.nt @@ -0,0 +1,2 @@ +_:` . + . diff --git a/testsuite/oxigraph-tests/parser-recovery/invalid_iri.nt b/testsuite/oxigraph-tests/parser-recovery/invalid_iri.nt new file mode 100644 index 00000000..37ddd42d --- /dev/null +++ b/testsuite/oxigraph-tests/parser-recovery/invalid_iri.nt @@ -0,0 +1,2 @@ + . + . diff --git a/testsuite/oxigraph-tests/parser-recovery/invalid_string.nt b/testsuite/oxigraph-tests/parser-recovery/invalid_string.nt new file mode 100644 index 00000000..19c3240c --- /dev/null +++ b/testsuite/oxigraph-tests/parser-recovery/invalid_string.nt @@ -0,0 +1,2 @@ + "\a" . + . diff --git a/testsuite/oxigraph-tests/parser-recovery/iri2_spo.nt b/testsuite/oxigraph-tests/parser-recovery/iri2_spo.nt new file mode 100644 index 00000000..ee1a2a6f --- /dev/null +++ b/testsuite/oxigraph-tests/parser-recovery/iri2_spo.nt @@ -0,0 +1,2 @@ + . + . diff --git a/testsuite/oxigraph-tests/parser-recovery/iri2_string_spo.nt b/testsuite/oxigraph-tests/parser-recovery/iri2_string_spo.nt new file mode 100644 index 00000000..2563a4f7 --- /dev/null +++ b/testsuite/oxigraph-tests/parser-recovery/iri2_string_spo.nt @@ -0,0 +1,2 @@ + "o" . + . diff --git a/testsuite/oxigraph-tests/parser-recovery/iri_spo.nt b/testsuite/oxigraph-tests/parser-recovery/iri_spo.nt new file mode 100644 index 00000000..8b7355c8 --- /dev/null +++ b/testsuite/oxigraph-tests/parser-recovery/iri_spo.nt @@ -0,0 +1 @@ + . diff --git a/testsuite/oxigraph-tests/parser-recovery/iri_string_spo.nt b/testsuite/oxigraph-tests/parser-recovery/iri_string_spo.nt new file mode 100644 index 00000000..24548279 --- /dev/null +++ b/testsuite/oxigraph-tests/parser-recovery/iri_string_spo.nt @@ -0,0 +1 @@ + "o" . diff --git a/testsuite/oxigraph-tests/parser-recovery/manifest.ttl b/testsuite/oxigraph-tests/parser-recovery/manifest.ttl new file mode 100644 index 00000000..37a102bc --- /dev/null +++ b/testsuite/oxigraph-tests/parser-recovery/manifest.ttl @@ -0,0 +1,129 @@ +@prefix mf: . +@prefix rdf: . +@prefix rdfs: . +@prefix ox: . + +<> + rdf:type mf:Manifest ; + rdfs:comment "Oxigraph parser recovery test cases" ; + mf:entries ( + <#invalid_iri_nt> + <#invalid_iri_ttl> + <#invalid_iri_n3> + <#invalid_bnode_nt> + <#invalid_bnode_ttl> + <#invalid_bnode_n3> + <#invalid_string_nt> + <#invalid_string_ttl> + <#invalid_string_n3> + <#missing_dot_at_end_of_triple_with_iri_middle_nt> + <#missing_dot_at_end_of_triple_with_iri_middle_ttl> + <#missing_dot_at_end_of_triple_with_iri_end_nt> + <#missing_dot_at_end_of_triple_with_iri_end_ttl> + <#missing_dot_at_end_of_triple_with_string_middle_nt> + <#missing_dot_at_end_of_triple_with_string_middle_ttl> + <#missing_dot_at_end_of_triple_with_string_end_nt> + <#missing_dot_at_end_of_triple_with_string_end_ttl> + ) . + +<#invalid_iri_nt> + rdf:type ox:TestNTripleRecovery ; + mf:name "IRI with space" ; + mf:action ; + mf:result . + +<#invalid_iri_ttl> + rdf:type ox:TestTurtleRecovery ; + mf:name "IRI with space" ; + mf:action ; + mf:result . + +<#invalid_iri_n3> + rdf:type ox:TestN3Recovery ; + mf:name "IRI with space" ; + mf:action ; + mf:result . + +<#invalid_bnode_nt> + rdf:type ox:TestNTripleRecovery ; + mf:name "bad character in blank node" ; + mf:action ; + mf:result . + +<#invalid_bnode_ttl> + rdf:type ox:TestTurtleRecovery ; + mf:name "bad character in blank node" ; + mf:action ; + mf:result . + +<#invalid_bnode_n3> + rdf:type ox:TestN3Recovery ; + mf:name "bad character in blank node" ; + mf:action ; + mf:result . + +<#invalid_string_nt> + rdf:type ox:TestNTripleRecovery ; + mf:name "invalid escape sequence in string" ; + mf:action ; + mf:result . + +<#invalid_string_ttl> + rdf:type ox:TestTurtleRecovery ; + mf:name "invalid escape sequence in string" ; + mf:action ; + mf:result . + +<#invalid_string_n3> + rdf:type ox:TestN3Recovery ; + mf:name "invalid escape sequence in string" ; + mf:action ; + mf:result . + +<#missing_dot_at_end_of_triple_with_iri_middle_nt> + rdf:type ox:TestNTripleRecovery ; + mf:name "missing dot at the end of a triple" ; + mf:action ; + mf:result . + +<#missing_dot_at_end_of_triple_with_iri_middle_ttl> + rdf:type ox:TestTurtleRecovery ; + mf:name "missing dot at the end of a triple" ; + mf:action ; + mf:result . + +<#missing_dot_at_end_of_triple_with_iri_end_nt> + rdf:type ox:TestNTripleRecovery ; + mf:name "missing dot at the end of a triple" ; + mf:action ; + mf:result . + +<#missing_dot_at_end_of_triple_with_iri_end_ttl> + rdf:type ox:TestTurtleRecovery ; + mf:name "missing dot at the end of a triple" ; + mf:action ; + mf:result . + +<#missing_dot_at_end_of_triple_with_string_middle_nt> + rdf:type ox:TestNTripleRecovery ; + mf:name "missing dot at the end of a triple" ; + mf:action ; + mf:result . + +<#missing_dot_at_end_of_triple_with_string_middle_ttl> + rdf:type ox:TestTurtleRecovery ; + mf:name "missing dot at the end of a triple" ; + mf:action ; + mf:result . + +<#missing_dot_at_end_of_triple_with_string_end_nt> + rdf:type ox:TestNTripleRecovery ; + mf:name "missing dot at the end of a triple" ; + mf:action ; + mf:result . + +<#missing_dot_at_end_of_triple_with_string_end_ttl> + rdf:type ox:TestTurtleRecovery ; + mf:name "missing dot at the end of a triple" ; + mf:action ; + mf:result . diff --git a/testsuite/oxigraph-tests/parser-recovery/missing_dot_at_end_of_triple_with_iri_end.nt b/testsuite/oxigraph-tests/parser-recovery/missing_dot_at_end_of_triple_with_iri_end.nt new file mode 100644 index 00000000..ff536e88 --- /dev/null +++ b/testsuite/oxigraph-tests/parser-recovery/missing_dot_at_end_of_triple_with_iri_end.nt @@ -0,0 +1 @@ + diff --git a/testsuite/oxigraph-tests/parser-recovery/missing_dot_at_end_of_triple_with_iri_middle.nt b/testsuite/oxigraph-tests/parser-recovery/missing_dot_at_end_of_triple_with_iri_middle.nt new file mode 100644 index 00000000..117f53b0 --- /dev/null +++ b/testsuite/oxigraph-tests/parser-recovery/missing_dot_at_end_of_triple_with_iri_middle.nt @@ -0,0 +1,2 @@ + + . diff --git a/testsuite/oxigraph-tests/parser-recovery/missing_dot_at_end_of_triple_with_string_end.nt b/testsuite/oxigraph-tests/parser-recovery/missing_dot_at_end_of_triple_with_string_end.nt new file mode 100644 index 00000000..ff0896c3 --- /dev/null +++ b/testsuite/oxigraph-tests/parser-recovery/missing_dot_at_end_of_triple_with_string_end.nt @@ -0,0 +1 @@ + "o" diff --git a/testsuite/oxigraph-tests/parser-recovery/missing_dot_at_end_of_triple_with_string_middle.nt b/testsuite/oxigraph-tests/parser-recovery/missing_dot_at_end_of_triple_with_string_middle.nt new file mode 100644 index 00000000..bdd80d3c --- /dev/null +++ b/testsuite/oxigraph-tests/parser-recovery/missing_dot_at_end_of_triple_with_string_middle.nt @@ -0,0 +1,2 @@ + "o" + . diff --git a/testsuite/oxigraph-tests/parser/at_keywords_as_lang_tag.nt b/testsuite/oxigraph-tests/parser/at_keywords_as_lang_tag.nt new file mode 100644 index 00000000..ebcd4fdf --- /dev/null +++ b/testsuite/oxigraph-tests/parser/at_keywords_as_lang_tag.nt @@ -0,0 +1,2 @@ + "foo"@base . + "bar"@prefix . diff --git a/testsuite/oxigraph-tests/parser/at_keywords_as_lang_tag.ttl b/testsuite/oxigraph-tests/parser/at_keywords_as_lang_tag.ttl new file mode 100644 index 00000000..183a0b35 --- /dev/null +++ b/testsuite/oxigraph-tests/parser/at_keywords_as_lang_tag.ttl @@ -0,0 +1,3 @@ +@prefix : . + +:s :p "foo"@base , "bar"@prefix . diff --git a/testsuite/oxigraph-tests/parser/bad_lang.ttl b/testsuite/oxigraph-tests/parser/bad_lang.ttl new file mode 100644 index 00000000..1491033d --- /dev/null +++ b/testsuite/oxigraph-tests/parser/bad_lang.ttl @@ -0,0 +1 @@ + "foo"@badlanguagetag . \ No newline at end of file diff --git a/testsuite/oxigraph-tests/parser/bad_parentheses.ttl b/testsuite/oxigraph-tests/parser/bad_parentheses.ttl new file mode 100644 index 00000000..32ae5d04 --- /dev/null +++ b/testsuite/oxigraph-tests/parser/bad_parentheses.ttl @@ -0,0 +1,2 @@ +(((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((( + diff --git a/testsuite/oxigraph-tests/parser/blank_node_with_linebreak.nt b/testsuite/oxigraph-tests/parser/blank_node_with_linebreak.nt new file mode 100644 index 00000000..c975cc52 --- /dev/null +++ b/testsuite/oxigraph-tests/parser/blank_node_with_linebreak.nt @@ -0,0 +1,2 @@ + _:1 . + diff --git a/testsuite/oxigraph-tests/parser/blank_node_with_linebreak.ttl b/testsuite/oxigraph-tests/parser/blank_node_with_linebreak.ttl new file mode 100644 index 00000000..5972050b --- /dev/null +++ b/testsuite/oxigraph-tests/parser/blank_node_with_linebreak.ttl @@ -0,0 +1,6 @@ +@prefix state: . + + + state:state [ + ] . + diff --git a/testsuite/oxigraph-tests/parser/keyword_vs_prefix.nq b/testsuite/oxigraph-tests/parser/keyword_vs_prefix.nq new file mode 100644 index 00000000..d789d008 --- /dev/null +++ b/testsuite/oxigraph-tests/parser/keyword_vs_prefix.nq @@ -0,0 +1,3 @@ + . + . + . diff --git a/testsuite/oxigraph-tests/parser/keyword_vs_prefix.nt b/testsuite/oxigraph-tests/parser/keyword_vs_prefix.nt new file mode 100644 index 00000000..30ae19ae --- /dev/null +++ b/testsuite/oxigraph-tests/parser/keyword_vs_prefix.nt @@ -0,0 +1,2 @@ + . + . diff --git a/testsuite/oxigraph-tests/parser/keyword_vs_prefix.trig b/testsuite/oxigraph-tests/parser/keyword_vs_prefix.trig new file mode 100644 index 00000000..b293904f --- /dev/null +++ b/testsuite/oxigraph-tests/parser/keyword_vs_prefix.trig @@ -0,0 +1,10 @@ +base +prefix prefix: +prefix base: +prefix graph: +prefix true: +prefix false: + +prefix:s prefix:p true:o . +base:s base:p false:o . +graph:g { graph:s graph:p graph:o . } diff --git a/testsuite/oxigraph-tests/parser/keyword_vs_prefix.ttl b/testsuite/oxigraph-tests/parser/keyword_vs_prefix.ttl new file mode 100644 index 00000000..37a94cd8 --- /dev/null +++ b/testsuite/oxigraph-tests/parser/keyword_vs_prefix.ttl @@ -0,0 +1,8 @@ +base +prefix prefix: +prefix base: +prefix true: +prefix false: + +prefix:s prefix:p true:o . +base:s base:p false:o . diff --git a/testsuite/oxigraph-tests/parser/language_normalization.nt b/testsuite/oxigraph-tests/parser/language_normalization.nt new file mode 100644 index 00000000..23267c1e --- /dev/null +++ b/testsuite/oxigraph-tests/parser/language_normalization.nt @@ -0,0 +1 @@ + "foo"@en-us . \ No newline at end of file diff --git a/testsuite/oxigraph-tests/parser/language_normalization.rdf b/testsuite/oxigraph-tests/parser/language_normalization.rdf new file mode 100644 index 00000000..56520f45 --- /dev/null +++ b/testsuite/oxigraph-tests/parser/language_normalization.rdf @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/testsuite/oxigraph-tests/parser/language_normalization.ttl b/testsuite/oxigraph-tests/parser/language_normalization.ttl new file mode 100644 index 00000000..ea2f650d --- /dev/null +++ b/testsuite/oxigraph-tests/parser/language_normalization.ttl @@ -0,0 +1 @@ + "foo"@en-US . \ No newline at end of file diff --git a/testsuite/oxigraph-tests/parser/literal_value_space.nt b/testsuite/oxigraph-tests/parser/literal_value_space.nt new file mode 100644 index 00000000..e91cd772 --- /dev/null +++ b/testsuite/oxigraph-tests/parser/literal_value_space.nt @@ -0,0 +1 @@ + " bar\n" . diff --git a/testsuite/oxigraph-tests/parser/literal_value_space.rdf b/testsuite/oxigraph-tests/parser/literal_value_space.rdf new file mode 100644 index 00000000..326d4194 --- /dev/null +++ b/testsuite/oxigraph-tests/parser/literal_value_space.rdf @@ -0,0 +1,7 @@ + + + + bar + + + diff --git a/testsuite/oxigraph-tests/parser/manifest.ttl b/testsuite/oxigraph-tests/parser/manifest.ttl new file mode 100644 index 00000000..ec90b2bf --- /dev/null +++ b/testsuite/oxigraph-tests/parser/manifest.ttl @@ -0,0 +1,90 @@ +@prefix mf: . +@prefix rdf: . +@prefix rdfs: . +@prefix rdft: . + +<> + rdf:type mf:Manifest ; + rdfs:comment "Oxigraph parsers test case" ; + mf:entries ( + <#blank_node_with_linebreak> + <#bad_lang> + <#language_normalization_ttl> + <#language_normalization_xml> + <#xml_entities> + <#xml_nested_entities> + <#literal_value_space> + <#bad_parentheses> + <#keyword_vs_prefix_ttl> + <#keyword_vs_prefix_trig> + <#at_keywords_as_lang_tag> + ) . + +<#no_end_line_jump> + rdf:type rdft:TestNTriplesPositiveSyntax ; + mf:name "No line jump at the end of the file" ; + mf:action . + +<#blank_node_with_linebreak> + rdf:type rdft:TestTurtleEval ; + mf:name "blank node with linebreak" ; + mf:action ; + mf:result . + +<#language_normalization_ttl> + rdf:type rdft:TestTurtleEval ; + mf:name "language case normalization" ; + mf:action ; + mf:result . + +<#language_normalization_xml> + rdf:type rdft:TestXMLEval ; + mf:name "language case normalization" ; + mf:action ; + mf:result . + +<#bad_lang> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad language tag" ; + mf:action . + +<#xml_entities> + rdf:type rdft:TestXMLEval ; + mf:name "custom XML entities" ; + mf:action ; + mf:result . + +<#xml_nested_entities> + rdf:type rdft:TestXMLEval ; + mf:name "custom XML entities with nested definitions" ; + mf:action ; + mf:result . + +<#literal_value_space> + rdf:type rdft:TestXMLEval ; + mf:name "spaces in literal values" ; + mf:action ; + mf:result . + +<#bad_parentheses> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "a lot of parentheses that might generate a stack overflow" ; + mf:action . + +<#keyword_vs_prefix_ttl> + rdf:type rdft:TestTurtleEval ; + mf:name "usage of keywords as prefix" ; + mf:action ; + mf:result . + +<#keyword_vs_prefix_trig> + rdf:type rdft:TestTrigEval ; + mf:name "usage of keywords as prefix" ; + mf:action ; + mf:result . + +<#at_keywords_as_lang_tag> + rdf:type rdft:TestTurtleEval ; + mf:name "usage of at keywords as language tags" ; + mf:action ; + mf:result . diff --git a/testsuite/oxigraph-tests/parser/no_end_line_jump.nt b/testsuite/oxigraph-tests/parser/no_end_line_jump.nt new file mode 100644 index 00000000..fc13f999 --- /dev/null +++ b/testsuite/oxigraph-tests/parser/no_end_line_jump.nt @@ -0,0 +1 @@ + . \ No newline at end of file diff --git a/testsuite/oxigraph-tests/parser/xml_entities.nt b/testsuite/oxigraph-tests/parser/xml_entities.nt new file mode 100644 index 00000000..d4ed18f0 --- /dev/null +++ b/testsuite/oxigraph-tests/parser/xml_entities.nt @@ -0,0 +1 @@ + "bar"^^ . diff --git a/testsuite/oxigraph-tests/parser/xml_entities.rdf b/testsuite/oxigraph-tests/parser/xml_entities.rdf new file mode 100644 index 00000000..9d14dfb3 --- /dev/null +++ b/testsuite/oxigraph-tests/parser/xml_entities.rdf @@ -0,0 +1,10 @@ + + + +]> + + + bar + + diff --git a/testsuite/oxigraph-tests/parser/xml_nested_entities.nt b/testsuite/oxigraph-tests/parser/xml_nested_entities.nt new file mode 100644 index 00000000..d4ed18f0 --- /dev/null +++ b/testsuite/oxigraph-tests/parser/xml_nested_entities.nt @@ -0,0 +1 @@ + "bar"^^ . diff --git a/testsuite/oxigraph-tests/parser/xml_nested_entities.rdf b/testsuite/oxigraph-tests/parser/xml_nested_entities.rdf new file mode 100644 index 00000000..5be2e9d7 --- /dev/null +++ b/testsuite/oxigraph-tests/parser/xml_nested_entities.rdf @@ -0,0 +1,15 @@ + + + + + + + +]> + + + + bar + + \ No newline at end of file diff --git a/testsuite/serd-tests/LICENSE b/testsuite/serd-tests/LICENSE new file mode 100644 index 00000000..977d7b6e --- /dev/null +++ b/testsuite/serd-tests/LICENSE @@ -0,0 +1,13 @@ +Copyright 2011-2022 David Robillard + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. diff --git a/testsuite/serd-tests/README.md b/testsuite/serd-tests/README.md new file mode 100644 index 00000000..985f0c59 --- /dev/null +++ b/testsuite/serd-tests/README.md @@ -0,0 +1 @@ +Testsuite from [Serd](https://drobilla.net/software/serd) project. diff --git a/testsuite/serd-tests/bad/bad-00.ttl b/testsuite/serd-tests/bad/bad-00.ttl new file mode 100644 index 00000000..b1a1b361 --- /dev/null +++ b/testsuite/serd-tests/bad/bad-00.ttl @@ -0,0 +1,2 @@ +# prefix name must end in a : +@prefix a <#> . diff --git a/testsuite/serd-tests/bad/bad-01.ttl b/testsuite/serd-tests/bad/bad-01.ttl new file mode 100644 index 00000000..533bc21d --- /dev/null +++ b/testsuite/serd-tests/bad/bad-01.ttl @@ -0,0 +1,3 @@ +# Forbidden by RDF - predicate cannot be blank +@prefix : . +:a [ :b :c ] :d . diff --git a/testsuite/serd-tests/bad/bad-02.ttl b/testsuite/serd-tests/bad/bad-02.ttl new file mode 100644 index 00000000..fd2014d0 --- /dev/null +++ b/testsuite/serd-tests/bad/bad-02.ttl @@ -0,0 +1,3 @@ +# Forbidden by RDF - predicate cannot be blank +@prefix : . +:a [] :b . diff --git a/testsuite/serd-tests/bad/bad-03.ttl b/testsuite/serd-tests/bad/bad-03.ttl new file mode 100644 index 00000000..07a372f1 --- /dev/null +++ b/testsuite/serd-tests/bad/bad-03.ttl @@ -0,0 +1,3 @@ +# 'a' only allowed as a predicate +@prefix : . +a :a :b . diff --git a/testsuite/serd-tests/bad/bad-04.ttl b/testsuite/serd-tests/bad/bad-04.ttl new file mode 100644 index 00000000..ee7246c8 --- /dev/null +++ b/testsuite/serd-tests/bad/bad-04.ttl @@ -0,0 +1,3 @@ +# No comma is allowed in collections +@prefix : . +:a :b ( "apple", "banana" ) . diff --git a/testsuite/serd-tests/bad/bad-05.ttl b/testsuite/serd-tests/bad/bad-05.ttl new file mode 100644 index 00000000..387015fe --- /dev/null +++ b/testsuite/serd-tests/bad/bad-05.ttl @@ -0,0 +1,4 @@ +# N3 {}s are not in Turtle +@prefix : . +{ :a :b :c . } :d :e . + diff --git a/testsuite/serd-tests/bad/bad-06.ttl b/testsuite/serd-tests/bad/bad-06.ttl new file mode 100644 index 00000000..a8f1a0fc --- /dev/null +++ b/testsuite/serd-tests/bad/bad-06.ttl @@ -0,0 +1,3 @@ +# is and of are not in turtle +@prefix : . +:a is :b of :c . diff --git a/testsuite/serd-tests/bad/bad-07.ttl b/testsuite/serd-tests/bad/bad-07.ttl new file mode 100644 index 00000000..18ea4fa6 --- /dev/null +++ b/testsuite/serd-tests/bad/bad-07.ttl @@ -0,0 +1,4 @@ +# paths are not in turtle +@prefix : . +:a.:b.:c . +:a^:b^:c . diff --git a/testsuite/serd-tests/bad/bad-08.ttl b/testsuite/serd-tests/bad/bad-08.ttl new file mode 100644 index 00000000..07918f9a --- /dev/null +++ b/testsuite/serd-tests/bad/bad-08.ttl @@ -0,0 +1,2 @@ +@keywords something. +# @keywords is not in turtle diff --git a/testsuite/serd-tests/bad/bad-09.ttl b/testsuite/serd-tests/bad/bad-09.ttl new file mode 100644 index 00000000..d2d76c13 --- /dev/null +++ b/testsuite/serd-tests/bad/bad-09.ttl @@ -0,0 +1,3 @@ +# implies is not in turtle +@prefix : . +:a => :b . diff --git a/testsuite/serd-tests/bad/bad-10.ttl b/testsuite/serd-tests/bad/bad-10.ttl new file mode 100644 index 00000000..b5ef1f55 --- /dev/null +++ b/testsuite/serd-tests/bad/bad-10.ttl @@ -0,0 +1,3 @@ +# equivalence is not in turtle +@prefix : . +:a = :b . diff --git a/testsuite/serd-tests/bad/bad-11.ttl b/testsuite/serd-tests/bad/bad-11.ttl new file mode 100644 index 00000000..09e7b6a0 --- /dev/null +++ b/testsuite/serd-tests/bad/bad-11.ttl @@ -0,0 +1,3 @@ +# @forAll is not in turtle +@prefix : . +@forAll :x . diff --git a/testsuite/serd-tests/bad/bad-12.ttl b/testsuite/serd-tests/bad/bad-12.ttl new file mode 100644 index 00000000..cef48ff8 --- /dev/null +++ b/testsuite/serd-tests/bad/bad-12.ttl @@ -0,0 +1,3 @@ +# @forSome is not in turtle +@prefix : . +@forSome :x . diff --git a/testsuite/serd-tests/bad/bad-13.ttl b/testsuite/serd-tests/bad/bad-13.ttl new file mode 100644 index 00000000..91accf2b --- /dev/null +++ b/testsuite/serd-tests/bad/bad-13.ttl @@ -0,0 +1,3 @@ +# <= is not in turtle +@prefix : . +:a <= :b . diff --git a/testsuite/serd-tests/bad/bad-14.ttl b/testsuite/serd-tests/bad/bad-14.ttl new file mode 100644 index 00000000..f4a7acf5 --- /dev/null +++ b/testsuite/serd-tests/bad/bad-14.ttl @@ -0,0 +1,6 @@ +# Test long literals with missing end +@prefix : . +:a :b """a long + literal +with +newlines diff --git a/testsuite/serd-tests/bad/bad-base.ttl b/testsuite/serd-tests/bad/bad-base.ttl new file mode 100644 index 00000000..85421999 --- /dev/null +++ b/testsuite/serd-tests/bad/bad-base.ttl @@ -0,0 +1 @@ +@base "I'm quite certain this is not a URI" . \ No newline at end of file diff --git a/testsuite/serd-tests/bad/bad-blank-syntax.ttl b/testsuite/serd-tests/bad/bad-blank-syntax.ttl new file mode 100644 index 00000000..6c7117a2 --- /dev/null +++ b/testsuite/serd-tests/bad/bad-blank-syntax.ttl @@ -0,0 +1 @@ + _|invalid . diff --git a/testsuite/serd-tests/bad/bad-blank.ttl b/testsuite/serd-tests/bad/bad-blank.ttl new file mode 100644 index 00000000..a6543f2d --- /dev/null +++ b/testsuite/serd-tests/bad/bad-blank.ttl @@ -0,0 +1,3 @@ +@prefix eg: . + +_:.bad a eg:Thing . \ No newline at end of file diff --git a/testsuite/serd-tests/bad/bad-bom.ttl b/testsuite/serd-tests/bad/bad-bom.ttl new file mode 100644 index 00000000..67993884 --- /dev/null +++ b/testsuite/serd-tests/bad/bad-bom.ttl @@ -0,0 +1,3 @@ +ï»´# This file starts with the first two bytes of the UTF-8 Byte Order Mark + + a . diff --git a/testsuite/serd-tests/bad/bad-char-in-local.ttl b/testsuite/serd-tests/bad/bad-char-in-local.ttl new file mode 100644 index 00000000..973cc0a2 --- /dev/null +++ b/testsuite/serd-tests/bad/bad-char-in-local.ttl @@ -0,0 +1,3 @@ +@prefix eg: . + +eg:†bad . diff --git a/testsuite/serd-tests/bad/bad-char-in-prefix.ttl b/testsuite/serd-tests/bad/bad-char-in-prefix.ttl new file mode 100644 index 00000000..b54e0274 --- /dev/null +++ b/testsuite/serd-tests/bad/bad-char-in-prefix.ttl @@ -0,0 +1 @@ +bad†:s . diff --git a/testsuite/serd-tests/bad/bad-char-in-uri.ttl b/testsuite/serd-tests/bad/bad-char-in-uri.ttl new file mode 100644 index 00000000..49f9c0d4 --- /dev/null +++ b/testsuite/serd-tests/bad/bad-char-in-uri.ttl @@ -0,0 +1 @@ +<ÿÿÿ://a.example/s> "\u0006!#[]\u007F" . diff --git a/testsuite/serd-tests/bad/bad-datatype-syntax.ttl b/testsuite/serd-tests/bad/bad-datatype-syntax.ttl new file mode 100644 index 00000000..541c25d4 --- /dev/null +++ b/testsuite/serd-tests/bad/bad-datatype-syntax.ttl @@ -0,0 +1 @@ + "value"^ . diff --git a/testsuite/serd-tests/bad/bad-datatype.ttl b/testsuite/serd-tests/bad/bad-datatype.ttl new file mode 100644 index 00000000..0dd6018f --- /dev/null +++ b/testsuite/serd-tests/bad/bad-datatype.ttl @@ -0,0 +1 @@ +<> "hello"^^"not-a-uri" . \ No newline at end of file diff --git a/testsuite/serd-tests/bad/bad-dot-after-subject.ttl b/testsuite/serd-tests/bad/bad-dot-after-subject.ttl new file mode 100644 index 00000000..e76e0ea2 --- /dev/null +++ b/testsuite/serd-tests/bad/bad-dot-after-subject.ttl @@ -0,0 +1 @@ + . . diff --git a/testsuite/serd-tests/bad/bad-dot-in-collection.ttl b/testsuite/serd-tests/bad/bad-dot-in-collection.ttl new file mode 100644 index 00000000..d2d35bc2 --- /dev/null +++ b/testsuite/serd-tests/bad/bad-dot-in-collection.ttl @@ -0,0 +1 @@ +[ (1. diff --git a/testsuite/serd-tests/bad/bad-eof-after-quotes.ttl b/testsuite/serd-tests/bad/bad-eof-after-quotes.ttl new file mode 100644 index 00000000..40e429cb --- /dev/null +++ b/testsuite/serd-tests/bad/bad-eof-after-quotes.ttl @@ -0,0 +1,3 @@ +@prefix eg: . + +<> eg:comment "" \ No newline at end of file diff --git a/testsuite/serd-tests/bad/bad-eof-at-string-start.ttl b/testsuite/serd-tests/bad/bad-eof-at-string-start.ttl new file mode 100644 index 00000000..93d20bcc --- /dev/null +++ b/testsuite/serd-tests/bad/bad-eof-at-string-start.ttl @@ -0,0 +1,3 @@ +@prefix eg: . + +<> eg:comment " \ No newline at end of file diff --git a/testsuite/serd-tests/bad/bad-eof-in-blank.ttl b/testsuite/serd-tests/bad/bad-eof-in-blank.ttl new file mode 100644 index 00000000..8cf4ee84 --- /dev/null +++ b/testsuite/serd-tests/bad/bad-eof-in-blank.ttl @@ -0,0 +1,3 @@ +@prefix eg: . + +<> eg:thing [ eg:comment "Thing" \ No newline at end of file diff --git a/testsuite/serd-tests/bad/bad-eof-in-escape.ttl b/testsuite/serd-tests/bad/bad-eof-in-escape.ttl new file mode 100644 index 00000000..24b4eec6 --- /dev/null +++ b/testsuite/serd-tests/bad/bad-eof-in-escape.ttl @@ -0,0 +1,3 @@ +@prefix eg: . + +<> eg:comment """\uA \ No newline at end of file diff --git a/testsuite/serd-tests/bad/bad-eof-in-lang-suffix.ttl b/testsuite/serd-tests/bad/bad-eof-in-lang-suffix.ttl new file mode 100644 index 00000000..f46a7763 --- /dev/null +++ b/testsuite/serd-tests/bad/bad-eof-in-lang-suffix.ttl @@ -0,0 +1,3 @@ +@prefix eg: . + +<> eg:comment "That ain't no language"@en-x \ No newline at end of file diff --git a/testsuite/serd-tests/bad/bad-eof-in-lang.ttl b/testsuite/serd-tests/bad/bad-eof-in-lang.ttl new file mode 100644 index 00000000..bfdffd02 --- /dev/null +++ b/testsuite/serd-tests/bad/bad-eof-in-lang.ttl @@ -0,0 +1,3 @@ +@prefix eg: . + +<> eg:comment "That ain't no language"@a \ No newline at end of file diff --git a/testsuite/serd-tests/bad/bad-eof-in-list.ttl b/testsuite/serd-tests/bad/bad-eof-in-list.ttl new file mode 100644 index 00000000..13eeb88d --- /dev/null +++ b/testsuite/serd-tests/bad/bad-eof-in-list.ttl @@ -0,0 +1,3 @@ +@prefix eg: . + +<> eg:thing ( eg:car eg:cdr \ No newline at end of file diff --git a/testsuite/serd-tests/bad/bad-eof-in-long-string.ttl b/testsuite/serd-tests/bad/bad-eof-in-long-string.ttl new file mode 100644 index 00000000..2ef179a8 --- /dev/null +++ b/testsuite/serd-tests/bad/bad-eof-in-long-string.ttl @@ -0,0 +1,3 @@ +@prefix eg: . + +<> eg:comment """This is the string that never ends \ No newline at end of file diff --git a/testsuite/serd-tests/bad/bad-eof-in-object-list.ttl b/testsuite/serd-tests/bad/bad-eof-in-object-list.ttl new file mode 100644 index 00000000..9bbcd17a --- /dev/null +++ b/testsuite/serd-tests/bad/bad-eof-in-object-list.ttl @@ -0,0 +1,2 @@ +@prefix eg: . +<> eg:p eg:o , \ No newline at end of file diff --git a/testsuite/serd-tests/bad/bad-eof-in-object-list2.ttl b/testsuite/serd-tests/bad/bad-eof-in-object-list2.ttl new file mode 100644 index 00000000..9186fb9f --- /dev/null +++ b/testsuite/serd-tests/bad/bad-eof-in-object-list2.ttl @@ -0,0 +1,2 @@ +@prefix eg: . +<> eg:p eg:o ; eg:p1 eg:o2 , \ No newline at end of file diff --git a/testsuite/serd-tests/bad/bad-eof-in-predicate-list.ttl b/testsuite/serd-tests/bad/bad-eof-in-predicate-list.ttl new file mode 100644 index 00000000..eab5b05b --- /dev/null +++ b/testsuite/serd-tests/bad/bad-eof-in-predicate-list.ttl @@ -0,0 +1,2 @@ +@prefix eg: . +<> eg:p eg:o ; \ No newline at end of file diff --git a/testsuite/serd-tests/bad/bad-eof-in-string.ttl b/testsuite/serd-tests/bad/bad-eof-in-string.ttl new file mode 100644 index 00000000..bb6e817f --- /dev/null +++ b/testsuite/serd-tests/bad/bad-eof-in-string.ttl @@ -0,0 +1,3 @@ +@prefix eg: . + +<> eg:comment "This is the string that never ends \ No newline at end of file diff --git a/testsuite/serd-tests/bad/bad-eof-in-triple-quote.ttl b/testsuite/serd-tests/bad/bad-eof-in-triple-quote.ttl new file mode 100644 index 00000000..fb935441 --- /dev/null +++ b/testsuite/serd-tests/bad/bad-eof-in-triple-quote.ttl @@ -0,0 +1,3 @@ +@prefix eg: . + +<> eg:comment """Hello"" \ No newline at end of file diff --git a/testsuite/serd-tests/bad/bad-eof-in-uri-scheme.nt b/testsuite/serd-tests/bad/bad-eof-in-uri-scheme.nt new file mode 100644 index 00000000..de892dcf --- /dev/null +++ b/testsuite/serd-tests/bad/bad-eof-in-uri-scheme.nt @@ -0,0 +1 @@ + . + +<> eg:uri """\!""" . diff --git a/testsuite/serd-tests/bad/bad-ext-namedblank-op.ttl b/testsuite/serd-tests/bad/bad-ext-namedblank-op.ttl new file mode 100644 index 00000000..5e648393 --- /dev/null +++ b/testsuite/serd-tests/bad/bad-ext-namedblank-op.ttl @@ -0,0 +1,3 @@ +@prefix eg: . + +eg:s eg:p [ =: _:o ; eg:name "o" ] . diff --git a/testsuite/serd-tests/bad/bad-graph-blank-label.trig b/testsuite/serd-tests/bad/bad-graph-blank-label.trig new file mode 100644 index 00000000..8b329db3 --- /dev/null +++ b/testsuite/serd-tests/bad/bad-graph-blank-label.trig @@ -0,0 +1,3 @@ +PREFIX : + +GRAPH [ :p :o ] { :s :p :o } diff --git a/testsuite/serd-tests/bad/bad-hex-digit.ttl b/testsuite/serd-tests/bad/bad-hex-digit.ttl new file mode 100644 index 00000000..85816ced --- /dev/null +++ b/testsuite/serd-tests/bad/bad-hex-digit.ttl @@ -0,0 +1 @@ + "\uABCG" . diff --git a/testsuite/serd-tests/bad/bad-id-clash.ttl b/testsuite/serd-tests/bad/bad-id-clash.ttl new file mode 100644 index 00000000..c1536840 --- /dev/null +++ b/testsuite/serd-tests/bad/bad-id-clash.ttl @@ -0,0 +1,2 @@ +_:b1 a _:b2 . +_:b2 a _:B1 . \ No newline at end of file diff --git a/testsuite/serd-tests/bad/bad-lang.ttl b/testsuite/serd-tests/bad/bad-lang.ttl new file mode 100644 index 00000000..01e04328 --- /dev/null +++ b/testsuite/serd-tests/bad/bad-lang.ttl @@ -0,0 +1 @@ +<> "hello"@\bad . \ No newline at end of file diff --git a/testsuite/serd-tests/bad/bad-list.ttl b/testsuite/serd-tests/bad/bad-list.ttl new file mode 100644 index 00000000..5606658e --- /dev/null +++ b/testsuite/serd-tests/bad/bad-list.ttl @@ -0,0 +1 @@ +<> , invalid . \ No newline at end of file diff --git a/testsuite/serd-tests/bad/bad-list2.ttl b/testsuite/serd-tests/bad/bad-list2.ttl new file mode 100644 index 00000000..18584209 --- /dev/null +++ b/testsuite/serd-tests/bad/bad-list2.ttl @@ -0,0 +1,3 @@ +@prefix eg: . + +<> eg:thing ( . \ No newline at end of file diff --git a/testsuite/serd-tests/bad/bad-long-literal-in-list.ttl b/testsuite/serd-tests/bad/bad-long-literal-in-list.ttl new file mode 100644 index 00000000..f10b4c3d --- /dev/null +++ b/testsuite/serd-tests/bad/bad-long-literal-in-list.ttl @@ -0,0 +1 @@ +<> ("""") . \ No newline at end of file diff --git a/testsuite/serd-tests/bad/bad-missing-semi.ttl b/testsuite/serd-tests/bad/bad-missing-semi.ttl new file mode 100644 index 00000000..f8838805 --- /dev/null +++ b/testsuite/serd-tests/bad/bad-missing-semi.ttl @@ -0,0 +1,3 @@ +@prefix a: . +a:s1 a:p1 a:o1 +a:s2 a:p2 a:o2 . diff --git a/testsuite/serd-tests/bad/bad-missing-uri-scheme.nt b/testsuite/serd-tests/bad/bad-missing-uri-scheme.nt new file mode 100644 index 00000000..5d7bc724 --- /dev/null +++ b/testsuite/serd-tests/bad/bad-missing-uri-scheme.nt @@ -0,0 +1 @@ + . diff --git a/testsuite/serd-tests/bad/bad-misspelled-base.ttl b/testsuite/serd-tests/bad/bad-misspelled-base.ttl new file mode 100644 index 00000000..c8ae227a --- /dev/null +++ b/testsuite/serd-tests/bad/bad-misspelled-base.ttl @@ -0,0 +1 @@ +@baze eg: . diff --git a/testsuite/serd-tests/bad/bad-misspelled-prefix.ttl b/testsuite/serd-tests/bad/bad-misspelled-prefix.ttl new file mode 100644 index 00000000..8c9d57f6 --- /dev/null +++ b/testsuite/serd-tests/bad/bad-misspelled-prefix.ttl @@ -0,0 +1 @@ +@prefox eg: . diff --git a/testsuite/serd-tests/bad/bad-namespace.ttl b/testsuite/serd-tests/bad/bad-namespace.ttl new file mode 100644 index 00000000..0dd78d33 --- /dev/null +++ b/testsuite/serd-tests/bad/bad-namespace.ttl @@ -0,0 +1 @@ +@prefix eg: "what?" . \ No newline at end of file diff --git a/testsuite/serd-tests/bad/bad-ns.ttl b/testsuite/serd-tests/bad/bad-ns.ttl new file mode 100644 index 00000000..4410573e --- /dev/null +++ b/testsuite/serd-tests/bad/bad-ns.ttl @@ -0,0 +1 @@ +<> a badprefix:Thing . \ No newline at end of file diff --git a/testsuite/serd-tests/bad/bad-null-byte.ttl b/testsuite/serd-tests/bad/bad-null-byte.ttl new file mode 100644 index 0000000000000000000000000000000000000000..4f2cb404d574b8d6f44d398ef1aeef7b144b3d91 GIT binary patch literal 27 fcmZQ5C`wJstWdC0uu-;C(Bq1ZRj>l`DuGM@W!46> literal 0 HcmV?d00001 diff --git a/testsuite/serd-tests/bad/bad-num.ttl b/testsuite/serd-tests/bad/bad-num.ttl new file mode 100644 index 00000000..7685f0e0 --- /dev/null +++ b/testsuite/serd-tests/bad/bad-num.ttl @@ -0,0 +1 @@ +<> .hello . \ No newline at end of file diff --git a/testsuite/serd-tests/bad/bad-object.ttl b/testsuite/serd-tests/bad/bad-object.ttl new file mode 100644 index 00000000..9fc6da18 --- /dev/null +++ b/testsuite/serd-tests/bad/bad-object.ttl @@ -0,0 +1,3 @@ +@prefix eg: . + +eg:thing a four . diff --git a/testsuite/serd-tests/bad/bad-object2.ttl b/testsuite/serd-tests/bad/bad-object2.ttl new file mode 100644 index 00000000..9293d168 --- /dev/null +++ b/testsuite/serd-tests/bad/bad-object2.ttl @@ -0,0 +1,3 @@ +@prefix eg: . + +eg:thing a fives . diff --git a/testsuite/serd-tests/bad/bad-pn-escape.ttl b/testsuite/serd-tests/bad/bad-pn-escape.ttl new file mode 100644 index 00000000..2b363e89 --- /dev/null +++ b/testsuite/serd-tests/bad/bad-pn-escape.ttl @@ -0,0 +1,2 @@ +@prefix : . +:s :p :\a diff --git a/testsuite/serd-tests/bad/bad-prefix.ttl b/testsuite/serd-tests/bad/bad-prefix.ttl new file mode 100644 index 00000000..6c286355 --- /dev/null +++ b/testsuite/serd-tests/bad/bad-prefix.ttl @@ -0,0 +1 @@ +@prefix _invalid . diff --git a/testsuite/serd-tests/bad/bad-quote-in-uri.ttl b/testsuite/serd-tests/bad/bad-quote-in-uri.ttl new file mode 100644 index 00000000..30711241 --- /dev/null +++ b/testsuite/serd-tests/bad/bad-quote-in-uri.ttl @@ -0,0 +1 @@ + . \ No newline at end of file diff --git a/testsuite/serd-tests/bad/bad-semicolon-after-subject.ttl b/testsuite/serd-tests/bad/bad-semicolon-after-subject.ttl new file mode 100644 index 00000000..582c6ea0 --- /dev/null +++ b/testsuite/serd-tests/bad/bad-semicolon-after-subject.ttl @@ -0,0 +1 @@ + ; . diff --git a/testsuite/serd-tests/bad/bad-string.ttl b/testsuite/serd-tests/bad/bad-string.ttl new file mode 100644 index 00000000..0bdea42c --- /dev/null +++ b/testsuite/serd-tests/bad/bad-string.ttl @@ -0,0 +1 @@ +<> "hello \ No newline at end of file diff --git a/testsuite/serd-tests/bad/bad-subject.ttl b/testsuite/serd-tests/bad/bad-subject.ttl new file mode 100644 index 00000000..b98ea39b --- /dev/null +++ b/testsuite/serd-tests/bad/bad-subject.ttl @@ -0,0 +1 @@ +invalid.:thing a invalid.Thing . diff --git a/testsuite/serd-tests/bad/bad-uri-escape.ttl b/testsuite/serd-tests/bad/bad-uri-escape.ttl new file mode 100644 index 00000000..16c63754 --- /dev/null +++ b/testsuite/serd-tests/bad/bad-uri-escape.ttl @@ -0,0 +1 @@ + . diff --git a/testsuite/serd-tests/bad/bad-uri-scheme-start.nt b/testsuite/serd-tests/bad/bad-uri-scheme-start.nt new file mode 100644 index 00000000..cd3fd70f --- /dev/null +++ b/testsuite/serd-tests/bad/bad-uri-scheme-start.nt @@ -0,0 +1 @@ +<2http://example.org/s> . diff --git a/testsuite/serd-tests/bad/bad-uri-scheme.nt b/testsuite/serd-tests/bad/bad-uri-scheme.nt new file mode 100644 index 00000000..1329edcd --- /dev/null +++ b/testsuite/serd-tests/bad/bad-uri-scheme.nt @@ -0,0 +1 @@ + . diff --git a/testsuite/serd-tests/bad/bad-uri-truncated.nt b/testsuite/serd-tests/bad/bad-uri-truncated.nt new file mode 100644 index 00000000..22d29e4b --- /dev/null +++ b/testsuite/serd-tests/bad/bad-uri-truncated.nt @@ -0,0 +1 @@ + . +a:thing x a:Thing . \ No newline at end of file diff --git a/testsuite/serd-tests/bad/invalid-char-in-local.ttl b/testsuite/serd-tests/bad/invalid-char-in-local.ttl new file mode 100644 index 00000000..520c2404 --- /dev/null +++ b/testsuite/serd-tests/bad/invalid-char-in-local.ttl @@ -0,0 +1,3 @@ +@prefix eg: . + +eg:¿invalid . diff --git a/testsuite/serd-tests/bad/invalid-char-in-prefix.ttl b/testsuite/serd-tests/bad/invalid-char-in-prefix.ttl new file mode 100644 index 00000000..79547803 --- /dev/null +++ b/testsuite/serd-tests/bad/invalid-char-in-prefix.ttl @@ -0,0 +1 @@ +invalid¿:s . diff --git a/testsuite/serd-tests/bad/manifest.ttl b/testsuite/serd-tests/bad/manifest.ttl new file mode 100644 index 00000000..6199c1f2 --- /dev/null +++ b/testsuite/serd-tests/bad/manifest.ttl @@ -0,0 +1,449 @@ +@prefix mf: . +@prefix rdf: . +@prefix rdfs: . +@prefix rdft: . + +<> + rdf:type mf:Manifest ; + rdfs:comment "Serd bad syntax test cases" ; + mf:entries ( + <#bad-00> + <#bad-01> + <#bad-02> + <#bad-03> + <#bad-04> + <#bad-05> + <#bad-06> + <#bad-07> + <#bad-08> + <#bad-09> + <#bad-10> + <#bad-11> + <#bad-12> + <#bad-13> + <#bad-14> + <#bad-base> + <#bad-blank> + <#bad-blank-syntax> + <#bad-bom> + <#bad-char-in-local> + <#bad-char-in-prefix> + <#bad-char-in-uri> + <#bad-datatype-syntax> + <#bad-datatype> + <#bad-dot-after-subject> + <#bad-dot-in-collection> + <#bad-eof-after-quotes> + <#bad-eof-at-string-start> + <#bad-eof-in-blank> + <#bad-eof-in-escape> + <#bad-eof-in-lang-suffix> + <#bad-eof-in-lang> + <#bad-eof-in-list> + <#bad-eof-in-object-list2> + <#bad-eof-in-object-list> + <#bad-eof-in-predicate-list> + <#bad-eof-in-long-string> + <#bad-eof-in-string> + <#bad-eof-in-triple-quote> + <#bad-eof-in-uri> + <#bad-eof-in-uri-scheme> + <#bad-escape> + <#bad-ext-namedblank-op> + <#bad-graph-blank-label> + <#bad-hex-digit> + # Not in the spec <#bad-id-clash> + <#bad-lang> + <#bad-list2> + <#bad-list> + <#bad-long-literal-in-list> + <#bad-missing-semi> + <#bad-misspelled-base> + <#bad-misspelled-prefix> + <#bad-namespace> + <#bad-ns> + <#bad-null-byte> + <#bad-num> + <#bad-object2> + <#bad-object> + <#bad-pn-escape> + <#bad-prefix> + <#bad-quote-in-uri> + <#bad-semicolon-after-subject> + <#bad-string> + <#bad-subject> + <#bad-uri-escape> + <#bad-verb> + <#invalid-char-in-local> + <#invalid-char-in-prefix> + <#bad-missing-uri-scheme.nt> + <#bad-uri-scheme.nt> + <#bad-uri-scheme-start.nt> + <#bad-uri-truncated.nt> + ) . + +<#bad-00> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-00" ; + mf:action . + +<#bad-01> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-01" ; + mf:action . + +<#bad-02> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-02" ; + mf:action . + +<#bad-03> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-03" ; + mf:action . + +<#bad-04> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-04" ; + mf:action . + +<#bad-05> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-05" ; + mf:action . + +<#bad-06> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-06" ; + mf:action . + +<#bad-07> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-07" ; + mf:action . + +<#bad-08> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-08" ; + mf:action . + +<#bad-09> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-09" ; + mf:action . + +<#bad-10> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-10" ; + mf:action . + +<#bad-11> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-11" ; + mf:action . + +<#bad-12> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-12" ; + mf:action . + +<#bad-13> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-13" ; + mf:action . + +<#bad-14> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-14" ; + mf:action . + +<#bad-base> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-base" ; + mf:action . + +<#bad-blank> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-blank" ; + mf:action . + +<#bad-blank-syntax> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-blank-syntax" ; + mf:action . + +<#bad-bom> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-bom" ; + mf:action . + +<#bad-char-in-local> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-char-in-local" ; + mf:action . + +<#bad-char-in-prefix> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-char-in-prefix" ; + mf:action . + +<#bad-char-in-uri> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-char-in-uri" ; + mf:action . + +<#bad-datatype-syntax> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-datatype-syntax" ; + mf:action . + +<#bad-datatype> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-datatype" ; + mf:action . + +<#bad-dot-after-subject> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-dot-after-subject" ; + mf:action . + +<#bad-dot-in-collection> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-dot-in-collection" ; + mf:action . + +<#bad-eof-after-quotes> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-eof-after-quotes" ; + mf:action . + +<#bad-eof-at-string-start> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-eof-at-string-start" ; + mf:action . + +<#bad-eof-in-blank> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-eof-in-blank" ; + mf:action . + +<#bad-eof-in-escape> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-eof-in-escape" ; + mf:action . + +<#bad-eof-in-lang-suffix> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-eof-in-lang-suffix" ; + mf:action . + +<#bad-eof-in-lang> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-eof-in-lang" ; + mf:action . + +<#bad-eof-in-list> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-eof-in-list" ; + mf:action . + +<#bad-eof-in-object-list2> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-eof-in-object-list2" ; + mf:action . + +<#bad-eof-in-object-list> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-eof-in-object-list" ; + mf:action . + +<#bad-eof-in-predicate-list> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-eof-in-predicate-list" ; + mf:action . + +<#bad-eof-in-long-string> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-eof-in-long-string" ; + mf:action . + +<#bad-eof-in-string> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-eof-in-string" ; + mf:action . + +<#bad-eof-in-triple-quote> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-eof-in-triple-quote" ; + mf:action . + +<#bad-eof-in-uri> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-eof-in-uri" ; + mf:action . + +<#bad-eof-in-uri-scheme> + rdf:type rdft:TestNTriplesNegativeSyntax ; + mf:name "bad-eof-in-uri-scheme" ; + mf:action . + +<#bad-escape> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-escape" ; + mf:action . + +<#bad-ext-namedblank-op> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-ext-namedblank-op" ; + mf:action . + +<#bad-graph-blank-label> + rdf:type rdft:TestTrigNegativeSyntax ; + mf:name "bad-graph-blank-label" ; + mf:action . + +<#bad-hex-digit> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-hex-digit" ; + mf:action . + +<#bad-id-clash> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-id-clash" ; + mf:action . + +<#bad-lang> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-lang" ; + mf:action . + +<#bad-list2> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-list2" ; + mf:action . + +<#bad-list> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-list" ; + mf:action . + +<#bad-long-literal-in-list> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-long-literal-in-list" ; + mf:action . + +<#bad-missing-semi> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-missing-semi" ; + mf:action . + +<#bad-misspelled-base> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-misspelled-base" ; + mf:action . + +<#bad-misspelled-prefix> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-misspelled-prefix" ; + mf:action . + +<#bad-namespace> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-namespace" ; + mf:action . + +<#bad-ns> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-ns" ; + mf:action . + +<#bad-null-byte> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-null-byte" ; + mf:action . + +<#bad-num> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-num" ; + mf:action . + +<#bad-object2> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-object2" ; + mf:action . + +<#bad-object> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-object" ; + mf:action . + +<#bad-pn-escape> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-pn-escape" ; + mf:action . + +<#bad-prefix> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-prefix" ; + mf:action . + +<#bad-quote-in-uri> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-quote-in-uri" ; + mf:action . + +<#bad-semicolon-after-subject> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-semicolon-after-subject" ; + mf:action . + +<#bad-string> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-string" ; + mf:action . + +<#bad-subject> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-subject" ; + mf:action . + +<#bad-uri-escape> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-uri-escape" ; + mf:action . + +<#bad-verb> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad-verb" ; + mf:action . + +<#invalid-char-in-local> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "invalid-char-in-local" ; + mf:action . + +<#invalid-char-in-prefix> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "invalid-char-in-prefix" ; + mf:action . + +<#bad-missing-uri-scheme.nt> + rdf:type rdft:TestNTriplesNegativeSyntax ; + mf:name "bad-missing-uri-scheme" ; + mf:action . + +<#bad-uri-scheme.nt> + rdf:type rdft:TestNTriplesNegativeSyntax ; + mf:name "bad-uri-scheme" ; + mf:action . + +<#bad-uri-scheme-start.nt> + rdf:type rdft:TestNTriplesNegativeSyntax ; + mf:name "bad-uri-scheme-start" ; + mf:action . + +<#bad-uri-truncated.nt> + rdf:type rdft:TestNTriplesNegativeSyntax ; + mf:name "bad-uri-truncated" ; + mf:action . + diff --git a/testsuite/serd-tests/good/README.txt b/testsuite/serd-tests/good/README.txt new file mode 100644 index 00000000..37460e60 --- /dev/null +++ b/testsuite/serd-tests/good/README.txt @@ -0,0 +1,20 @@ +These are the tests for the Turtle Terse RDF Triple Language +that must be passed by conformant systems. See + http://www.dajobe.org/2004/01/turtle/ +for the full conformance information. + +The format is a set of good tests and bad tests. + +Good tests are a pair of files: + abc.ttl abc.out +which are the input Turtle file and the expected output RDF triples, +written in N-Triples. + +bad tests are of the form + bad-XX.ttl +which must fail. + +The tests should be performed with an assumed base URI +of http://www.w3.org/2001/sw/DataAccess/df1/tests/ + +Dave diff --git a/testsuite/serd-tests/good/UTF-8.nt b/testsuite/serd-tests/good/UTF-8.nt new file mode 100644 index 00000000..c6c67bcb --- /dev/null +++ b/testsuite/serd-tests/good/UTF-8.nt @@ -0,0 +1,2 @@ + "\nUTF-8 encoded sample plain-text file\n\u203E\u203E\u203E\u203E\u203E\u203E\u203E\u203E\u203E\u203E\u203E\u203E\u203E\u203E\u203E\u203E\u203E\u203E\u203E\u203E\u203E\u203E\u203E\u203E\u203E\u203E\u203E\u203E\u203E\u203E\u203E\u203E\u203E\u203E\u203E\u203E\n\nMarkus Kuhn [\u02C8ma\u02B3k\u028As ku\u02D0n] \u2014 2002-07-25\n\n\nThe ASCII compatible UTF-8 encoding used in this plain-text file\nis defined in Unicode, ISO 10646-1, and RFC 2279.\n\n\nUsing Unicode/UTF-8, you can write in emails and source code things such as\n\nMathematics and sciences:\n\n \u222E E\u22C5da = Q, n \u2192 \u221E, \u2211 f(i) = \u220F g(i), \u23A7\u23A1\u239B\u250C\u2500\u2500\u2500\u2500\u2500\u2510\u239E\u23A4\u23AB\n \u23AA\u23A2\u239C\u2502a\u00B2+b\u00B3 \u239F\u23A5\u23AA\n \u2200x\u2208\u211D: \u2308x\u2309 = \u2212\u230A\u2212x\u230B, \u03B1 \u2227 \u00AC\u03B2 = \u00AC(\u00AC\u03B1 \u2228 \u03B2), \u23AA\u23A2\u239C\u2502\u2500\u2500\u2500\u2500\u2500 \u239F\u23A5\u23AA\n \u23AA\u23A2\u239C\u23B7 c\u2088 \u239F\u23A5\u23AA\n \u2115 \u2286 \u2115\u2080 \u2282 \u2124 \u2282 \u211A \u2282 \u211D \u2282 \u2102, \u23A8\u23A2\u239C \u239F\u23A5\u23AC\n \u23AA\u23A2\u239C \u221E \u239F\u23A5\u23AA\n \u22A5 < a \u2260 b \u2261 c \u2264 d \u226A \u22A4 \u21D2 (\u27E6A\u27E7 \u21D4 \u27EAB\u27EB), \u23AA\u23A2\u239C \u23B2 \u239F\u23A5\u23AA\n \u23AA\u23A2\u239C \u23B3a\u2071-b\u2071\u239F\u23A5\u23AA\n 2H\u2082 + O\u2082 \u21CC 2H\u2082O, R = 4.7 k\u03A9, \u2300 200 mm \u23A9\u23A3\u239Di=1 \u23A0\u23A6\u23AD\n\nLinguistics and dictionaries:\n\n \u00F0i \u0131nt\u0259\u02C8n\u00E6\u0283\u0259n\u0259l f\u0259\u02C8n\u025Bt\u0131k \u0259so\u028Asi\u02C8e\u0131\u0283n\n Y [\u02C8\u028Fpsil\u0254n], Yen [j\u025Bn], Yoga [\u02C8jo\u02D0g\u0251]\n\nAPL:\n\n ((V\u2373V)=\u2373\u2374V)/V\u2190,V \u2337\u2190\u2373\u2192\u2374\u2206\u2207\u2283\u203E\u234E\u2355\u2308\n\nNicer typography in plain text files:\n\n \u2554\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2557\n \u2551 \u2551\n \u2551 \u2022 \u2018single\u2019 and \u201Cdouble\u201D quotes \u2551\n \u2551 \u2551\n \u2551 \u2022 Curly apostrophes: \u201CWe\u2019ve been here\u201D \u2551\n \u2551 \u2551\n \u2551 \u2022 Latin-1 apostrophe and accents: '\u00B4` \u2551\n \u2551 \u2551\n \u2551 \u2022 \u201Adeutsche\u2018 \u201EAnf\u00FChrungszeichen\u201C \u2551\n \u2551 \u2551\n \u2551 \u2022 \u2020, \u2021, \u2030, \u2022, 3\u20134, \u2014, \u22125/+5, \u2122, \u2026 \u2551\n \u2551 \u2551\n \u2551 \u2022 ASCII safety test: 1lI|, 0OD, 8B \u2551\n \u2551 \u256D\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u256E \u2551\n \u2551 \u2022 the euro symbol: \u2502 14.95 \u20AC \u2502 \u2551\n \u2551 \u2570\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u256F \u2551\n \u255A\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u255D\n\nCombining characters:\n\n STARG\u039B\u030ATE SG-1, a = v\u0307 = r\u0308, a\u20D1 \u22A5 b\u20D1\n\nGreek (in Polytonic):\n\n The Greek anthem:\n\n \u03A3\u1F72 \u03B3\u03BD\u03C9\u03C1\u1F77\u03B6\u03C9 \u1F00\u03C0\u1F78 \u03C4\u1F74\u03BD \u03BA\u1F79\u03C8\u03B7\n \u03C4\u03BF\u1FE6 \u03C3\u03C0\u03B1\u03B8\u03B9\u03BF\u1FE6 \u03C4\u1F74\u03BD \u03C4\u03C1\u03BF\u03BC\u03B5\u03C1\u1F75,\n \u03C3\u1F72 \u03B3\u03BD\u03C9\u03C1\u1F77\u03B6\u03C9 \u1F00\u03C0\u1F78 \u03C4\u1F74\u03BD \u1F44\u03C8\u03B7\n \u03C0\u03BF\u1F7A \u03BC\u1F72 \u03B2\u1F77\u03B1 \u03BC\u03B5\u03C4\u03C1\u1F71\u03B5\u03B9 \u03C4\u1F74 \u03B3\u1FC6.\n\n \u1FBF\u0391\u03C0\u1FBF \u03C4\u1F70 \u03BA\u1F79\u03BA\u03BA\u03B1\u03BB\u03B1 \u03B2\u03B3\u03B1\u03BB\u03BC\u1F73\u03BD\u03B7\n \u03C4\u1FF6\u03BD \u1FFE\u0395\u03BB\u03BB\u1F75\u03BD\u03C9\u03BD \u03C4\u1F70 \u1F31\u03B5\u03C1\u1F71\n \u03BA\u03B1\u1F76 \u03C3\u1F70\u03BD \u03C0\u03C1\u1FF6\u03C4\u03B1 \u1F00\u03BD\u03B4\u03C1\u03B5\u03B9\u03C9\u03BC\u1F73\u03BD\u03B7\n \u03C7\u03B1\u1FD6\u03C1\u03B5, \u1F66 \u03C7\u03B1\u1FD6\u03C1\u03B5, \u1FBF\u0395\u03BB\u03B5\u03C5\u03B8\u03B5\u03C1\u03B9\u1F71!\n\n From a speech of Demosthenes in the 4th century BC:\n\n \u039F\u1F50\u03C7\u1F76 \u03C4\u03B1\u1F50\u03C4\u1F70 \u03C0\u03B1\u03C1\u1F77\u03C3\u03C4\u03B1\u03C4\u03B1\u1F77 \u03BC\u03BF\u03B9 \u03B3\u03B9\u03B3\u03BD\u1F7D\u03C3\u03BA\u03B5\u03B9\u03BD, \u1F66 \u1F04\u03BD\u03B4\u03C1\u03B5\u03C2 \u1FBF\u0391\u03B8\u03B7\u03BD\u03B1\u1FD6\u03BF\u03B9,\n \u1F45\u03C4\u03B1\u03BD \u03C4\u1FBF \u03B5\u1F30\u03C2 \u03C4\u1F70 \u03C0\u03C1\u1F71\u03B3\u03BC\u03B1\u03C4\u03B1 \u1F00\u03C0\u03BF\u03B2\u03BB\u1F73\u03C8\u03C9 \u03BA\u03B1\u1F76 \u1F45\u03C4\u03B1\u03BD \u03C0\u03C1\u1F78\u03C2 \u03C4\u03BF\u1F7A\u03C2\n \u03BB\u1F79\u03B3\u03BF\u03C5\u03C2 \u03BF\u1F53\u03C2 \u1F00\u03BA\u03BF\u1F7B\u03C9\u0387 \u03C4\u03BF\u1F7A\u03C2 \u03BC\u1F72\u03BD \u03B3\u1F70\u03C1 \u03BB\u1F79\u03B3\u03BF\u03C5\u03C2 \u03C0\u03B5\u03C1\u1F76 \u03C4\u03BF\u1FE6\n \u03C4\u03B9\u03BC\u03C9\u03C1\u1F75\u03C3\u03B1\u03C3\u03B8\u03B1\u03B9 \u03A6\u1F77\u03BB\u03B9\u03C0\u03C0\u03BF\u03BD \u1F41\u03C1\u1FF6 \u03B3\u03B9\u03B3\u03BD\u03BF\u03BC\u1F73\u03BD\u03BF\u03C5\u03C2, \u03C4\u1F70 \u03B4\u1F72 \u03C0\u03C1\u1F71\u03B3\u03BC\u03B1\u03C4\u1FBF\n \u03B5\u1F30\u03C2 \u03C4\u03BF\u1FE6\u03C4\u03BF \u03C0\u03C1\u03BF\u1F75\u03BA\u03BF\u03BD\u03C4\u03B1, \u1F65\u03C3\u03B8\u1FBF \u1F45\u03C0\u03C9\u03C2 \u03BC\u1F74 \u03C0\u03B5\u03B9\u03C3\u1F79\u03BC\u03B5\u03B8\u1FBF \u03B1\u1F50\u03C4\u03BF\u1F76\n \u03C0\u03C1\u1F79\u03C4\u03B5\u03C1\u03BF\u03BD \u03BA\u03B1\u03BA\u1FF6\u03C2 \u03C3\u03BA\u1F73\u03C8\u03B1\u03C3\u03B8\u03B1\u03B9 \u03B4\u1F73\u03BF\u03BD. \u03BF\u1F50\u03B4\u1F73\u03BD \u03BF\u1F56\u03BD \u1F04\u03BB\u03BB\u03BF \u03BC\u03BF\u03B9 \u03B4\u03BF\u03BA\u03BF\u1FE6\u03C3\u03B9\u03BD\n \u03BF\u1F31 \u03C4\u1F70 \u03C4\u03BF\u03B9\u03B1\u1FE6\u03C4\u03B1 \u03BB\u1F73\u03B3\u03BF\u03BD\u03C4\u03B5\u03C2 \u1F22 \u03C4\u1F74\u03BD \u1F51\u03C0\u1F79\u03B8\u03B5\u03C3\u03B9\u03BD, \u03C0\u03B5\u03C1\u1F76 \u1F27\u03C2 \u03B2\u03BF\u03C5\u03BB\u03B5\u1F7B\u03B5\u03C3\u03B8\u03B1\u03B9,\n \u03BF\u1F50\u03C7\u1F76 \u03C4\u1F74\u03BD \u03BF\u1F56\u03C3\u03B1\u03BD \u03C0\u03B1\u03C1\u03B9\u03C3\u03C4\u1F71\u03BD\u03C4\u03B5\u03C2 \u1F51\u03BC\u1FD6\u03BD \u1F01\u03BC\u03B1\u03C1\u03C4\u1F71\u03BD\u03B5\u03B9\u03BD. \u1F10\u03B3\u1F7C \u03B4\u1F73, \u1F45\u03C4\u03B9 \u03BC\u1F73\u03BD\n \u03C0\u03BF\u03C4\u1FBF \u1F10\u03BE\u1FC6\u03BD \u03C4\u1FC7 \u03C0\u1F79\u03BB\u03B5\u03B9 \u03BA\u03B1\u1F76 \u03C4\u1F70 \u03B1\u1F51\u03C4\u1FC6\u03C2 \u1F14\u03C7\u03B5\u03B9\u03BD \u1F00\u03C3\u03C6\u03B1\u03BB\u1FF6\u03C2 \u03BA\u03B1\u1F76 \u03A6\u1F77\u03BB\u03B9\u03C0\u03C0\u03BF\u03BD\n \u03C4\u03B9\u03BC\u03C9\u03C1\u1F75\u03C3\u03B1\u03C3\u03B8\u03B1\u03B9, \u03BA\u03B1\u1F76 \u03BC\u1F71\u03BB\u1FBF \u1F00\u03BA\u03C1\u03B9\u03B2\u1FF6\u03C2 \u03BF\u1F36\u03B4\u03B1\u0387 \u1F10\u03C0\u1FBF \u1F10\u03BC\u03BF\u1FE6 \u03B3\u1F71\u03C1, \u03BF\u1F50 \u03C0\u1F71\u03BB\u03B1\u03B9\n \u03B3\u1F73\u03B3\u03BF\u03BD\u03B5\u03BD \u03C4\u03B1\u1FE6\u03C4\u1FBF \u1F00\u03BC\u03C6\u1F79\u03C4\u03B5\u03C1\u03B1\u0387 \u03BD\u1FE6\u03BD \u03BC\u1F73\u03BD\u03C4\u03BF\u03B9 \u03C0\u1F73\u03C0\u03B5\u03B9\u03C3\u03BC\u03B1\u03B9 \u03C4\u03BF\u1FE6\u03B8\u1FBF \u1F31\u03BA\u03B1\u03BD\u1F78\u03BD\n \u03C0\u03C1\u03BF\u03BB\u03B1\u03B2\u03B5\u1FD6\u03BD \u1F21\u03BC\u1FD6\u03BD \u03B5\u1F36\u03BD\u03B1\u03B9 \u03C4\u1F74\u03BD \u03C0\u03C1\u1F7D\u03C4\u03B7\u03BD, \u1F45\u03C0\u03C9\u03C2 \u03C4\u03BF\u1F7A\u03C2 \u03C3\u03C5\u03BC\u03BC\u1F71\u03C7\u03BF\u03C5\u03C2\n \u03C3\u1F7D\u03C3\u03BF\u03BC\u03B5\u03BD. \u1F10\u1F70\u03BD \u03B3\u1F70\u03C1 \u03C4\u03BF\u1FE6\u03C4\u03BF \u03B2\u03B5\u03B2\u03B1\u1F77\u03C9\u03C2 \u1F51\u03C0\u1F71\u03C1\u03BE\u1FC3, \u03C4\u1F79\u03C4\u03B5 \u03BA\u03B1\u1F76 \u03C0\u03B5\u03C1\u1F76 \u03C4\u03BF\u1FE6\n \u03C4\u1F77\u03BD\u03B1 \u03C4\u03B9\u03BC\u03C9\u03C1\u1F75\u03C3\u03B5\u03C4\u03B1\u1F77 \u03C4\u03B9\u03C2 \u03BA\u03B1\u1F76 \u1F43\u03BD \u03C4\u03C1\u1F79\u03C0\u03BF\u03BD \u1F10\u03BE\u1F73\u03C3\u03C4\u03B1\u03B9 \u03C3\u03BA\u03BF\u03C0\u03B5\u1FD6\u03BD\u0387 \u03C0\u03C1\u1F76\u03BD \u03B4\u1F72\n \u03C4\u1F74\u03BD \u1F00\u03C1\u03C7\u1F74\u03BD \u1F40\u03C1\u03B8\u1FF6\u03C2 \u1F51\u03C0\u03BF\u03B8\u1F73\u03C3\u03B8\u03B1\u03B9, \u03BC\u1F71\u03C4\u03B1\u03B9\u03BF\u03BD \u1F21\u03B3\u03BF\u1FE6\u03BC\u03B1\u03B9 \u03C0\u03B5\u03C1\u1F76 \u03C4\u1FC6\u03C2\n \u03C4\u03B5\u03BB\u03B5\u03C5\u03C4\u1FC6\u03C2 \u1F41\u03BD\u03C4\u03B9\u03BD\u03BF\u1FE6\u03BD \u03C0\u03BF\u03B9\u03B5\u1FD6\u03C3\u03B8\u03B1\u03B9 \u03BB\u1F79\u03B3\u03BF\u03BD.\n\n \u0394\u03B7\u03BC\u03BF\u03C3\u03B8\u1F73\u03BD\u03BF\u03C5\u03C2, \u0393\u1FFD \u1FBF\u039F\u03BB\u03C5\u03BD\u03B8\u03B9\u03B1\u03BA\u1F78\u03C2\n\nGeorgian:\n\n From a Unicode conference invitation:\n\n \u10D2\u10D7\u10EE\u10DD\u10D5\u10D7 \u10D0\u10EE\u10DA\u10D0\u10D5\u10D4 \u10D2\u10D0\u10D8\u10D0\u10E0\u10DD\u10D7 \u10E0\u10D4\u10D2\u10D8\u10E1\u10E2\u10E0\u10D0\u10EA\u10D8\u10D0 Unicode-\u10D8\u10E1 \u10DB\u10D4\u10D0\u10D7\u10D4 \u10E1\u10D0\u10D4\u10E0\u10D7\u10D0\u10E8\u10DD\u10E0\u10D8\u10E1\u10DD\n \u10D9\u10DD\u10DC\u10E4\u10D4\u10E0\u10D4\u10DC\u10EA\u10D8\u10D0\u10D6\u10D4 \u10D3\u10D0\u10E1\u10D0\u10E1\u10EC\u10E0\u10D4\u10D1\u10D0\u10D3, \u10E0\u10DD\u10DB\u10D4\u10DA\u10D8\u10EA \u10D2\u10D0\u10D8\u10DB\u10D0\u10E0\u10D7\u10D4\u10D1\u10D0 10-12 \u10DB\u10D0\u10E0\u10E2\u10E1,\n \u10E5. \u10DB\u10D0\u10D8\u10DC\u10EA\u10E8\u10D8, \u10D2\u10D4\u10E0\u10DB\u10D0\u10DC\u10D8\u10D0\u10E8\u10D8. \u10D9\u10DD\u10DC\u10E4\u10D4\u10E0\u10D4\u10DC\u10EA\u10D8\u10D0 \u10E8\u10D4\u10F0\u10D9\u10E0\u10D4\u10D1\u10E1 \u10D4\u10E0\u10D7\u10D0\u10D3 \u10DB\u10E1\u10DD\u10E4\u10DA\u10D8\u10DD\u10E1\n \u10D4\u10E5\u10E1\u10DE\u10D4\u10E0\u10E2\u10D4\u10D1\u10E1 \u10D8\u10E1\u10D4\u10D7 \u10D3\u10D0\u10E0\u10D2\u10D4\u10D1\u10E8\u10D8 \u10E0\u10DD\u10D2\u10DD\u10E0\u10D8\u10EA\u10D0\u10D0 \u10D8\u10DC\u10E2\u10D4\u10E0\u10DC\u10D4\u10E2\u10D8 \u10D3\u10D0 Unicode-\u10D8,\n \u10D8\u10DC\u10E2\u10D4\u10E0\u10DC\u10D0\u10EA\u10D8\u10DD\u10DC\u10D0\u10DA\u10D8\u10D6\u10D0\u10EA\u10D8\u10D0 \u10D3\u10D0 \u10DA\u10DD\u10D9\u10D0\u10DA\u10D8\u10D6\u10D0\u10EA\u10D8\u10D0, Unicode-\u10D8\u10E1 \u10D2\u10D0\u10DB\u10DD\u10E7\u10D4\u10DC\u10D4\u10D1\u10D0\n \u10DD\u10DE\u10D4\u10E0\u10D0\u10EA\u10D8\u10E3\u10DA \u10E1\u10D8\u10E1\u10E2\u10D4\u10DB\u10D4\u10D1\u10E1\u10D0, \u10D3\u10D0 \u10D2\u10D0\u10DB\u10DD\u10E7\u10D4\u10DC\u10D4\u10D1\u10D8\u10D7 \u10DE\u10E0\u10DD\u10D2\u10E0\u10D0\u10DB\u10D4\u10D1\u10E8\u10D8, \u10E8\u10E0\u10D8\u10E4\u10E2\u10D4\u10D1\u10E8\u10D8,\n \u10E2\u10D4\u10E5\u10E1\u10E2\u10D4\u10D1\u10D8\u10E1 \u10D3\u10D0\u10DB\u10E3\u10E8\u10D0\u10D5\u10D4\u10D1\u10D0\u10E1\u10D0 \u10D3\u10D0 \u10DB\u10E0\u10D0\u10D5\u10D0\u10DA\u10D4\u10DC\u10DD\u10D5\u10D0\u10DC \u10D9\u10DD\u10DB\u10DE\u10D8\u10E3\u10E2\u10D4\u10E0\u10E3\u10DA \u10E1\u10D8\u10E1\u10E2\u10D4\u10DB\u10D4\u10D1\u10E8\u10D8.\n\nRussian:\n\n From a Unicode conference invitation:\n\n \u0417\u0430\u0440\u0435\u0433\u0438\u0441\u0442\u0440\u0438\u0440\u0443\u0439\u0442\u0435\u0441\u044C \u0441\u0435\u0439\u0447\u0430\u0441 \u043D\u0430 \u0414\u0435\u0441\u044F\u0442\u0443\u044E \u041C\u0435\u0436\u0434\u0443\u043D\u0430\u0440\u043E\u0434\u043D\u0443\u044E \u041A\u043E\u043D\u0444\u0435\u0440\u0435\u043D\u0446\u0438\u044E \u043F\u043E\n Unicode, \u043A\u043E\u0442\u043E\u0440\u0430\u044F \u0441\u043E\u0441\u0442\u043E\u0438\u0442\u0441\u044F 10-12 \u043C\u0430\u0440\u0442\u0430 1997 \u0433\u043E\u0434\u0430 \u0432 \u041C\u0430\u0439\u043D\u0446\u0435 \u0432 \u0413\u0435\u0440\u043C\u0430\u043D\u0438\u0438.\n \u041A\u043E\u043D\u0444\u0435\u0440\u0435\u043D\u0446\u0438\u044F \u0441\u043E\u0431\u0435\u0440\u0435\u0442 \u0448\u0438\u0440\u043E\u043A\u0438\u0439 \u043A\u0440\u0443\u0433 \u044D\u043A\u0441\u043F\u0435\u0440\u0442\u043E\u0432 \u043F\u043E \u0432\u043E\u043F\u0440\u043E\u0441\u0430\u043C \u0433\u043B\u043E\u0431\u0430\u043B\u044C\u043D\u043E\u0433\u043E\n \u0418\u043D\u0442\u0435\u0440\u043D\u0435\u0442\u0430 \u0438 Unicode, \u043B\u043E\u043A\u0430\u043B\u0438\u0437\u0430\u0446\u0438\u0438 \u0438 \u0438\u043D\u0442\u0435\u0440\u043D\u0430\u0446\u0438\u043E\u043D\u0430\u043B\u0438\u0437\u0430\u0446\u0438\u0438, \u0432\u043E\u043F\u043B\u043E\u0449\u0435\u043D\u0438\u044E \u0438\n \u043F\u0440\u0438\u043C\u0435\u043D\u0435\u043D\u0438\u044E Unicode \u0432 \u0440\u0430\u0437\u043B\u0438\u0447\u043D\u044B\u0445 \u043E\u043F\u0435\u0440\u0430\u0446\u0438\u043E\u043D\u043D\u044B\u0445 \u0441\u0438\u0441\u0442\u0435\u043C\u0430\u0445 \u0438 \u043F\u0440\u043E\u0433\u0440\u0430\u043C\u043C\u043D\u044B\u0445\n \u043F\u0440\u0438\u043B\u043E\u0436\u0435\u043D\u0438\u044F\u0445, \u0448\u0440\u0438\u0444\u0442\u0430\u0445, \u0432\u0435\u0440\u0441\u0442\u043A\u0435 \u0438 \u043C\u043D\u043E\u0433\u043E\u044F\u0437\u044B\u0447\u043D\u044B\u0445 \u043A\u043E\u043C\u043F\u044C\u044E\u0442\u0435\u0440\u043D\u044B\u0445 \u0441\u0438\u0441\u0442\u0435\u043C\u0430\u0445.\n\nThai (UCS Level 2):\n\n Excerpt from a poetry on The Romance of The Three Kingdoms (a Chinese\n classic 'San Gua'):\n\n [----------------------------|------------------------]\n \u0E4F \u0E41\u0E1C\u0E48\u0E19\u0E14\u0E34\u0E19\u0E2E\u0E31\u0E48\u0E19\u0E40\u0E2A\u0E37\u0E48\u0E2D\u0E21\u0E42\u0E17\u0E23\u0E21\u0E41\u0E2A\u0E19\u0E2A\u0E31\u0E07\u0E40\u0E27\u0E0A \u0E1E\u0E23\u0E30\u0E1B\u0E01\u0E40\u0E01\u0E28\u0E01\u0E2D\u0E07\u0E1A\u0E39\u0E4A\u0E01\u0E39\u0E49\u0E02\u0E36\u0E49\u0E19\u0E43\u0E2B\u0E21\u0E48\n \u0E2A\u0E34\u0E1A\u0E2A\u0E2D\u0E07\u0E01\u0E29\u0E31\u0E15\u0E23\u0E34\u0E22\u0E4C\u0E01\u0E48\u0E2D\u0E19\u0E2B\u0E19\u0E49\u0E32\u0E41\u0E25\u0E16\u0E31\u0E14\u0E44\u0E1B \u0E2A\u0E2D\u0E07\u0E2D\u0E07\u0E04\u0E4C\u0E44\u0E0B\u0E23\u0E49\u0E42\u0E07\u0E48\u0E40\u0E02\u0E25\u0E32\u0E40\u0E1A\u0E32\u0E1B\u0E31\u0E0D\u0E0D\u0E32\n \u0E17\u0E23\u0E07\u0E19\u0E31\u0E1A\u0E16\u0E37\u0E2D\u0E02\u0E31\u0E19\u0E17\u0E35\u0E40\u0E1B\u0E47\u0E19\u0E17\u0E35\u0E48\u0E1E\u0E36\u0E48\u0E07 \u0E1A\u0E49\u0E32\u0E19\u0E40\u0E21\u0E37\u0E2D\u0E07\u0E08\u0E36\u0E07\u0E27\u0E34\u0E1B\u0E23\u0E34\u0E15\u0E40\u0E1B\u0E47\u0E19\u0E19\u0E31\u0E01\u0E2B\u0E19\u0E32\n \u0E42\u0E2E\u0E08\u0E34\u0E4B\u0E19\u0E40\u0E23\u0E35\u0E22\u0E01\u0E17\u0E31\u0E1E\u0E17\u0E31\u0E48\u0E27\u0E2B\u0E31\u0E27\u0E40\u0E21\u0E37\u0E2D\u0E07\u0E21\u0E32 \u0E2B\u0E21\u0E32\u0E22\u0E08\u0E30\u0E06\u0E48\u0E32\u0E21\u0E14\u0E0A\u0E31\u0E48\u0E27\u0E15\u0E31\u0E27\u0E2A\u0E33\u0E04\u0E31\u0E0D\n \u0E40\u0E2B\u0E21\u0E37\u0E2D\u0E19\u0E02\u0E31\u0E1A\u0E44\u0E2A\u0E44\u0E25\u0E48\u0E40\u0E2A\u0E37\u0E2D\u0E08\u0E32\u0E01\u0E40\u0E04\u0E2B\u0E32 \u0E23\u0E31\u0E1A\u0E2B\u0E21\u0E32\u0E1B\u0E48\u0E32\u0E40\u0E02\u0E49\u0E32\u0E21\u0E32\u0E40\u0E25\u0E22\u0E2D\u0E32\u0E2A\u0E31\u0E0D\n \u0E1D\u0E48\u0E32\u0E22\u0E2D\u0E49\u0E2D\u0E07\u0E2D\u0E38\u0E49\u0E19\u0E22\u0E38\u0E41\u0E22\u0E01\u0E43\u0E2B\u0E49\u0E41\u0E15\u0E01\u0E01\u0E31\u0E19 \u0E43\u0E0A\u0E49\u0E2A\u0E32\u0E27\u0E19\u0E31\u0E49\u0E19\u0E40\u0E1B\u0E47\u0E19\u0E0A\u0E19\u0E27\u0E19\u0E0A\u0E37\u0E48\u0E19\u0E0A\u0E27\u0E19\u0E43\u0E08\n \u0E1E\u0E25\u0E31\u0E19\u0E25\u0E34\u0E09\u0E38\u0E22\u0E01\u0E38\u0E22\u0E01\u0E35\u0E01\u0E25\u0E31\u0E1A\u0E01\u0E48\u0E2D\u0E40\u0E2B\u0E15\u0E38 \u0E0A\u0E48\u0E32\u0E07\u0E2D\u0E32\u0E40\u0E1E\u0E28\u0E08\u0E23\u0E34\u0E07\u0E2B\u0E19\u0E32\u0E1F\u0E49\u0E32\u0E23\u0E49\u0E2D\u0E07\u0E44\u0E2B\u0E49\n \u0E15\u0E49\u0E2D\u0E07\u0E23\u0E1A\u0E23\u0E32\u0E06\u0E48\u0E32\u0E1F\u0E31\u0E19\u0E08\u0E19\u0E1A\u0E23\u0E23\u0E25\u0E31\u0E22 \u0E24\u0E45\u0E2B\u0E32\u0E43\u0E04\u0E23\u0E04\u0E49\u0E33\u0E0A\u0E39\u0E01\u0E39\u0E49\u0E1A\u0E23\u0E23\u0E25\u0E31\u0E07\u0E01\u0E4C \u0E2F\n\n (The above is a two-column text. If combining characters are handled\n correctly, the lines of the second column should be aligned with the\n | character above.)\n\nEthiopian:\n\n Proverbs in the Amharic language:\n\n \u1230\u121B\u12ED \u12A0\u12ED\u1273\u1228\u1235 \u1295\u1309\u1225 \u12A0\u12ED\u12A8\u1230\u1235\u1362\n \u1265\u120B \u12AB\u1208\u129D \u12A5\u1295\u12F0\u12A0\u1263\u1274 \u1260\u1246\u1218\u1320\u129D\u1362\n \u130C\u1325 \u12EB\u1208\u1264\u1271 \u1241\u121D\u1325\u1293 \u1290\u12CD\u1362\n \u12F0\u1200 \u1260\u1215\u120D\u1219 \u1245\u1264 \u1263\u12ED\u1320\u1323 \u1295\u1323\u1275 \u1260\u1308\u12F0\u1208\u12CD\u1362\n \u12E8\u12A0\u134D \u12C8\u1208\u121D\u1273 \u1260\u1245\u1264 \u12A0\u12ED\u1273\u123D\u121D\u1362\n \u12A0\u12ED\u1325 \u1260\u1260\u120B \u12F3\u12CB \u1270\u1218\u1273\u1362\n \u1232\u1270\u1228\u1309\u1219 \u12ED\u12F0\u1228\u130D\u1219\u1362\n \u1240\u1235 \u1260\u1240\u1235\u1365 \u12D5\u1295\u1241\u120B\u120D \u1260\u12A5\u130D\u1229 \u12ED\u1204\u12F3\u120D\u1362\n \u12F5\u122D \u1262\u12EB\u1265\u122D \u12A0\u1295\u1260\u1233 \u12EB\u1235\u122D\u1362\n \u1230\u12CD \u12A5\u1295\u12F0\u1264\u1271 \u12A5\u1295\u1305 \u12A5\u1295\u12F0 \u1309\u1228\u1264\u1271 \u12A0\u12ED\u1270\u12F3\u12F0\u122D\u121D\u1362\n \u12A5\u130D\u12DC\u122D \u12E8\u12A8\u1348\u1270\u12CD\u1295 \u1309\u122E\u122E \u1233\u12ED\u12D8\u130B\u12CD \u12A0\u12ED\u12F5\u122D\u121D\u1362\n \u12E8\u130E\u1228\u1264\u1275 \u120C\u1263\u1365 \u1262\u12EB\u12E9\u1275 \u12ED\u1235\u1245 \u1263\u12EB\u12E9\u1275 \u12EB\u1320\u120D\u1245\u1362\n \u1225\u122B \u12A8\u1218\u134D\u1273\u1275 \u120D\u1304\u1295 \u120B\u134B\u1273\u1275\u1362\n \u12D3\u1263\u12ED \u121B\u12F0\u122A\u12EB \u12E8\u1208\u12CD\u1365 \u130D\u1295\u12F5 \u12ED\u12DE \u12ED\u12DE\u122B\u120D\u1362\n \u12E8\u12A5\u1235\u120B\u121D \u12A0\u1308\u1229 \u1218\u12AB \u12E8\u12A0\u121E\u122B \u12A0\u1308\u1229 \u12CB\u122D\u12AB\u1362\n \u1270\u1295\u130B\u120E \u1262\u1270\u1349 \u1270\u1218\u120D\u1236 \u1263\u1349\u1362\n \u12C8\u12F3\u1305\u1205 \u121B\u122D \u1262\u1206\u1295 \u1328\u122D\u1235\u1205 \u12A0\u1275\u120B\u1230\u12CD\u1362\n \u12A5\u130D\u122D\u1205\u1295 \u1260\u134D\u122B\u123D\u1205 \u120D\u12AD \u12D8\u122D\u130B\u1362\n\nRunes:\n\n \u16BB\u16D6 \u16B3\u16B9\u16AB\u16A6 \u16A6\u16AB\u16CF \u16BB\u16D6 \u16D2\u16A2\u16DE\u16D6 \u16A9\u16BE \u16A6\u16AB\u16D7 \u16DA\u16AA\u16BE\u16DE\u16D6 \u16BE\u16A9\u16B1\u16A6\u16B9\u16D6\u16AA\u16B1\u16DE\u16A2\u16D7 \u16B9\u16C1\u16A6 \u16A6\u16AA \u16B9\u16D6\u16E5\u16AB\n\n (Old English, which transcribed into Latin reads 'He cwaeth that he\n bude thaem lande northweardum with that Westsae.' and means 'He said\n that he lived in the northern land near the Western Sea.')\n\nBraille:\n\n \u284C\u2801\u2827\u2811 \u283C\u2801\u2812 \u284D\u281C\u2807\u2811\u2839\u2830\u280E \u2863\u2815\u280C\n\n \u284D\u281C\u2807\u2811\u2839 \u283A\u2801\u280E \u2819\u2811\u2801\u2819\u2812 \u281E\u2815 \u2803\u2811\u281B\u2814 \u283A\u280A\u2839\u2832 \u2879\u283B\u2811 \u280A\u280E \u281D\u2815 \u2819\u2833\u2803\u281E\n \u2831\u2801\u281E\u2811\u2827\u283B \u2801\u2803\u2833\u281E \u2839\u2801\u281E\u2832 \u2879\u2811 \u2817\u2811\u281B\u280A\u280C\u283B \u2815\u280B \u2819\u280A\u280E \u2803\u2825\u2817\u280A\u2801\u2807 \u283A\u2801\u280E\n \u280E\u280A\u281B\u281D\u282B \u2803\u2839 \u2839\u2811 \u280A\u2807\u283B\u281B\u2839\u280D\u2801\u281D\u2802 \u2839\u2811 \u280A\u2807\u283B\u2805\u2802 \u2839\u2811 \u2825\u281D\u2819\u283B\u281E\u2801\u2805\u283B\u2802\n \u2801\u281D\u2819 \u2839\u2811 \u2821\u280A\u2811\u280B \u280D\u2833\u2817\u281D\u283B\u2832 \u284E\u280A\u2817\u2815\u2815\u281B\u2811 \u280E\u280A\u281B\u281D\u282B \u280A\u281E\u2832 \u2841\u281D\u2819\n \u284E\u280A\u2817\u2815\u2815\u281B\u2811\u2830\u280E \u281D\u2801\u280D\u2811 \u283A\u2801\u280E \u281B\u2815\u2815\u2819 \u2825\u280F\u2815\u281D \u2830\u2861\u2801\u281D\u281B\u2811\u2802 \u280B\u2815\u2817 \u2801\u281D\u2839\u2839\u2814\u281B \u2819\u2811\n \u2821\u2815\u280E\u2811 \u281E\u2815 \u280F\u2825\u281E \u2819\u280A\u280E \u2819\u2801\u281D\u2819 \u281E\u2815\u2832\n\n \u2855\u2807\u2819 \u284D\u281C\u2807\u2811\u2839 \u283A\u2801\u280E \u2801\u280E \u2819\u2811\u2801\u2819 \u2801\u280E \u2801 \u2819\u2815\u2815\u2817\u2824\u281D\u2801\u280A\u2807\u2832\n\n \u284D\u2814\u2819\u2816 \u284A \u2819\u2815\u281D\u2830\u281E \u280D\u2811\u2801\u281D \u281E\u2815 \u280E\u2801\u2839 \u2839\u2801\u281E \u284A \u2805\u281D\u282A\u2802 \u2815\u280B \u280D\u2839\n \u282A\u281D \u2805\u281D\u282A\u2807\u282B\u281B\u2811\u2802 \u2831\u2801\u281E \u2839\u283B\u2811 \u280A\u280E \u280F\u281C\u281E\u280A\u280A\u2825\u2807\u281C\u2807\u2839 \u2819\u2811\u2801\u2819 \u2801\u2803\u2833\u281E\n \u2801 \u2819\u2815\u2815\u2817\u2824\u281D\u2801\u280A\u2807\u2832 \u284A \u280D\u280A\u2823\u281E \u2819\u2801\u2827\u2811 \u2803\u2811\u2832 \u2814\u280A\u2807\u2814\u282B\u2802 \u280D\u2839\u280E\u2811\u2807\u280B\u2802 \u281E\u2815\n \u2817\u2811\u281B\u281C\u2819 \u2801 \u280A\u2815\u280B\u280B\u2814\u2824\u281D\u2801\u280A\u2807 \u2801\u280E \u2839\u2811 \u2819\u2811\u2801\u2819\u2811\u280C \u280F\u280A\u2811\u280A\u2811 \u2815\u280B \u280A\u2817\u2815\u281D\u280D\u2815\u281D\u281B\u283B\u2839\n \u2814 \u2839\u2811 \u281E\u2817\u2801\u2819\u2811\u2832 \u2843\u2825\u281E \u2839\u2811 \u283A\u280A\u280E\u2819\u2815\u280D \u2815\u280B \u2833\u2817 \u2801\u281D\u280A\u2811\u280C\u2815\u2817\u280E\n \u280A\u280E \u2814 \u2839\u2811 \u280E\u280A\u280D\u280A\u2807\u2811\u2806 \u2801\u281D\u2819 \u280D\u2839 \u2825\u281D\u2819\u2801\u2807\u2807\u282A\u282B \u2819\u2801\u281D\u2819\u280E\n \u2829\u2801\u2807\u2807 \u281D\u2815\u281E \u2819\u280A\u280C\u2825\u2817\u2803 \u280A\u281E\u2802 \u2815\u2817 \u2839\u2811 \u284A\u2833\u281D\u281E\u2817\u2839\u2830\u280E \u2819\u2815\u281D\u2811 \u280B\u2815\u2817\u2832 \u2879\u2833\n \u283A\u280A\u2807\u2807 \u2839\u283B\u2811\u280B\u2815\u2817\u2811 \u280F\u283B\u280D\u280A\u281E \u280D\u2811 \u281E\u2815 \u2817\u2811\u280F\u2811\u2801\u281E\u2802 \u2811\u280D\u280F\u2819\u2801\u281E\u280A\u280A\u2801\u2807\u2807\u2839\u2802 \u2839\u2801\u281E\n \u284D\u281C\u2807\u2811\u2839 \u283A\u2801\u280E \u2801\u280E \u2819\u2811\u2801\u2819 \u2801\u280E \u2801 \u2819\u2815\u2815\u2817\u2824\u281D\u2801\u280A\u2807\u2832\n\n (The first couple of paragraphs of \"A Christmas Carol\" by Dickens)\n\nCompact font selection example text:\n\n ABCDEFGHIJKLMNOPQRSTUVWXYZ /0123456789\n abcdefghijklmnopqrstuvwxyz \u00A3\u00A9\u00B5\u00C0\u00C6\u00D6\u00DE\u00DF\u00E9\u00F6\u00FF\n \u2013\u2014\u2018\u201C\u201D\u201E\u2020\u2022\u2026\u2030\u2122\u0153\u0160\u0178\u017E\u20AC \u0391\u0392\u0393\u0394\u03A9\u03B1\u03B2\u03B3\u03B4\u03C9 \u0410\u0411\u0412\u0413\u0414\u0430\u0431\u0432\u0433\u0434\n \u2200\u2202\u2208\u211D\u2227\u222A\u2261\u221E \u2191\u2197\u21A8\u21BB\u21E3 \u2510\u253C\u2554\u2558\u2591\u25BA\u263A\u2640 \uFB01\uFFFD\u2440\u2082\u1F20\u1E02\u04E5\u1E84\u0250\u02D0\u234E\u05D0\u0531\u10D0\n\nGreetings in various languages:\n\n Hello world, \u039A\u03B1\u03BB\u03B7\u03BC\u1F73\u03C1\u03B1 \u03BA\u1F79\u03C3\u03BC\u03B5, \u30B3\u30F3\u30CB\u30C1\u30CF\n\nBox drawing alignment tests: \u2588\n \u2589\n \u2554\u2550\u2550\u2566\u2550\u2550\u2557 \u250C\u2500\u2500\u252C\u2500\u2500\u2510 \u256D\u2500\u2500\u252C\u2500\u2500\u256E \u256D\u2500\u2500\u252C\u2500\u2500\u256E \u250F\u2501\u2501\u2533\u2501\u2501\u2513 \u250E\u2512\u250F\u2511 \u2577 \u257B \u250F\u252F\u2513 \u250C\u2530\u2510 \u258A \u2571\u2572\u2571\u2572\u2573\u2573\u2573\n \u2551\u250C\u2500\u2568\u2500\u2510\u2551 \u2502\u2554\u2550\u2567\u2550\u2557\u2502 \u2502\u2552\u2550\u256A\u2550\u2555\u2502 \u2502\u2553\u2500\u2541\u2500\u2556\u2502 \u2503\u250C\u2500\u2542\u2500\u2510\u2503 \u2517\u2543\u2544\u2519 \u2576\u253C\u2574\u257A\u254B\u2578\u2520\u253C\u2528 \u251D\u254B\u2525 \u258B \u2572\u2571\u2572\u2571\u2573\u2573\u2573\n \u2551\u2502\u2572 \u2571\u2502\u2551 \u2502\u2551 \u2551\u2502 \u2502\u2502 \u2502 \u2502\u2502 \u2502\u2551 \u2503 \u2551\u2502 \u2503\u2502 \u257F \u2502\u2503 \u250D\u2545\u2546\u2513 \u2575 \u2579 \u2517\u2537\u251B \u2514\u2538\u2518 \u258C \u2571\u2572\u2571\u2572\u2573\u2573\u2573\n \u2560\u2561 \u2573 \u255E\u2563 \u251C\u2562 \u255F\u2524 \u251C\u253C\u2500\u253C\u2500\u253C\u2524 \u251C\u256B\u2500\u2542\u2500\u256B\u2524 \u2523\u253F\u257E\u253C\u257C\u253F\u252B \u2515\u251B\u2516\u251A \u250C\u2504\u2504\u2510 \u254E \u250F\u2505\u2505\u2513 \u250B \u258D \u2572\u2571\u2572\u2571\u2573\u2573\u2573\n \u2551\u2502\u2571 \u2572\u2502\u2551 \u2502\u2551 \u2551\u2502 \u2502\u2502 \u2502 \u2502\u2502 \u2502\u2551 \u2503 \u2551\u2502 \u2503\u2502 \u257D \u2502\u2503 \u2591\u2591\u2592\u2592\u2593\u2593\u2588\u2588 \u250A \u2506 \u254E \u254F \u2507 \u250B \u258E\n \u2551\u2514\u2500\u2565\u2500\u2518\u2551 \u2502\u255A\u2550\u2564\u2550\u255D\u2502 \u2502\u2558\u2550\u256A\u2550\u255B\u2502 \u2502\u2559\u2500\u2540\u2500\u255C\u2502 \u2503\u2514\u2500\u2542\u2500\u2518\u2503 \u2591\u2591\u2592\u2592\u2593\u2593\u2588\u2588 \u250A \u2506 \u254E \u254F \u2507 \u250B \u258F\n \u255A\u2550\u2550\u2569\u2550\u2550\u255D \u2514\u2500\u2500\u2534\u2500\u2500\u2518 \u2570\u2500\u2500\u2534\u2500\u2500\u256F \u2570\u2500\u2500\u2534\u2500\u2500\u256F \u2517\u2501\u2501\u253B\u2501\u2501\u251B \u2597\u2584\u2596\u259B\u2580\u259C \u2514\u254C\u254C\u2518 \u254E \u2517\u254D\u254D\u251B \u250B \u2581\u2582\u2583\u2584\u2585\u2586\u2587\u2588\n \u259D\u2580\u2598\u2599\u2584\u259F\n" . + "\n Two byte Unicode escape: \u00E0\n Largest Unicode escape in Turtle: \U0010FFFF\n" . diff --git a/testsuite/serd-tests/good/UTF-8.ttl b/testsuite/serd-tests/good/UTF-8.ttl new file mode 100644 index 00000000..01ab2bab --- /dev/null +++ b/testsuite/serd-tests/good/UTF-8.ttl @@ -0,0 +1,219 @@ +@prefix rdfs: . + +<> rdfs:comment """ +UTF-8 encoded sample plain-text file +‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾ + +Markus Kuhn [ˈmaʳkÊŠs kuËn] — 2002-07-25 + + +The ASCII compatible UTF-8 encoding used in this plain-text file +is defined in Unicode, ISO 10646-1, and RFC 2279. + + +Using Unicode/UTF-8, you can write in emails and source code things such as + +Mathematics and sciences: + + ∮ Eâ‹…da = Q, n → ∞, ∑ f(i) = ∠g(i), ⎧⎡⎛┌─────â”⎞⎤⎫ + ⎪⎢⎜│a²+b³ ⎟⎥⎪ + ∀x∈â„: ⌈x⌉ = −⌊−x⌋, α ∧ ¬β = ¬(¬α ∨ β), ⎪⎢⎜│───── ⎟⎥⎪ + ⎪⎢⎜⎷ c₈ ⎟⎥⎪ + â„• ⊆ â„•â‚€ ⊂ ℤ ⊂ â„š ⊂ ℠⊂ â„‚, ⎨⎢⎜ ⎟⎥⎬ + ⎪⎢⎜ ∞ ⎟⎥⎪ + ⊥ < a ≠ b ≡ c ≤ d ≪ ⊤ ⇒ (⟦A⟧ ⇔ ⟪B⟫), ⎪⎢⎜ ⎲ ⎟⎥⎪ + ⎪⎢⎜ ⎳aâ±-bâ±âŽŸâŽ¥âŽª + 2Hâ‚‚ + Oâ‚‚ ⇌ 2Hâ‚‚O, R = 4.7 kΩ, ⌀ 200 mm ⎩⎣âŽi=1 ⎠⎦⎭ + +Linguistics and dictionaries: + + ði ıntəˈnæʃənÉ™l fəˈnÉ›tık É™soÊŠsiˈeıʃn + Y [ˈÊpsilÉ”n], Yen [jÉ›n], Yoga [ˈjoËgÉ‘] + +APL: + + ((Vâ³V)=â³â´V)/Vâ†,V ⌷â†â³â†’â´âˆ†âˆ‡âŠƒâ€¾âŽâ•âŒˆ + +Nicer typography in plain text files: + + â•”â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•— + â•‘ â•‘ + â•‘ • ‘single’ and “double†quotes â•‘ + â•‘ â•‘ + â•‘ • Curly apostrophes: “We’ve been here†║ + â•‘ â•‘ + â•‘ • Latin-1 apostrophe and accents: '´` â•‘ + â•‘ â•‘ + â•‘ • ‚deutsche‘ „Anführungszeichen“ â•‘ + â•‘ â•‘ + â•‘ • †, ‡, ‰, •, 3–4, —, −5/+5, â„¢, … â•‘ + â•‘ â•‘ + â•‘ • ASCII safety test: 1lI|, 0OD, 8B â•‘ + â•‘ ╭─────────╮ â•‘ + â•‘ • the euro symbol: │ 14.95 € │ â•‘ + â•‘ ╰─────────╯ â•‘ + â•šâ•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â• + +Combining characters: + + STARGΛ̊TE SG-1, a = v̇ = r̈, a⃑ ⊥ b⃑ + +Greek (in Polytonic): + + The Greek anthem: + + Σὲ γνωÏίζω ἀπὸ τὴν κόψη + τοῦ σπαθιοῦ τὴν Ï„ÏομεÏá½µ, + σὲ γνωÏίζω ἀπὸ τὴν ὄψη + ποὺ μὲ βία μετÏάει Ï„á½´ γῆ. + + ᾿Απ᾿ Ï„á½° κόκκαλα βγαλμένη + τῶν ῾Ελλήνων Ï„á½° ἱεÏá½± + καὶ σὰν Ï€Ïῶτα ἀνδÏειωμένη + χαῖÏε, ὦ χαῖÏε, ᾿ΕλευθεÏιά! + + From a speech of Demosthenes in the 4th century BC: + + Οá½Ï‡á½¶ ταá½Ï„á½° παÏίσταταί μοι γιγνώσκειν, ὦ ἄνδÏες ᾿Αθηναῖοι, + ὅταν τ᾿ εἰς Ï„á½° Ï€Ïάγματα ἀποβλέψω καὶ ὅταν Ï€Ïὸς τοὺς + λόγους οὓς ἀκούω· τοὺς μὲν Î³á½°Ï Î»á½¹Î³Î¿Ï…Ï‚ πεÏὶ τοῦ + τιμωÏήσασθαι Φίλιππον á½Ïῶ γιγνομένους, Ï„á½° δὲ Ï€Ïάγματ᾿ + εἰς τοῦτο Ï€Ïοήκοντα, ὥσθ᾿ ὅπως μὴ πεισόμεθ᾿ αá½Ï„οὶ + Ï€ÏότεÏον κακῶς σκέψασθαι δέον. οá½Î´á½³Î½ οὖν ἄλλο μοι δοκοῦσιν + οἱ Ï„á½° τοιαῦτα λέγοντες á¼¢ τὴν ὑπόθεσιν, πεÏὶ ἧς βουλεύεσθαι, + οá½Ï‡á½¶ τὴν οὖσαν παÏιστάντες ὑμῖν á¼Î¼Î±Ïτάνειν. á¼Î³á½¼ δέ, ὅτι μέν + ποτ᾿ á¼Î¾á¿†Î½ τῇ πόλει καὶ Ï„á½° αὑτῆς ἔχειν ἀσφαλῶς καὶ Φίλιππον + τιμωÏήσασθαι, καὶ μάλ᾿ ἀκÏιβῶς οἶδα· á¼Ï€á¾¿ á¼Î¼Î¿á¿¦ γάÏ, οὠπάλαι + γέγονεν ταῦτ᾿ ἀμφότεÏα· νῦν μέντοι πέπεισμαι τοῦθ᾿ ἱκανὸν + Ï€Ïολαβεῖν ἡμῖν εἶναι τὴν Ï€Ïώτην, ὅπως τοὺς συμμάχους + σώσομεν. á¼á½°Î½ Î³á½°Ï Ï„Î¿á¿¦Ï„Î¿ βεβαίως ὑπάÏξῃ, τότε καὶ πεÏὶ τοῦ + τίνα τιμωÏήσεταί τις καὶ ὃν Ï„Ïόπον á¼Î¾á½³ÏƒÏ„αι σκοπεῖν· Ï€Ïὶν δὲ + τὴν á¼€Ïχὴν á½€Ïθῶς ὑποθέσθαι, μάταιον ἡγοῦμαι πεÏὶ τῆς + τελευτῆς á½Î½Ï„ινοῦν ποιεῖσθαι λόγον. + + Δημοσθένους, Γ´ ᾿Ολυνθιακὸς + +Georgian: + + From a Unicode conference invitation: + + გთხáƒáƒ•áƒ— áƒáƒ®áƒšáƒáƒ•áƒ” გáƒáƒ˜áƒáƒ áƒáƒ— რეგისტრáƒáƒªáƒ˜áƒ Unicode-ის მეáƒáƒ—ე სáƒáƒ”რთáƒáƒ¨áƒáƒ áƒ˜áƒ¡áƒ + კáƒáƒœáƒ¤áƒ”რენციáƒáƒ–ე დáƒáƒ¡áƒáƒ¡áƒ¬áƒ áƒ”ბáƒáƒ“, რáƒáƒ›áƒ”ლიც გáƒáƒ˜áƒ›áƒáƒ áƒ—ებრ10-12 მáƒáƒ áƒ¢áƒ¡, + ქ. მáƒáƒ˜áƒœáƒªáƒ¨áƒ˜, გერმáƒáƒœáƒ˜áƒáƒ¨áƒ˜. კáƒáƒœáƒ¤áƒ”რენცირშეჰკრებს ერთáƒáƒ“ მსáƒáƒ¤áƒšáƒ˜áƒáƒ¡ + ექსპერტებს ისეთ დáƒáƒ áƒ’ებში რáƒáƒ’áƒáƒ áƒ˜áƒªáƒáƒ ინტერნეტი დრUnicode-ი, + ინტერნáƒáƒªáƒ˜áƒáƒœáƒáƒšáƒ˜áƒ–áƒáƒªáƒ˜áƒ დრლáƒáƒ™áƒáƒšáƒ˜áƒ–áƒáƒªáƒ˜áƒ, Unicode-ის გáƒáƒ›áƒáƒ§áƒ”ნებრ+ áƒáƒžáƒ”რáƒáƒªáƒ˜áƒ£áƒš სისტემებსáƒ, დრგáƒáƒ›áƒáƒ§áƒ”ნებით პრáƒáƒ’რáƒáƒ›áƒ”ბში, შრიფტებში, + ტექსტების დáƒáƒ›áƒ£áƒ¨áƒáƒ•áƒ”ბáƒáƒ¡áƒ დრმრáƒáƒ•áƒáƒšáƒ”ნáƒáƒ•áƒáƒœ კáƒáƒ›áƒžáƒ˜áƒ£áƒ¢áƒ”რულ სისტემებში. + +Russian: + + From a Unicode conference invitation: + + ЗарегиÑтрируйтеÑÑŒ ÑÐµÐ¹Ñ‡Ð°Ñ Ð½Ð° ДеÑÑтую Международную Конференцию по + Unicode, ÐºÐ¾Ñ‚Ð¾Ñ€Ð°Ñ ÑоÑтоитÑÑ 10-12 марта 1997 года в Майнце в Германии. + ÐšÐ¾Ð½Ñ„ÐµÑ€ÐµÐ½Ñ†Ð¸Ñ Ñоберет широкий круг ÑкÑпертов по вопроÑам глобального + Интернета и Unicode, локализации и интернационализации, воплощению и + применению Unicode в различных операционных ÑиÑтемах и программных + приложениÑÑ…, шрифтах, верÑтке и многоÑзычных компьютерных ÑиÑтемах. + +Thai (UCS Level 2): + + Excerpt from a poetry on The Romance of The Three Kingdoms (a Chinese + classic 'San Gua'): + + [----------------------------|------------------------] + ๠à¹à¸œà¹ˆà¸™à¸”ินฮั่นเสื่อมโทรมà¹à¸ªà¸™à¸ªà¸±à¸‡à¹€à¸§à¸Š พระปà¸à¹€à¸à¸¨à¸à¸­à¸‡à¸šà¸¹à¹Šà¸à¸¹à¹‰à¸‚ึ้นใหม่ + สิบสองà¸à¸©à¸±à¸•à¸£à¸´à¸¢à¹Œà¸à¹ˆà¸­à¸™à¸«à¸™à¹‰à¸²à¹à¸¥à¸–ัดไป สององค์ไซร้โง่เขลาเบาปัà¸à¸à¸² + ทรงนับถือขันทีเป็นที่พึ่ง บ้านเมืองจึงวิปริตเป็นนัà¸à¸«à¸™à¸² + โฮจิ๋นเรียà¸à¸—ัพทั่วหัวเมืองมา หมายจะฆ่ามดชั่วตัวสำคัภ+ เหมือนขับไสไล่เสือจาà¸à¹€à¸„หา รับหมาป่าเข้ามาเลยอาสัภ+ à¸à¹ˆà¸²à¸¢à¸­à¹‰à¸­à¸‡à¸­à¸¸à¹‰à¸™à¸¢à¸¸à¹à¸¢à¸à¹ƒà¸«à¹‰à¹à¸•à¸à¸à¸±à¸™ ใช้สาวนั้นเป็นชนวนชื่นชวนใจ + พลันลิฉุยà¸à¸¸à¸¢à¸à¸µà¸à¸¥à¸±à¸šà¸à¹ˆà¸­à¹€à¸«à¸•à¸¸ ช่างอาเพศจริงหนาฟ้าร้องไห้ + ต้องรบราฆ่าฟันจนบรรลัย ฤๅหาใครค้ำชูà¸à¸¹à¹‰à¸šà¸£à¸£à¸¥à¸±à¸‡à¸à¹Œ ฯ + + (The above is a two-column text. If combining characters are handled + correctly, the lines of the second column should be aligned with the + | character above.) + +Ethiopian: + + Proverbs in the Amharic language: + + ሰማይ አይታረስ ንጉሥ አይከሰስᢠ+ ብላ ካለአእንደአባቴ በቆመጠáŠá¢ + ጌጥ ያለቤቱ á‰áˆáŒ¥áŠ“ áŠá‹á¢ + ደሀ በሕáˆáˆ™ ቅቤ ባይጠጣ ንጣት በገደለá‹á¢ + የአá ወለáˆá‰³ በቅቤ አይታሽáˆá¢ + አይጥ በበላ ዳዋ ተመታᢠ+ ሲተረጉሙ ይደረáŒáˆ™á¢ + ቀስ በቀስᥠዕንá‰áˆ‹áˆ በእáŒáˆ© ይሄዳáˆá¢ + ድር ቢያብር አንበሳ ያስርᢠ+ ሰዠእንደቤቱ እንጅ እንደ ጉረቤቱ አይተዳደርáˆá¢ + እáŒá‹œáˆ­ የከáˆá‰°á‹áŠ• ጉሮሮ ሳይዘጋዠአይድርáˆá¢ + የጎረቤት ሌባᥠቢያዩት ይስቅ ባያዩት ያጠáˆá‰…ᢠ+ ሥራ ከመáታት áˆáŒ„ን ላá‹á‰³á‰µá¢ + ዓባይ ማደሪያ የለá‹á¥ áŒáŠ•á‹µ á‹­á‹ž ይዞራáˆá¢ + የእስላሠአገሩ መካ የአሞራ አገሩ ዋርካᢠ+ ተንጋሎ ቢተበተመáˆáˆ¶ ባá‰á¢ + ወዳጅህ ማር ቢሆን ጨርስህ አትላሰá‹á¢ + እáŒáˆ­áˆ…ን በáራሽህ áˆáŠ­ ዘርጋᢠ+ +Runes: + + ᚻᛖ ᚳᚹᚫᚦ ᚦᚫᛠᚻᛖ ᛒᚢᛞᛖ áš©áš¾ ᚦᚫᛗ ᛚᚪᚾᛞᛖ ᚾᚩᚱᚦᚹᛖᚪᚱᛞᚢᛗ áš¹á›áš¦ ᚦᚪ ᚹᛖᛥᚫ + + (Old English, which transcribed into Latin reads 'He cwaeth that he + bude thaem lande northweardum with that Westsae.' and means 'He said + that he lived in the northern land near the Western Sea.') + +Braille: + + â¡Œâ â §â ‘ â ¼â â ’ â¡â œâ ‡â ‘⠹⠰⠎ ⡣⠕⠌ + + â¡â œâ ‡â ‘â ¹ â ºâ â Ž ⠙⠑â â ™â ’ â žâ • ⠃⠑⠛⠔ ⠺⠊⠹⠲ ⡹⠻⠑ â Šâ Ž â â • ⠙⠳⠃⠞ + â ±â â žâ ‘⠧⠻ â â ƒâ ³â ž â ¹â â žâ ² ⡹⠑ ⠗⠑⠛⠊⠌⠻ â •â ‹ ⠙⠊⠎ ⠃⠥⠗⠊â â ‡ â ºâ â Ž + â Žâ Šâ ›â â « ⠃⠹ ⠹⠑ ⠊⠇⠻⠛⠹â â â â ‚ ⠹⠑ ⠊⠇⠻⠅⠂ ⠹⠑ â ¥â â ™â »â žâ â …⠻⠂ + â â â ™ ⠹⠑ â ¡â Šâ ‘â ‹ â â ³â —â â »â ² ⡎⠊⠗⠕⠕⠛⠑ â Žâ Šâ ›â â « â Šâ žâ ² â¡â â ™ + ⡎⠊⠗⠕⠕⠛⠑⠰⠎ â â â â ‘ â ºâ â Ž ⠛⠕⠕⠙ â ¥â â •â  â °â¡¡â â â ›â ‘â ‚ â ‹â •â — â â â ¹â ¹â ”â › ⠙⠑ + â ¡â •â Žâ ‘ â žâ • â â ¥â ž ⠙⠊⠎ â ™â â â ™ â žâ •â ² + + ⡕⠇⠙ â¡â œâ ‡â ‘â ¹ â ºâ â Ž â â Ž ⠙⠑â â ™ â â Ž â  â ™â •â •â —â ¤â â â Šâ ‡â ² + + â¡â ”⠙⠖ â¡Š ⠙⠕â â °â ž â â ‘â â  â žâ • â Žâ â ¹ â ¹â â ž â¡Š â …â â ªâ ‚ â •â ‹ â â ¹ + â ªâ  â …â â ªâ ‡â «â ›â ‘â ‚ â ±â â ž ⠹⠻⠑ â Šâ Ž â â œâ žâ Šâ Šâ ¥â ‡â œâ ‡â ¹ ⠙⠑â â ™ â â ƒâ ³â ž + â  â ™â •â •â —â ¤â â â Šâ ‡â ² â¡Š â â Šâ £â ž â ™â â §â ‘ ⠃⠑⠲ ⠔⠊⠇⠔⠫⠂ â â ¹â Žâ ‘⠇⠋⠂ â žâ • + ⠗⠑⠛⠜⠙ â  â Šâ •â ‹â ‹â ”â ¤â â â Šâ ‡ â â Ž ⠹⠑ ⠙⠑â â ™â ‘â Œ â â Šâ ‘â Šâ ‘ â •â ‹ â Šâ —â •â â â •â â ›â »â ¹ + â ” ⠹⠑ â žâ —â â ™â ‘â ² ⡃⠥⠞ ⠹⠑ â ºâ Šâ Žâ ™â •â  â •â ‹ ⠳⠗ â â â Šâ ‘⠌⠕⠗⠎ + â Šâ Ž â ” ⠹⠑ â Žâ Šâ â Šâ ‡â ‘â † â â â ™ â â ¹ â ¥â â ™â â ‡â ‡â ªâ « â ™â â â ™â Ž + â ©â â ‡â ‡ â â •â ž ⠙⠊⠌⠥⠗⠃ â Šâ žâ ‚ â •â — ⠹⠑ â¡Šâ ³â â žâ —⠹⠰⠎ ⠙⠕â â ‘ â ‹â •â —â ² ⡹⠳ + ⠺⠊⠇⠇ ⠹⠻⠑⠋⠕⠗⠑ â â »â â Šâ ž â â ‘ â žâ • â —â ‘â â ‘â â žâ ‚ â ‘â â â ™â â žâ Šâ Šâ â ‡â ‡â ¹â ‚ â ¹â â ž + â¡â œâ ‡â ‘â ¹ â ºâ â Ž â â Ž ⠙⠑â â ™ â â Ž â  â ™â •â •â —â ¤â â â Šâ ‡â ² + + (The first couple of paragraphs of "A Christmas Carol" by Dickens) + +Compact font selection example text: + + ABCDEFGHIJKLMNOPQRSTUVWXYZ /0123456789 + abcdefghijklmnopqrstuvwxyz £©µÀÆÖÞßéöÿ + –—‘“â€â€žâ€ â€¢â€¦â€°â„¢Å“ŠŸž€ ΑΒΓΔΩαβγδω ÐБВГДабвгд + ∀∂∈â„∧∪≡∞ ↑↗↨↻⇣ â”┼╔╘░►☺♀ ï¬ï¿½â‘€â‚‚ἠḂӥẄÉËâŽ×Աრ+ +Greetings in various languages: + + Hello world, ΚαλημέÏα κόσμε, コンニãƒãƒ + +Box drawing alignment tests: â–ˆ + â–‰ + â•”â•â•â•¦â•â•â•— ┌──┬──┠╭──┬──╮ ╭──┬──╮ â”â”â”┳â”â”┓ ┎┒â”┑ â•· â•» â”┯┓ ┌┰┠▊ ╱╲╱╲╳╳╳ + ║┌─╨─â”â•‘ │╔â•â•§â•â•—│ │╒â•â•ªâ•â••â”‚ │╓─â•â”€â•–│ ┃┌─╂─â”┃ ┗╃╄┙ ╶┼╴╺╋╸┠┼┨ â”╋┥ â–‹ ╲╱╲╱╳╳╳ + ║│╲ ╱│║ │║ ║│ ││ │ ││ │║ ┃ ║│ ┃│ â•¿ │┃ â”╅╆┓ ╵ ╹ â”—â”·â”› └┸┘ â–Œ ╱╲╱╲╳╳╳ + â• â•¡ ╳ â•žâ•£ ├╢ ╟┤ ├┼─┼─┼┤ ├╫─╂─╫┤ ┣┿╾┼╼┿┫ ┕┛┖┚ ┌┄┄┠╎ â”┅┅┓ ┋ ■╲╱╲╱╳╳╳ + ║│╱ ╲│║ │║ ║│ ││ │ ││ │║ ┃ ║│ ┃│ ╽ │┃ ░░▒▒▓▓██ ┊ ┆ â•Ž ╠┇ ┋ â–Ž + ║└─╥─┘║ │╚â•â•¤â•â•â”‚ │╘â•â•ªâ•â•›â”‚ │╙─╀─╜│ ┃└─╂─┘┃ ░░▒▒▓▓██ ┊ ┆ â•Ž ╠┇ ┋ â– + â•šâ•â•â•©â•â•â• └──┴──┘ ╰──┴──╯ ╰──┴──╯ â”—â”â”â”»â”â”â”› ▗▄▖▛▀▜ └╌╌┘ â•Ž â”—â•â•â”› ┋ â–▂▃▄▅▆▇█ + â–▀▘▙▄▟ +""" . +<> rdfs:comment """ + Two byte Unicode escape: \u00E0 + Largest Unicode escape in Turtle: \U0010FFFF +""" . diff --git a/testsuite/serd-tests/good/base.nt b/testsuite/serd-tests/good/base.nt new file mode 100644 index 00000000..4e8e4c82 --- /dev/null +++ b/testsuite/serd-tests/good/base.nt @@ -0,0 +1 @@ + . diff --git a/testsuite/serd-tests/good/base.ttl b/testsuite/serd-tests/good/base.ttl new file mode 100644 index 00000000..4c437937 --- /dev/null +++ b/testsuite/serd-tests/good/base.ttl @@ -0,0 +1,3 @@ + + a . + diff --git a/testsuite/serd-tests/good/manifest.ttl b/testsuite/serd-tests/good/manifest.ttl new file mode 100644 index 00000000..085878f0 --- /dev/null +++ b/testsuite/serd-tests/good/manifest.ttl @@ -0,0 +1,319 @@ +@prefix mf: . +@prefix rdf: . +@prefix rdfs: . +@prefix rdft: . + +<> + rdf:type mf:Manifest ; + rdfs:comment "Serd good syntax test cases" ; + mf:entries ( + <#base> + <#test-14> + <#test-15> + <#test-16> + <#test-18> + <#test-30> + <#test-a-without-whitespace> + <#test-backspace> + <#test-base-nopath> + <#test-base-query> + <#test-blank-cont> + <#test-blank-in-list> + <#test-blank-node-statement> + <#test-blankdot> + # We do not support BOM <#test-bom> + <#test-cr> + <#test-delete> + <#test-digit-start-pname> + <#test-empty-path-base> + <#test-empty> + <#test-eof-at-page-end> + # We prefer to decode before IRI validation <#test-escapes> + # this is an extension <#test-ext-namedblank-iri> + # this is an extension <#test-ext-namedblank-prefix> + <#test-form-feed> + <#test-id> + <#test-lang> + <#test-list-in-blank> + <#test-list-subject> + <#test-list> + <#test-long-string> + <#test-several-eaten-dots> + <#test-no-spaces> + <#test-non-curie-uri> + <#test-num> + # We prefer to raise an error on invalid unicode code points <#test-out-of-range-unicode> + <#test-prefix> + <#test-pretty> + <#test-rel> + <#test-semi-dot> + # We validate URI escaping <#test-uri-escape> + <#test-uri> + <#test-utf8-uri> + <#UTF-8> + ) . + +<#base> + rdf:type rdft:TestTurtleEval ; + mf:name "base" ; + mf:action ; + mf:result . + +<#test-14> + rdf:type rdft:TestTurtleEval ; + mf:name "test-14" ; + mf:action ; + mf:result . + +<#test-15> + rdf:type rdft:TestTurtleEval ; + mf:name "test-15" ; + mf:action ; + mf:result . + +<#test-16> + rdf:type rdft:TestTurtleEval ; + mf:name "test-16" ; + mf:action ; + mf:result . + +<#test-18> + rdf:type rdft:TestTurtleEval ; + mf:name "test-18" ; + mf:action ; + mf:result . + +<#test-30> + rdf:type rdft:TestTurtleEval ; + mf:name "test-30" ; + mf:action ; + mf:result . + +<#test-a-without-whitespace> + rdf:type rdft:TestTurtleEval ; + mf:name "test-a-without-whitespace" ; + mf:action ; + mf:result . + +<#test-backspace> + rdf:type rdft:TestTurtleEval ; + mf:name "test-backspace" ; + mf:action ; + mf:result . + +<#test-base-nopath> + rdf:type rdft:TestTurtleEval ; + mf:name "test-base-nopath" ; + mf:action ; + mf:result . + +<#test-base-query> + rdf:type rdft:TestTurtleEval ; + mf:name "test-base-query" ; + mf:action ; + mf:result . + +<#test-blank-cont> + rdf:type rdft:TestTurtleEval ; + mf:name "test-blank-cont" ; + mf:action ; + mf:result . + +<#test-blankdot> + rdf:type rdft:TestTurtleEval ; + mf:name "test-blankdot" ; + mf:action ; + mf:result . + +<#test-blank-in-list> + rdf:type rdft:TestTurtleEval ; + mf:name "test-blank-in-list" ; + mf:action ; + mf:result . + +<#test-blank-node-statement> + rdf:type rdft:TestTurtleEval ; + mf:name "test-blank-node-statement" ; + mf:action ; + mf:result . + +<#test-bom> + rdf:type rdft:TestTurtleEval ; + mf:name "test-bom" ; + mf:action ; + mf:result . + +<#test-cr> + rdf:type rdft:TestTurtleEval ; + mf:name "test-cr" ; + mf:action ; + mf:result . + +<#test-delete> + rdf:type rdft:TestTurtleEval ; + mf:name "test-delete" ; + mf:action ; + mf:result . + +<#test-digit-start-pname> + rdf:type rdft:TestTurtleEval ; + mf:name "test-digit-start-pname" ; + mf:action ; + mf:result . + +<#test-empty-path-base> + rdf:type rdft:TestTurtleEval ; + mf:name "test-empty-path-base" ; + mf:action ; + mf:result . + +<#test-empty> + rdf:type rdft:TestTurtleEval ; + mf:name "test-empty" ; + mf:action ; + mf:result . + +<#test-eof-at-page-end> + rdf:type rdft:TestTurtleEval ; + mf:name "test-eof-at-page-end" ; + mf:action ; + mf:result . + +<#test-escapes> + rdf:type rdft:TestTurtleEval ; + mf:name "test-escapes" ; + mf:action ; + mf:result . + +<#test-ext-namedblank-iri> + rdf:type rdft:TestTurtleEval ; + mf:name "test-ext-namedblank-iri" ; + mf:action ; + mf:result . + +<#test-ext-namedblank-prefix> + rdf:type rdft:TestTurtleEval ; + mf:name "test-ext-namedblank-prefix" ; + mf:action ; + mf:result . + +<#test-form-feed> + rdf:type rdft:TestTurtleEval ; + mf:name "test-form-feed" ; + mf:action ; + mf:result . + +<#test-id> + rdf:type rdft:TestTurtleEval ; + mf:name "test-id" ; + mf:action ; + mf:result . + +<#test-lang> + rdf:type rdft:TestTurtleEval ; + mf:name "test-lang" ; + mf:action ; + mf:result . + +<#test-list-in-blank> + rdf:type rdft:TestTurtleEval ; + mf:name "test-list-in-blank" ; + mf:action ; + mf:result . + +<#test-list-subject> + rdf:type rdft:TestTurtleEval ; + mf:name "test-list-subject" ; + mf:action ; + mf:result . + +<#test-list> + rdf:type rdft:TestTurtleEval ; + mf:name "test-list" ; + mf:action ; + mf:result . + +<#test-long-string> + rdf:type rdft:TestTurtleEval ; + mf:name "test-long-string" ; + mf:action ; + mf:result . + +<#test-several-eaten-dots> + rdf:type rdft:TestTrigEval ; + mf:name "test-several-eaten-dots" ; + mf:action ; + mf:result . + +<#test-no-spaces> + rdf:type rdft:TestTurtleEval ; + mf:name "test-no-spaces" ; + mf:action ; + mf:result . + +<#test-non-curie-uri> + rdf:type rdft:TestTurtleEval ; + mf:name "test-non-curie-uri" ; + mf:action ; + mf:result . + +<#test-num> + rdf:type rdft:TestTurtleEval ; + mf:name "test-num" ; + mf:action ; + mf:result . + +<#test-out-of-range-unicode> + rdf:type rdft:TestTurtleEval ; + mf:name "test-out-of-range-unicode" ; + mf:action ; + mf:result . + +<#test-prefix> + rdf:type rdft:TestTurtleEval ; + mf:name "test-prefix" ; + mf:action ; + mf:result . + +<#test-pretty> + rdf:type rdft:TestTurtleEval ; + mf:name "test-pretty" ; + mf:action ; + mf:result . + +<#test-rel> + rdf:type rdft:TestTurtleEval ; + mf:name "test-rel" ; + mf:action ; + mf:result . + +<#test-semi-dot> + rdf:type rdft:TestTurtleEval ; + mf:name "test-semi-dot" ; + mf:action ; + mf:result . + +<#test-uri-escape> + rdf:type rdft:TestTurtleEval ; + mf:name "test-uri-escape" ; + mf:action ; + mf:result . + +<#test-uri> + rdf:type rdft:TestTurtleEval ; + mf:name "test-uri" ; + mf:action ; + mf:result . + +<#test-utf8-uri> + rdf:type rdft:TestTurtleEval ; + mf:name "test-utf8-uri" ; + mf:action ; + mf:result . + +<#UTF-8> + rdf:type rdft:TestTurtleEval ; + mf:name "UTF-8" ; + mf:action ; + mf:result . + diff --git a/testsuite/serd-tests/good/qualify-in.ttl b/testsuite/serd-tests/good/qualify-in.ttl new file mode 100644 index 00000000..d938b7bd --- /dev/null +++ b/testsuite/serd-tests/good/qualify-in.ttl @@ -0,0 +1,3 @@ +@prefix eg: . + + . diff --git a/testsuite/serd-tests/good/qualify-out.ttl b/testsuite/serd-tests/good/qualify-out.ttl new file mode 100644 index 00000000..97f67a53 --- /dev/null +++ b/testsuite/serd-tests/good/qualify-out.ttl @@ -0,0 +1,5 @@ +@prefix eg: . + +eg:s + eg:p eg:o . + diff --git a/testsuite/serd-tests/good/test-14.nt b/testsuite/serd-tests/good/test-14.nt new file mode 100644 index 00000000..791b3a7a --- /dev/null +++ b/testsuite/serd-tests/good/test-14.nt @@ -0,0 +1,10000 @@ + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . diff --git a/testsuite/serd-tests/good/test-14.ttl b/testsuite/serd-tests/good/test-14.ttl new file mode 100644 index 00000000..ad9dbde8 --- /dev/null +++ b/testsuite/serd-tests/good/test-14.ttl @@ -0,0 +1,10002 @@ +# 10000 triples, more than the default Bison stack size +@prefix : . +:a1 :a1 :a1. +:a2 :a2 :a2. +:a3 :a3 :a3. +:a4 :a4 :a4. +:a5 :a5 :a5. +:a6 :a6 :a6. +:a7 :a7 :a7. +:a8 :a8 :a8. +:a9 :a9 :a9. +:a10 :a10 :a10. +:a11 :a11 :a11. +:a12 :a12 :a12. +:a13 :a13 :a13. +:a14 :a14 :a14. +:a15 :a15 :a15. +:a16 :a16 :a16. +:a17 :a17 :a17. +:a18 :a18 :a18. +:a19 :a19 :a19. +:a20 :a20 :a20. +:a21 :a21 :a21. +:a22 :a22 :a22. +:a23 :a23 :a23. +:a24 :a24 :a24. +:a25 :a25 :a25. +:a26 :a26 :a26. +:a27 :a27 :a27. +:a28 :a28 :a28. +:a29 :a29 :a29. +:a30 :a30 :a30. +:a31 :a31 :a31. +:a32 :a32 :a32. +:a33 :a33 :a33. +:a34 :a34 :a34. +:a35 :a35 :a35. +:a36 :a36 :a36. +:a37 :a37 :a37. +:a38 :a38 :a38. +:a39 :a39 :a39. +:a40 :a40 :a40. +:a41 :a41 :a41. +:a42 :a42 :a42. +:a43 :a43 :a43. +:a44 :a44 :a44. +:a45 :a45 :a45. +:a46 :a46 :a46. +:a47 :a47 :a47. +:a48 :a48 :a48. +:a49 :a49 :a49. +:a50 :a50 :a50. +:a51 :a51 :a51. +:a52 :a52 :a52. +:a53 :a53 :a53. +:a54 :a54 :a54. +:a55 :a55 :a55. +:a56 :a56 :a56. +:a57 :a57 :a57. +:a58 :a58 :a58. +:a59 :a59 :a59. +:a60 :a60 :a60. +:a61 :a61 :a61. +:a62 :a62 :a62. +:a63 :a63 :a63. +:a64 :a64 :a64. +:a65 :a65 :a65. +:a66 :a66 :a66. +:a67 :a67 :a67. +:a68 :a68 :a68. +:a69 :a69 :a69. +:a70 :a70 :a70. +:a71 :a71 :a71. +:a72 :a72 :a72. +:a73 :a73 :a73. +:a74 :a74 :a74. +:a75 :a75 :a75. +:a76 :a76 :a76. +:a77 :a77 :a77. +:a78 :a78 :a78. +:a79 :a79 :a79. +:a80 :a80 :a80. +:a81 :a81 :a81. +:a82 :a82 :a82. +:a83 :a83 :a83. +:a84 :a84 :a84. +:a85 :a85 :a85. +:a86 :a86 :a86. +:a87 :a87 :a87. +:a88 :a88 :a88. +:a89 :a89 :a89. +:a90 :a90 :a90. +:a91 :a91 :a91. +:a92 :a92 :a92. +:a93 :a93 :a93. +:a94 :a94 :a94. +:a95 :a95 :a95. +:a96 :a96 :a96. +:a97 :a97 :a97. +:a98 :a98 :a98. +:a99 :a99 :a99. +:a100 :a100 :a100. +:a101 :a101 :a101. +:a102 :a102 :a102. +:a103 :a103 :a103. +:a104 :a104 :a104. +:a105 :a105 :a105. +:a106 :a106 :a106. +:a107 :a107 :a107. +:a108 :a108 :a108. +:a109 :a109 :a109. +:a110 :a110 :a110. +:a111 :a111 :a111. +:a112 :a112 :a112. +:a113 :a113 :a113. +:a114 :a114 :a114. +:a115 :a115 :a115. +:a116 :a116 :a116. +:a117 :a117 :a117. +:a118 :a118 :a118. +:a119 :a119 :a119. +:a120 :a120 :a120. +:a121 :a121 :a121. +:a122 :a122 :a122. +:a123 :a123 :a123. +:a124 :a124 :a124. +:a125 :a125 :a125. +:a126 :a126 :a126. +:a127 :a127 :a127. +:a128 :a128 :a128. +:a129 :a129 :a129. +:a130 :a130 :a130. +:a131 :a131 :a131. +:a132 :a132 :a132. +:a133 :a133 :a133. +:a134 :a134 :a134. +:a135 :a135 :a135. +:a136 :a136 :a136. +:a137 :a137 :a137. +:a138 :a138 :a138. +:a139 :a139 :a139. +:a140 :a140 :a140. +:a141 :a141 :a141. +:a142 :a142 :a142. +:a143 :a143 :a143. +:a144 :a144 :a144. +:a145 :a145 :a145. +:a146 :a146 :a146. +:a147 :a147 :a147. +:a148 :a148 :a148. +:a149 :a149 :a149. +:a150 :a150 :a150. +:a151 :a151 :a151. +:a152 :a152 :a152. +:a153 :a153 :a153. +:a154 :a154 :a154. +:a155 :a155 :a155. +:a156 :a156 :a156. +:a157 :a157 :a157. +:a158 :a158 :a158. +:a159 :a159 :a159. +:a160 :a160 :a160. +:a161 :a161 :a161. +:a162 :a162 :a162. +:a163 :a163 :a163. +:a164 :a164 :a164. +:a165 :a165 :a165. +:a166 :a166 :a166. +:a167 :a167 :a167. +:a168 :a168 :a168. +:a169 :a169 :a169. +:a170 :a170 :a170. +:a171 :a171 :a171. +:a172 :a172 :a172. +:a173 :a173 :a173. +:a174 :a174 :a174. +:a175 :a175 :a175. +:a176 :a176 :a176. +:a177 :a177 :a177. +:a178 :a178 :a178. +:a179 :a179 :a179. +:a180 :a180 :a180. +:a181 :a181 :a181. +:a182 :a182 :a182. +:a183 :a183 :a183. +:a184 :a184 :a184. +:a185 :a185 :a185. +:a186 :a186 :a186. +:a187 :a187 :a187. +:a188 :a188 :a188. +:a189 :a189 :a189. +:a190 :a190 :a190. +:a191 :a191 :a191. +:a192 :a192 :a192. +:a193 :a193 :a193. +:a194 :a194 :a194. +:a195 :a195 :a195. +:a196 :a196 :a196. +:a197 :a197 :a197. +:a198 :a198 :a198. +:a199 :a199 :a199. +:a200 :a200 :a200. +:a201 :a201 :a201. +:a202 :a202 :a202. +:a203 :a203 :a203. +:a204 :a204 :a204. +:a205 :a205 :a205. +:a206 :a206 :a206. +:a207 :a207 :a207. +:a208 :a208 :a208. +:a209 :a209 :a209. +:a210 :a210 :a210. +:a211 :a211 :a211. +:a212 :a212 :a212. +:a213 :a213 :a213. +:a214 :a214 :a214. +:a215 :a215 :a215. +:a216 :a216 :a216. +:a217 :a217 :a217. +:a218 :a218 :a218. +:a219 :a219 :a219. +:a220 :a220 :a220. +:a221 :a221 :a221. +:a222 :a222 :a222. +:a223 :a223 :a223. +:a224 :a224 :a224. +:a225 :a225 :a225. +:a226 :a226 :a226. +:a227 :a227 :a227. +:a228 :a228 :a228. +:a229 :a229 :a229. +:a230 :a230 :a230. +:a231 :a231 :a231. +:a232 :a232 :a232. +:a233 :a233 :a233. +:a234 :a234 :a234. +:a235 :a235 :a235. +:a236 :a236 :a236. +:a237 :a237 :a237. +:a238 :a238 :a238. +:a239 :a239 :a239. +:a240 :a240 :a240. +:a241 :a241 :a241. +:a242 :a242 :a242. +:a243 :a243 :a243. +:a244 :a244 :a244. +:a245 :a245 :a245. +:a246 :a246 :a246. +:a247 :a247 :a247. +:a248 :a248 :a248. +:a249 :a249 :a249. +:a250 :a250 :a250. +:a251 :a251 :a251. +:a252 :a252 :a252. +:a253 :a253 :a253. +:a254 :a254 :a254. +:a255 :a255 :a255. +:a256 :a256 :a256. +:a257 :a257 :a257. +:a258 :a258 :a258. +:a259 :a259 :a259. +:a260 :a260 :a260. +:a261 :a261 :a261. +:a262 :a262 :a262. +:a263 :a263 :a263. +:a264 :a264 :a264. +:a265 :a265 :a265. +:a266 :a266 :a266. +:a267 :a267 :a267. +:a268 :a268 :a268. +:a269 :a269 :a269. +:a270 :a270 :a270. +:a271 :a271 :a271. +:a272 :a272 :a272. +:a273 :a273 :a273. +:a274 :a274 :a274. +:a275 :a275 :a275. +:a276 :a276 :a276. +:a277 :a277 :a277. +:a278 :a278 :a278. +:a279 :a279 :a279. +:a280 :a280 :a280. +:a281 :a281 :a281. +:a282 :a282 :a282. +:a283 :a283 :a283. +:a284 :a284 :a284. +:a285 :a285 :a285. +:a286 :a286 :a286. +:a287 :a287 :a287. +:a288 :a288 :a288. +:a289 :a289 :a289. +:a290 :a290 :a290. +:a291 :a291 :a291. +:a292 :a292 :a292. +:a293 :a293 :a293. +:a294 :a294 :a294. +:a295 :a295 :a295. +:a296 :a296 :a296. +:a297 :a297 :a297. +:a298 :a298 :a298. +:a299 :a299 :a299. +:a300 :a300 :a300. +:a301 :a301 :a301. +:a302 :a302 :a302. +:a303 :a303 :a303. +:a304 :a304 :a304. +:a305 :a305 :a305. +:a306 :a306 :a306. +:a307 :a307 :a307. +:a308 :a308 :a308. +:a309 :a309 :a309. +:a310 :a310 :a310. +:a311 :a311 :a311. +:a312 :a312 :a312. +:a313 :a313 :a313. +:a314 :a314 :a314. +:a315 :a315 :a315. +:a316 :a316 :a316. +:a317 :a317 :a317. +:a318 :a318 :a318. +:a319 :a319 :a319. +:a320 :a320 :a320. +:a321 :a321 :a321. +:a322 :a322 :a322. +:a323 :a323 :a323. +:a324 :a324 :a324. +:a325 :a325 :a325. +:a326 :a326 :a326. +:a327 :a327 :a327. +:a328 :a328 :a328. +:a329 :a329 :a329. +:a330 :a330 :a330. +:a331 :a331 :a331. +:a332 :a332 :a332. +:a333 :a333 :a333. +:a334 :a334 :a334. +:a335 :a335 :a335. +:a336 :a336 :a336. +:a337 :a337 :a337. +:a338 :a338 :a338. +:a339 :a339 :a339. +:a340 :a340 :a340. +:a341 :a341 :a341. +:a342 :a342 :a342. +:a343 :a343 :a343. +:a344 :a344 :a344. +:a345 :a345 :a345. +:a346 :a346 :a346. +:a347 :a347 :a347. +:a348 :a348 :a348. +:a349 :a349 :a349. +:a350 :a350 :a350. +:a351 :a351 :a351. +:a352 :a352 :a352. +:a353 :a353 :a353. +:a354 :a354 :a354. +:a355 :a355 :a355. +:a356 :a356 :a356. +:a357 :a357 :a357. +:a358 :a358 :a358. +:a359 :a359 :a359. +:a360 :a360 :a360. +:a361 :a361 :a361. +:a362 :a362 :a362. +:a363 :a363 :a363. +:a364 :a364 :a364. +:a365 :a365 :a365. +:a366 :a366 :a366. +:a367 :a367 :a367. +:a368 :a368 :a368. +:a369 :a369 :a369. +:a370 :a370 :a370. +:a371 :a371 :a371. +:a372 :a372 :a372. +:a373 :a373 :a373. +:a374 :a374 :a374. +:a375 :a375 :a375. +:a376 :a376 :a376. +:a377 :a377 :a377. +:a378 :a378 :a378. +:a379 :a379 :a379. +:a380 :a380 :a380. +:a381 :a381 :a381. +:a382 :a382 :a382. +:a383 :a383 :a383. +:a384 :a384 :a384. +:a385 :a385 :a385. +:a386 :a386 :a386. +:a387 :a387 :a387. +:a388 :a388 :a388. +:a389 :a389 :a389. +:a390 :a390 :a390. +:a391 :a391 :a391. +:a392 :a392 :a392. +:a393 :a393 :a393. +:a394 :a394 :a394. +:a395 :a395 :a395. +:a396 :a396 :a396. +:a397 :a397 :a397. +:a398 :a398 :a398. +:a399 :a399 :a399. +:a400 :a400 :a400. +:a401 :a401 :a401. +:a402 :a402 :a402. +:a403 :a403 :a403. +:a404 :a404 :a404. +:a405 :a405 :a405. +:a406 :a406 :a406. +:a407 :a407 :a407. +:a408 :a408 :a408. +:a409 :a409 :a409. +:a410 :a410 :a410. +:a411 :a411 :a411. +:a412 :a412 :a412. +:a413 :a413 :a413. +:a414 :a414 :a414. +:a415 :a415 :a415. +:a416 :a416 :a416. +:a417 :a417 :a417. +:a418 :a418 :a418. +:a419 :a419 :a419. +:a420 :a420 :a420. +:a421 :a421 :a421. +:a422 :a422 :a422. +:a423 :a423 :a423. +:a424 :a424 :a424. +:a425 :a425 :a425. +:a426 :a426 :a426. +:a427 :a427 :a427. +:a428 :a428 :a428. +:a429 :a429 :a429. +:a430 :a430 :a430. +:a431 :a431 :a431. +:a432 :a432 :a432. +:a433 :a433 :a433. +:a434 :a434 :a434. +:a435 :a435 :a435. +:a436 :a436 :a436. +:a437 :a437 :a437. +:a438 :a438 :a438. +:a439 :a439 :a439. +:a440 :a440 :a440. +:a441 :a441 :a441. +:a442 :a442 :a442. +:a443 :a443 :a443. +:a444 :a444 :a444. +:a445 :a445 :a445. +:a446 :a446 :a446. +:a447 :a447 :a447. +:a448 :a448 :a448. +:a449 :a449 :a449. +:a450 :a450 :a450. +:a451 :a451 :a451. +:a452 :a452 :a452. +:a453 :a453 :a453. +:a454 :a454 :a454. +:a455 :a455 :a455. +:a456 :a456 :a456. +:a457 :a457 :a457. +:a458 :a458 :a458. +:a459 :a459 :a459. +:a460 :a460 :a460. +:a461 :a461 :a461. +:a462 :a462 :a462. +:a463 :a463 :a463. +:a464 :a464 :a464. +:a465 :a465 :a465. +:a466 :a466 :a466. +:a467 :a467 :a467. +:a468 :a468 :a468. +:a469 :a469 :a469. +:a470 :a470 :a470. +:a471 :a471 :a471. +:a472 :a472 :a472. +:a473 :a473 :a473. +:a474 :a474 :a474. +:a475 :a475 :a475. +:a476 :a476 :a476. +:a477 :a477 :a477. +:a478 :a478 :a478. +:a479 :a479 :a479. +:a480 :a480 :a480. +:a481 :a481 :a481. +:a482 :a482 :a482. +:a483 :a483 :a483. +:a484 :a484 :a484. +:a485 :a485 :a485. +:a486 :a486 :a486. +:a487 :a487 :a487. +:a488 :a488 :a488. +:a489 :a489 :a489. +:a490 :a490 :a490. +:a491 :a491 :a491. +:a492 :a492 :a492. +:a493 :a493 :a493. +:a494 :a494 :a494. +:a495 :a495 :a495. +:a496 :a496 :a496. +:a497 :a497 :a497. +:a498 :a498 :a498. +:a499 :a499 :a499. +:a500 :a500 :a500. +:a501 :a501 :a501. +:a502 :a502 :a502. +:a503 :a503 :a503. +:a504 :a504 :a504. +:a505 :a505 :a505. +:a506 :a506 :a506. +:a507 :a507 :a507. +:a508 :a508 :a508. +:a509 :a509 :a509. +:a510 :a510 :a510. +:a511 :a511 :a511. +:a512 :a512 :a512. +:a513 :a513 :a513. +:a514 :a514 :a514. +:a515 :a515 :a515. +:a516 :a516 :a516. +:a517 :a517 :a517. +:a518 :a518 :a518. +:a519 :a519 :a519. +:a520 :a520 :a520. +:a521 :a521 :a521. +:a522 :a522 :a522. +:a523 :a523 :a523. +:a524 :a524 :a524. +:a525 :a525 :a525. +:a526 :a526 :a526. +:a527 :a527 :a527. +:a528 :a528 :a528. +:a529 :a529 :a529. +:a530 :a530 :a530. +:a531 :a531 :a531. +:a532 :a532 :a532. +:a533 :a533 :a533. +:a534 :a534 :a534. +:a535 :a535 :a535. +:a536 :a536 :a536. +:a537 :a537 :a537. +:a538 :a538 :a538. +:a539 :a539 :a539. +:a540 :a540 :a540. +:a541 :a541 :a541. +:a542 :a542 :a542. +:a543 :a543 :a543. +:a544 :a544 :a544. +:a545 :a545 :a545. +:a546 :a546 :a546. +:a547 :a547 :a547. +:a548 :a548 :a548. +:a549 :a549 :a549. +:a550 :a550 :a550. +:a551 :a551 :a551. +:a552 :a552 :a552. +:a553 :a553 :a553. +:a554 :a554 :a554. +:a555 :a555 :a555. +:a556 :a556 :a556. +:a557 :a557 :a557. +:a558 :a558 :a558. +:a559 :a559 :a559. +:a560 :a560 :a560. +:a561 :a561 :a561. +:a562 :a562 :a562. +:a563 :a563 :a563. +:a564 :a564 :a564. +:a565 :a565 :a565. +:a566 :a566 :a566. +:a567 :a567 :a567. +:a568 :a568 :a568. +:a569 :a569 :a569. +:a570 :a570 :a570. +:a571 :a571 :a571. +:a572 :a572 :a572. +:a573 :a573 :a573. +:a574 :a574 :a574. +:a575 :a575 :a575. +:a576 :a576 :a576. +:a577 :a577 :a577. +:a578 :a578 :a578. +:a579 :a579 :a579. +:a580 :a580 :a580. +:a581 :a581 :a581. +:a582 :a582 :a582. +:a583 :a583 :a583. +:a584 :a584 :a584. +:a585 :a585 :a585. +:a586 :a586 :a586. +:a587 :a587 :a587. +:a588 :a588 :a588. +:a589 :a589 :a589. +:a590 :a590 :a590. +:a591 :a591 :a591. +:a592 :a592 :a592. +:a593 :a593 :a593. +:a594 :a594 :a594. +:a595 :a595 :a595. +:a596 :a596 :a596. +:a597 :a597 :a597. +:a598 :a598 :a598. +:a599 :a599 :a599. +:a600 :a600 :a600. +:a601 :a601 :a601. +:a602 :a602 :a602. +:a603 :a603 :a603. +:a604 :a604 :a604. +:a605 :a605 :a605. +:a606 :a606 :a606. +:a607 :a607 :a607. +:a608 :a608 :a608. +:a609 :a609 :a609. +:a610 :a610 :a610. +:a611 :a611 :a611. +:a612 :a612 :a612. +:a613 :a613 :a613. +:a614 :a614 :a614. +:a615 :a615 :a615. +:a616 :a616 :a616. +:a617 :a617 :a617. +:a618 :a618 :a618. +:a619 :a619 :a619. +:a620 :a620 :a620. +:a621 :a621 :a621. +:a622 :a622 :a622. +:a623 :a623 :a623. +:a624 :a624 :a624. +:a625 :a625 :a625. +:a626 :a626 :a626. +:a627 :a627 :a627. +:a628 :a628 :a628. +:a629 :a629 :a629. +:a630 :a630 :a630. +:a631 :a631 :a631. +:a632 :a632 :a632. +:a633 :a633 :a633. +:a634 :a634 :a634. +:a635 :a635 :a635. +:a636 :a636 :a636. +:a637 :a637 :a637. +:a638 :a638 :a638. +:a639 :a639 :a639. +:a640 :a640 :a640. +:a641 :a641 :a641. +:a642 :a642 :a642. +:a643 :a643 :a643. +:a644 :a644 :a644. +:a645 :a645 :a645. +:a646 :a646 :a646. +:a647 :a647 :a647. +:a648 :a648 :a648. +:a649 :a649 :a649. +:a650 :a650 :a650. +:a651 :a651 :a651. +:a652 :a652 :a652. +:a653 :a653 :a653. +:a654 :a654 :a654. +:a655 :a655 :a655. +:a656 :a656 :a656. +:a657 :a657 :a657. +:a658 :a658 :a658. +:a659 :a659 :a659. +:a660 :a660 :a660. +:a661 :a661 :a661. +:a662 :a662 :a662. +:a663 :a663 :a663. +:a664 :a664 :a664. +:a665 :a665 :a665. +:a666 :a666 :a666. +:a667 :a667 :a667. +:a668 :a668 :a668. +:a669 :a669 :a669. +:a670 :a670 :a670. +:a671 :a671 :a671. +:a672 :a672 :a672. +:a673 :a673 :a673. +:a674 :a674 :a674. +:a675 :a675 :a675. +:a676 :a676 :a676. +:a677 :a677 :a677. +:a678 :a678 :a678. +:a679 :a679 :a679. +:a680 :a680 :a680. +:a681 :a681 :a681. +:a682 :a682 :a682. +:a683 :a683 :a683. +:a684 :a684 :a684. +:a685 :a685 :a685. +:a686 :a686 :a686. +:a687 :a687 :a687. +:a688 :a688 :a688. +:a689 :a689 :a689. +:a690 :a690 :a690. +:a691 :a691 :a691. +:a692 :a692 :a692. +:a693 :a693 :a693. +:a694 :a694 :a694. +:a695 :a695 :a695. +:a696 :a696 :a696. +:a697 :a697 :a697. +:a698 :a698 :a698. +:a699 :a699 :a699. +:a700 :a700 :a700. +:a701 :a701 :a701. +:a702 :a702 :a702. +:a703 :a703 :a703. +:a704 :a704 :a704. +:a705 :a705 :a705. +:a706 :a706 :a706. +:a707 :a707 :a707. +:a708 :a708 :a708. +:a709 :a709 :a709. +:a710 :a710 :a710. +:a711 :a711 :a711. +:a712 :a712 :a712. +:a713 :a713 :a713. +:a714 :a714 :a714. +:a715 :a715 :a715. +:a716 :a716 :a716. +:a717 :a717 :a717. +:a718 :a718 :a718. +:a719 :a719 :a719. +:a720 :a720 :a720. +:a721 :a721 :a721. +:a722 :a722 :a722. +:a723 :a723 :a723. +:a724 :a724 :a724. +:a725 :a725 :a725. +:a726 :a726 :a726. +:a727 :a727 :a727. +:a728 :a728 :a728. +:a729 :a729 :a729. +:a730 :a730 :a730. +:a731 :a731 :a731. +:a732 :a732 :a732. +:a733 :a733 :a733. +:a734 :a734 :a734. +:a735 :a735 :a735. +:a736 :a736 :a736. +:a737 :a737 :a737. +:a738 :a738 :a738. +:a739 :a739 :a739. +:a740 :a740 :a740. +:a741 :a741 :a741. +:a742 :a742 :a742. +:a743 :a743 :a743. +:a744 :a744 :a744. +:a745 :a745 :a745. +:a746 :a746 :a746. +:a747 :a747 :a747. +:a748 :a748 :a748. +:a749 :a749 :a749. +:a750 :a750 :a750. +:a751 :a751 :a751. +:a752 :a752 :a752. +:a753 :a753 :a753. +:a754 :a754 :a754. +:a755 :a755 :a755. +:a756 :a756 :a756. +:a757 :a757 :a757. +:a758 :a758 :a758. +:a759 :a759 :a759. +:a760 :a760 :a760. +:a761 :a761 :a761. +:a762 :a762 :a762. +:a763 :a763 :a763. +:a764 :a764 :a764. +:a765 :a765 :a765. +:a766 :a766 :a766. +:a767 :a767 :a767. +:a768 :a768 :a768. +:a769 :a769 :a769. +:a770 :a770 :a770. +:a771 :a771 :a771. +:a772 :a772 :a772. +:a773 :a773 :a773. +:a774 :a774 :a774. +:a775 :a775 :a775. +:a776 :a776 :a776. +:a777 :a777 :a777. +:a778 :a778 :a778. +:a779 :a779 :a779. +:a780 :a780 :a780. +:a781 :a781 :a781. +:a782 :a782 :a782. +:a783 :a783 :a783. +:a784 :a784 :a784. +:a785 :a785 :a785. +:a786 :a786 :a786. +:a787 :a787 :a787. +:a788 :a788 :a788. +:a789 :a789 :a789. +:a790 :a790 :a790. +:a791 :a791 :a791. +:a792 :a792 :a792. +:a793 :a793 :a793. +:a794 :a794 :a794. +:a795 :a795 :a795. +:a796 :a796 :a796. +:a797 :a797 :a797. +:a798 :a798 :a798. +:a799 :a799 :a799. +:a800 :a800 :a800. +:a801 :a801 :a801. +:a802 :a802 :a802. +:a803 :a803 :a803. +:a804 :a804 :a804. +:a805 :a805 :a805. +:a806 :a806 :a806. +:a807 :a807 :a807. +:a808 :a808 :a808. +:a809 :a809 :a809. +:a810 :a810 :a810. +:a811 :a811 :a811. +:a812 :a812 :a812. +:a813 :a813 :a813. +:a814 :a814 :a814. +:a815 :a815 :a815. +:a816 :a816 :a816. +:a817 :a817 :a817. +:a818 :a818 :a818. +:a819 :a819 :a819. +:a820 :a820 :a820. +:a821 :a821 :a821. +:a822 :a822 :a822. +:a823 :a823 :a823. +:a824 :a824 :a824. +:a825 :a825 :a825. +:a826 :a826 :a826. +:a827 :a827 :a827. +:a828 :a828 :a828. +:a829 :a829 :a829. +:a830 :a830 :a830. +:a831 :a831 :a831. +:a832 :a832 :a832. +:a833 :a833 :a833. +:a834 :a834 :a834. +:a835 :a835 :a835. +:a836 :a836 :a836. +:a837 :a837 :a837. +:a838 :a838 :a838. +:a839 :a839 :a839. +:a840 :a840 :a840. +:a841 :a841 :a841. +:a842 :a842 :a842. +:a843 :a843 :a843. +:a844 :a844 :a844. +:a845 :a845 :a845. +:a846 :a846 :a846. +:a847 :a847 :a847. +:a848 :a848 :a848. +:a849 :a849 :a849. +:a850 :a850 :a850. +:a851 :a851 :a851. +:a852 :a852 :a852. +:a853 :a853 :a853. +:a854 :a854 :a854. +:a855 :a855 :a855. +:a856 :a856 :a856. +:a857 :a857 :a857. +:a858 :a858 :a858. +:a859 :a859 :a859. +:a860 :a860 :a860. +:a861 :a861 :a861. +:a862 :a862 :a862. +:a863 :a863 :a863. +:a864 :a864 :a864. +:a865 :a865 :a865. +:a866 :a866 :a866. +:a867 :a867 :a867. +:a868 :a868 :a868. +:a869 :a869 :a869. +:a870 :a870 :a870. +:a871 :a871 :a871. +:a872 :a872 :a872. +:a873 :a873 :a873. +:a874 :a874 :a874. +:a875 :a875 :a875. +:a876 :a876 :a876. +:a877 :a877 :a877. +:a878 :a878 :a878. +:a879 :a879 :a879. +:a880 :a880 :a880. +:a881 :a881 :a881. +:a882 :a882 :a882. +:a883 :a883 :a883. +:a884 :a884 :a884. +:a885 :a885 :a885. +:a886 :a886 :a886. +:a887 :a887 :a887. +:a888 :a888 :a888. +:a889 :a889 :a889. +:a890 :a890 :a890. +:a891 :a891 :a891. +:a892 :a892 :a892. +:a893 :a893 :a893. +:a894 :a894 :a894. +:a895 :a895 :a895. +:a896 :a896 :a896. +:a897 :a897 :a897. +:a898 :a898 :a898. +:a899 :a899 :a899. +:a900 :a900 :a900. +:a901 :a901 :a901. +:a902 :a902 :a902. +:a903 :a903 :a903. +:a904 :a904 :a904. +:a905 :a905 :a905. +:a906 :a906 :a906. +:a907 :a907 :a907. +:a908 :a908 :a908. +:a909 :a909 :a909. +:a910 :a910 :a910. +:a911 :a911 :a911. +:a912 :a912 :a912. +:a913 :a913 :a913. +:a914 :a914 :a914. +:a915 :a915 :a915. +:a916 :a916 :a916. +:a917 :a917 :a917. +:a918 :a918 :a918. +:a919 :a919 :a919. +:a920 :a920 :a920. +:a921 :a921 :a921. +:a922 :a922 :a922. +:a923 :a923 :a923. +:a924 :a924 :a924. +:a925 :a925 :a925. +:a926 :a926 :a926. +:a927 :a927 :a927. +:a928 :a928 :a928. +:a929 :a929 :a929. +:a930 :a930 :a930. +:a931 :a931 :a931. +:a932 :a932 :a932. +:a933 :a933 :a933. +:a934 :a934 :a934. +:a935 :a935 :a935. +:a936 :a936 :a936. +:a937 :a937 :a937. +:a938 :a938 :a938. +:a939 :a939 :a939. +:a940 :a940 :a940. +:a941 :a941 :a941. +:a942 :a942 :a942. +:a943 :a943 :a943. +:a944 :a944 :a944. +:a945 :a945 :a945. +:a946 :a946 :a946. +:a947 :a947 :a947. +:a948 :a948 :a948. +:a949 :a949 :a949. +:a950 :a950 :a950. +:a951 :a951 :a951. +:a952 :a952 :a952. +:a953 :a953 :a953. +:a954 :a954 :a954. +:a955 :a955 :a955. +:a956 :a956 :a956. +:a957 :a957 :a957. +:a958 :a958 :a958. +:a959 :a959 :a959. +:a960 :a960 :a960. +:a961 :a961 :a961. +:a962 :a962 :a962. +:a963 :a963 :a963. +:a964 :a964 :a964. +:a965 :a965 :a965. +:a966 :a966 :a966. +:a967 :a967 :a967. +:a968 :a968 :a968. +:a969 :a969 :a969. +:a970 :a970 :a970. +:a971 :a971 :a971. +:a972 :a972 :a972. +:a973 :a973 :a973. +:a974 :a974 :a974. +:a975 :a975 :a975. +:a976 :a976 :a976. +:a977 :a977 :a977. +:a978 :a978 :a978. +:a979 :a979 :a979. +:a980 :a980 :a980. +:a981 :a981 :a981. +:a982 :a982 :a982. +:a983 :a983 :a983. +:a984 :a984 :a984. +:a985 :a985 :a985. +:a986 :a986 :a986. +:a987 :a987 :a987. +:a988 :a988 :a988. +:a989 :a989 :a989. +:a990 :a990 :a990. +:a991 :a991 :a991. +:a992 :a992 :a992. +:a993 :a993 :a993. +:a994 :a994 :a994. +:a995 :a995 :a995. +:a996 :a996 :a996. +:a997 :a997 :a997. +:a998 :a998 :a998. +:a999 :a999 :a999. +:a1000 :a1000 :a1000. +:a1001 :a1001 :a1001. +:a1002 :a1002 :a1002. +:a1003 :a1003 :a1003. +:a1004 :a1004 :a1004. +:a1005 :a1005 :a1005. +:a1006 :a1006 :a1006. +:a1007 :a1007 :a1007. +:a1008 :a1008 :a1008. +:a1009 :a1009 :a1009. +:a1010 :a1010 :a1010. +:a1011 :a1011 :a1011. +:a1012 :a1012 :a1012. +:a1013 :a1013 :a1013. +:a1014 :a1014 :a1014. +:a1015 :a1015 :a1015. +:a1016 :a1016 :a1016. +:a1017 :a1017 :a1017. +:a1018 :a1018 :a1018. +:a1019 :a1019 :a1019. +:a1020 :a1020 :a1020. +:a1021 :a1021 :a1021. +:a1022 :a1022 :a1022. +:a1023 :a1023 :a1023. +:a1024 :a1024 :a1024. +:a1025 :a1025 :a1025. +:a1026 :a1026 :a1026. +:a1027 :a1027 :a1027. +:a1028 :a1028 :a1028. +:a1029 :a1029 :a1029. +:a1030 :a1030 :a1030. +:a1031 :a1031 :a1031. +:a1032 :a1032 :a1032. +:a1033 :a1033 :a1033. +:a1034 :a1034 :a1034. +:a1035 :a1035 :a1035. +:a1036 :a1036 :a1036. +:a1037 :a1037 :a1037. +:a1038 :a1038 :a1038. +:a1039 :a1039 :a1039. +:a1040 :a1040 :a1040. +:a1041 :a1041 :a1041. +:a1042 :a1042 :a1042. +:a1043 :a1043 :a1043. +:a1044 :a1044 :a1044. +:a1045 :a1045 :a1045. +:a1046 :a1046 :a1046. +:a1047 :a1047 :a1047. +:a1048 :a1048 :a1048. +:a1049 :a1049 :a1049. +:a1050 :a1050 :a1050. +:a1051 :a1051 :a1051. +:a1052 :a1052 :a1052. +:a1053 :a1053 :a1053. +:a1054 :a1054 :a1054. +:a1055 :a1055 :a1055. +:a1056 :a1056 :a1056. +:a1057 :a1057 :a1057. +:a1058 :a1058 :a1058. +:a1059 :a1059 :a1059. +:a1060 :a1060 :a1060. +:a1061 :a1061 :a1061. +:a1062 :a1062 :a1062. +:a1063 :a1063 :a1063. +:a1064 :a1064 :a1064. +:a1065 :a1065 :a1065. +:a1066 :a1066 :a1066. +:a1067 :a1067 :a1067. +:a1068 :a1068 :a1068. +:a1069 :a1069 :a1069. +:a1070 :a1070 :a1070. +:a1071 :a1071 :a1071. +:a1072 :a1072 :a1072. +:a1073 :a1073 :a1073. +:a1074 :a1074 :a1074. +:a1075 :a1075 :a1075. +:a1076 :a1076 :a1076. +:a1077 :a1077 :a1077. +:a1078 :a1078 :a1078. +:a1079 :a1079 :a1079. +:a1080 :a1080 :a1080. +:a1081 :a1081 :a1081. +:a1082 :a1082 :a1082. +:a1083 :a1083 :a1083. +:a1084 :a1084 :a1084. +:a1085 :a1085 :a1085. +:a1086 :a1086 :a1086. +:a1087 :a1087 :a1087. +:a1088 :a1088 :a1088. +:a1089 :a1089 :a1089. +:a1090 :a1090 :a1090. +:a1091 :a1091 :a1091. +:a1092 :a1092 :a1092. +:a1093 :a1093 :a1093. +:a1094 :a1094 :a1094. +:a1095 :a1095 :a1095. +:a1096 :a1096 :a1096. +:a1097 :a1097 :a1097. +:a1098 :a1098 :a1098. +:a1099 :a1099 :a1099. +:a1100 :a1100 :a1100. +:a1101 :a1101 :a1101. +:a1102 :a1102 :a1102. +:a1103 :a1103 :a1103. +:a1104 :a1104 :a1104. +:a1105 :a1105 :a1105. +:a1106 :a1106 :a1106. +:a1107 :a1107 :a1107. +:a1108 :a1108 :a1108. +:a1109 :a1109 :a1109. +:a1110 :a1110 :a1110. +:a1111 :a1111 :a1111. +:a1112 :a1112 :a1112. +:a1113 :a1113 :a1113. +:a1114 :a1114 :a1114. +:a1115 :a1115 :a1115. +:a1116 :a1116 :a1116. +:a1117 :a1117 :a1117. +:a1118 :a1118 :a1118. +:a1119 :a1119 :a1119. +:a1120 :a1120 :a1120. +:a1121 :a1121 :a1121. +:a1122 :a1122 :a1122. +:a1123 :a1123 :a1123. +:a1124 :a1124 :a1124. +:a1125 :a1125 :a1125. +:a1126 :a1126 :a1126. +:a1127 :a1127 :a1127. +:a1128 :a1128 :a1128. +:a1129 :a1129 :a1129. +:a1130 :a1130 :a1130. +:a1131 :a1131 :a1131. +:a1132 :a1132 :a1132. +:a1133 :a1133 :a1133. +:a1134 :a1134 :a1134. +:a1135 :a1135 :a1135. +:a1136 :a1136 :a1136. +:a1137 :a1137 :a1137. +:a1138 :a1138 :a1138. +:a1139 :a1139 :a1139. +:a1140 :a1140 :a1140. +:a1141 :a1141 :a1141. +:a1142 :a1142 :a1142. +:a1143 :a1143 :a1143. +:a1144 :a1144 :a1144. +:a1145 :a1145 :a1145. +:a1146 :a1146 :a1146. +:a1147 :a1147 :a1147. +:a1148 :a1148 :a1148. +:a1149 :a1149 :a1149. +:a1150 :a1150 :a1150. +:a1151 :a1151 :a1151. +:a1152 :a1152 :a1152. +:a1153 :a1153 :a1153. +:a1154 :a1154 :a1154. +:a1155 :a1155 :a1155. +:a1156 :a1156 :a1156. +:a1157 :a1157 :a1157. +:a1158 :a1158 :a1158. +:a1159 :a1159 :a1159. +:a1160 :a1160 :a1160. +:a1161 :a1161 :a1161. +:a1162 :a1162 :a1162. +:a1163 :a1163 :a1163. +:a1164 :a1164 :a1164. +:a1165 :a1165 :a1165. +:a1166 :a1166 :a1166. +:a1167 :a1167 :a1167. +:a1168 :a1168 :a1168. +:a1169 :a1169 :a1169. +:a1170 :a1170 :a1170. +:a1171 :a1171 :a1171. +:a1172 :a1172 :a1172. +:a1173 :a1173 :a1173. +:a1174 :a1174 :a1174. +:a1175 :a1175 :a1175. +:a1176 :a1176 :a1176. +:a1177 :a1177 :a1177. +:a1178 :a1178 :a1178. +:a1179 :a1179 :a1179. +:a1180 :a1180 :a1180. +:a1181 :a1181 :a1181. +:a1182 :a1182 :a1182. +:a1183 :a1183 :a1183. +:a1184 :a1184 :a1184. +:a1185 :a1185 :a1185. +:a1186 :a1186 :a1186. +:a1187 :a1187 :a1187. +:a1188 :a1188 :a1188. +:a1189 :a1189 :a1189. +:a1190 :a1190 :a1190. +:a1191 :a1191 :a1191. +:a1192 :a1192 :a1192. +:a1193 :a1193 :a1193. +:a1194 :a1194 :a1194. +:a1195 :a1195 :a1195. +:a1196 :a1196 :a1196. +:a1197 :a1197 :a1197. +:a1198 :a1198 :a1198. +:a1199 :a1199 :a1199. +:a1200 :a1200 :a1200. +:a1201 :a1201 :a1201. +:a1202 :a1202 :a1202. +:a1203 :a1203 :a1203. +:a1204 :a1204 :a1204. +:a1205 :a1205 :a1205. +:a1206 :a1206 :a1206. +:a1207 :a1207 :a1207. +:a1208 :a1208 :a1208. +:a1209 :a1209 :a1209. +:a1210 :a1210 :a1210. +:a1211 :a1211 :a1211. +:a1212 :a1212 :a1212. +:a1213 :a1213 :a1213. +:a1214 :a1214 :a1214. +:a1215 :a1215 :a1215. +:a1216 :a1216 :a1216. +:a1217 :a1217 :a1217. +:a1218 :a1218 :a1218. +:a1219 :a1219 :a1219. +:a1220 :a1220 :a1220. +:a1221 :a1221 :a1221. +:a1222 :a1222 :a1222. +:a1223 :a1223 :a1223. +:a1224 :a1224 :a1224. +:a1225 :a1225 :a1225. +:a1226 :a1226 :a1226. +:a1227 :a1227 :a1227. +:a1228 :a1228 :a1228. +:a1229 :a1229 :a1229. +:a1230 :a1230 :a1230. +:a1231 :a1231 :a1231. +:a1232 :a1232 :a1232. +:a1233 :a1233 :a1233. +:a1234 :a1234 :a1234. +:a1235 :a1235 :a1235. +:a1236 :a1236 :a1236. +:a1237 :a1237 :a1237. +:a1238 :a1238 :a1238. +:a1239 :a1239 :a1239. +:a1240 :a1240 :a1240. +:a1241 :a1241 :a1241. +:a1242 :a1242 :a1242. +:a1243 :a1243 :a1243. +:a1244 :a1244 :a1244. +:a1245 :a1245 :a1245. +:a1246 :a1246 :a1246. +:a1247 :a1247 :a1247. +:a1248 :a1248 :a1248. +:a1249 :a1249 :a1249. +:a1250 :a1250 :a1250. +:a1251 :a1251 :a1251. +:a1252 :a1252 :a1252. +:a1253 :a1253 :a1253. +:a1254 :a1254 :a1254. +:a1255 :a1255 :a1255. +:a1256 :a1256 :a1256. +:a1257 :a1257 :a1257. +:a1258 :a1258 :a1258. +:a1259 :a1259 :a1259. +:a1260 :a1260 :a1260. +:a1261 :a1261 :a1261. +:a1262 :a1262 :a1262. +:a1263 :a1263 :a1263. +:a1264 :a1264 :a1264. +:a1265 :a1265 :a1265. +:a1266 :a1266 :a1266. +:a1267 :a1267 :a1267. +:a1268 :a1268 :a1268. +:a1269 :a1269 :a1269. +:a1270 :a1270 :a1270. +:a1271 :a1271 :a1271. +:a1272 :a1272 :a1272. +:a1273 :a1273 :a1273. +:a1274 :a1274 :a1274. +:a1275 :a1275 :a1275. +:a1276 :a1276 :a1276. +:a1277 :a1277 :a1277. +:a1278 :a1278 :a1278. +:a1279 :a1279 :a1279. +:a1280 :a1280 :a1280. +:a1281 :a1281 :a1281. +:a1282 :a1282 :a1282. +:a1283 :a1283 :a1283. +:a1284 :a1284 :a1284. +:a1285 :a1285 :a1285. +:a1286 :a1286 :a1286. +:a1287 :a1287 :a1287. +:a1288 :a1288 :a1288. +:a1289 :a1289 :a1289. +:a1290 :a1290 :a1290. +:a1291 :a1291 :a1291. +:a1292 :a1292 :a1292. +:a1293 :a1293 :a1293. +:a1294 :a1294 :a1294. +:a1295 :a1295 :a1295. +:a1296 :a1296 :a1296. +:a1297 :a1297 :a1297. +:a1298 :a1298 :a1298. +:a1299 :a1299 :a1299. +:a1300 :a1300 :a1300. +:a1301 :a1301 :a1301. +:a1302 :a1302 :a1302. +:a1303 :a1303 :a1303. +:a1304 :a1304 :a1304. +:a1305 :a1305 :a1305. +:a1306 :a1306 :a1306. +:a1307 :a1307 :a1307. +:a1308 :a1308 :a1308. +:a1309 :a1309 :a1309. +:a1310 :a1310 :a1310. +:a1311 :a1311 :a1311. +:a1312 :a1312 :a1312. +:a1313 :a1313 :a1313. +:a1314 :a1314 :a1314. +:a1315 :a1315 :a1315. +:a1316 :a1316 :a1316. +:a1317 :a1317 :a1317. +:a1318 :a1318 :a1318. +:a1319 :a1319 :a1319. +:a1320 :a1320 :a1320. +:a1321 :a1321 :a1321. +:a1322 :a1322 :a1322. +:a1323 :a1323 :a1323. +:a1324 :a1324 :a1324. +:a1325 :a1325 :a1325. +:a1326 :a1326 :a1326. +:a1327 :a1327 :a1327. +:a1328 :a1328 :a1328. +:a1329 :a1329 :a1329. +:a1330 :a1330 :a1330. +:a1331 :a1331 :a1331. +:a1332 :a1332 :a1332. +:a1333 :a1333 :a1333. +:a1334 :a1334 :a1334. +:a1335 :a1335 :a1335. +:a1336 :a1336 :a1336. +:a1337 :a1337 :a1337. +:a1338 :a1338 :a1338. +:a1339 :a1339 :a1339. +:a1340 :a1340 :a1340. +:a1341 :a1341 :a1341. +:a1342 :a1342 :a1342. +:a1343 :a1343 :a1343. +:a1344 :a1344 :a1344. +:a1345 :a1345 :a1345. +:a1346 :a1346 :a1346. +:a1347 :a1347 :a1347. +:a1348 :a1348 :a1348. +:a1349 :a1349 :a1349. +:a1350 :a1350 :a1350. +:a1351 :a1351 :a1351. +:a1352 :a1352 :a1352. +:a1353 :a1353 :a1353. +:a1354 :a1354 :a1354. +:a1355 :a1355 :a1355. +:a1356 :a1356 :a1356. +:a1357 :a1357 :a1357. +:a1358 :a1358 :a1358. +:a1359 :a1359 :a1359. +:a1360 :a1360 :a1360. +:a1361 :a1361 :a1361. +:a1362 :a1362 :a1362. +:a1363 :a1363 :a1363. +:a1364 :a1364 :a1364. +:a1365 :a1365 :a1365. +:a1366 :a1366 :a1366. +:a1367 :a1367 :a1367. +:a1368 :a1368 :a1368. +:a1369 :a1369 :a1369. +:a1370 :a1370 :a1370. +:a1371 :a1371 :a1371. +:a1372 :a1372 :a1372. +:a1373 :a1373 :a1373. +:a1374 :a1374 :a1374. +:a1375 :a1375 :a1375. +:a1376 :a1376 :a1376. +:a1377 :a1377 :a1377. +:a1378 :a1378 :a1378. +:a1379 :a1379 :a1379. +:a1380 :a1380 :a1380. +:a1381 :a1381 :a1381. +:a1382 :a1382 :a1382. +:a1383 :a1383 :a1383. +:a1384 :a1384 :a1384. +:a1385 :a1385 :a1385. +:a1386 :a1386 :a1386. +:a1387 :a1387 :a1387. +:a1388 :a1388 :a1388. +:a1389 :a1389 :a1389. +:a1390 :a1390 :a1390. +:a1391 :a1391 :a1391. +:a1392 :a1392 :a1392. +:a1393 :a1393 :a1393. +:a1394 :a1394 :a1394. +:a1395 :a1395 :a1395. +:a1396 :a1396 :a1396. +:a1397 :a1397 :a1397. +:a1398 :a1398 :a1398. +:a1399 :a1399 :a1399. +:a1400 :a1400 :a1400. +:a1401 :a1401 :a1401. +:a1402 :a1402 :a1402. +:a1403 :a1403 :a1403. +:a1404 :a1404 :a1404. +:a1405 :a1405 :a1405. +:a1406 :a1406 :a1406. +:a1407 :a1407 :a1407. +:a1408 :a1408 :a1408. +:a1409 :a1409 :a1409. +:a1410 :a1410 :a1410. +:a1411 :a1411 :a1411. +:a1412 :a1412 :a1412. +:a1413 :a1413 :a1413. +:a1414 :a1414 :a1414. +:a1415 :a1415 :a1415. +:a1416 :a1416 :a1416. +:a1417 :a1417 :a1417. +:a1418 :a1418 :a1418. +:a1419 :a1419 :a1419. +:a1420 :a1420 :a1420. +:a1421 :a1421 :a1421. +:a1422 :a1422 :a1422. +:a1423 :a1423 :a1423. +:a1424 :a1424 :a1424. +:a1425 :a1425 :a1425. +:a1426 :a1426 :a1426. +:a1427 :a1427 :a1427. +:a1428 :a1428 :a1428. +:a1429 :a1429 :a1429. +:a1430 :a1430 :a1430. +:a1431 :a1431 :a1431. +:a1432 :a1432 :a1432. +:a1433 :a1433 :a1433. +:a1434 :a1434 :a1434. +:a1435 :a1435 :a1435. +:a1436 :a1436 :a1436. +:a1437 :a1437 :a1437. +:a1438 :a1438 :a1438. +:a1439 :a1439 :a1439. +:a1440 :a1440 :a1440. +:a1441 :a1441 :a1441. +:a1442 :a1442 :a1442. +:a1443 :a1443 :a1443. +:a1444 :a1444 :a1444. +:a1445 :a1445 :a1445. +:a1446 :a1446 :a1446. +:a1447 :a1447 :a1447. +:a1448 :a1448 :a1448. +:a1449 :a1449 :a1449. +:a1450 :a1450 :a1450. +:a1451 :a1451 :a1451. +:a1452 :a1452 :a1452. +:a1453 :a1453 :a1453. +:a1454 :a1454 :a1454. +:a1455 :a1455 :a1455. +:a1456 :a1456 :a1456. +:a1457 :a1457 :a1457. +:a1458 :a1458 :a1458. +:a1459 :a1459 :a1459. +:a1460 :a1460 :a1460. +:a1461 :a1461 :a1461. +:a1462 :a1462 :a1462. +:a1463 :a1463 :a1463. +:a1464 :a1464 :a1464. +:a1465 :a1465 :a1465. +:a1466 :a1466 :a1466. +:a1467 :a1467 :a1467. +:a1468 :a1468 :a1468. +:a1469 :a1469 :a1469. +:a1470 :a1470 :a1470. +:a1471 :a1471 :a1471. +:a1472 :a1472 :a1472. +:a1473 :a1473 :a1473. +:a1474 :a1474 :a1474. +:a1475 :a1475 :a1475. +:a1476 :a1476 :a1476. +:a1477 :a1477 :a1477. +:a1478 :a1478 :a1478. +:a1479 :a1479 :a1479. +:a1480 :a1480 :a1480. +:a1481 :a1481 :a1481. +:a1482 :a1482 :a1482. +:a1483 :a1483 :a1483. +:a1484 :a1484 :a1484. +:a1485 :a1485 :a1485. +:a1486 :a1486 :a1486. +:a1487 :a1487 :a1487. +:a1488 :a1488 :a1488. +:a1489 :a1489 :a1489. +:a1490 :a1490 :a1490. +:a1491 :a1491 :a1491. +:a1492 :a1492 :a1492. +:a1493 :a1493 :a1493. +:a1494 :a1494 :a1494. +:a1495 :a1495 :a1495. +:a1496 :a1496 :a1496. +:a1497 :a1497 :a1497. +:a1498 :a1498 :a1498. +:a1499 :a1499 :a1499. +:a1500 :a1500 :a1500. +:a1501 :a1501 :a1501. +:a1502 :a1502 :a1502. +:a1503 :a1503 :a1503. +:a1504 :a1504 :a1504. +:a1505 :a1505 :a1505. +:a1506 :a1506 :a1506. +:a1507 :a1507 :a1507. +:a1508 :a1508 :a1508. +:a1509 :a1509 :a1509. +:a1510 :a1510 :a1510. +:a1511 :a1511 :a1511. +:a1512 :a1512 :a1512. +:a1513 :a1513 :a1513. +:a1514 :a1514 :a1514. +:a1515 :a1515 :a1515. +:a1516 :a1516 :a1516. +:a1517 :a1517 :a1517. +:a1518 :a1518 :a1518. +:a1519 :a1519 :a1519. +:a1520 :a1520 :a1520. +:a1521 :a1521 :a1521. +:a1522 :a1522 :a1522. +:a1523 :a1523 :a1523. +:a1524 :a1524 :a1524. +:a1525 :a1525 :a1525. +:a1526 :a1526 :a1526. +:a1527 :a1527 :a1527. +:a1528 :a1528 :a1528. +:a1529 :a1529 :a1529. +:a1530 :a1530 :a1530. +:a1531 :a1531 :a1531. +:a1532 :a1532 :a1532. +:a1533 :a1533 :a1533. +:a1534 :a1534 :a1534. +:a1535 :a1535 :a1535. +:a1536 :a1536 :a1536. +:a1537 :a1537 :a1537. +:a1538 :a1538 :a1538. +:a1539 :a1539 :a1539. +:a1540 :a1540 :a1540. +:a1541 :a1541 :a1541. +:a1542 :a1542 :a1542. +:a1543 :a1543 :a1543. +:a1544 :a1544 :a1544. +:a1545 :a1545 :a1545. +:a1546 :a1546 :a1546. +:a1547 :a1547 :a1547. +:a1548 :a1548 :a1548. +:a1549 :a1549 :a1549. +:a1550 :a1550 :a1550. +:a1551 :a1551 :a1551. +:a1552 :a1552 :a1552. +:a1553 :a1553 :a1553. +:a1554 :a1554 :a1554. +:a1555 :a1555 :a1555. +:a1556 :a1556 :a1556. +:a1557 :a1557 :a1557. +:a1558 :a1558 :a1558. +:a1559 :a1559 :a1559. +:a1560 :a1560 :a1560. +:a1561 :a1561 :a1561. +:a1562 :a1562 :a1562. +:a1563 :a1563 :a1563. +:a1564 :a1564 :a1564. +:a1565 :a1565 :a1565. +:a1566 :a1566 :a1566. +:a1567 :a1567 :a1567. +:a1568 :a1568 :a1568. +:a1569 :a1569 :a1569. +:a1570 :a1570 :a1570. +:a1571 :a1571 :a1571. +:a1572 :a1572 :a1572. +:a1573 :a1573 :a1573. +:a1574 :a1574 :a1574. +:a1575 :a1575 :a1575. +:a1576 :a1576 :a1576. +:a1577 :a1577 :a1577. +:a1578 :a1578 :a1578. +:a1579 :a1579 :a1579. +:a1580 :a1580 :a1580. +:a1581 :a1581 :a1581. +:a1582 :a1582 :a1582. +:a1583 :a1583 :a1583. +:a1584 :a1584 :a1584. +:a1585 :a1585 :a1585. +:a1586 :a1586 :a1586. +:a1587 :a1587 :a1587. +:a1588 :a1588 :a1588. +:a1589 :a1589 :a1589. +:a1590 :a1590 :a1590. +:a1591 :a1591 :a1591. +:a1592 :a1592 :a1592. +:a1593 :a1593 :a1593. +:a1594 :a1594 :a1594. +:a1595 :a1595 :a1595. +:a1596 :a1596 :a1596. +:a1597 :a1597 :a1597. +:a1598 :a1598 :a1598. +:a1599 :a1599 :a1599. +:a1600 :a1600 :a1600. +:a1601 :a1601 :a1601. +:a1602 :a1602 :a1602. +:a1603 :a1603 :a1603. +:a1604 :a1604 :a1604. +:a1605 :a1605 :a1605. +:a1606 :a1606 :a1606. +:a1607 :a1607 :a1607. +:a1608 :a1608 :a1608. +:a1609 :a1609 :a1609. +:a1610 :a1610 :a1610. +:a1611 :a1611 :a1611. +:a1612 :a1612 :a1612. +:a1613 :a1613 :a1613. +:a1614 :a1614 :a1614. +:a1615 :a1615 :a1615. +:a1616 :a1616 :a1616. +:a1617 :a1617 :a1617. +:a1618 :a1618 :a1618. +:a1619 :a1619 :a1619. +:a1620 :a1620 :a1620. +:a1621 :a1621 :a1621. +:a1622 :a1622 :a1622. +:a1623 :a1623 :a1623. +:a1624 :a1624 :a1624. +:a1625 :a1625 :a1625. +:a1626 :a1626 :a1626. +:a1627 :a1627 :a1627. +:a1628 :a1628 :a1628. +:a1629 :a1629 :a1629. +:a1630 :a1630 :a1630. +:a1631 :a1631 :a1631. +:a1632 :a1632 :a1632. +:a1633 :a1633 :a1633. +:a1634 :a1634 :a1634. +:a1635 :a1635 :a1635. +:a1636 :a1636 :a1636. +:a1637 :a1637 :a1637. +:a1638 :a1638 :a1638. +:a1639 :a1639 :a1639. +:a1640 :a1640 :a1640. +:a1641 :a1641 :a1641. +:a1642 :a1642 :a1642. +:a1643 :a1643 :a1643. +:a1644 :a1644 :a1644. +:a1645 :a1645 :a1645. +:a1646 :a1646 :a1646. +:a1647 :a1647 :a1647. +:a1648 :a1648 :a1648. +:a1649 :a1649 :a1649. +:a1650 :a1650 :a1650. +:a1651 :a1651 :a1651. +:a1652 :a1652 :a1652. +:a1653 :a1653 :a1653. +:a1654 :a1654 :a1654. +:a1655 :a1655 :a1655. +:a1656 :a1656 :a1656. +:a1657 :a1657 :a1657. +:a1658 :a1658 :a1658. +:a1659 :a1659 :a1659. +:a1660 :a1660 :a1660. +:a1661 :a1661 :a1661. +:a1662 :a1662 :a1662. +:a1663 :a1663 :a1663. +:a1664 :a1664 :a1664. +:a1665 :a1665 :a1665. +:a1666 :a1666 :a1666. +:a1667 :a1667 :a1667. +:a1668 :a1668 :a1668. +:a1669 :a1669 :a1669. +:a1670 :a1670 :a1670. +:a1671 :a1671 :a1671. +:a1672 :a1672 :a1672. +:a1673 :a1673 :a1673. +:a1674 :a1674 :a1674. +:a1675 :a1675 :a1675. +:a1676 :a1676 :a1676. +:a1677 :a1677 :a1677. +:a1678 :a1678 :a1678. +:a1679 :a1679 :a1679. +:a1680 :a1680 :a1680. +:a1681 :a1681 :a1681. +:a1682 :a1682 :a1682. +:a1683 :a1683 :a1683. +:a1684 :a1684 :a1684. +:a1685 :a1685 :a1685. +:a1686 :a1686 :a1686. +:a1687 :a1687 :a1687. +:a1688 :a1688 :a1688. +:a1689 :a1689 :a1689. +:a1690 :a1690 :a1690. +:a1691 :a1691 :a1691. +:a1692 :a1692 :a1692. +:a1693 :a1693 :a1693. +:a1694 :a1694 :a1694. +:a1695 :a1695 :a1695. +:a1696 :a1696 :a1696. +:a1697 :a1697 :a1697. +:a1698 :a1698 :a1698. +:a1699 :a1699 :a1699. +:a1700 :a1700 :a1700. +:a1701 :a1701 :a1701. +:a1702 :a1702 :a1702. +:a1703 :a1703 :a1703. +:a1704 :a1704 :a1704. +:a1705 :a1705 :a1705. +:a1706 :a1706 :a1706. +:a1707 :a1707 :a1707. +:a1708 :a1708 :a1708. +:a1709 :a1709 :a1709. +:a1710 :a1710 :a1710. +:a1711 :a1711 :a1711. +:a1712 :a1712 :a1712. +:a1713 :a1713 :a1713. +:a1714 :a1714 :a1714. +:a1715 :a1715 :a1715. +:a1716 :a1716 :a1716. +:a1717 :a1717 :a1717. +:a1718 :a1718 :a1718. +:a1719 :a1719 :a1719. +:a1720 :a1720 :a1720. +:a1721 :a1721 :a1721. +:a1722 :a1722 :a1722. +:a1723 :a1723 :a1723. +:a1724 :a1724 :a1724. +:a1725 :a1725 :a1725. +:a1726 :a1726 :a1726. +:a1727 :a1727 :a1727. +:a1728 :a1728 :a1728. +:a1729 :a1729 :a1729. +:a1730 :a1730 :a1730. +:a1731 :a1731 :a1731. +:a1732 :a1732 :a1732. +:a1733 :a1733 :a1733. +:a1734 :a1734 :a1734. +:a1735 :a1735 :a1735. +:a1736 :a1736 :a1736. +:a1737 :a1737 :a1737. +:a1738 :a1738 :a1738. +:a1739 :a1739 :a1739. +:a1740 :a1740 :a1740. +:a1741 :a1741 :a1741. +:a1742 :a1742 :a1742. +:a1743 :a1743 :a1743. +:a1744 :a1744 :a1744. +:a1745 :a1745 :a1745. +:a1746 :a1746 :a1746. +:a1747 :a1747 :a1747. +:a1748 :a1748 :a1748. +:a1749 :a1749 :a1749. +:a1750 :a1750 :a1750. +:a1751 :a1751 :a1751. +:a1752 :a1752 :a1752. +:a1753 :a1753 :a1753. +:a1754 :a1754 :a1754. +:a1755 :a1755 :a1755. +:a1756 :a1756 :a1756. +:a1757 :a1757 :a1757. +:a1758 :a1758 :a1758. +:a1759 :a1759 :a1759. +:a1760 :a1760 :a1760. +:a1761 :a1761 :a1761. +:a1762 :a1762 :a1762. +:a1763 :a1763 :a1763. +:a1764 :a1764 :a1764. +:a1765 :a1765 :a1765. +:a1766 :a1766 :a1766. +:a1767 :a1767 :a1767. +:a1768 :a1768 :a1768. +:a1769 :a1769 :a1769. +:a1770 :a1770 :a1770. +:a1771 :a1771 :a1771. +:a1772 :a1772 :a1772. +:a1773 :a1773 :a1773. +:a1774 :a1774 :a1774. +:a1775 :a1775 :a1775. +:a1776 :a1776 :a1776. +:a1777 :a1777 :a1777. +:a1778 :a1778 :a1778. +:a1779 :a1779 :a1779. +:a1780 :a1780 :a1780. +:a1781 :a1781 :a1781. +:a1782 :a1782 :a1782. +:a1783 :a1783 :a1783. +:a1784 :a1784 :a1784. +:a1785 :a1785 :a1785. +:a1786 :a1786 :a1786. +:a1787 :a1787 :a1787. +:a1788 :a1788 :a1788. +:a1789 :a1789 :a1789. +:a1790 :a1790 :a1790. +:a1791 :a1791 :a1791. +:a1792 :a1792 :a1792. +:a1793 :a1793 :a1793. +:a1794 :a1794 :a1794. +:a1795 :a1795 :a1795. +:a1796 :a1796 :a1796. +:a1797 :a1797 :a1797. +:a1798 :a1798 :a1798. +:a1799 :a1799 :a1799. +:a1800 :a1800 :a1800. +:a1801 :a1801 :a1801. +:a1802 :a1802 :a1802. +:a1803 :a1803 :a1803. +:a1804 :a1804 :a1804. +:a1805 :a1805 :a1805. +:a1806 :a1806 :a1806. +:a1807 :a1807 :a1807. +:a1808 :a1808 :a1808. +:a1809 :a1809 :a1809. +:a1810 :a1810 :a1810. +:a1811 :a1811 :a1811. +:a1812 :a1812 :a1812. +:a1813 :a1813 :a1813. +:a1814 :a1814 :a1814. +:a1815 :a1815 :a1815. +:a1816 :a1816 :a1816. +:a1817 :a1817 :a1817. +:a1818 :a1818 :a1818. +:a1819 :a1819 :a1819. +:a1820 :a1820 :a1820. +:a1821 :a1821 :a1821. +:a1822 :a1822 :a1822. +:a1823 :a1823 :a1823. +:a1824 :a1824 :a1824. +:a1825 :a1825 :a1825. +:a1826 :a1826 :a1826. +:a1827 :a1827 :a1827. +:a1828 :a1828 :a1828. +:a1829 :a1829 :a1829. +:a1830 :a1830 :a1830. +:a1831 :a1831 :a1831. +:a1832 :a1832 :a1832. +:a1833 :a1833 :a1833. +:a1834 :a1834 :a1834. +:a1835 :a1835 :a1835. +:a1836 :a1836 :a1836. +:a1837 :a1837 :a1837. +:a1838 :a1838 :a1838. +:a1839 :a1839 :a1839. +:a1840 :a1840 :a1840. +:a1841 :a1841 :a1841. +:a1842 :a1842 :a1842. +:a1843 :a1843 :a1843. +:a1844 :a1844 :a1844. +:a1845 :a1845 :a1845. +:a1846 :a1846 :a1846. +:a1847 :a1847 :a1847. +:a1848 :a1848 :a1848. +:a1849 :a1849 :a1849. +:a1850 :a1850 :a1850. +:a1851 :a1851 :a1851. +:a1852 :a1852 :a1852. +:a1853 :a1853 :a1853. +:a1854 :a1854 :a1854. +:a1855 :a1855 :a1855. +:a1856 :a1856 :a1856. +:a1857 :a1857 :a1857. +:a1858 :a1858 :a1858. +:a1859 :a1859 :a1859. +:a1860 :a1860 :a1860. +:a1861 :a1861 :a1861. +:a1862 :a1862 :a1862. +:a1863 :a1863 :a1863. +:a1864 :a1864 :a1864. +:a1865 :a1865 :a1865. +:a1866 :a1866 :a1866. +:a1867 :a1867 :a1867. +:a1868 :a1868 :a1868. +:a1869 :a1869 :a1869. +:a1870 :a1870 :a1870. +:a1871 :a1871 :a1871. +:a1872 :a1872 :a1872. +:a1873 :a1873 :a1873. +:a1874 :a1874 :a1874. +:a1875 :a1875 :a1875. +:a1876 :a1876 :a1876. +:a1877 :a1877 :a1877. +:a1878 :a1878 :a1878. +:a1879 :a1879 :a1879. +:a1880 :a1880 :a1880. +:a1881 :a1881 :a1881. +:a1882 :a1882 :a1882. +:a1883 :a1883 :a1883. +:a1884 :a1884 :a1884. +:a1885 :a1885 :a1885. +:a1886 :a1886 :a1886. +:a1887 :a1887 :a1887. +:a1888 :a1888 :a1888. +:a1889 :a1889 :a1889. +:a1890 :a1890 :a1890. +:a1891 :a1891 :a1891. +:a1892 :a1892 :a1892. +:a1893 :a1893 :a1893. +:a1894 :a1894 :a1894. +:a1895 :a1895 :a1895. +:a1896 :a1896 :a1896. +:a1897 :a1897 :a1897. +:a1898 :a1898 :a1898. +:a1899 :a1899 :a1899. +:a1900 :a1900 :a1900. +:a1901 :a1901 :a1901. +:a1902 :a1902 :a1902. +:a1903 :a1903 :a1903. +:a1904 :a1904 :a1904. +:a1905 :a1905 :a1905. +:a1906 :a1906 :a1906. +:a1907 :a1907 :a1907. +:a1908 :a1908 :a1908. +:a1909 :a1909 :a1909. +:a1910 :a1910 :a1910. +:a1911 :a1911 :a1911. +:a1912 :a1912 :a1912. +:a1913 :a1913 :a1913. +:a1914 :a1914 :a1914. +:a1915 :a1915 :a1915. +:a1916 :a1916 :a1916. +:a1917 :a1917 :a1917. +:a1918 :a1918 :a1918. +:a1919 :a1919 :a1919. +:a1920 :a1920 :a1920. +:a1921 :a1921 :a1921. +:a1922 :a1922 :a1922. +:a1923 :a1923 :a1923. +:a1924 :a1924 :a1924. +:a1925 :a1925 :a1925. +:a1926 :a1926 :a1926. +:a1927 :a1927 :a1927. +:a1928 :a1928 :a1928. +:a1929 :a1929 :a1929. +:a1930 :a1930 :a1930. +:a1931 :a1931 :a1931. +:a1932 :a1932 :a1932. +:a1933 :a1933 :a1933. +:a1934 :a1934 :a1934. +:a1935 :a1935 :a1935. +:a1936 :a1936 :a1936. +:a1937 :a1937 :a1937. +:a1938 :a1938 :a1938. +:a1939 :a1939 :a1939. +:a1940 :a1940 :a1940. +:a1941 :a1941 :a1941. +:a1942 :a1942 :a1942. +:a1943 :a1943 :a1943. +:a1944 :a1944 :a1944. +:a1945 :a1945 :a1945. +:a1946 :a1946 :a1946. +:a1947 :a1947 :a1947. +:a1948 :a1948 :a1948. +:a1949 :a1949 :a1949. +:a1950 :a1950 :a1950. +:a1951 :a1951 :a1951. +:a1952 :a1952 :a1952. +:a1953 :a1953 :a1953. +:a1954 :a1954 :a1954. +:a1955 :a1955 :a1955. +:a1956 :a1956 :a1956. +:a1957 :a1957 :a1957. +:a1958 :a1958 :a1958. +:a1959 :a1959 :a1959. +:a1960 :a1960 :a1960. +:a1961 :a1961 :a1961. +:a1962 :a1962 :a1962. +:a1963 :a1963 :a1963. +:a1964 :a1964 :a1964. +:a1965 :a1965 :a1965. +:a1966 :a1966 :a1966. +:a1967 :a1967 :a1967. +:a1968 :a1968 :a1968. +:a1969 :a1969 :a1969. +:a1970 :a1970 :a1970. +:a1971 :a1971 :a1971. +:a1972 :a1972 :a1972. +:a1973 :a1973 :a1973. +:a1974 :a1974 :a1974. +:a1975 :a1975 :a1975. +:a1976 :a1976 :a1976. +:a1977 :a1977 :a1977. +:a1978 :a1978 :a1978. +:a1979 :a1979 :a1979. +:a1980 :a1980 :a1980. +:a1981 :a1981 :a1981. +:a1982 :a1982 :a1982. +:a1983 :a1983 :a1983. +:a1984 :a1984 :a1984. +:a1985 :a1985 :a1985. +:a1986 :a1986 :a1986. +:a1987 :a1987 :a1987. +:a1988 :a1988 :a1988. +:a1989 :a1989 :a1989. +:a1990 :a1990 :a1990. +:a1991 :a1991 :a1991. +:a1992 :a1992 :a1992. +:a1993 :a1993 :a1993. +:a1994 :a1994 :a1994. +:a1995 :a1995 :a1995. +:a1996 :a1996 :a1996. +:a1997 :a1997 :a1997. +:a1998 :a1998 :a1998. +:a1999 :a1999 :a1999. +:a2000 :a2000 :a2000. +:a2001 :a2001 :a2001. +:a2002 :a2002 :a2002. +:a2003 :a2003 :a2003. +:a2004 :a2004 :a2004. +:a2005 :a2005 :a2005. +:a2006 :a2006 :a2006. +:a2007 :a2007 :a2007. +:a2008 :a2008 :a2008. +:a2009 :a2009 :a2009. +:a2010 :a2010 :a2010. +:a2011 :a2011 :a2011. +:a2012 :a2012 :a2012. +:a2013 :a2013 :a2013. +:a2014 :a2014 :a2014. +:a2015 :a2015 :a2015. +:a2016 :a2016 :a2016. +:a2017 :a2017 :a2017. +:a2018 :a2018 :a2018. +:a2019 :a2019 :a2019. +:a2020 :a2020 :a2020. +:a2021 :a2021 :a2021. +:a2022 :a2022 :a2022. +:a2023 :a2023 :a2023. +:a2024 :a2024 :a2024. +:a2025 :a2025 :a2025. +:a2026 :a2026 :a2026. +:a2027 :a2027 :a2027. +:a2028 :a2028 :a2028. +:a2029 :a2029 :a2029. +:a2030 :a2030 :a2030. +:a2031 :a2031 :a2031. +:a2032 :a2032 :a2032. +:a2033 :a2033 :a2033. +:a2034 :a2034 :a2034. +:a2035 :a2035 :a2035. +:a2036 :a2036 :a2036. +:a2037 :a2037 :a2037. +:a2038 :a2038 :a2038. +:a2039 :a2039 :a2039. +:a2040 :a2040 :a2040. +:a2041 :a2041 :a2041. +:a2042 :a2042 :a2042. +:a2043 :a2043 :a2043. +:a2044 :a2044 :a2044. +:a2045 :a2045 :a2045. +:a2046 :a2046 :a2046. +:a2047 :a2047 :a2047. +:a2048 :a2048 :a2048. +:a2049 :a2049 :a2049. +:a2050 :a2050 :a2050. +:a2051 :a2051 :a2051. +:a2052 :a2052 :a2052. +:a2053 :a2053 :a2053. +:a2054 :a2054 :a2054. +:a2055 :a2055 :a2055. +:a2056 :a2056 :a2056. +:a2057 :a2057 :a2057. +:a2058 :a2058 :a2058. +:a2059 :a2059 :a2059. +:a2060 :a2060 :a2060. +:a2061 :a2061 :a2061. +:a2062 :a2062 :a2062. +:a2063 :a2063 :a2063. +:a2064 :a2064 :a2064. +:a2065 :a2065 :a2065. +:a2066 :a2066 :a2066. +:a2067 :a2067 :a2067. +:a2068 :a2068 :a2068. +:a2069 :a2069 :a2069. +:a2070 :a2070 :a2070. +:a2071 :a2071 :a2071. +:a2072 :a2072 :a2072. +:a2073 :a2073 :a2073. +:a2074 :a2074 :a2074. +:a2075 :a2075 :a2075. +:a2076 :a2076 :a2076. +:a2077 :a2077 :a2077. +:a2078 :a2078 :a2078. +:a2079 :a2079 :a2079. +:a2080 :a2080 :a2080. +:a2081 :a2081 :a2081. +:a2082 :a2082 :a2082. +:a2083 :a2083 :a2083. +:a2084 :a2084 :a2084. +:a2085 :a2085 :a2085. +:a2086 :a2086 :a2086. +:a2087 :a2087 :a2087. +:a2088 :a2088 :a2088. +:a2089 :a2089 :a2089. +:a2090 :a2090 :a2090. +:a2091 :a2091 :a2091. +:a2092 :a2092 :a2092. +:a2093 :a2093 :a2093. +:a2094 :a2094 :a2094. +:a2095 :a2095 :a2095. +:a2096 :a2096 :a2096. +:a2097 :a2097 :a2097. +:a2098 :a2098 :a2098. +:a2099 :a2099 :a2099. +:a2100 :a2100 :a2100. +:a2101 :a2101 :a2101. +:a2102 :a2102 :a2102. +:a2103 :a2103 :a2103. +:a2104 :a2104 :a2104. +:a2105 :a2105 :a2105. +:a2106 :a2106 :a2106. +:a2107 :a2107 :a2107. +:a2108 :a2108 :a2108. +:a2109 :a2109 :a2109. +:a2110 :a2110 :a2110. +:a2111 :a2111 :a2111. +:a2112 :a2112 :a2112. +:a2113 :a2113 :a2113. +:a2114 :a2114 :a2114. +:a2115 :a2115 :a2115. +:a2116 :a2116 :a2116. +:a2117 :a2117 :a2117. +:a2118 :a2118 :a2118. +:a2119 :a2119 :a2119. +:a2120 :a2120 :a2120. +:a2121 :a2121 :a2121. +:a2122 :a2122 :a2122. +:a2123 :a2123 :a2123. +:a2124 :a2124 :a2124. +:a2125 :a2125 :a2125. +:a2126 :a2126 :a2126. +:a2127 :a2127 :a2127. +:a2128 :a2128 :a2128. +:a2129 :a2129 :a2129. +:a2130 :a2130 :a2130. +:a2131 :a2131 :a2131. +:a2132 :a2132 :a2132. +:a2133 :a2133 :a2133. +:a2134 :a2134 :a2134. +:a2135 :a2135 :a2135. +:a2136 :a2136 :a2136. +:a2137 :a2137 :a2137. +:a2138 :a2138 :a2138. +:a2139 :a2139 :a2139. +:a2140 :a2140 :a2140. +:a2141 :a2141 :a2141. +:a2142 :a2142 :a2142. +:a2143 :a2143 :a2143. +:a2144 :a2144 :a2144. +:a2145 :a2145 :a2145. +:a2146 :a2146 :a2146. +:a2147 :a2147 :a2147. +:a2148 :a2148 :a2148. +:a2149 :a2149 :a2149. +:a2150 :a2150 :a2150. +:a2151 :a2151 :a2151. +:a2152 :a2152 :a2152. +:a2153 :a2153 :a2153. +:a2154 :a2154 :a2154. +:a2155 :a2155 :a2155. +:a2156 :a2156 :a2156. +:a2157 :a2157 :a2157. +:a2158 :a2158 :a2158. +:a2159 :a2159 :a2159. +:a2160 :a2160 :a2160. +:a2161 :a2161 :a2161. +:a2162 :a2162 :a2162. +:a2163 :a2163 :a2163. +:a2164 :a2164 :a2164. +:a2165 :a2165 :a2165. +:a2166 :a2166 :a2166. +:a2167 :a2167 :a2167. +:a2168 :a2168 :a2168. +:a2169 :a2169 :a2169. +:a2170 :a2170 :a2170. +:a2171 :a2171 :a2171. +:a2172 :a2172 :a2172. +:a2173 :a2173 :a2173. +:a2174 :a2174 :a2174. +:a2175 :a2175 :a2175. +:a2176 :a2176 :a2176. +:a2177 :a2177 :a2177. +:a2178 :a2178 :a2178. +:a2179 :a2179 :a2179. +:a2180 :a2180 :a2180. +:a2181 :a2181 :a2181. +:a2182 :a2182 :a2182. +:a2183 :a2183 :a2183. +:a2184 :a2184 :a2184. +:a2185 :a2185 :a2185. +:a2186 :a2186 :a2186. +:a2187 :a2187 :a2187. +:a2188 :a2188 :a2188. +:a2189 :a2189 :a2189. +:a2190 :a2190 :a2190. +:a2191 :a2191 :a2191. +:a2192 :a2192 :a2192. +:a2193 :a2193 :a2193. +:a2194 :a2194 :a2194. +:a2195 :a2195 :a2195. +:a2196 :a2196 :a2196. +:a2197 :a2197 :a2197. +:a2198 :a2198 :a2198. +:a2199 :a2199 :a2199. +:a2200 :a2200 :a2200. +:a2201 :a2201 :a2201. +:a2202 :a2202 :a2202. +:a2203 :a2203 :a2203. +:a2204 :a2204 :a2204. +:a2205 :a2205 :a2205. +:a2206 :a2206 :a2206. +:a2207 :a2207 :a2207. +:a2208 :a2208 :a2208. +:a2209 :a2209 :a2209. +:a2210 :a2210 :a2210. +:a2211 :a2211 :a2211. +:a2212 :a2212 :a2212. +:a2213 :a2213 :a2213. +:a2214 :a2214 :a2214. +:a2215 :a2215 :a2215. +:a2216 :a2216 :a2216. +:a2217 :a2217 :a2217. +:a2218 :a2218 :a2218. +:a2219 :a2219 :a2219. +:a2220 :a2220 :a2220. +:a2221 :a2221 :a2221. +:a2222 :a2222 :a2222. +:a2223 :a2223 :a2223. +:a2224 :a2224 :a2224. +:a2225 :a2225 :a2225. +:a2226 :a2226 :a2226. +:a2227 :a2227 :a2227. +:a2228 :a2228 :a2228. +:a2229 :a2229 :a2229. +:a2230 :a2230 :a2230. +:a2231 :a2231 :a2231. +:a2232 :a2232 :a2232. +:a2233 :a2233 :a2233. +:a2234 :a2234 :a2234. +:a2235 :a2235 :a2235. +:a2236 :a2236 :a2236. +:a2237 :a2237 :a2237. +:a2238 :a2238 :a2238. +:a2239 :a2239 :a2239. +:a2240 :a2240 :a2240. +:a2241 :a2241 :a2241. +:a2242 :a2242 :a2242. +:a2243 :a2243 :a2243. +:a2244 :a2244 :a2244. +:a2245 :a2245 :a2245. +:a2246 :a2246 :a2246. +:a2247 :a2247 :a2247. +:a2248 :a2248 :a2248. +:a2249 :a2249 :a2249. +:a2250 :a2250 :a2250. +:a2251 :a2251 :a2251. +:a2252 :a2252 :a2252. +:a2253 :a2253 :a2253. +:a2254 :a2254 :a2254. +:a2255 :a2255 :a2255. +:a2256 :a2256 :a2256. +:a2257 :a2257 :a2257. +:a2258 :a2258 :a2258. +:a2259 :a2259 :a2259. +:a2260 :a2260 :a2260. +:a2261 :a2261 :a2261. +:a2262 :a2262 :a2262. +:a2263 :a2263 :a2263. +:a2264 :a2264 :a2264. +:a2265 :a2265 :a2265. +:a2266 :a2266 :a2266. +:a2267 :a2267 :a2267. +:a2268 :a2268 :a2268. +:a2269 :a2269 :a2269. +:a2270 :a2270 :a2270. +:a2271 :a2271 :a2271. +:a2272 :a2272 :a2272. +:a2273 :a2273 :a2273. +:a2274 :a2274 :a2274. +:a2275 :a2275 :a2275. +:a2276 :a2276 :a2276. +:a2277 :a2277 :a2277. +:a2278 :a2278 :a2278. +:a2279 :a2279 :a2279. +:a2280 :a2280 :a2280. +:a2281 :a2281 :a2281. +:a2282 :a2282 :a2282. +:a2283 :a2283 :a2283. +:a2284 :a2284 :a2284. +:a2285 :a2285 :a2285. +:a2286 :a2286 :a2286. +:a2287 :a2287 :a2287. +:a2288 :a2288 :a2288. +:a2289 :a2289 :a2289. +:a2290 :a2290 :a2290. +:a2291 :a2291 :a2291. +:a2292 :a2292 :a2292. +:a2293 :a2293 :a2293. +:a2294 :a2294 :a2294. +:a2295 :a2295 :a2295. +:a2296 :a2296 :a2296. +:a2297 :a2297 :a2297. +:a2298 :a2298 :a2298. +:a2299 :a2299 :a2299. +:a2300 :a2300 :a2300. +:a2301 :a2301 :a2301. +:a2302 :a2302 :a2302. +:a2303 :a2303 :a2303. +:a2304 :a2304 :a2304. +:a2305 :a2305 :a2305. +:a2306 :a2306 :a2306. +:a2307 :a2307 :a2307. +:a2308 :a2308 :a2308. +:a2309 :a2309 :a2309. +:a2310 :a2310 :a2310. +:a2311 :a2311 :a2311. +:a2312 :a2312 :a2312. +:a2313 :a2313 :a2313. +:a2314 :a2314 :a2314. +:a2315 :a2315 :a2315. +:a2316 :a2316 :a2316. +:a2317 :a2317 :a2317. +:a2318 :a2318 :a2318. +:a2319 :a2319 :a2319. +:a2320 :a2320 :a2320. +:a2321 :a2321 :a2321. +:a2322 :a2322 :a2322. +:a2323 :a2323 :a2323. +:a2324 :a2324 :a2324. +:a2325 :a2325 :a2325. +:a2326 :a2326 :a2326. +:a2327 :a2327 :a2327. +:a2328 :a2328 :a2328. +:a2329 :a2329 :a2329. +:a2330 :a2330 :a2330. +:a2331 :a2331 :a2331. +:a2332 :a2332 :a2332. +:a2333 :a2333 :a2333. +:a2334 :a2334 :a2334. +:a2335 :a2335 :a2335. +:a2336 :a2336 :a2336. +:a2337 :a2337 :a2337. +:a2338 :a2338 :a2338. +:a2339 :a2339 :a2339. +:a2340 :a2340 :a2340. +:a2341 :a2341 :a2341. +:a2342 :a2342 :a2342. +:a2343 :a2343 :a2343. +:a2344 :a2344 :a2344. +:a2345 :a2345 :a2345. +:a2346 :a2346 :a2346. +:a2347 :a2347 :a2347. +:a2348 :a2348 :a2348. +:a2349 :a2349 :a2349. +:a2350 :a2350 :a2350. +:a2351 :a2351 :a2351. +:a2352 :a2352 :a2352. +:a2353 :a2353 :a2353. +:a2354 :a2354 :a2354. +:a2355 :a2355 :a2355. +:a2356 :a2356 :a2356. +:a2357 :a2357 :a2357. +:a2358 :a2358 :a2358. +:a2359 :a2359 :a2359. +:a2360 :a2360 :a2360. +:a2361 :a2361 :a2361. +:a2362 :a2362 :a2362. +:a2363 :a2363 :a2363. +:a2364 :a2364 :a2364. +:a2365 :a2365 :a2365. +:a2366 :a2366 :a2366. +:a2367 :a2367 :a2367. +:a2368 :a2368 :a2368. +:a2369 :a2369 :a2369. +:a2370 :a2370 :a2370. +:a2371 :a2371 :a2371. +:a2372 :a2372 :a2372. +:a2373 :a2373 :a2373. +:a2374 :a2374 :a2374. +:a2375 :a2375 :a2375. +:a2376 :a2376 :a2376. +:a2377 :a2377 :a2377. +:a2378 :a2378 :a2378. +:a2379 :a2379 :a2379. +:a2380 :a2380 :a2380. +:a2381 :a2381 :a2381. +:a2382 :a2382 :a2382. +:a2383 :a2383 :a2383. +:a2384 :a2384 :a2384. +:a2385 :a2385 :a2385. +:a2386 :a2386 :a2386. +:a2387 :a2387 :a2387. +:a2388 :a2388 :a2388. +:a2389 :a2389 :a2389. +:a2390 :a2390 :a2390. +:a2391 :a2391 :a2391. +:a2392 :a2392 :a2392. +:a2393 :a2393 :a2393. +:a2394 :a2394 :a2394. +:a2395 :a2395 :a2395. +:a2396 :a2396 :a2396. +:a2397 :a2397 :a2397. +:a2398 :a2398 :a2398. +:a2399 :a2399 :a2399. +:a2400 :a2400 :a2400. +:a2401 :a2401 :a2401. +:a2402 :a2402 :a2402. +:a2403 :a2403 :a2403. +:a2404 :a2404 :a2404. +:a2405 :a2405 :a2405. +:a2406 :a2406 :a2406. +:a2407 :a2407 :a2407. +:a2408 :a2408 :a2408. +:a2409 :a2409 :a2409. +:a2410 :a2410 :a2410. +:a2411 :a2411 :a2411. +:a2412 :a2412 :a2412. +:a2413 :a2413 :a2413. +:a2414 :a2414 :a2414. +:a2415 :a2415 :a2415. +:a2416 :a2416 :a2416. +:a2417 :a2417 :a2417. +:a2418 :a2418 :a2418. +:a2419 :a2419 :a2419. +:a2420 :a2420 :a2420. +:a2421 :a2421 :a2421. +:a2422 :a2422 :a2422. +:a2423 :a2423 :a2423. +:a2424 :a2424 :a2424. +:a2425 :a2425 :a2425. +:a2426 :a2426 :a2426. +:a2427 :a2427 :a2427. +:a2428 :a2428 :a2428. +:a2429 :a2429 :a2429. +:a2430 :a2430 :a2430. +:a2431 :a2431 :a2431. +:a2432 :a2432 :a2432. +:a2433 :a2433 :a2433. +:a2434 :a2434 :a2434. +:a2435 :a2435 :a2435. +:a2436 :a2436 :a2436. +:a2437 :a2437 :a2437. +:a2438 :a2438 :a2438. +:a2439 :a2439 :a2439. +:a2440 :a2440 :a2440. +:a2441 :a2441 :a2441. +:a2442 :a2442 :a2442. +:a2443 :a2443 :a2443. +:a2444 :a2444 :a2444. +:a2445 :a2445 :a2445. +:a2446 :a2446 :a2446. +:a2447 :a2447 :a2447. +:a2448 :a2448 :a2448. +:a2449 :a2449 :a2449. +:a2450 :a2450 :a2450. +:a2451 :a2451 :a2451. +:a2452 :a2452 :a2452. +:a2453 :a2453 :a2453. +:a2454 :a2454 :a2454. +:a2455 :a2455 :a2455. +:a2456 :a2456 :a2456. +:a2457 :a2457 :a2457. +:a2458 :a2458 :a2458. +:a2459 :a2459 :a2459. +:a2460 :a2460 :a2460. +:a2461 :a2461 :a2461. +:a2462 :a2462 :a2462. +:a2463 :a2463 :a2463. +:a2464 :a2464 :a2464. +:a2465 :a2465 :a2465. +:a2466 :a2466 :a2466. +:a2467 :a2467 :a2467. +:a2468 :a2468 :a2468. +:a2469 :a2469 :a2469. +:a2470 :a2470 :a2470. +:a2471 :a2471 :a2471. +:a2472 :a2472 :a2472. +:a2473 :a2473 :a2473. +:a2474 :a2474 :a2474. +:a2475 :a2475 :a2475. +:a2476 :a2476 :a2476. +:a2477 :a2477 :a2477. +:a2478 :a2478 :a2478. +:a2479 :a2479 :a2479. +:a2480 :a2480 :a2480. +:a2481 :a2481 :a2481. +:a2482 :a2482 :a2482. +:a2483 :a2483 :a2483. +:a2484 :a2484 :a2484. +:a2485 :a2485 :a2485. +:a2486 :a2486 :a2486. +:a2487 :a2487 :a2487. +:a2488 :a2488 :a2488. +:a2489 :a2489 :a2489. +:a2490 :a2490 :a2490. +:a2491 :a2491 :a2491. +:a2492 :a2492 :a2492. +:a2493 :a2493 :a2493. +:a2494 :a2494 :a2494. +:a2495 :a2495 :a2495. +:a2496 :a2496 :a2496. +:a2497 :a2497 :a2497. +:a2498 :a2498 :a2498. +:a2499 :a2499 :a2499. +:a2500 :a2500 :a2500. +:a2501 :a2501 :a2501. +:a2502 :a2502 :a2502. +:a2503 :a2503 :a2503. +:a2504 :a2504 :a2504. +:a2505 :a2505 :a2505. +:a2506 :a2506 :a2506. +:a2507 :a2507 :a2507. +:a2508 :a2508 :a2508. +:a2509 :a2509 :a2509. +:a2510 :a2510 :a2510. +:a2511 :a2511 :a2511. +:a2512 :a2512 :a2512. +:a2513 :a2513 :a2513. +:a2514 :a2514 :a2514. +:a2515 :a2515 :a2515. +:a2516 :a2516 :a2516. +:a2517 :a2517 :a2517. +:a2518 :a2518 :a2518. +:a2519 :a2519 :a2519. +:a2520 :a2520 :a2520. +:a2521 :a2521 :a2521. +:a2522 :a2522 :a2522. +:a2523 :a2523 :a2523. +:a2524 :a2524 :a2524. +:a2525 :a2525 :a2525. +:a2526 :a2526 :a2526. +:a2527 :a2527 :a2527. +:a2528 :a2528 :a2528. +:a2529 :a2529 :a2529. +:a2530 :a2530 :a2530. +:a2531 :a2531 :a2531. +:a2532 :a2532 :a2532. +:a2533 :a2533 :a2533. +:a2534 :a2534 :a2534. +:a2535 :a2535 :a2535. +:a2536 :a2536 :a2536. +:a2537 :a2537 :a2537. +:a2538 :a2538 :a2538. +:a2539 :a2539 :a2539. +:a2540 :a2540 :a2540. +:a2541 :a2541 :a2541. +:a2542 :a2542 :a2542. +:a2543 :a2543 :a2543. +:a2544 :a2544 :a2544. +:a2545 :a2545 :a2545. +:a2546 :a2546 :a2546. +:a2547 :a2547 :a2547. +:a2548 :a2548 :a2548. +:a2549 :a2549 :a2549. +:a2550 :a2550 :a2550. +:a2551 :a2551 :a2551. +:a2552 :a2552 :a2552. +:a2553 :a2553 :a2553. +:a2554 :a2554 :a2554. +:a2555 :a2555 :a2555. +:a2556 :a2556 :a2556. +:a2557 :a2557 :a2557. +:a2558 :a2558 :a2558. +:a2559 :a2559 :a2559. +:a2560 :a2560 :a2560. +:a2561 :a2561 :a2561. +:a2562 :a2562 :a2562. +:a2563 :a2563 :a2563. +:a2564 :a2564 :a2564. +:a2565 :a2565 :a2565. +:a2566 :a2566 :a2566. +:a2567 :a2567 :a2567. +:a2568 :a2568 :a2568. +:a2569 :a2569 :a2569. +:a2570 :a2570 :a2570. +:a2571 :a2571 :a2571. +:a2572 :a2572 :a2572. +:a2573 :a2573 :a2573. +:a2574 :a2574 :a2574. +:a2575 :a2575 :a2575. +:a2576 :a2576 :a2576. +:a2577 :a2577 :a2577. +:a2578 :a2578 :a2578. +:a2579 :a2579 :a2579. +:a2580 :a2580 :a2580. +:a2581 :a2581 :a2581. +:a2582 :a2582 :a2582. +:a2583 :a2583 :a2583. +:a2584 :a2584 :a2584. +:a2585 :a2585 :a2585. +:a2586 :a2586 :a2586. +:a2587 :a2587 :a2587. +:a2588 :a2588 :a2588. +:a2589 :a2589 :a2589. +:a2590 :a2590 :a2590. +:a2591 :a2591 :a2591. +:a2592 :a2592 :a2592. +:a2593 :a2593 :a2593. +:a2594 :a2594 :a2594. +:a2595 :a2595 :a2595. +:a2596 :a2596 :a2596. +:a2597 :a2597 :a2597. +:a2598 :a2598 :a2598. +:a2599 :a2599 :a2599. +:a2600 :a2600 :a2600. +:a2601 :a2601 :a2601. +:a2602 :a2602 :a2602. +:a2603 :a2603 :a2603. +:a2604 :a2604 :a2604. +:a2605 :a2605 :a2605. +:a2606 :a2606 :a2606. +:a2607 :a2607 :a2607. +:a2608 :a2608 :a2608. +:a2609 :a2609 :a2609. +:a2610 :a2610 :a2610. +:a2611 :a2611 :a2611. +:a2612 :a2612 :a2612. +:a2613 :a2613 :a2613. +:a2614 :a2614 :a2614. +:a2615 :a2615 :a2615. +:a2616 :a2616 :a2616. +:a2617 :a2617 :a2617. +:a2618 :a2618 :a2618. +:a2619 :a2619 :a2619. +:a2620 :a2620 :a2620. +:a2621 :a2621 :a2621. +:a2622 :a2622 :a2622. +:a2623 :a2623 :a2623. +:a2624 :a2624 :a2624. +:a2625 :a2625 :a2625. +:a2626 :a2626 :a2626. +:a2627 :a2627 :a2627. +:a2628 :a2628 :a2628. +:a2629 :a2629 :a2629. +:a2630 :a2630 :a2630. +:a2631 :a2631 :a2631. +:a2632 :a2632 :a2632. +:a2633 :a2633 :a2633. +:a2634 :a2634 :a2634. +:a2635 :a2635 :a2635. +:a2636 :a2636 :a2636. +:a2637 :a2637 :a2637. +:a2638 :a2638 :a2638. +:a2639 :a2639 :a2639. +:a2640 :a2640 :a2640. +:a2641 :a2641 :a2641. +:a2642 :a2642 :a2642. +:a2643 :a2643 :a2643. +:a2644 :a2644 :a2644. +:a2645 :a2645 :a2645. +:a2646 :a2646 :a2646. +:a2647 :a2647 :a2647. +:a2648 :a2648 :a2648. +:a2649 :a2649 :a2649. +:a2650 :a2650 :a2650. +:a2651 :a2651 :a2651. +:a2652 :a2652 :a2652. +:a2653 :a2653 :a2653. +:a2654 :a2654 :a2654. +:a2655 :a2655 :a2655. +:a2656 :a2656 :a2656. +:a2657 :a2657 :a2657. +:a2658 :a2658 :a2658. +:a2659 :a2659 :a2659. +:a2660 :a2660 :a2660. +:a2661 :a2661 :a2661. +:a2662 :a2662 :a2662. +:a2663 :a2663 :a2663. +:a2664 :a2664 :a2664. +:a2665 :a2665 :a2665. +:a2666 :a2666 :a2666. +:a2667 :a2667 :a2667. +:a2668 :a2668 :a2668. +:a2669 :a2669 :a2669. +:a2670 :a2670 :a2670. +:a2671 :a2671 :a2671. +:a2672 :a2672 :a2672. +:a2673 :a2673 :a2673. +:a2674 :a2674 :a2674. +:a2675 :a2675 :a2675. +:a2676 :a2676 :a2676. +:a2677 :a2677 :a2677. +:a2678 :a2678 :a2678. +:a2679 :a2679 :a2679. +:a2680 :a2680 :a2680. +:a2681 :a2681 :a2681. +:a2682 :a2682 :a2682. +:a2683 :a2683 :a2683. +:a2684 :a2684 :a2684. +:a2685 :a2685 :a2685. +:a2686 :a2686 :a2686. +:a2687 :a2687 :a2687. +:a2688 :a2688 :a2688. +:a2689 :a2689 :a2689. +:a2690 :a2690 :a2690. +:a2691 :a2691 :a2691. +:a2692 :a2692 :a2692. +:a2693 :a2693 :a2693. +:a2694 :a2694 :a2694. +:a2695 :a2695 :a2695. +:a2696 :a2696 :a2696. +:a2697 :a2697 :a2697. +:a2698 :a2698 :a2698. +:a2699 :a2699 :a2699. +:a2700 :a2700 :a2700. +:a2701 :a2701 :a2701. +:a2702 :a2702 :a2702. +:a2703 :a2703 :a2703. +:a2704 :a2704 :a2704. +:a2705 :a2705 :a2705. +:a2706 :a2706 :a2706. +:a2707 :a2707 :a2707. +:a2708 :a2708 :a2708. +:a2709 :a2709 :a2709. +:a2710 :a2710 :a2710. +:a2711 :a2711 :a2711. +:a2712 :a2712 :a2712. +:a2713 :a2713 :a2713. +:a2714 :a2714 :a2714. +:a2715 :a2715 :a2715. +:a2716 :a2716 :a2716. +:a2717 :a2717 :a2717. +:a2718 :a2718 :a2718. +:a2719 :a2719 :a2719. +:a2720 :a2720 :a2720. +:a2721 :a2721 :a2721. +:a2722 :a2722 :a2722. +:a2723 :a2723 :a2723. +:a2724 :a2724 :a2724. +:a2725 :a2725 :a2725. +:a2726 :a2726 :a2726. +:a2727 :a2727 :a2727. +:a2728 :a2728 :a2728. +:a2729 :a2729 :a2729. +:a2730 :a2730 :a2730. +:a2731 :a2731 :a2731. +:a2732 :a2732 :a2732. +:a2733 :a2733 :a2733. +:a2734 :a2734 :a2734. +:a2735 :a2735 :a2735. +:a2736 :a2736 :a2736. +:a2737 :a2737 :a2737. +:a2738 :a2738 :a2738. +:a2739 :a2739 :a2739. +:a2740 :a2740 :a2740. +:a2741 :a2741 :a2741. +:a2742 :a2742 :a2742. +:a2743 :a2743 :a2743. +:a2744 :a2744 :a2744. +:a2745 :a2745 :a2745. +:a2746 :a2746 :a2746. +:a2747 :a2747 :a2747. +:a2748 :a2748 :a2748. +:a2749 :a2749 :a2749. +:a2750 :a2750 :a2750. +:a2751 :a2751 :a2751. +:a2752 :a2752 :a2752. +:a2753 :a2753 :a2753. +:a2754 :a2754 :a2754. +:a2755 :a2755 :a2755. +:a2756 :a2756 :a2756. +:a2757 :a2757 :a2757. +:a2758 :a2758 :a2758. +:a2759 :a2759 :a2759. +:a2760 :a2760 :a2760. +:a2761 :a2761 :a2761. +:a2762 :a2762 :a2762. +:a2763 :a2763 :a2763. +:a2764 :a2764 :a2764. +:a2765 :a2765 :a2765. +:a2766 :a2766 :a2766. +:a2767 :a2767 :a2767. +:a2768 :a2768 :a2768. +:a2769 :a2769 :a2769. +:a2770 :a2770 :a2770. +:a2771 :a2771 :a2771. +:a2772 :a2772 :a2772. +:a2773 :a2773 :a2773. +:a2774 :a2774 :a2774. +:a2775 :a2775 :a2775. +:a2776 :a2776 :a2776. +:a2777 :a2777 :a2777. +:a2778 :a2778 :a2778. +:a2779 :a2779 :a2779. +:a2780 :a2780 :a2780. +:a2781 :a2781 :a2781. +:a2782 :a2782 :a2782. +:a2783 :a2783 :a2783. +:a2784 :a2784 :a2784. +:a2785 :a2785 :a2785. +:a2786 :a2786 :a2786. +:a2787 :a2787 :a2787. +:a2788 :a2788 :a2788. +:a2789 :a2789 :a2789. +:a2790 :a2790 :a2790. +:a2791 :a2791 :a2791. +:a2792 :a2792 :a2792. +:a2793 :a2793 :a2793. +:a2794 :a2794 :a2794. +:a2795 :a2795 :a2795. +:a2796 :a2796 :a2796. +:a2797 :a2797 :a2797. +:a2798 :a2798 :a2798. +:a2799 :a2799 :a2799. +:a2800 :a2800 :a2800. +:a2801 :a2801 :a2801. +:a2802 :a2802 :a2802. +:a2803 :a2803 :a2803. +:a2804 :a2804 :a2804. +:a2805 :a2805 :a2805. +:a2806 :a2806 :a2806. +:a2807 :a2807 :a2807. +:a2808 :a2808 :a2808. +:a2809 :a2809 :a2809. +:a2810 :a2810 :a2810. +:a2811 :a2811 :a2811. +:a2812 :a2812 :a2812. +:a2813 :a2813 :a2813. +:a2814 :a2814 :a2814. +:a2815 :a2815 :a2815. +:a2816 :a2816 :a2816. +:a2817 :a2817 :a2817. +:a2818 :a2818 :a2818. +:a2819 :a2819 :a2819. +:a2820 :a2820 :a2820. +:a2821 :a2821 :a2821. +:a2822 :a2822 :a2822. +:a2823 :a2823 :a2823. +:a2824 :a2824 :a2824. +:a2825 :a2825 :a2825. +:a2826 :a2826 :a2826. +:a2827 :a2827 :a2827. +:a2828 :a2828 :a2828. +:a2829 :a2829 :a2829. +:a2830 :a2830 :a2830. +:a2831 :a2831 :a2831. +:a2832 :a2832 :a2832. +:a2833 :a2833 :a2833. +:a2834 :a2834 :a2834. +:a2835 :a2835 :a2835. +:a2836 :a2836 :a2836. +:a2837 :a2837 :a2837. +:a2838 :a2838 :a2838. +:a2839 :a2839 :a2839. +:a2840 :a2840 :a2840. +:a2841 :a2841 :a2841. +:a2842 :a2842 :a2842. +:a2843 :a2843 :a2843. +:a2844 :a2844 :a2844. +:a2845 :a2845 :a2845. +:a2846 :a2846 :a2846. +:a2847 :a2847 :a2847. +:a2848 :a2848 :a2848. +:a2849 :a2849 :a2849. +:a2850 :a2850 :a2850. +:a2851 :a2851 :a2851. +:a2852 :a2852 :a2852. +:a2853 :a2853 :a2853. +:a2854 :a2854 :a2854. +:a2855 :a2855 :a2855. +:a2856 :a2856 :a2856. +:a2857 :a2857 :a2857. +:a2858 :a2858 :a2858. +:a2859 :a2859 :a2859. +:a2860 :a2860 :a2860. +:a2861 :a2861 :a2861. +:a2862 :a2862 :a2862. +:a2863 :a2863 :a2863. +:a2864 :a2864 :a2864. +:a2865 :a2865 :a2865. +:a2866 :a2866 :a2866. +:a2867 :a2867 :a2867. +:a2868 :a2868 :a2868. +:a2869 :a2869 :a2869. +:a2870 :a2870 :a2870. +:a2871 :a2871 :a2871. +:a2872 :a2872 :a2872. +:a2873 :a2873 :a2873. +:a2874 :a2874 :a2874. +:a2875 :a2875 :a2875. +:a2876 :a2876 :a2876. +:a2877 :a2877 :a2877. +:a2878 :a2878 :a2878. +:a2879 :a2879 :a2879. +:a2880 :a2880 :a2880. +:a2881 :a2881 :a2881. +:a2882 :a2882 :a2882. +:a2883 :a2883 :a2883. +:a2884 :a2884 :a2884. +:a2885 :a2885 :a2885. +:a2886 :a2886 :a2886. +:a2887 :a2887 :a2887. +:a2888 :a2888 :a2888. +:a2889 :a2889 :a2889. +:a2890 :a2890 :a2890. +:a2891 :a2891 :a2891. +:a2892 :a2892 :a2892. +:a2893 :a2893 :a2893. +:a2894 :a2894 :a2894. +:a2895 :a2895 :a2895. +:a2896 :a2896 :a2896. +:a2897 :a2897 :a2897. +:a2898 :a2898 :a2898. +:a2899 :a2899 :a2899. +:a2900 :a2900 :a2900. +:a2901 :a2901 :a2901. +:a2902 :a2902 :a2902. +:a2903 :a2903 :a2903. +:a2904 :a2904 :a2904. +:a2905 :a2905 :a2905. +:a2906 :a2906 :a2906. +:a2907 :a2907 :a2907. +:a2908 :a2908 :a2908. +:a2909 :a2909 :a2909. +:a2910 :a2910 :a2910. +:a2911 :a2911 :a2911. +:a2912 :a2912 :a2912. +:a2913 :a2913 :a2913. +:a2914 :a2914 :a2914. +:a2915 :a2915 :a2915. +:a2916 :a2916 :a2916. +:a2917 :a2917 :a2917. +:a2918 :a2918 :a2918. +:a2919 :a2919 :a2919. +:a2920 :a2920 :a2920. +:a2921 :a2921 :a2921. +:a2922 :a2922 :a2922. +:a2923 :a2923 :a2923. +:a2924 :a2924 :a2924. +:a2925 :a2925 :a2925. +:a2926 :a2926 :a2926. +:a2927 :a2927 :a2927. +:a2928 :a2928 :a2928. +:a2929 :a2929 :a2929. +:a2930 :a2930 :a2930. +:a2931 :a2931 :a2931. +:a2932 :a2932 :a2932. +:a2933 :a2933 :a2933. +:a2934 :a2934 :a2934. +:a2935 :a2935 :a2935. +:a2936 :a2936 :a2936. +:a2937 :a2937 :a2937. +:a2938 :a2938 :a2938. +:a2939 :a2939 :a2939. +:a2940 :a2940 :a2940. +:a2941 :a2941 :a2941. +:a2942 :a2942 :a2942. +:a2943 :a2943 :a2943. +:a2944 :a2944 :a2944. +:a2945 :a2945 :a2945. +:a2946 :a2946 :a2946. +:a2947 :a2947 :a2947. +:a2948 :a2948 :a2948. +:a2949 :a2949 :a2949. +:a2950 :a2950 :a2950. +:a2951 :a2951 :a2951. +:a2952 :a2952 :a2952. +:a2953 :a2953 :a2953. +:a2954 :a2954 :a2954. +:a2955 :a2955 :a2955. +:a2956 :a2956 :a2956. +:a2957 :a2957 :a2957. +:a2958 :a2958 :a2958. +:a2959 :a2959 :a2959. +:a2960 :a2960 :a2960. +:a2961 :a2961 :a2961. +:a2962 :a2962 :a2962. +:a2963 :a2963 :a2963. +:a2964 :a2964 :a2964. +:a2965 :a2965 :a2965. +:a2966 :a2966 :a2966. +:a2967 :a2967 :a2967. +:a2968 :a2968 :a2968. +:a2969 :a2969 :a2969. +:a2970 :a2970 :a2970. +:a2971 :a2971 :a2971. +:a2972 :a2972 :a2972. +:a2973 :a2973 :a2973. +:a2974 :a2974 :a2974. +:a2975 :a2975 :a2975. +:a2976 :a2976 :a2976. +:a2977 :a2977 :a2977. +:a2978 :a2978 :a2978. +:a2979 :a2979 :a2979. +:a2980 :a2980 :a2980. +:a2981 :a2981 :a2981. +:a2982 :a2982 :a2982. +:a2983 :a2983 :a2983. +:a2984 :a2984 :a2984. +:a2985 :a2985 :a2985. +:a2986 :a2986 :a2986. +:a2987 :a2987 :a2987. +:a2988 :a2988 :a2988. +:a2989 :a2989 :a2989. +:a2990 :a2990 :a2990. +:a2991 :a2991 :a2991. +:a2992 :a2992 :a2992. +:a2993 :a2993 :a2993. +:a2994 :a2994 :a2994. +:a2995 :a2995 :a2995. +:a2996 :a2996 :a2996. +:a2997 :a2997 :a2997. +:a2998 :a2998 :a2998. +:a2999 :a2999 :a2999. +:a3000 :a3000 :a3000. +:a3001 :a3001 :a3001. +:a3002 :a3002 :a3002. +:a3003 :a3003 :a3003. +:a3004 :a3004 :a3004. +:a3005 :a3005 :a3005. +:a3006 :a3006 :a3006. +:a3007 :a3007 :a3007. +:a3008 :a3008 :a3008. +:a3009 :a3009 :a3009. +:a3010 :a3010 :a3010. +:a3011 :a3011 :a3011. +:a3012 :a3012 :a3012. +:a3013 :a3013 :a3013. +:a3014 :a3014 :a3014. +:a3015 :a3015 :a3015. +:a3016 :a3016 :a3016. +:a3017 :a3017 :a3017. +:a3018 :a3018 :a3018. +:a3019 :a3019 :a3019. +:a3020 :a3020 :a3020. +:a3021 :a3021 :a3021. +:a3022 :a3022 :a3022. +:a3023 :a3023 :a3023. +:a3024 :a3024 :a3024. +:a3025 :a3025 :a3025. +:a3026 :a3026 :a3026. +:a3027 :a3027 :a3027. +:a3028 :a3028 :a3028. +:a3029 :a3029 :a3029. +:a3030 :a3030 :a3030. +:a3031 :a3031 :a3031. +:a3032 :a3032 :a3032. +:a3033 :a3033 :a3033. +:a3034 :a3034 :a3034. +:a3035 :a3035 :a3035. +:a3036 :a3036 :a3036. +:a3037 :a3037 :a3037. +:a3038 :a3038 :a3038. +:a3039 :a3039 :a3039. +:a3040 :a3040 :a3040. +:a3041 :a3041 :a3041. +:a3042 :a3042 :a3042. +:a3043 :a3043 :a3043. +:a3044 :a3044 :a3044. +:a3045 :a3045 :a3045. +:a3046 :a3046 :a3046. +:a3047 :a3047 :a3047. +:a3048 :a3048 :a3048. +:a3049 :a3049 :a3049. +:a3050 :a3050 :a3050. +:a3051 :a3051 :a3051. +:a3052 :a3052 :a3052. +:a3053 :a3053 :a3053. +:a3054 :a3054 :a3054. +:a3055 :a3055 :a3055. +:a3056 :a3056 :a3056. +:a3057 :a3057 :a3057. +:a3058 :a3058 :a3058. +:a3059 :a3059 :a3059. +:a3060 :a3060 :a3060. +:a3061 :a3061 :a3061. +:a3062 :a3062 :a3062. +:a3063 :a3063 :a3063. +:a3064 :a3064 :a3064. +:a3065 :a3065 :a3065. +:a3066 :a3066 :a3066. +:a3067 :a3067 :a3067. +:a3068 :a3068 :a3068. +:a3069 :a3069 :a3069. +:a3070 :a3070 :a3070. +:a3071 :a3071 :a3071. +:a3072 :a3072 :a3072. +:a3073 :a3073 :a3073. +:a3074 :a3074 :a3074. +:a3075 :a3075 :a3075. +:a3076 :a3076 :a3076. +:a3077 :a3077 :a3077. +:a3078 :a3078 :a3078. +:a3079 :a3079 :a3079. +:a3080 :a3080 :a3080. +:a3081 :a3081 :a3081. +:a3082 :a3082 :a3082. +:a3083 :a3083 :a3083. +:a3084 :a3084 :a3084. +:a3085 :a3085 :a3085. +:a3086 :a3086 :a3086. +:a3087 :a3087 :a3087. +:a3088 :a3088 :a3088. +:a3089 :a3089 :a3089. +:a3090 :a3090 :a3090. +:a3091 :a3091 :a3091. +:a3092 :a3092 :a3092. +:a3093 :a3093 :a3093. +:a3094 :a3094 :a3094. +:a3095 :a3095 :a3095. +:a3096 :a3096 :a3096. +:a3097 :a3097 :a3097. +:a3098 :a3098 :a3098. +:a3099 :a3099 :a3099. +:a3100 :a3100 :a3100. +:a3101 :a3101 :a3101. +:a3102 :a3102 :a3102. +:a3103 :a3103 :a3103. +:a3104 :a3104 :a3104. +:a3105 :a3105 :a3105. +:a3106 :a3106 :a3106. +:a3107 :a3107 :a3107. +:a3108 :a3108 :a3108. +:a3109 :a3109 :a3109. +:a3110 :a3110 :a3110. +:a3111 :a3111 :a3111. +:a3112 :a3112 :a3112. +:a3113 :a3113 :a3113. +:a3114 :a3114 :a3114. +:a3115 :a3115 :a3115. +:a3116 :a3116 :a3116. +:a3117 :a3117 :a3117. +:a3118 :a3118 :a3118. +:a3119 :a3119 :a3119. +:a3120 :a3120 :a3120. +:a3121 :a3121 :a3121. +:a3122 :a3122 :a3122. +:a3123 :a3123 :a3123. +:a3124 :a3124 :a3124. +:a3125 :a3125 :a3125. +:a3126 :a3126 :a3126. +:a3127 :a3127 :a3127. +:a3128 :a3128 :a3128. +:a3129 :a3129 :a3129. +:a3130 :a3130 :a3130. +:a3131 :a3131 :a3131. +:a3132 :a3132 :a3132. +:a3133 :a3133 :a3133. +:a3134 :a3134 :a3134. +:a3135 :a3135 :a3135. +:a3136 :a3136 :a3136. +:a3137 :a3137 :a3137. +:a3138 :a3138 :a3138. +:a3139 :a3139 :a3139. +:a3140 :a3140 :a3140. +:a3141 :a3141 :a3141. +:a3142 :a3142 :a3142. +:a3143 :a3143 :a3143. +:a3144 :a3144 :a3144. +:a3145 :a3145 :a3145. +:a3146 :a3146 :a3146. +:a3147 :a3147 :a3147. +:a3148 :a3148 :a3148. +:a3149 :a3149 :a3149. +:a3150 :a3150 :a3150. +:a3151 :a3151 :a3151. +:a3152 :a3152 :a3152. +:a3153 :a3153 :a3153. +:a3154 :a3154 :a3154. +:a3155 :a3155 :a3155. +:a3156 :a3156 :a3156. +:a3157 :a3157 :a3157. +:a3158 :a3158 :a3158. +:a3159 :a3159 :a3159. +:a3160 :a3160 :a3160. +:a3161 :a3161 :a3161. +:a3162 :a3162 :a3162. +:a3163 :a3163 :a3163. +:a3164 :a3164 :a3164. +:a3165 :a3165 :a3165. +:a3166 :a3166 :a3166. +:a3167 :a3167 :a3167. +:a3168 :a3168 :a3168. +:a3169 :a3169 :a3169. +:a3170 :a3170 :a3170. +:a3171 :a3171 :a3171. +:a3172 :a3172 :a3172. +:a3173 :a3173 :a3173. +:a3174 :a3174 :a3174. +:a3175 :a3175 :a3175. +:a3176 :a3176 :a3176. +:a3177 :a3177 :a3177. +:a3178 :a3178 :a3178. +:a3179 :a3179 :a3179. +:a3180 :a3180 :a3180. +:a3181 :a3181 :a3181. +:a3182 :a3182 :a3182. +:a3183 :a3183 :a3183. +:a3184 :a3184 :a3184. +:a3185 :a3185 :a3185. +:a3186 :a3186 :a3186. +:a3187 :a3187 :a3187. +:a3188 :a3188 :a3188. +:a3189 :a3189 :a3189. +:a3190 :a3190 :a3190. +:a3191 :a3191 :a3191. +:a3192 :a3192 :a3192. +:a3193 :a3193 :a3193. +:a3194 :a3194 :a3194. +:a3195 :a3195 :a3195. +:a3196 :a3196 :a3196. +:a3197 :a3197 :a3197. +:a3198 :a3198 :a3198. +:a3199 :a3199 :a3199. +:a3200 :a3200 :a3200. +:a3201 :a3201 :a3201. +:a3202 :a3202 :a3202. +:a3203 :a3203 :a3203. +:a3204 :a3204 :a3204. +:a3205 :a3205 :a3205. +:a3206 :a3206 :a3206. +:a3207 :a3207 :a3207. +:a3208 :a3208 :a3208. +:a3209 :a3209 :a3209. +:a3210 :a3210 :a3210. +:a3211 :a3211 :a3211. +:a3212 :a3212 :a3212. +:a3213 :a3213 :a3213. +:a3214 :a3214 :a3214. +:a3215 :a3215 :a3215. +:a3216 :a3216 :a3216. +:a3217 :a3217 :a3217. +:a3218 :a3218 :a3218. +:a3219 :a3219 :a3219. +:a3220 :a3220 :a3220. +:a3221 :a3221 :a3221. +:a3222 :a3222 :a3222. +:a3223 :a3223 :a3223. +:a3224 :a3224 :a3224. +:a3225 :a3225 :a3225. +:a3226 :a3226 :a3226. +:a3227 :a3227 :a3227. +:a3228 :a3228 :a3228. +:a3229 :a3229 :a3229. +:a3230 :a3230 :a3230. +:a3231 :a3231 :a3231. +:a3232 :a3232 :a3232. +:a3233 :a3233 :a3233. +:a3234 :a3234 :a3234. +:a3235 :a3235 :a3235. +:a3236 :a3236 :a3236. +:a3237 :a3237 :a3237. +:a3238 :a3238 :a3238. +:a3239 :a3239 :a3239. +:a3240 :a3240 :a3240. +:a3241 :a3241 :a3241. +:a3242 :a3242 :a3242. +:a3243 :a3243 :a3243. +:a3244 :a3244 :a3244. +:a3245 :a3245 :a3245. +:a3246 :a3246 :a3246. +:a3247 :a3247 :a3247. +:a3248 :a3248 :a3248. +:a3249 :a3249 :a3249. +:a3250 :a3250 :a3250. +:a3251 :a3251 :a3251. +:a3252 :a3252 :a3252. +:a3253 :a3253 :a3253. +:a3254 :a3254 :a3254. +:a3255 :a3255 :a3255. +:a3256 :a3256 :a3256. +:a3257 :a3257 :a3257. +:a3258 :a3258 :a3258. +:a3259 :a3259 :a3259. +:a3260 :a3260 :a3260. +:a3261 :a3261 :a3261. +:a3262 :a3262 :a3262. +:a3263 :a3263 :a3263. +:a3264 :a3264 :a3264. +:a3265 :a3265 :a3265. +:a3266 :a3266 :a3266. +:a3267 :a3267 :a3267. +:a3268 :a3268 :a3268. +:a3269 :a3269 :a3269. +:a3270 :a3270 :a3270. +:a3271 :a3271 :a3271. +:a3272 :a3272 :a3272. +:a3273 :a3273 :a3273. +:a3274 :a3274 :a3274. +:a3275 :a3275 :a3275. +:a3276 :a3276 :a3276. +:a3277 :a3277 :a3277. +:a3278 :a3278 :a3278. +:a3279 :a3279 :a3279. +:a3280 :a3280 :a3280. +:a3281 :a3281 :a3281. +:a3282 :a3282 :a3282. +:a3283 :a3283 :a3283. +:a3284 :a3284 :a3284. +:a3285 :a3285 :a3285. +:a3286 :a3286 :a3286. +:a3287 :a3287 :a3287. +:a3288 :a3288 :a3288. +:a3289 :a3289 :a3289. +:a3290 :a3290 :a3290. +:a3291 :a3291 :a3291. +:a3292 :a3292 :a3292. +:a3293 :a3293 :a3293. +:a3294 :a3294 :a3294. +:a3295 :a3295 :a3295. +:a3296 :a3296 :a3296. +:a3297 :a3297 :a3297. +:a3298 :a3298 :a3298. +:a3299 :a3299 :a3299. +:a3300 :a3300 :a3300. +:a3301 :a3301 :a3301. +:a3302 :a3302 :a3302. +:a3303 :a3303 :a3303. +:a3304 :a3304 :a3304. +:a3305 :a3305 :a3305. +:a3306 :a3306 :a3306. +:a3307 :a3307 :a3307. +:a3308 :a3308 :a3308. +:a3309 :a3309 :a3309. +:a3310 :a3310 :a3310. +:a3311 :a3311 :a3311. +:a3312 :a3312 :a3312. +:a3313 :a3313 :a3313. +:a3314 :a3314 :a3314. +:a3315 :a3315 :a3315. +:a3316 :a3316 :a3316. +:a3317 :a3317 :a3317. +:a3318 :a3318 :a3318. +:a3319 :a3319 :a3319. +:a3320 :a3320 :a3320. +:a3321 :a3321 :a3321. +:a3322 :a3322 :a3322. +:a3323 :a3323 :a3323. +:a3324 :a3324 :a3324. +:a3325 :a3325 :a3325. +:a3326 :a3326 :a3326. +:a3327 :a3327 :a3327. +:a3328 :a3328 :a3328. +:a3329 :a3329 :a3329. +:a3330 :a3330 :a3330. +:a3331 :a3331 :a3331. +:a3332 :a3332 :a3332. +:a3333 :a3333 :a3333. +:a3334 :a3334 :a3334. +:a3335 :a3335 :a3335. +:a3336 :a3336 :a3336. +:a3337 :a3337 :a3337. +:a3338 :a3338 :a3338. +:a3339 :a3339 :a3339. +:a3340 :a3340 :a3340. +:a3341 :a3341 :a3341. +:a3342 :a3342 :a3342. +:a3343 :a3343 :a3343. +:a3344 :a3344 :a3344. +:a3345 :a3345 :a3345. +:a3346 :a3346 :a3346. +:a3347 :a3347 :a3347. +:a3348 :a3348 :a3348. +:a3349 :a3349 :a3349. +:a3350 :a3350 :a3350. +:a3351 :a3351 :a3351. +:a3352 :a3352 :a3352. +:a3353 :a3353 :a3353. +:a3354 :a3354 :a3354. +:a3355 :a3355 :a3355. +:a3356 :a3356 :a3356. +:a3357 :a3357 :a3357. +:a3358 :a3358 :a3358. +:a3359 :a3359 :a3359. +:a3360 :a3360 :a3360. +:a3361 :a3361 :a3361. +:a3362 :a3362 :a3362. +:a3363 :a3363 :a3363. +:a3364 :a3364 :a3364. +:a3365 :a3365 :a3365. +:a3366 :a3366 :a3366. +:a3367 :a3367 :a3367. +:a3368 :a3368 :a3368. +:a3369 :a3369 :a3369. +:a3370 :a3370 :a3370. +:a3371 :a3371 :a3371. +:a3372 :a3372 :a3372. +:a3373 :a3373 :a3373. +:a3374 :a3374 :a3374. +:a3375 :a3375 :a3375. +:a3376 :a3376 :a3376. +:a3377 :a3377 :a3377. +:a3378 :a3378 :a3378. +:a3379 :a3379 :a3379. +:a3380 :a3380 :a3380. +:a3381 :a3381 :a3381. +:a3382 :a3382 :a3382. +:a3383 :a3383 :a3383. +:a3384 :a3384 :a3384. +:a3385 :a3385 :a3385. +:a3386 :a3386 :a3386. +:a3387 :a3387 :a3387. +:a3388 :a3388 :a3388. +:a3389 :a3389 :a3389. +:a3390 :a3390 :a3390. +:a3391 :a3391 :a3391. +:a3392 :a3392 :a3392. +:a3393 :a3393 :a3393. +:a3394 :a3394 :a3394. +:a3395 :a3395 :a3395. +:a3396 :a3396 :a3396. +:a3397 :a3397 :a3397. +:a3398 :a3398 :a3398. +:a3399 :a3399 :a3399. +:a3400 :a3400 :a3400. +:a3401 :a3401 :a3401. +:a3402 :a3402 :a3402. +:a3403 :a3403 :a3403. +:a3404 :a3404 :a3404. +:a3405 :a3405 :a3405. +:a3406 :a3406 :a3406. +:a3407 :a3407 :a3407. +:a3408 :a3408 :a3408. +:a3409 :a3409 :a3409. +:a3410 :a3410 :a3410. +:a3411 :a3411 :a3411. +:a3412 :a3412 :a3412. +:a3413 :a3413 :a3413. +:a3414 :a3414 :a3414. +:a3415 :a3415 :a3415. +:a3416 :a3416 :a3416. +:a3417 :a3417 :a3417. +:a3418 :a3418 :a3418. +:a3419 :a3419 :a3419. +:a3420 :a3420 :a3420. +:a3421 :a3421 :a3421. +:a3422 :a3422 :a3422. +:a3423 :a3423 :a3423. +:a3424 :a3424 :a3424. +:a3425 :a3425 :a3425. +:a3426 :a3426 :a3426. +:a3427 :a3427 :a3427. +:a3428 :a3428 :a3428. +:a3429 :a3429 :a3429. +:a3430 :a3430 :a3430. +:a3431 :a3431 :a3431. +:a3432 :a3432 :a3432. +:a3433 :a3433 :a3433. +:a3434 :a3434 :a3434. +:a3435 :a3435 :a3435. +:a3436 :a3436 :a3436. +:a3437 :a3437 :a3437. +:a3438 :a3438 :a3438. +:a3439 :a3439 :a3439. +:a3440 :a3440 :a3440. +:a3441 :a3441 :a3441. +:a3442 :a3442 :a3442. +:a3443 :a3443 :a3443. +:a3444 :a3444 :a3444. +:a3445 :a3445 :a3445. +:a3446 :a3446 :a3446. +:a3447 :a3447 :a3447. +:a3448 :a3448 :a3448. +:a3449 :a3449 :a3449. +:a3450 :a3450 :a3450. +:a3451 :a3451 :a3451. +:a3452 :a3452 :a3452. +:a3453 :a3453 :a3453. +:a3454 :a3454 :a3454. +:a3455 :a3455 :a3455. +:a3456 :a3456 :a3456. +:a3457 :a3457 :a3457. +:a3458 :a3458 :a3458. +:a3459 :a3459 :a3459. +:a3460 :a3460 :a3460. +:a3461 :a3461 :a3461. +:a3462 :a3462 :a3462. +:a3463 :a3463 :a3463. +:a3464 :a3464 :a3464. +:a3465 :a3465 :a3465. +:a3466 :a3466 :a3466. +:a3467 :a3467 :a3467. +:a3468 :a3468 :a3468. +:a3469 :a3469 :a3469. +:a3470 :a3470 :a3470. +:a3471 :a3471 :a3471. +:a3472 :a3472 :a3472. +:a3473 :a3473 :a3473. +:a3474 :a3474 :a3474. +:a3475 :a3475 :a3475. +:a3476 :a3476 :a3476. +:a3477 :a3477 :a3477. +:a3478 :a3478 :a3478. +:a3479 :a3479 :a3479. +:a3480 :a3480 :a3480. +:a3481 :a3481 :a3481. +:a3482 :a3482 :a3482. +:a3483 :a3483 :a3483. +:a3484 :a3484 :a3484. +:a3485 :a3485 :a3485. +:a3486 :a3486 :a3486. +:a3487 :a3487 :a3487. +:a3488 :a3488 :a3488. +:a3489 :a3489 :a3489. +:a3490 :a3490 :a3490. +:a3491 :a3491 :a3491. +:a3492 :a3492 :a3492. +:a3493 :a3493 :a3493. +:a3494 :a3494 :a3494. +:a3495 :a3495 :a3495. +:a3496 :a3496 :a3496. +:a3497 :a3497 :a3497. +:a3498 :a3498 :a3498. +:a3499 :a3499 :a3499. +:a3500 :a3500 :a3500. +:a3501 :a3501 :a3501. +:a3502 :a3502 :a3502. +:a3503 :a3503 :a3503. +:a3504 :a3504 :a3504. +:a3505 :a3505 :a3505. +:a3506 :a3506 :a3506. +:a3507 :a3507 :a3507. +:a3508 :a3508 :a3508. +:a3509 :a3509 :a3509. +:a3510 :a3510 :a3510. +:a3511 :a3511 :a3511. +:a3512 :a3512 :a3512. +:a3513 :a3513 :a3513. +:a3514 :a3514 :a3514. +:a3515 :a3515 :a3515. +:a3516 :a3516 :a3516. +:a3517 :a3517 :a3517. +:a3518 :a3518 :a3518. +:a3519 :a3519 :a3519. +:a3520 :a3520 :a3520. +:a3521 :a3521 :a3521. +:a3522 :a3522 :a3522. +:a3523 :a3523 :a3523. +:a3524 :a3524 :a3524. +:a3525 :a3525 :a3525. +:a3526 :a3526 :a3526. +:a3527 :a3527 :a3527. +:a3528 :a3528 :a3528. +:a3529 :a3529 :a3529. +:a3530 :a3530 :a3530. +:a3531 :a3531 :a3531. +:a3532 :a3532 :a3532. +:a3533 :a3533 :a3533. +:a3534 :a3534 :a3534. +:a3535 :a3535 :a3535. +:a3536 :a3536 :a3536. +:a3537 :a3537 :a3537. +:a3538 :a3538 :a3538. +:a3539 :a3539 :a3539. +:a3540 :a3540 :a3540. +:a3541 :a3541 :a3541. +:a3542 :a3542 :a3542. +:a3543 :a3543 :a3543. +:a3544 :a3544 :a3544. +:a3545 :a3545 :a3545. +:a3546 :a3546 :a3546. +:a3547 :a3547 :a3547. +:a3548 :a3548 :a3548. +:a3549 :a3549 :a3549. +:a3550 :a3550 :a3550. +:a3551 :a3551 :a3551. +:a3552 :a3552 :a3552. +:a3553 :a3553 :a3553. +:a3554 :a3554 :a3554. +:a3555 :a3555 :a3555. +:a3556 :a3556 :a3556. +:a3557 :a3557 :a3557. +:a3558 :a3558 :a3558. +:a3559 :a3559 :a3559. +:a3560 :a3560 :a3560. +:a3561 :a3561 :a3561. +:a3562 :a3562 :a3562. +:a3563 :a3563 :a3563. +:a3564 :a3564 :a3564. +:a3565 :a3565 :a3565. +:a3566 :a3566 :a3566. +:a3567 :a3567 :a3567. +:a3568 :a3568 :a3568. +:a3569 :a3569 :a3569. +:a3570 :a3570 :a3570. +:a3571 :a3571 :a3571. +:a3572 :a3572 :a3572. +:a3573 :a3573 :a3573. +:a3574 :a3574 :a3574. +:a3575 :a3575 :a3575. +:a3576 :a3576 :a3576. +:a3577 :a3577 :a3577. +:a3578 :a3578 :a3578. +:a3579 :a3579 :a3579. +:a3580 :a3580 :a3580. +:a3581 :a3581 :a3581. +:a3582 :a3582 :a3582. +:a3583 :a3583 :a3583. +:a3584 :a3584 :a3584. +:a3585 :a3585 :a3585. +:a3586 :a3586 :a3586. +:a3587 :a3587 :a3587. +:a3588 :a3588 :a3588. +:a3589 :a3589 :a3589. +:a3590 :a3590 :a3590. +:a3591 :a3591 :a3591. +:a3592 :a3592 :a3592. +:a3593 :a3593 :a3593. +:a3594 :a3594 :a3594. +:a3595 :a3595 :a3595. +:a3596 :a3596 :a3596. +:a3597 :a3597 :a3597. +:a3598 :a3598 :a3598. +:a3599 :a3599 :a3599. +:a3600 :a3600 :a3600. +:a3601 :a3601 :a3601. +:a3602 :a3602 :a3602. +:a3603 :a3603 :a3603. +:a3604 :a3604 :a3604. +:a3605 :a3605 :a3605. +:a3606 :a3606 :a3606. +:a3607 :a3607 :a3607. +:a3608 :a3608 :a3608. +:a3609 :a3609 :a3609. +:a3610 :a3610 :a3610. +:a3611 :a3611 :a3611. +:a3612 :a3612 :a3612. +:a3613 :a3613 :a3613. +:a3614 :a3614 :a3614. +:a3615 :a3615 :a3615. +:a3616 :a3616 :a3616. +:a3617 :a3617 :a3617. +:a3618 :a3618 :a3618. +:a3619 :a3619 :a3619. +:a3620 :a3620 :a3620. +:a3621 :a3621 :a3621. +:a3622 :a3622 :a3622. +:a3623 :a3623 :a3623. +:a3624 :a3624 :a3624. +:a3625 :a3625 :a3625. +:a3626 :a3626 :a3626. +:a3627 :a3627 :a3627. +:a3628 :a3628 :a3628. +:a3629 :a3629 :a3629. +:a3630 :a3630 :a3630. +:a3631 :a3631 :a3631. +:a3632 :a3632 :a3632. +:a3633 :a3633 :a3633. +:a3634 :a3634 :a3634. +:a3635 :a3635 :a3635. +:a3636 :a3636 :a3636. +:a3637 :a3637 :a3637. +:a3638 :a3638 :a3638. +:a3639 :a3639 :a3639. +:a3640 :a3640 :a3640. +:a3641 :a3641 :a3641. +:a3642 :a3642 :a3642. +:a3643 :a3643 :a3643. +:a3644 :a3644 :a3644. +:a3645 :a3645 :a3645. +:a3646 :a3646 :a3646. +:a3647 :a3647 :a3647. +:a3648 :a3648 :a3648. +:a3649 :a3649 :a3649. +:a3650 :a3650 :a3650. +:a3651 :a3651 :a3651. +:a3652 :a3652 :a3652. +:a3653 :a3653 :a3653. +:a3654 :a3654 :a3654. +:a3655 :a3655 :a3655. +:a3656 :a3656 :a3656. +:a3657 :a3657 :a3657. +:a3658 :a3658 :a3658. +:a3659 :a3659 :a3659. +:a3660 :a3660 :a3660. +:a3661 :a3661 :a3661. +:a3662 :a3662 :a3662. +:a3663 :a3663 :a3663. +:a3664 :a3664 :a3664. +:a3665 :a3665 :a3665. +:a3666 :a3666 :a3666. +:a3667 :a3667 :a3667. +:a3668 :a3668 :a3668. +:a3669 :a3669 :a3669. +:a3670 :a3670 :a3670. +:a3671 :a3671 :a3671. +:a3672 :a3672 :a3672. +:a3673 :a3673 :a3673. +:a3674 :a3674 :a3674. +:a3675 :a3675 :a3675. +:a3676 :a3676 :a3676. +:a3677 :a3677 :a3677. +:a3678 :a3678 :a3678. +:a3679 :a3679 :a3679. +:a3680 :a3680 :a3680. +:a3681 :a3681 :a3681. +:a3682 :a3682 :a3682. +:a3683 :a3683 :a3683. +:a3684 :a3684 :a3684. +:a3685 :a3685 :a3685. +:a3686 :a3686 :a3686. +:a3687 :a3687 :a3687. +:a3688 :a3688 :a3688. +:a3689 :a3689 :a3689. +:a3690 :a3690 :a3690. +:a3691 :a3691 :a3691. +:a3692 :a3692 :a3692. +:a3693 :a3693 :a3693. +:a3694 :a3694 :a3694. +:a3695 :a3695 :a3695. +:a3696 :a3696 :a3696. +:a3697 :a3697 :a3697. +:a3698 :a3698 :a3698. +:a3699 :a3699 :a3699. +:a3700 :a3700 :a3700. +:a3701 :a3701 :a3701. +:a3702 :a3702 :a3702. +:a3703 :a3703 :a3703. +:a3704 :a3704 :a3704. +:a3705 :a3705 :a3705. +:a3706 :a3706 :a3706. +:a3707 :a3707 :a3707. +:a3708 :a3708 :a3708. +:a3709 :a3709 :a3709. +:a3710 :a3710 :a3710. +:a3711 :a3711 :a3711. +:a3712 :a3712 :a3712. +:a3713 :a3713 :a3713. +:a3714 :a3714 :a3714. +:a3715 :a3715 :a3715. +:a3716 :a3716 :a3716. +:a3717 :a3717 :a3717. +:a3718 :a3718 :a3718. +:a3719 :a3719 :a3719. +:a3720 :a3720 :a3720. +:a3721 :a3721 :a3721. +:a3722 :a3722 :a3722. +:a3723 :a3723 :a3723. +:a3724 :a3724 :a3724. +:a3725 :a3725 :a3725. +:a3726 :a3726 :a3726. +:a3727 :a3727 :a3727. +:a3728 :a3728 :a3728. +:a3729 :a3729 :a3729. +:a3730 :a3730 :a3730. +:a3731 :a3731 :a3731. +:a3732 :a3732 :a3732. +:a3733 :a3733 :a3733. +:a3734 :a3734 :a3734. +:a3735 :a3735 :a3735. +:a3736 :a3736 :a3736. +:a3737 :a3737 :a3737. +:a3738 :a3738 :a3738. +:a3739 :a3739 :a3739. +:a3740 :a3740 :a3740. +:a3741 :a3741 :a3741. +:a3742 :a3742 :a3742. +:a3743 :a3743 :a3743. +:a3744 :a3744 :a3744. +:a3745 :a3745 :a3745. +:a3746 :a3746 :a3746. +:a3747 :a3747 :a3747. +:a3748 :a3748 :a3748. +:a3749 :a3749 :a3749. +:a3750 :a3750 :a3750. +:a3751 :a3751 :a3751. +:a3752 :a3752 :a3752. +:a3753 :a3753 :a3753. +:a3754 :a3754 :a3754. +:a3755 :a3755 :a3755. +:a3756 :a3756 :a3756. +:a3757 :a3757 :a3757. +:a3758 :a3758 :a3758. +:a3759 :a3759 :a3759. +:a3760 :a3760 :a3760. +:a3761 :a3761 :a3761. +:a3762 :a3762 :a3762. +:a3763 :a3763 :a3763. +:a3764 :a3764 :a3764. +:a3765 :a3765 :a3765. +:a3766 :a3766 :a3766. +:a3767 :a3767 :a3767. +:a3768 :a3768 :a3768. +:a3769 :a3769 :a3769. +:a3770 :a3770 :a3770. +:a3771 :a3771 :a3771. +:a3772 :a3772 :a3772. +:a3773 :a3773 :a3773. +:a3774 :a3774 :a3774. +:a3775 :a3775 :a3775. +:a3776 :a3776 :a3776. +:a3777 :a3777 :a3777. +:a3778 :a3778 :a3778. +:a3779 :a3779 :a3779. +:a3780 :a3780 :a3780. +:a3781 :a3781 :a3781. +:a3782 :a3782 :a3782. +:a3783 :a3783 :a3783. +:a3784 :a3784 :a3784. +:a3785 :a3785 :a3785. +:a3786 :a3786 :a3786. +:a3787 :a3787 :a3787. +:a3788 :a3788 :a3788. +:a3789 :a3789 :a3789. +:a3790 :a3790 :a3790. +:a3791 :a3791 :a3791. +:a3792 :a3792 :a3792. +:a3793 :a3793 :a3793. +:a3794 :a3794 :a3794. +:a3795 :a3795 :a3795. +:a3796 :a3796 :a3796. +:a3797 :a3797 :a3797. +:a3798 :a3798 :a3798. +:a3799 :a3799 :a3799. +:a3800 :a3800 :a3800. +:a3801 :a3801 :a3801. +:a3802 :a3802 :a3802. +:a3803 :a3803 :a3803. +:a3804 :a3804 :a3804. +:a3805 :a3805 :a3805. +:a3806 :a3806 :a3806. +:a3807 :a3807 :a3807. +:a3808 :a3808 :a3808. +:a3809 :a3809 :a3809. +:a3810 :a3810 :a3810. +:a3811 :a3811 :a3811. +:a3812 :a3812 :a3812. +:a3813 :a3813 :a3813. +:a3814 :a3814 :a3814. +:a3815 :a3815 :a3815. +:a3816 :a3816 :a3816. +:a3817 :a3817 :a3817. +:a3818 :a3818 :a3818. +:a3819 :a3819 :a3819. +:a3820 :a3820 :a3820. +:a3821 :a3821 :a3821. +:a3822 :a3822 :a3822. +:a3823 :a3823 :a3823. +:a3824 :a3824 :a3824. +:a3825 :a3825 :a3825. +:a3826 :a3826 :a3826. +:a3827 :a3827 :a3827. +:a3828 :a3828 :a3828. +:a3829 :a3829 :a3829. +:a3830 :a3830 :a3830. +:a3831 :a3831 :a3831. +:a3832 :a3832 :a3832. +:a3833 :a3833 :a3833. +:a3834 :a3834 :a3834. +:a3835 :a3835 :a3835. +:a3836 :a3836 :a3836. +:a3837 :a3837 :a3837. +:a3838 :a3838 :a3838. +:a3839 :a3839 :a3839. +:a3840 :a3840 :a3840. +:a3841 :a3841 :a3841. +:a3842 :a3842 :a3842. +:a3843 :a3843 :a3843. +:a3844 :a3844 :a3844. +:a3845 :a3845 :a3845. +:a3846 :a3846 :a3846. +:a3847 :a3847 :a3847. +:a3848 :a3848 :a3848. +:a3849 :a3849 :a3849. +:a3850 :a3850 :a3850. +:a3851 :a3851 :a3851. +:a3852 :a3852 :a3852. +:a3853 :a3853 :a3853. +:a3854 :a3854 :a3854. +:a3855 :a3855 :a3855. +:a3856 :a3856 :a3856. +:a3857 :a3857 :a3857. +:a3858 :a3858 :a3858. +:a3859 :a3859 :a3859. +:a3860 :a3860 :a3860. +:a3861 :a3861 :a3861. +:a3862 :a3862 :a3862. +:a3863 :a3863 :a3863. +:a3864 :a3864 :a3864. +:a3865 :a3865 :a3865. +:a3866 :a3866 :a3866. +:a3867 :a3867 :a3867. +:a3868 :a3868 :a3868. +:a3869 :a3869 :a3869. +:a3870 :a3870 :a3870. +:a3871 :a3871 :a3871. +:a3872 :a3872 :a3872. +:a3873 :a3873 :a3873. +:a3874 :a3874 :a3874. +:a3875 :a3875 :a3875. +:a3876 :a3876 :a3876. +:a3877 :a3877 :a3877. +:a3878 :a3878 :a3878. +:a3879 :a3879 :a3879. +:a3880 :a3880 :a3880. +:a3881 :a3881 :a3881. +:a3882 :a3882 :a3882. +:a3883 :a3883 :a3883. +:a3884 :a3884 :a3884. +:a3885 :a3885 :a3885. +:a3886 :a3886 :a3886. +:a3887 :a3887 :a3887. +:a3888 :a3888 :a3888. +:a3889 :a3889 :a3889. +:a3890 :a3890 :a3890. +:a3891 :a3891 :a3891. +:a3892 :a3892 :a3892. +:a3893 :a3893 :a3893. +:a3894 :a3894 :a3894. +:a3895 :a3895 :a3895. +:a3896 :a3896 :a3896. +:a3897 :a3897 :a3897. +:a3898 :a3898 :a3898. +:a3899 :a3899 :a3899. +:a3900 :a3900 :a3900. +:a3901 :a3901 :a3901. +:a3902 :a3902 :a3902. +:a3903 :a3903 :a3903. +:a3904 :a3904 :a3904. +:a3905 :a3905 :a3905. +:a3906 :a3906 :a3906. +:a3907 :a3907 :a3907. +:a3908 :a3908 :a3908. +:a3909 :a3909 :a3909. +:a3910 :a3910 :a3910. +:a3911 :a3911 :a3911. +:a3912 :a3912 :a3912. +:a3913 :a3913 :a3913. +:a3914 :a3914 :a3914. +:a3915 :a3915 :a3915. +:a3916 :a3916 :a3916. +:a3917 :a3917 :a3917. +:a3918 :a3918 :a3918. +:a3919 :a3919 :a3919. +:a3920 :a3920 :a3920. +:a3921 :a3921 :a3921. +:a3922 :a3922 :a3922. +:a3923 :a3923 :a3923. +:a3924 :a3924 :a3924. +:a3925 :a3925 :a3925. +:a3926 :a3926 :a3926. +:a3927 :a3927 :a3927. +:a3928 :a3928 :a3928. +:a3929 :a3929 :a3929. +:a3930 :a3930 :a3930. +:a3931 :a3931 :a3931. +:a3932 :a3932 :a3932. +:a3933 :a3933 :a3933. +:a3934 :a3934 :a3934. +:a3935 :a3935 :a3935. +:a3936 :a3936 :a3936. +:a3937 :a3937 :a3937. +:a3938 :a3938 :a3938. +:a3939 :a3939 :a3939. +:a3940 :a3940 :a3940. +:a3941 :a3941 :a3941. +:a3942 :a3942 :a3942. +:a3943 :a3943 :a3943. +:a3944 :a3944 :a3944. +:a3945 :a3945 :a3945. +:a3946 :a3946 :a3946. +:a3947 :a3947 :a3947. +:a3948 :a3948 :a3948. +:a3949 :a3949 :a3949. +:a3950 :a3950 :a3950. +:a3951 :a3951 :a3951. +:a3952 :a3952 :a3952. +:a3953 :a3953 :a3953. +:a3954 :a3954 :a3954. +:a3955 :a3955 :a3955. +:a3956 :a3956 :a3956. +:a3957 :a3957 :a3957. +:a3958 :a3958 :a3958. +:a3959 :a3959 :a3959. +:a3960 :a3960 :a3960. +:a3961 :a3961 :a3961. +:a3962 :a3962 :a3962. +:a3963 :a3963 :a3963. +:a3964 :a3964 :a3964. +:a3965 :a3965 :a3965. +:a3966 :a3966 :a3966. +:a3967 :a3967 :a3967. +:a3968 :a3968 :a3968. +:a3969 :a3969 :a3969. +:a3970 :a3970 :a3970. +:a3971 :a3971 :a3971. +:a3972 :a3972 :a3972. +:a3973 :a3973 :a3973. +:a3974 :a3974 :a3974. +:a3975 :a3975 :a3975. +:a3976 :a3976 :a3976. +:a3977 :a3977 :a3977. +:a3978 :a3978 :a3978. +:a3979 :a3979 :a3979. +:a3980 :a3980 :a3980. +:a3981 :a3981 :a3981. +:a3982 :a3982 :a3982. +:a3983 :a3983 :a3983. +:a3984 :a3984 :a3984. +:a3985 :a3985 :a3985. +:a3986 :a3986 :a3986. +:a3987 :a3987 :a3987. +:a3988 :a3988 :a3988. +:a3989 :a3989 :a3989. +:a3990 :a3990 :a3990. +:a3991 :a3991 :a3991. +:a3992 :a3992 :a3992. +:a3993 :a3993 :a3993. +:a3994 :a3994 :a3994. +:a3995 :a3995 :a3995. +:a3996 :a3996 :a3996. +:a3997 :a3997 :a3997. +:a3998 :a3998 :a3998. +:a3999 :a3999 :a3999. +:a4000 :a4000 :a4000. +:a4001 :a4001 :a4001. +:a4002 :a4002 :a4002. +:a4003 :a4003 :a4003. +:a4004 :a4004 :a4004. +:a4005 :a4005 :a4005. +:a4006 :a4006 :a4006. +:a4007 :a4007 :a4007. +:a4008 :a4008 :a4008. +:a4009 :a4009 :a4009. +:a4010 :a4010 :a4010. +:a4011 :a4011 :a4011. +:a4012 :a4012 :a4012. +:a4013 :a4013 :a4013. +:a4014 :a4014 :a4014. +:a4015 :a4015 :a4015. +:a4016 :a4016 :a4016. +:a4017 :a4017 :a4017. +:a4018 :a4018 :a4018. +:a4019 :a4019 :a4019. +:a4020 :a4020 :a4020. +:a4021 :a4021 :a4021. +:a4022 :a4022 :a4022. +:a4023 :a4023 :a4023. +:a4024 :a4024 :a4024. +:a4025 :a4025 :a4025. +:a4026 :a4026 :a4026. +:a4027 :a4027 :a4027. +:a4028 :a4028 :a4028. +:a4029 :a4029 :a4029. +:a4030 :a4030 :a4030. +:a4031 :a4031 :a4031. +:a4032 :a4032 :a4032. +:a4033 :a4033 :a4033. +:a4034 :a4034 :a4034. +:a4035 :a4035 :a4035. +:a4036 :a4036 :a4036. +:a4037 :a4037 :a4037. +:a4038 :a4038 :a4038. +:a4039 :a4039 :a4039. +:a4040 :a4040 :a4040. +:a4041 :a4041 :a4041. +:a4042 :a4042 :a4042. +:a4043 :a4043 :a4043. +:a4044 :a4044 :a4044. +:a4045 :a4045 :a4045. +:a4046 :a4046 :a4046. +:a4047 :a4047 :a4047. +:a4048 :a4048 :a4048. +:a4049 :a4049 :a4049. +:a4050 :a4050 :a4050. +:a4051 :a4051 :a4051. +:a4052 :a4052 :a4052. +:a4053 :a4053 :a4053. +:a4054 :a4054 :a4054. +:a4055 :a4055 :a4055. +:a4056 :a4056 :a4056. +:a4057 :a4057 :a4057. +:a4058 :a4058 :a4058. +:a4059 :a4059 :a4059. +:a4060 :a4060 :a4060. +:a4061 :a4061 :a4061. +:a4062 :a4062 :a4062. +:a4063 :a4063 :a4063. +:a4064 :a4064 :a4064. +:a4065 :a4065 :a4065. +:a4066 :a4066 :a4066. +:a4067 :a4067 :a4067. +:a4068 :a4068 :a4068. +:a4069 :a4069 :a4069. +:a4070 :a4070 :a4070. +:a4071 :a4071 :a4071. +:a4072 :a4072 :a4072. +:a4073 :a4073 :a4073. +:a4074 :a4074 :a4074. +:a4075 :a4075 :a4075. +:a4076 :a4076 :a4076. +:a4077 :a4077 :a4077. +:a4078 :a4078 :a4078. +:a4079 :a4079 :a4079. +:a4080 :a4080 :a4080. +:a4081 :a4081 :a4081. +:a4082 :a4082 :a4082. +:a4083 :a4083 :a4083. +:a4084 :a4084 :a4084. +:a4085 :a4085 :a4085. +:a4086 :a4086 :a4086. +:a4087 :a4087 :a4087. +:a4088 :a4088 :a4088. +:a4089 :a4089 :a4089. +:a4090 :a4090 :a4090. +:a4091 :a4091 :a4091. +:a4092 :a4092 :a4092. +:a4093 :a4093 :a4093. +:a4094 :a4094 :a4094. +:a4095 :a4095 :a4095. +:a4096 :a4096 :a4096. +:a4097 :a4097 :a4097. +:a4098 :a4098 :a4098. +:a4099 :a4099 :a4099. +:a4100 :a4100 :a4100. +:a4101 :a4101 :a4101. +:a4102 :a4102 :a4102. +:a4103 :a4103 :a4103. +:a4104 :a4104 :a4104. +:a4105 :a4105 :a4105. +:a4106 :a4106 :a4106. +:a4107 :a4107 :a4107. +:a4108 :a4108 :a4108. +:a4109 :a4109 :a4109. +:a4110 :a4110 :a4110. +:a4111 :a4111 :a4111. +:a4112 :a4112 :a4112. +:a4113 :a4113 :a4113. +:a4114 :a4114 :a4114. +:a4115 :a4115 :a4115. +:a4116 :a4116 :a4116. +:a4117 :a4117 :a4117. +:a4118 :a4118 :a4118. +:a4119 :a4119 :a4119. +:a4120 :a4120 :a4120. +:a4121 :a4121 :a4121. +:a4122 :a4122 :a4122. +:a4123 :a4123 :a4123. +:a4124 :a4124 :a4124. +:a4125 :a4125 :a4125. +:a4126 :a4126 :a4126. +:a4127 :a4127 :a4127. +:a4128 :a4128 :a4128. +:a4129 :a4129 :a4129. +:a4130 :a4130 :a4130. +:a4131 :a4131 :a4131. +:a4132 :a4132 :a4132. +:a4133 :a4133 :a4133. +:a4134 :a4134 :a4134. +:a4135 :a4135 :a4135. +:a4136 :a4136 :a4136. +:a4137 :a4137 :a4137. +:a4138 :a4138 :a4138. +:a4139 :a4139 :a4139. +:a4140 :a4140 :a4140. +:a4141 :a4141 :a4141. +:a4142 :a4142 :a4142. +:a4143 :a4143 :a4143. +:a4144 :a4144 :a4144. +:a4145 :a4145 :a4145. +:a4146 :a4146 :a4146. +:a4147 :a4147 :a4147. +:a4148 :a4148 :a4148. +:a4149 :a4149 :a4149. +:a4150 :a4150 :a4150. +:a4151 :a4151 :a4151. +:a4152 :a4152 :a4152. +:a4153 :a4153 :a4153. +:a4154 :a4154 :a4154. +:a4155 :a4155 :a4155. +:a4156 :a4156 :a4156. +:a4157 :a4157 :a4157. +:a4158 :a4158 :a4158. +:a4159 :a4159 :a4159. +:a4160 :a4160 :a4160. +:a4161 :a4161 :a4161. +:a4162 :a4162 :a4162. +:a4163 :a4163 :a4163. +:a4164 :a4164 :a4164. +:a4165 :a4165 :a4165. +:a4166 :a4166 :a4166. +:a4167 :a4167 :a4167. +:a4168 :a4168 :a4168. +:a4169 :a4169 :a4169. +:a4170 :a4170 :a4170. +:a4171 :a4171 :a4171. +:a4172 :a4172 :a4172. +:a4173 :a4173 :a4173. +:a4174 :a4174 :a4174. +:a4175 :a4175 :a4175. +:a4176 :a4176 :a4176. +:a4177 :a4177 :a4177. +:a4178 :a4178 :a4178. +:a4179 :a4179 :a4179. +:a4180 :a4180 :a4180. +:a4181 :a4181 :a4181. +:a4182 :a4182 :a4182. +:a4183 :a4183 :a4183. +:a4184 :a4184 :a4184. +:a4185 :a4185 :a4185. +:a4186 :a4186 :a4186. +:a4187 :a4187 :a4187. +:a4188 :a4188 :a4188. +:a4189 :a4189 :a4189. +:a4190 :a4190 :a4190. +:a4191 :a4191 :a4191. +:a4192 :a4192 :a4192. +:a4193 :a4193 :a4193. +:a4194 :a4194 :a4194. +:a4195 :a4195 :a4195. +:a4196 :a4196 :a4196. +:a4197 :a4197 :a4197. +:a4198 :a4198 :a4198. +:a4199 :a4199 :a4199. +:a4200 :a4200 :a4200. +:a4201 :a4201 :a4201. +:a4202 :a4202 :a4202. +:a4203 :a4203 :a4203. +:a4204 :a4204 :a4204. +:a4205 :a4205 :a4205. +:a4206 :a4206 :a4206. +:a4207 :a4207 :a4207. +:a4208 :a4208 :a4208. +:a4209 :a4209 :a4209. +:a4210 :a4210 :a4210. +:a4211 :a4211 :a4211. +:a4212 :a4212 :a4212. +:a4213 :a4213 :a4213. +:a4214 :a4214 :a4214. +:a4215 :a4215 :a4215. +:a4216 :a4216 :a4216. +:a4217 :a4217 :a4217. +:a4218 :a4218 :a4218. +:a4219 :a4219 :a4219. +:a4220 :a4220 :a4220. +:a4221 :a4221 :a4221. +:a4222 :a4222 :a4222. +:a4223 :a4223 :a4223. +:a4224 :a4224 :a4224. +:a4225 :a4225 :a4225. +:a4226 :a4226 :a4226. +:a4227 :a4227 :a4227. +:a4228 :a4228 :a4228. +:a4229 :a4229 :a4229. +:a4230 :a4230 :a4230. +:a4231 :a4231 :a4231. +:a4232 :a4232 :a4232. +:a4233 :a4233 :a4233. +:a4234 :a4234 :a4234. +:a4235 :a4235 :a4235. +:a4236 :a4236 :a4236. +:a4237 :a4237 :a4237. +:a4238 :a4238 :a4238. +:a4239 :a4239 :a4239. +:a4240 :a4240 :a4240. +:a4241 :a4241 :a4241. +:a4242 :a4242 :a4242. +:a4243 :a4243 :a4243. +:a4244 :a4244 :a4244. +:a4245 :a4245 :a4245. +:a4246 :a4246 :a4246. +:a4247 :a4247 :a4247. +:a4248 :a4248 :a4248. +:a4249 :a4249 :a4249. +:a4250 :a4250 :a4250. +:a4251 :a4251 :a4251. +:a4252 :a4252 :a4252. +:a4253 :a4253 :a4253. +:a4254 :a4254 :a4254. +:a4255 :a4255 :a4255. +:a4256 :a4256 :a4256. +:a4257 :a4257 :a4257. +:a4258 :a4258 :a4258. +:a4259 :a4259 :a4259. +:a4260 :a4260 :a4260. +:a4261 :a4261 :a4261. +:a4262 :a4262 :a4262. +:a4263 :a4263 :a4263. +:a4264 :a4264 :a4264. +:a4265 :a4265 :a4265. +:a4266 :a4266 :a4266. +:a4267 :a4267 :a4267. +:a4268 :a4268 :a4268. +:a4269 :a4269 :a4269. +:a4270 :a4270 :a4270. +:a4271 :a4271 :a4271. +:a4272 :a4272 :a4272. +:a4273 :a4273 :a4273. +:a4274 :a4274 :a4274. +:a4275 :a4275 :a4275. +:a4276 :a4276 :a4276. +:a4277 :a4277 :a4277. +:a4278 :a4278 :a4278. +:a4279 :a4279 :a4279. +:a4280 :a4280 :a4280. +:a4281 :a4281 :a4281. +:a4282 :a4282 :a4282. +:a4283 :a4283 :a4283. +:a4284 :a4284 :a4284. +:a4285 :a4285 :a4285. +:a4286 :a4286 :a4286. +:a4287 :a4287 :a4287. +:a4288 :a4288 :a4288. +:a4289 :a4289 :a4289. +:a4290 :a4290 :a4290. +:a4291 :a4291 :a4291. +:a4292 :a4292 :a4292. +:a4293 :a4293 :a4293. +:a4294 :a4294 :a4294. +:a4295 :a4295 :a4295. +:a4296 :a4296 :a4296. +:a4297 :a4297 :a4297. +:a4298 :a4298 :a4298. +:a4299 :a4299 :a4299. +:a4300 :a4300 :a4300. +:a4301 :a4301 :a4301. +:a4302 :a4302 :a4302. +:a4303 :a4303 :a4303. +:a4304 :a4304 :a4304. +:a4305 :a4305 :a4305. +:a4306 :a4306 :a4306. +:a4307 :a4307 :a4307. +:a4308 :a4308 :a4308. +:a4309 :a4309 :a4309. +:a4310 :a4310 :a4310. +:a4311 :a4311 :a4311. +:a4312 :a4312 :a4312. +:a4313 :a4313 :a4313. +:a4314 :a4314 :a4314. +:a4315 :a4315 :a4315. +:a4316 :a4316 :a4316. +:a4317 :a4317 :a4317. +:a4318 :a4318 :a4318. +:a4319 :a4319 :a4319. +:a4320 :a4320 :a4320. +:a4321 :a4321 :a4321. +:a4322 :a4322 :a4322. +:a4323 :a4323 :a4323. +:a4324 :a4324 :a4324. +:a4325 :a4325 :a4325. +:a4326 :a4326 :a4326. +:a4327 :a4327 :a4327. +:a4328 :a4328 :a4328. +:a4329 :a4329 :a4329. +:a4330 :a4330 :a4330. +:a4331 :a4331 :a4331. +:a4332 :a4332 :a4332. +:a4333 :a4333 :a4333. +:a4334 :a4334 :a4334. +:a4335 :a4335 :a4335. +:a4336 :a4336 :a4336. +:a4337 :a4337 :a4337. +:a4338 :a4338 :a4338. +:a4339 :a4339 :a4339. +:a4340 :a4340 :a4340. +:a4341 :a4341 :a4341. +:a4342 :a4342 :a4342. +:a4343 :a4343 :a4343. +:a4344 :a4344 :a4344. +:a4345 :a4345 :a4345. +:a4346 :a4346 :a4346. +:a4347 :a4347 :a4347. +:a4348 :a4348 :a4348. +:a4349 :a4349 :a4349. +:a4350 :a4350 :a4350. +:a4351 :a4351 :a4351. +:a4352 :a4352 :a4352. +:a4353 :a4353 :a4353. +:a4354 :a4354 :a4354. +:a4355 :a4355 :a4355. +:a4356 :a4356 :a4356. +:a4357 :a4357 :a4357. +:a4358 :a4358 :a4358. +:a4359 :a4359 :a4359. +:a4360 :a4360 :a4360. +:a4361 :a4361 :a4361. +:a4362 :a4362 :a4362. +:a4363 :a4363 :a4363. +:a4364 :a4364 :a4364. +:a4365 :a4365 :a4365. +:a4366 :a4366 :a4366. +:a4367 :a4367 :a4367. +:a4368 :a4368 :a4368. +:a4369 :a4369 :a4369. +:a4370 :a4370 :a4370. +:a4371 :a4371 :a4371. +:a4372 :a4372 :a4372. +:a4373 :a4373 :a4373. +:a4374 :a4374 :a4374. +:a4375 :a4375 :a4375. +:a4376 :a4376 :a4376. +:a4377 :a4377 :a4377. +:a4378 :a4378 :a4378. +:a4379 :a4379 :a4379. +:a4380 :a4380 :a4380. +:a4381 :a4381 :a4381. +:a4382 :a4382 :a4382. +:a4383 :a4383 :a4383. +:a4384 :a4384 :a4384. +:a4385 :a4385 :a4385. +:a4386 :a4386 :a4386. +:a4387 :a4387 :a4387. +:a4388 :a4388 :a4388. +:a4389 :a4389 :a4389. +:a4390 :a4390 :a4390. +:a4391 :a4391 :a4391. +:a4392 :a4392 :a4392. +:a4393 :a4393 :a4393. +:a4394 :a4394 :a4394. +:a4395 :a4395 :a4395. +:a4396 :a4396 :a4396. +:a4397 :a4397 :a4397. +:a4398 :a4398 :a4398. +:a4399 :a4399 :a4399. +:a4400 :a4400 :a4400. +:a4401 :a4401 :a4401. +:a4402 :a4402 :a4402. +:a4403 :a4403 :a4403. +:a4404 :a4404 :a4404. +:a4405 :a4405 :a4405. +:a4406 :a4406 :a4406. +:a4407 :a4407 :a4407. +:a4408 :a4408 :a4408. +:a4409 :a4409 :a4409. +:a4410 :a4410 :a4410. +:a4411 :a4411 :a4411. +:a4412 :a4412 :a4412. +:a4413 :a4413 :a4413. +:a4414 :a4414 :a4414. +:a4415 :a4415 :a4415. +:a4416 :a4416 :a4416. +:a4417 :a4417 :a4417. +:a4418 :a4418 :a4418. +:a4419 :a4419 :a4419. +:a4420 :a4420 :a4420. +:a4421 :a4421 :a4421. +:a4422 :a4422 :a4422. +:a4423 :a4423 :a4423. +:a4424 :a4424 :a4424. +:a4425 :a4425 :a4425. +:a4426 :a4426 :a4426. +:a4427 :a4427 :a4427. +:a4428 :a4428 :a4428. +:a4429 :a4429 :a4429. +:a4430 :a4430 :a4430. +:a4431 :a4431 :a4431. +:a4432 :a4432 :a4432. +:a4433 :a4433 :a4433. +:a4434 :a4434 :a4434. +:a4435 :a4435 :a4435. +:a4436 :a4436 :a4436. +:a4437 :a4437 :a4437. +:a4438 :a4438 :a4438. +:a4439 :a4439 :a4439. +:a4440 :a4440 :a4440. +:a4441 :a4441 :a4441. +:a4442 :a4442 :a4442. +:a4443 :a4443 :a4443. +:a4444 :a4444 :a4444. +:a4445 :a4445 :a4445. +:a4446 :a4446 :a4446. +:a4447 :a4447 :a4447. +:a4448 :a4448 :a4448. +:a4449 :a4449 :a4449. +:a4450 :a4450 :a4450. +:a4451 :a4451 :a4451. +:a4452 :a4452 :a4452. +:a4453 :a4453 :a4453. +:a4454 :a4454 :a4454. +:a4455 :a4455 :a4455. +:a4456 :a4456 :a4456. +:a4457 :a4457 :a4457. +:a4458 :a4458 :a4458. +:a4459 :a4459 :a4459. +:a4460 :a4460 :a4460. +:a4461 :a4461 :a4461. +:a4462 :a4462 :a4462. +:a4463 :a4463 :a4463. +:a4464 :a4464 :a4464. +:a4465 :a4465 :a4465. +:a4466 :a4466 :a4466. +:a4467 :a4467 :a4467. +:a4468 :a4468 :a4468. +:a4469 :a4469 :a4469. +:a4470 :a4470 :a4470. +:a4471 :a4471 :a4471. +:a4472 :a4472 :a4472. +:a4473 :a4473 :a4473. +:a4474 :a4474 :a4474. +:a4475 :a4475 :a4475. +:a4476 :a4476 :a4476. +:a4477 :a4477 :a4477. +:a4478 :a4478 :a4478. +:a4479 :a4479 :a4479. +:a4480 :a4480 :a4480. +:a4481 :a4481 :a4481. +:a4482 :a4482 :a4482. +:a4483 :a4483 :a4483. +:a4484 :a4484 :a4484. +:a4485 :a4485 :a4485. +:a4486 :a4486 :a4486. +:a4487 :a4487 :a4487. +:a4488 :a4488 :a4488. +:a4489 :a4489 :a4489. +:a4490 :a4490 :a4490. +:a4491 :a4491 :a4491. +:a4492 :a4492 :a4492. +:a4493 :a4493 :a4493. +:a4494 :a4494 :a4494. +:a4495 :a4495 :a4495. +:a4496 :a4496 :a4496. +:a4497 :a4497 :a4497. +:a4498 :a4498 :a4498. +:a4499 :a4499 :a4499. +:a4500 :a4500 :a4500. +:a4501 :a4501 :a4501. +:a4502 :a4502 :a4502. +:a4503 :a4503 :a4503. +:a4504 :a4504 :a4504. +:a4505 :a4505 :a4505. +:a4506 :a4506 :a4506. +:a4507 :a4507 :a4507. +:a4508 :a4508 :a4508. +:a4509 :a4509 :a4509. +:a4510 :a4510 :a4510. +:a4511 :a4511 :a4511. +:a4512 :a4512 :a4512. +:a4513 :a4513 :a4513. +:a4514 :a4514 :a4514. +:a4515 :a4515 :a4515. +:a4516 :a4516 :a4516. +:a4517 :a4517 :a4517. +:a4518 :a4518 :a4518. +:a4519 :a4519 :a4519. +:a4520 :a4520 :a4520. +:a4521 :a4521 :a4521. +:a4522 :a4522 :a4522. +:a4523 :a4523 :a4523. +:a4524 :a4524 :a4524. +:a4525 :a4525 :a4525. +:a4526 :a4526 :a4526. +:a4527 :a4527 :a4527. +:a4528 :a4528 :a4528. +:a4529 :a4529 :a4529. +:a4530 :a4530 :a4530. +:a4531 :a4531 :a4531. +:a4532 :a4532 :a4532. +:a4533 :a4533 :a4533. +:a4534 :a4534 :a4534. +:a4535 :a4535 :a4535. +:a4536 :a4536 :a4536. +:a4537 :a4537 :a4537. +:a4538 :a4538 :a4538. +:a4539 :a4539 :a4539. +:a4540 :a4540 :a4540. +:a4541 :a4541 :a4541. +:a4542 :a4542 :a4542. +:a4543 :a4543 :a4543. +:a4544 :a4544 :a4544. +:a4545 :a4545 :a4545. +:a4546 :a4546 :a4546. +:a4547 :a4547 :a4547. +:a4548 :a4548 :a4548. +:a4549 :a4549 :a4549. +:a4550 :a4550 :a4550. +:a4551 :a4551 :a4551. +:a4552 :a4552 :a4552. +:a4553 :a4553 :a4553. +:a4554 :a4554 :a4554. +:a4555 :a4555 :a4555. +:a4556 :a4556 :a4556. +:a4557 :a4557 :a4557. +:a4558 :a4558 :a4558. +:a4559 :a4559 :a4559. +:a4560 :a4560 :a4560. +:a4561 :a4561 :a4561. +:a4562 :a4562 :a4562. +:a4563 :a4563 :a4563. +:a4564 :a4564 :a4564. +:a4565 :a4565 :a4565. +:a4566 :a4566 :a4566. +:a4567 :a4567 :a4567. +:a4568 :a4568 :a4568. +:a4569 :a4569 :a4569. +:a4570 :a4570 :a4570. +:a4571 :a4571 :a4571. +:a4572 :a4572 :a4572. +:a4573 :a4573 :a4573. +:a4574 :a4574 :a4574. +:a4575 :a4575 :a4575. +:a4576 :a4576 :a4576. +:a4577 :a4577 :a4577. +:a4578 :a4578 :a4578. +:a4579 :a4579 :a4579. +:a4580 :a4580 :a4580. +:a4581 :a4581 :a4581. +:a4582 :a4582 :a4582. +:a4583 :a4583 :a4583. +:a4584 :a4584 :a4584. +:a4585 :a4585 :a4585. +:a4586 :a4586 :a4586. +:a4587 :a4587 :a4587. +:a4588 :a4588 :a4588. +:a4589 :a4589 :a4589. +:a4590 :a4590 :a4590. +:a4591 :a4591 :a4591. +:a4592 :a4592 :a4592. +:a4593 :a4593 :a4593. +:a4594 :a4594 :a4594. +:a4595 :a4595 :a4595. +:a4596 :a4596 :a4596. +:a4597 :a4597 :a4597. +:a4598 :a4598 :a4598. +:a4599 :a4599 :a4599. +:a4600 :a4600 :a4600. +:a4601 :a4601 :a4601. +:a4602 :a4602 :a4602. +:a4603 :a4603 :a4603. +:a4604 :a4604 :a4604. +:a4605 :a4605 :a4605. +:a4606 :a4606 :a4606. +:a4607 :a4607 :a4607. +:a4608 :a4608 :a4608. +:a4609 :a4609 :a4609. +:a4610 :a4610 :a4610. +:a4611 :a4611 :a4611. +:a4612 :a4612 :a4612. +:a4613 :a4613 :a4613. +:a4614 :a4614 :a4614. +:a4615 :a4615 :a4615. +:a4616 :a4616 :a4616. +:a4617 :a4617 :a4617. +:a4618 :a4618 :a4618. +:a4619 :a4619 :a4619. +:a4620 :a4620 :a4620. +:a4621 :a4621 :a4621. +:a4622 :a4622 :a4622. +:a4623 :a4623 :a4623. +:a4624 :a4624 :a4624. +:a4625 :a4625 :a4625. +:a4626 :a4626 :a4626. +:a4627 :a4627 :a4627. +:a4628 :a4628 :a4628. +:a4629 :a4629 :a4629. +:a4630 :a4630 :a4630. +:a4631 :a4631 :a4631. +:a4632 :a4632 :a4632. +:a4633 :a4633 :a4633. +:a4634 :a4634 :a4634. +:a4635 :a4635 :a4635. +:a4636 :a4636 :a4636. +:a4637 :a4637 :a4637. +:a4638 :a4638 :a4638. +:a4639 :a4639 :a4639. +:a4640 :a4640 :a4640. +:a4641 :a4641 :a4641. +:a4642 :a4642 :a4642. +:a4643 :a4643 :a4643. +:a4644 :a4644 :a4644. +:a4645 :a4645 :a4645. +:a4646 :a4646 :a4646. +:a4647 :a4647 :a4647. +:a4648 :a4648 :a4648. +:a4649 :a4649 :a4649. +:a4650 :a4650 :a4650. +:a4651 :a4651 :a4651. +:a4652 :a4652 :a4652. +:a4653 :a4653 :a4653. +:a4654 :a4654 :a4654. +:a4655 :a4655 :a4655. +:a4656 :a4656 :a4656. +:a4657 :a4657 :a4657. +:a4658 :a4658 :a4658. +:a4659 :a4659 :a4659. +:a4660 :a4660 :a4660. +:a4661 :a4661 :a4661. +:a4662 :a4662 :a4662. +:a4663 :a4663 :a4663. +:a4664 :a4664 :a4664. +:a4665 :a4665 :a4665. +:a4666 :a4666 :a4666. +:a4667 :a4667 :a4667. +:a4668 :a4668 :a4668. +:a4669 :a4669 :a4669. +:a4670 :a4670 :a4670. +:a4671 :a4671 :a4671. +:a4672 :a4672 :a4672. +:a4673 :a4673 :a4673. +:a4674 :a4674 :a4674. +:a4675 :a4675 :a4675. +:a4676 :a4676 :a4676. +:a4677 :a4677 :a4677. +:a4678 :a4678 :a4678. +:a4679 :a4679 :a4679. +:a4680 :a4680 :a4680. +:a4681 :a4681 :a4681. +:a4682 :a4682 :a4682. +:a4683 :a4683 :a4683. +:a4684 :a4684 :a4684. +:a4685 :a4685 :a4685. +:a4686 :a4686 :a4686. +:a4687 :a4687 :a4687. +:a4688 :a4688 :a4688. +:a4689 :a4689 :a4689. +:a4690 :a4690 :a4690. +:a4691 :a4691 :a4691. +:a4692 :a4692 :a4692. +:a4693 :a4693 :a4693. +:a4694 :a4694 :a4694. +:a4695 :a4695 :a4695. +:a4696 :a4696 :a4696. +:a4697 :a4697 :a4697. +:a4698 :a4698 :a4698. +:a4699 :a4699 :a4699. +:a4700 :a4700 :a4700. +:a4701 :a4701 :a4701. +:a4702 :a4702 :a4702. +:a4703 :a4703 :a4703. +:a4704 :a4704 :a4704. +:a4705 :a4705 :a4705. +:a4706 :a4706 :a4706. +:a4707 :a4707 :a4707. +:a4708 :a4708 :a4708. +:a4709 :a4709 :a4709. +:a4710 :a4710 :a4710. +:a4711 :a4711 :a4711. +:a4712 :a4712 :a4712. +:a4713 :a4713 :a4713. +:a4714 :a4714 :a4714. +:a4715 :a4715 :a4715. +:a4716 :a4716 :a4716. +:a4717 :a4717 :a4717. +:a4718 :a4718 :a4718. +:a4719 :a4719 :a4719. +:a4720 :a4720 :a4720. +:a4721 :a4721 :a4721. +:a4722 :a4722 :a4722. +:a4723 :a4723 :a4723. +:a4724 :a4724 :a4724. +:a4725 :a4725 :a4725. +:a4726 :a4726 :a4726. +:a4727 :a4727 :a4727. +:a4728 :a4728 :a4728. +:a4729 :a4729 :a4729. +:a4730 :a4730 :a4730. +:a4731 :a4731 :a4731. +:a4732 :a4732 :a4732. +:a4733 :a4733 :a4733. +:a4734 :a4734 :a4734. +:a4735 :a4735 :a4735. +:a4736 :a4736 :a4736. +:a4737 :a4737 :a4737. +:a4738 :a4738 :a4738. +:a4739 :a4739 :a4739. +:a4740 :a4740 :a4740. +:a4741 :a4741 :a4741. +:a4742 :a4742 :a4742. +:a4743 :a4743 :a4743. +:a4744 :a4744 :a4744. +:a4745 :a4745 :a4745. +:a4746 :a4746 :a4746. +:a4747 :a4747 :a4747. +:a4748 :a4748 :a4748. +:a4749 :a4749 :a4749. +:a4750 :a4750 :a4750. +:a4751 :a4751 :a4751. +:a4752 :a4752 :a4752. +:a4753 :a4753 :a4753. +:a4754 :a4754 :a4754. +:a4755 :a4755 :a4755. +:a4756 :a4756 :a4756. +:a4757 :a4757 :a4757. +:a4758 :a4758 :a4758. +:a4759 :a4759 :a4759. +:a4760 :a4760 :a4760. +:a4761 :a4761 :a4761. +:a4762 :a4762 :a4762. +:a4763 :a4763 :a4763. +:a4764 :a4764 :a4764. +:a4765 :a4765 :a4765. +:a4766 :a4766 :a4766. +:a4767 :a4767 :a4767. +:a4768 :a4768 :a4768. +:a4769 :a4769 :a4769. +:a4770 :a4770 :a4770. +:a4771 :a4771 :a4771. +:a4772 :a4772 :a4772. +:a4773 :a4773 :a4773. +:a4774 :a4774 :a4774. +:a4775 :a4775 :a4775. +:a4776 :a4776 :a4776. +:a4777 :a4777 :a4777. +:a4778 :a4778 :a4778. +:a4779 :a4779 :a4779. +:a4780 :a4780 :a4780. +:a4781 :a4781 :a4781. +:a4782 :a4782 :a4782. +:a4783 :a4783 :a4783. +:a4784 :a4784 :a4784. +:a4785 :a4785 :a4785. +:a4786 :a4786 :a4786. +:a4787 :a4787 :a4787. +:a4788 :a4788 :a4788. +:a4789 :a4789 :a4789. +:a4790 :a4790 :a4790. +:a4791 :a4791 :a4791. +:a4792 :a4792 :a4792. +:a4793 :a4793 :a4793. +:a4794 :a4794 :a4794. +:a4795 :a4795 :a4795. +:a4796 :a4796 :a4796. +:a4797 :a4797 :a4797. +:a4798 :a4798 :a4798. +:a4799 :a4799 :a4799. +:a4800 :a4800 :a4800. +:a4801 :a4801 :a4801. +:a4802 :a4802 :a4802. +:a4803 :a4803 :a4803. +:a4804 :a4804 :a4804. +:a4805 :a4805 :a4805. +:a4806 :a4806 :a4806. +:a4807 :a4807 :a4807. +:a4808 :a4808 :a4808. +:a4809 :a4809 :a4809. +:a4810 :a4810 :a4810. +:a4811 :a4811 :a4811. +:a4812 :a4812 :a4812. +:a4813 :a4813 :a4813. +:a4814 :a4814 :a4814. +:a4815 :a4815 :a4815. +:a4816 :a4816 :a4816. +:a4817 :a4817 :a4817. +:a4818 :a4818 :a4818. +:a4819 :a4819 :a4819. +:a4820 :a4820 :a4820. +:a4821 :a4821 :a4821. +:a4822 :a4822 :a4822. +:a4823 :a4823 :a4823. +:a4824 :a4824 :a4824. +:a4825 :a4825 :a4825. +:a4826 :a4826 :a4826. +:a4827 :a4827 :a4827. +:a4828 :a4828 :a4828. +:a4829 :a4829 :a4829. +:a4830 :a4830 :a4830. +:a4831 :a4831 :a4831. +:a4832 :a4832 :a4832. +:a4833 :a4833 :a4833. +:a4834 :a4834 :a4834. +:a4835 :a4835 :a4835. +:a4836 :a4836 :a4836. +:a4837 :a4837 :a4837. +:a4838 :a4838 :a4838. +:a4839 :a4839 :a4839. +:a4840 :a4840 :a4840. +:a4841 :a4841 :a4841. +:a4842 :a4842 :a4842. +:a4843 :a4843 :a4843. +:a4844 :a4844 :a4844. +:a4845 :a4845 :a4845. +:a4846 :a4846 :a4846. +:a4847 :a4847 :a4847. +:a4848 :a4848 :a4848. +:a4849 :a4849 :a4849. +:a4850 :a4850 :a4850. +:a4851 :a4851 :a4851. +:a4852 :a4852 :a4852. +:a4853 :a4853 :a4853. +:a4854 :a4854 :a4854. +:a4855 :a4855 :a4855. +:a4856 :a4856 :a4856. +:a4857 :a4857 :a4857. +:a4858 :a4858 :a4858. +:a4859 :a4859 :a4859. +:a4860 :a4860 :a4860. +:a4861 :a4861 :a4861. +:a4862 :a4862 :a4862. +:a4863 :a4863 :a4863. +:a4864 :a4864 :a4864. +:a4865 :a4865 :a4865. +:a4866 :a4866 :a4866. +:a4867 :a4867 :a4867. +:a4868 :a4868 :a4868. +:a4869 :a4869 :a4869. +:a4870 :a4870 :a4870. +:a4871 :a4871 :a4871. +:a4872 :a4872 :a4872. +:a4873 :a4873 :a4873. +:a4874 :a4874 :a4874. +:a4875 :a4875 :a4875. +:a4876 :a4876 :a4876. +:a4877 :a4877 :a4877. +:a4878 :a4878 :a4878. +:a4879 :a4879 :a4879. +:a4880 :a4880 :a4880. +:a4881 :a4881 :a4881. +:a4882 :a4882 :a4882. +:a4883 :a4883 :a4883. +:a4884 :a4884 :a4884. +:a4885 :a4885 :a4885. +:a4886 :a4886 :a4886. +:a4887 :a4887 :a4887. +:a4888 :a4888 :a4888. +:a4889 :a4889 :a4889. +:a4890 :a4890 :a4890. +:a4891 :a4891 :a4891. +:a4892 :a4892 :a4892. +:a4893 :a4893 :a4893. +:a4894 :a4894 :a4894. +:a4895 :a4895 :a4895. +:a4896 :a4896 :a4896. +:a4897 :a4897 :a4897. +:a4898 :a4898 :a4898. +:a4899 :a4899 :a4899. +:a4900 :a4900 :a4900. +:a4901 :a4901 :a4901. +:a4902 :a4902 :a4902. +:a4903 :a4903 :a4903. +:a4904 :a4904 :a4904. +:a4905 :a4905 :a4905. +:a4906 :a4906 :a4906. +:a4907 :a4907 :a4907. +:a4908 :a4908 :a4908. +:a4909 :a4909 :a4909. +:a4910 :a4910 :a4910. +:a4911 :a4911 :a4911. +:a4912 :a4912 :a4912. +:a4913 :a4913 :a4913. +:a4914 :a4914 :a4914. +:a4915 :a4915 :a4915. +:a4916 :a4916 :a4916. +:a4917 :a4917 :a4917. +:a4918 :a4918 :a4918. +:a4919 :a4919 :a4919. +:a4920 :a4920 :a4920. +:a4921 :a4921 :a4921. +:a4922 :a4922 :a4922. +:a4923 :a4923 :a4923. +:a4924 :a4924 :a4924. +:a4925 :a4925 :a4925. +:a4926 :a4926 :a4926. +:a4927 :a4927 :a4927. +:a4928 :a4928 :a4928. +:a4929 :a4929 :a4929. +:a4930 :a4930 :a4930. +:a4931 :a4931 :a4931. +:a4932 :a4932 :a4932. +:a4933 :a4933 :a4933. +:a4934 :a4934 :a4934. +:a4935 :a4935 :a4935. +:a4936 :a4936 :a4936. +:a4937 :a4937 :a4937. +:a4938 :a4938 :a4938. +:a4939 :a4939 :a4939. +:a4940 :a4940 :a4940. +:a4941 :a4941 :a4941. +:a4942 :a4942 :a4942. +:a4943 :a4943 :a4943. +:a4944 :a4944 :a4944. +:a4945 :a4945 :a4945. +:a4946 :a4946 :a4946. +:a4947 :a4947 :a4947. +:a4948 :a4948 :a4948. +:a4949 :a4949 :a4949. +:a4950 :a4950 :a4950. +:a4951 :a4951 :a4951. +:a4952 :a4952 :a4952. +:a4953 :a4953 :a4953. +:a4954 :a4954 :a4954. +:a4955 :a4955 :a4955. +:a4956 :a4956 :a4956. +:a4957 :a4957 :a4957. +:a4958 :a4958 :a4958. +:a4959 :a4959 :a4959. +:a4960 :a4960 :a4960. +:a4961 :a4961 :a4961. +:a4962 :a4962 :a4962. +:a4963 :a4963 :a4963. +:a4964 :a4964 :a4964. +:a4965 :a4965 :a4965. +:a4966 :a4966 :a4966. +:a4967 :a4967 :a4967. +:a4968 :a4968 :a4968. +:a4969 :a4969 :a4969. +:a4970 :a4970 :a4970. +:a4971 :a4971 :a4971. +:a4972 :a4972 :a4972. +:a4973 :a4973 :a4973. +:a4974 :a4974 :a4974. +:a4975 :a4975 :a4975. +:a4976 :a4976 :a4976. +:a4977 :a4977 :a4977. +:a4978 :a4978 :a4978. +:a4979 :a4979 :a4979. +:a4980 :a4980 :a4980. +:a4981 :a4981 :a4981. +:a4982 :a4982 :a4982. +:a4983 :a4983 :a4983. +:a4984 :a4984 :a4984. +:a4985 :a4985 :a4985. +:a4986 :a4986 :a4986. +:a4987 :a4987 :a4987. +:a4988 :a4988 :a4988. +:a4989 :a4989 :a4989. +:a4990 :a4990 :a4990. +:a4991 :a4991 :a4991. +:a4992 :a4992 :a4992. +:a4993 :a4993 :a4993. +:a4994 :a4994 :a4994. +:a4995 :a4995 :a4995. +:a4996 :a4996 :a4996. +:a4997 :a4997 :a4997. +:a4998 :a4998 :a4998. +:a4999 :a4999 :a4999. +:a5000 :a5000 :a5000. +:a5001 :a5001 :a5001. +:a5002 :a5002 :a5002. +:a5003 :a5003 :a5003. +:a5004 :a5004 :a5004. +:a5005 :a5005 :a5005. +:a5006 :a5006 :a5006. +:a5007 :a5007 :a5007. +:a5008 :a5008 :a5008. +:a5009 :a5009 :a5009. +:a5010 :a5010 :a5010. +:a5011 :a5011 :a5011. +:a5012 :a5012 :a5012. +:a5013 :a5013 :a5013. +:a5014 :a5014 :a5014. +:a5015 :a5015 :a5015. +:a5016 :a5016 :a5016. +:a5017 :a5017 :a5017. +:a5018 :a5018 :a5018. +:a5019 :a5019 :a5019. +:a5020 :a5020 :a5020. +:a5021 :a5021 :a5021. +:a5022 :a5022 :a5022. +:a5023 :a5023 :a5023. +:a5024 :a5024 :a5024. +:a5025 :a5025 :a5025. +:a5026 :a5026 :a5026. +:a5027 :a5027 :a5027. +:a5028 :a5028 :a5028. +:a5029 :a5029 :a5029. +:a5030 :a5030 :a5030. +:a5031 :a5031 :a5031. +:a5032 :a5032 :a5032. +:a5033 :a5033 :a5033. +:a5034 :a5034 :a5034. +:a5035 :a5035 :a5035. +:a5036 :a5036 :a5036. +:a5037 :a5037 :a5037. +:a5038 :a5038 :a5038. +:a5039 :a5039 :a5039. +:a5040 :a5040 :a5040. +:a5041 :a5041 :a5041. +:a5042 :a5042 :a5042. +:a5043 :a5043 :a5043. +:a5044 :a5044 :a5044. +:a5045 :a5045 :a5045. +:a5046 :a5046 :a5046. +:a5047 :a5047 :a5047. +:a5048 :a5048 :a5048. +:a5049 :a5049 :a5049. +:a5050 :a5050 :a5050. +:a5051 :a5051 :a5051. +:a5052 :a5052 :a5052. +:a5053 :a5053 :a5053. +:a5054 :a5054 :a5054. +:a5055 :a5055 :a5055. +:a5056 :a5056 :a5056. +:a5057 :a5057 :a5057. +:a5058 :a5058 :a5058. +:a5059 :a5059 :a5059. +:a5060 :a5060 :a5060. +:a5061 :a5061 :a5061. +:a5062 :a5062 :a5062. +:a5063 :a5063 :a5063. +:a5064 :a5064 :a5064. +:a5065 :a5065 :a5065. +:a5066 :a5066 :a5066. +:a5067 :a5067 :a5067. +:a5068 :a5068 :a5068. +:a5069 :a5069 :a5069. +:a5070 :a5070 :a5070. +:a5071 :a5071 :a5071. +:a5072 :a5072 :a5072. +:a5073 :a5073 :a5073. +:a5074 :a5074 :a5074. +:a5075 :a5075 :a5075. +:a5076 :a5076 :a5076. +:a5077 :a5077 :a5077. +:a5078 :a5078 :a5078. +:a5079 :a5079 :a5079. +:a5080 :a5080 :a5080. +:a5081 :a5081 :a5081. +:a5082 :a5082 :a5082. +:a5083 :a5083 :a5083. +:a5084 :a5084 :a5084. +:a5085 :a5085 :a5085. +:a5086 :a5086 :a5086. +:a5087 :a5087 :a5087. +:a5088 :a5088 :a5088. +:a5089 :a5089 :a5089. +:a5090 :a5090 :a5090. +:a5091 :a5091 :a5091. +:a5092 :a5092 :a5092. +:a5093 :a5093 :a5093. +:a5094 :a5094 :a5094. +:a5095 :a5095 :a5095. +:a5096 :a5096 :a5096. +:a5097 :a5097 :a5097. +:a5098 :a5098 :a5098. +:a5099 :a5099 :a5099. +:a5100 :a5100 :a5100. +:a5101 :a5101 :a5101. +:a5102 :a5102 :a5102. +:a5103 :a5103 :a5103. +:a5104 :a5104 :a5104. +:a5105 :a5105 :a5105. +:a5106 :a5106 :a5106. +:a5107 :a5107 :a5107. +:a5108 :a5108 :a5108. +:a5109 :a5109 :a5109. +:a5110 :a5110 :a5110. +:a5111 :a5111 :a5111. +:a5112 :a5112 :a5112. +:a5113 :a5113 :a5113. +:a5114 :a5114 :a5114. +:a5115 :a5115 :a5115. +:a5116 :a5116 :a5116. +:a5117 :a5117 :a5117. +:a5118 :a5118 :a5118. +:a5119 :a5119 :a5119. +:a5120 :a5120 :a5120. +:a5121 :a5121 :a5121. +:a5122 :a5122 :a5122. +:a5123 :a5123 :a5123. +:a5124 :a5124 :a5124. +:a5125 :a5125 :a5125. +:a5126 :a5126 :a5126. +:a5127 :a5127 :a5127. +:a5128 :a5128 :a5128. +:a5129 :a5129 :a5129. +:a5130 :a5130 :a5130. +:a5131 :a5131 :a5131. +:a5132 :a5132 :a5132. +:a5133 :a5133 :a5133. +:a5134 :a5134 :a5134. +:a5135 :a5135 :a5135. +:a5136 :a5136 :a5136. +:a5137 :a5137 :a5137. +:a5138 :a5138 :a5138. +:a5139 :a5139 :a5139. +:a5140 :a5140 :a5140. +:a5141 :a5141 :a5141. +:a5142 :a5142 :a5142. +:a5143 :a5143 :a5143. +:a5144 :a5144 :a5144. +:a5145 :a5145 :a5145. +:a5146 :a5146 :a5146. +:a5147 :a5147 :a5147. +:a5148 :a5148 :a5148. +:a5149 :a5149 :a5149. +:a5150 :a5150 :a5150. +:a5151 :a5151 :a5151. +:a5152 :a5152 :a5152. +:a5153 :a5153 :a5153. +:a5154 :a5154 :a5154. +:a5155 :a5155 :a5155. +:a5156 :a5156 :a5156. +:a5157 :a5157 :a5157. +:a5158 :a5158 :a5158. +:a5159 :a5159 :a5159. +:a5160 :a5160 :a5160. +:a5161 :a5161 :a5161. +:a5162 :a5162 :a5162. +:a5163 :a5163 :a5163. +:a5164 :a5164 :a5164. +:a5165 :a5165 :a5165. +:a5166 :a5166 :a5166. +:a5167 :a5167 :a5167. +:a5168 :a5168 :a5168. +:a5169 :a5169 :a5169. +:a5170 :a5170 :a5170. +:a5171 :a5171 :a5171. +:a5172 :a5172 :a5172. +:a5173 :a5173 :a5173. +:a5174 :a5174 :a5174. +:a5175 :a5175 :a5175. +:a5176 :a5176 :a5176. +:a5177 :a5177 :a5177. +:a5178 :a5178 :a5178. +:a5179 :a5179 :a5179. +:a5180 :a5180 :a5180. +:a5181 :a5181 :a5181. +:a5182 :a5182 :a5182. +:a5183 :a5183 :a5183. +:a5184 :a5184 :a5184. +:a5185 :a5185 :a5185. +:a5186 :a5186 :a5186. +:a5187 :a5187 :a5187. +:a5188 :a5188 :a5188. +:a5189 :a5189 :a5189. +:a5190 :a5190 :a5190. +:a5191 :a5191 :a5191. +:a5192 :a5192 :a5192. +:a5193 :a5193 :a5193. +:a5194 :a5194 :a5194. +:a5195 :a5195 :a5195. +:a5196 :a5196 :a5196. +:a5197 :a5197 :a5197. +:a5198 :a5198 :a5198. +:a5199 :a5199 :a5199. +:a5200 :a5200 :a5200. +:a5201 :a5201 :a5201. +:a5202 :a5202 :a5202. +:a5203 :a5203 :a5203. +:a5204 :a5204 :a5204. +:a5205 :a5205 :a5205. +:a5206 :a5206 :a5206. +:a5207 :a5207 :a5207. +:a5208 :a5208 :a5208. +:a5209 :a5209 :a5209. +:a5210 :a5210 :a5210. +:a5211 :a5211 :a5211. +:a5212 :a5212 :a5212. +:a5213 :a5213 :a5213. +:a5214 :a5214 :a5214. +:a5215 :a5215 :a5215. +:a5216 :a5216 :a5216. +:a5217 :a5217 :a5217. +:a5218 :a5218 :a5218. +:a5219 :a5219 :a5219. +:a5220 :a5220 :a5220. +:a5221 :a5221 :a5221. +:a5222 :a5222 :a5222. +:a5223 :a5223 :a5223. +:a5224 :a5224 :a5224. +:a5225 :a5225 :a5225. +:a5226 :a5226 :a5226. +:a5227 :a5227 :a5227. +:a5228 :a5228 :a5228. +:a5229 :a5229 :a5229. +:a5230 :a5230 :a5230. +:a5231 :a5231 :a5231. +:a5232 :a5232 :a5232. +:a5233 :a5233 :a5233. +:a5234 :a5234 :a5234. +:a5235 :a5235 :a5235. +:a5236 :a5236 :a5236. +:a5237 :a5237 :a5237. +:a5238 :a5238 :a5238. +:a5239 :a5239 :a5239. +:a5240 :a5240 :a5240. +:a5241 :a5241 :a5241. +:a5242 :a5242 :a5242. +:a5243 :a5243 :a5243. +:a5244 :a5244 :a5244. +:a5245 :a5245 :a5245. +:a5246 :a5246 :a5246. +:a5247 :a5247 :a5247. +:a5248 :a5248 :a5248. +:a5249 :a5249 :a5249. +:a5250 :a5250 :a5250. +:a5251 :a5251 :a5251. +:a5252 :a5252 :a5252. +:a5253 :a5253 :a5253. +:a5254 :a5254 :a5254. +:a5255 :a5255 :a5255. +:a5256 :a5256 :a5256. +:a5257 :a5257 :a5257. +:a5258 :a5258 :a5258. +:a5259 :a5259 :a5259. +:a5260 :a5260 :a5260. +:a5261 :a5261 :a5261. +:a5262 :a5262 :a5262. +:a5263 :a5263 :a5263. +:a5264 :a5264 :a5264. +:a5265 :a5265 :a5265. +:a5266 :a5266 :a5266. +:a5267 :a5267 :a5267. +:a5268 :a5268 :a5268. +:a5269 :a5269 :a5269. +:a5270 :a5270 :a5270. +:a5271 :a5271 :a5271. +:a5272 :a5272 :a5272. +:a5273 :a5273 :a5273. +:a5274 :a5274 :a5274. +:a5275 :a5275 :a5275. +:a5276 :a5276 :a5276. +:a5277 :a5277 :a5277. +:a5278 :a5278 :a5278. +:a5279 :a5279 :a5279. +:a5280 :a5280 :a5280. +:a5281 :a5281 :a5281. +:a5282 :a5282 :a5282. +:a5283 :a5283 :a5283. +:a5284 :a5284 :a5284. +:a5285 :a5285 :a5285. +:a5286 :a5286 :a5286. +:a5287 :a5287 :a5287. +:a5288 :a5288 :a5288. +:a5289 :a5289 :a5289. +:a5290 :a5290 :a5290. +:a5291 :a5291 :a5291. +:a5292 :a5292 :a5292. +:a5293 :a5293 :a5293. +:a5294 :a5294 :a5294. +:a5295 :a5295 :a5295. +:a5296 :a5296 :a5296. +:a5297 :a5297 :a5297. +:a5298 :a5298 :a5298. +:a5299 :a5299 :a5299. +:a5300 :a5300 :a5300. +:a5301 :a5301 :a5301. +:a5302 :a5302 :a5302. +:a5303 :a5303 :a5303. +:a5304 :a5304 :a5304. +:a5305 :a5305 :a5305. +:a5306 :a5306 :a5306. +:a5307 :a5307 :a5307. +:a5308 :a5308 :a5308. +:a5309 :a5309 :a5309. +:a5310 :a5310 :a5310. +:a5311 :a5311 :a5311. +:a5312 :a5312 :a5312. +:a5313 :a5313 :a5313. +:a5314 :a5314 :a5314. +:a5315 :a5315 :a5315. +:a5316 :a5316 :a5316. +:a5317 :a5317 :a5317. +:a5318 :a5318 :a5318. +:a5319 :a5319 :a5319. +:a5320 :a5320 :a5320. +:a5321 :a5321 :a5321. +:a5322 :a5322 :a5322. +:a5323 :a5323 :a5323. +:a5324 :a5324 :a5324. +:a5325 :a5325 :a5325. +:a5326 :a5326 :a5326. +:a5327 :a5327 :a5327. +:a5328 :a5328 :a5328. +:a5329 :a5329 :a5329. +:a5330 :a5330 :a5330. +:a5331 :a5331 :a5331. +:a5332 :a5332 :a5332. +:a5333 :a5333 :a5333. +:a5334 :a5334 :a5334. +:a5335 :a5335 :a5335. +:a5336 :a5336 :a5336. +:a5337 :a5337 :a5337. +:a5338 :a5338 :a5338. +:a5339 :a5339 :a5339. +:a5340 :a5340 :a5340. +:a5341 :a5341 :a5341. +:a5342 :a5342 :a5342. +:a5343 :a5343 :a5343. +:a5344 :a5344 :a5344. +:a5345 :a5345 :a5345. +:a5346 :a5346 :a5346. +:a5347 :a5347 :a5347. +:a5348 :a5348 :a5348. +:a5349 :a5349 :a5349. +:a5350 :a5350 :a5350. +:a5351 :a5351 :a5351. +:a5352 :a5352 :a5352. +:a5353 :a5353 :a5353. +:a5354 :a5354 :a5354. +:a5355 :a5355 :a5355. +:a5356 :a5356 :a5356. +:a5357 :a5357 :a5357. +:a5358 :a5358 :a5358. +:a5359 :a5359 :a5359. +:a5360 :a5360 :a5360. +:a5361 :a5361 :a5361. +:a5362 :a5362 :a5362. +:a5363 :a5363 :a5363. +:a5364 :a5364 :a5364. +:a5365 :a5365 :a5365. +:a5366 :a5366 :a5366. +:a5367 :a5367 :a5367. +:a5368 :a5368 :a5368. +:a5369 :a5369 :a5369. +:a5370 :a5370 :a5370. +:a5371 :a5371 :a5371. +:a5372 :a5372 :a5372. +:a5373 :a5373 :a5373. +:a5374 :a5374 :a5374. +:a5375 :a5375 :a5375. +:a5376 :a5376 :a5376. +:a5377 :a5377 :a5377. +:a5378 :a5378 :a5378. +:a5379 :a5379 :a5379. +:a5380 :a5380 :a5380. +:a5381 :a5381 :a5381. +:a5382 :a5382 :a5382. +:a5383 :a5383 :a5383. +:a5384 :a5384 :a5384. +:a5385 :a5385 :a5385. +:a5386 :a5386 :a5386. +:a5387 :a5387 :a5387. +:a5388 :a5388 :a5388. +:a5389 :a5389 :a5389. +:a5390 :a5390 :a5390. +:a5391 :a5391 :a5391. +:a5392 :a5392 :a5392. +:a5393 :a5393 :a5393. +:a5394 :a5394 :a5394. +:a5395 :a5395 :a5395. +:a5396 :a5396 :a5396. +:a5397 :a5397 :a5397. +:a5398 :a5398 :a5398. +:a5399 :a5399 :a5399. +:a5400 :a5400 :a5400. +:a5401 :a5401 :a5401. +:a5402 :a5402 :a5402. +:a5403 :a5403 :a5403. +:a5404 :a5404 :a5404. +:a5405 :a5405 :a5405. +:a5406 :a5406 :a5406. +:a5407 :a5407 :a5407. +:a5408 :a5408 :a5408. +:a5409 :a5409 :a5409. +:a5410 :a5410 :a5410. +:a5411 :a5411 :a5411. +:a5412 :a5412 :a5412. +:a5413 :a5413 :a5413. +:a5414 :a5414 :a5414. +:a5415 :a5415 :a5415. +:a5416 :a5416 :a5416. +:a5417 :a5417 :a5417. +:a5418 :a5418 :a5418. +:a5419 :a5419 :a5419. +:a5420 :a5420 :a5420. +:a5421 :a5421 :a5421. +:a5422 :a5422 :a5422. +:a5423 :a5423 :a5423. +:a5424 :a5424 :a5424. +:a5425 :a5425 :a5425. +:a5426 :a5426 :a5426. +:a5427 :a5427 :a5427. +:a5428 :a5428 :a5428. +:a5429 :a5429 :a5429. +:a5430 :a5430 :a5430. +:a5431 :a5431 :a5431. +:a5432 :a5432 :a5432. +:a5433 :a5433 :a5433. +:a5434 :a5434 :a5434. +:a5435 :a5435 :a5435. +:a5436 :a5436 :a5436. +:a5437 :a5437 :a5437. +:a5438 :a5438 :a5438. +:a5439 :a5439 :a5439. +:a5440 :a5440 :a5440. +:a5441 :a5441 :a5441. +:a5442 :a5442 :a5442. +:a5443 :a5443 :a5443. +:a5444 :a5444 :a5444. +:a5445 :a5445 :a5445. +:a5446 :a5446 :a5446. +:a5447 :a5447 :a5447. +:a5448 :a5448 :a5448. +:a5449 :a5449 :a5449. +:a5450 :a5450 :a5450. +:a5451 :a5451 :a5451. +:a5452 :a5452 :a5452. +:a5453 :a5453 :a5453. +:a5454 :a5454 :a5454. +:a5455 :a5455 :a5455. +:a5456 :a5456 :a5456. +:a5457 :a5457 :a5457. +:a5458 :a5458 :a5458. +:a5459 :a5459 :a5459. +:a5460 :a5460 :a5460. +:a5461 :a5461 :a5461. +:a5462 :a5462 :a5462. +:a5463 :a5463 :a5463. +:a5464 :a5464 :a5464. +:a5465 :a5465 :a5465. +:a5466 :a5466 :a5466. +:a5467 :a5467 :a5467. +:a5468 :a5468 :a5468. +:a5469 :a5469 :a5469. +:a5470 :a5470 :a5470. +:a5471 :a5471 :a5471. +:a5472 :a5472 :a5472. +:a5473 :a5473 :a5473. +:a5474 :a5474 :a5474. +:a5475 :a5475 :a5475. +:a5476 :a5476 :a5476. +:a5477 :a5477 :a5477. +:a5478 :a5478 :a5478. +:a5479 :a5479 :a5479. +:a5480 :a5480 :a5480. +:a5481 :a5481 :a5481. +:a5482 :a5482 :a5482. +:a5483 :a5483 :a5483. +:a5484 :a5484 :a5484. +:a5485 :a5485 :a5485. +:a5486 :a5486 :a5486. +:a5487 :a5487 :a5487. +:a5488 :a5488 :a5488. +:a5489 :a5489 :a5489. +:a5490 :a5490 :a5490. +:a5491 :a5491 :a5491. +:a5492 :a5492 :a5492. +:a5493 :a5493 :a5493. +:a5494 :a5494 :a5494. +:a5495 :a5495 :a5495. +:a5496 :a5496 :a5496. +:a5497 :a5497 :a5497. +:a5498 :a5498 :a5498. +:a5499 :a5499 :a5499. +:a5500 :a5500 :a5500. +:a5501 :a5501 :a5501. +:a5502 :a5502 :a5502. +:a5503 :a5503 :a5503. +:a5504 :a5504 :a5504. +:a5505 :a5505 :a5505. +:a5506 :a5506 :a5506. +:a5507 :a5507 :a5507. +:a5508 :a5508 :a5508. +:a5509 :a5509 :a5509. +:a5510 :a5510 :a5510. +:a5511 :a5511 :a5511. +:a5512 :a5512 :a5512. +:a5513 :a5513 :a5513. +:a5514 :a5514 :a5514. +:a5515 :a5515 :a5515. +:a5516 :a5516 :a5516. +:a5517 :a5517 :a5517. +:a5518 :a5518 :a5518. +:a5519 :a5519 :a5519. +:a5520 :a5520 :a5520. +:a5521 :a5521 :a5521. +:a5522 :a5522 :a5522. +:a5523 :a5523 :a5523. +:a5524 :a5524 :a5524. +:a5525 :a5525 :a5525. +:a5526 :a5526 :a5526. +:a5527 :a5527 :a5527. +:a5528 :a5528 :a5528. +:a5529 :a5529 :a5529. +:a5530 :a5530 :a5530. +:a5531 :a5531 :a5531. +:a5532 :a5532 :a5532. +:a5533 :a5533 :a5533. +:a5534 :a5534 :a5534. +:a5535 :a5535 :a5535. +:a5536 :a5536 :a5536. +:a5537 :a5537 :a5537. +:a5538 :a5538 :a5538. +:a5539 :a5539 :a5539. +:a5540 :a5540 :a5540. +:a5541 :a5541 :a5541. +:a5542 :a5542 :a5542. +:a5543 :a5543 :a5543. +:a5544 :a5544 :a5544. +:a5545 :a5545 :a5545. +:a5546 :a5546 :a5546. +:a5547 :a5547 :a5547. +:a5548 :a5548 :a5548. +:a5549 :a5549 :a5549. +:a5550 :a5550 :a5550. +:a5551 :a5551 :a5551. +:a5552 :a5552 :a5552. +:a5553 :a5553 :a5553. +:a5554 :a5554 :a5554. +:a5555 :a5555 :a5555. +:a5556 :a5556 :a5556. +:a5557 :a5557 :a5557. +:a5558 :a5558 :a5558. +:a5559 :a5559 :a5559. +:a5560 :a5560 :a5560. +:a5561 :a5561 :a5561. +:a5562 :a5562 :a5562. +:a5563 :a5563 :a5563. +:a5564 :a5564 :a5564. +:a5565 :a5565 :a5565. +:a5566 :a5566 :a5566. +:a5567 :a5567 :a5567. +:a5568 :a5568 :a5568. +:a5569 :a5569 :a5569. +:a5570 :a5570 :a5570. +:a5571 :a5571 :a5571. +:a5572 :a5572 :a5572. +:a5573 :a5573 :a5573. +:a5574 :a5574 :a5574. +:a5575 :a5575 :a5575. +:a5576 :a5576 :a5576. +:a5577 :a5577 :a5577. +:a5578 :a5578 :a5578. +:a5579 :a5579 :a5579. +:a5580 :a5580 :a5580. +:a5581 :a5581 :a5581. +:a5582 :a5582 :a5582. +:a5583 :a5583 :a5583. +:a5584 :a5584 :a5584. +:a5585 :a5585 :a5585. +:a5586 :a5586 :a5586. +:a5587 :a5587 :a5587. +:a5588 :a5588 :a5588. +:a5589 :a5589 :a5589. +:a5590 :a5590 :a5590. +:a5591 :a5591 :a5591. +:a5592 :a5592 :a5592. +:a5593 :a5593 :a5593. +:a5594 :a5594 :a5594. +:a5595 :a5595 :a5595. +:a5596 :a5596 :a5596. +:a5597 :a5597 :a5597. +:a5598 :a5598 :a5598. +:a5599 :a5599 :a5599. +:a5600 :a5600 :a5600. +:a5601 :a5601 :a5601. +:a5602 :a5602 :a5602. +:a5603 :a5603 :a5603. +:a5604 :a5604 :a5604. +:a5605 :a5605 :a5605. +:a5606 :a5606 :a5606. +:a5607 :a5607 :a5607. +:a5608 :a5608 :a5608. +:a5609 :a5609 :a5609. +:a5610 :a5610 :a5610. +:a5611 :a5611 :a5611. +:a5612 :a5612 :a5612. +:a5613 :a5613 :a5613. +:a5614 :a5614 :a5614. +:a5615 :a5615 :a5615. +:a5616 :a5616 :a5616. +:a5617 :a5617 :a5617. +:a5618 :a5618 :a5618. +:a5619 :a5619 :a5619. +:a5620 :a5620 :a5620. +:a5621 :a5621 :a5621. +:a5622 :a5622 :a5622. +:a5623 :a5623 :a5623. +:a5624 :a5624 :a5624. +:a5625 :a5625 :a5625. +:a5626 :a5626 :a5626. +:a5627 :a5627 :a5627. +:a5628 :a5628 :a5628. +:a5629 :a5629 :a5629. +:a5630 :a5630 :a5630. +:a5631 :a5631 :a5631. +:a5632 :a5632 :a5632. +:a5633 :a5633 :a5633. +:a5634 :a5634 :a5634. +:a5635 :a5635 :a5635. +:a5636 :a5636 :a5636. +:a5637 :a5637 :a5637. +:a5638 :a5638 :a5638. +:a5639 :a5639 :a5639. +:a5640 :a5640 :a5640. +:a5641 :a5641 :a5641. +:a5642 :a5642 :a5642. +:a5643 :a5643 :a5643. +:a5644 :a5644 :a5644. +:a5645 :a5645 :a5645. +:a5646 :a5646 :a5646. +:a5647 :a5647 :a5647. +:a5648 :a5648 :a5648. +:a5649 :a5649 :a5649. +:a5650 :a5650 :a5650. +:a5651 :a5651 :a5651. +:a5652 :a5652 :a5652. +:a5653 :a5653 :a5653. +:a5654 :a5654 :a5654. +:a5655 :a5655 :a5655. +:a5656 :a5656 :a5656. +:a5657 :a5657 :a5657. +:a5658 :a5658 :a5658. +:a5659 :a5659 :a5659. +:a5660 :a5660 :a5660. +:a5661 :a5661 :a5661. +:a5662 :a5662 :a5662. +:a5663 :a5663 :a5663. +:a5664 :a5664 :a5664. +:a5665 :a5665 :a5665. +:a5666 :a5666 :a5666. +:a5667 :a5667 :a5667. +:a5668 :a5668 :a5668. +:a5669 :a5669 :a5669. +:a5670 :a5670 :a5670. +:a5671 :a5671 :a5671. +:a5672 :a5672 :a5672. +:a5673 :a5673 :a5673. +:a5674 :a5674 :a5674. +:a5675 :a5675 :a5675. +:a5676 :a5676 :a5676. +:a5677 :a5677 :a5677. +:a5678 :a5678 :a5678. +:a5679 :a5679 :a5679. +:a5680 :a5680 :a5680. +:a5681 :a5681 :a5681. +:a5682 :a5682 :a5682. +:a5683 :a5683 :a5683. +:a5684 :a5684 :a5684. +:a5685 :a5685 :a5685. +:a5686 :a5686 :a5686. +:a5687 :a5687 :a5687. +:a5688 :a5688 :a5688. +:a5689 :a5689 :a5689. +:a5690 :a5690 :a5690. +:a5691 :a5691 :a5691. +:a5692 :a5692 :a5692. +:a5693 :a5693 :a5693. +:a5694 :a5694 :a5694. +:a5695 :a5695 :a5695. +:a5696 :a5696 :a5696. +:a5697 :a5697 :a5697. +:a5698 :a5698 :a5698. +:a5699 :a5699 :a5699. +:a5700 :a5700 :a5700. +:a5701 :a5701 :a5701. +:a5702 :a5702 :a5702. +:a5703 :a5703 :a5703. +:a5704 :a5704 :a5704. +:a5705 :a5705 :a5705. +:a5706 :a5706 :a5706. +:a5707 :a5707 :a5707. +:a5708 :a5708 :a5708. +:a5709 :a5709 :a5709. +:a5710 :a5710 :a5710. +:a5711 :a5711 :a5711. +:a5712 :a5712 :a5712. +:a5713 :a5713 :a5713. +:a5714 :a5714 :a5714. +:a5715 :a5715 :a5715. +:a5716 :a5716 :a5716. +:a5717 :a5717 :a5717. +:a5718 :a5718 :a5718. +:a5719 :a5719 :a5719. +:a5720 :a5720 :a5720. +:a5721 :a5721 :a5721. +:a5722 :a5722 :a5722. +:a5723 :a5723 :a5723. +:a5724 :a5724 :a5724. +:a5725 :a5725 :a5725. +:a5726 :a5726 :a5726. +:a5727 :a5727 :a5727. +:a5728 :a5728 :a5728. +:a5729 :a5729 :a5729. +:a5730 :a5730 :a5730. +:a5731 :a5731 :a5731. +:a5732 :a5732 :a5732. +:a5733 :a5733 :a5733. +:a5734 :a5734 :a5734. +:a5735 :a5735 :a5735. +:a5736 :a5736 :a5736. +:a5737 :a5737 :a5737. +:a5738 :a5738 :a5738. +:a5739 :a5739 :a5739. +:a5740 :a5740 :a5740. +:a5741 :a5741 :a5741. +:a5742 :a5742 :a5742. +:a5743 :a5743 :a5743. +:a5744 :a5744 :a5744. +:a5745 :a5745 :a5745. +:a5746 :a5746 :a5746. +:a5747 :a5747 :a5747. +:a5748 :a5748 :a5748. +:a5749 :a5749 :a5749. +:a5750 :a5750 :a5750. +:a5751 :a5751 :a5751. +:a5752 :a5752 :a5752. +:a5753 :a5753 :a5753. +:a5754 :a5754 :a5754. +:a5755 :a5755 :a5755. +:a5756 :a5756 :a5756. +:a5757 :a5757 :a5757. +:a5758 :a5758 :a5758. +:a5759 :a5759 :a5759. +:a5760 :a5760 :a5760. +:a5761 :a5761 :a5761. +:a5762 :a5762 :a5762. +:a5763 :a5763 :a5763. +:a5764 :a5764 :a5764. +:a5765 :a5765 :a5765. +:a5766 :a5766 :a5766. +:a5767 :a5767 :a5767. +:a5768 :a5768 :a5768. +:a5769 :a5769 :a5769. +:a5770 :a5770 :a5770. +:a5771 :a5771 :a5771. +:a5772 :a5772 :a5772. +:a5773 :a5773 :a5773. +:a5774 :a5774 :a5774. +:a5775 :a5775 :a5775. +:a5776 :a5776 :a5776. +:a5777 :a5777 :a5777. +:a5778 :a5778 :a5778. +:a5779 :a5779 :a5779. +:a5780 :a5780 :a5780. +:a5781 :a5781 :a5781. +:a5782 :a5782 :a5782. +:a5783 :a5783 :a5783. +:a5784 :a5784 :a5784. +:a5785 :a5785 :a5785. +:a5786 :a5786 :a5786. +:a5787 :a5787 :a5787. +:a5788 :a5788 :a5788. +:a5789 :a5789 :a5789. +:a5790 :a5790 :a5790. +:a5791 :a5791 :a5791. +:a5792 :a5792 :a5792. +:a5793 :a5793 :a5793. +:a5794 :a5794 :a5794. +:a5795 :a5795 :a5795. +:a5796 :a5796 :a5796. +:a5797 :a5797 :a5797. +:a5798 :a5798 :a5798. +:a5799 :a5799 :a5799. +:a5800 :a5800 :a5800. +:a5801 :a5801 :a5801. +:a5802 :a5802 :a5802. +:a5803 :a5803 :a5803. +:a5804 :a5804 :a5804. +:a5805 :a5805 :a5805. +:a5806 :a5806 :a5806. +:a5807 :a5807 :a5807. +:a5808 :a5808 :a5808. +:a5809 :a5809 :a5809. +:a5810 :a5810 :a5810. +:a5811 :a5811 :a5811. +:a5812 :a5812 :a5812. +:a5813 :a5813 :a5813. +:a5814 :a5814 :a5814. +:a5815 :a5815 :a5815. +:a5816 :a5816 :a5816. +:a5817 :a5817 :a5817. +:a5818 :a5818 :a5818. +:a5819 :a5819 :a5819. +:a5820 :a5820 :a5820. +:a5821 :a5821 :a5821. +:a5822 :a5822 :a5822. +:a5823 :a5823 :a5823. +:a5824 :a5824 :a5824. +:a5825 :a5825 :a5825. +:a5826 :a5826 :a5826. +:a5827 :a5827 :a5827. +:a5828 :a5828 :a5828. +:a5829 :a5829 :a5829. +:a5830 :a5830 :a5830. +:a5831 :a5831 :a5831. +:a5832 :a5832 :a5832. +:a5833 :a5833 :a5833. +:a5834 :a5834 :a5834. +:a5835 :a5835 :a5835. +:a5836 :a5836 :a5836. +:a5837 :a5837 :a5837. +:a5838 :a5838 :a5838. +:a5839 :a5839 :a5839. +:a5840 :a5840 :a5840. +:a5841 :a5841 :a5841. +:a5842 :a5842 :a5842. +:a5843 :a5843 :a5843. +:a5844 :a5844 :a5844. +:a5845 :a5845 :a5845. +:a5846 :a5846 :a5846. +:a5847 :a5847 :a5847. +:a5848 :a5848 :a5848. +:a5849 :a5849 :a5849. +:a5850 :a5850 :a5850. +:a5851 :a5851 :a5851. +:a5852 :a5852 :a5852. +:a5853 :a5853 :a5853. +:a5854 :a5854 :a5854. +:a5855 :a5855 :a5855. +:a5856 :a5856 :a5856. +:a5857 :a5857 :a5857. +:a5858 :a5858 :a5858. +:a5859 :a5859 :a5859. +:a5860 :a5860 :a5860. +:a5861 :a5861 :a5861. +:a5862 :a5862 :a5862. +:a5863 :a5863 :a5863. +:a5864 :a5864 :a5864. +:a5865 :a5865 :a5865. +:a5866 :a5866 :a5866. +:a5867 :a5867 :a5867. +:a5868 :a5868 :a5868. +:a5869 :a5869 :a5869. +:a5870 :a5870 :a5870. +:a5871 :a5871 :a5871. +:a5872 :a5872 :a5872. +:a5873 :a5873 :a5873. +:a5874 :a5874 :a5874. +:a5875 :a5875 :a5875. +:a5876 :a5876 :a5876. +:a5877 :a5877 :a5877. +:a5878 :a5878 :a5878. +:a5879 :a5879 :a5879. +:a5880 :a5880 :a5880. +:a5881 :a5881 :a5881. +:a5882 :a5882 :a5882. +:a5883 :a5883 :a5883. +:a5884 :a5884 :a5884. +:a5885 :a5885 :a5885. +:a5886 :a5886 :a5886. +:a5887 :a5887 :a5887. +:a5888 :a5888 :a5888. +:a5889 :a5889 :a5889. +:a5890 :a5890 :a5890. +:a5891 :a5891 :a5891. +:a5892 :a5892 :a5892. +:a5893 :a5893 :a5893. +:a5894 :a5894 :a5894. +:a5895 :a5895 :a5895. +:a5896 :a5896 :a5896. +:a5897 :a5897 :a5897. +:a5898 :a5898 :a5898. +:a5899 :a5899 :a5899. +:a5900 :a5900 :a5900. +:a5901 :a5901 :a5901. +:a5902 :a5902 :a5902. +:a5903 :a5903 :a5903. +:a5904 :a5904 :a5904. +:a5905 :a5905 :a5905. +:a5906 :a5906 :a5906. +:a5907 :a5907 :a5907. +:a5908 :a5908 :a5908. +:a5909 :a5909 :a5909. +:a5910 :a5910 :a5910. +:a5911 :a5911 :a5911. +:a5912 :a5912 :a5912. +:a5913 :a5913 :a5913. +:a5914 :a5914 :a5914. +:a5915 :a5915 :a5915. +:a5916 :a5916 :a5916. +:a5917 :a5917 :a5917. +:a5918 :a5918 :a5918. +:a5919 :a5919 :a5919. +:a5920 :a5920 :a5920. +:a5921 :a5921 :a5921. +:a5922 :a5922 :a5922. +:a5923 :a5923 :a5923. +:a5924 :a5924 :a5924. +:a5925 :a5925 :a5925. +:a5926 :a5926 :a5926. +:a5927 :a5927 :a5927. +:a5928 :a5928 :a5928. +:a5929 :a5929 :a5929. +:a5930 :a5930 :a5930. +:a5931 :a5931 :a5931. +:a5932 :a5932 :a5932. +:a5933 :a5933 :a5933. +:a5934 :a5934 :a5934. +:a5935 :a5935 :a5935. +:a5936 :a5936 :a5936. +:a5937 :a5937 :a5937. +:a5938 :a5938 :a5938. +:a5939 :a5939 :a5939. +:a5940 :a5940 :a5940. +:a5941 :a5941 :a5941. +:a5942 :a5942 :a5942. +:a5943 :a5943 :a5943. +:a5944 :a5944 :a5944. +:a5945 :a5945 :a5945. +:a5946 :a5946 :a5946. +:a5947 :a5947 :a5947. +:a5948 :a5948 :a5948. +:a5949 :a5949 :a5949. +:a5950 :a5950 :a5950. +:a5951 :a5951 :a5951. +:a5952 :a5952 :a5952. +:a5953 :a5953 :a5953. +:a5954 :a5954 :a5954. +:a5955 :a5955 :a5955. +:a5956 :a5956 :a5956. +:a5957 :a5957 :a5957. +:a5958 :a5958 :a5958. +:a5959 :a5959 :a5959. +:a5960 :a5960 :a5960. +:a5961 :a5961 :a5961. +:a5962 :a5962 :a5962. +:a5963 :a5963 :a5963. +:a5964 :a5964 :a5964. +:a5965 :a5965 :a5965. +:a5966 :a5966 :a5966. +:a5967 :a5967 :a5967. +:a5968 :a5968 :a5968. +:a5969 :a5969 :a5969. +:a5970 :a5970 :a5970. +:a5971 :a5971 :a5971. +:a5972 :a5972 :a5972. +:a5973 :a5973 :a5973. +:a5974 :a5974 :a5974. +:a5975 :a5975 :a5975. +:a5976 :a5976 :a5976. +:a5977 :a5977 :a5977. +:a5978 :a5978 :a5978. +:a5979 :a5979 :a5979. +:a5980 :a5980 :a5980. +:a5981 :a5981 :a5981. +:a5982 :a5982 :a5982. +:a5983 :a5983 :a5983. +:a5984 :a5984 :a5984. +:a5985 :a5985 :a5985. +:a5986 :a5986 :a5986. +:a5987 :a5987 :a5987. +:a5988 :a5988 :a5988. +:a5989 :a5989 :a5989. +:a5990 :a5990 :a5990. +:a5991 :a5991 :a5991. +:a5992 :a5992 :a5992. +:a5993 :a5993 :a5993. +:a5994 :a5994 :a5994. +:a5995 :a5995 :a5995. +:a5996 :a5996 :a5996. +:a5997 :a5997 :a5997. +:a5998 :a5998 :a5998. +:a5999 :a5999 :a5999. +:a6000 :a6000 :a6000. +:a6001 :a6001 :a6001. +:a6002 :a6002 :a6002. +:a6003 :a6003 :a6003. +:a6004 :a6004 :a6004. +:a6005 :a6005 :a6005. +:a6006 :a6006 :a6006. +:a6007 :a6007 :a6007. +:a6008 :a6008 :a6008. +:a6009 :a6009 :a6009. +:a6010 :a6010 :a6010. +:a6011 :a6011 :a6011. +:a6012 :a6012 :a6012. +:a6013 :a6013 :a6013. +:a6014 :a6014 :a6014. +:a6015 :a6015 :a6015. +:a6016 :a6016 :a6016. +:a6017 :a6017 :a6017. +:a6018 :a6018 :a6018. +:a6019 :a6019 :a6019. +:a6020 :a6020 :a6020. +:a6021 :a6021 :a6021. +:a6022 :a6022 :a6022. +:a6023 :a6023 :a6023. +:a6024 :a6024 :a6024. +:a6025 :a6025 :a6025. +:a6026 :a6026 :a6026. +:a6027 :a6027 :a6027. +:a6028 :a6028 :a6028. +:a6029 :a6029 :a6029. +:a6030 :a6030 :a6030. +:a6031 :a6031 :a6031. +:a6032 :a6032 :a6032. +:a6033 :a6033 :a6033. +:a6034 :a6034 :a6034. +:a6035 :a6035 :a6035. +:a6036 :a6036 :a6036. +:a6037 :a6037 :a6037. +:a6038 :a6038 :a6038. +:a6039 :a6039 :a6039. +:a6040 :a6040 :a6040. +:a6041 :a6041 :a6041. +:a6042 :a6042 :a6042. +:a6043 :a6043 :a6043. +:a6044 :a6044 :a6044. +:a6045 :a6045 :a6045. +:a6046 :a6046 :a6046. +:a6047 :a6047 :a6047. +:a6048 :a6048 :a6048. +:a6049 :a6049 :a6049. +:a6050 :a6050 :a6050. +:a6051 :a6051 :a6051. +:a6052 :a6052 :a6052. +:a6053 :a6053 :a6053. +:a6054 :a6054 :a6054. +:a6055 :a6055 :a6055. +:a6056 :a6056 :a6056. +:a6057 :a6057 :a6057. +:a6058 :a6058 :a6058. +:a6059 :a6059 :a6059. +:a6060 :a6060 :a6060. +:a6061 :a6061 :a6061. +:a6062 :a6062 :a6062. +:a6063 :a6063 :a6063. +:a6064 :a6064 :a6064. +:a6065 :a6065 :a6065. +:a6066 :a6066 :a6066. +:a6067 :a6067 :a6067. +:a6068 :a6068 :a6068. +:a6069 :a6069 :a6069. +:a6070 :a6070 :a6070. +:a6071 :a6071 :a6071. +:a6072 :a6072 :a6072. +:a6073 :a6073 :a6073. +:a6074 :a6074 :a6074. +:a6075 :a6075 :a6075. +:a6076 :a6076 :a6076. +:a6077 :a6077 :a6077. +:a6078 :a6078 :a6078. +:a6079 :a6079 :a6079. +:a6080 :a6080 :a6080. +:a6081 :a6081 :a6081. +:a6082 :a6082 :a6082. +:a6083 :a6083 :a6083. +:a6084 :a6084 :a6084. +:a6085 :a6085 :a6085. +:a6086 :a6086 :a6086. +:a6087 :a6087 :a6087. +:a6088 :a6088 :a6088. +:a6089 :a6089 :a6089. +:a6090 :a6090 :a6090. +:a6091 :a6091 :a6091. +:a6092 :a6092 :a6092. +:a6093 :a6093 :a6093. +:a6094 :a6094 :a6094. +:a6095 :a6095 :a6095. +:a6096 :a6096 :a6096. +:a6097 :a6097 :a6097. +:a6098 :a6098 :a6098. +:a6099 :a6099 :a6099. +:a6100 :a6100 :a6100. +:a6101 :a6101 :a6101. +:a6102 :a6102 :a6102. +:a6103 :a6103 :a6103. +:a6104 :a6104 :a6104. +:a6105 :a6105 :a6105. +:a6106 :a6106 :a6106. +:a6107 :a6107 :a6107. +:a6108 :a6108 :a6108. +:a6109 :a6109 :a6109. +:a6110 :a6110 :a6110. +:a6111 :a6111 :a6111. +:a6112 :a6112 :a6112. +:a6113 :a6113 :a6113. +:a6114 :a6114 :a6114. +:a6115 :a6115 :a6115. +:a6116 :a6116 :a6116. +:a6117 :a6117 :a6117. +:a6118 :a6118 :a6118. +:a6119 :a6119 :a6119. +:a6120 :a6120 :a6120. +:a6121 :a6121 :a6121. +:a6122 :a6122 :a6122. +:a6123 :a6123 :a6123. +:a6124 :a6124 :a6124. +:a6125 :a6125 :a6125. +:a6126 :a6126 :a6126. +:a6127 :a6127 :a6127. +:a6128 :a6128 :a6128. +:a6129 :a6129 :a6129. +:a6130 :a6130 :a6130. +:a6131 :a6131 :a6131. +:a6132 :a6132 :a6132. +:a6133 :a6133 :a6133. +:a6134 :a6134 :a6134. +:a6135 :a6135 :a6135. +:a6136 :a6136 :a6136. +:a6137 :a6137 :a6137. +:a6138 :a6138 :a6138. +:a6139 :a6139 :a6139. +:a6140 :a6140 :a6140. +:a6141 :a6141 :a6141. +:a6142 :a6142 :a6142. +:a6143 :a6143 :a6143. +:a6144 :a6144 :a6144. +:a6145 :a6145 :a6145. +:a6146 :a6146 :a6146. +:a6147 :a6147 :a6147. +:a6148 :a6148 :a6148. +:a6149 :a6149 :a6149. +:a6150 :a6150 :a6150. +:a6151 :a6151 :a6151. +:a6152 :a6152 :a6152. +:a6153 :a6153 :a6153. +:a6154 :a6154 :a6154. +:a6155 :a6155 :a6155. +:a6156 :a6156 :a6156. +:a6157 :a6157 :a6157. +:a6158 :a6158 :a6158. +:a6159 :a6159 :a6159. +:a6160 :a6160 :a6160. +:a6161 :a6161 :a6161. +:a6162 :a6162 :a6162. +:a6163 :a6163 :a6163. +:a6164 :a6164 :a6164. +:a6165 :a6165 :a6165. +:a6166 :a6166 :a6166. +:a6167 :a6167 :a6167. +:a6168 :a6168 :a6168. +:a6169 :a6169 :a6169. +:a6170 :a6170 :a6170. +:a6171 :a6171 :a6171. +:a6172 :a6172 :a6172. +:a6173 :a6173 :a6173. +:a6174 :a6174 :a6174. +:a6175 :a6175 :a6175. +:a6176 :a6176 :a6176. +:a6177 :a6177 :a6177. +:a6178 :a6178 :a6178. +:a6179 :a6179 :a6179. +:a6180 :a6180 :a6180. +:a6181 :a6181 :a6181. +:a6182 :a6182 :a6182. +:a6183 :a6183 :a6183. +:a6184 :a6184 :a6184. +:a6185 :a6185 :a6185. +:a6186 :a6186 :a6186. +:a6187 :a6187 :a6187. +:a6188 :a6188 :a6188. +:a6189 :a6189 :a6189. +:a6190 :a6190 :a6190. +:a6191 :a6191 :a6191. +:a6192 :a6192 :a6192. +:a6193 :a6193 :a6193. +:a6194 :a6194 :a6194. +:a6195 :a6195 :a6195. +:a6196 :a6196 :a6196. +:a6197 :a6197 :a6197. +:a6198 :a6198 :a6198. +:a6199 :a6199 :a6199. +:a6200 :a6200 :a6200. +:a6201 :a6201 :a6201. +:a6202 :a6202 :a6202. +:a6203 :a6203 :a6203. +:a6204 :a6204 :a6204. +:a6205 :a6205 :a6205. +:a6206 :a6206 :a6206. +:a6207 :a6207 :a6207. +:a6208 :a6208 :a6208. +:a6209 :a6209 :a6209. +:a6210 :a6210 :a6210. +:a6211 :a6211 :a6211. +:a6212 :a6212 :a6212. +:a6213 :a6213 :a6213. +:a6214 :a6214 :a6214. +:a6215 :a6215 :a6215. +:a6216 :a6216 :a6216. +:a6217 :a6217 :a6217. +:a6218 :a6218 :a6218. +:a6219 :a6219 :a6219. +:a6220 :a6220 :a6220. +:a6221 :a6221 :a6221. +:a6222 :a6222 :a6222. +:a6223 :a6223 :a6223. +:a6224 :a6224 :a6224. +:a6225 :a6225 :a6225. +:a6226 :a6226 :a6226. +:a6227 :a6227 :a6227. +:a6228 :a6228 :a6228. +:a6229 :a6229 :a6229. +:a6230 :a6230 :a6230. +:a6231 :a6231 :a6231. +:a6232 :a6232 :a6232. +:a6233 :a6233 :a6233. +:a6234 :a6234 :a6234. +:a6235 :a6235 :a6235. +:a6236 :a6236 :a6236. +:a6237 :a6237 :a6237. +:a6238 :a6238 :a6238. +:a6239 :a6239 :a6239. +:a6240 :a6240 :a6240. +:a6241 :a6241 :a6241. +:a6242 :a6242 :a6242. +:a6243 :a6243 :a6243. +:a6244 :a6244 :a6244. +:a6245 :a6245 :a6245. +:a6246 :a6246 :a6246. +:a6247 :a6247 :a6247. +:a6248 :a6248 :a6248. +:a6249 :a6249 :a6249. +:a6250 :a6250 :a6250. +:a6251 :a6251 :a6251. +:a6252 :a6252 :a6252. +:a6253 :a6253 :a6253. +:a6254 :a6254 :a6254. +:a6255 :a6255 :a6255. +:a6256 :a6256 :a6256. +:a6257 :a6257 :a6257. +:a6258 :a6258 :a6258. +:a6259 :a6259 :a6259. +:a6260 :a6260 :a6260. +:a6261 :a6261 :a6261. +:a6262 :a6262 :a6262. +:a6263 :a6263 :a6263. +:a6264 :a6264 :a6264. +:a6265 :a6265 :a6265. +:a6266 :a6266 :a6266. +:a6267 :a6267 :a6267. +:a6268 :a6268 :a6268. +:a6269 :a6269 :a6269. +:a6270 :a6270 :a6270. +:a6271 :a6271 :a6271. +:a6272 :a6272 :a6272. +:a6273 :a6273 :a6273. +:a6274 :a6274 :a6274. +:a6275 :a6275 :a6275. +:a6276 :a6276 :a6276. +:a6277 :a6277 :a6277. +:a6278 :a6278 :a6278. +:a6279 :a6279 :a6279. +:a6280 :a6280 :a6280. +:a6281 :a6281 :a6281. +:a6282 :a6282 :a6282. +:a6283 :a6283 :a6283. +:a6284 :a6284 :a6284. +:a6285 :a6285 :a6285. +:a6286 :a6286 :a6286. +:a6287 :a6287 :a6287. +:a6288 :a6288 :a6288. +:a6289 :a6289 :a6289. +:a6290 :a6290 :a6290. +:a6291 :a6291 :a6291. +:a6292 :a6292 :a6292. +:a6293 :a6293 :a6293. +:a6294 :a6294 :a6294. +:a6295 :a6295 :a6295. +:a6296 :a6296 :a6296. +:a6297 :a6297 :a6297. +:a6298 :a6298 :a6298. +:a6299 :a6299 :a6299. +:a6300 :a6300 :a6300. +:a6301 :a6301 :a6301. +:a6302 :a6302 :a6302. +:a6303 :a6303 :a6303. +:a6304 :a6304 :a6304. +:a6305 :a6305 :a6305. +:a6306 :a6306 :a6306. +:a6307 :a6307 :a6307. +:a6308 :a6308 :a6308. +:a6309 :a6309 :a6309. +:a6310 :a6310 :a6310. +:a6311 :a6311 :a6311. +:a6312 :a6312 :a6312. +:a6313 :a6313 :a6313. +:a6314 :a6314 :a6314. +:a6315 :a6315 :a6315. +:a6316 :a6316 :a6316. +:a6317 :a6317 :a6317. +:a6318 :a6318 :a6318. +:a6319 :a6319 :a6319. +:a6320 :a6320 :a6320. +:a6321 :a6321 :a6321. +:a6322 :a6322 :a6322. +:a6323 :a6323 :a6323. +:a6324 :a6324 :a6324. +:a6325 :a6325 :a6325. +:a6326 :a6326 :a6326. +:a6327 :a6327 :a6327. +:a6328 :a6328 :a6328. +:a6329 :a6329 :a6329. +:a6330 :a6330 :a6330. +:a6331 :a6331 :a6331. +:a6332 :a6332 :a6332. +:a6333 :a6333 :a6333. +:a6334 :a6334 :a6334. +:a6335 :a6335 :a6335. +:a6336 :a6336 :a6336. +:a6337 :a6337 :a6337. +:a6338 :a6338 :a6338. +:a6339 :a6339 :a6339. +:a6340 :a6340 :a6340. +:a6341 :a6341 :a6341. +:a6342 :a6342 :a6342. +:a6343 :a6343 :a6343. +:a6344 :a6344 :a6344. +:a6345 :a6345 :a6345. +:a6346 :a6346 :a6346. +:a6347 :a6347 :a6347. +:a6348 :a6348 :a6348. +:a6349 :a6349 :a6349. +:a6350 :a6350 :a6350. +:a6351 :a6351 :a6351. +:a6352 :a6352 :a6352. +:a6353 :a6353 :a6353. +:a6354 :a6354 :a6354. +:a6355 :a6355 :a6355. +:a6356 :a6356 :a6356. +:a6357 :a6357 :a6357. +:a6358 :a6358 :a6358. +:a6359 :a6359 :a6359. +:a6360 :a6360 :a6360. +:a6361 :a6361 :a6361. +:a6362 :a6362 :a6362. +:a6363 :a6363 :a6363. +:a6364 :a6364 :a6364. +:a6365 :a6365 :a6365. +:a6366 :a6366 :a6366. +:a6367 :a6367 :a6367. +:a6368 :a6368 :a6368. +:a6369 :a6369 :a6369. +:a6370 :a6370 :a6370. +:a6371 :a6371 :a6371. +:a6372 :a6372 :a6372. +:a6373 :a6373 :a6373. +:a6374 :a6374 :a6374. +:a6375 :a6375 :a6375. +:a6376 :a6376 :a6376. +:a6377 :a6377 :a6377. +:a6378 :a6378 :a6378. +:a6379 :a6379 :a6379. +:a6380 :a6380 :a6380. +:a6381 :a6381 :a6381. +:a6382 :a6382 :a6382. +:a6383 :a6383 :a6383. +:a6384 :a6384 :a6384. +:a6385 :a6385 :a6385. +:a6386 :a6386 :a6386. +:a6387 :a6387 :a6387. +:a6388 :a6388 :a6388. +:a6389 :a6389 :a6389. +:a6390 :a6390 :a6390. +:a6391 :a6391 :a6391. +:a6392 :a6392 :a6392. +:a6393 :a6393 :a6393. +:a6394 :a6394 :a6394. +:a6395 :a6395 :a6395. +:a6396 :a6396 :a6396. +:a6397 :a6397 :a6397. +:a6398 :a6398 :a6398. +:a6399 :a6399 :a6399. +:a6400 :a6400 :a6400. +:a6401 :a6401 :a6401. +:a6402 :a6402 :a6402. +:a6403 :a6403 :a6403. +:a6404 :a6404 :a6404. +:a6405 :a6405 :a6405. +:a6406 :a6406 :a6406. +:a6407 :a6407 :a6407. +:a6408 :a6408 :a6408. +:a6409 :a6409 :a6409. +:a6410 :a6410 :a6410. +:a6411 :a6411 :a6411. +:a6412 :a6412 :a6412. +:a6413 :a6413 :a6413. +:a6414 :a6414 :a6414. +:a6415 :a6415 :a6415. +:a6416 :a6416 :a6416. +:a6417 :a6417 :a6417. +:a6418 :a6418 :a6418. +:a6419 :a6419 :a6419. +:a6420 :a6420 :a6420. +:a6421 :a6421 :a6421. +:a6422 :a6422 :a6422. +:a6423 :a6423 :a6423. +:a6424 :a6424 :a6424. +:a6425 :a6425 :a6425. +:a6426 :a6426 :a6426. +:a6427 :a6427 :a6427. +:a6428 :a6428 :a6428. +:a6429 :a6429 :a6429. +:a6430 :a6430 :a6430. +:a6431 :a6431 :a6431. +:a6432 :a6432 :a6432. +:a6433 :a6433 :a6433. +:a6434 :a6434 :a6434. +:a6435 :a6435 :a6435. +:a6436 :a6436 :a6436. +:a6437 :a6437 :a6437. +:a6438 :a6438 :a6438. +:a6439 :a6439 :a6439. +:a6440 :a6440 :a6440. +:a6441 :a6441 :a6441. +:a6442 :a6442 :a6442. +:a6443 :a6443 :a6443. +:a6444 :a6444 :a6444. +:a6445 :a6445 :a6445. +:a6446 :a6446 :a6446. +:a6447 :a6447 :a6447. +:a6448 :a6448 :a6448. +:a6449 :a6449 :a6449. +:a6450 :a6450 :a6450. +:a6451 :a6451 :a6451. +:a6452 :a6452 :a6452. +:a6453 :a6453 :a6453. +:a6454 :a6454 :a6454. +:a6455 :a6455 :a6455. +:a6456 :a6456 :a6456. +:a6457 :a6457 :a6457. +:a6458 :a6458 :a6458. +:a6459 :a6459 :a6459. +:a6460 :a6460 :a6460. +:a6461 :a6461 :a6461. +:a6462 :a6462 :a6462. +:a6463 :a6463 :a6463. +:a6464 :a6464 :a6464. +:a6465 :a6465 :a6465. +:a6466 :a6466 :a6466. +:a6467 :a6467 :a6467. +:a6468 :a6468 :a6468. +:a6469 :a6469 :a6469. +:a6470 :a6470 :a6470. +:a6471 :a6471 :a6471. +:a6472 :a6472 :a6472. +:a6473 :a6473 :a6473. +:a6474 :a6474 :a6474. +:a6475 :a6475 :a6475. +:a6476 :a6476 :a6476. +:a6477 :a6477 :a6477. +:a6478 :a6478 :a6478. +:a6479 :a6479 :a6479. +:a6480 :a6480 :a6480. +:a6481 :a6481 :a6481. +:a6482 :a6482 :a6482. +:a6483 :a6483 :a6483. +:a6484 :a6484 :a6484. +:a6485 :a6485 :a6485. +:a6486 :a6486 :a6486. +:a6487 :a6487 :a6487. +:a6488 :a6488 :a6488. +:a6489 :a6489 :a6489. +:a6490 :a6490 :a6490. +:a6491 :a6491 :a6491. +:a6492 :a6492 :a6492. +:a6493 :a6493 :a6493. +:a6494 :a6494 :a6494. +:a6495 :a6495 :a6495. +:a6496 :a6496 :a6496. +:a6497 :a6497 :a6497. +:a6498 :a6498 :a6498. +:a6499 :a6499 :a6499. +:a6500 :a6500 :a6500. +:a6501 :a6501 :a6501. +:a6502 :a6502 :a6502. +:a6503 :a6503 :a6503. +:a6504 :a6504 :a6504. +:a6505 :a6505 :a6505. +:a6506 :a6506 :a6506. +:a6507 :a6507 :a6507. +:a6508 :a6508 :a6508. +:a6509 :a6509 :a6509. +:a6510 :a6510 :a6510. +:a6511 :a6511 :a6511. +:a6512 :a6512 :a6512. +:a6513 :a6513 :a6513. +:a6514 :a6514 :a6514. +:a6515 :a6515 :a6515. +:a6516 :a6516 :a6516. +:a6517 :a6517 :a6517. +:a6518 :a6518 :a6518. +:a6519 :a6519 :a6519. +:a6520 :a6520 :a6520. +:a6521 :a6521 :a6521. +:a6522 :a6522 :a6522. +:a6523 :a6523 :a6523. +:a6524 :a6524 :a6524. +:a6525 :a6525 :a6525. +:a6526 :a6526 :a6526. +:a6527 :a6527 :a6527. +:a6528 :a6528 :a6528. +:a6529 :a6529 :a6529. +:a6530 :a6530 :a6530. +:a6531 :a6531 :a6531. +:a6532 :a6532 :a6532. +:a6533 :a6533 :a6533. +:a6534 :a6534 :a6534. +:a6535 :a6535 :a6535. +:a6536 :a6536 :a6536. +:a6537 :a6537 :a6537. +:a6538 :a6538 :a6538. +:a6539 :a6539 :a6539. +:a6540 :a6540 :a6540. +:a6541 :a6541 :a6541. +:a6542 :a6542 :a6542. +:a6543 :a6543 :a6543. +:a6544 :a6544 :a6544. +:a6545 :a6545 :a6545. +:a6546 :a6546 :a6546. +:a6547 :a6547 :a6547. +:a6548 :a6548 :a6548. +:a6549 :a6549 :a6549. +:a6550 :a6550 :a6550. +:a6551 :a6551 :a6551. +:a6552 :a6552 :a6552. +:a6553 :a6553 :a6553. +:a6554 :a6554 :a6554. +:a6555 :a6555 :a6555. +:a6556 :a6556 :a6556. +:a6557 :a6557 :a6557. +:a6558 :a6558 :a6558. +:a6559 :a6559 :a6559. +:a6560 :a6560 :a6560. +:a6561 :a6561 :a6561. +:a6562 :a6562 :a6562. +:a6563 :a6563 :a6563. +:a6564 :a6564 :a6564. +:a6565 :a6565 :a6565. +:a6566 :a6566 :a6566. +:a6567 :a6567 :a6567. +:a6568 :a6568 :a6568. +:a6569 :a6569 :a6569. +:a6570 :a6570 :a6570. +:a6571 :a6571 :a6571. +:a6572 :a6572 :a6572. +:a6573 :a6573 :a6573. +:a6574 :a6574 :a6574. +:a6575 :a6575 :a6575. +:a6576 :a6576 :a6576. +:a6577 :a6577 :a6577. +:a6578 :a6578 :a6578. +:a6579 :a6579 :a6579. +:a6580 :a6580 :a6580. +:a6581 :a6581 :a6581. +:a6582 :a6582 :a6582. +:a6583 :a6583 :a6583. +:a6584 :a6584 :a6584. +:a6585 :a6585 :a6585. +:a6586 :a6586 :a6586. +:a6587 :a6587 :a6587. +:a6588 :a6588 :a6588. +:a6589 :a6589 :a6589. +:a6590 :a6590 :a6590. +:a6591 :a6591 :a6591. +:a6592 :a6592 :a6592. +:a6593 :a6593 :a6593. +:a6594 :a6594 :a6594. +:a6595 :a6595 :a6595. +:a6596 :a6596 :a6596. +:a6597 :a6597 :a6597. +:a6598 :a6598 :a6598. +:a6599 :a6599 :a6599. +:a6600 :a6600 :a6600. +:a6601 :a6601 :a6601. +:a6602 :a6602 :a6602. +:a6603 :a6603 :a6603. +:a6604 :a6604 :a6604. +:a6605 :a6605 :a6605. +:a6606 :a6606 :a6606. +:a6607 :a6607 :a6607. +:a6608 :a6608 :a6608. +:a6609 :a6609 :a6609. +:a6610 :a6610 :a6610. +:a6611 :a6611 :a6611. +:a6612 :a6612 :a6612. +:a6613 :a6613 :a6613. +:a6614 :a6614 :a6614. +:a6615 :a6615 :a6615. +:a6616 :a6616 :a6616. +:a6617 :a6617 :a6617. +:a6618 :a6618 :a6618. +:a6619 :a6619 :a6619. +:a6620 :a6620 :a6620. +:a6621 :a6621 :a6621. +:a6622 :a6622 :a6622. +:a6623 :a6623 :a6623. +:a6624 :a6624 :a6624. +:a6625 :a6625 :a6625. +:a6626 :a6626 :a6626. +:a6627 :a6627 :a6627. +:a6628 :a6628 :a6628. +:a6629 :a6629 :a6629. +:a6630 :a6630 :a6630. +:a6631 :a6631 :a6631. +:a6632 :a6632 :a6632. +:a6633 :a6633 :a6633. +:a6634 :a6634 :a6634. +:a6635 :a6635 :a6635. +:a6636 :a6636 :a6636. +:a6637 :a6637 :a6637. +:a6638 :a6638 :a6638. +:a6639 :a6639 :a6639. +:a6640 :a6640 :a6640. +:a6641 :a6641 :a6641. +:a6642 :a6642 :a6642. +:a6643 :a6643 :a6643. +:a6644 :a6644 :a6644. +:a6645 :a6645 :a6645. +:a6646 :a6646 :a6646. +:a6647 :a6647 :a6647. +:a6648 :a6648 :a6648. +:a6649 :a6649 :a6649. +:a6650 :a6650 :a6650. +:a6651 :a6651 :a6651. +:a6652 :a6652 :a6652. +:a6653 :a6653 :a6653. +:a6654 :a6654 :a6654. +:a6655 :a6655 :a6655. +:a6656 :a6656 :a6656. +:a6657 :a6657 :a6657. +:a6658 :a6658 :a6658. +:a6659 :a6659 :a6659. +:a6660 :a6660 :a6660. +:a6661 :a6661 :a6661. +:a6662 :a6662 :a6662. +:a6663 :a6663 :a6663. +:a6664 :a6664 :a6664. +:a6665 :a6665 :a6665. +:a6666 :a6666 :a6666. +:a6667 :a6667 :a6667. +:a6668 :a6668 :a6668. +:a6669 :a6669 :a6669. +:a6670 :a6670 :a6670. +:a6671 :a6671 :a6671. +:a6672 :a6672 :a6672. +:a6673 :a6673 :a6673. +:a6674 :a6674 :a6674. +:a6675 :a6675 :a6675. +:a6676 :a6676 :a6676. +:a6677 :a6677 :a6677. +:a6678 :a6678 :a6678. +:a6679 :a6679 :a6679. +:a6680 :a6680 :a6680. +:a6681 :a6681 :a6681. +:a6682 :a6682 :a6682. +:a6683 :a6683 :a6683. +:a6684 :a6684 :a6684. +:a6685 :a6685 :a6685. +:a6686 :a6686 :a6686. +:a6687 :a6687 :a6687. +:a6688 :a6688 :a6688. +:a6689 :a6689 :a6689. +:a6690 :a6690 :a6690. +:a6691 :a6691 :a6691. +:a6692 :a6692 :a6692. +:a6693 :a6693 :a6693. +:a6694 :a6694 :a6694. +:a6695 :a6695 :a6695. +:a6696 :a6696 :a6696. +:a6697 :a6697 :a6697. +:a6698 :a6698 :a6698. +:a6699 :a6699 :a6699. +:a6700 :a6700 :a6700. +:a6701 :a6701 :a6701. +:a6702 :a6702 :a6702. +:a6703 :a6703 :a6703. +:a6704 :a6704 :a6704. +:a6705 :a6705 :a6705. +:a6706 :a6706 :a6706. +:a6707 :a6707 :a6707. +:a6708 :a6708 :a6708. +:a6709 :a6709 :a6709. +:a6710 :a6710 :a6710. +:a6711 :a6711 :a6711. +:a6712 :a6712 :a6712. +:a6713 :a6713 :a6713. +:a6714 :a6714 :a6714. +:a6715 :a6715 :a6715. +:a6716 :a6716 :a6716. +:a6717 :a6717 :a6717. +:a6718 :a6718 :a6718. +:a6719 :a6719 :a6719. +:a6720 :a6720 :a6720. +:a6721 :a6721 :a6721. +:a6722 :a6722 :a6722. +:a6723 :a6723 :a6723. +:a6724 :a6724 :a6724. +:a6725 :a6725 :a6725. +:a6726 :a6726 :a6726. +:a6727 :a6727 :a6727. +:a6728 :a6728 :a6728. +:a6729 :a6729 :a6729. +:a6730 :a6730 :a6730. +:a6731 :a6731 :a6731. +:a6732 :a6732 :a6732. +:a6733 :a6733 :a6733. +:a6734 :a6734 :a6734. +:a6735 :a6735 :a6735. +:a6736 :a6736 :a6736. +:a6737 :a6737 :a6737. +:a6738 :a6738 :a6738. +:a6739 :a6739 :a6739. +:a6740 :a6740 :a6740. +:a6741 :a6741 :a6741. +:a6742 :a6742 :a6742. +:a6743 :a6743 :a6743. +:a6744 :a6744 :a6744. +:a6745 :a6745 :a6745. +:a6746 :a6746 :a6746. +:a6747 :a6747 :a6747. +:a6748 :a6748 :a6748. +:a6749 :a6749 :a6749. +:a6750 :a6750 :a6750. +:a6751 :a6751 :a6751. +:a6752 :a6752 :a6752. +:a6753 :a6753 :a6753. +:a6754 :a6754 :a6754. +:a6755 :a6755 :a6755. +:a6756 :a6756 :a6756. +:a6757 :a6757 :a6757. +:a6758 :a6758 :a6758. +:a6759 :a6759 :a6759. +:a6760 :a6760 :a6760. +:a6761 :a6761 :a6761. +:a6762 :a6762 :a6762. +:a6763 :a6763 :a6763. +:a6764 :a6764 :a6764. +:a6765 :a6765 :a6765. +:a6766 :a6766 :a6766. +:a6767 :a6767 :a6767. +:a6768 :a6768 :a6768. +:a6769 :a6769 :a6769. +:a6770 :a6770 :a6770. +:a6771 :a6771 :a6771. +:a6772 :a6772 :a6772. +:a6773 :a6773 :a6773. +:a6774 :a6774 :a6774. +:a6775 :a6775 :a6775. +:a6776 :a6776 :a6776. +:a6777 :a6777 :a6777. +:a6778 :a6778 :a6778. +:a6779 :a6779 :a6779. +:a6780 :a6780 :a6780. +:a6781 :a6781 :a6781. +:a6782 :a6782 :a6782. +:a6783 :a6783 :a6783. +:a6784 :a6784 :a6784. +:a6785 :a6785 :a6785. +:a6786 :a6786 :a6786. +:a6787 :a6787 :a6787. +:a6788 :a6788 :a6788. +:a6789 :a6789 :a6789. +:a6790 :a6790 :a6790. +:a6791 :a6791 :a6791. +:a6792 :a6792 :a6792. +:a6793 :a6793 :a6793. +:a6794 :a6794 :a6794. +:a6795 :a6795 :a6795. +:a6796 :a6796 :a6796. +:a6797 :a6797 :a6797. +:a6798 :a6798 :a6798. +:a6799 :a6799 :a6799. +:a6800 :a6800 :a6800. +:a6801 :a6801 :a6801. +:a6802 :a6802 :a6802. +:a6803 :a6803 :a6803. +:a6804 :a6804 :a6804. +:a6805 :a6805 :a6805. +:a6806 :a6806 :a6806. +:a6807 :a6807 :a6807. +:a6808 :a6808 :a6808. +:a6809 :a6809 :a6809. +:a6810 :a6810 :a6810. +:a6811 :a6811 :a6811. +:a6812 :a6812 :a6812. +:a6813 :a6813 :a6813. +:a6814 :a6814 :a6814. +:a6815 :a6815 :a6815. +:a6816 :a6816 :a6816. +:a6817 :a6817 :a6817. +:a6818 :a6818 :a6818. +:a6819 :a6819 :a6819. +:a6820 :a6820 :a6820. +:a6821 :a6821 :a6821. +:a6822 :a6822 :a6822. +:a6823 :a6823 :a6823. +:a6824 :a6824 :a6824. +:a6825 :a6825 :a6825. +:a6826 :a6826 :a6826. +:a6827 :a6827 :a6827. +:a6828 :a6828 :a6828. +:a6829 :a6829 :a6829. +:a6830 :a6830 :a6830. +:a6831 :a6831 :a6831. +:a6832 :a6832 :a6832. +:a6833 :a6833 :a6833. +:a6834 :a6834 :a6834. +:a6835 :a6835 :a6835. +:a6836 :a6836 :a6836. +:a6837 :a6837 :a6837. +:a6838 :a6838 :a6838. +:a6839 :a6839 :a6839. +:a6840 :a6840 :a6840. +:a6841 :a6841 :a6841. +:a6842 :a6842 :a6842. +:a6843 :a6843 :a6843. +:a6844 :a6844 :a6844. +:a6845 :a6845 :a6845. +:a6846 :a6846 :a6846. +:a6847 :a6847 :a6847. +:a6848 :a6848 :a6848. +:a6849 :a6849 :a6849. +:a6850 :a6850 :a6850. +:a6851 :a6851 :a6851. +:a6852 :a6852 :a6852. +:a6853 :a6853 :a6853. +:a6854 :a6854 :a6854. +:a6855 :a6855 :a6855. +:a6856 :a6856 :a6856. +:a6857 :a6857 :a6857. +:a6858 :a6858 :a6858. +:a6859 :a6859 :a6859. +:a6860 :a6860 :a6860. +:a6861 :a6861 :a6861. +:a6862 :a6862 :a6862. +:a6863 :a6863 :a6863. +:a6864 :a6864 :a6864. +:a6865 :a6865 :a6865. +:a6866 :a6866 :a6866. +:a6867 :a6867 :a6867. +:a6868 :a6868 :a6868. +:a6869 :a6869 :a6869. +:a6870 :a6870 :a6870. +:a6871 :a6871 :a6871. +:a6872 :a6872 :a6872. +:a6873 :a6873 :a6873. +:a6874 :a6874 :a6874. +:a6875 :a6875 :a6875. +:a6876 :a6876 :a6876. +:a6877 :a6877 :a6877. +:a6878 :a6878 :a6878. +:a6879 :a6879 :a6879. +:a6880 :a6880 :a6880. +:a6881 :a6881 :a6881. +:a6882 :a6882 :a6882. +:a6883 :a6883 :a6883. +:a6884 :a6884 :a6884. +:a6885 :a6885 :a6885. +:a6886 :a6886 :a6886. +:a6887 :a6887 :a6887. +:a6888 :a6888 :a6888. +:a6889 :a6889 :a6889. +:a6890 :a6890 :a6890. +:a6891 :a6891 :a6891. +:a6892 :a6892 :a6892. +:a6893 :a6893 :a6893. +:a6894 :a6894 :a6894. +:a6895 :a6895 :a6895. +:a6896 :a6896 :a6896. +:a6897 :a6897 :a6897. +:a6898 :a6898 :a6898. +:a6899 :a6899 :a6899. +:a6900 :a6900 :a6900. +:a6901 :a6901 :a6901. +:a6902 :a6902 :a6902. +:a6903 :a6903 :a6903. +:a6904 :a6904 :a6904. +:a6905 :a6905 :a6905. +:a6906 :a6906 :a6906. +:a6907 :a6907 :a6907. +:a6908 :a6908 :a6908. +:a6909 :a6909 :a6909. +:a6910 :a6910 :a6910. +:a6911 :a6911 :a6911. +:a6912 :a6912 :a6912. +:a6913 :a6913 :a6913. +:a6914 :a6914 :a6914. +:a6915 :a6915 :a6915. +:a6916 :a6916 :a6916. +:a6917 :a6917 :a6917. +:a6918 :a6918 :a6918. +:a6919 :a6919 :a6919. +:a6920 :a6920 :a6920. +:a6921 :a6921 :a6921. +:a6922 :a6922 :a6922. +:a6923 :a6923 :a6923. +:a6924 :a6924 :a6924. +:a6925 :a6925 :a6925. +:a6926 :a6926 :a6926. +:a6927 :a6927 :a6927. +:a6928 :a6928 :a6928. +:a6929 :a6929 :a6929. +:a6930 :a6930 :a6930. +:a6931 :a6931 :a6931. +:a6932 :a6932 :a6932. +:a6933 :a6933 :a6933. +:a6934 :a6934 :a6934. +:a6935 :a6935 :a6935. +:a6936 :a6936 :a6936. +:a6937 :a6937 :a6937. +:a6938 :a6938 :a6938. +:a6939 :a6939 :a6939. +:a6940 :a6940 :a6940. +:a6941 :a6941 :a6941. +:a6942 :a6942 :a6942. +:a6943 :a6943 :a6943. +:a6944 :a6944 :a6944. +:a6945 :a6945 :a6945. +:a6946 :a6946 :a6946. +:a6947 :a6947 :a6947. +:a6948 :a6948 :a6948. +:a6949 :a6949 :a6949. +:a6950 :a6950 :a6950. +:a6951 :a6951 :a6951. +:a6952 :a6952 :a6952. +:a6953 :a6953 :a6953. +:a6954 :a6954 :a6954. +:a6955 :a6955 :a6955. +:a6956 :a6956 :a6956. +:a6957 :a6957 :a6957. +:a6958 :a6958 :a6958. +:a6959 :a6959 :a6959. +:a6960 :a6960 :a6960. +:a6961 :a6961 :a6961. +:a6962 :a6962 :a6962. +:a6963 :a6963 :a6963. +:a6964 :a6964 :a6964. +:a6965 :a6965 :a6965. +:a6966 :a6966 :a6966. +:a6967 :a6967 :a6967. +:a6968 :a6968 :a6968. +:a6969 :a6969 :a6969. +:a6970 :a6970 :a6970. +:a6971 :a6971 :a6971. +:a6972 :a6972 :a6972. +:a6973 :a6973 :a6973. +:a6974 :a6974 :a6974. +:a6975 :a6975 :a6975. +:a6976 :a6976 :a6976. +:a6977 :a6977 :a6977. +:a6978 :a6978 :a6978. +:a6979 :a6979 :a6979. +:a6980 :a6980 :a6980. +:a6981 :a6981 :a6981. +:a6982 :a6982 :a6982. +:a6983 :a6983 :a6983. +:a6984 :a6984 :a6984. +:a6985 :a6985 :a6985. +:a6986 :a6986 :a6986. +:a6987 :a6987 :a6987. +:a6988 :a6988 :a6988. +:a6989 :a6989 :a6989. +:a6990 :a6990 :a6990. +:a6991 :a6991 :a6991. +:a6992 :a6992 :a6992. +:a6993 :a6993 :a6993. +:a6994 :a6994 :a6994. +:a6995 :a6995 :a6995. +:a6996 :a6996 :a6996. +:a6997 :a6997 :a6997. +:a6998 :a6998 :a6998. +:a6999 :a6999 :a6999. +:a7000 :a7000 :a7000. +:a7001 :a7001 :a7001. +:a7002 :a7002 :a7002. +:a7003 :a7003 :a7003. +:a7004 :a7004 :a7004. +:a7005 :a7005 :a7005. +:a7006 :a7006 :a7006. +:a7007 :a7007 :a7007. +:a7008 :a7008 :a7008. +:a7009 :a7009 :a7009. +:a7010 :a7010 :a7010. +:a7011 :a7011 :a7011. +:a7012 :a7012 :a7012. +:a7013 :a7013 :a7013. +:a7014 :a7014 :a7014. +:a7015 :a7015 :a7015. +:a7016 :a7016 :a7016. +:a7017 :a7017 :a7017. +:a7018 :a7018 :a7018. +:a7019 :a7019 :a7019. +:a7020 :a7020 :a7020. +:a7021 :a7021 :a7021. +:a7022 :a7022 :a7022. +:a7023 :a7023 :a7023. +:a7024 :a7024 :a7024. +:a7025 :a7025 :a7025. +:a7026 :a7026 :a7026. +:a7027 :a7027 :a7027. +:a7028 :a7028 :a7028. +:a7029 :a7029 :a7029. +:a7030 :a7030 :a7030. +:a7031 :a7031 :a7031. +:a7032 :a7032 :a7032. +:a7033 :a7033 :a7033. +:a7034 :a7034 :a7034. +:a7035 :a7035 :a7035. +:a7036 :a7036 :a7036. +:a7037 :a7037 :a7037. +:a7038 :a7038 :a7038. +:a7039 :a7039 :a7039. +:a7040 :a7040 :a7040. +:a7041 :a7041 :a7041. +:a7042 :a7042 :a7042. +:a7043 :a7043 :a7043. +:a7044 :a7044 :a7044. +:a7045 :a7045 :a7045. +:a7046 :a7046 :a7046. +:a7047 :a7047 :a7047. +:a7048 :a7048 :a7048. +:a7049 :a7049 :a7049. +:a7050 :a7050 :a7050. +:a7051 :a7051 :a7051. +:a7052 :a7052 :a7052. +:a7053 :a7053 :a7053. +:a7054 :a7054 :a7054. +:a7055 :a7055 :a7055. +:a7056 :a7056 :a7056. +:a7057 :a7057 :a7057. +:a7058 :a7058 :a7058. +:a7059 :a7059 :a7059. +:a7060 :a7060 :a7060. +:a7061 :a7061 :a7061. +:a7062 :a7062 :a7062. +:a7063 :a7063 :a7063. +:a7064 :a7064 :a7064. +:a7065 :a7065 :a7065. +:a7066 :a7066 :a7066. +:a7067 :a7067 :a7067. +:a7068 :a7068 :a7068. +:a7069 :a7069 :a7069. +:a7070 :a7070 :a7070. +:a7071 :a7071 :a7071. +:a7072 :a7072 :a7072. +:a7073 :a7073 :a7073. +:a7074 :a7074 :a7074. +:a7075 :a7075 :a7075. +:a7076 :a7076 :a7076. +:a7077 :a7077 :a7077. +:a7078 :a7078 :a7078. +:a7079 :a7079 :a7079. +:a7080 :a7080 :a7080. +:a7081 :a7081 :a7081. +:a7082 :a7082 :a7082. +:a7083 :a7083 :a7083. +:a7084 :a7084 :a7084. +:a7085 :a7085 :a7085. +:a7086 :a7086 :a7086. +:a7087 :a7087 :a7087. +:a7088 :a7088 :a7088. +:a7089 :a7089 :a7089. +:a7090 :a7090 :a7090. +:a7091 :a7091 :a7091. +:a7092 :a7092 :a7092. +:a7093 :a7093 :a7093. +:a7094 :a7094 :a7094. +:a7095 :a7095 :a7095. +:a7096 :a7096 :a7096. +:a7097 :a7097 :a7097. +:a7098 :a7098 :a7098. +:a7099 :a7099 :a7099. +:a7100 :a7100 :a7100. +:a7101 :a7101 :a7101. +:a7102 :a7102 :a7102. +:a7103 :a7103 :a7103. +:a7104 :a7104 :a7104. +:a7105 :a7105 :a7105. +:a7106 :a7106 :a7106. +:a7107 :a7107 :a7107. +:a7108 :a7108 :a7108. +:a7109 :a7109 :a7109. +:a7110 :a7110 :a7110. +:a7111 :a7111 :a7111. +:a7112 :a7112 :a7112. +:a7113 :a7113 :a7113. +:a7114 :a7114 :a7114. +:a7115 :a7115 :a7115. +:a7116 :a7116 :a7116. +:a7117 :a7117 :a7117. +:a7118 :a7118 :a7118. +:a7119 :a7119 :a7119. +:a7120 :a7120 :a7120. +:a7121 :a7121 :a7121. +:a7122 :a7122 :a7122. +:a7123 :a7123 :a7123. +:a7124 :a7124 :a7124. +:a7125 :a7125 :a7125. +:a7126 :a7126 :a7126. +:a7127 :a7127 :a7127. +:a7128 :a7128 :a7128. +:a7129 :a7129 :a7129. +:a7130 :a7130 :a7130. +:a7131 :a7131 :a7131. +:a7132 :a7132 :a7132. +:a7133 :a7133 :a7133. +:a7134 :a7134 :a7134. +:a7135 :a7135 :a7135. +:a7136 :a7136 :a7136. +:a7137 :a7137 :a7137. +:a7138 :a7138 :a7138. +:a7139 :a7139 :a7139. +:a7140 :a7140 :a7140. +:a7141 :a7141 :a7141. +:a7142 :a7142 :a7142. +:a7143 :a7143 :a7143. +:a7144 :a7144 :a7144. +:a7145 :a7145 :a7145. +:a7146 :a7146 :a7146. +:a7147 :a7147 :a7147. +:a7148 :a7148 :a7148. +:a7149 :a7149 :a7149. +:a7150 :a7150 :a7150. +:a7151 :a7151 :a7151. +:a7152 :a7152 :a7152. +:a7153 :a7153 :a7153. +:a7154 :a7154 :a7154. +:a7155 :a7155 :a7155. +:a7156 :a7156 :a7156. +:a7157 :a7157 :a7157. +:a7158 :a7158 :a7158. +:a7159 :a7159 :a7159. +:a7160 :a7160 :a7160. +:a7161 :a7161 :a7161. +:a7162 :a7162 :a7162. +:a7163 :a7163 :a7163. +:a7164 :a7164 :a7164. +:a7165 :a7165 :a7165. +:a7166 :a7166 :a7166. +:a7167 :a7167 :a7167. +:a7168 :a7168 :a7168. +:a7169 :a7169 :a7169. +:a7170 :a7170 :a7170. +:a7171 :a7171 :a7171. +:a7172 :a7172 :a7172. +:a7173 :a7173 :a7173. +:a7174 :a7174 :a7174. +:a7175 :a7175 :a7175. +:a7176 :a7176 :a7176. +:a7177 :a7177 :a7177. +:a7178 :a7178 :a7178. +:a7179 :a7179 :a7179. +:a7180 :a7180 :a7180. +:a7181 :a7181 :a7181. +:a7182 :a7182 :a7182. +:a7183 :a7183 :a7183. +:a7184 :a7184 :a7184. +:a7185 :a7185 :a7185. +:a7186 :a7186 :a7186. +:a7187 :a7187 :a7187. +:a7188 :a7188 :a7188. +:a7189 :a7189 :a7189. +:a7190 :a7190 :a7190. +:a7191 :a7191 :a7191. +:a7192 :a7192 :a7192. +:a7193 :a7193 :a7193. +:a7194 :a7194 :a7194. +:a7195 :a7195 :a7195. +:a7196 :a7196 :a7196. +:a7197 :a7197 :a7197. +:a7198 :a7198 :a7198. +:a7199 :a7199 :a7199. +:a7200 :a7200 :a7200. +:a7201 :a7201 :a7201. +:a7202 :a7202 :a7202. +:a7203 :a7203 :a7203. +:a7204 :a7204 :a7204. +:a7205 :a7205 :a7205. +:a7206 :a7206 :a7206. +:a7207 :a7207 :a7207. +:a7208 :a7208 :a7208. +:a7209 :a7209 :a7209. +:a7210 :a7210 :a7210. +:a7211 :a7211 :a7211. +:a7212 :a7212 :a7212. +:a7213 :a7213 :a7213. +:a7214 :a7214 :a7214. +:a7215 :a7215 :a7215. +:a7216 :a7216 :a7216. +:a7217 :a7217 :a7217. +:a7218 :a7218 :a7218. +:a7219 :a7219 :a7219. +:a7220 :a7220 :a7220. +:a7221 :a7221 :a7221. +:a7222 :a7222 :a7222. +:a7223 :a7223 :a7223. +:a7224 :a7224 :a7224. +:a7225 :a7225 :a7225. +:a7226 :a7226 :a7226. +:a7227 :a7227 :a7227. +:a7228 :a7228 :a7228. +:a7229 :a7229 :a7229. +:a7230 :a7230 :a7230. +:a7231 :a7231 :a7231. +:a7232 :a7232 :a7232. +:a7233 :a7233 :a7233. +:a7234 :a7234 :a7234. +:a7235 :a7235 :a7235. +:a7236 :a7236 :a7236. +:a7237 :a7237 :a7237. +:a7238 :a7238 :a7238. +:a7239 :a7239 :a7239. +:a7240 :a7240 :a7240. +:a7241 :a7241 :a7241. +:a7242 :a7242 :a7242. +:a7243 :a7243 :a7243. +:a7244 :a7244 :a7244. +:a7245 :a7245 :a7245. +:a7246 :a7246 :a7246. +:a7247 :a7247 :a7247. +:a7248 :a7248 :a7248. +:a7249 :a7249 :a7249. +:a7250 :a7250 :a7250. +:a7251 :a7251 :a7251. +:a7252 :a7252 :a7252. +:a7253 :a7253 :a7253. +:a7254 :a7254 :a7254. +:a7255 :a7255 :a7255. +:a7256 :a7256 :a7256. +:a7257 :a7257 :a7257. +:a7258 :a7258 :a7258. +:a7259 :a7259 :a7259. +:a7260 :a7260 :a7260. +:a7261 :a7261 :a7261. +:a7262 :a7262 :a7262. +:a7263 :a7263 :a7263. +:a7264 :a7264 :a7264. +:a7265 :a7265 :a7265. +:a7266 :a7266 :a7266. +:a7267 :a7267 :a7267. +:a7268 :a7268 :a7268. +:a7269 :a7269 :a7269. +:a7270 :a7270 :a7270. +:a7271 :a7271 :a7271. +:a7272 :a7272 :a7272. +:a7273 :a7273 :a7273. +:a7274 :a7274 :a7274. +:a7275 :a7275 :a7275. +:a7276 :a7276 :a7276. +:a7277 :a7277 :a7277. +:a7278 :a7278 :a7278. +:a7279 :a7279 :a7279. +:a7280 :a7280 :a7280. +:a7281 :a7281 :a7281. +:a7282 :a7282 :a7282. +:a7283 :a7283 :a7283. +:a7284 :a7284 :a7284. +:a7285 :a7285 :a7285. +:a7286 :a7286 :a7286. +:a7287 :a7287 :a7287. +:a7288 :a7288 :a7288. +:a7289 :a7289 :a7289. +:a7290 :a7290 :a7290. +:a7291 :a7291 :a7291. +:a7292 :a7292 :a7292. +:a7293 :a7293 :a7293. +:a7294 :a7294 :a7294. +:a7295 :a7295 :a7295. +:a7296 :a7296 :a7296. +:a7297 :a7297 :a7297. +:a7298 :a7298 :a7298. +:a7299 :a7299 :a7299. +:a7300 :a7300 :a7300. +:a7301 :a7301 :a7301. +:a7302 :a7302 :a7302. +:a7303 :a7303 :a7303. +:a7304 :a7304 :a7304. +:a7305 :a7305 :a7305. +:a7306 :a7306 :a7306. +:a7307 :a7307 :a7307. +:a7308 :a7308 :a7308. +:a7309 :a7309 :a7309. +:a7310 :a7310 :a7310. +:a7311 :a7311 :a7311. +:a7312 :a7312 :a7312. +:a7313 :a7313 :a7313. +:a7314 :a7314 :a7314. +:a7315 :a7315 :a7315. +:a7316 :a7316 :a7316. +:a7317 :a7317 :a7317. +:a7318 :a7318 :a7318. +:a7319 :a7319 :a7319. +:a7320 :a7320 :a7320. +:a7321 :a7321 :a7321. +:a7322 :a7322 :a7322. +:a7323 :a7323 :a7323. +:a7324 :a7324 :a7324. +:a7325 :a7325 :a7325. +:a7326 :a7326 :a7326. +:a7327 :a7327 :a7327. +:a7328 :a7328 :a7328. +:a7329 :a7329 :a7329. +:a7330 :a7330 :a7330. +:a7331 :a7331 :a7331. +:a7332 :a7332 :a7332. +:a7333 :a7333 :a7333. +:a7334 :a7334 :a7334. +:a7335 :a7335 :a7335. +:a7336 :a7336 :a7336. +:a7337 :a7337 :a7337. +:a7338 :a7338 :a7338. +:a7339 :a7339 :a7339. +:a7340 :a7340 :a7340. +:a7341 :a7341 :a7341. +:a7342 :a7342 :a7342. +:a7343 :a7343 :a7343. +:a7344 :a7344 :a7344. +:a7345 :a7345 :a7345. +:a7346 :a7346 :a7346. +:a7347 :a7347 :a7347. +:a7348 :a7348 :a7348. +:a7349 :a7349 :a7349. +:a7350 :a7350 :a7350. +:a7351 :a7351 :a7351. +:a7352 :a7352 :a7352. +:a7353 :a7353 :a7353. +:a7354 :a7354 :a7354. +:a7355 :a7355 :a7355. +:a7356 :a7356 :a7356. +:a7357 :a7357 :a7357. +:a7358 :a7358 :a7358. +:a7359 :a7359 :a7359. +:a7360 :a7360 :a7360. +:a7361 :a7361 :a7361. +:a7362 :a7362 :a7362. +:a7363 :a7363 :a7363. +:a7364 :a7364 :a7364. +:a7365 :a7365 :a7365. +:a7366 :a7366 :a7366. +:a7367 :a7367 :a7367. +:a7368 :a7368 :a7368. +:a7369 :a7369 :a7369. +:a7370 :a7370 :a7370. +:a7371 :a7371 :a7371. +:a7372 :a7372 :a7372. +:a7373 :a7373 :a7373. +:a7374 :a7374 :a7374. +:a7375 :a7375 :a7375. +:a7376 :a7376 :a7376. +:a7377 :a7377 :a7377. +:a7378 :a7378 :a7378. +:a7379 :a7379 :a7379. +:a7380 :a7380 :a7380. +:a7381 :a7381 :a7381. +:a7382 :a7382 :a7382. +:a7383 :a7383 :a7383. +:a7384 :a7384 :a7384. +:a7385 :a7385 :a7385. +:a7386 :a7386 :a7386. +:a7387 :a7387 :a7387. +:a7388 :a7388 :a7388. +:a7389 :a7389 :a7389. +:a7390 :a7390 :a7390. +:a7391 :a7391 :a7391. +:a7392 :a7392 :a7392. +:a7393 :a7393 :a7393. +:a7394 :a7394 :a7394. +:a7395 :a7395 :a7395. +:a7396 :a7396 :a7396. +:a7397 :a7397 :a7397. +:a7398 :a7398 :a7398. +:a7399 :a7399 :a7399. +:a7400 :a7400 :a7400. +:a7401 :a7401 :a7401. +:a7402 :a7402 :a7402. +:a7403 :a7403 :a7403. +:a7404 :a7404 :a7404. +:a7405 :a7405 :a7405. +:a7406 :a7406 :a7406. +:a7407 :a7407 :a7407. +:a7408 :a7408 :a7408. +:a7409 :a7409 :a7409. +:a7410 :a7410 :a7410. +:a7411 :a7411 :a7411. +:a7412 :a7412 :a7412. +:a7413 :a7413 :a7413. +:a7414 :a7414 :a7414. +:a7415 :a7415 :a7415. +:a7416 :a7416 :a7416. +:a7417 :a7417 :a7417. +:a7418 :a7418 :a7418. +:a7419 :a7419 :a7419. +:a7420 :a7420 :a7420. +:a7421 :a7421 :a7421. +:a7422 :a7422 :a7422. +:a7423 :a7423 :a7423. +:a7424 :a7424 :a7424. +:a7425 :a7425 :a7425. +:a7426 :a7426 :a7426. +:a7427 :a7427 :a7427. +:a7428 :a7428 :a7428. +:a7429 :a7429 :a7429. +:a7430 :a7430 :a7430. +:a7431 :a7431 :a7431. +:a7432 :a7432 :a7432. +:a7433 :a7433 :a7433. +:a7434 :a7434 :a7434. +:a7435 :a7435 :a7435. +:a7436 :a7436 :a7436. +:a7437 :a7437 :a7437. +:a7438 :a7438 :a7438. +:a7439 :a7439 :a7439. +:a7440 :a7440 :a7440. +:a7441 :a7441 :a7441. +:a7442 :a7442 :a7442. +:a7443 :a7443 :a7443. +:a7444 :a7444 :a7444. +:a7445 :a7445 :a7445. +:a7446 :a7446 :a7446. +:a7447 :a7447 :a7447. +:a7448 :a7448 :a7448. +:a7449 :a7449 :a7449. +:a7450 :a7450 :a7450. +:a7451 :a7451 :a7451. +:a7452 :a7452 :a7452. +:a7453 :a7453 :a7453. +:a7454 :a7454 :a7454. +:a7455 :a7455 :a7455. +:a7456 :a7456 :a7456. +:a7457 :a7457 :a7457. +:a7458 :a7458 :a7458. +:a7459 :a7459 :a7459. +:a7460 :a7460 :a7460. +:a7461 :a7461 :a7461. +:a7462 :a7462 :a7462. +:a7463 :a7463 :a7463. +:a7464 :a7464 :a7464. +:a7465 :a7465 :a7465. +:a7466 :a7466 :a7466. +:a7467 :a7467 :a7467. +:a7468 :a7468 :a7468. +:a7469 :a7469 :a7469. +:a7470 :a7470 :a7470. +:a7471 :a7471 :a7471. +:a7472 :a7472 :a7472. +:a7473 :a7473 :a7473. +:a7474 :a7474 :a7474. +:a7475 :a7475 :a7475. +:a7476 :a7476 :a7476. +:a7477 :a7477 :a7477. +:a7478 :a7478 :a7478. +:a7479 :a7479 :a7479. +:a7480 :a7480 :a7480. +:a7481 :a7481 :a7481. +:a7482 :a7482 :a7482. +:a7483 :a7483 :a7483. +:a7484 :a7484 :a7484. +:a7485 :a7485 :a7485. +:a7486 :a7486 :a7486. +:a7487 :a7487 :a7487. +:a7488 :a7488 :a7488. +:a7489 :a7489 :a7489. +:a7490 :a7490 :a7490. +:a7491 :a7491 :a7491. +:a7492 :a7492 :a7492. +:a7493 :a7493 :a7493. +:a7494 :a7494 :a7494. +:a7495 :a7495 :a7495. +:a7496 :a7496 :a7496. +:a7497 :a7497 :a7497. +:a7498 :a7498 :a7498. +:a7499 :a7499 :a7499. +:a7500 :a7500 :a7500. +:a7501 :a7501 :a7501. +:a7502 :a7502 :a7502. +:a7503 :a7503 :a7503. +:a7504 :a7504 :a7504. +:a7505 :a7505 :a7505. +:a7506 :a7506 :a7506. +:a7507 :a7507 :a7507. +:a7508 :a7508 :a7508. +:a7509 :a7509 :a7509. +:a7510 :a7510 :a7510. +:a7511 :a7511 :a7511. +:a7512 :a7512 :a7512. +:a7513 :a7513 :a7513. +:a7514 :a7514 :a7514. +:a7515 :a7515 :a7515. +:a7516 :a7516 :a7516. +:a7517 :a7517 :a7517. +:a7518 :a7518 :a7518. +:a7519 :a7519 :a7519. +:a7520 :a7520 :a7520. +:a7521 :a7521 :a7521. +:a7522 :a7522 :a7522. +:a7523 :a7523 :a7523. +:a7524 :a7524 :a7524. +:a7525 :a7525 :a7525. +:a7526 :a7526 :a7526. +:a7527 :a7527 :a7527. +:a7528 :a7528 :a7528. +:a7529 :a7529 :a7529. +:a7530 :a7530 :a7530. +:a7531 :a7531 :a7531. +:a7532 :a7532 :a7532. +:a7533 :a7533 :a7533. +:a7534 :a7534 :a7534. +:a7535 :a7535 :a7535. +:a7536 :a7536 :a7536. +:a7537 :a7537 :a7537. +:a7538 :a7538 :a7538. +:a7539 :a7539 :a7539. +:a7540 :a7540 :a7540. +:a7541 :a7541 :a7541. +:a7542 :a7542 :a7542. +:a7543 :a7543 :a7543. +:a7544 :a7544 :a7544. +:a7545 :a7545 :a7545. +:a7546 :a7546 :a7546. +:a7547 :a7547 :a7547. +:a7548 :a7548 :a7548. +:a7549 :a7549 :a7549. +:a7550 :a7550 :a7550. +:a7551 :a7551 :a7551. +:a7552 :a7552 :a7552. +:a7553 :a7553 :a7553. +:a7554 :a7554 :a7554. +:a7555 :a7555 :a7555. +:a7556 :a7556 :a7556. +:a7557 :a7557 :a7557. +:a7558 :a7558 :a7558. +:a7559 :a7559 :a7559. +:a7560 :a7560 :a7560. +:a7561 :a7561 :a7561. +:a7562 :a7562 :a7562. +:a7563 :a7563 :a7563. +:a7564 :a7564 :a7564. +:a7565 :a7565 :a7565. +:a7566 :a7566 :a7566. +:a7567 :a7567 :a7567. +:a7568 :a7568 :a7568. +:a7569 :a7569 :a7569. +:a7570 :a7570 :a7570. +:a7571 :a7571 :a7571. +:a7572 :a7572 :a7572. +:a7573 :a7573 :a7573. +:a7574 :a7574 :a7574. +:a7575 :a7575 :a7575. +:a7576 :a7576 :a7576. +:a7577 :a7577 :a7577. +:a7578 :a7578 :a7578. +:a7579 :a7579 :a7579. +:a7580 :a7580 :a7580. +:a7581 :a7581 :a7581. +:a7582 :a7582 :a7582. +:a7583 :a7583 :a7583. +:a7584 :a7584 :a7584. +:a7585 :a7585 :a7585. +:a7586 :a7586 :a7586. +:a7587 :a7587 :a7587. +:a7588 :a7588 :a7588. +:a7589 :a7589 :a7589. +:a7590 :a7590 :a7590. +:a7591 :a7591 :a7591. +:a7592 :a7592 :a7592. +:a7593 :a7593 :a7593. +:a7594 :a7594 :a7594. +:a7595 :a7595 :a7595. +:a7596 :a7596 :a7596. +:a7597 :a7597 :a7597. +:a7598 :a7598 :a7598. +:a7599 :a7599 :a7599. +:a7600 :a7600 :a7600. +:a7601 :a7601 :a7601. +:a7602 :a7602 :a7602. +:a7603 :a7603 :a7603. +:a7604 :a7604 :a7604. +:a7605 :a7605 :a7605. +:a7606 :a7606 :a7606. +:a7607 :a7607 :a7607. +:a7608 :a7608 :a7608. +:a7609 :a7609 :a7609. +:a7610 :a7610 :a7610. +:a7611 :a7611 :a7611. +:a7612 :a7612 :a7612. +:a7613 :a7613 :a7613. +:a7614 :a7614 :a7614. +:a7615 :a7615 :a7615. +:a7616 :a7616 :a7616. +:a7617 :a7617 :a7617. +:a7618 :a7618 :a7618. +:a7619 :a7619 :a7619. +:a7620 :a7620 :a7620. +:a7621 :a7621 :a7621. +:a7622 :a7622 :a7622. +:a7623 :a7623 :a7623. +:a7624 :a7624 :a7624. +:a7625 :a7625 :a7625. +:a7626 :a7626 :a7626. +:a7627 :a7627 :a7627. +:a7628 :a7628 :a7628. +:a7629 :a7629 :a7629. +:a7630 :a7630 :a7630. +:a7631 :a7631 :a7631. +:a7632 :a7632 :a7632. +:a7633 :a7633 :a7633. +:a7634 :a7634 :a7634. +:a7635 :a7635 :a7635. +:a7636 :a7636 :a7636. +:a7637 :a7637 :a7637. +:a7638 :a7638 :a7638. +:a7639 :a7639 :a7639. +:a7640 :a7640 :a7640. +:a7641 :a7641 :a7641. +:a7642 :a7642 :a7642. +:a7643 :a7643 :a7643. +:a7644 :a7644 :a7644. +:a7645 :a7645 :a7645. +:a7646 :a7646 :a7646. +:a7647 :a7647 :a7647. +:a7648 :a7648 :a7648. +:a7649 :a7649 :a7649. +:a7650 :a7650 :a7650. +:a7651 :a7651 :a7651. +:a7652 :a7652 :a7652. +:a7653 :a7653 :a7653. +:a7654 :a7654 :a7654. +:a7655 :a7655 :a7655. +:a7656 :a7656 :a7656. +:a7657 :a7657 :a7657. +:a7658 :a7658 :a7658. +:a7659 :a7659 :a7659. +:a7660 :a7660 :a7660. +:a7661 :a7661 :a7661. +:a7662 :a7662 :a7662. +:a7663 :a7663 :a7663. +:a7664 :a7664 :a7664. +:a7665 :a7665 :a7665. +:a7666 :a7666 :a7666. +:a7667 :a7667 :a7667. +:a7668 :a7668 :a7668. +:a7669 :a7669 :a7669. +:a7670 :a7670 :a7670. +:a7671 :a7671 :a7671. +:a7672 :a7672 :a7672. +:a7673 :a7673 :a7673. +:a7674 :a7674 :a7674. +:a7675 :a7675 :a7675. +:a7676 :a7676 :a7676. +:a7677 :a7677 :a7677. +:a7678 :a7678 :a7678. +:a7679 :a7679 :a7679. +:a7680 :a7680 :a7680. +:a7681 :a7681 :a7681. +:a7682 :a7682 :a7682. +:a7683 :a7683 :a7683. +:a7684 :a7684 :a7684. +:a7685 :a7685 :a7685. +:a7686 :a7686 :a7686. +:a7687 :a7687 :a7687. +:a7688 :a7688 :a7688. +:a7689 :a7689 :a7689. +:a7690 :a7690 :a7690. +:a7691 :a7691 :a7691. +:a7692 :a7692 :a7692. +:a7693 :a7693 :a7693. +:a7694 :a7694 :a7694. +:a7695 :a7695 :a7695. +:a7696 :a7696 :a7696. +:a7697 :a7697 :a7697. +:a7698 :a7698 :a7698. +:a7699 :a7699 :a7699. +:a7700 :a7700 :a7700. +:a7701 :a7701 :a7701. +:a7702 :a7702 :a7702. +:a7703 :a7703 :a7703. +:a7704 :a7704 :a7704. +:a7705 :a7705 :a7705. +:a7706 :a7706 :a7706. +:a7707 :a7707 :a7707. +:a7708 :a7708 :a7708. +:a7709 :a7709 :a7709. +:a7710 :a7710 :a7710. +:a7711 :a7711 :a7711. +:a7712 :a7712 :a7712. +:a7713 :a7713 :a7713. +:a7714 :a7714 :a7714. +:a7715 :a7715 :a7715. +:a7716 :a7716 :a7716. +:a7717 :a7717 :a7717. +:a7718 :a7718 :a7718. +:a7719 :a7719 :a7719. +:a7720 :a7720 :a7720. +:a7721 :a7721 :a7721. +:a7722 :a7722 :a7722. +:a7723 :a7723 :a7723. +:a7724 :a7724 :a7724. +:a7725 :a7725 :a7725. +:a7726 :a7726 :a7726. +:a7727 :a7727 :a7727. +:a7728 :a7728 :a7728. +:a7729 :a7729 :a7729. +:a7730 :a7730 :a7730. +:a7731 :a7731 :a7731. +:a7732 :a7732 :a7732. +:a7733 :a7733 :a7733. +:a7734 :a7734 :a7734. +:a7735 :a7735 :a7735. +:a7736 :a7736 :a7736. +:a7737 :a7737 :a7737. +:a7738 :a7738 :a7738. +:a7739 :a7739 :a7739. +:a7740 :a7740 :a7740. +:a7741 :a7741 :a7741. +:a7742 :a7742 :a7742. +:a7743 :a7743 :a7743. +:a7744 :a7744 :a7744. +:a7745 :a7745 :a7745. +:a7746 :a7746 :a7746. +:a7747 :a7747 :a7747. +:a7748 :a7748 :a7748. +:a7749 :a7749 :a7749. +:a7750 :a7750 :a7750. +:a7751 :a7751 :a7751. +:a7752 :a7752 :a7752. +:a7753 :a7753 :a7753. +:a7754 :a7754 :a7754. +:a7755 :a7755 :a7755. +:a7756 :a7756 :a7756. +:a7757 :a7757 :a7757. +:a7758 :a7758 :a7758. +:a7759 :a7759 :a7759. +:a7760 :a7760 :a7760. +:a7761 :a7761 :a7761. +:a7762 :a7762 :a7762. +:a7763 :a7763 :a7763. +:a7764 :a7764 :a7764. +:a7765 :a7765 :a7765. +:a7766 :a7766 :a7766. +:a7767 :a7767 :a7767. +:a7768 :a7768 :a7768. +:a7769 :a7769 :a7769. +:a7770 :a7770 :a7770. +:a7771 :a7771 :a7771. +:a7772 :a7772 :a7772. +:a7773 :a7773 :a7773. +:a7774 :a7774 :a7774. +:a7775 :a7775 :a7775. +:a7776 :a7776 :a7776. +:a7777 :a7777 :a7777. +:a7778 :a7778 :a7778. +:a7779 :a7779 :a7779. +:a7780 :a7780 :a7780. +:a7781 :a7781 :a7781. +:a7782 :a7782 :a7782. +:a7783 :a7783 :a7783. +:a7784 :a7784 :a7784. +:a7785 :a7785 :a7785. +:a7786 :a7786 :a7786. +:a7787 :a7787 :a7787. +:a7788 :a7788 :a7788. +:a7789 :a7789 :a7789. +:a7790 :a7790 :a7790. +:a7791 :a7791 :a7791. +:a7792 :a7792 :a7792. +:a7793 :a7793 :a7793. +:a7794 :a7794 :a7794. +:a7795 :a7795 :a7795. +:a7796 :a7796 :a7796. +:a7797 :a7797 :a7797. +:a7798 :a7798 :a7798. +:a7799 :a7799 :a7799. +:a7800 :a7800 :a7800. +:a7801 :a7801 :a7801. +:a7802 :a7802 :a7802. +:a7803 :a7803 :a7803. +:a7804 :a7804 :a7804. +:a7805 :a7805 :a7805. +:a7806 :a7806 :a7806. +:a7807 :a7807 :a7807. +:a7808 :a7808 :a7808. +:a7809 :a7809 :a7809. +:a7810 :a7810 :a7810. +:a7811 :a7811 :a7811. +:a7812 :a7812 :a7812. +:a7813 :a7813 :a7813. +:a7814 :a7814 :a7814. +:a7815 :a7815 :a7815. +:a7816 :a7816 :a7816. +:a7817 :a7817 :a7817. +:a7818 :a7818 :a7818. +:a7819 :a7819 :a7819. +:a7820 :a7820 :a7820. +:a7821 :a7821 :a7821. +:a7822 :a7822 :a7822. +:a7823 :a7823 :a7823. +:a7824 :a7824 :a7824. +:a7825 :a7825 :a7825. +:a7826 :a7826 :a7826. +:a7827 :a7827 :a7827. +:a7828 :a7828 :a7828. +:a7829 :a7829 :a7829. +:a7830 :a7830 :a7830. +:a7831 :a7831 :a7831. +:a7832 :a7832 :a7832. +:a7833 :a7833 :a7833. +:a7834 :a7834 :a7834. +:a7835 :a7835 :a7835. +:a7836 :a7836 :a7836. +:a7837 :a7837 :a7837. +:a7838 :a7838 :a7838. +:a7839 :a7839 :a7839. +:a7840 :a7840 :a7840. +:a7841 :a7841 :a7841. +:a7842 :a7842 :a7842. +:a7843 :a7843 :a7843. +:a7844 :a7844 :a7844. +:a7845 :a7845 :a7845. +:a7846 :a7846 :a7846. +:a7847 :a7847 :a7847. +:a7848 :a7848 :a7848. +:a7849 :a7849 :a7849. +:a7850 :a7850 :a7850. +:a7851 :a7851 :a7851. +:a7852 :a7852 :a7852. +:a7853 :a7853 :a7853. +:a7854 :a7854 :a7854. +:a7855 :a7855 :a7855. +:a7856 :a7856 :a7856. +:a7857 :a7857 :a7857. +:a7858 :a7858 :a7858. +:a7859 :a7859 :a7859. +:a7860 :a7860 :a7860. +:a7861 :a7861 :a7861. +:a7862 :a7862 :a7862. +:a7863 :a7863 :a7863. +:a7864 :a7864 :a7864. +:a7865 :a7865 :a7865. +:a7866 :a7866 :a7866. +:a7867 :a7867 :a7867. +:a7868 :a7868 :a7868. +:a7869 :a7869 :a7869. +:a7870 :a7870 :a7870. +:a7871 :a7871 :a7871. +:a7872 :a7872 :a7872. +:a7873 :a7873 :a7873. +:a7874 :a7874 :a7874. +:a7875 :a7875 :a7875. +:a7876 :a7876 :a7876. +:a7877 :a7877 :a7877. +:a7878 :a7878 :a7878. +:a7879 :a7879 :a7879. +:a7880 :a7880 :a7880. +:a7881 :a7881 :a7881. +:a7882 :a7882 :a7882. +:a7883 :a7883 :a7883. +:a7884 :a7884 :a7884. +:a7885 :a7885 :a7885. +:a7886 :a7886 :a7886. +:a7887 :a7887 :a7887. +:a7888 :a7888 :a7888. +:a7889 :a7889 :a7889. +:a7890 :a7890 :a7890. +:a7891 :a7891 :a7891. +:a7892 :a7892 :a7892. +:a7893 :a7893 :a7893. +:a7894 :a7894 :a7894. +:a7895 :a7895 :a7895. +:a7896 :a7896 :a7896. +:a7897 :a7897 :a7897. +:a7898 :a7898 :a7898. +:a7899 :a7899 :a7899. +:a7900 :a7900 :a7900. +:a7901 :a7901 :a7901. +:a7902 :a7902 :a7902. +:a7903 :a7903 :a7903. +:a7904 :a7904 :a7904. +:a7905 :a7905 :a7905. +:a7906 :a7906 :a7906. +:a7907 :a7907 :a7907. +:a7908 :a7908 :a7908. +:a7909 :a7909 :a7909. +:a7910 :a7910 :a7910. +:a7911 :a7911 :a7911. +:a7912 :a7912 :a7912. +:a7913 :a7913 :a7913. +:a7914 :a7914 :a7914. +:a7915 :a7915 :a7915. +:a7916 :a7916 :a7916. +:a7917 :a7917 :a7917. +:a7918 :a7918 :a7918. +:a7919 :a7919 :a7919. +:a7920 :a7920 :a7920. +:a7921 :a7921 :a7921. +:a7922 :a7922 :a7922. +:a7923 :a7923 :a7923. +:a7924 :a7924 :a7924. +:a7925 :a7925 :a7925. +:a7926 :a7926 :a7926. +:a7927 :a7927 :a7927. +:a7928 :a7928 :a7928. +:a7929 :a7929 :a7929. +:a7930 :a7930 :a7930. +:a7931 :a7931 :a7931. +:a7932 :a7932 :a7932. +:a7933 :a7933 :a7933. +:a7934 :a7934 :a7934. +:a7935 :a7935 :a7935. +:a7936 :a7936 :a7936. +:a7937 :a7937 :a7937. +:a7938 :a7938 :a7938. +:a7939 :a7939 :a7939. +:a7940 :a7940 :a7940. +:a7941 :a7941 :a7941. +:a7942 :a7942 :a7942. +:a7943 :a7943 :a7943. +:a7944 :a7944 :a7944. +:a7945 :a7945 :a7945. +:a7946 :a7946 :a7946. +:a7947 :a7947 :a7947. +:a7948 :a7948 :a7948. +:a7949 :a7949 :a7949. +:a7950 :a7950 :a7950. +:a7951 :a7951 :a7951. +:a7952 :a7952 :a7952. +:a7953 :a7953 :a7953. +:a7954 :a7954 :a7954. +:a7955 :a7955 :a7955. +:a7956 :a7956 :a7956. +:a7957 :a7957 :a7957. +:a7958 :a7958 :a7958. +:a7959 :a7959 :a7959. +:a7960 :a7960 :a7960. +:a7961 :a7961 :a7961. +:a7962 :a7962 :a7962. +:a7963 :a7963 :a7963. +:a7964 :a7964 :a7964. +:a7965 :a7965 :a7965. +:a7966 :a7966 :a7966. +:a7967 :a7967 :a7967. +:a7968 :a7968 :a7968. +:a7969 :a7969 :a7969. +:a7970 :a7970 :a7970. +:a7971 :a7971 :a7971. +:a7972 :a7972 :a7972. +:a7973 :a7973 :a7973. +:a7974 :a7974 :a7974. +:a7975 :a7975 :a7975. +:a7976 :a7976 :a7976. +:a7977 :a7977 :a7977. +:a7978 :a7978 :a7978. +:a7979 :a7979 :a7979. +:a7980 :a7980 :a7980. +:a7981 :a7981 :a7981. +:a7982 :a7982 :a7982. +:a7983 :a7983 :a7983. +:a7984 :a7984 :a7984. +:a7985 :a7985 :a7985. +:a7986 :a7986 :a7986. +:a7987 :a7987 :a7987. +:a7988 :a7988 :a7988. +:a7989 :a7989 :a7989. +:a7990 :a7990 :a7990. +:a7991 :a7991 :a7991. +:a7992 :a7992 :a7992. +:a7993 :a7993 :a7993. +:a7994 :a7994 :a7994. +:a7995 :a7995 :a7995. +:a7996 :a7996 :a7996. +:a7997 :a7997 :a7997. +:a7998 :a7998 :a7998. +:a7999 :a7999 :a7999. +:a8000 :a8000 :a8000. +:a8001 :a8001 :a8001. +:a8002 :a8002 :a8002. +:a8003 :a8003 :a8003. +:a8004 :a8004 :a8004. +:a8005 :a8005 :a8005. +:a8006 :a8006 :a8006. +:a8007 :a8007 :a8007. +:a8008 :a8008 :a8008. +:a8009 :a8009 :a8009. +:a8010 :a8010 :a8010. +:a8011 :a8011 :a8011. +:a8012 :a8012 :a8012. +:a8013 :a8013 :a8013. +:a8014 :a8014 :a8014. +:a8015 :a8015 :a8015. +:a8016 :a8016 :a8016. +:a8017 :a8017 :a8017. +:a8018 :a8018 :a8018. +:a8019 :a8019 :a8019. +:a8020 :a8020 :a8020. +:a8021 :a8021 :a8021. +:a8022 :a8022 :a8022. +:a8023 :a8023 :a8023. +:a8024 :a8024 :a8024. +:a8025 :a8025 :a8025. +:a8026 :a8026 :a8026. +:a8027 :a8027 :a8027. +:a8028 :a8028 :a8028. +:a8029 :a8029 :a8029. +:a8030 :a8030 :a8030. +:a8031 :a8031 :a8031. +:a8032 :a8032 :a8032. +:a8033 :a8033 :a8033. +:a8034 :a8034 :a8034. +:a8035 :a8035 :a8035. +:a8036 :a8036 :a8036. +:a8037 :a8037 :a8037. +:a8038 :a8038 :a8038. +:a8039 :a8039 :a8039. +:a8040 :a8040 :a8040. +:a8041 :a8041 :a8041. +:a8042 :a8042 :a8042. +:a8043 :a8043 :a8043. +:a8044 :a8044 :a8044. +:a8045 :a8045 :a8045. +:a8046 :a8046 :a8046. +:a8047 :a8047 :a8047. +:a8048 :a8048 :a8048. +:a8049 :a8049 :a8049. +:a8050 :a8050 :a8050. +:a8051 :a8051 :a8051. +:a8052 :a8052 :a8052. +:a8053 :a8053 :a8053. +:a8054 :a8054 :a8054. +:a8055 :a8055 :a8055. +:a8056 :a8056 :a8056. +:a8057 :a8057 :a8057. +:a8058 :a8058 :a8058. +:a8059 :a8059 :a8059. +:a8060 :a8060 :a8060. +:a8061 :a8061 :a8061. +:a8062 :a8062 :a8062. +:a8063 :a8063 :a8063. +:a8064 :a8064 :a8064. +:a8065 :a8065 :a8065. +:a8066 :a8066 :a8066. +:a8067 :a8067 :a8067. +:a8068 :a8068 :a8068. +:a8069 :a8069 :a8069. +:a8070 :a8070 :a8070. +:a8071 :a8071 :a8071. +:a8072 :a8072 :a8072. +:a8073 :a8073 :a8073. +:a8074 :a8074 :a8074. +:a8075 :a8075 :a8075. +:a8076 :a8076 :a8076. +:a8077 :a8077 :a8077. +:a8078 :a8078 :a8078. +:a8079 :a8079 :a8079. +:a8080 :a8080 :a8080. +:a8081 :a8081 :a8081. +:a8082 :a8082 :a8082. +:a8083 :a8083 :a8083. +:a8084 :a8084 :a8084. +:a8085 :a8085 :a8085. +:a8086 :a8086 :a8086. +:a8087 :a8087 :a8087. +:a8088 :a8088 :a8088. +:a8089 :a8089 :a8089. +:a8090 :a8090 :a8090. +:a8091 :a8091 :a8091. +:a8092 :a8092 :a8092. +:a8093 :a8093 :a8093. +:a8094 :a8094 :a8094. +:a8095 :a8095 :a8095. +:a8096 :a8096 :a8096. +:a8097 :a8097 :a8097. +:a8098 :a8098 :a8098. +:a8099 :a8099 :a8099. +:a8100 :a8100 :a8100. +:a8101 :a8101 :a8101. +:a8102 :a8102 :a8102. +:a8103 :a8103 :a8103. +:a8104 :a8104 :a8104. +:a8105 :a8105 :a8105. +:a8106 :a8106 :a8106. +:a8107 :a8107 :a8107. +:a8108 :a8108 :a8108. +:a8109 :a8109 :a8109. +:a8110 :a8110 :a8110. +:a8111 :a8111 :a8111. +:a8112 :a8112 :a8112. +:a8113 :a8113 :a8113. +:a8114 :a8114 :a8114. +:a8115 :a8115 :a8115. +:a8116 :a8116 :a8116. +:a8117 :a8117 :a8117. +:a8118 :a8118 :a8118. +:a8119 :a8119 :a8119. +:a8120 :a8120 :a8120. +:a8121 :a8121 :a8121. +:a8122 :a8122 :a8122. +:a8123 :a8123 :a8123. +:a8124 :a8124 :a8124. +:a8125 :a8125 :a8125. +:a8126 :a8126 :a8126. +:a8127 :a8127 :a8127. +:a8128 :a8128 :a8128. +:a8129 :a8129 :a8129. +:a8130 :a8130 :a8130. +:a8131 :a8131 :a8131. +:a8132 :a8132 :a8132. +:a8133 :a8133 :a8133. +:a8134 :a8134 :a8134. +:a8135 :a8135 :a8135. +:a8136 :a8136 :a8136. +:a8137 :a8137 :a8137. +:a8138 :a8138 :a8138. +:a8139 :a8139 :a8139. +:a8140 :a8140 :a8140. +:a8141 :a8141 :a8141. +:a8142 :a8142 :a8142. +:a8143 :a8143 :a8143. +:a8144 :a8144 :a8144. +:a8145 :a8145 :a8145. +:a8146 :a8146 :a8146. +:a8147 :a8147 :a8147. +:a8148 :a8148 :a8148. +:a8149 :a8149 :a8149. +:a8150 :a8150 :a8150. +:a8151 :a8151 :a8151. +:a8152 :a8152 :a8152. +:a8153 :a8153 :a8153. +:a8154 :a8154 :a8154. +:a8155 :a8155 :a8155. +:a8156 :a8156 :a8156. +:a8157 :a8157 :a8157. +:a8158 :a8158 :a8158. +:a8159 :a8159 :a8159. +:a8160 :a8160 :a8160. +:a8161 :a8161 :a8161. +:a8162 :a8162 :a8162. +:a8163 :a8163 :a8163. +:a8164 :a8164 :a8164. +:a8165 :a8165 :a8165. +:a8166 :a8166 :a8166. +:a8167 :a8167 :a8167. +:a8168 :a8168 :a8168. +:a8169 :a8169 :a8169. +:a8170 :a8170 :a8170. +:a8171 :a8171 :a8171. +:a8172 :a8172 :a8172. +:a8173 :a8173 :a8173. +:a8174 :a8174 :a8174. +:a8175 :a8175 :a8175. +:a8176 :a8176 :a8176. +:a8177 :a8177 :a8177. +:a8178 :a8178 :a8178. +:a8179 :a8179 :a8179. +:a8180 :a8180 :a8180. +:a8181 :a8181 :a8181. +:a8182 :a8182 :a8182. +:a8183 :a8183 :a8183. +:a8184 :a8184 :a8184. +:a8185 :a8185 :a8185. +:a8186 :a8186 :a8186. +:a8187 :a8187 :a8187. +:a8188 :a8188 :a8188. +:a8189 :a8189 :a8189. +:a8190 :a8190 :a8190. +:a8191 :a8191 :a8191. +:a8192 :a8192 :a8192. +:a8193 :a8193 :a8193. +:a8194 :a8194 :a8194. +:a8195 :a8195 :a8195. +:a8196 :a8196 :a8196. +:a8197 :a8197 :a8197. +:a8198 :a8198 :a8198. +:a8199 :a8199 :a8199. +:a8200 :a8200 :a8200. +:a8201 :a8201 :a8201. +:a8202 :a8202 :a8202. +:a8203 :a8203 :a8203. +:a8204 :a8204 :a8204. +:a8205 :a8205 :a8205. +:a8206 :a8206 :a8206. +:a8207 :a8207 :a8207. +:a8208 :a8208 :a8208. +:a8209 :a8209 :a8209. +:a8210 :a8210 :a8210. +:a8211 :a8211 :a8211. +:a8212 :a8212 :a8212. +:a8213 :a8213 :a8213. +:a8214 :a8214 :a8214. +:a8215 :a8215 :a8215. +:a8216 :a8216 :a8216. +:a8217 :a8217 :a8217. +:a8218 :a8218 :a8218. +:a8219 :a8219 :a8219. +:a8220 :a8220 :a8220. +:a8221 :a8221 :a8221. +:a8222 :a8222 :a8222. +:a8223 :a8223 :a8223. +:a8224 :a8224 :a8224. +:a8225 :a8225 :a8225. +:a8226 :a8226 :a8226. +:a8227 :a8227 :a8227. +:a8228 :a8228 :a8228. +:a8229 :a8229 :a8229. +:a8230 :a8230 :a8230. +:a8231 :a8231 :a8231. +:a8232 :a8232 :a8232. +:a8233 :a8233 :a8233. +:a8234 :a8234 :a8234. +:a8235 :a8235 :a8235. +:a8236 :a8236 :a8236. +:a8237 :a8237 :a8237. +:a8238 :a8238 :a8238. +:a8239 :a8239 :a8239. +:a8240 :a8240 :a8240. +:a8241 :a8241 :a8241. +:a8242 :a8242 :a8242. +:a8243 :a8243 :a8243. +:a8244 :a8244 :a8244. +:a8245 :a8245 :a8245. +:a8246 :a8246 :a8246. +:a8247 :a8247 :a8247. +:a8248 :a8248 :a8248. +:a8249 :a8249 :a8249. +:a8250 :a8250 :a8250. +:a8251 :a8251 :a8251. +:a8252 :a8252 :a8252. +:a8253 :a8253 :a8253. +:a8254 :a8254 :a8254. +:a8255 :a8255 :a8255. +:a8256 :a8256 :a8256. +:a8257 :a8257 :a8257. +:a8258 :a8258 :a8258. +:a8259 :a8259 :a8259. +:a8260 :a8260 :a8260. +:a8261 :a8261 :a8261. +:a8262 :a8262 :a8262. +:a8263 :a8263 :a8263. +:a8264 :a8264 :a8264. +:a8265 :a8265 :a8265. +:a8266 :a8266 :a8266. +:a8267 :a8267 :a8267. +:a8268 :a8268 :a8268. +:a8269 :a8269 :a8269. +:a8270 :a8270 :a8270. +:a8271 :a8271 :a8271. +:a8272 :a8272 :a8272. +:a8273 :a8273 :a8273. +:a8274 :a8274 :a8274. +:a8275 :a8275 :a8275. +:a8276 :a8276 :a8276. +:a8277 :a8277 :a8277. +:a8278 :a8278 :a8278. +:a8279 :a8279 :a8279. +:a8280 :a8280 :a8280. +:a8281 :a8281 :a8281. +:a8282 :a8282 :a8282. +:a8283 :a8283 :a8283. +:a8284 :a8284 :a8284. +:a8285 :a8285 :a8285. +:a8286 :a8286 :a8286. +:a8287 :a8287 :a8287. +:a8288 :a8288 :a8288. +:a8289 :a8289 :a8289. +:a8290 :a8290 :a8290. +:a8291 :a8291 :a8291. +:a8292 :a8292 :a8292. +:a8293 :a8293 :a8293. +:a8294 :a8294 :a8294. +:a8295 :a8295 :a8295. +:a8296 :a8296 :a8296. +:a8297 :a8297 :a8297. +:a8298 :a8298 :a8298. +:a8299 :a8299 :a8299. +:a8300 :a8300 :a8300. +:a8301 :a8301 :a8301. +:a8302 :a8302 :a8302. +:a8303 :a8303 :a8303. +:a8304 :a8304 :a8304. +:a8305 :a8305 :a8305. +:a8306 :a8306 :a8306. +:a8307 :a8307 :a8307. +:a8308 :a8308 :a8308. +:a8309 :a8309 :a8309. +:a8310 :a8310 :a8310. +:a8311 :a8311 :a8311. +:a8312 :a8312 :a8312. +:a8313 :a8313 :a8313. +:a8314 :a8314 :a8314. +:a8315 :a8315 :a8315. +:a8316 :a8316 :a8316. +:a8317 :a8317 :a8317. +:a8318 :a8318 :a8318. +:a8319 :a8319 :a8319. +:a8320 :a8320 :a8320. +:a8321 :a8321 :a8321. +:a8322 :a8322 :a8322. +:a8323 :a8323 :a8323. +:a8324 :a8324 :a8324. +:a8325 :a8325 :a8325. +:a8326 :a8326 :a8326. +:a8327 :a8327 :a8327. +:a8328 :a8328 :a8328. +:a8329 :a8329 :a8329. +:a8330 :a8330 :a8330. +:a8331 :a8331 :a8331. +:a8332 :a8332 :a8332. +:a8333 :a8333 :a8333. +:a8334 :a8334 :a8334. +:a8335 :a8335 :a8335. +:a8336 :a8336 :a8336. +:a8337 :a8337 :a8337. +:a8338 :a8338 :a8338. +:a8339 :a8339 :a8339. +:a8340 :a8340 :a8340. +:a8341 :a8341 :a8341. +:a8342 :a8342 :a8342. +:a8343 :a8343 :a8343. +:a8344 :a8344 :a8344. +:a8345 :a8345 :a8345. +:a8346 :a8346 :a8346. +:a8347 :a8347 :a8347. +:a8348 :a8348 :a8348. +:a8349 :a8349 :a8349. +:a8350 :a8350 :a8350. +:a8351 :a8351 :a8351. +:a8352 :a8352 :a8352. +:a8353 :a8353 :a8353. +:a8354 :a8354 :a8354. +:a8355 :a8355 :a8355. +:a8356 :a8356 :a8356. +:a8357 :a8357 :a8357. +:a8358 :a8358 :a8358. +:a8359 :a8359 :a8359. +:a8360 :a8360 :a8360. +:a8361 :a8361 :a8361. +:a8362 :a8362 :a8362. +:a8363 :a8363 :a8363. +:a8364 :a8364 :a8364. +:a8365 :a8365 :a8365. +:a8366 :a8366 :a8366. +:a8367 :a8367 :a8367. +:a8368 :a8368 :a8368. +:a8369 :a8369 :a8369. +:a8370 :a8370 :a8370. +:a8371 :a8371 :a8371. +:a8372 :a8372 :a8372. +:a8373 :a8373 :a8373. +:a8374 :a8374 :a8374. +:a8375 :a8375 :a8375. +:a8376 :a8376 :a8376. +:a8377 :a8377 :a8377. +:a8378 :a8378 :a8378. +:a8379 :a8379 :a8379. +:a8380 :a8380 :a8380. +:a8381 :a8381 :a8381. +:a8382 :a8382 :a8382. +:a8383 :a8383 :a8383. +:a8384 :a8384 :a8384. +:a8385 :a8385 :a8385. +:a8386 :a8386 :a8386. +:a8387 :a8387 :a8387. +:a8388 :a8388 :a8388. +:a8389 :a8389 :a8389. +:a8390 :a8390 :a8390. +:a8391 :a8391 :a8391. +:a8392 :a8392 :a8392. +:a8393 :a8393 :a8393. +:a8394 :a8394 :a8394. +:a8395 :a8395 :a8395. +:a8396 :a8396 :a8396. +:a8397 :a8397 :a8397. +:a8398 :a8398 :a8398. +:a8399 :a8399 :a8399. +:a8400 :a8400 :a8400. +:a8401 :a8401 :a8401. +:a8402 :a8402 :a8402. +:a8403 :a8403 :a8403. +:a8404 :a8404 :a8404. +:a8405 :a8405 :a8405. +:a8406 :a8406 :a8406. +:a8407 :a8407 :a8407. +:a8408 :a8408 :a8408. +:a8409 :a8409 :a8409. +:a8410 :a8410 :a8410. +:a8411 :a8411 :a8411. +:a8412 :a8412 :a8412. +:a8413 :a8413 :a8413. +:a8414 :a8414 :a8414. +:a8415 :a8415 :a8415. +:a8416 :a8416 :a8416. +:a8417 :a8417 :a8417. +:a8418 :a8418 :a8418. +:a8419 :a8419 :a8419. +:a8420 :a8420 :a8420. +:a8421 :a8421 :a8421. +:a8422 :a8422 :a8422. +:a8423 :a8423 :a8423. +:a8424 :a8424 :a8424. +:a8425 :a8425 :a8425. +:a8426 :a8426 :a8426. +:a8427 :a8427 :a8427. +:a8428 :a8428 :a8428. +:a8429 :a8429 :a8429. +:a8430 :a8430 :a8430. +:a8431 :a8431 :a8431. +:a8432 :a8432 :a8432. +:a8433 :a8433 :a8433. +:a8434 :a8434 :a8434. +:a8435 :a8435 :a8435. +:a8436 :a8436 :a8436. +:a8437 :a8437 :a8437. +:a8438 :a8438 :a8438. +:a8439 :a8439 :a8439. +:a8440 :a8440 :a8440. +:a8441 :a8441 :a8441. +:a8442 :a8442 :a8442. +:a8443 :a8443 :a8443. +:a8444 :a8444 :a8444. +:a8445 :a8445 :a8445. +:a8446 :a8446 :a8446. +:a8447 :a8447 :a8447. +:a8448 :a8448 :a8448. +:a8449 :a8449 :a8449. +:a8450 :a8450 :a8450. +:a8451 :a8451 :a8451. +:a8452 :a8452 :a8452. +:a8453 :a8453 :a8453. +:a8454 :a8454 :a8454. +:a8455 :a8455 :a8455. +:a8456 :a8456 :a8456. +:a8457 :a8457 :a8457. +:a8458 :a8458 :a8458. +:a8459 :a8459 :a8459. +:a8460 :a8460 :a8460. +:a8461 :a8461 :a8461. +:a8462 :a8462 :a8462. +:a8463 :a8463 :a8463. +:a8464 :a8464 :a8464. +:a8465 :a8465 :a8465. +:a8466 :a8466 :a8466. +:a8467 :a8467 :a8467. +:a8468 :a8468 :a8468. +:a8469 :a8469 :a8469. +:a8470 :a8470 :a8470. +:a8471 :a8471 :a8471. +:a8472 :a8472 :a8472. +:a8473 :a8473 :a8473. +:a8474 :a8474 :a8474. +:a8475 :a8475 :a8475. +:a8476 :a8476 :a8476. +:a8477 :a8477 :a8477. +:a8478 :a8478 :a8478. +:a8479 :a8479 :a8479. +:a8480 :a8480 :a8480. +:a8481 :a8481 :a8481. +:a8482 :a8482 :a8482. +:a8483 :a8483 :a8483. +:a8484 :a8484 :a8484. +:a8485 :a8485 :a8485. +:a8486 :a8486 :a8486. +:a8487 :a8487 :a8487. +:a8488 :a8488 :a8488. +:a8489 :a8489 :a8489. +:a8490 :a8490 :a8490. +:a8491 :a8491 :a8491. +:a8492 :a8492 :a8492. +:a8493 :a8493 :a8493. +:a8494 :a8494 :a8494. +:a8495 :a8495 :a8495. +:a8496 :a8496 :a8496. +:a8497 :a8497 :a8497. +:a8498 :a8498 :a8498. +:a8499 :a8499 :a8499. +:a8500 :a8500 :a8500. +:a8501 :a8501 :a8501. +:a8502 :a8502 :a8502. +:a8503 :a8503 :a8503. +:a8504 :a8504 :a8504. +:a8505 :a8505 :a8505. +:a8506 :a8506 :a8506. +:a8507 :a8507 :a8507. +:a8508 :a8508 :a8508. +:a8509 :a8509 :a8509. +:a8510 :a8510 :a8510. +:a8511 :a8511 :a8511. +:a8512 :a8512 :a8512. +:a8513 :a8513 :a8513. +:a8514 :a8514 :a8514. +:a8515 :a8515 :a8515. +:a8516 :a8516 :a8516. +:a8517 :a8517 :a8517. +:a8518 :a8518 :a8518. +:a8519 :a8519 :a8519. +:a8520 :a8520 :a8520. +:a8521 :a8521 :a8521. +:a8522 :a8522 :a8522. +:a8523 :a8523 :a8523. +:a8524 :a8524 :a8524. +:a8525 :a8525 :a8525. +:a8526 :a8526 :a8526. +:a8527 :a8527 :a8527. +:a8528 :a8528 :a8528. +:a8529 :a8529 :a8529. +:a8530 :a8530 :a8530. +:a8531 :a8531 :a8531. +:a8532 :a8532 :a8532. +:a8533 :a8533 :a8533. +:a8534 :a8534 :a8534. +:a8535 :a8535 :a8535. +:a8536 :a8536 :a8536. +:a8537 :a8537 :a8537. +:a8538 :a8538 :a8538. +:a8539 :a8539 :a8539. +:a8540 :a8540 :a8540. +:a8541 :a8541 :a8541. +:a8542 :a8542 :a8542. +:a8543 :a8543 :a8543. +:a8544 :a8544 :a8544. +:a8545 :a8545 :a8545. +:a8546 :a8546 :a8546. +:a8547 :a8547 :a8547. +:a8548 :a8548 :a8548. +:a8549 :a8549 :a8549. +:a8550 :a8550 :a8550. +:a8551 :a8551 :a8551. +:a8552 :a8552 :a8552. +:a8553 :a8553 :a8553. +:a8554 :a8554 :a8554. +:a8555 :a8555 :a8555. +:a8556 :a8556 :a8556. +:a8557 :a8557 :a8557. +:a8558 :a8558 :a8558. +:a8559 :a8559 :a8559. +:a8560 :a8560 :a8560. +:a8561 :a8561 :a8561. +:a8562 :a8562 :a8562. +:a8563 :a8563 :a8563. +:a8564 :a8564 :a8564. +:a8565 :a8565 :a8565. +:a8566 :a8566 :a8566. +:a8567 :a8567 :a8567. +:a8568 :a8568 :a8568. +:a8569 :a8569 :a8569. +:a8570 :a8570 :a8570. +:a8571 :a8571 :a8571. +:a8572 :a8572 :a8572. +:a8573 :a8573 :a8573. +:a8574 :a8574 :a8574. +:a8575 :a8575 :a8575. +:a8576 :a8576 :a8576. +:a8577 :a8577 :a8577. +:a8578 :a8578 :a8578. +:a8579 :a8579 :a8579. +:a8580 :a8580 :a8580. +:a8581 :a8581 :a8581. +:a8582 :a8582 :a8582. +:a8583 :a8583 :a8583. +:a8584 :a8584 :a8584. +:a8585 :a8585 :a8585. +:a8586 :a8586 :a8586. +:a8587 :a8587 :a8587. +:a8588 :a8588 :a8588. +:a8589 :a8589 :a8589. +:a8590 :a8590 :a8590. +:a8591 :a8591 :a8591. +:a8592 :a8592 :a8592. +:a8593 :a8593 :a8593. +:a8594 :a8594 :a8594. +:a8595 :a8595 :a8595. +:a8596 :a8596 :a8596. +:a8597 :a8597 :a8597. +:a8598 :a8598 :a8598. +:a8599 :a8599 :a8599. +:a8600 :a8600 :a8600. +:a8601 :a8601 :a8601. +:a8602 :a8602 :a8602. +:a8603 :a8603 :a8603. +:a8604 :a8604 :a8604. +:a8605 :a8605 :a8605. +:a8606 :a8606 :a8606. +:a8607 :a8607 :a8607. +:a8608 :a8608 :a8608. +:a8609 :a8609 :a8609. +:a8610 :a8610 :a8610. +:a8611 :a8611 :a8611. +:a8612 :a8612 :a8612. +:a8613 :a8613 :a8613. +:a8614 :a8614 :a8614. +:a8615 :a8615 :a8615. +:a8616 :a8616 :a8616. +:a8617 :a8617 :a8617. +:a8618 :a8618 :a8618. +:a8619 :a8619 :a8619. +:a8620 :a8620 :a8620. +:a8621 :a8621 :a8621. +:a8622 :a8622 :a8622. +:a8623 :a8623 :a8623. +:a8624 :a8624 :a8624. +:a8625 :a8625 :a8625. +:a8626 :a8626 :a8626. +:a8627 :a8627 :a8627. +:a8628 :a8628 :a8628. +:a8629 :a8629 :a8629. +:a8630 :a8630 :a8630. +:a8631 :a8631 :a8631. +:a8632 :a8632 :a8632. +:a8633 :a8633 :a8633. +:a8634 :a8634 :a8634. +:a8635 :a8635 :a8635. +:a8636 :a8636 :a8636. +:a8637 :a8637 :a8637. +:a8638 :a8638 :a8638. +:a8639 :a8639 :a8639. +:a8640 :a8640 :a8640. +:a8641 :a8641 :a8641. +:a8642 :a8642 :a8642. +:a8643 :a8643 :a8643. +:a8644 :a8644 :a8644. +:a8645 :a8645 :a8645. +:a8646 :a8646 :a8646. +:a8647 :a8647 :a8647. +:a8648 :a8648 :a8648. +:a8649 :a8649 :a8649. +:a8650 :a8650 :a8650. +:a8651 :a8651 :a8651. +:a8652 :a8652 :a8652. +:a8653 :a8653 :a8653. +:a8654 :a8654 :a8654. +:a8655 :a8655 :a8655. +:a8656 :a8656 :a8656. +:a8657 :a8657 :a8657. +:a8658 :a8658 :a8658. +:a8659 :a8659 :a8659. +:a8660 :a8660 :a8660. +:a8661 :a8661 :a8661. +:a8662 :a8662 :a8662. +:a8663 :a8663 :a8663. +:a8664 :a8664 :a8664. +:a8665 :a8665 :a8665. +:a8666 :a8666 :a8666. +:a8667 :a8667 :a8667. +:a8668 :a8668 :a8668. +:a8669 :a8669 :a8669. +:a8670 :a8670 :a8670. +:a8671 :a8671 :a8671. +:a8672 :a8672 :a8672. +:a8673 :a8673 :a8673. +:a8674 :a8674 :a8674. +:a8675 :a8675 :a8675. +:a8676 :a8676 :a8676. +:a8677 :a8677 :a8677. +:a8678 :a8678 :a8678. +:a8679 :a8679 :a8679. +:a8680 :a8680 :a8680. +:a8681 :a8681 :a8681. +:a8682 :a8682 :a8682. +:a8683 :a8683 :a8683. +:a8684 :a8684 :a8684. +:a8685 :a8685 :a8685. +:a8686 :a8686 :a8686. +:a8687 :a8687 :a8687. +:a8688 :a8688 :a8688. +:a8689 :a8689 :a8689. +:a8690 :a8690 :a8690. +:a8691 :a8691 :a8691. +:a8692 :a8692 :a8692. +:a8693 :a8693 :a8693. +:a8694 :a8694 :a8694. +:a8695 :a8695 :a8695. +:a8696 :a8696 :a8696. +:a8697 :a8697 :a8697. +:a8698 :a8698 :a8698. +:a8699 :a8699 :a8699. +:a8700 :a8700 :a8700. +:a8701 :a8701 :a8701. +:a8702 :a8702 :a8702. +:a8703 :a8703 :a8703. +:a8704 :a8704 :a8704. +:a8705 :a8705 :a8705. +:a8706 :a8706 :a8706. +:a8707 :a8707 :a8707. +:a8708 :a8708 :a8708. +:a8709 :a8709 :a8709. +:a8710 :a8710 :a8710. +:a8711 :a8711 :a8711. +:a8712 :a8712 :a8712. +:a8713 :a8713 :a8713. +:a8714 :a8714 :a8714. +:a8715 :a8715 :a8715. +:a8716 :a8716 :a8716. +:a8717 :a8717 :a8717. +:a8718 :a8718 :a8718. +:a8719 :a8719 :a8719. +:a8720 :a8720 :a8720. +:a8721 :a8721 :a8721. +:a8722 :a8722 :a8722. +:a8723 :a8723 :a8723. +:a8724 :a8724 :a8724. +:a8725 :a8725 :a8725. +:a8726 :a8726 :a8726. +:a8727 :a8727 :a8727. +:a8728 :a8728 :a8728. +:a8729 :a8729 :a8729. +:a8730 :a8730 :a8730. +:a8731 :a8731 :a8731. +:a8732 :a8732 :a8732. +:a8733 :a8733 :a8733. +:a8734 :a8734 :a8734. +:a8735 :a8735 :a8735. +:a8736 :a8736 :a8736. +:a8737 :a8737 :a8737. +:a8738 :a8738 :a8738. +:a8739 :a8739 :a8739. +:a8740 :a8740 :a8740. +:a8741 :a8741 :a8741. +:a8742 :a8742 :a8742. +:a8743 :a8743 :a8743. +:a8744 :a8744 :a8744. +:a8745 :a8745 :a8745. +:a8746 :a8746 :a8746. +:a8747 :a8747 :a8747. +:a8748 :a8748 :a8748. +:a8749 :a8749 :a8749. +:a8750 :a8750 :a8750. +:a8751 :a8751 :a8751. +:a8752 :a8752 :a8752. +:a8753 :a8753 :a8753. +:a8754 :a8754 :a8754. +:a8755 :a8755 :a8755. +:a8756 :a8756 :a8756. +:a8757 :a8757 :a8757. +:a8758 :a8758 :a8758. +:a8759 :a8759 :a8759. +:a8760 :a8760 :a8760. +:a8761 :a8761 :a8761. +:a8762 :a8762 :a8762. +:a8763 :a8763 :a8763. +:a8764 :a8764 :a8764. +:a8765 :a8765 :a8765. +:a8766 :a8766 :a8766. +:a8767 :a8767 :a8767. +:a8768 :a8768 :a8768. +:a8769 :a8769 :a8769. +:a8770 :a8770 :a8770. +:a8771 :a8771 :a8771. +:a8772 :a8772 :a8772. +:a8773 :a8773 :a8773. +:a8774 :a8774 :a8774. +:a8775 :a8775 :a8775. +:a8776 :a8776 :a8776. +:a8777 :a8777 :a8777. +:a8778 :a8778 :a8778. +:a8779 :a8779 :a8779. +:a8780 :a8780 :a8780. +:a8781 :a8781 :a8781. +:a8782 :a8782 :a8782. +:a8783 :a8783 :a8783. +:a8784 :a8784 :a8784. +:a8785 :a8785 :a8785. +:a8786 :a8786 :a8786. +:a8787 :a8787 :a8787. +:a8788 :a8788 :a8788. +:a8789 :a8789 :a8789. +:a8790 :a8790 :a8790. +:a8791 :a8791 :a8791. +:a8792 :a8792 :a8792. +:a8793 :a8793 :a8793. +:a8794 :a8794 :a8794. +:a8795 :a8795 :a8795. +:a8796 :a8796 :a8796. +:a8797 :a8797 :a8797. +:a8798 :a8798 :a8798. +:a8799 :a8799 :a8799. +:a8800 :a8800 :a8800. +:a8801 :a8801 :a8801. +:a8802 :a8802 :a8802. +:a8803 :a8803 :a8803. +:a8804 :a8804 :a8804. +:a8805 :a8805 :a8805. +:a8806 :a8806 :a8806. +:a8807 :a8807 :a8807. +:a8808 :a8808 :a8808. +:a8809 :a8809 :a8809. +:a8810 :a8810 :a8810. +:a8811 :a8811 :a8811. +:a8812 :a8812 :a8812. +:a8813 :a8813 :a8813. +:a8814 :a8814 :a8814. +:a8815 :a8815 :a8815. +:a8816 :a8816 :a8816. +:a8817 :a8817 :a8817. +:a8818 :a8818 :a8818. +:a8819 :a8819 :a8819. +:a8820 :a8820 :a8820. +:a8821 :a8821 :a8821. +:a8822 :a8822 :a8822. +:a8823 :a8823 :a8823. +:a8824 :a8824 :a8824. +:a8825 :a8825 :a8825. +:a8826 :a8826 :a8826. +:a8827 :a8827 :a8827. +:a8828 :a8828 :a8828. +:a8829 :a8829 :a8829. +:a8830 :a8830 :a8830. +:a8831 :a8831 :a8831. +:a8832 :a8832 :a8832. +:a8833 :a8833 :a8833. +:a8834 :a8834 :a8834. +:a8835 :a8835 :a8835. +:a8836 :a8836 :a8836. +:a8837 :a8837 :a8837. +:a8838 :a8838 :a8838. +:a8839 :a8839 :a8839. +:a8840 :a8840 :a8840. +:a8841 :a8841 :a8841. +:a8842 :a8842 :a8842. +:a8843 :a8843 :a8843. +:a8844 :a8844 :a8844. +:a8845 :a8845 :a8845. +:a8846 :a8846 :a8846. +:a8847 :a8847 :a8847. +:a8848 :a8848 :a8848. +:a8849 :a8849 :a8849. +:a8850 :a8850 :a8850. +:a8851 :a8851 :a8851. +:a8852 :a8852 :a8852. +:a8853 :a8853 :a8853. +:a8854 :a8854 :a8854. +:a8855 :a8855 :a8855. +:a8856 :a8856 :a8856. +:a8857 :a8857 :a8857. +:a8858 :a8858 :a8858. +:a8859 :a8859 :a8859. +:a8860 :a8860 :a8860. +:a8861 :a8861 :a8861. +:a8862 :a8862 :a8862. +:a8863 :a8863 :a8863. +:a8864 :a8864 :a8864. +:a8865 :a8865 :a8865. +:a8866 :a8866 :a8866. +:a8867 :a8867 :a8867. +:a8868 :a8868 :a8868. +:a8869 :a8869 :a8869. +:a8870 :a8870 :a8870. +:a8871 :a8871 :a8871. +:a8872 :a8872 :a8872. +:a8873 :a8873 :a8873. +:a8874 :a8874 :a8874. +:a8875 :a8875 :a8875. +:a8876 :a8876 :a8876. +:a8877 :a8877 :a8877. +:a8878 :a8878 :a8878. +:a8879 :a8879 :a8879. +:a8880 :a8880 :a8880. +:a8881 :a8881 :a8881. +:a8882 :a8882 :a8882. +:a8883 :a8883 :a8883. +:a8884 :a8884 :a8884. +:a8885 :a8885 :a8885. +:a8886 :a8886 :a8886. +:a8887 :a8887 :a8887. +:a8888 :a8888 :a8888. +:a8889 :a8889 :a8889. +:a8890 :a8890 :a8890. +:a8891 :a8891 :a8891. +:a8892 :a8892 :a8892. +:a8893 :a8893 :a8893. +:a8894 :a8894 :a8894. +:a8895 :a8895 :a8895. +:a8896 :a8896 :a8896. +:a8897 :a8897 :a8897. +:a8898 :a8898 :a8898. +:a8899 :a8899 :a8899. +:a8900 :a8900 :a8900. +:a8901 :a8901 :a8901. +:a8902 :a8902 :a8902. +:a8903 :a8903 :a8903. +:a8904 :a8904 :a8904. +:a8905 :a8905 :a8905. +:a8906 :a8906 :a8906. +:a8907 :a8907 :a8907. +:a8908 :a8908 :a8908. +:a8909 :a8909 :a8909. +:a8910 :a8910 :a8910. +:a8911 :a8911 :a8911. +:a8912 :a8912 :a8912. +:a8913 :a8913 :a8913. +:a8914 :a8914 :a8914. +:a8915 :a8915 :a8915. +:a8916 :a8916 :a8916. +:a8917 :a8917 :a8917. +:a8918 :a8918 :a8918. +:a8919 :a8919 :a8919. +:a8920 :a8920 :a8920. +:a8921 :a8921 :a8921. +:a8922 :a8922 :a8922. +:a8923 :a8923 :a8923. +:a8924 :a8924 :a8924. +:a8925 :a8925 :a8925. +:a8926 :a8926 :a8926. +:a8927 :a8927 :a8927. +:a8928 :a8928 :a8928. +:a8929 :a8929 :a8929. +:a8930 :a8930 :a8930. +:a8931 :a8931 :a8931. +:a8932 :a8932 :a8932. +:a8933 :a8933 :a8933. +:a8934 :a8934 :a8934. +:a8935 :a8935 :a8935. +:a8936 :a8936 :a8936. +:a8937 :a8937 :a8937. +:a8938 :a8938 :a8938. +:a8939 :a8939 :a8939. +:a8940 :a8940 :a8940. +:a8941 :a8941 :a8941. +:a8942 :a8942 :a8942. +:a8943 :a8943 :a8943. +:a8944 :a8944 :a8944. +:a8945 :a8945 :a8945. +:a8946 :a8946 :a8946. +:a8947 :a8947 :a8947. +:a8948 :a8948 :a8948. +:a8949 :a8949 :a8949. +:a8950 :a8950 :a8950. +:a8951 :a8951 :a8951. +:a8952 :a8952 :a8952. +:a8953 :a8953 :a8953. +:a8954 :a8954 :a8954. +:a8955 :a8955 :a8955. +:a8956 :a8956 :a8956. +:a8957 :a8957 :a8957. +:a8958 :a8958 :a8958. +:a8959 :a8959 :a8959. +:a8960 :a8960 :a8960. +:a8961 :a8961 :a8961. +:a8962 :a8962 :a8962. +:a8963 :a8963 :a8963. +:a8964 :a8964 :a8964. +:a8965 :a8965 :a8965. +:a8966 :a8966 :a8966. +:a8967 :a8967 :a8967. +:a8968 :a8968 :a8968. +:a8969 :a8969 :a8969. +:a8970 :a8970 :a8970. +:a8971 :a8971 :a8971. +:a8972 :a8972 :a8972. +:a8973 :a8973 :a8973. +:a8974 :a8974 :a8974. +:a8975 :a8975 :a8975. +:a8976 :a8976 :a8976. +:a8977 :a8977 :a8977. +:a8978 :a8978 :a8978. +:a8979 :a8979 :a8979. +:a8980 :a8980 :a8980. +:a8981 :a8981 :a8981. +:a8982 :a8982 :a8982. +:a8983 :a8983 :a8983. +:a8984 :a8984 :a8984. +:a8985 :a8985 :a8985. +:a8986 :a8986 :a8986. +:a8987 :a8987 :a8987. +:a8988 :a8988 :a8988. +:a8989 :a8989 :a8989. +:a8990 :a8990 :a8990. +:a8991 :a8991 :a8991. +:a8992 :a8992 :a8992. +:a8993 :a8993 :a8993. +:a8994 :a8994 :a8994. +:a8995 :a8995 :a8995. +:a8996 :a8996 :a8996. +:a8997 :a8997 :a8997. +:a8998 :a8998 :a8998. +:a8999 :a8999 :a8999. +:a9000 :a9000 :a9000. +:a9001 :a9001 :a9001. +:a9002 :a9002 :a9002. +:a9003 :a9003 :a9003. +:a9004 :a9004 :a9004. +:a9005 :a9005 :a9005. +:a9006 :a9006 :a9006. +:a9007 :a9007 :a9007. +:a9008 :a9008 :a9008. +:a9009 :a9009 :a9009. +:a9010 :a9010 :a9010. +:a9011 :a9011 :a9011. +:a9012 :a9012 :a9012. +:a9013 :a9013 :a9013. +:a9014 :a9014 :a9014. +:a9015 :a9015 :a9015. +:a9016 :a9016 :a9016. +:a9017 :a9017 :a9017. +:a9018 :a9018 :a9018. +:a9019 :a9019 :a9019. +:a9020 :a9020 :a9020. +:a9021 :a9021 :a9021. +:a9022 :a9022 :a9022. +:a9023 :a9023 :a9023. +:a9024 :a9024 :a9024. +:a9025 :a9025 :a9025. +:a9026 :a9026 :a9026. +:a9027 :a9027 :a9027. +:a9028 :a9028 :a9028. +:a9029 :a9029 :a9029. +:a9030 :a9030 :a9030. +:a9031 :a9031 :a9031. +:a9032 :a9032 :a9032. +:a9033 :a9033 :a9033. +:a9034 :a9034 :a9034. +:a9035 :a9035 :a9035. +:a9036 :a9036 :a9036. +:a9037 :a9037 :a9037. +:a9038 :a9038 :a9038. +:a9039 :a9039 :a9039. +:a9040 :a9040 :a9040. +:a9041 :a9041 :a9041. +:a9042 :a9042 :a9042. +:a9043 :a9043 :a9043. +:a9044 :a9044 :a9044. +:a9045 :a9045 :a9045. +:a9046 :a9046 :a9046. +:a9047 :a9047 :a9047. +:a9048 :a9048 :a9048. +:a9049 :a9049 :a9049. +:a9050 :a9050 :a9050. +:a9051 :a9051 :a9051. +:a9052 :a9052 :a9052. +:a9053 :a9053 :a9053. +:a9054 :a9054 :a9054. +:a9055 :a9055 :a9055. +:a9056 :a9056 :a9056. +:a9057 :a9057 :a9057. +:a9058 :a9058 :a9058. +:a9059 :a9059 :a9059. +:a9060 :a9060 :a9060. +:a9061 :a9061 :a9061. +:a9062 :a9062 :a9062. +:a9063 :a9063 :a9063. +:a9064 :a9064 :a9064. +:a9065 :a9065 :a9065. +:a9066 :a9066 :a9066. +:a9067 :a9067 :a9067. +:a9068 :a9068 :a9068. +:a9069 :a9069 :a9069. +:a9070 :a9070 :a9070. +:a9071 :a9071 :a9071. +:a9072 :a9072 :a9072. +:a9073 :a9073 :a9073. +:a9074 :a9074 :a9074. +:a9075 :a9075 :a9075. +:a9076 :a9076 :a9076. +:a9077 :a9077 :a9077. +:a9078 :a9078 :a9078. +:a9079 :a9079 :a9079. +:a9080 :a9080 :a9080. +:a9081 :a9081 :a9081. +:a9082 :a9082 :a9082. +:a9083 :a9083 :a9083. +:a9084 :a9084 :a9084. +:a9085 :a9085 :a9085. +:a9086 :a9086 :a9086. +:a9087 :a9087 :a9087. +:a9088 :a9088 :a9088. +:a9089 :a9089 :a9089. +:a9090 :a9090 :a9090. +:a9091 :a9091 :a9091. +:a9092 :a9092 :a9092. +:a9093 :a9093 :a9093. +:a9094 :a9094 :a9094. +:a9095 :a9095 :a9095. +:a9096 :a9096 :a9096. +:a9097 :a9097 :a9097. +:a9098 :a9098 :a9098. +:a9099 :a9099 :a9099. +:a9100 :a9100 :a9100. +:a9101 :a9101 :a9101. +:a9102 :a9102 :a9102. +:a9103 :a9103 :a9103. +:a9104 :a9104 :a9104. +:a9105 :a9105 :a9105. +:a9106 :a9106 :a9106. +:a9107 :a9107 :a9107. +:a9108 :a9108 :a9108. +:a9109 :a9109 :a9109. +:a9110 :a9110 :a9110. +:a9111 :a9111 :a9111. +:a9112 :a9112 :a9112. +:a9113 :a9113 :a9113. +:a9114 :a9114 :a9114. +:a9115 :a9115 :a9115. +:a9116 :a9116 :a9116. +:a9117 :a9117 :a9117. +:a9118 :a9118 :a9118. +:a9119 :a9119 :a9119. +:a9120 :a9120 :a9120. +:a9121 :a9121 :a9121. +:a9122 :a9122 :a9122. +:a9123 :a9123 :a9123. +:a9124 :a9124 :a9124. +:a9125 :a9125 :a9125. +:a9126 :a9126 :a9126. +:a9127 :a9127 :a9127. +:a9128 :a9128 :a9128. +:a9129 :a9129 :a9129. +:a9130 :a9130 :a9130. +:a9131 :a9131 :a9131. +:a9132 :a9132 :a9132. +:a9133 :a9133 :a9133. +:a9134 :a9134 :a9134. +:a9135 :a9135 :a9135. +:a9136 :a9136 :a9136. +:a9137 :a9137 :a9137. +:a9138 :a9138 :a9138. +:a9139 :a9139 :a9139. +:a9140 :a9140 :a9140. +:a9141 :a9141 :a9141. +:a9142 :a9142 :a9142. +:a9143 :a9143 :a9143. +:a9144 :a9144 :a9144. +:a9145 :a9145 :a9145. +:a9146 :a9146 :a9146. +:a9147 :a9147 :a9147. +:a9148 :a9148 :a9148. +:a9149 :a9149 :a9149. +:a9150 :a9150 :a9150. +:a9151 :a9151 :a9151. +:a9152 :a9152 :a9152. +:a9153 :a9153 :a9153. +:a9154 :a9154 :a9154. +:a9155 :a9155 :a9155. +:a9156 :a9156 :a9156. +:a9157 :a9157 :a9157. +:a9158 :a9158 :a9158. +:a9159 :a9159 :a9159. +:a9160 :a9160 :a9160. +:a9161 :a9161 :a9161. +:a9162 :a9162 :a9162. +:a9163 :a9163 :a9163. +:a9164 :a9164 :a9164. +:a9165 :a9165 :a9165. +:a9166 :a9166 :a9166. +:a9167 :a9167 :a9167. +:a9168 :a9168 :a9168. +:a9169 :a9169 :a9169. +:a9170 :a9170 :a9170. +:a9171 :a9171 :a9171. +:a9172 :a9172 :a9172. +:a9173 :a9173 :a9173. +:a9174 :a9174 :a9174. +:a9175 :a9175 :a9175. +:a9176 :a9176 :a9176. +:a9177 :a9177 :a9177. +:a9178 :a9178 :a9178. +:a9179 :a9179 :a9179. +:a9180 :a9180 :a9180. +:a9181 :a9181 :a9181. +:a9182 :a9182 :a9182. +:a9183 :a9183 :a9183. +:a9184 :a9184 :a9184. +:a9185 :a9185 :a9185. +:a9186 :a9186 :a9186. +:a9187 :a9187 :a9187. +:a9188 :a9188 :a9188. +:a9189 :a9189 :a9189. +:a9190 :a9190 :a9190. +:a9191 :a9191 :a9191. +:a9192 :a9192 :a9192. +:a9193 :a9193 :a9193. +:a9194 :a9194 :a9194. +:a9195 :a9195 :a9195. +:a9196 :a9196 :a9196. +:a9197 :a9197 :a9197. +:a9198 :a9198 :a9198. +:a9199 :a9199 :a9199. +:a9200 :a9200 :a9200. +:a9201 :a9201 :a9201. +:a9202 :a9202 :a9202. +:a9203 :a9203 :a9203. +:a9204 :a9204 :a9204. +:a9205 :a9205 :a9205. +:a9206 :a9206 :a9206. +:a9207 :a9207 :a9207. +:a9208 :a9208 :a9208. +:a9209 :a9209 :a9209. +:a9210 :a9210 :a9210. +:a9211 :a9211 :a9211. +:a9212 :a9212 :a9212. +:a9213 :a9213 :a9213. +:a9214 :a9214 :a9214. +:a9215 :a9215 :a9215. +:a9216 :a9216 :a9216. +:a9217 :a9217 :a9217. +:a9218 :a9218 :a9218. +:a9219 :a9219 :a9219. +:a9220 :a9220 :a9220. +:a9221 :a9221 :a9221. +:a9222 :a9222 :a9222. +:a9223 :a9223 :a9223. +:a9224 :a9224 :a9224. +:a9225 :a9225 :a9225. +:a9226 :a9226 :a9226. +:a9227 :a9227 :a9227. +:a9228 :a9228 :a9228. +:a9229 :a9229 :a9229. +:a9230 :a9230 :a9230. +:a9231 :a9231 :a9231. +:a9232 :a9232 :a9232. +:a9233 :a9233 :a9233. +:a9234 :a9234 :a9234. +:a9235 :a9235 :a9235. +:a9236 :a9236 :a9236. +:a9237 :a9237 :a9237. +:a9238 :a9238 :a9238. +:a9239 :a9239 :a9239. +:a9240 :a9240 :a9240. +:a9241 :a9241 :a9241. +:a9242 :a9242 :a9242. +:a9243 :a9243 :a9243. +:a9244 :a9244 :a9244. +:a9245 :a9245 :a9245. +:a9246 :a9246 :a9246. +:a9247 :a9247 :a9247. +:a9248 :a9248 :a9248. +:a9249 :a9249 :a9249. +:a9250 :a9250 :a9250. +:a9251 :a9251 :a9251. +:a9252 :a9252 :a9252. +:a9253 :a9253 :a9253. +:a9254 :a9254 :a9254. +:a9255 :a9255 :a9255. +:a9256 :a9256 :a9256. +:a9257 :a9257 :a9257. +:a9258 :a9258 :a9258. +:a9259 :a9259 :a9259. +:a9260 :a9260 :a9260. +:a9261 :a9261 :a9261. +:a9262 :a9262 :a9262. +:a9263 :a9263 :a9263. +:a9264 :a9264 :a9264. +:a9265 :a9265 :a9265. +:a9266 :a9266 :a9266. +:a9267 :a9267 :a9267. +:a9268 :a9268 :a9268. +:a9269 :a9269 :a9269. +:a9270 :a9270 :a9270. +:a9271 :a9271 :a9271. +:a9272 :a9272 :a9272. +:a9273 :a9273 :a9273. +:a9274 :a9274 :a9274. +:a9275 :a9275 :a9275. +:a9276 :a9276 :a9276. +:a9277 :a9277 :a9277. +:a9278 :a9278 :a9278. +:a9279 :a9279 :a9279. +:a9280 :a9280 :a9280. +:a9281 :a9281 :a9281. +:a9282 :a9282 :a9282. +:a9283 :a9283 :a9283. +:a9284 :a9284 :a9284. +:a9285 :a9285 :a9285. +:a9286 :a9286 :a9286. +:a9287 :a9287 :a9287. +:a9288 :a9288 :a9288. +:a9289 :a9289 :a9289. +:a9290 :a9290 :a9290. +:a9291 :a9291 :a9291. +:a9292 :a9292 :a9292. +:a9293 :a9293 :a9293. +:a9294 :a9294 :a9294. +:a9295 :a9295 :a9295. +:a9296 :a9296 :a9296. +:a9297 :a9297 :a9297. +:a9298 :a9298 :a9298. +:a9299 :a9299 :a9299. +:a9300 :a9300 :a9300. +:a9301 :a9301 :a9301. +:a9302 :a9302 :a9302. +:a9303 :a9303 :a9303. +:a9304 :a9304 :a9304. +:a9305 :a9305 :a9305. +:a9306 :a9306 :a9306. +:a9307 :a9307 :a9307. +:a9308 :a9308 :a9308. +:a9309 :a9309 :a9309. +:a9310 :a9310 :a9310. +:a9311 :a9311 :a9311. +:a9312 :a9312 :a9312. +:a9313 :a9313 :a9313. +:a9314 :a9314 :a9314. +:a9315 :a9315 :a9315. +:a9316 :a9316 :a9316. +:a9317 :a9317 :a9317. +:a9318 :a9318 :a9318. +:a9319 :a9319 :a9319. +:a9320 :a9320 :a9320. +:a9321 :a9321 :a9321. +:a9322 :a9322 :a9322. +:a9323 :a9323 :a9323. +:a9324 :a9324 :a9324. +:a9325 :a9325 :a9325. +:a9326 :a9326 :a9326. +:a9327 :a9327 :a9327. +:a9328 :a9328 :a9328. +:a9329 :a9329 :a9329. +:a9330 :a9330 :a9330. +:a9331 :a9331 :a9331. +:a9332 :a9332 :a9332. +:a9333 :a9333 :a9333. +:a9334 :a9334 :a9334. +:a9335 :a9335 :a9335. +:a9336 :a9336 :a9336. +:a9337 :a9337 :a9337. +:a9338 :a9338 :a9338. +:a9339 :a9339 :a9339. +:a9340 :a9340 :a9340. +:a9341 :a9341 :a9341. +:a9342 :a9342 :a9342. +:a9343 :a9343 :a9343. +:a9344 :a9344 :a9344. +:a9345 :a9345 :a9345. +:a9346 :a9346 :a9346. +:a9347 :a9347 :a9347. +:a9348 :a9348 :a9348. +:a9349 :a9349 :a9349. +:a9350 :a9350 :a9350. +:a9351 :a9351 :a9351. +:a9352 :a9352 :a9352. +:a9353 :a9353 :a9353. +:a9354 :a9354 :a9354. +:a9355 :a9355 :a9355. +:a9356 :a9356 :a9356. +:a9357 :a9357 :a9357. +:a9358 :a9358 :a9358. +:a9359 :a9359 :a9359. +:a9360 :a9360 :a9360. +:a9361 :a9361 :a9361. +:a9362 :a9362 :a9362. +:a9363 :a9363 :a9363. +:a9364 :a9364 :a9364. +:a9365 :a9365 :a9365. +:a9366 :a9366 :a9366. +:a9367 :a9367 :a9367. +:a9368 :a9368 :a9368. +:a9369 :a9369 :a9369. +:a9370 :a9370 :a9370. +:a9371 :a9371 :a9371. +:a9372 :a9372 :a9372. +:a9373 :a9373 :a9373. +:a9374 :a9374 :a9374. +:a9375 :a9375 :a9375. +:a9376 :a9376 :a9376. +:a9377 :a9377 :a9377. +:a9378 :a9378 :a9378. +:a9379 :a9379 :a9379. +:a9380 :a9380 :a9380. +:a9381 :a9381 :a9381. +:a9382 :a9382 :a9382. +:a9383 :a9383 :a9383. +:a9384 :a9384 :a9384. +:a9385 :a9385 :a9385. +:a9386 :a9386 :a9386. +:a9387 :a9387 :a9387. +:a9388 :a9388 :a9388. +:a9389 :a9389 :a9389. +:a9390 :a9390 :a9390. +:a9391 :a9391 :a9391. +:a9392 :a9392 :a9392. +:a9393 :a9393 :a9393. +:a9394 :a9394 :a9394. +:a9395 :a9395 :a9395. +:a9396 :a9396 :a9396. +:a9397 :a9397 :a9397. +:a9398 :a9398 :a9398. +:a9399 :a9399 :a9399. +:a9400 :a9400 :a9400. +:a9401 :a9401 :a9401. +:a9402 :a9402 :a9402. +:a9403 :a9403 :a9403. +:a9404 :a9404 :a9404. +:a9405 :a9405 :a9405. +:a9406 :a9406 :a9406. +:a9407 :a9407 :a9407. +:a9408 :a9408 :a9408. +:a9409 :a9409 :a9409. +:a9410 :a9410 :a9410. +:a9411 :a9411 :a9411. +:a9412 :a9412 :a9412. +:a9413 :a9413 :a9413. +:a9414 :a9414 :a9414. +:a9415 :a9415 :a9415. +:a9416 :a9416 :a9416. +:a9417 :a9417 :a9417. +:a9418 :a9418 :a9418. +:a9419 :a9419 :a9419. +:a9420 :a9420 :a9420. +:a9421 :a9421 :a9421. +:a9422 :a9422 :a9422. +:a9423 :a9423 :a9423. +:a9424 :a9424 :a9424. +:a9425 :a9425 :a9425. +:a9426 :a9426 :a9426. +:a9427 :a9427 :a9427. +:a9428 :a9428 :a9428. +:a9429 :a9429 :a9429. +:a9430 :a9430 :a9430. +:a9431 :a9431 :a9431. +:a9432 :a9432 :a9432. +:a9433 :a9433 :a9433. +:a9434 :a9434 :a9434. +:a9435 :a9435 :a9435. +:a9436 :a9436 :a9436. +:a9437 :a9437 :a9437. +:a9438 :a9438 :a9438. +:a9439 :a9439 :a9439. +:a9440 :a9440 :a9440. +:a9441 :a9441 :a9441. +:a9442 :a9442 :a9442. +:a9443 :a9443 :a9443. +:a9444 :a9444 :a9444. +:a9445 :a9445 :a9445. +:a9446 :a9446 :a9446. +:a9447 :a9447 :a9447. +:a9448 :a9448 :a9448. +:a9449 :a9449 :a9449. +:a9450 :a9450 :a9450. +:a9451 :a9451 :a9451. +:a9452 :a9452 :a9452. +:a9453 :a9453 :a9453. +:a9454 :a9454 :a9454. +:a9455 :a9455 :a9455. +:a9456 :a9456 :a9456. +:a9457 :a9457 :a9457. +:a9458 :a9458 :a9458. +:a9459 :a9459 :a9459. +:a9460 :a9460 :a9460. +:a9461 :a9461 :a9461. +:a9462 :a9462 :a9462. +:a9463 :a9463 :a9463. +:a9464 :a9464 :a9464. +:a9465 :a9465 :a9465. +:a9466 :a9466 :a9466. +:a9467 :a9467 :a9467. +:a9468 :a9468 :a9468. +:a9469 :a9469 :a9469. +:a9470 :a9470 :a9470. +:a9471 :a9471 :a9471. +:a9472 :a9472 :a9472. +:a9473 :a9473 :a9473. +:a9474 :a9474 :a9474. +:a9475 :a9475 :a9475. +:a9476 :a9476 :a9476. +:a9477 :a9477 :a9477. +:a9478 :a9478 :a9478. +:a9479 :a9479 :a9479. +:a9480 :a9480 :a9480. +:a9481 :a9481 :a9481. +:a9482 :a9482 :a9482. +:a9483 :a9483 :a9483. +:a9484 :a9484 :a9484. +:a9485 :a9485 :a9485. +:a9486 :a9486 :a9486. +:a9487 :a9487 :a9487. +:a9488 :a9488 :a9488. +:a9489 :a9489 :a9489. +:a9490 :a9490 :a9490. +:a9491 :a9491 :a9491. +:a9492 :a9492 :a9492. +:a9493 :a9493 :a9493. +:a9494 :a9494 :a9494. +:a9495 :a9495 :a9495. +:a9496 :a9496 :a9496. +:a9497 :a9497 :a9497. +:a9498 :a9498 :a9498. +:a9499 :a9499 :a9499. +:a9500 :a9500 :a9500. +:a9501 :a9501 :a9501. +:a9502 :a9502 :a9502. +:a9503 :a9503 :a9503. +:a9504 :a9504 :a9504. +:a9505 :a9505 :a9505. +:a9506 :a9506 :a9506. +:a9507 :a9507 :a9507. +:a9508 :a9508 :a9508. +:a9509 :a9509 :a9509. +:a9510 :a9510 :a9510. +:a9511 :a9511 :a9511. +:a9512 :a9512 :a9512. +:a9513 :a9513 :a9513. +:a9514 :a9514 :a9514. +:a9515 :a9515 :a9515. +:a9516 :a9516 :a9516. +:a9517 :a9517 :a9517. +:a9518 :a9518 :a9518. +:a9519 :a9519 :a9519. +:a9520 :a9520 :a9520. +:a9521 :a9521 :a9521. +:a9522 :a9522 :a9522. +:a9523 :a9523 :a9523. +:a9524 :a9524 :a9524. +:a9525 :a9525 :a9525. +:a9526 :a9526 :a9526. +:a9527 :a9527 :a9527. +:a9528 :a9528 :a9528. +:a9529 :a9529 :a9529. +:a9530 :a9530 :a9530. +:a9531 :a9531 :a9531. +:a9532 :a9532 :a9532. +:a9533 :a9533 :a9533. +:a9534 :a9534 :a9534. +:a9535 :a9535 :a9535. +:a9536 :a9536 :a9536. +:a9537 :a9537 :a9537. +:a9538 :a9538 :a9538. +:a9539 :a9539 :a9539. +:a9540 :a9540 :a9540. +:a9541 :a9541 :a9541. +:a9542 :a9542 :a9542. +:a9543 :a9543 :a9543. +:a9544 :a9544 :a9544. +:a9545 :a9545 :a9545. +:a9546 :a9546 :a9546. +:a9547 :a9547 :a9547. +:a9548 :a9548 :a9548. +:a9549 :a9549 :a9549. +:a9550 :a9550 :a9550. +:a9551 :a9551 :a9551. +:a9552 :a9552 :a9552. +:a9553 :a9553 :a9553. +:a9554 :a9554 :a9554. +:a9555 :a9555 :a9555. +:a9556 :a9556 :a9556. +:a9557 :a9557 :a9557. +:a9558 :a9558 :a9558. +:a9559 :a9559 :a9559. +:a9560 :a9560 :a9560. +:a9561 :a9561 :a9561. +:a9562 :a9562 :a9562. +:a9563 :a9563 :a9563. +:a9564 :a9564 :a9564. +:a9565 :a9565 :a9565. +:a9566 :a9566 :a9566. +:a9567 :a9567 :a9567. +:a9568 :a9568 :a9568. +:a9569 :a9569 :a9569. +:a9570 :a9570 :a9570. +:a9571 :a9571 :a9571. +:a9572 :a9572 :a9572. +:a9573 :a9573 :a9573. +:a9574 :a9574 :a9574. +:a9575 :a9575 :a9575. +:a9576 :a9576 :a9576. +:a9577 :a9577 :a9577. +:a9578 :a9578 :a9578. +:a9579 :a9579 :a9579. +:a9580 :a9580 :a9580. +:a9581 :a9581 :a9581. +:a9582 :a9582 :a9582. +:a9583 :a9583 :a9583. +:a9584 :a9584 :a9584. +:a9585 :a9585 :a9585. +:a9586 :a9586 :a9586. +:a9587 :a9587 :a9587. +:a9588 :a9588 :a9588. +:a9589 :a9589 :a9589. +:a9590 :a9590 :a9590. +:a9591 :a9591 :a9591. +:a9592 :a9592 :a9592. +:a9593 :a9593 :a9593. +:a9594 :a9594 :a9594. +:a9595 :a9595 :a9595. +:a9596 :a9596 :a9596. +:a9597 :a9597 :a9597. +:a9598 :a9598 :a9598. +:a9599 :a9599 :a9599. +:a9600 :a9600 :a9600. +:a9601 :a9601 :a9601. +:a9602 :a9602 :a9602. +:a9603 :a9603 :a9603. +:a9604 :a9604 :a9604. +:a9605 :a9605 :a9605. +:a9606 :a9606 :a9606. +:a9607 :a9607 :a9607. +:a9608 :a9608 :a9608. +:a9609 :a9609 :a9609. +:a9610 :a9610 :a9610. +:a9611 :a9611 :a9611. +:a9612 :a9612 :a9612. +:a9613 :a9613 :a9613. +:a9614 :a9614 :a9614. +:a9615 :a9615 :a9615. +:a9616 :a9616 :a9616. +:a9617 :a9617 :a9617. +:a9618 :a9618 :a9618. +:a9619 :a9619 :a9619. +:a9620 :a9620 :a9620. +:a9621 :a9621 :a9621. +:a9622 :a9622 :a9622. +:a9623 :a9623 :a9623. +:a9624 :a9624 :a9624. +:a9625 :a9625 :a9625. +:a9626 :a9626 :a9626. +:a9627 :a9627 :a9627. +:a9628 :a9628 :a9628. +:a9629 :a9629 :a9629. +:a9630 :a9630 :a9630. +:a9631 :a9631 :a9631. +:a9632 :a9632 :a9632. +:a9633 :a9633 :a9633. +:a9634 :a9634 :a9634. +:a9635 :a9635 :a9635. +:a9636 :a9636 :a9636. +:a9637 :a9637 :a9637. +:a9638 :a9638 :a9638. +:a9639 :a9639 :a9639. +:a9640 :a9640 :a9640. +:a9641 :a9641 :a9641. +:a9642 :a9642 :a9642. +:a9643 :a9643 :a9643. +:a9644 :a9644 :a9644. +:a9645 :a9645 :a9645. +:a9646 :a9646 :a9646. +:a9647 :a9647 :a9647. +:a9648 :a9648 :a9648. +:a9649 :a9649 :a9649. +:a9650 :a9650 :a9650. +:a9651 :a9651 :a9651. +:a9652 :a9652 :a9652. +:a9653 :a9653 :a9653. +:a9654 :a9654 :a9654. +:a9655 :a9655 :a9655. +:a9656 :a9656 :a9656. +:a9657 :a9657 :a9657. +:a9658 :a9658 :a9658. +:a9659 :a9659 :a9659. +:a9660 :a9660 :a9660. +:a9661 :a9661 :a9661. +:a9662 :a9662 :a9662. +:a9663 :a9663 :a9663. +:a9664 :a9664 :a9664. +:a9665 :a9665 :a9665. +:a9666 :a9666 :a9666. +:a9667 :a9667 :a9667. +:a9668 :a9668 :a9668. +:a9669 :a9669 :a9669. +:a9670 :a9670 :a9670. +:a9671 :a9671 :a9671. +:a9672 :a9672 :a9672. +:a9673 :a9673 :a9673. +:a9674 :a9674 :a9674. +:a9675 :a9675 :a9675. +:a9676 :a9676 :a9676. +:a9677 :a9677 :a9677. +:a9678 :a9678 :a9678. +:a9679 :a9679 :a9679. +:a9680 :a9680 :a9680. +:a9681 :a9681 :a9681. +:a9682 :a9682 :a9682. +:a9683 :a9683 :a9683. +:a9684 :a9684 :a9684. +:a9685 :a9685 :a9685. +:a9686 :a9686 :a9686. +:a9687 :a9687 :a9687. +:a9688 :a9688 :a9688. +:a9689 :a9689 :a9689. +:a9690 :a9690 :a9690. +:a9691 :a9691 :a9691. +:a9692 :a9692 :a9692. +:a9693 :a9693 :a9693. +:a9694 :a9694 :a9694. +:a9695 :a9695 :a9695. +:a9696 :a9696 :a9696. +:a9697 :a9697 :a9697. +:a9698 :a9698 :a9698. +:a9699 :a9699 :a9699. +:a9700 :a9700 :a9700. +:a9701 :a9701 :a9701. +:a9702 :a9702 :a9702. +:a9703 :a9703 :a9703. +:a9704 :a9704 :a9704. +:a9705 :a9705 :a9705. +:a9706 :a9706 :a9706. +:a9707 :a9707 :a9707. +:a9708 :a9708 :a9708. +:a9709 :a9709 :a9709. +:a9710 :a9710 :a9710. +:a9711 :a9711 :a9711. +:a9712 :a9712 :a9712. +:a9713 :a9713 :a9713. +:a9714 :a9714 :a9714. +:a9715 :a9715 :a9715. +:a9716 :a9716 :a9716. +:a9717 :a9717 :a9717. +:a9718 :a9718 :a9718. +:a9719 :a9719 :a9719. +:a9720 :a9720 :a9720. +:a9721 :a9721 :a9721. +:a9722 :a9722 :a9722. +:a9723 :a9723 :a9723. +:a9724 :a9724 :a9724. +:a9725 :a9725 :a9725. +:a9726 :a9726 :a9726. +:a9727 :a9727 :a9727. +:a9728 :a9728 :a9728. +:a9729 :a9729 :a9729. +:a9730 :a9730 :a9730. +:a9731 :a9731 :a9731. +:a9732 :a9732 :a9732. +:a9733 :a9733 :a9733. +:a9734 :a9734 :a9734. +:a9735 :a9735 :a9735. +:a9736 :a9736 :a9736. +:a9737 :a9737 :a9737. +:a9738 :a9738 :a9738. +:a9739 :a9739 :a9739. +:a9740 :a9740 :a9740. +:a9741 :a9741 :a9741. +:a9742 :a9742 :a9742. +:a9743 :a9743 :a9743. +:a9744 :a9744 :a9744. +:a9745 :a9745 :a9745. +:a9746 :a9746 :a9746. +:a9747 :a9747 :a9747. +:a9748 :a9748 :a9748. +:a9749 :a9749 :a9749. +:a9750 :a9750 :a9750. +:a9751 :a9751 :a9751. +:a9752 :a9752 :a9752. +:a9753 :a9753 :a9753. +:a9754 :a9754 :a9754. +:a9755 :a9755 :a9755. +:a9756 :a9756 :a9756. +:a9757 :a9757 :a9757. +:a9758 :a9758 :a9758. +:a9759 :a9759 :a9759. +:a9760 :a9760 :a9760. +:a9761 :a9761 :a9761. +:a9762 :a9762 :a9762. +:a9763 :a9763 :a9763. +:a9764 :a9764 :a9764. +:a9765 :a9765 :a9765. +:a9766 :a9766 :a9766. +:a9767 :a9767 :a9767. +:a9768 :a9768 :a9768. +:a9769 :a9769 :a9769. +:a9770 :a9770 :a9770. +:a9771 :a9771 :a9771. +:a9772 :a9772 :a9772. +:a9773 :a9773 :a9773. +:a9774 :a9774 :a9774. +:a9775 :a9775 :a9775. +:a9776 :a9776 :a9776. +:a9777 :a9777 :a9777. +:a9778 :a9778 :a9778. +:a9779 :a9779 :a9779. +:a9780 :a9780 :a9780. +:a9781 :a9781 :a9781. +:a9782 :a9782 :a9782. +:a9783 :a9783 :a9783. +:a9784 :a9784 :a9784. +:a9785 :a9785 :a9785. +:a9786 :a9786 :a9786. +:a9787 :a9787 :a9787. +:a9788 :a9788 :a9788. +:a9789 :a9789 :a9789. +:a9790 :a9790 :a9790. +:a9791 :a9791 :a9791. +:a9792 :a9792 :a9792. +:a9793 :a9793 :a9793. +:a9794 :a9794 :a9794. +:a9795 :a9795 :a9795. +:a9796 :a9796 :a9796. +:a9797 :a9797 :a9797. +:a9798 :a9798 :a9798. +:a9799 :a9799 :a9799. +:a9800 :a9800 :a9800. +:a9801 :a9801 :a9801. +:a9802 :a9802 :a9802. +:a9803 :a9803 :a9803. +:a9804 :a9804 :a9804. +:a9805 :a9805 :a9805. +:a9806 :a9806 :a9806. +:a9807 :a9807 :a9807. +:a9808 :a9808 :a9808. +:a9809 :a9809 :a9809. +:a9810 :a9810 :a9810. +:a9811 :a9811 :a9811. +:a9812 :a9812 :a9812. +:a9813 :a9813 :a9813. +:a9814 :a9814 :a9814. +:a9815 :a9815 :a9815. +:a9816 :a9816 :a9816. +:a9817 :a9817 :a9817. +:a9818 :a9818 :a9818. +:a9819 :a9819 :a9819. +:a9820 :a9820 :a9820. +:a9821 :a9821 :a9821. +:a9822 :a9822 :a9822. +:a9823 :a9823 :a9823. +:a9824 :a9824 :a9824. +:a9825 :a9825 :a9825. +:a9826 :a9826 :a9826. +:a9827 :a9827 :a9827. +:a9828 :a9828 :a9828. +:a9829 :a9829 :a9829. +:a9830 :a9830 :a9830. +:a9831 :a9831 :a9831. +:a9832 :a9832 :a9832. +:a9833 :a9833 :a9833. +:a9834 :a9834 :a9834. +:a9835 :a9835 :a9835. +:a9836 :a9836 :a9836. +:a9837 :a9837 :a9837. +:a9838 :a9838 :a9838. +:a9839 :a9839 :a9839. +:a9840 :a9840 :a9840. +:a9841 :a9841 :a9841. +:a9842 :a9842 :a9842. +:a9843 :a9843 :a9843. +:a9844 :a9844 :a9844. +:a9845 :a9845 :a9845. +:a9846 :a9846 :a9846. +:a9847 :a9847 :a9847. +:a9848 :a9848 :a9848. +:a9849 :a9849 :a9849. +:a9850 :a9850 :a9850. +:a9851 :a9851 :a9851. +:a9852 :a9852 :a9852. +:a9853 :a9853 :a9853. +:a9854 :a9854 :a9854. +:a9855 :a9855 :a9855. +:a9856 :a9856 :a9856. +:a9857 :a9857 :a9857. +:a9858 :a9858 :a9858. +:a9859 :a9859 :a9859. +:a9860 :a9860 :a9860. +:a9861 :a9861 :a9861. +:a9862 :a9862 :a9862. +:a9863 :a9863 :a9863. +:a9864 :a9864 :a9864. +:a9865 :a9865 :a9865. +:a9866 :a9866 :a9866. +:a9867 :a9867 :a9867. +:a9868 :a9868 :a9868. +:a9869 :a9869 :a9869. +:a9870 :a9870 :a9870. +:a9871 :a9871 :a9871. +:a9872 :a9872 :a9872. +:a9873 :a9873 :a9873. +:a9874 :a9874 :a9874. +:a9875 :a9875 :a9875. +:a9876 :a9876 :a9876. +:a9877 :a9877 :a9877. +:a9878 :a9878 :a9878. +:a9879 :a9879 :a9879. +:a9880 :a9880 :a9880. +:a9881 :a9881 :a9881. +:a9882 :a9882 :a9882. +:a9883 :a9883 :a9883. +:a9884 :a9884 :a9884. +:a9885 :a9885 :a9885. +:a9886 :a9886 :a9886. +:a9887 :a9887 :a9887. +:a9888 :a9888 :a9888. +:a9889 :a9889 :a9889. +:a9890 :a9890 :a9890. +:a9891 :a9891 :a9891. +:a9892 :a9892 :a9892. +:a9893 :a9893 :a9893. +:a9894 :a9894 :a9894. +:a9895 :a9895 :a9895. +:a9896 :a9896 :a9896. +:a9897 :a9897 :a9897. +:a9898 :a9898 :a9898. +:a9899 :a9899 :a9899. +:a9900 :a9900 :a9900. +:a9901 :a9901 :a9901. +:a9902 :a9902 :a9902. +:a9903 :a9903 :a9903. +:a9904 :a9904 :a9904. +:a9905 :a9905 :a9905. +:a9906 :a9906 :a9906. +:a9907 :a9907 :a9907. +:a9908 :a9908 :a9908. +:a9909 :a9909 :a9909. +:a9910 :a9910 :a9910. +:a9911 :a9911 :a9911. +:a9912 :a9912 :a9912. +:a9913 :a9913 :a9913. +:a9914 :a9914 :a9914. +:a9915 :a9915 :a9915. +:a9916 :a9916 :a9916. +:a9917 :a9917 :a9917. +:a9918 :a9918 :a9918. +:a9919 :a9919 :a9919. +:a9920 :a9920 :a9920. +:a9921 :a9921 :a9921. +:a9922 :a9922 :a9922. +:a9923 :a9923 :a9923. +:a9924 :a9924 :a9924. +:a9925 :a9925 :a9925. +:a9926 :a9926 :a9926. +:a9927 :a9927 :a9927. +:a9928 :a9928 :a9928. +:a9929 :a9929 :a9929. +:a9930 :a9930 :a9930. +:a9931 :a9931 :a9931. +:a9932 :a9932 :a9932. +:a9933 :a9933 :a9933. +:a9934 :a9934 :a9934. +:a9935 :a9935 :a9935. +:a9936 :a9936 :a9936. +:a9937 :a9937 :a9937. +:a9938 :a9938 :a9938. +:a9939 :a9939 :a9939. +:a9940 :a9940 :a9940. +:a9941 :a9941 :a9941. +:a9942 :a9942 :a9942. +:a9943 :a9943 :a9943. +:a9944 :a9944 :a9944. +:a9945 :a9945 :a9945. +:a9946 :a9946 :a9946. +:a9947 :a9947 :a9947. +:a9948 :a9948 :a9948. +:a9949 :a9949 :a9949. +:a9950 :a9950 :a9950. +:a9951 :a9951 :a9951. +:a9952 :a9952 :a9952. +:a9953 :a9953 :a9953. +:a9954 :a9954 :a9954. +:a9955 :a9955 :a9955. +:a9956 :a9956 :a9956. +:a9957 :a9957 :a9957. +:a9958 :a9958 :a9958. +:a9959 :a9959 :a9959. +:a9960 :a9960 :a9960. +:a9961 :a9961 :a9961. +:a9962 :a9962 :a9962. +:a9963 :a9963 :a9963. +:a9964 :a9964 :a9964. +:a9965 :a9965 :a9965. +:a9966 :a9966 :a9966. +:a9967 :a9967 :a9967. +:a9968 :a9968 :a9968. +:a9969 :a9969 :a9969. +:a9970 :a9970 :a9970. +:a9971 :a9971 :a9971. +:a9972 :a9972 :a9972. +:a9973 :a9973 :a9973. +:a9974 :a9974 :a9974. +:a9975 :a9975 :a9975. +:a9976 :a9976 :a9976. +:a9977 :a9977 :a9977. +:a9978 :a9978 :a9978. +:a9979 :a9979 :a9979. +:a9980 :a9980 :a9980. +:a9981 :a9981 :a9981. +:a9982 :a9982 :a9982. +:a9983 :a9983 :a9983. +:a9984 :a9984 :a9984. +:a9985 :a9985 :a9985. +:a9986 :a9986 :a9986. +:a9987 :a9987 :a9987. +:a9988 :a9988 :a9988. +:a9989 :a9989 :a9989. +:a9990 :a9990 :a9990. +:a9991 :a9991 :a9991. +:a9992 :a9992 :a9992. +:a9993 :a9993 :a9993. +:a9994 :a9994 :a9994. +:a9995 :a9995 :a9995. +:a9996 :a9996 :a9996. +:a9997 :a9997 :a9997. +:a9998 :a9998 :a9998. +:a9999 :a9999 :a9999. +:a10000 :a10000 :a10000. diff --git a/testsuite/serd-tests/good/test-15.nt b/testsuite/serd-tests/good/test-15.nt new file mode 100644 index 00000000..c0604b3b --- /dev/null +++ b/testsuite/serd-tests/good/test-15.nt @@ -0,0 +1,10000 @@ + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . diff --git a/testsuite/serd-tests/good/test-15.ttl b/testsuite/serd-tests/good/test-15.ttl new file mode 100644 index 00000000..10df17df --- /dev/null +++ b/testsuite/serd-tests/good/test-15.ttl @@ -0,0 +1,3 @@ +# 10000 triple objects (10000 triples) more than the default Bison stack size +@prefix : . +:a :b :c1, :c2, :c3, :c4, :c5, :c6, :c7, :c8, :c9, :c10, :c11, :c12, :c13, :c14, :c15, :c16, :c17, :c18, :c19, :c20, :c21, :c22, :c23, :c24, :c25, :c26, :c27, :c28, :c29, :c30, :c31, :c32, :c33, :c34, :c35, :c36, :c37, :c38, :c39, :c40, :c41, :c42, :c43, :c44, :c45, :c46, :c47, :c48, :c49, :c50, :c51, :c52, :c53, :c54, :c55, :c56, :c57, :c58, :c59, :c60, :c61, :c62, :c63, :c64, :c65, :c66, :c67, :c68, :c69, :c70, :c71, :c72, :c73, :c74, :c75, :c76, :c77, :c78, :c79, :c80, :c81, :c82, :c83, :c84, :c85, :c86, :c87, :c88, :c89, :c90, :c91, :c92, :c93, :c94, :c95, :c96, :c97, :c98, :c99, :c100, :c101, :c102, :c103, :c104, :c105, :c106, :c107, :c108, :c109, :c110, :c111, :c112, :c113, :c114, :c115, :c116, :c117, :c118, :c119, :c120, :c121, :c122, :c123, :c124, :c125, :c126, :c127, :c128, :c129, :c130, :c131, :c132, :c133, :c134, :c135, :c136, :c137, :c138, :c139, :c140, :c141, :c142, :c143, :c144, :c145, :c146, :c147, :c148, :c149, :c150, :c151, :c152, :c153, :c154, :c155, :c156, :c157, :c158, :c159, :c160, :c161, :c162, :c163, :c164, :c165, :c166, :c167, :c168, :c169, :c170, :c171, :c172, :c173, :c174, :c175, :c176, :c177, :c178, :c179, :c180, :c181, :c182, :c183, :c184, :c185, :c186, :c187, :c188, :c189, :c190, :c191, :c192, :c193, :c194, :c195, :c196, :c197, :c198, :c199, :c200, :c201, :c202, :c203, :c204, :c205, :c206, :c207, :c208, :c209, :c210, :c211, :c212, :c213, :c214, :c215, :c216, :c217, :c218, :c219, :c220, :c221, :c222, :c223, :c224, :c225, :c226, :c227, :c228, :c229, :c230, :c231, :c232, :c233, :c234, :c235, :c236, :c237, :c238, :c239, :c240, :c241, :c242, :c243, :c244, :c245, :c246, :c247, :c248, :c249, :c250, :c251, :c252, :c253, :c254, :c255, :c256, :c257, :c258, :c259, :c260, :c261, :c262, :c263, :c264, :c265, :c266, :c267, :c268, :c269, :c270, :c271, :c272, :c273, :c274, :c275, :c276, :c277, :c278, :c279, :c280, :c281, :c282, :c283, :c284, :c285, :c286, :c287, :c288, :c289, :c290, :c291, :c292, :c293, :c294, :c295, :c296, :c297, :c298, :c299, :c300, :c301, :c302, :c303, :c304, :c305, :c306, :c307, :c308, :c309, :c310, :c311, :c312, :c313, :c314, :c315, :c316, :c317, :c318, :c319, :c320, :c321, :c322, :c323, :c324, :c325, :c326, :c327, :c328, :c329, :c330, :c331, :c332, :c333, :c334, :c335, :c336, :c337, :c338, :c339, :c340, :c341, :c342, :c343, :c344, :c345, :c346, :c347, :c348, :c349, :c350, :c351, :c352, :c353, :c354, :c355, :c356, :c357, :c358, :c359, :c360, :c361, :c362, :c363, :c364, :c365, :c366, :c367, :c368, :c369, :c370, :c371, :c372, :c373, :c374, :c375, :c376, :c377, :c378, :c379, :c380, :c381, :c382, :c383, :c384, :c385, :c386, :c387, :c388, :c389, :c390, :c391, :c392, :c393, :c394, :c395, :c396, :c397, :c398, :c399, :c400, :c401, :c402, :c403, :c404, :c405, :c406, :c407, :c408, :c409, :c410, :c411, :c412, :c413, :c414, :c415, :c416, :c417, :c418, :c419, :c420, :c421, :c422, :c423, :c424, :c425, :c426, :c427, :c428, :c429, :c430, :c431, :c432, :c433, :c434, :c435, :c436, :c437, :c438, :c439, :c440, :c441, :c442, :c443, :c444, :c445, :c446, :c447, :c448, :c449, :c450, :c451, :c452, :c453, :c454, :c455, :c456, :c457, :c458, :c459, :c460, :c461, :c462, :c463, :c464, :c465, :c466, :c467, :c468, :c469, :c470, :c471, :c472, :c473, :c474, :c475, :c476, :c477, :c478, :c479, :c480, :c481, :c482, :c483, :c484, :c485, :c486, :c487, :c488, :c489, :c490, :c491, :c492, :c493, :c494, :c495, :c496, :c497, :c498, :c499, :c500, :c501, :c502, :c503, :c504, :c505, :c506, :c507, :c508, :c509, :c510, :c511, :c512, :c513, :c514, :c515, :c516, :c517, :c518, :c519, :c520, :c521, :c522, :c523, :c524, :c525, :c526, :c527, :c528, :c529, :c530, :c531, :c532, :c533, :c534, :c535, :c536, :c537, :c538, :c539, :c540, :c541, :c542, :c543, :c544, :c545, :c546, :c547, :c548, :c549, :c550, :c551, :c552, :c553, :c554, :c555, :c556, :c557, :c558, :c559, :c560, :c561, :c562, :c563, :c564, :c565, :c566, :c567, :c568, :c569, :c570, :c571, :c572, :c573, :c574, :c575, :c576, :c577, :c578, :c579, :c580, :c581, :c582, :c583, :c584, :c585, :c586, :c587, :c588, :c589, :c590, :c591, :c592, :c593, :c594, :c595, :c596, :c597, :c598, :c599, :c600, :c601, :c602, :c603, :c604, :c605, :c606, :c607, :c608, :c609, :c610, :c611, :c612, :c613, :c614, :c615, :c616, :c617, :c618, :c619, :c620, :c621, :c622, :c623, :c624, :c625, :c626, :c627, :c628, :c629, :c630, :c631, :c632, :c633, :c634, :c635, :c636, :c637, :c638, :c639, :c640, :c641, :c642, :c643, :c644, :c645, :c646, :c647, :c648, :c649, :c650, :c651, :c652, :c653, :c654, :c655, :c656, :c657, :c658, :c659, :c660, :c661, :c662, :c663, :c664, :c665, :c666, :c667, :c668, :c669, :c670, :c671, :c672, :c673, :c674, :c675, :c676, :c677, :c678, :c679, :c680, :c681, :c682, :c683, :c684, :c685, :c686, :c687, :c688, :c689, :c690, :c691, :c692, :c693, :c694, :c695, :c696, :c697, :c698, :c699, :c700, :c701, :c702, :c703, :c704, :c705, :c706, :c707, :c708, :c709, :c710, :c711, :c712, :c713, :c714, :c715, :c716, :c717, :c718, :c719, :c720, :c721, :c722, :c723, :c724, :c725, :c726, :c727, :c728, :c729, :c730, :c731, :c732, :c733, :c734, :c735, :c736, :c737, :c738, :c739, :c740, :c741, :c742, :c743, :c744, :c745, :c746, :c747, :c748, :c749, :c750, :c751, :c752, :c753, :c754, :c755, :c756, :c757, :c758, :c759, :c760, :c761, :c762, :c763, :c764, :c765, :c766, :c767, :c768, :c769, :c770, :c771, :c772, :c773, :c774, :c775, :c776, :c777, :c778, :c779, :c780, :c781, :c782, :c783, :c784, :c785, :c786, :c787, :c788, :c789, :c790, :c791, :c792, :c793, :c794, :c795, :c796, :c797, :c798, :c799, :c800, :c801, :c802, :c803, :c804, :c805, :c806, :c807, :c808, :c809, :c810, :c811, :c812, :c813, :c814, :c815, :c816, :c817, :c818, :c819, :c820, :c821, :c822, :c823, :c824, :c825, :c826, :c827, :c828, :c829, :c830, :c831, :c832, :c833, :c834, :c835, :c836, :c837, :c838, :c839, :c840, :c841, :c842, :c843, :c844, :c845, :c846, :c847, :c848, :c849, :c850, :c851, :c852, :c853, :c854, :c855, :c856, :c857, :c858, :c859, :c860, :c861, :c862, :c863, :c864, :c865, :c866, :c867, :c868, :c869, :c870, :c871, :c872, :c873, :c874, :c875, :c876, :c877, :c878, :c879, :c880, :c881, :c882, :c883, :c884, :c885, :c886, :c887, :c888, :c889, :c890, :c891, :c892, :c893, :c894, :c895, :c896, :c897, :c898, :c899, :c900, :c901, :c902, :c903, :c904, :c905, :c906, :c907, :c908, :c909, :c910, :c911, :c912, :c913, :c914, :c915, :c916, :c917, :c918, :c919, :c920, :c921, :c922, :c923, :c924, :c925, :c926, :c927, :c928, :c929, :c930, :c931, :c932, :c933, :c934, :c935, :c936, :c937, :c938, :c939, :c940, :c941, :c942, :c943, :c944, :c945, :c946, :c947, :c948, :c949, :c950, :c951, :c952, :c953, :c954, :c955, :c956, :c957, :c958, :c959, :c960, :c961, :c962, :c963, :c964, :c965, :c966, :c967, :c968, :c969, :c970, :c971, :c972, :c973, :c974, :c975, :c976, :c977, :c978, :c979, :c980, :c981, :c982, :c983, :c984, :c985, :c986, :c987, :c988, :c989, :c990, :c991, :c992, :c993, :c994, :c995, :c996, :c997, :c998, :c999, :c1000, :c1001, :c1002, :c1003, :c1004, :c1005, :c1006, :c1007, :c1008, :c1009, :c1010, :c1011, :c1012, :c1013, :c1014, :c1015, :c1016, :c1017, :c1018, :c1019, :c1020, :c1021, :c1022, :c1023, :c1024, :c1025, :c1026, :c1027, :c1028, :c1029, :c1030, :c1031, :c1032, :c1033, :c1034, :c1035, :c1036, :c1037, :c1038, :c1039, :c1040, :c1041, :c1042, :c1043, :c1044, :c1045, :c1046, :c1047, :c1048, :c1049, :c1050, :c1051, :c1052, :c1053, :c1054, :c1055, :c1056, :c1057, :c1058, :c1059, :c1060, :c1061, :c1062, :c1063, :c1064, :c1065, :c1066, :c1067, :c1068, :c1069, :c1070, :c1071, :c1072, :c1073, :c1074, :c1075, :c1076, :c1077, :c1078, :c1079, :c1080, :c1081, :c1082, :c1083, :c1084, :c1085, :c1086, :c1087, :c1088, :c1089, :c1090, :c1091, :c1092, :c1093, :c1094, :c1095, :c1096, :c1097, :c1098, :c1099, :c1100, :c1101, :c1102, :c1103, :c1104, :c1105, :c1106, :c1107, :c1108, :c1109, :c1110, :c1111, :c1112, :c1113, :c1114, :c1115, :c1116, :c1117, :c1118, :c1119, :c1120, :c1121, :c1122, :c1123, :c1124, :c1125, :c1126, :c1127, :c1128, :c1129, :c1130, :c1131, :c1132, :c1133, :c1134, :c1135, :c1136, :c1137, :c1138, :c1139, :c1140, :c1141, :c1142, :c1143, :c1144, :c1145, :c1146, :c1147, :c1148, :c1149, :c1150, :c1151, :c1152, :c1153, :c1154, :c1155, :c1156, :c1157, :c1158, :c1159, :c1160, :c1161, :c1162, :c1163, :c1164, :c1165, :c1166, :c1167, :c1168, :c1169, :c1170, :c1171, :c1172, :c1173, :c1174, :c1175, :c1176, :c1177, :c1178, :c1179, :c1180, :c1181, :c1182, :c1183, :c1184, :c1185, :c1186, :c1187, :c1188, :c1189, :c1190, :c1191, :c1192, :c1193, :c1194, :c1195, :c1196, :c1197, :c1198, :c1199, :c1200, :c1201, :c1202, :c1203, :c1204, :c1205, :c1206, :c1207, :c1208, :c1209, :c1210, :c1211, :c1212, :c1213, :c1214, :c1215, :c1216, :c1217, :c1218, :c1219, :c1220, :c1221, :c1222, :c1223, :c1224, :c1225, :c1226, :c1227, :c1228, :c1229, :c1230, :c1231, :c1232, :c1233, :c1234, :c1235, :c1236, :c1237, :c1238, :c1239, :c1240, :c1241, :c1242, :c1243, :c1244, :c1245, :c1246, :c1247, :c1248, :c1249, :c1250, :c1251, :c1252, :c1253, :c1254, :c1255, :c1256, :c1257, :c1258, :c1259, :c1260, :c1261, :c1262, :c1263, :c1264, :c1265, :c1266, :c1267, :c1268, :c1269, :c1270, :c1271, :c1272, :c1273, :c1274, :c1275, :c1276, :c1277, :c1278, :c1279, :c1280, :c1281, :c1282, :c1283, :c1284, :c1285, :c1286, :c1287, :c1288, :c1289, :c1290, :c1291, :c1292, :c1293, :c1294, :c1295, :c1296, :c1297, :c1298, :c1299, :c1300, :c1301, :c1302, :c1303, :c1304, :c1305, :c1306, :c1307, :c1308, :c1309, :c1310, :c1311, :c1312, :c1313, :c1314, :c1315, :c1316, :c1317, :c1318, :c1319, :c1320, :c1321, :c1322, :c1323, :c1324, :c1325, :c1326, :c1327, :c1328, :c1329, :c1330, :c1331, :c1332, :c1333, :c1334, :c1335, :c1336, :c1337, :c1338, :c1339, :c1340, :c1341, :c1342, :c1343, :c1344, :c1345, :c1346, :c1347, :c1348, :c1349, :c1350, :c1351, :c1352, :c1353, :c1354, :c1355, :c1356, :c1357, :c1358, :c1359, :c1360, :c1361, :c1362, :c1363, :c1364, :c1365, :c1366, :c1367, :c1368, :c1369, :c1370, :c1371, :c1372, :c1373, :c1374, :c1375, :c1376, :c1377, :c1378, :c1379, :c1380, :c1381, :c1382, :c1383, :c1384, :c1385, :c1386, :c1387, :c1388, :c1389, :c1390, :c1391, :c1392, :c1393, :c1394, :c1395, :c1396, :c1397, :c1398, :c1399, :c1400, :c1401, :c1402, :c1403, :c1404, :c1405, :c1406, :c1407, :c1408, :c1409, :c1410, :c1411, :c1412, :c1413, :c1414, :c1415, :c1416, :c1417, :c1418, :c1419, :c1420, :c1421, :c1422, :c1423, :c1424, :c1425, :c1426, :c1427, :c1428, :c1429, :c1430, :c1431, :c1432, :c1433, :c1434, :c1435, :c1436, :c1437, :c1438, :c1439, :c1440, :c1441, :c1442, :c1443, :c1444, :c1445, :c1446, :c1447, :c1448, :c1449, :c1450, :c1451, :c1452, :c1453, :c1454, :c1455, :c1456, :c1457, :c1458, :c1459, :c1460, :c1461, :c1462, :c1463, :c1464, :c1465, :c1466, :c1467, :c1468, :c1469, :c1470, :c1471, :c1472, :c1473, :c1474, :c1475, :c1476, :c1477, :c1478, :c1479, :c1480, :c1481, :c1482, :c1483, :c1484, :c1485, :c1486, :c1487, :c1488, :c1489, :c1490, :c1491, :c1492, :c1493, :c1494, :c1495, :c1496, :c1497, :c1498, :c1499, :c1500, :c1501, :c1502, :c1503, :c1504, :c1505, :c1506, :c1507, :c1508, :c1509, :c1510, :c1511, :c1512, :c1513, :c1514, :c1515, :c1516, :c1517, :c1518, :c1519, :c1520, :c1521, :c1522, :c1523, :c1524, :c1525, :c1526, :c1527, :c1528, :c1529, :c1530, :c1531, :c1532, :c1533, :c1534, :c1535, :c1536, :c1537, :c1538, :c1539, :c1540, :c1541, :c1542, :c1543, :c1544, :c1545, :c1546, :c1547, :c1548, :c1549, :c1550, :c1551, :c1552, :c1553, :c1554, :c1555, :c1556, :c1557, :c1558, :c1559, :c1560, :c1561, :c1562, :c1563, :c1564, :c1565, :c1566, :c1567, :c1568, :c1569, :c1570, :c1571, :c1572, :c1573, :c1574, :c1575, :c1576, :c1577, :c1578, :c1579, :c1580, :c1581, :c1582, :c1583, :c1584, :c1585, :c1586, :c1587, :c1588, :c1589, :c1590, :c1591, :c1592, :c1593, :c1594, :c1595, :c1596, :c1597, :c1598, :c1599, :c1600, :c1601, :c1602, :c1603, :c1604, :c1605, :c1606, :c1607, :c1608, :c1609, :c1610, :c1611, :c1612, :c1613, :c1614, :c1615, :c1616, :c1617, :c1618, :c1619, :c1620, :c1621, :c1622, :c1623, :c1624, :c1625, :c1626, :c1627, :c1628, :c1629, :c1630, :c1631, :c1632, :c1633, :c1634, :c1635, :c1636, :c1637, :c1638, :c1639, :c1640, :c1641, :c1642, :c1643, :c1644, :c1645, :c1646, :c1647, :c1648, :c1649, :c1650, :c1651, :c1652, :c1653, :c1654, :c1655, :c1656, :c1657, :c1658, :c1659, :c1660, :c1661, :c1662, :c1663, :c1664, :c1665, :c1666, :c1667, :c1668, :c1669, :c1670, :c1671, :c1672, :c1673, :c1674, :c1675, :c1676, :c1677, :c1678, :c1679, :c1680, :c1681, :c1682, :c1683, :c1684, :c1685, :c1686, :c1687, :c1688, :c1689, :c1690, :c1691, :c1692, :c1693, :c1694, :c1695, :c1696, :c1697, :c1698, :c1699, :c1700, :c1701, :c1702, :c1703, :c1704, :c1705, :c1706, :c1707, :c1708, :c1709, :c1710, :c1711, :c1712, :c1713, :c1714, :c1715, :c1716, :c1717, :c1718, :c1719, :c1720, :c1721, :c1722, :c1723, :c1724, :c1725, :c1726, :c1727, :c1728, :c1729, :c1730, :c1731, :c1732, :c1733, :c1734, :c1735, :c1736, :c1737, :c1738, :c1739, :c1740, :c1741, :c1742, :c1743, :c1744, :c1745, :c1746, :c1747, :c1748, :c1749, :c1750, :c1751, :c1752, :c1753, :c1754, :c1755, :c1756, :c1757, :c1758, :c1759, :c1760, :c1761, :c1762, :c1763, :c1764, :c1765, :c1766, :c1767, :c1768, :c1769, :c1770, :c1771, :c1772, :c1773, :c1774, :c1775, :c1776, :c1777, :c1778, :c1779, :c1780, :c1781, :c1782, :c1783, :c1784, :c1785, :c1786, :c1787, :c1788, :c1789, :c1790, :c1791, :c1792, :c1793, :c1794, :c1795, :c1796, :c1797, :c1798, :c1799, :c1800, :c1801, :c1802, :c1803, :c1804, :c1805, :c1806, :c1807, :c1808, :c1809, :c1810, :c1811, :c1812, :c1813, :c1814, :c1815, :c1816, :c1817, :c1818, :c1819, :c1820, :c1821, :c1822, :c1823, :c1824, :c1825, :c1826, :c1827, :c1828, :c1829, :c1830, :c1831, :c1832, :c1833, :c1834, :c1835, :c1836, :c1837, :c1838, :c1839, :c1840, :c1841, :c1842, :c1843, :c1844, :c1845, :c1846, :c1847, :c1848, :c1849, :c1850, :c1851, :c1852, :c1853, :c1854, :c1855, :c1856, :c1857, :c1858, :c1859, :c1860, :c1861, :c1862, :c1863, :c1864, :c1865, :c1866, :c1867, :c1868, :c1869, :c1870, :c1871, :c1872, :c1873, :c1874, :c1875, :c1876, :c1877, :c1878, :c1879, :c1880, :c1881, :c1882, :c1883, :c1884, :c1885, :c1886, :c1887, :c1888, :c1889, :c1890, :c1891, :c1892, :c1893, :c1894, :c1895, :c1896, :c1897, :c1898, :c1899, :c1900, :c1901, :c1902, :c1903, :c1904, :c1905, :c1906, :c1907, :c1908, :c1909, :c1910, :c1911, :c1912, :c1913, :c1914, :c1915, :c1916, :c1917, :c1918, :c1919, :c1920, :c1921, :c1922, :c1923, :c1924, :c1925, :c1926, :c1927, :c1928, :c1929, :c1930, :c1931, :c1932, :c1933, :c1934, :c1935, :c1936, :c1937, :c1938, :c1939, :c1940, :c1941, :c1942, :c1943, :c1944, :c1945, :c1946, :c1947, :c1948, :c1949, :c1950, :c1951, :c1952, :c1953, :c1954, :c1955, :c1956, :c1957, :c1958, :c1959, :c1960, :c1961, :c1962, :c1963, :c1964, :c1965, :c1966, :c1967, :c1968, :c1969, :c1970, :c1971, :c1972, :c1973, :c1974, :c1975, :c1976, :c1977, :c1978, :c1979, :c1980, :c1981, :c1982, :c1983, :c1984, :c1985, :c1986, :c1987, :c1988, :c1989, :c1990, :c1991, :c1992, :c1993, :c1994, :c1995, :c1996, :c1997, :c1998, :c1999, :c2000, :c2001, :c2002, :c2003, :c2004, :c2005, :c2006, :c2007, :c2008, :c2009, :c2010, :c2011, :c2012, :c2013, :c2014, :c2015, :c2016, :c2017, :c2018, :c2019, :c2020, :c2021, :c2022, :c2023, :c2024, :c2025, :c2026, :c2027, :c2028, :c2029, :c2030, :c2031, :c2032, :c2033, :c2034, :c2035, :c2036, :c2037, :c2038, :c2039, :c2040, :c2041, :c2042, :c2043, :c2044, :c2045, :c2046, :c2047, :c2048, :c2049, :c2050, :c2051, :c2052, :c2053, :c2054, :c2055, :c2056, :c2057, :c2058, :c2059, :c2060, :c2061, :c2062, :c2063, :c2064, :c2065, :c2066, :c2067, :c2068, :c2069, :c2070, :c2071, :c2072, :c2073, :c2074, :c2075, :c2076, :c2077, :c2078, :c2079, :c2080, :c2081, :c2082, :c2083, :c2084, :c2085, :c2086, :c2087, :c2088, :c2089, :c2090, :c2091, :c2092, :c2093, :c2094, :c2095, :c2096, :c2097, :c2098, :c2099, :c2100, :c2101, :c2102, :c2103, :c2104, :c2105, :c2106, :c2107, :c2108, :c2109, :c2110, :c2111, :c2112, :c2113, :c2114, :c2115, :c2116, :c2117, :c2118, :c2119, :c2120, :c2121, :c2122, :c2123, :c2124, :c2125, :c2126, :c2127, :c2128, :c2129, :c2130, :c2131, :c2132, :c2133, :c2134, :c2135, :c2136, :c2137, :c2138, :c2139, :c2140, :c2141, :c2142, :c2143, :c2144, :c2145, :c2146, :c2147, :c2148, :c2149, :c2150, :c2151, :c2152, :c2153, :c2154, :c2155, :c2156, :c2157, :c2158, :c2159, :c2160, :c2161, :c2162, :c2163, :c2164, :c2165, :c2166, :c2167, :c2168, :c2169, :c2170, :c2171, :c2172, :c2173, :c2174, :c2175, :c2176, :c2177, :c2178, :c2179, :c2180, :c2181, :c2182, :c2183, :c2184, :c2185, :c2186, :c2187, :c2188, :c2189, :c2190, :c2191, :c2192, :c2193, :c2194, :c2195, :c2196, :c2197, :c2198, :c2199, :c2200, :c2201, :c2202, :c2203, :c2204, :c2205, :c2206, :c2207, :c2208, :c2209, :c2210, :c2211, :c2212, :c2213, :c2214, :c2215, :c2216, :c2217, :c2218, :c2219, :c2220, :c2221, :c2222, :c2223, :c2224, :c2225, :c2226, :c2227, :c2228, :c2229, :c2230, :c2231, :c2232, :c2233, :c2234, :c2235, :c2236, :c2237, :c2238, :c2239, :c2240, :c2241, :c2242, :c2243, :c2244, :c2245, :c2246, :c2247, :c2248, :c2249, :c2250, :c2251, :c2252, :c2253, :c2254, :c2255, :c2256, :c2257, :c2258, :c2259, :c2260, :c2261, :c2262, :c2263, :c2264, :c2265, :c2266, :c2267, :c2268, :c2269, :c2270, :c2271, :c2272, :c2273, :c2274, :c2275, :c2276, :c2277, :c2278, :c2279, :c2280, :c2281, :c2282, :c2283, :c2284, :c2285, :c2286, :c2287, :c2288, :c2289, :c2290, :c2291, :c2292, :c2293, :c2294, :c2295, :c2296, :c2297, :c2298, :c2299, :c2300, :c2301, :c2302, :c2303, :c2304, :c2305, :c2306, :c2307, :c2308, :c2309, :c2310, :c2311, :c2312, :c2313, :c2314, :c2315, :c2316, :c2317, :c2318, :c2319, :c2320, :c2321, :c2322, :c2323, :c2324, :c2325, :c2326, :c2327, :c2328, :c2329, :c2330, :c2331, :c2332, :c2333, :c2334, :c2335, :c2336, :c2337, :c2338, :c2339, :c2340, :c2341, :c2342, :c2343, :c2344, :c2345, :c2346, :c2347, :c2348, :c2349, :c2350, :c2351, :c2352, :c2353, :c2354, :c2355, :c2356, :c2357, :c2358, :c2359, :c2360, :c2361, :c2362, :c2363, :c2364, :c2365, :c2366, :c2367, :c2368, :c2369, :c2370, :c2371, :c2372, :c2373, :c2374, :c2375, :c2376, :c2377, :c2378, :c2379, :c2380, :c2381, :c2382, :c2383, :c2384, :c2385, :c2386, :c2387, :c2388, :c2389, :c2390, :c2391, :c2392, :c2393, :c2394, :c2395, :c2396, :c2397, :c2398, :c2399, :c2400, :c2401, :c2402, :c2403, :c2404, :c2405, :c2406, :c2407, :c2408, :c2409, :c2410, :c2411, :c2412, :c2413, :c2414, :c2415, :c2416, :c2417, :c2418, :c2419, :c2420, :c2421, :c2422, :c2423, :c2424, :c2425, :c2426, :c2427, :c2428, :c2429, :c2430, :c2431, :c2432, :c2433, :c2434, :c2435, :c2436, :c2437, :c2438, :c2439, :c2440, :c2441, :c2442, :c2443, :c2444, :c2445, :c2446, :c2447, :c2448, :c2449, :c2450, :c2451, :c2452, :c2453, :c2454, :c2455, :c2456, :c2457, :c2458, :c2459, :c2460, :c2461, :c2462, :c2463, :c2464, :c2465, :c2466, :c2467, :c2468, :c2469, :c2470, :c2471, :c2472, :c2473, :c2474, :c2475, :c2476, :c2477, :c2478, :c2479, :c2480, :c2481, :c2482, :c2483, :c2484, :c2485, :c2486, :c2487, :c2488, :c2489, :c2490, :c2491, :c2492, :c2493, :c2494, :c2495, :c2496, :c2497, :c2498, :c2499, :c2500, :c2501, :c2502, :c2503, :c2504, :c2505, :c2506, :c2507, :c2508, :c2509, :c2510, :c2511, :c2512, :c2513, :c2514, :c2515, :c2516, :c2517, :c2518, :c2519, :c2520, :c2521, :c2522, :c2523, :c2524, :c2525, :c2526, :c2527, :c2528, :c2529, :c2530, :c2531, :c2532, :c2533, :c2534, :c2535, :c2536, :c2537, :c2538, :c2539, :c2540, :c2541, :c2542, :c2543, :c2544, :c2545, :c2546, :c2547, :c2548, :c2549, :c2550, :c2551, :c2552, :c2553, :c2554, :c2555, :c2556, :c2557, :c2558, :c2559, :c2560, :c2561, :c2562, :c2563, :c2564, :c2565, :c2566, :c2567, :c2568, :c2569, :c2570, :c2571, :c2572, :c2573, :c2574, :c2575, :c2576, :c2577, :c2578, :c2579, :c2580, :c2581, :c2582, :c2583, :c2584, :c2585, :c2586, :c2587, :c2588, :c2589, :c2590, :c2591, :c2592, :c2593, :c2594, :c2595, :c2596, :c2597, :c2598, :c2599, :c2600, :c2601, :c2602, :c2603, :c2604, :c2605, :c2606, :c2607, :c2608, :c2609, :c2610, :c2611, :c2612, :c2613, :c2614, :c2615, :c2616, :c2617, :c2618, :c2619, :c2620, :c2621, :c2622, :c2623, :c2624, :c2625, :c2626, :c2627, :c2628, :c2629, :c2630, :c2631, :c2632, :c2633, :c2634, :c2635, :c2636, :c2637, :c2638, :c2639, :c2640, :c2641, :c2642, :c2643, :c2644, :c2645, :c2646, :c2647, :c2648, :c2649, :c2650, :c2651, :c2652, :c2653, :c2654, :c2655, :c2656, :c2657, :c2658, :c2659, :c2660, :c2661, :c2662, :c2663, :c2664, :c2665, :c2666, :c2667, :c2668, :c2669, :c2670, :c2671, :c2672, :c2673, :c2674, :c2675, :c2676, :c2677, :c2678, :c2679, :c2680, :c2681, :c2682, :c2683, :c2684, :c2685, :c2686, :c2687, :c2688, :c2689, :c2690, :c2691, :c2692, :c2693, :c2694, :c2695, :c2696, :c2697, :c2698, :c2699, :c2700, :c2701, :c2702, :c2703, :c2704, :c2705, :c2706, :c2707, :c2708, :c2709, :c2710, :c2711, :c2712, :c2713, :c2714, :c2715, :c2716, :c2717, :c2718, :c2719, :c2720, :c2721, :c2722, :c2723, :c2724, :c2725, :c2726, :c2727, :c2728, :c2729, :c2730, :c2731, :c2732, :c2733, :c2734, :c2735, :c2736, :c2737, :c2738, :c2739, :c2740, :c2741, :c2742, :c2743, :c2744, :c2745, :c2746, :c2747, :c2748, :c2749, :c2750, :c2751, :c2752, :c2753, :c2754, :c2755, :c2756, :c2757, :c2758, :c2759, :c2760, :c2761, :c2762, :c2763, :c2764, :c2765, :c2766, :c2767, :c2768, :c2769, :c2770, :c2771, :c2772, :c2773, :c2774, :c2775, :c2776, :c2777, :c2778, :c2779, :c2780, :c2781, :c2782, :c2783, :c2784, :c2785, :c2786, :c2787, :c2788, :c2789, :c2790, :c2791, :c2792, :c2793, :c2794, :c2795, :c2796, :c2797, :c2798, :c2799, :c2800, :c2801, :c2802, :c2803, :c2804, :c2805, :c2806, :c2807, :c2808, :c2809, :c2810, :c2811, :c2812, :c2813, :c2814, :c2815, :c2816, :c2817, :c2818, :c2819, :c2820, :c2821, :c2822, :c2823, :c2824, :c2825, :c2826, :c2827, :c2828, :c2829, :c2830, :c2831, :c2832, :c2833, :c2834, :c2835, :c2836, :c2837, :c2838, :c2839, :c2840, :c2841, :c2842, :c2843, :c2844, :c2845, :c2846, :c2847, :c2848, :c2849, :c2850, :c2851, :c2852, :c2853, :c2854, :c2855, :c2856, :c2857, :c2858, :c2859, :c2860, :c2861, :c2862, :c2863, :c2864, :c2865, :c2866, :c2867, :c2868, :c2869, :c2870, :c2871, :c2872, :c2873, :c2874, :c2875, :c2876, :c2877, :c2878, :c2879, :c2880, :c2881, :c2882, :c2883, :c2884, :c2885, :c2886, :c2887, :c2888, :c2889, :c2890, :c2891, :c2892, :c2893, :c2894, :c2895, :c2896, :c2897, :c2898, :c2899, :c2900, :c2901, :c2902, :c2903, :c2904, :c2905, :c2906, :c2907, :c2908, :c2909, :c2910, :c2911, :c2912, :c2913, :c2914, :c2915, :c2916, :c2917, :c2918, :c2919, :c2920, :c2921, :c2922, :c2923, :c2924, :c2925, :c2926, :c2927, :c2928, :c2929, :c2930, :c2931, :c2932, :c2933, :c2934, :c2935, :c2936, :c2937, :c2938, :c2939, :c2940, :c2941, :c2942, :c2943, :c2944, :c2945, :c2946, :c2947, :c2948, :c2949, :c2950, :c2951, :c2952, :c2953, :c2954, :c2955, :c2956, :c2957, :c2958, :c2959, :c2960, :c2961, :c2962, :c2963, :c2964, :c2965, :c2966, :c2967, :c2968, :c2969, :c2970, :c2971, :c2972, :c2973, :c2974, :c2975, :c2976, :c2977, :c2978, :c2979, :c2980, :c2981, :c2982, :c2983, :c2984, :c2985, :c2986, :c2987, :c2988, :c2989, :c2990, :c2991, :c2992, :c2993, :c2994, :c2995, :c2996, :c2997, :c2998, :c2999, :c3000, :c3001, :c3002, :c3003, :c3004, :c3005, :c3006, :c3007, :c3008, :c3009, :c3010, :c3011, :c3012, :c3013, :c3014, :c3015, :c3016, :c3017, :c3018, :c3019, :c3020, :c3021, :c3022, :c3023, :c3024, :c3025, :c3026, :c3027, :c3028, :c3029, :c3030, :c3031, :c3032, :c3033, :c3034, :c3035, :c3036, :c3037, :c3038, :c3039, :c3040, :c3041, :c3042, :c3043, :c3044, :c3045, :c3046, :c3047, :c3048, :c3049, :c3050, :c3051, :c3052, :c3053, :c3054, :c3055, :c3056, :c3057, :c3058, :c3059, :c3060, :c3061, :c3062, :c3063, :c3064, :c3065, :c3066, :c3067, :c3068, :c3069, :c3070, :c3071, :c3072, :c3073, :c3074, :c3075, :c3076, :c3077, :c3078, :c3079, :c3080, :c3081, :c3082, :c3083, :c3084, :c3085, :c3086, :c3087, :c3088, :c3089, :c3090, :c3091, :c3092, :c3093, :c3094, :c3095, :c3096, :c3097, :c3098, :c3099, :c3100, :c3101, :c3102, :c3103, :c3104, :c3105, :c3106, :c3107, :c3108, :c3109, :c3110, :c3111, :c3112, :c3113, :c3114, :c3115, :c3116, :c3117, :c3118, :c3119, :c3120, :c3121, :c3122, :c3123, :c3124, :c3125, :c3126, :c3127, :c3128, :c3129, :c3130, :c3131, :c3132, :c3133, :c3134, :c3135, :c3136, :c3137, :c3138, :c3139, :c3140, :c3141, :c3142, :c3143, :c3144, :c3145, :c3146, :c3147, :c3148, :c3149, :c3150, :c3151, :c3152, :c3153, :c3154, :c3155, :c3156, :c3157, :c3158, :c3159, :c3160, :c3161, :c3162, :c3163, :c3164, :c3165, :c3166, :c3167, :c3168, :c3169, :c3170, :c3171, :c3172, :c3173, :c3174, :c3175, :c3176, :c3177, :c3178, :c3179, :c3180, :c3181, :c3182, :c3183, :c3184, :c3185, :c3186, :c3187, :c3188, :c3189, :c3190, :c3191, :c3192, :c3193, :c3194, :c3195, :c3196, :c3197, :c3198, :c3199, :c3200, :c3201, :c3202, :c3203, :c3204, :c3205, :c3206, :c3207, :c3208, :c3209, :c3210, :c3211, :c3212, :c3213, :c3214, :c3215, :c3216, :c3217, :c3218, :c3219, :c3220, :c3221, :c3222, :c3223, :c3224, :c3225, :c3226, :c3227, :c3228, :c3229, :c3230, :c3231, :c3232, :c3233, :c3234, :c3235, :c3236, :c3237, :c3238, :c3239, :c3240, :c3241, :c3242, :c3243, :c3244, :c3245, :c3246, :c3247, :c3248, :c3249, :c3250, :c3251, :c3252, :c3253, :c3254, :c3255, :c3256, :c3257, :c3258, :c3259, :c3260, :c3261, :c3262, :c3263, :c3264, :c3265, :c3266, :c3267, :c3268, :c3269, :c3270, :c3271, :c3272, :c3273, :c3274, :c3275, :c3276, :c3277, :c3278, :c3279, :c3280, :c3281, :c3282, :c3283, :c3284, :c3285, :c3286, :c3287, :c3288, :c3289, :c3290, :c3291, :c3292, :c3293, :c3294, :c3295, :c3296, :c3297, :c3298, :c3299, :c3300, :c3301, :c3302, :c3303, :c3304, :c3305, :c3306, :c3307, :c3308, :c3309, :c3310, :c3311, :c3312, :c3313, :c3314, :c3315, :c3316, :c3317, :c3318, :c3319, :c3320, :c3321, :c3322, :c3323, :c3324, :c3325, :c3326, :c3327, :c3328, :c3329, :c3330, :c3331, :c3332, :c3333, :c3334, :c3335, :c3336, :c3337, :c3338, :c3339, :c3340, :c3341, :c3342, :c3343, :c3344, :c3345, :c3346, :c3347, :c3348, :c3349, :c3350, :c3351, :c3352, :c3353, :c3354, :c3355, :c3356, :c3357, :c3358, :c3359, :c3360, :c3361, :c3362, :c3363, :c3364, :c3365, :c3366, :c3367, :c3368, :c3369, :c3370, :c3371, :c3372, :c3373, :c3374, :c3375, :c3376, :c3377, :c3378, :c3379, :c3380, :c3381, :c3382, :c3383, :c3384, :c3385, :c3386, :c3387, :c3388, :c3389, :c3390, :c3391, :c3392, :c3393, :c3394, :c3395, :c3396, :c3397, :c3398, :c3399, :c3400, :c3401, :c3402, :c3403, :c3404, :c3405, :c3406, :c3407, :c3408, :c3409, :c3410, :c3411, :c3412, :c3413, :c3414, :c3415, :c3416, :c3417, :c3418, :c3419, :c3420, :c3421, :c3422, :c3423, :c3424, :c3425, :c3426, :c3427, :c3428, :c3429, :c3430, :c3431, :c3432, :c3433, :c3434, :c3435, :c3436, :c3437, :c3438, :c3439, :c3440, :c3441, :c3442, :c3443, :c3444, :c3445, :c3446, :c3447, :c3448, :c3449, :c3450, :c3451, :c3452, :c3453, :c3454, :c3455, :c3456, :c3457, :c3458, :c3459, :c3460, :c3461, :c3462, :c3463, :c3464, :c3465, :c3466, :c3467, :c3468, :c3469, :c3470, :c3471, :c3472, :c3473, :c3474, :c3475, :c3476, :c3477, :c3478, :c3479, :c3480, :c3481, :c3482, :c3483, :c3484, :c3485, :c3486, :c3487, :c3488, :c3489, :c3490, :c3491, :c3492, :c3493, :c3494, :c3495, :c3496, :c3497, :c3498, :c3499, :c3500, :c3501, :c3502, :c3503, :c3504, :c3505, :c3506, :c3507, :c3508, :c3509, :c3510, :c3511, :c3512, :c3513, :c3514, :c3515, :c3516, :c3517, :c3518, :c3519, :c3520, :c3521, :c3522, :c3523, :c3524, :c3525, :c3526, :c3527, :c3528, :c3529, :c3530, :c3531, :c3532, :c3533, :c3534, :c3535, :c3536, :c3537, :c3538, :c3539, :c3540, :c3541, :c3542, :c3543, :c3544, :c3545, :c3546, :c3547, :c3548, :c3549, :c3550, :c3551, :c3552, :c3553, :c3554, :c3555, :c3556, :c3557, :c3558, :c3559, :c3560, :c3561, :c3562, :c3563, :c3564, :c3565, :c3566, :c3567, :c3568, :c3569, :c3570, :c3571, :c3572, :c3573, :c3574, :c3575, :c3576, :c3577, :c3578, :c3579, :c3580, :c3581, :c3582, :c3583, :c3584, :c3585, :c3586, :c3587, :c3588, :c3589, :c3590, :c3591, :c3592, :c3593, :c3594, :c3595, :c3596, :c3597, :c3598, :c3599, :c3600, :c3601, :c3602, :c3603, :c3604, :c3605, :c3606, :c3607, :c3608, :c3609, :c3610, :c3611, :c3612, :c3613, :c3614, :c3615, :c3616, :c3617, :c3618, :c3619, :c3620, :c3621, :c3622, :c3623, :c3624, :c3625, :c3626, :c3627, :c3628, :c3629, :c3630, :c3631, :c3632, :c3633, :c3634, :c3635, :c3636, :c3637, :c3638, :c3639, :c3640, :c3641, :c3642, :c3643, :c3644, :c3645, :c3646, :c3647, :c3648, :c3649, :c3650, :c3651, :c3652, :c3653, :c3654, :c3655, :c3656, :c3657, :c3658, :c3659, :c3660, :c3661, :c3662, :c3663, :c3664, :c3665, :c3666, :c3667, :c3668, :c3669, :c3670, :c3671, :c3672, :c3673, :c3674, :c3675, :c3676, :c3677, :c3678, :c3679, :c3680, :c3681, :c3682, :c3683, :c3684, :c3685, :c3686, :c3687, :c3688, :c3689, :c3690, :c3691, :c3692, :c3693, :c3694, :c3695, :c3696, :c3697, :c3698, :c3699, :c3700, :c3701, :c3702, :c3703, :c3704, :c3705, :c3706, :c3707, :c3708, :c3709, :c3710, :c3711, :c3712, :c3713, :c3714, :c3715, :c3716, :c3717, :c3718, :c3719, :c3720, :c3721, :c3722, :c3723, :c3724, :c3725, :c3726, :c3727, :c3728, :c3729, :c3730, :c3731, :c3732, :c3733, :c3734, :c3735, :c3736, :c3737, :c3738, :c3739, :c3740, :c3741, :c3742, :c3743, :c3744, :c3745, :c3746, :c3747, :c3748, :c3749, :c3750, :c3751, :c3752, :c3753, :c3754, :c3755, :c3756, :c3757, :c3758, :c3759, :c3760, :c3761, :c3762, :c3763, :c3764, :c3765, :c3766, :c3767, :c3768, :c3769, :c3770, :c3771, :c3772, :c3773, :c3774, :c3775, :c3776, :c3777, :c3778, :c3779, :c3780, :c3781, :c3782, :c3783, :c3784, :c3785, :c3786, :c3787, :c3788, :c3789, :c3790, :c3791, :c3792, :c3793, :c3794, :c3795, :c3796, :c3797, :c3798, :c3799, :c3800, :c3801, :c3802, :c3803, :c3804, :c3805, :c3806, :c3807, :c3808, :c3809, :c3810, :c3811, :c3812, :c3813, :c3814, :c3815, :c3816, :c3817, :c3818, :c3819, :c3820, :c3821, :c3822, :c3823, :c3824, :c3825, :c3826, :c3827, :c3828, :c3829, :c3830, :c3831, :c3832, :c3833, :c3834, :c3835, :c3836, :c3837, :c3838, :c3839, :c3840, :c3841, :c3842, :c3843, :c3844, :c3845, :c3846, :c3847, :c3848, :c3849, :c3850, :c3851, :c3852, :c3853, :c3854, :c3855, :c3856, :c3857, :c3858, :c3859, :c3860, :c3861, :c3862, :c3863, :c3864, :c3865, :c3866, :c3867, :c3868, :c3869, :c3870, :c3871, :c3872, :c3873, :c3874, :c3875, :c3876, :c3877, :c3878, :c3879, :c3880, :c3881, :c3882, :c3883, :c3884, :c3885, :c3886, :c3887, :c3888, :c3889, :c3890, :c3891, :c3892, :c3893, :c3894, :c3895, :c3896, :c3897, :c3898, :c3899, :c3900, :c3901, :c3902, :c3903, :c3904, :c3905, :c3906, :c3907, :c3908, :c3909, :c3910, :c3911, :c3912, :c3913, :c3914, :c3915, :c3916, :c3917, :c3918, :c3919, :c3920, :c3921, :c3922, :c3923, :c3924, :c3925, :c3926, :c3927, :c3928, :c3929, :c3930, :c3931, :c3932, :c3933, :c3934, :c3935, :c3936, :c3937, :c3938, :c3939, :c3940, :c3941, :c3942, :c3943, :c3944, :c3945, :c3946, :c3947, :c3948, :c3949, :c3950, :c3951, :c3952, :c3953, :c3954, :c3955, :c3956, :c3957, :c3958, :c3959, :c3960, :c3961, :c3962, :c3963, :c3964, :c3965, :c3966, :c3967, :c3968, :c3969, :c3970, :c3971, :c3972, :c3973, :c3974, :c3975, :c3976, :c3977, :c3978, :c3979, :c3980, :c3981, :c3982, :c3983, :c3984, :c3985, :c3986, :c3987, :c3988, :c3989, :c3990, :c3991, :c3992, :c3993, :c3994, :c3995, :c3996, :c3997, :c3998, :c3999, :c4000, :c4001, :c4002, :c4003, :c4004, :c4005, :c4006, :c4007, :c4008, :c4009, :c4010, :c4011, :c4012, :c4013, :c4014, :c4015, :c4016, :c4017, :c4018, :c4019, :c4020, :c4021, :c4022, :c4023, :c4024, :c4025, :c4026, :c4027, :c4028, :c4029, :c4030, :c4031, :c4032, :c4033, :c4034, :c4035, :c4036, :c4037, :c4038, :c4039, :c4040, :c4041, :c4042, :c4043, :c4044, :c4045, :c4046, :c4047, :c4048, :c4049, :c4050, :c4051, :c4052, :c4053, :c4054, :c4055, :c4056, :c4057, :c4058, :c4059, :c4060, :c4061, :c4062, :c4063, :c4064, :c4065, :c4066, :c4067, :c4068, :c4069, :c4070, :c4071, :c4072, :c4073, :c4074, :c4075, :c4076, :c4077, :c4078, :c4079, :c4080, :c4081, :c4082, :c4083, :c4084, :c4085, :c4086, :c4087, :c4088, :c4089, :c4090, :c4091, :c4092, :c4093, :c4094, :c4095, :c4096, :c4097, :c4098, :c4099, :c4100, :c4101, :c4102, :c4103, :c4104, :c4105, :c4106, :c4107, :c4108, :c4109, :c4110, :c4111, :c4112, :c4113, :c4114, :c4115, :c4116, :c4117, :c4118, :c4119, :c4120, :c4121, :c4122, :c4123, :c4124, :c4125, :c4126, :c4127, :c4128, :c4129, :c4130, :c4131, :c4132, :c4133, :c4134, :c4135, :c4136, :c4137, :c4138, :c4139, :c4140, :c4141, :c4142, :c4143, :c4144, :c4145, :c4146, :c4147, :c4148, :c4149, :c4150, :c4151, :c4152, :c4153, :c4154, :c4155, :c4156, :c4157, :c4158, :c4159, :c4160, :c4161, :c4162, :c4163, :c4164, :c4165, :c4166, :c4167, :c4168, :c4169, :c4170, :c4171, :c4172, :c4173, :c4174, :c4175, :c4176, :c4177, :c4178, :c4179, :c4180, :c4181, :c4182, :c4183, :c4184, :c4185, :c4186, :c4187, :c4188, :c4189, :c4190, :c4191, :c4192, :c4193, :c4194, :c4195, :c4196, :c4197, :c4198, :c4199, :c4200, :c4201, :c4202, :c4203, :c4204, :c4205, :c4206, :c4207, :c4208, :c4209, :c4210, :c4211, :c4212, :c4213, :c4214, :c4215, :c4216, :c4217, :c4218, :c4219, :c4220, :c4221, :c4222, :c4223, :c4224, :c4225, :c4226, :c4227, :c4228, :c4229, :c4230, :c4231, :c4232, :c4233, :c4234, :c4235, :c4236, :c4237, :c4238, :c4239, :c4240, :c4241, :c4242, :c4243, :c4244, :c4245, :c4246, :c4247, :c4248, :c4249, :c4250, :c4251, :c4252, :c4253, :c4254, :c4255, :c4256, :c4257, :c4258, :c4259, :c4260, :c4261, :c4262, :c4263, :c4264, :c4265, :c4266, :c4267, :c4268, :c4269, :c4270, :c4271, :c4272, :c4273, :c4274, :c4275, :c4276, :c4277, :c4278, :c4279, :c4280, :c4281, :c4282, :c4283, :c4284, :c4285, :c4286, :c4287, :c4288, :c4289, :c4290, :c4291, :c4292, :c4293, :c4294, :c4295, :c4296, :c4297, :c4298, :c4299, :c4300, :c4301, :c4302, :c4303, :c4304, :c4305, :c4306, :c4307, :c4308, :c4309, :c4310, :c4311, :c4312, :c4313, :c4314, :c4315, :c4316, :c4317, :c4318, :c4319, :c4320, :c4321, :c4322, :c4323, :c4324, :c4325, :c4326, :c4327, :c4328, :c4329, :c4330, :c4331, :c4332, :c4333, :c4334, :c4335, :c4336, :c4337, :c4338, :c4339, :c4340, :c4341, :c4342, :c4343, :c4344, :c4345, :c4346, :c4347, :c4348, :c4349, :c4350, :c4351, :c4352, :c4353, :c4354, :c4355, :c4356, :c4357, :c4358, :c4359, :c4360, :c4361, :c4362, :c4363, :c4364, :c4365, :c4366, :c4367, :c4368, :c4369, :c4370, :c4371, :c4372, :c4373, :c4374, :c4375, :c4376, :c4377, :c4378, :c4379, :c4380, :c4381, :c4382, :c4383, :c4384, :c4385, :c4386, :c4387, :c4388, :c4389, :c4390, :c4391, :c4392, :c4393, :c4394, :c4395, :c4396, :c4397, :c4398, :c4399, :c4400, :c4401, :c4402, :c4403, :c4404, :c4405, :c4406, :c4407, :c4408, :c4409, :c4410, :c4411, :c4412, :c4413, :c4414, :c4415, :c4416, :c4417, :c4418, :c4419, :c4420, :c4421, :c4422, :c4423, :c4424, :c4425, :c4426, :c4427, :c4428, :c4429, :c4430, :c4431, :c4432, :c4433, :c4434, :c4435, :c4436, :c4437, :c4438, :c4439, :c4440, :c4441, :c4442, :c4443, :c4444, :c4445, :c4446, :c4447, :c4448, :c4449, :c4450, :c4451, :c4452, :c4453, :c4454, :c4455, :c4456, :c4457, :c4458, :c4459, :c4460, :c4461, :c4462, :c4463, :c4464, :c4465, :c4466, :c4467, :c4468, :c4469, :c4470, :c4471, :c4472, :c4473, :c4474, :c4475, :c4476, :c4477, :c4478, :c4479, :c4480, :c4481, :c4482, :c4483, :c4484, :c4485, :c4486, :c4487, :c4488, :c4489, :c4490, :c4491, :c4492, :c4493, :c4494, :c4495, :c4496, :c4497, :c4498, :c4499, :c4500, :c4501, :c4502, :c4503, :c4504, :c4505, :c4506, :c4507, :c4508, :c4509, :c4510, :c4511, :c4512, :c4513, :c4514, :c4515, :c4516, :c4517, :c4518, :c4519, :c4520, :c4521, :c4522, :c4523, :c4524, :c4525, :c4526, :c4527, :c4528, :c4529, :c4530, :c4531, :c4532, :c4533, :c4534, :c4535, :c4536, :c4537, :c4538, :c4539, :c4540, :c4541, :c4542, :c4543, :c4544, :c4545, :c4546, :c4547, :c4548, :c4549, :c4550, :c4551, :c4552, :c4553, :c4554, :c4555, :c4556, :c4557, :c4558, :c4559, :c4560, :c4561, :c4562, :c4563, :c4564, :c4565, :c4566, :c4567, :c4568, :c4569, :c4570, :c4571, :c4572, :c4573, :c4574, :c4575, :c4576, :c4577, :c4578, :c4579, :c4580, :c4581, :c4582, :c4583, :c4584, :c4585, :c4586, :c4587, :c4588, :c4589, :c4590, :c4591, :c4592, :c4593, :c4594, :c4595, :c4596, :c4597, :c4598, :c4599, :c4600, :c4601, :c4602, :c4603, :c4604, :c4605, :c4606, :c4607, :c4608, :c4609, :c4610, :c4611, :c4612, :c4613, :c4614, :c4615, :c4616, :c4617, :c4618, :c4619, :c4620, :c4621, :c4622, :c4623, :c4624, :c4625, :c4626, :c4627, :c4628, :c4629, :c4630, :c4631, :c4632, :c4633, :c4634, :c4635, :c4636, :c4637, :c4638, :c4639, :c4640, :c4641, :c4642, :c4643, :c4644, :c4645, :c4646, :c4647, :c4648, :c4649, :c4650, :c4651, :c4652, :c4653, :c4654, :c4655, :c4656, :c4657, :c4658, :c4659, :c4660, :c4661, :c4662, :c4663, :c4664, :c4665, :c4666, :c4667, :c4668, :c4669, :c4670, :c4671, :c4672, :c4673, :c4674, :c4675, :c4676, :c4677, :c4678, :c4679, :c4680, :c4681, :c4682, :c4683, :c4684, :c4685, :c4686, :c4687, :c4688, :c4689, :c4690, :c4691, :c4692, :c4693, :c4694, :c4695, :c4696, :c4697, :c4698, :c4699, :c4700, :c4701, :c4702, :c4703, :c4704, :c4705, :c4706, :c4707, :c4708, :c4709, :c4710, :c4711, :c4712, :c4713, :c4714, :c4715, :c4716, :c4717, :c4718, :c4719, :c4720, :c4721, :c4722, :c4723, :c4724, :c4725, :c4726, :c4727, :c4728, :c4729, :c4730, :c4731, :c4732, :c4733, :c4734, :c4735, :c4736, :c4737, :c4738, :c4739, :c4740, :c4741, :c4742, :c4743, :c4744, :c4745, :c4746, :c4747, :c4748, :c4749, :c4750, :c4751, :c4752, :c4753, :c4754, :c4755, :c4756, :c4757, :c4758, :c4759, :c4760, :c4761, :c4762, :c4763, :c4764, :c4765, :c4766, :c4767, :c4768, :c4769, :c4770, :c4771, :c4772, :c4773, :c4774, :c4775, :c4776, :c4777, :c4778, :c4779, :c4780, :c4781, :c4782, :c4783, :c4784, :c4785, :c4786, :c4787, :c4788, :c4789, :c4790, :c4791, :c4792, :c4793, :c4794, :c4795, :c4796, :c4797, :c4798, :c4799, :c4800, :c4801, :c4802, :c4803, :c4804, :c4805, :c4806, :c4807, :c4808, :c4809, :c4810, :c4811, :c4812, :c4813, :c4814, :c4815, :c4816, :c4817, :c4818, :c4819, :c4820, :c4821, :c4822, :c4823, :c4824, :c4825, :c4826, :c4827, :c4828, :c4829, :c4830, :c4831, :c4832, :c4833, :c4834, :c4835, :c4836, :c4837, :c4838, :c4839, :c4840, :c4841, :c4842, :c4843, :c4844, :c4845, :c4846, :c4847, :c4848, :c4849, :c4850, :c4851, :c4852, :c4853, :c4854, :c4855, :c4856, :c4857, :c4858, :c4859, :c4860, :c4861, :c4862, :c4863, :c4864, :c4865, :c4866, :c4867, :c4868, :c4869, :c4870, :c4871, :c4872, :c4873, :c4874, :c4875, :c4876, :c4877, :c4878, :c4879, :c4880, :c4881, :c4882, :c4883, :c4884, :c4885, :c4886, :c4887, :c4888, :c4889, :c4890, :c4891, :c4892, :c4893, :c4894, :c4895, :c4896, :c4897, :c4898, :c4899, :c4900, :c4901, :c4902, :c4903, :c4904, :c4905, :c4906, :c4907, :c4908, :c4909, :c4910, :c4911, :c4912, :c4913, :c4914, :c4915, :c4916, :c4917, :c4918, :c4919, :c4920, :c4921, :c4922, :c4923, :c4924, :c4925, :c4926, :c4927, :c4928, :c4929, :c4930, :c4931, :c4932, :c4933, :c4934, :c4935, :c4936, :c4937, :c4938, :c4939, :c4940, :c4941, :c4942, :c4943, :c4944, :c4945, :c4946, :c4947, :c4948, :c4949, :c4950, :c4951, :c4952, :c4953, :c4954, :c4955, :c4956, :c4957, :c4958, :c4959, :c4960, :c4961, :c4962, :c4963, :c4964, :c4965, :c4966, :c4967, :c4968, :c4969, :c4970, :c4971, :c4972, :c4973, :c4974, :c4975, :c4976, :c4977, :c4978, :c4979, :c4980, :c4981, :c4982, :c4983, :c4984, :c4985, :c4986, :c4987, :c4988, :c4989, :c4990, :c4991, :c4992, :c4993, :c4994, :c4995, :c4996, :c4997, :c4998, :c4999, :c5000, :c5001, :c5002, :c5003, :c5004, :c5005, :c5006, :c5007, :c5008, :c5009, :c5010, :c5011, :c5012, :c5013, :c5014, :c5015, :c5016, :c5017, :c5018, :c5019, :c5020, :c5021, :c5022, :c5023, :c5024, :c5025, :c5026, :c5027, :c5028, :c5029, :c5030, :c5031, :c5032, :c5033, :c5034, :c5035, :c5036, :c5037, :c5038, :c5039, :c5040, :c5041, :c5042, :c5043, :c5044, :c5045, :c5046, :c5047, :c5048, :c5049, :c5050, :c5051, :c5052, :c5053, :c5054, :c5055, :c5056, :c5057, :c5058, :c5059, :c5060, :c5061, :c5062, :c5063, :c5064, :c5065, :c5066, :c5067, :c5068, :c5069, :c5070, :c5071, :c5072, :c5073, :c5074, :c5075, :c5076, :c5077, :c5078, :c5079, :c5080, :c5081, :c5082, :c5083, :c5084, :c5085, :c5086, :c5087, :c5088, :c5089, :c5090, :c5091, :c5092, :c5093, :c5094, :c5095, :c5096, :c5097, :c5098, :c5099, :c5100, :c5101, :c5102, :c5103, :c5104, :c5105, :c5106, :c5107, :c5108, :c5109, :c5110, :c5111, :c5112, :c5113, :c5114, :c5115, :c5116, :c5117, :c5118, :c5119, :c5120, :c5121, :c5122, :c5123, :c5124, :c5125, :c5126, :c5127, :c5128, :c5129, :c5130, :c5131, :c5132, :c5133, :c5134, :c5135, :c5136, :c5137, :c5138, :c5139, :c5140, :c5141, :c5142, :c5143, :c5144, :c5145, :c5146, :c5147, :c5148, :c5149, :c5150, :c5151, :c5152, :c5153, :c5154, :c5155, :c5156, :c5157, :c5158, :c5159, :c5160, :c5161, :c5162, :c5163, :c5164, :c5165, :c5166, :c5167, :c5168, :c5169, :c5170, :c5171, :c5172, :c5173, :c5174, :c5175, :c5176, :c5177, :c5178, :c5179, :c5180, :c5181, :c5182, :c5183, :c5184, :c5185, :c5186, :c5187, :c5188, :c5189, :c5190, :c5191, :c5192, :c5193, :c5194, :c5195, :c5196, :c5197, :c5198, :c5199, :c5200, :c5201, :c5202, :c5203, :c5204, :c5205, :c5206, :c5207, :c5208, :c5209, :c5210, :c5211, :c5212, :c5213, :c5214, :c5215, :c5216, :c5217, :c5218, :c5219, :c5220, :c5221, :c5222, :c5223, :c5224, :c5225, :c5226, :c5227, :c5228, :c5229, :c5230, :c5231, :c5232, :c5233, :c5234, :c5235, :c5236, :c5237, :c5238, :c5239, :c5240, :c5241, :c5242, :c5243, :c5244, :c5245, :c5246, :c5247, :c5248, :c5249, :c5250, :c5251, :c5252, :c5253, :c5254, :c5255, :c5256, :c5257, :c5258, :c5259, :c5260, :c5261, :c5262, :c5263, :c5264, :c5265, :c5266, :c5267, :c5268, :c5269, :c5270, :c5271, :c5272, :c5273, :c5274, :c5275, :c5276, :c5277, :c5278, :c5279, :c5280, :c5281, :c5282, :c5283, :c5284, :c5285, :c5286, :c5287, :c5288, :c5289, :c5290, :c5291, :c5292, :c5293, :c5294, :c5295, :c5296, :c5297, :c5298, :c5299, :c5300, :c5301, :c5302, :c5303, :c5304, :c5305, :c5306, :c5307, :c5308, :c5309, :c5310, :c5311, :c5312, :c5313, :c5314, :c5315, :c5316, :c5317, :c5318, :c5319, :c5320, :c5321, :c5322, :c5323, :c5324, :c5325, :c5326, :c5327, :c5328, :c5329, :c5330, :c5331, :c5332, :c5333, :c5334, :c5335, :c5336, :c5337, :c5338, :c5339, :c5340, :c5341, :c5342, :c5343, :c5344, :c5345, :c5346, :c5347, :c5348, :c5349, :c5350, :c5351, :c5352, :c5353, :c5354, :c5355, :c5356, :c5357, :c5358, :c5359, :c5360, :c5361, :c5362, :c5363, :c5364, :c5365, :c5366, :c5367, :c5368, :c5369, :c5370, :c5371, :c5372, :c5373, :c5374, :c5375, :c5376, :c5377, :c5378, :c5379, :c5380, :c5381, :c5382, :c5383, :c5384, :c5385, :c5386, :c5387, :c5388, :c5389, :c5390, :c5391, :c5392, :c5393, :c5394, :c5395, :c5396, :c5397, :c5398, :c5399, :c5400, :c5401, :c5402, :c5403, :c5404, :c5405, :c5406, :c5407, :c5408, :c5409, :c5410, :c5411, :c5412, :c5413, :c5414, :c5415, :c5416, :c5417, :c5418, :c5419, :c5420, :c5421, :c5422, :c5423, :c5424, :c5425, :c5426, :c5427, :c5428, :c5429, :c5430, :c5431, :c5432, :c5433, :c5434, :c5435, :c5436, :c5437, :c5438, :c5439, :c5440, :c5441, :c5442, :c5443, :c5444, :c5445, :c5446, :c5447, :c5448, :c5449, :c5450, :c5451, :c5452, :c5453, :c5454, :c5455, :c5456, :c5457, :c5458, :c5459, :c5460, :c5461, :c5462, :c5463, :c5464, :c5465, :c5466, :c5467, :c5468, :c5469, :c5470, :c5471, :c5472, :c5473, :c5474, :c5475, :c5476, :c5477, :c5478, :c5479, :c5480, :c5481, :c5482, :c5483, :c5484, :c5485, :c5486, :c5487, :c5488, :c5489, :c5490, :c5491, :c5492, :c5493, :c5494, :c5495, :c5496, :c5497, :c5498, :c5499, :c5500, :c5501, :c5502, :c5503, :c5504, :c5505, :c5506, :c5507, :c5508, :c5509, :c5510, :c5511, :c5512, :c5513, :c5514, :c5515, :c5516, :c5517, :c5518, :c5519, :c5520, :c5521, :c5522, :c5523, :c5524, :c5525, :c5526, :c5527, :c5528, :c5529, :c5530, :c5531, :c5532, :c5533, :c5534, :c5535, :c5536, :c5537, :c5538, :c5539, :c5540, :c5541, :c5542, :c5543, :c5544, :c5545, :c5546, :c5547, :c5548, :c5549, :c5550, :c5551, :c5552, :c5553, :c5554, :c5555, :c5556, :c5557, :c5558, :c5559, :c5560, :c5561, :c5562, :c5563, :c5564, :c5565, :c5566, :c5567, :c5568, :c5569, :c5570, :c5571, :c5572, :c5573, :c5574, :c5575, :c5576, :c5577, :c5578, :c5579, :c5580, :c5581, :c5582, :c5583, :c5584, :c5585, :c5586, :c5587, :c5588, :c5589, :c5590, :c5591, :c5592, :c5593, :c5594, :c5595, :c5596, :c5597, :c5598, :c5599, :c5600, :c5601, :c5602, :c5603, :c5604, :c5605, :c5606, :c5607, :c5608, :c5609, :c5610, :c5611, :c5612, :c5613, :c5614, :c5615, :c5616, :c5617, :c5618, :c5619, :c5620, :c5621, :c5622, :c5623, :c5624, :c5625, :c5626, :c5627, :c5628, :c5629, :c5630, :c5631, :c5632, :c5633, :c5634, :c5635, :c5636, :c5637, :c5638, :c5639, :c5640, :c5641, :c5642, :c5643, :c5644, :c5645, :c5646, :c5647, :c5648, :c5649, :c5650, :c5651, :c5652, :c5653, :c5654, :c5655, :c5656, :c5657, :c5658, :c5659, :c5660, :c5661, :c5662, :c5663, :c5664, :c5665, :c5666, :c5667, :c5668, :c5669, :c5670, :c5671, :c5672, :c5673, :c5674, :c5675, :c5676, :c5677, :c5678, :c5679, :c5680, :c5681, :c5682, :c5683, :c5684, :c5685, :c5686, :c5687, :c5688, :c5689, :c5690, :c5691, :c5692, :c5693, :c5694, :c5695, :c5696, :c5697, :c5698, :c5699, :c5700, :c5701, :c5702, :c5703, :c5704, :c5705, :c5706, :c5707, :c5708, :c5709, :c5710, :c5711, :c5712, :c5713, :c5714, :c5715, :c5716, :c5717, :c5718, :c5719, :c5720, :c5721, :c5722, :c5723, :c5724, :c5725, :c5726, :c5727, :c5728, :c5729, :c5730, :c5731, :c5732, :c5733, :c5734, :c5735, :c5736, :c5737, :c5738, :c5739, :c5740, :c5741, :c5742, :c5743, :c5744, :c5745, :c5746, :c5747, :c5748, :c5749, :c5750, :c5751, :c5752, :c5753, :c5754, :c5755, :c5756, :c5757, :c5758, :c5759, :c5760, :c5761, :c5762, :c5763, :c5764, :c5765, :c5766, :c5767, :c5768, :c5769, :c5770, :c5771, :c5772, :c5773, :c5774, :c5775, :c5776, :c5777, :c5778, :c5779, :c5780, :c5781, :c5782, :c5783, :c5784, :c5785, :c5786, :c5787, :c5788, :c5789, :c5790, :c5791, :c5792, :c5793, :c5794, :c5795, :c5796, :c5797, :c5798, :c5799, :c5800, :c5801, :c5802, :c5803, :c5804, :c5805, :c5806, :c5807, :c5808, :c5809, :c5810, :c5811, :c5812, :c5813, :c5814, :c5815, :c5816, :c5817, :c5818, :c5819, :c5820, :c5821, :c5822, :c5823, :c5824, :c5825, :c5826, :c5827, :c5828, :c5829, :c5830, :c5831, :c5832, :c5833, :c5834, :c5835, :c5836, :c5837, :c5838, :c5839, :c5840, :c5841, :c5842, :c5843, :c5844, :c5845, :c5846, :c5847, :c5848, :c5849, :c5850, :c5851, :c5852, :c5853, :c5854, :c5855, :c5856, :c5857, :c5858, :c5859, :c5860, :c5861, :c5862, :c5863, :c5864, :c5865, :c5866, :c5867, :c5868, :c5869, :c5870, :c5871, :c5872, :c5873, :c5874, :c5875, :c5876, :c5877, :c5878, :c5879, :c5880, :c5881, :c5882, :c5883, :c5884, :c5885, :c5886, :c5887, :c5888, :c5889, :c5890, :c5891, :c5892, :c5893, :c5894, :c5895, :c5896, :c5897, :c5898, :c5899, :c5900, :c5901, :c5902, :c5903, :c5904, :c5905, :c5906, :c5907, :c5908, :c5909, :c5910, :c5911, :c5912, :c5913, :c5914, :c5915, :c5916, :c5917, :c5918, :c5919, :c5920, :c5921, :c5922, :c5923, :c5924, :c5925, :c5926, :c5927, :c5928, :c5929, :c5930, :c5931, :c5932, :c5933, :c5934, :c5935, :c5936, :c5937, :c5938, :c5939, :c5940, :c5941, :c5942, :c5943, :c5944, :c5945, :c5946, :c5947, :c5948, :c5949, :c5950, :c5951, :c5952, :c5953, :c5954, :c5955, :c5956, :c5957, :c5958, :c5959, :c5960, :c5961, :c5962, :c5963, :c5964, :c5965, :c5966, :c5967, :c5968, :c5969, :c5970, :c5971, :c5972, :c5973, :c5974, :c5975, :c5976, :c5977, :c5978, :c5979, :c5980, :c5981, :c5982, :c5983, :c5984, :c5985, :c5986, :c5987, :c5988, :c5989, :c5990, :c5991, :c5992, :c5993, :c5994, :c5995, :c5996, :c5997, :c5998, :c5999, :c6000, :c6001, :c6002, :c6003, :c6004, :c6005, :c6006, :c6007, :c6008, :c6009, :c6010, :c6011, :c6012, :c6013, :c6014, :c6015, :c6016, :c6017, :c6018, :c6019, :c6020, :c6021, :c6022, :c6023, :c6024, :c6025, :c6026, :c6027, :c6028, :c6029, :c6030, :c6031, :c6032, :c6033, :c6034, :c6035, :c6036, :c6037, :c6038, :c6039, :c6040, :c6041, :c6042, :c6043, :c6044, :c6045, :c6046, :c6047, :c6048, :c6049, :c6050, :c6051, :c6052, :c6053, :c6054, :c6055, :c6056, :c6057, :c6058, :c6059, :c6060, :c6061, :c6062, :c6063, :c6064, :c6065, :c6066, :c6067, :c6068, :c6069, :c6070, :c6071, :c6072, :c6073, :c6074, :c6075, :c6076, :c6077, :c6078, :c6079, :c6080, :c6081, :c6082, :c6083, :c6084, :c6085, :c6086, :c6087, :c6088, :c6089, :c6090, :c6091, :c6092, :c6093, :c6094, :c6095, :c6096, :c6097, :c6098, :c6099, :c6100, :c6101, :c6102, :c6103, :c6104, :c6105, :c6106, :c6107, :c6108, :c6109, :c6110, :c6111, :c6112, :c6113, :c6114, :c6115, :c6116, :c6117, :c6118, :c6119, :c6120, :c6121, :c6122, :c6123, :c6124, :c6125, :c6126, :c6127, :c6128, :c6129, :c6130, :c6131, :c6132, :c6133, :c6134, :c6135, :c6136, :c6137, :c6138, :c6139, :c6140, :c6141, :c6142, :c6143, :c6144, :c6145, :c6146, :c6147, :c6148, :c6149, :c6150, :c6151, :c6152, :c6153, :c6154, :c6155, :c6156, :c6157, :c6158, :c6159, :c6160, :c6161, :c6162, :c6163, :c6164, :c6165, :c6166, :c6167, :c6168, :c6169, :c6170, :c6171, :c6172, :c6173, :c6174, :c6175, :c6176, :c6177, :c6178, :c6179, :c6180, :c6181, :c6182, :c6183, :c6184, :c6185, :c6186, :c6187, :c6188, :c6189, :c6190, :c6191, :c6192, :c6193, :c6194, :c6195, :c6196, :c6197, :c6198, :c6199, :c6200, :c6201, :c6202, :c6203, :c6204, :c6205, :c6206, :c6207, :c6208, :c6209, :c6210, :c6211, :c6212, :c6213, :c6214, :c6215, :c6216, :c6217, :c6218, :c6219, :c6220, :c6221, :c6222, :c6223, :c6224, :c6225, :c6226, :c6227, :c6228, :c6229, :c6230, :c6231, :c6232, :c6233, :c6234, :c6235, :c6236, :c6237, :c6238, :c6239, :c6240, :c6241, :c6242, :c6243, :c6244, :c6245, :c6246, :c6247, :c6248, :c6249, :c6250, :c6251, :c6252, :c6253, :c6254, :c6255, :c6256, :c6257, :c6258, :c6259, :c6260, :c6261, :c6262, :c6263, :c6264, :c6265, :c6266, :c6267, :c6268, :c6269, :c6270, :c6271, :c6272, :c6273, :c6274, :c6275, :c6276, :c6277, :c6278, :c6279, :c6280, :c6281, :c6282, :c6283, :c6284, :c6285, :c6286, :c6287, :c6288, :c6289, :c6290, :c6291, :c6292, :c6293, :c6294, :c6295, :c6296, :c6297, :c6298, :c6299, :c6300, :c6301, :c6302, :c6303, :c6304, :c6305, :c6306, :c6307, :c6308, :c6309, :c6310, :c6311, :c6312, :c6313, :c6314, :c6315, :c6316, :c6317, :c6318, :c6319, :c6320, :c6321, :c6322, :c6323, :c6324, :c6325, :c6326, :c6327, :c6328, :c6329, :c6330, :c6331, :c6332, :c6333, :c6334, :c6335, :c6336, :c6337, :c6338, :c6339, :c6340, :c6341, :c6342, :c6343, :c6344, :c6345, :c6346, :c6347, :c6348, :c6349, :c6350, :c6351, :c6352, :c6353, :c6354, :c6355, :c6356, :c6357, :c6358, :c6359, :c6360, :c6361, :c6362, :c6363, :c6364, :c6365, :c6366, :c6367, :c6368, :c6369, :c6370, :c6371, :c6372, :c6373, :c6374, :c6375, :c6376, :c6377, :c6378, :c6379, :c6380, :c6381, :c6382, :c6383, :c6384, :c6385, :c6386, :c6387, :c6388, :c6389, :c6390, :c6391, :c6392, :c6393, :c6394, :c6395, :c6396, :c6397, :c6398, :c6399, :c6400, :c6401, :c6402, :c6403, :c6404, :c6405, :c6406, :c6407, :c6408, :c6409, :c6410, :c6411, :c6412, :c6413, :c6414, :c6415, :c6416, :c6417, :c6418, :c6419, :c6420, :c6421, :c6422, :c6423, :c6424, :c6425, :c6426, :c6427, :c6428, :c6429, :c6430, :c6431, :c6432, :c6433, :c6434, :c6435, :c6436, :c6437, :c6438, :c6439, :c6440, :c6441, :c6442, :c6443, :c6444, :c6445, :c6446, :c6447, :c6448, :c6449, :c6450, :c6451, :c6452, :c6453, :c6454, :c6455, :c6456, :c6457, :c6458, :c6459, :c6460, :c6461, :c6462, :c6463, :c6464, :c6465, :c6466, :c6467, :c6468, :c6469, :c6470, :c6471, :c6472, :c6473, :c6474, :c6475, :c6476, :c6477, :c6478, :c6479, :c6480, :c6481, :c6482, :c6483, :c6484, :c6485, :c6486, :c6487, :c6488, :c6489, :c6490, :c6491, :c6492, :c6493, :c6494, :c6495, :c6496, :c6497, :c6498, :c6499, :c6500, :c6501, :c6502, :c6503, :c6504, :c6505, :c6506, :c6507, :c6508, :c6509, :c6510, :c6511, :c6512, :c6513, :c6514, :c6515, :c6516, :c6517, :c6518, :c6519, :c6520, :c6521, :c6522, :c6523, :c6524, :c6525, :c6526, :c6527, :c6528, :c6529, :c6530, :c6531, :c6532, :c6533, :c6534, :c6535, :c6536, :c6537, :c6538, :c6539, :c6540, :c6541, :c6542, :c6543, :c6544, :c6545, :c6546, :c6547, :c6548, :c6549, :c6550, :c6551, :c6552, :c6553, :c6554, :c6555, :c6556, :c6557, :c6558, :c6559, :c6560, :c6561, :c6562, :c6563, :c6564, :c6565, :c6566, :c6567, :c6568, :c6569, :c6570, :c6571, :c6572, :c6573, :c6574, :c6575, :c6576, :c6577, :c6578, :c6579, :c6580, :c6581, :c6582, :c6583, :c6584, :c6585, :c6586, :c6587, :c6588, :c6589, :c6590, :c6591, :c6592, :c6593, :c6594, :c6595, :c6596, :c6597, :c6598, :c6599, :c6600, :c6601, :c6602, :c6603, :c6604, :c6605, :c6606, :c6607, :c6608, :c6609, :c6610, :c6611, :c6612, :c6613, :c6614, :c6615, :c6616, :c6617, :c6618, :c6619, :c6620, :c6621, :c6622, :c6623, :c6624, :c6625, :c6626, :c6627, :c6628, :c6629, :c6630, :c6631, :c6632, :c6633, :c6634, :c6635, :c6636, :c6637, :c6638, :c6639, :c6640, :c6641, :c6642, :c6643, :c6644, :c6645, :c6646, :c6647, :c6648, :c6649, :c6650, :c6651, :c6652, :c6653, :c6654, :c6655, :c6656, :c6657, :c6658, :c6659, :c6660, :c6661, :c6662, :c6663, :c6664, :c6665, :c6666, :c6667, :c6668, :c6669, :c6670, :c6671, :c6672, :c6673, :c6674, :c6675, :c6676, :c6677, :c6678, :c6679, :c6680, :c6681, :c6682, :c6683, :c6684, :c6685, :c6686, :c6687, :c6688, :c6689, :c6690, :c6691, :c6692, :c6693, :c6694, :c6695, :c6696, :c6697, :c6698, :c6699, :c6700, :c6701, :c6702, :c6703, :c6704, :c6705, :c6706, :c6707, :c6708, :c6709, :c6710, :c6711, :c6712, :c6713, :c6714, :c6715, :c6716, :c6717, :c6718, :c6719, :c6720, :c6721, :c6722, :c6723, :c6724, :c6725, :c6726, :c6727, :c6728, :c6729, :c6730, :c6731, :c6732, :c6733, :c6734, :c6735, :c6736, :c6737, :c6738, :c6739, :c6740, :c6741, :c6742, :c6743, :c6744, :c6745, :c6746, :c6747, :c6748, :c6749, :c6750, :c6751, :c6752, :c6753, :c6754, :c6755, :c6756, :c6757, :c6758, :c6759, :c6760, :c6761, :c6762, :c6763, :c6764, :c6765, :c6766, :c6767, :c6768, :c6769, :c6770, :c6771, :c6772, :c6773, :c6774, :c6775, :c6776, :c6777, :c6778, :c6779, :c6780, :c6781, :c6782, :c6783, :c6784, :c6785, :c6786, :c6787, :c6788, :c6789, :c6790, :c6791, :c6792, :c6793, :c6794, :c6795, :c6796, :c6797, :c6798, :c6799, :c6800, :c6801, :c6802, :c6803, :c6804, :c6805, :c6806, :c6807, :c6808, :c6809, :c6810, :c6811, :c6812, :c6813, :c6814, :c6815, :c6816, :c6817, :c6818, :c6819, :c6820, :c6821, :c6822, :c6823, :c6824, :c6825, :c6826, :c6827, :c6828, :c6829, :c6830, :c6831, :c6832, :c6833, :c6834, :c6835, :c6836, :c6837, :c6838, :c6839, :c6840, :c6841, :c6842, :c6843, :c6844, :c6845, :c6846, :c6847, :c6848, :c6849, :c6850, :c6851, :c6852, :c6853, :c6854, :c6855, :c6856, :c6857, :c6858, :c6859, :c6860, :c6861, :c6862, :c6863, :c6864, :c6865, :c6866, :c6867, :c6868, :c6869, :c6870, :c6871, :c6872, :c6873, :c6874, :c6875, :c6876, :c6877, :c6878, :c6879, :c6880, :c6881, :c6882, :c6883, :c6884, :c6885, :c6886, :c6887, :c6888, :c6889, :c6890, :c6891, :c6892, :c6893, :c6894, :c6895, :c6896, :c6897, :c6898, :c6899, :c6900, :c6901, :c6902, :c6903, :c6904, :c6905, :c6906, :c6907, :c6908, :c6909, :c6910, :c6911, :c6912, :c6913, :c6914, :c6915, :c6916, :c6917, :c6918, :c6919, :c6920, :c6921, :c6922, :c6923, :c6924, :c6925, :c6926, :c6927, :c6928, :c6929, :c6930, :c6931, :c6932, :c6933, :c6934, :c6935, :c6936, :c6937, :c6938, :c6939, :c6940, :c6941, :c6942, :c6943, :c6944, :c6945, :c6946, :c6947, :c6948, :c6949, :c6950, :c6951, :c6952, :c6953, :c6954, :c6955, :c6956, :c6957, :c6958, :c6959, :c6960, :c6961, :c6962, :c6963, :c6964, :c6965, :c6966, :c6967, :c6968, :c6969, :c6970, :c6971, :c6972, :c6973, :c6974, :c6975, :c6976, :c6977, :c6978, :c6979, :c6980, :c6981, :c6982, :c6983, :c6984, :c6985, :c6986, :c6987, :c6988, :c6989, :c6990, :c6991, :c6992, :c6993, :c6994, :c6995, :c6996, :c6997, :c6998, :c6999, :c7000, :c7001, :c7002, :c7003, :c7004, :c7005, :c7006, :c7007, :c7008, :c7009, :c7010, :c7011, :c7012, :c7013, :c7014, :c7015, :c7016, :c7017, :c7018, :c7019, :c7020, :c7021, :c7022, :c7023, :c7024, :c7025, :c7026, :c7027, :c7028, :c7029, :c7030, :c7031, :c7032, :c7033, :c7034, :c7035, :c7036, :c7037, :c7038, :c7039, :c7040, :c7041, :c7042, :c7043, :c7044, :c7045, :c7046, :c7047, :c7048, :c7049, :c7050, :c7051, :c7052, :c7053, :c7054, :c7055, :c7056, :c7057, :c7058, :c7059, :c7060, :c7061, :c7062, :c7063, :c7064, :c7065, :c7066, :c7067, :c7068, :c7069, :c7070, :c7071, :c7072, :c7073, :c7074, :c7075, :c7076, :c7077, :c7078, :c7079, :c7080, :c7081, :c7082, :c7083, :c7084, :c7085, :c7086, :c7087, :c7088, :c7089, :c7090, :c7091, :c7092, :c7093, :c7094, :c7095, :c7096, :c7097, :c7098, :c7099, :c7100, :c7101, :c7102, :c7103, :c7104, :c7105, :c7106, :c7107, :c7108, :c7109, :c7110, :c7111, :c7112, :c7113, :c7114, :c7115, :c7116, :c7117, :c7118, :c7119, :c7120, :c7121, :c7122, :c7123, :c7124, :c7125, :c7126, :c7127, :c7128, :c7129, :c7130, :c7131, :c7132, :c7133, :c7134, :c7135, :c7136, :c7137, :c7138, :c7139, :c7140, :c7141, :c7142, :c7143, :c7144, :c7145, :c7146, :c7147, :c7148, :c7149, :c7150, :c7151, :c7152, :c7153, :c7154, :c7155, :c7156, :c7157, :c7158, :c7159, :c7160, :c7161, :c7162, :c7163, :c7164, :c7165, :c7166, :c7167, :c7168, :c7169, :c7170, :c7171, :c7172, :c7173, :c7174, :c7175, :c7176, :c7177, :c7178, :c7179, :c7180, :c7181, :c7182, :c7183, :c7184, :c7185, :c7186, :c7187, :c7188, :c7189, :c7190, :c7191, :c7192, :c7193, :c7194, :c7195, :c7196, :c7197, :c7198, :c7199, :c7200, :c7201, :c7202, :c7203, :c7204, :c7205, :c7206, :c7207, :c7208, :c7209, :c7210, :c7211, :c7212, :c7213, :c7214, :c7215, :c7216, :c7217, :c7218, :c7219, :c7220, :c7221, :c7222, :c7223, :c7224, :c7225, :c7226, :c7227, :c7228, :c7229, :c7230, :c7231, :c7232, :c7233, :c7234, :c7235, :c7236, :c7237, :c7238, :c7239, :c7240, :c7241, :c7242, :c7243, :c7244, :c7245, :c7246, :c7247, :c7248, :c7249, :c7250, :c7251, :c7252, :c7253, :c7254, :c7255, :c7256, :c7257, :c7258, :c7259, :c7260, :c7261, :c7262, :c7263, :c7264, :c7265, :c7266, :c7267, :c7268, :c7269, :c7270, :c7271, :c7272, :c7273, :c7274, :c7275, :c7276, :c7277, :c7278, :c7279, :c7280, :c7281, :c7282, :c7283, :c7284, :c7285, :c7286, :c7287, :c7288, :c7289, :c7290, :c7291, :c7292, :c7293, :c7294, :c7295, :c7296, :c7297, :c7298, :c7299, :c7300, :c7301, :c7302, :c7303, :c7304, :c7305, :c7306, :c7307, :c7308, :c7309, :c7310, :c7311, :c7312, :c7313, :c7314, :c7315, :c7316, :c7317, :c7318, :c7319, :c7320, :c7321, :c7322, :c7323, :c7324, :c7325, :c7326, :c7327, :c7328, :c7329, :c7330, :c7331, :c7332, :c7333, :c7334, :c7335, :c7336, :c7337, :c7338, :c7339, :c7340, :c7341, :c7342, :c7343, :c7344, :c7345, :c7346, :c7347, :c7348, :c7349, :c7350, :c7351, :c7352, :c7353, :c7354, :c7355, :c7356, :c7357, :c7358, :c7359, :c7360, :c7361, :c7362, :c7363, :c7364, :c7365, :c7366, :c7367, :c7368, :c7369, :c7370, :c7371, :c7372, :c7373, :c7374, :c7375, :c7376, :c7377, :c7378, :c7379, :c7380, :c7381, :c7382, :c7383, :c7384, :c7385, :c7386, :c7387, :c7388, :c7389, :c7390, :c7391, :c7392, :c7393, :c7394, :c7395, :c7396, :c7397, :c7398, :c7399, :c7400, :c7401, :c7402, :c7403, :c7404, :c7405, :c7406, :c7407, :c7408, :c7409, :c7410, :c7411, :c7412, :c7413, :c7414, :c7415, :c7416, :c7417, :c7418, :c7419, :c7420, :c7421, :c7422, :c7423, :c7424, :c7425, :c7426, :c7427, :c7428, :c7429, :c7430, :c7431, :c7432, :c7433, :c7434, :c7435, :c7436, :c7437, :c7438, :c7439, :c7440, :c7441, :c7442, :c7443, :c7444, :c7445, :c7446, :c7447, :c7448, :c7449, :c7450, :c7451, :c7452, :c7453, :c7454, :c7455, :c7456, :c7457, :c7458, :c7459, :c7460, :c7461, :c7462, :c7463, :c7464, :c7465, :c7466, :c7467, :c7468, :c7469, :c7470, :c7471, :c7472, :c7473, :c7474, :c7475, :c7476, :c7477, :c7478, :c7479, :c7480, :c7481, :c7482, :c7483, :c7484, :c7485, :c7486, :c7487, :c7488, :c7489, :c7490, :c7491, :c7492, :c7493, :c7494, :c7495, :c7496, :c7497, :c7498, :c7499, :c7500, :c7501, :c7502, :c7503, :c7504, :c7505, :c7506, :c7507, :c7508, :c7509, :c7510, :c7511, :c7512, :c7513, :c7514, :c7515, :c7516, :c7517, :c7518, :c7519, :c7520, :c7521, :c7522, :c7523, :c7524, :c7525, :c7526, :c7527, :c7528, :c7529, :c7530, :c7531, :c7532, :c7533, :c7534, :c7535, :c7536, :c7537, :c7538, :c7539, :c7540, :c7541, :c7542, :c7543, :c7544, :c7545, :c7546, :c7547, :c7548, :c7549, :c7550, :c7551, :c7552, :c7553, :c7554, :c7555, :c7556, :c7557, :c7558, :c7559, :c7560, :c7561, :c7562, :c7563, :c7564, :c7565, :c7566, :c7567, :c7568, :c7569, :c7570, :c7571, :c7572, :c7573, :c7574, :c7575, :c7576, :c7577, :c7578, :c7579, :c7580, :c7581, :c7582, :c7583, :c7584, :c7585, :c7586, :c7587, :c7588, :c7589, :c7590, :c7591, :c7592, :c7593, :c7594, :c7595, :c7596, :c7597, :c7598, :c7599, :c7600, :c7601, :c7602, :c7603, :c7604, :c7605, :c7606, :c7607, :c7608, :c7609, :c7610, :c7611, :c7612, :c7613, :c7614, :c7615, :c7616, :c7617, :c7618, :c7619, :c7620, :c7621, :c7622, :c7623, :c7624, :c7625, :c7626, :c7627, :c7628, :c7629, :c7630, :c7631, :c7632, :c7633, :c7634, :c7635, :c7636, :c7637, :c7638, :c7639, :c7640, :c7641, :c7642, :c7643, :c7644, :c7645, :c7646, :c7647, :c7648, :c7649, :c7650, :c7651, :c7652, :c7653, :c7654, :c7655, :c7656, :c7657, :c7658, :c7659, :c7660, :c7661, :c7662, :c7663, :c7664, :c7665, :c7666, :c7667, :c7668, :c7669, :c7670, :c7671, :c7672, :c7673, :c7674, :c7675, :c7676, :c7677, :c7678, :c7679, :c7680, :c7681, :c7682, :c7683, :c7684, :c7685, :c7686, :c7687, :c7688, :c7689, :c7690, :c7691, :c7692, :c7693, :c7694, :c7695, :c7696, :c7697, :c7698, :c7699, :c7700, :c7701, :c7702, :c7703, :c7704, :c7705, :c7706, :c7707, :c7708, :c7709, :c7710, :c7711, :c7712, :c7713, :c7714, :c7715, :c7716, :c7717, :c7718, :c7719, :c7720, :c7721, :c7722, :c7723, :c7724, :c7725, :c7726, :c7727, :c7728, :c7729, :c7730, :c7731, :c7732, :c7733, :c7734, :c7735, :c7736, :c7737, :c7738, :c7739, :c7740, :c7741, :c7742, :c7743, :c7744, :c7745, :c7746, :c7747, :c7748, :c7749, :c7750, :c7751, :c7752, :c7753, :c7754, :c7755, :c7756, :c7757, :c7758, :c7759, :c7760, :c7761, :c7762, :c7763, :c7764, :c7765, :c7766, :c7767, :c7768, :c7769, :c7770, :c7771, :c7772, :c7773, :c7774, :c7775, :c7776, :c7777, :c7778, :c7779, :c7780, :c7781, :c7782, :c7783, :c7784, :c7785, :c7786, :c7787, :c7788, :c7789, :c7790, :c7791, :c7792, :c7793, :c7794, :c7795, :c7796, :c7797, :c7798, :c7799, :c7800, :c7801, :c7802, :c7803, :c7804, :c7805, :c7806, :c7807, :c7808, :c7809, :c7810, :c7811, :c7812, :c7813, :c7814, :c7815, :c7816, :c7817, :c7818, :c7819, :c7820, :c7821, :c7822, :c7823, :c7824, :c7825, :c7826, :c7827, :c7828, :c7829, :c7830, :c7831, :c7832, :c7833, :c7834, :c7835, :c7836, :c7837, :c7838, :c7839, :c7840, :c7841, :c7842, :c7843, :c7844, :c7845, :c7846, :c7847, :c7848, :c7849, :c7850, :c7851, :c7852, :c7853, :c7854, :c7855, :c7856, :c7857, :c7858, :c7859, :c7860, :c7861, :c7862, :c7863, :c7864, :c7865, :c7866, :c7867, :c7868, :c7869, :c7870, :c7871, :c7872, :c7873, :c7874, :c7875, :c7876, :c7877, :c7878, :c7879, :c7880, :c7881, :c7882, :c7883, :c7884, :c7885, :c7886, :c7887, :c7888, :c7889, :c7890, :c7891, :c7892, :c7893, :c7894, :c7895, :c7896, :c7897, :c7898, :c7899, :c7900, :c7901, :c7902, :c7903, :c7904, :c7905, :c7906, :c7907, :c7908, :c7909, :c7910, :c7911, :c7912, :c7913, :c7914, :c7915, :c7916, :c7917, :c7918, :c7919, :c7920, :c7921, :c7922, :c7923, :c7924, :c7925, :c7926, :c7927, :c7928, :c7929, :c7930, :c7931, :c7932, :c7933, :c7934, :c7935, :c7936, :c7937, :c7938, :c7939, :c7940, :c7941, :c7942, :c7943, :c7944, :c7945, :c7946, :c7947, :c7948, :c7949, :c7950, :c7951, :c7952, :c7953, :c7954, :c7955, :c7956, :c7957, :c7958, :c7959, :c7960, :c7961, :c7962, :c7963, :c7964, :c7965, :c7966, :c7967, :c7968, :c7969, :c7970, :c7971, :c7972, :c7973, :c7974, :c7975, :c7976, :c7977, :c7978, :c7979, :c7980, :c7981, :c7982, :c7983, :c7984, :c7985, :c7986, :c7987, :c7988, :c7989, :c7990, :c7991, :c7992, :c7993, :c7994, :c7995, :c7996, :c7997, :c7998, :c7999, :c8000, :c8001, :c8002, :c8003, :c8004, :c8005, :c8006, :c8007, :c8008, :c8009, :c8010, :c8011, :c8012, :c8013, :c8014, :c8015, :c8016, :c8017, :c8018, :c8019, :c8020, :c8021, :c8022, :c8023, :c8024, :c8025, :c8026, :c8027, :c8028, :c8029, :c8030, :c8031, :c8032, :c8033, :c8034, :c8035, :c8036, :c8037, :c8038, :c8039, :c8040, :c8041, :c8042, :c8043, :c8044, :c8045, :c8046, :c8047, :c8048, :c8049, :c8050, :c8051, :c8052, :c8053, :c8054, :c8055, :c8056, :c8057, :c8058, :c8059, :c8060, :c8061, :c8062, :c8063, :c8064, :c8065, :c8066, :c8067, :c8068, :c8069, :c8070, :c8071, :c8072, :c8073, :c8074, :c8075, :c8076, :c8077, :c8078, :c8079, :c8080, :c8081, :c8082, :c8083, :c8084, :c8085, :c8086, :c8087, :c8088, :c8089, :c8090, :c8091, :c8092, :c8093, :c8094, :c8095, :c8096, :c8097, :c8098, :c8099, :c8100, :c8101, :c8102, :c8103, :c8104, :c8105, :c8106, :c8107, :c8108, :c8109, :c8110, :c8111, :c8112, :c8113, :c8114, :c8115, :c8116, :c8117, :c8118, :c8119, :c8120, :c8121, :c8122, :c8123, :c8124, :c8125, :c8126, :c8127, :c8128, :c8129, :c8130, :c8131, :c8132, :c8133, :c8134, :c8135, :c8136, :c8137, :c8138, :c8139, :c8140, :c8141, :c8142, :c8143, :c8144, :c8145, :c8146, :c8147, :c8148, :c8149, :c8150, :c8151, :c8152, :c8153, :c8154, :c8155, :c8156, :c8157, :c8158, :c8159, :c8160, :c8161, :c8162, :c8163, :c8164, :c8165, :c8166, :c8167, :c8168, :c8169, :c8170, :c8171, :c8172, :c8173, :c8174, :c8175, :c8176, :c8177, :c8178, :c8179, :c8180, :c8181, :c8182, :c8183, :c8184, :c8185, :c8186, :c8187, :c8188, :c8189, :c8190, :c8191, :c8192, :c8193, :c8194, :c8195, :c8196, :c8197, :c8198, :c8199, :c8200, :c8201, :c8202, :c8203, :c8204, :c8205, :c8206, :c8207, :c8208, :c8209, :c8210, :c8211, :c8212, :c8213, :c8214, :c8215, :c8216, :c8217, :c8218, :c8219, :c8220, :c8221, :c8222, :c8223, :c8224, :c8225, :c8226, :c8227, :c8228, :c8229, :c8230, :c8231, :c8232, :c8233, :c8234, :c8235, :c8236, :c8237, :c8238, :c8239, :c8240, :c8241, :c8242, :c8243, :c8244, :c8245, :c8246, :c8247, :c8248, :c8249, :c8250, :c8251, :c8252, :c8253, :c8254, :c8255, :c8256, :c8257, :c8258, :c8259, :c8260, :c8261, :c8262, :c8263, :c8264, :c8265, :c8266, :c8267, :c8268, :c8269, :c8270, :c8271, :c8272, :c8273, :c8274, :c8275, :c8276, :c8277, :c8278, :c8279, :c8280, :c8281, :c8282, :c8283, :c8284, :c8285, :c8286, :c8287, :c8288, :c8289, :c8290, :c8291, :c8292, :c8293, :c8294, :c8295, :c8296, :c8297, :c8298, :c8299, :c8300, :c8301, :c8302, :c8303, :c8304, :c8305, :c8306, :c8307, :c8308, :c8309, :c8310, :c8311, :c8312, :c8313, :c8314, :c8315, :c8316, :c8317, :c8318, :c8319, :c8320, :c8321, :c8322, :c8323, :c8324, :c8325, :c8326, :c8327, :c8328, :c8329, :c8330, :c8331, :c8332, :c8333, :c8334, :c8335, :c8336, :c8337, :c8338, :c8339, :c8340, :c8341, :c8342, :c8343, :c8344, :c8345, :c8346, :c8347, :c8348, :c8349, :c8350, :c8351, :c8352, :c8353, :c8354, :c8355, :c8356, :c8357, :c8358, :c8359, :c8360, :c8361, :c8362, :c8363, :c8364, :c8365, :c8366, :c8367, :c8368, :c8369, :c8370, :c8371, :c8372, :c8373, :c8374, :c8375, :c8376, :c8377, :c8378, :c8379, :c8380, :c8381, :c8382, :c8383, :c8384, :c8385, :c8386, :c8387, :c8388, :c8389, :c8390, :c8391, :c8392, :c8393, :c8394, :c8395, :c8396, :c8397, :c8398, :c8399, :c8400, :c8401, :c8402, :c8403, :c8404, :c8405, :c8406, :c8407, :c8408, :c8409, :c8410, :c8411, :c8412, :c8413, :c8414, :c8415, :c8416, :c8417, :c8418, :c8419, :c8420, :c8421, :c8422, :c8423, :c8424, :c8425, :c8426, :c8427, :c8428, :c8429, :c8430, :c8431, :c8432, :c8433, :c8434, :c8435, :c8436, :c8437, :c8438, :c8439, :c8440, :c8441, :c8442, :c8443, :c8444, :c8445, :c8446, :c8447, :c8448, :c8449, :c8450, :c8451, :c8452, :c8453, :c8454, :c8455, :c8456, :c8457, :c8458, :c8459, :c8460, :c8461, :c8462, :c8463, :c8464, :c8465, :c8466, :c8467, :c8468, :c8469, :c8470, :c8471, :c8472, :c8473, :c8474, :c8475, :c8476, :c8477, :c8478, :c8479, :c8480, :c8481, :c8482, :c8483, :c8484, :c8485, :c8486, :c8487, :c8488, :c8489, :c8490, :c8491, :c8492, :c8493, :c8494, :c8495, :c8496, :c8497, :c8498, :c8499, :c8500, :c8501, :c8502, :c8503, :c8504, :c8505, :c8506, :c8507, :c8508, :c8509, :c8510, :c8511, :c8512, :c8513, :c8514, :c8515, :c8516, :c8517, :c8518, :c8519, :c8520, :c8521, :c8522, :c8523, :c8524, :c8525, :c8526, :c8527, :c8528, :c8529, :c8530, :c8531, :c8532, :c8533, :c8534, :c8535, :c8536, :c8537, :c8538, :c8539, :c8540, :c8541, :c8542, :c8543, :c8544, :c8545, :c8546, :c8547, :c8548, :c8549, :c8550, :c8551, :c8552, :c8553, :c8554, :c8555, :c8556, :c8557, :c8558, :c8559, :c8560, :c8561, :c8562, :c8563, :c8564, :c8565, :c8566, :c8567, :c8568, :c8569, :c8570, :c8571, :c8572, :c8573, :c8574, :c8575, :c8576, :c8577, :c8578, :c8579, :c8580, :c8581, :c8582, :c8583, :c8584, :c8585, :c8586, :c8587, :c8588, :c8589, :c8590, :c8591, :c8592, :c8593, :c8594, :c8595, :c8596, :c8597, :c8598, :c8599, :c8600, :c8601, :c8602, :c8603, :c8604, :c8605, :c8606, :c8607, :c8608, :c8609, :c8610, :c8611, :c8612, :c8613, :c8614, :c8615, :c8616, :c8617, :c8618, :c8619, :c8620, :c8621, :c8622, :c8623, :c8624, :c8625, :c8626, :c8627, :c8628, :c8629, :c8630, :c8631, :c8632, :c8633, :c8634, :c8635, :c8636, :c8637, :c8638, :c8639, :c8640, :c8641, :c8642, :c8643, :c8644, :c8645, :c8646, :c8647, :c8648, :c8649, :c8650, :c8651, :c8652, :c8653, :c8654, :c8655, :c8656, :c8657, :c8658, :c8659, :c8660, :c8661, :c8662, :c8663, :c8664, :c8665, :c8666, :c8667, :c8668, :c8669, :c8670, :c8671, :c8672, :c8673, :c8674, :c8675, :c8676, :c8677, :c8678, :c8679, :c8680, :c8681, :c8682, :c8683, :c8684, :c8685, :c8686, :c8687, :c8688, :c8689, :c8690, :c8691, :c8692, :c8693, :c8694, :c8695, :c8696, :c8697, :c8698, :c8699, :c8700, :c8701, :c8702, :c8703, :c8704, :c8705, :c8706, :c8707, :c8708, :c8709, :c8710, :c8711, :c8712, :c8713, :c8714, :c8715, :c8716, :c8717, :c8718, :c8719, :c8720, :c8721, :c8722, :c8723, :c8724, :c8725, :c8726, :c8727, :c8728, :c8729, :c8730, :c8731, :c8732, :c8733, :c8734, :c8735, :c8736, :c8737, :c8738, :c8739, :c8740, :c8741, :c8742, :c8743, :c8744, :c8745, :c8746, :c8747, :c8748, :c8749, :c8750, :c8751, :c8752, :c8753, :c8754, :c8755, :c8756, :c8757, :c8758, :c8759, :c8760, :c8761, :c8762, :c8763, :c8764, :c8765, :c8766, :c8767, :c8768, :c8769, :c8770, :c8771, :c8772, :c8773, :c8774, :c8775, :c8776, :c8777, :c8778, :c8779, :c8780, :c8781, :c8782, :c8783, :c8784, :c8785, :c8786, :c8787, :c8788, :c8789, :c8790, :c8791, :c8792, :c8793, :c8794, :c8795, :c8796, :c8797, :c8798, :c8799, :c8800, :c8801, :c8802, :c8803, :c8804, :c8805, :c8806, :c8807, :c8808, :c8809, :c8810, :c8811, :c8812, :c8813, :c8814, :c8815, :c8816, :c8817, :c8818, :c8819, :c8820, :c8821, :c8822, :c8823, :c8824, :c8825, :c8826, :c8827, :c8828, :c8829, :c8830, :c8831, :c8832, :c8833, :c8834, :c8835, :c8836, :c8837, :c8838, :c8839, :c8840, :c8841, :c8842, :c8843, :c8844, :c8845, :c8846, :c8847, :c8848, :c8849, :c8850, :c8851, :c8852, :c8853, :c8854, :c8855, :c8856, :c8857, :c8858, :c8859, :c8860, :c8861, :c8862, :c8863, :c8864, :c8865, :c8866, :c8867, :c8868, :c8869, :c8870, :c8871, :c8872, :c8873, :c8874, :c8875, :c8876, :c8877, :c8878, :c8879, :c8880, :c8881, :c8882, :c8883, :c8884, :c8885, :c8886, :c8887, :c8888, :c8889, :c8890, :c8891, :c8892, :c8893, :c8894, :c8895, :c8896, :c8897, :c8898, :c8899, :c8900, :c8901, :c8902, :c8903, :c8904, :c8905, :c8906, :c8907, :c8908, :c8909, :c8910, :c8911, :c8912, :c8913, :c8914, :c8915, :c8916, :c8917, :c8918, :c8919, :c8920, :c8921, :c8922, :c8923, :c8924, :c8925, :c8926, :c8927, :c8928, :c8929, :c8930, :c8931, :c8932, :c8933, :c8934, :c8935, :c8936, :c8937, :c8938, :c8939, :c8940, :c8941, :c8942, :c8943, :c8944, :c8945, :c8946, :c8947, :c8948, :c8949, :c8950, :c8951, :c8952, :c8953, :c8954, :c8955, :c8956, :c8957, :c8958, :c8959, :c8960, :c8961, :c8962, :c8963, :c8964, :c8965, :c8966, :c8967, :c8968, :c8969, :c8970, :c8971, :c8972, :c8973, :c8974, :c8975, :c8976, :c8977, :c8978, :c8979, :c8980, :c8981, :c8982, :c8983, :c8984, :c8985, :c8986, :c8987, :c8988, :c8989, :c8990, :c8991, :c8992, :c8993, :c8994, :c8995, :c8996, :c8997, :c8998, :c8999, :c9000, :c9001, :c9002, :c9003, :c9004, :c9005, :c9006, :c9007, :c9008, :c9009, :c9010, :c9011, :c9012, :c9013, :c9014, :c9015, :c9016, :c9017, :c9018, :c9019, :c9020, :c9021, :c9022, :c9023, :c9024, :c9025, :c9026, :c9027, :c9028, :c9029, :c9030, :c9031, :c9032, :c9033, :c9034, :c9035, :c9036, :c9037, :c9038, :c9039, :c9040, :c9041, :c9042, :c9043, :c9044, :c9045, :c9046, :c9047, :c9048, :c9049, :c9050, :c9051, :c9052, :c9053, :c9054, :c9055, :c9056, :c9057, :c9058, :c9059, :c9060, :c9061, :c9062, :c9063, :c9064, :c9065, :c9066, :c9067, :c9068, :c9069, :c9070, :c9071, :c9072, :c9073, :c9074, :c9075, :c9076, :c9077, :c9078, :c9079, :c9080, :c9081, :c9082, :c9083, :c9084, :c9085, :c9086, :c9087, :c9088, :c9089, :c9090, :c9091, :c9092, :c9093, :c9094, :c9095, :c9096, :c9097, :c9098, :c9099, :c9100, :c9101, :c9102, :c9103, :c9104, :c9105, :c9106, :c9107, :c9108, :c9109, :c9110, :c9111, :c9112, :c9113, :c9114, :c9115, :c9116, :c9117, :c9118, :c9119, :c9120, :c9121, :c9122, :c9123, :c9124, :c9125, :c9126, :c9127, :c9128, :c9129, :c9130, :c9131, :c9132, :c9133, :c9134, :c9135, :c9136, :c9137, :c9138, :c9139, :c9140, :c9141, :c9142, :c9143, :c9144, :c9145, :c9146, :c9147, :c9148, :c9149, :c9150, :c9151, :c9152, :c9153, :c9154, :c9155, :c9156, :c9157, :c9158, :c9159, :c9160, :c9161, :c9162, :c9163, :c9164, :c9165, :c9166, :c9167, :c9168, :c9169, :c9170, :c9171, :c9172, :c9173, :c9174, :c9175, :c9176, :c9177, :c9178, :c9179, :c9180, :c9181, :c9182, :c9183, :c9184, :c9185, :c9186, :c9187, :c9188, :c9189, :c9190, :c9191, :c9192, :c9193, :c9194, :c9195, :c9196, :c9197, :c9198, :c9199, :c9200, :c9201, :c9202, :c9203, :c9204, :c9205, :c9206, :c9207, :c9208, :c9209, :c9210, :c9211, :c9212, :c9213, :c9214, :c9215, :c9216, :c9217, :c9218, :c9219, :c9220, :c9221, :c9222, :c9223, :c9224, :c9225, :c9226, :c9227, :c9228, :c9229, :c9230, :c9231, :c9232, :c9233, :c9234, :c9235, :c9236, :c9237, :c9238, :c9239, :c9240, :c9241, :c9242, :c9243, :c9244, :c9245, :c9246, :c9247, :c9248, :c9249, :c9250, :c9251, :c9252, :c9253, :c9254, :c9255, :c9256, :c9257, :c9258, :c9259, :c9260, :c9261, :c9262, :c9263, :c9264, :c9265, :c9266, :c9267, :c9268, :c9269, :c9270, :c9271, :c9272, :c9273, :c9274, :c9275, :c9276, :c9277, :c9278, :c9279, :c9280, :c9281, :c9282, :c9283, :c9284, :c9285, :c9286, :c9287, :c9288, :c9289, :c9290, :c9291, :c9292, :c9293, :c9294, :c9295, :c9296, :c9297, :c9298, :c9299, :c9300, :c9301, :c9302, :c9303, :c9304, :c9305, :c9306, :c9307, :c9308, :c9309, :c9310, :c9311, :c9312, :c9313, :c9314, :c9315, :c9316, :c9317, :c9318, :c9319, :c9320, :c9321, :c9322, :c9323, :c9324, :c9325, :c9326, :c9327, :c9328, :c9329, :c9330, :c9331, :c9332, :c9333, :c9334, :c9335, :c9336, :c9337, :c9338, :c9339, :c9340, :c9341, :c9342, :c9343, :c9344, :c9345, :c9346, :c9347, :c9348, :c9349, :c9350, :c9351, :c9352, :c9353, :c9354, :c9355, :c9356, :c9357, :c9358, :c9359, :c9360, :c9361, :c9362, :c9363, :c9364, :c9365, :c9366, :c9367, :c9368, :c9369, :c9370, :c9371, :c9372, :c9373, :c9374, :c9375, :c9376, :c9377, :c9378, :c9379, :c9380, :c9381, :c9382, :c9383, :c9384, :c9385, :c9386, :c9387, :c9388, :c9389, :c9390, :c9391, :c9392, :c9393, :c9394, :c9395, :c9396, :c9397, :c9398, :c9399, :c9400, :c9401, :c9402, :c9403, :c9404, :c9405, :c9406, :c9407, :c9408, :c9409, :c9410, :c9411, :c9412, :c9413, :c9414, :c9415, :c9416, :c9417, :c9418, :c9419, :c9420, :c9421, :c9422, :c9423, :c9424, :c9425, :c9426, :c9427, :c9428, :c9429, :c9430, :c9431, :c9432, :c9433, :c9434, :c9435, :c9436, :c9437, :c9438, :c9439, :c9440, :c9441, :c9442, :c9443, :c9444, :c9445, :c9446, :c9447, :c9448, :c9449, :c9450, :c9451, :c9452, :c9453, :c9454, :c9455, :c9456, :c9457, :c9458, :c9459, :c9460, :c9461, :c9462, :c9463, :c9464, :c9465, :c9466, :c9467, :c9468, :c9469, :c9470, :c9471, :c9472, :c9473, :c9474, :c9475, :c9476, :c9477, :c9478, :c9479, :c9480, :c9481, :c9482, :c9483, :c9484, :c9485, :c9486, :c9487, :c9488, :c9489, :c9490, :c9491, :c9492, :c9493, :c9494, :c9495, :c9496, :c9497, :c9498, :c9499, :c9500, :c9501, :c9502, :c9503, :c9504, :c9505, :c9506, :c9507, :c9508, :c9509, :c9510, :c9511, :c9512, :c9513, :c9514, :c9515, :c9516, :c9517, :c9518, :c9519, :c9520, :c9521, :c9522, :c9523, :c9524, :c9525, :c9526, :c9527, :c9528, :c9529, :c9530, :c9531, :c9532, :c9533, :c9534, :c9535, :c9536, :c9537, :c9538, :c9539, :c9540, :c9541, :c9542, :c9543, :c9544, :c9545, :c9546, :c9547, :c9548, :c9549, :c9550, :c9551, :c9552, :c9553, :c9554, :c9555, :c9556, :c9557, :c9558, :c9559, :c9560, :c9561, :c9562, :c9563, :c9564, :c9565, :c9566, :c9567, :c9568, :c9569, :c9570, :c9571, :c9572, :c9573, :c9574, :c9575, :c9576, :c9577, :c9578, :c9579, :c9580, :c9581, :c9582, :c9583, :c9584, :c9585, :c9586, :c9587, :c9588, :c9589, :c9590, :c9591, :c9592, :c9593, :c9594, :c9595, :c9596, :c9597, :c9598, :c9599, :c9600, :c9601, :c9602, :c9603, :c9604, :c9605, :c9606, :c9607, :c9608, :c9609, :c9610, :c9611, :c9612, :c9613, :c9614, :c9615, :c9616, :c9617, :c9618, :c9619, :c9620, :c9621, :c9622, :c9623, :c9624, :c9625, :c9626, :c9627, :c9628, :c9629, :c9630, :c9631, :c9632, :c9633, :c9634, :c9635, :c9636, :c9637, :c9638, :c9639, :c9640, :c9641, :c9642, :c9643, :c9644, :c9645, :c9646, :c9647, :c9648, :c9649, :c9650, :c9651, :c9652, :c9653, :c9654, :c9655, :c9656, :c9657, :c9658, :c9659, :c9660, :c9661, :c9662, :c9663, :c9664, :c9665, :c9666, :c9667, :c9668, :c9669, :c9670, :c9671, :c9672, :c9673, :c9674, :c9675, :c9676, :c9677, :c9678, :c9679, :c9680, :c9681, :c9682, :c9683, :c9684, :c9685, :c9686, :c9687, :c9688, :c9689, :c9690, :c9691, :c9692, :c9693, :c9694, :c9695, :c9696, :c9697, :c9698, :c9699, :c9700, :c9701, :c9702, :c9703, :c9704, :c9705, :c9706, :c9707, :c9708, :c9709, :c9710, :c9711, :c9712, :c9713, :c9714, :c9715, :c9716, :c9717, :c9718, :c9719, :c9720, :c9721, :c9722, :c9723, :c9724, :c9725, :c9726, :c9727, :c9728, :c9729, :c9730, :c9731, :c9732, :c9733, :c9734, :c9735, :c9736, :c9737, :c9738, :c9739, :c9740, :c9741, :c9742, :c9743, :c9744, :c9745, :c9746, :c9747, :c9748, :c9749, :c9750, :c9751, :c9752, :c9753, :c9754, :c9755, :c9756, :c9757, :c9758, :c9759, :c9760, :c9761, :c9762, :c9763, :c9764, :c9765, :c9766, :c9767, :c9768, :c9769, :c9770, :c9771, :c9772, :c9773, :c9774, :c9775, :c9776, :c9777, :c9778, :c9779, :c9780, :c9781, :c9782, :c9783, :c9784, :c9785, :c9786, :c9787, :c9788, :c9789, :c9790, :c9791, :c9792, :c9793, :c9794, :c9795, :c9796, :c9797, :c9798, :c9799, :c9800, :c9801, :c9802, :c9803, :c9804, :c9805, :c9806, :c9807, :c9808, :c9809, :c9810, :c9811, :c9812, :c9813, :c9814, :c9815, :c9816, :c9817, :c9818, :c9819, :c9820, :c9821, :c9822, :c9823, :c9824, :c9825, :c9826, :c9827, :c9828, :c9829, :c9830, :c9831, :c9832, :c9833, :c9834, :c9835, :c9836, :c9837, :c9838, :c9839, :c9840, :c9841, :c9842, :c9843, :c9844, :c9845, :c9846, :c9847, :c9848, :c9849, :c9850, :c9851, :c9852, :c9853, :c9854, :c9855, :c9856, :c9857, :c9858, :c9859, :c9860, :c9861, :c9862, :c9863, :c9864, :c9865, :c9866, :c9867, :c9868, :c9869, :c9870, :c9871, :c9872, :c9873, :c9874, :c9875, :c9876, :c9877, :c9878, :c9879, :c9880, :c9881, :c9882, :c9883, :c9884, :c9885, :c9886, :c9887, :c9888, :c9889, :c9890, :c9891, :c9892, :c9893, :c9894, :c9895, :c9896, :c9897, :c9898, :c9899, :c9900, :c9901, :c9902, :c9903, :c9904, :c9905, :c9906, :c9907, :c9908, :c9909, :c9910, :c9911, :c9912, :c9913, :c9914, :c9915, :c9916, :c9917, :c9918, :c9919, :c9920, :c9921, :c9922, :c9923, :c9924, :c9925, :c9926, :c9927, :c9928, :c9929, :c9930, :c9931, :c9932, :c9933, :c9934, :c9935, :c9936, :c9937, :c9938, :c9939, :c9940, :c9941, :c9942, :c9943, :c9944, :c9945, :c9946, :c9947, :c9948, :c9949, :c9950, :c9951, :c9952, :c9953, :c9954, :c9955, :c9956, :c9957, :c9958, :c9959, :c9960, :c9961, :c9962, :c9963, :c9964, :c9965, :c9966, :c9967, :c9968, :c9969, :c9970, :c9971, :c9972, :c9973, :c9974, :c9975, :c9976, :c9977, :c9978, :c9979, :c9980, :c9981, :c9982, :c9983, :c9984, :c9985, :c9986, :c9987, :c9988, :c9989, :c9990, :c9991, :c9992, :c9993, :c9994, :c9995, :c9996, :c9997, :c9998, :c9999, :c10000 . diff --git a/testsuite/serd-tests/good/test-16.nt b/testsuite/serd-tests/good/test-16.nt new file mode 100644 index 00000000..c0604b3b --- /dev/null +++ b/testsuite/serd-tests/good/test-16.nt @@ -0,0 +1,10000 @@ + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . diff --git a/testsuite/serd-tests/good/test-16.ttl b/testsuite/serd-tests/good/test-16.ttl new file mode 100644 index 00000000..a9ac37fb --- /dev/null +++ b/testsuite/serd-tests/good/test-16.ttl @@ -0,0 +1,10002 @@ +# 10000 items (10000 triples) more than the default Bison stack size +@prefix : . +:a :b :c1; + :b :c2; + :b :c3; + :b :c4; + :b :c5; + :b :c6; + :b :c7; + :b :c8; + :b :c9; + :b :c10; + :b :c11; + :b :c12; + :b :c13; + :b :c14; + :b :c15; + :b :c16; + :b :c17; + :b :c18; + :b :c19; + :b :c20; + :b :c21; + :b :c22; + :b :c23; + :b :c24; + :b :c25; + :b :c26; + :b :c27; + :b :c28; + :b :c29; + :b :c30; + :b :c31; + :b :c32; + :b :c33; + :b :c34; + :b :c35; + :b :c36; + :b :c37; + :b :c38; + :b :c39; + :b :c40; + :b :c41; + :b :c42; + :b :c43; + :b :c44; + :b :c45; + :b :c46; + :b :c47; + :b :c48; + :b :c49; + :b :c50; + :b :c51; + :b :c52; + :b :c53; + :b :c54; + :b :c55; + :b :c56; + :b :c57; + :b :c58; + :b :c59; + :b :c60; + :b :c61; + :b :c62; + :b :c63; + :b :c64; + :b :c65; + :b :c66; + :b :c67; + :b :c68; + :b :c69; + :b :c70; + :b :c71; + :b :c72; + :b :c73; + :b :c74; + :b :c75; + :b :c76; + :b :c77; + :b :c78; + :b :c79; + :b :c80; + :b :c81; + :b :c82; + :b :c83; + :b :c84; + :b :c85; + :b :c86; + :b :c87; + :b :c88; + :b :c89; + :b :c90; + :b :c91; + :b :c92; + :b :c93; + :b :c94; + :b :c95; + :b :c96; + :b :c97; + :b :c98; + :b :c99; + :b :c100; + :b :c101; + :b :c102; + :b :c103; + :b :c104; + :b :c105; + :b :c106; + :b :c107; + :b :c108; + :b :c109; + :b :c110; + :b :c111; + :b :c112; + :b :c113; + :b :c114; + :b :c115; + :b :c116; + :b :c117; + :b :c118; + :b :c119; + :b :c120; + :b :c121; + :b :c122; + :b :c123; + :b :c124; + :b :c125; + :b :c126; + :b :c127; + :b :c128; + :b :c129; + :b :c130; + :b :c131; + :b :c132; + :b :c133; + :b :c134; + :b :c135; + :b :c136; + :b :c137; + :b :c138; + :b :c139; + :b :c140; + :b :c141; + :b :c142; + :b :c143; + :b :c144; + :b :c145; + :b :c146; + :b :c147; + :b :c148; + :b :c149; + :b :c150; + :b :c151; + :b :c152; + :b :c153; + :b :c154; + :b :c155; + :b :c156; + :b :c157; + :b :c158; + :b :c159; + :b :c160; + :b :c161; + :b :c162; + :b :c163; + :b :c164; + :b :c165; + :b :c166; + :b :c167; + :b :c168; + :b :c169; + :b :c170; + :b :c171; + :b :c172; + :b :c173; + :b :c174; + :b :c175; + :b :c176; + :b :c177; + :b :c178; + :b :c179; + :b :c180; + :b :c181; + :b :c182; + :b :c183; + :b :c184; + :b :c185; + :b :c186; + :b :c187; + :b :c188; + :b :c189; + :b :c190; + :b :c191; + :b :c192; + :b :c193; + :b :c194; + :b :c195; + :b :c196; + :b :c197; + :b :c198; + :b :c199; + :b :c200; + :b :c201; + :b :c202; + :b :c203; + :b :c204; + :b :c205; + :b :c206; + :b :c207; + :b :c208; + :b :c209; + :b :c210; + :b :c211; + :b :c212; + :b :c213; + :b :c214; + :b :c215; + :b :c216; + :b :c217; + :b :c218; + :b :c219; + :b :c220; + :b :c221; + :b :c222; + :b :c223; + :b :c224; + :b :c225; + :b :c226; + :b :c227; + :b :c228; + :b :c229; + :b :c230; + :b :c231; + :b :c232; + :b :c233; + :b :c234; + :b :c235; + :b :c236; + :b :c237; + :b :c238; + :b :c239; + :b :c240; + :b :c241; + :b :c242; + :b :c243; + :b :c244; + :b :c245; + :b :c246; + :b :c247; + :b :c248; + :b :c249; + :b :c250; + :b :c251; + :b :c252; + :b :c253; + :b :c254; + :b :c255; + :b :c256; + :b :c257; + :b :c258; + :b :c259; + :b :c260; + :b :c261; + :b :c262; + :b :c263; + :b :c264; + :b :c265; + :b :c266; + :b :c267; + :b :c268; + :b :c269; + :b :c270; + :b :c271; + :b :c272; + :b :c273; + :b :c274; + :b :c275; + :b :c276; + :b :c277; + :b :c278; + :b :c279; + :b :c280; + :b :c281; + :b :c282; + :b :c283; + :b :c284; + :b :c285; + :b :c286; + :b :c287; + :b :c288; + :b :c289; + :b :c290; + :b :c291; + :b :c292; + :b :c293; + :b :c294; + :b :c295; + :b :c296; + :b :c297; + :b :c298; + :b :c299; + :b :c300; + :b :c301; + :b :c302; + :b :c303; + :b :c304; + :b :c305; + :b :c306; + :b :c307; + :b :c308; + :b :c309; + :b :c310; + :b :c311; + :b :c312; + :b :c313; + :b :c314; + :b :c315; + :b :c316; + :b :c317; + :b :c318; + :b :c319; + :b :c320; + :b :c321; + :b :c322; + :b :c323; + :b :c324; + :b :c325; + :b :c326; + :b :c327; + :b :c328; + :b :c329; + :b :c330; + :b :c331; + :b :c332; + :b :c333; + :b :c334; + :b :c335; + :b :c336; + :b :c337; + :b :c338; + :b :c339; + :b :c340; + :b :c341; + :b :c342; + :b :c343; + :b :c344; + :b :c345; + :b :c346; + :b :c347; + :b :c348; + :b :c349; + :b :c350; + :b :c351; + :b :c352; + :b :c353; + :b :c354; + :b :c355; + :b :c356; + :b :c357; + :b :c358; + :b :c359; + :b :c360; + :b :c361; + :b :c362; + :b :c363; + :b :c364; + :b :c365; + :b :c366; + :b :c367; + :b :c368; + :b :c369; + :b :c370; + :b :c371; + :b :c372; + :b :c373; + :b :c374; + :b :c375; + :b :c376; + :b :c377; + :b :c378; + :b :c379; + :b :c380; + :b :c381; + :b :c382; + :b :c383; + :b :c384; + :b :c385; + :b :c386; + :b :c387; + :b :c388; + :b :c389; + :b :c390; + :b :c391; + :b :c392; + :b :c393; + :b :c394; + :b :c395; + :b :c396; + :b :c397; + :b :c398; + :b :c399; + :b :c400; + :b :c401; + :b :c402; + :b :c403; + :b :c404; + :b :c405; + :b :c406; + :b :c407; + :b :c408; + :b :c409; + :b :c410; + :b :c411; + :b :c412; + :b :c413; + :b :c414; + :b :c415; + :b :c416; + :b :c417; + :b :c418; + :b :c419; + :b :c420; + :b :c421; + :b :c422; + :b :c423; + :b :c424; + :b :c425; + :b :c426; + :b :c427; + :b :c428; + :b :c429; + :b :c430; + :b :c431; + :b :c432; + :b :c433; + :b :c434; + :b :c435; + :b :c436; + :b :c437; + :b :c438; + :b :c439; + :b :c440; + :b :c441; + :b :c442; + :b :c443; + :b :c444; + :b :c445; + :b :c446; + :b :c447; + :b :c448; + :b :c449; + :b :c450; + :b :c451; + :b :c452; + :b :c453; + :b :c454; + :b :c455; + :b :c456; + :b :c457; + :b :c458; + :b :c459; + :b :c460; + :b :c461; + :b :c462; + :b :c463; + :b :c464; + :b :c465; + :b :c466; + :b :c467; + :b :c468; + :b :c469; + :b :c470; + :b :c471; + :b :c472; + :b :c473; + :b :c474; + :b :c475; + :b :c476; + :b :c477; + :b :c478; + :b :c479; + :b :c480; + :b :c481; + :b :c482; + :b :c483; + :b :c484; + :b :c485; + :b :c486; + :b :c487; + :b :c488; + :b :c489; + :b :c490; + :b :c491; + :b :c492; + :b :c493; + :b :c494; + :b :c495; + :b :c496; + :b :c497; + :b :c498; + :b :c499; + :b :c500; + :b :c501; + :b :c502; + :b :c503; + :b :c504; + :b :c505; + :b :c506; + :b :c507; + :b :c508; + :b :c509; + :b :c510; + :b :c511; + :b :c512; + :b :c513; + :b :c514; + :b :c515; + :b :c516; + :b :c517; + :b :c518; + :b :c519; + :b :c520; + :b :c521; + :b :c522; + :b :c523; + :b :c524; + :b :c525; + :b :c526; + :b :c527; + :b :c528; + :b :c529; + :b :c530; + :b :c531; + :b :c532; + :b :c533; + :b :c534; + :b :c535; + :b :c536; + :b :c537; + :b :c538; + :b :c539; + :b :c540; + :b :c541; + :b :c542; + :b :c543; + :b :c544; + :b :c545; + :b :c546; + :b :c547; + :b :c548; + :b :c549; + :b :c550; + :b :c551; + :b :c552; + :b :c553; + :b :c554; + :b :c555; + :b :c556; + :b :c557; + :b :c558; + :b :c559; + :b :c560; + :b :c561; + :b :c562; + :b :c563; + :b :c564; + :b :c565; + :b :c566; + :b :c567; + :b :c568; + :b :c569; + :b :c570; + :b :c571; + :b :c572; + :b :c573; + :b :c574; + :b :c575; + :b :c576; + :b :c577; + :b :c578; + :b :c579; + :b :c580; + :b :c581; + :b :c582; + :b :c583; + :b :c584; + :b :c585; + :b :c586; + :b :c587; + :b :c588; + :b :c589; + :b :c590; + :b :c591; + :b :c592; + :b :c593; + :b :c594; + :b :c595; + :b :c596; + :b :c597; + :b :c598; + :b :c599; + :b :c600; + :b :c601; + :b :c602; + :b :c603; + :b :c604; + :b :c605; + :b :c606; + :b :c607; + :b :c608; + :b :c609; + :b :c610; + :b :c611; + :b :c612; + :b :c613; + :b :c614; + :b :c615; + :b :c616; + :b :c617; + :b :c618; + :b :c619; + :b :c620; + :b :c621; + :b :c622; + :b :c623; + :b :c624; + :b :c625; + :b :c626; + :b :c627; + :b :c628; + :b :c629; + :b :c630; + :b :c631; + :b :c632; + :b :c633; + :b :c634; + :b :c635; + :b :c636; + :b :c637; + :b :c638; + :b :c639; + :b :c640; + :b :c641; + :b :c642; + :b :c643; + :b :c644; + :b :c645; + :b :c646; + :b :c647; + :b :c648; + :b :c649; + :b :c650; + :b :c651; + :b :c652; + :b :c653; + :b :c654; + :b :c655; + :b :c656; + :b :c657; + :b :c658; + :b :c659; + :b :c660; + :b :c661; + :b :c662; + :b :c663; + :b :c664; + :b :c665; + :b :c666; + :b :c667; + :b :c668; + :b :c669; + :b :c670; + :b :c671; + :b :c672; + :b :c673; + :b :c674; + :b :c675; + :b :c676; + :b :c677; + :b :c678; + :b :c679; + :b :c680; + :b :c681; + :b :c682; + :b :c683; + :b :c684; + :b :c685; + :b :c686; + :b :c687; + :b :c688; + :b :c689; + :b :c690; + :b :c691; + :b :c692; + :b :c693; + :b :c694; + :b :c695; + :b :c696; + :b :c697; + :b :c698; + :b :c699; + :b :c700; + :b :c701; + :b :c702; + :b :c703; + :b :c704; + :b :c705; + :b :c706; + :b :c707; + :b :c708; + :b :c709; + :b :c710; + :b :c711; + :b :c712; + :b :c713; + :b :c714; + :b :c715; + :b :c716; + :b :c717; + :b :c718; + :b :c719; + :b :c720; + :b :c721; + :b :c722; + :b :c723; + :b :c724; + :b :c725; + :b :c726; + :b :c727; + :b :c728; + :b :c729; + :b :c730; + :b :c731; + :b :c732; + :b :c733; + :b :c734; + :b :c735; + :b :c736; + :b :c737; + :b :c738; + :b :c739; + :b :c740; + :b :c741; + :b :c742; + :b :c743; + :b :c744; + :b :c745; + :b :c746; + :b :c747; + :b :c748; + :b :c749; + :b :c750; + :b :c751; + :b :c752; + :b :c753; + :b :c754; + :b :c755; + :b :c756; + :b :c757; + :b :c758; + :b :c759; + :b :c760; + :b :c761; + :b :c762; + :b :c763; + :b :c764; + :b :c765; + :b :c766; + :b :c767; + :b :c768; + :b :c769; + :b :c770; + :b :c771; + :b :c772; + :b :c773; + :b :c774; + :b :c775; + :b :c776; + :b :c777; + :b :c778; + :b :c779; + :b :c780; + :b :c781; + :b :c782; + :b :c783; + :b :c784; + :b :c785; + :b :c786; + :b :c787; + :b :c788; + :b :c789; + :b :c790; + :b :c791; + :b :c792; + :b :c793; + :b :c794; + :b :c795; + :b :c796; + :b :c797; + :b :c798; + :b :c799; + :b :c800; + :b :c801; + :b :c802; + :b :c803; + :b :c804; + :b :c805; + :b :c806; + :b :c807; + :b :c808; + :b :c809; + :b :c810; + :b :c811; + :b :c812; + :b :c813; + :b :c814; + :b :c815; + :b :c816; + :b :c817; + :b :c818; + :b :c819; + :b :c820; + :b :c821; + :b :c822; + :b :c823; + :b :c824; + :b :c825; + :b :c826; + :b :c827; + :b :c828; + :b :c829; + :b :c830; + :b :c831; + :b :c832; + :b :c833; + :b :c834; + :b :c835; + :b :c836; + :b :c837; + :b :c838; + :b :c839; + :b :c840; + :b :c841; + :b :c842; + :b :c843; + :b :c844; + :b :c845; + :b :c846; + :b :c847; + :b :c848; + :b :c849; + :b :c850; + :b :c851; + :b :c852; + :b :c853; + :b :c854; + :b :c855; + :b :c856; + :b :c857; + :b :c858; + :b :c859; + :b :c860; + :b :c861; + :b :c862; + :b :c863; + :b :c864; + :b :c865; + :b :c866; + :b :c867; + :b :c868; + :b :c869; + :b :c870; + :b :c871; + :b :c872; + :b :c873; + :b :c874; + :b :c875; + :b :c876; + :b :c877; + :b :c878; + :b :c879; + :b :c880; + :b :c881; + :b :c882; + :b :c883; + :b :c884; + :b :c885; + :b :c886; + :b :c887; + :b :c888; + :b :c889; + :b :c890; + :b :c891; + :b :c892; + :b :c893; + :b :c894; + :b :c895; + :b :c896; + :b :c897; + :b :c898; + :b :c899; + :b :c900; + :b :c901; + :b :c902; + :b :c903; + :b :c904; + :b :c905; + :b :c906; + :b :c907; + :b :c908; + :b :c909; + :b :c910; + :b :c911; + :b :c912; + :b :c913; + :b :c914; + :b :c915; + :b :c916; + :b :c917; + :b :c918; + :b :c919; + :b :c920; + :b :c921; + :b :c922; + :b :c923; + :b :c924; + :b :c925; + :b :c926; + :b :c927; + :b :c928; + :b :c929; + :b :c930; + :b :c931; + :b :c932; + :b :c933; + :b :c934; + :b :c935; + :b :c936; + :b :c937; + :b :c938; + :b :c939; + :b :c940; + :b :c941; + :b :c942; + :b :c943; + :b :c944; + :b :c945; + :b :c946; + :b :c947; + :b :c948; + :b :c949; + :b :c950; + :b :c951; + :b :c952; + :b :c953; + :b :c954; + :b :c955; + :b :c956; + :b :c957; + :b :c958; + :b :c959; + :b :c960; + :b :c961; + :b :c962; + :b :c963; + :b :c964; + :b :c965; + :b :c966; + :b :c967; + :b :c968; + :b :c969; + :b :c970; + :b :c971; + :b :c972; + :b :c973; + :b :c974; + :b :c975; + :b :c976; + :b :c977; + :b :c978; + :b :c979; + :b :c980; + :b :c981; + :b :c982; + :b :c983; + :b :c984; + :b :c985; + :b :c986; + :b :c987; + :b :c988; + :b :c989; + :b :c990; + :b :c991; + :b :c992; + :b :c993; + :b :c994; + :b :c995; + :b :c996; + :b :c997; + :b :c998; + :b :c999; + :b :c1000; + :b :c1001; + :b :c1002; + :b :c1003; + :b :c1004; + :b :c1005; + :b :c1006; + :b :c1007; + :b :c1008; + :b :c1009; + :b :c1010; + :b :c1011; + :b :c1012; + :b :c1013; + :b :c1014; + :b :c1015; + :b :c1016; + :b :c1017; + :b :c1018; + :b :c1019; + :b :c1020; + :b :c1021; + :b :c1022; + :b :c1023; + :b :c1024; + :b :c1025; + :b :c1026; + :b :c1027; + :b :c1028; + :b :c1029; + :b :c1030; + :b :c1031; + :b :c1032; + :b :c1033; + :b :c1034; + :b :c1035; + :b :c1036; + :b :c1037; + :b :c1038; + :b :c1039; + :b :c1040; + :b :c1041; + :b :c1042; + :b :c1043; + :b :c1044; + :b :c1045; + :b :c1046; + :b :c1047; + :b :c1048; + :b :c1049; + :b :c1050; + :b :c1051; + :b :c1052; + :b :c1053; + :b :c1054; + :b :c1055; + :b :c1056; + :b :c1057; + :b :c1058; + :b :c1059; + :b :c1060; + :b :c1061; + :b :c1062; + :b :c1063; + :b :c1064; + :b :c1065; + :b :c1066; + :b :c1067; + :b :c1068; + :b :c1069; + :b :c1070; + :b :c1071; + :b :c1072; + :b :c1073; + :b :c1074; + :b :c1075; + :b :c1076; + :b :c1077; + :b :c1078; + :b :c1079; + :b :c1080; + :b :c1081; + :b :c1082; + :b :c1083; + :b :c1084; + :b :c1085; + :b :c1086; + :b :c1087; + :b :c1088; + :b :c1089; + :b :c1090; + :b :c1091; + :b :c1092; + :b :c1093; + :b :c1094; + :b :c1095; + :b :c1096; + :b :c1097; + :b :c1098; + :b :c1099; + :b :c1100; + :b :c1101; + :b :c1102; + :b :c1103; + :b :c1104; + :b :c1105; + :b :c1106; + :b :c1107; + :b :c1108; + :b :c1109; + :b :c1110; + :b :c1111; + :b :c1112; + :b :c1113; + :b :c1114; + :b :c1115; + :b :c1116; + :b :c1117; + :b :c1118; + :b :c1119; + :b :c1120; + :b :c1121; + :b :c1122; + :b :c1123; + :b :c1124; + :b :c1125; + :b :c1126; + :b :c1127; + :b :c1128; + :b :c1129; + :b :c1130; + :b :c1131; + :b :c1132; + :b :c1133; + :b :c1134; + :b :c1135; + :b :c1136; + :b :c1137; + :b :c1138; + :b :c1139; + :b :c1140; + :b :c1141; + :b :c1142; + :b :c1143; + :b :c1144; + :b :c1145; + :b :c1146; + :b :c1147; + :b :c1148; + :b :c1149; + :b :c1150; + :b :c1151; + :b :c1152; + :b :c1153; + :b :c1154; + :b :c1155; + :b :c1156; + :b :c1157; + :b :c1158; + :b :c1159; + :b :c1160; + :b :c1161; + :b :c1162; + :b :c1163; + :b :c1164; + :b :c1165; + :b :c1166; + :b :c1167; + :b :c1168; + :b :c1169; + :b :c1170; + :b :c1171; + :b :c1172; + :b :c1173; + :b :c1174; + :b :c1175; + :b :c1176; + :b :c1177; + :b :c1178; + :b :c1179; + :b :c1180; + :b :c1181; + :b :c1182; + :b :c1183; + :b :c1184; + :b :c1185; + :b :c1186; + :b :c1187; + :b :c1188; + :b :c1189; + :b :c1190; + :b :c1191; + :b :c1192; + :b :c1193; + :b :c1194; + :b :c1195; + :b :c1196; + :b :c1197; + :b :c1198; + :b :c1199; + :b :c1200; + :b :c1201; + :b :c1202; + :b :c1203; + :b :c1204; + :b :c1205; + :b :c1206; + :b :c1207; + :b :c1208; + :b :c1209; + :b :c1210; + :b :c1211; + :b :c1212; + :b :c1213; + :b :c1214; + :b :c1215; + :b :c1216; + :b :c1217; + :b :c1218; + :b :c1219; + :b :c1220; + :b :c1221; + :b :c1222; + :b :c1223; + :b :c1224; + :b :c1225; + :b :c1226; + :b :c1227; + :b :c1228; + :b :c1229; + :b :c1230; + :b :c1231; + :b :c1232; + :b :c1233; + :b :c1234; + :b :c1235; + :b :c1236; + :b :c1237; + :b :c1238; + :b :c1239; + :b :c1240; + :b :c1241; + :b :c1242; + :b :c1243; + :b :c1244; + :b :c1245; + :b :c1246; + :b :c1247; + :b :c1248; + :b :c1249; + :b :c1250; + :b :c1251; + :b :c1252; + :b :c1253; + :b :c1254; + :b :c1255; + :b :c1256; + :b :c1257; + :b :c1258; + :b :c1259; + :b :c1260; + :b :c1261; + :b :c1262; + :b :c1263; + :b :c1264; + :b :c1265; + :b :c1266; + :b :c1267; + :b :c1268; + :b :c1269; + :b :c1270; + :b :c1271; + :b :c1272; + :b :c1273; + :b :c1274; + :b :c1275; + :b :c1276; + :b :c1277; + :b :c1278; + :b :c1279; + :b :c1280; + :b :c1281; + :b :c1282; + :b :c1283; + :b :c1284; + :b :c1285; + :b :c1286; + :b :c1287; + :b :c1288; + :b :c1289; + :b :c1290; + :b :c1291; + :b :c1292; + :b :c1293; + :b :c1294; + :b :c1295; + :b :c1296; + :b :c1297; + :b :c1298; + :b :c1299; + :b :c1300; + :b :c1301; + :b :c1302; + :b :c1303; + :b :c1304; + :b :c1305; + :b :c1306; + :b :c1307; + :b :c1308; + :b :c1309; + :b :c1310; + :b :c1311; + :b :c1312; + :b :c1313; + :b :c1314; + :b :c1315; + :b :c1316; + :b :c1317; + :b :c1318; + :b :c1319; + :b :c1320; + :b :c1321; + :b :c1322; + :b :c1323; + :b :c1324; + :b :c1325; + :b :c1326; + :b :c1327; + :b :c1328; + :b :c1329; + :b :c1330; + :b :c1331; + :b :c1332; + :b :c1333; + :b :c1334; + :b :c1335; + :b :c1336; + :b :c1337; + :b :c1338; + :b :c1339; + :b :c1340; + :b :c1341; + :b :c1342; + :b :c1343; + :b :c1344; + :b :c1345; + :b :c1346; + :b :c1347; + :b :c1348; + :b :c1349; + :b :c1350; + :b :c1351; + :b :c1352; + :b :c1353; + :b :c1354; + :b :c1355; + :b :c1356; + :b :c1357; + :b :c1358; + :b :c1359; + :b :c1360; + :b :c1361; + :b :c1362; + :b :c1363; + :b :c1364; + :b :c1365; + :b :c1366; + :b :c1367; + :b :c1368; + :b :c1369; + :b :c1370; + :b :c1371; + :b :c1372; + :b :c1373; + :b :c1374; + :b :c1375; + :b :c1376; + :b :c1377; + :b :c1378; + :b :c1379; + :b :c1380; + :b :c1381; + :b :c1382; + :b :c1383; + :b :c1384; + :b :c1385; + :b :c1386; + :b :c1387; + :b :c1388; + :b :c1389; + :b :c1390; + :b :c1391; + :b :c1392; + :b :c1393; + :b :c1394; + :b :c1395; + :b :c1396; + :b :c1397; + :b :c1398; + :b :c1399; + :b :c1400; + :b :c1401; + :b :c1402; + :b :c1403; + :b :c1404; + :b :c1405; + :b :c1406; + :b :c1407; + :b :c1408; + :b :c1409; + :b :c1410; + :b :c1411; + :b :c1412; + :b :c1413; + :b :c1414; + :b :c1415; + :b :c1416; + :b :c1417; + :b :c1418; + :b :c1419; + :b :c1420; + :b :c1421; + :b :c1422; + :b :c1423; + :b :c1424; + :b :c1425; + :b :c1426; + :b :c1427; + :b :c1428; + :b :c1429; + :b :c1430; + :b :c1431; + :b :c1432; + :b :c1433; + :b :c1434; + :b :c1435; + :b :c1436; + :b :c1437; + :b :c1438; + :b :c1439; + :b :c1440; + :b :c1441; + :b :c1442; + :b :c1443; + :b :c1444; + :b :c1445; + :b :c1446; + :b :c1447; + :b :c1448; + :b :c1449; + :b :c1450; + :b :c1451; + :b :c1452; + :b :c1453; + :b :c1454; + :b :c1455; + :b :c1456; + :b :c1457; + :b :c1458; + :b :c1459; + :b :c1460; + :b :c1461; + :b :c1462; + :b :c1463; + :b :c1464; + :b :c1465; + :b :c1466; + :b :c1467; + :b :c1468; + :b :c1469; + :b :c1470; + :b :c1471; + :b :c1472; + :b :c1473; + :b :c1474; + :b :c1475; + :b :c1476; + :b :c1477; + :b :c1478; + :b :c1479; + :b :c1480; + :b :c1481; + :b :c1482; + :b :c1483; + :b :c1484; + :b :c1485; + :b :c1486; + :b :c1487; + :b :c1488; + :b :c1489; + :b :c1490; + :b :c1491; + :b :c1492; + :b :c1493; + :b :c1494; + :b :c1495; + :b :c1496; + :b :c1497; + :b :c1498; + :b :c1499; + :b :c1500; + :b :c1501; + :b :c1502; + :b :c1503; + :b :c1504; + :b :c1505; + :b :c1506; + :b :c1507; + :b :c1508; + :b :c1509; + :b :c1510; + :b :c1511; + :b :c1512; + :b :c1513; + :b :c1514; + :b :c1515; + :b :c1516; + :b :c1517; + :b :c1518; + :b :c1519; + :b :c1520; + :b :c1521; + :b :c1522; + :b :c1523; + :b :c1524; + :b :c1525; + :b :c1526; + :b :c1527; + :b :c1528; + :b :c1529; + :b :c1530; + :b :c1531; + :b :c1532; + :b :c1533; + :b :c1534; + :b :c1535; + :b :c1536; + :b :c1537; + :b :c1538; + :b :c1539; + :b :c1540; + :b :c1541; + :b :c1542; + :b :c1543; + :b :c1544; + :b :c1545; + :b :c1546; + :b :c1547; + :b :c1548; + :b :c1549; + :b :c1550; + :b :c1551; + :b :c1552; + :b :c1553; + :b :c1554; + :b :c1555; + :b :c1556; + :b :c1557; + :b :c1558; + :b :c1559; + :b :c1560; + :b :c1561; + :b :c1562; + :b :c1563; + :b :c1564; + :b :c1565; + :b :c1566; + :b :c1567; + :b :c1568; + :b :c1569; + :b :c1570; + :b :c1571; + :b :c1572; + :b :c1573; + :b :c1574; + :b :c1575; + :b :c1576; + :b :c1577; + :b :c1578; + :b :c1579; + :b :c1580; + :b :c1581; + :b :c1582; + :b :c1583; + :b :c1584; + :b :c1585; + :b :c1586; + :b :c1587; + :b :c1588; + :b :c1589; + :b :c1590; + :b :c1591; + :b :c1592; + :b :c1593; + :b :c1594; + :b :c1595; + :b :c1596; + :b :c1597; + :b :c1598; + :b :c1599; + :b :c1600; + :b :c1601; + :b :c1602; + :b :c1603; + :b :c1604; + :b :c1605; + :b :c1606; + :b :c1607; + :b :c1608; + :b :c1609; + :b :c1610; + :b :c1611; + :b :c1612; + :b :c1613; + :b :c1614; + :b :c1615; + :b :c1616; + :b :c1617; + :b :c1618; + :b :c1619; + :b :c1620; + :b :c1621; + :b :c1622; + :b :c1623; + :b :c1624; + :b :c1625; + :b :c1626; + :b :c1627; + :b :c1628; + :b :c1629; + :b :c1630; + :b :c1631; + :b :c1632; + :b :c1633; + :b :c1634; + :b :c1635; + :b :c1636; + :b :c1637; + :b :c1638; + :b :c1639; + :b :c1640; + :b :c1641; + :b :c1642; + :b :c1643; + :b :c1644; + :b :c1645; + :b :c1646; + :b :c1647; + :b :c1648; + :b :c1649; + :b :c1650; + :b :c1651; + :b :c1652; + :b :c1653; + :b :c1654; + :b :c1655; + :b :c1656; + :b :c1657; + :b :c1658; + :b :c1659; + :b :c1660; + :b :c1661; + :b :c1662; + :b :c1663; + :b :c1664; + :b :c1665; + :b :c1666; + :b :c1667; + :b :c1668; + :b :c1669; + :b :c1670; + :b :c1671; + :b :c1672; + :b :c1673; + :b :c1674; + :b :c1675; + :b :c1676; + :b :c1677; + :b :c1678; + :b :c1679; + :b :c1680; + :b :c1681; + :b :c1682; + :b :c1683; + :b :c1684; + :b :c1685; + :b :c1686; + :b :c1687; + :b :c1688; + :b :c1689; + :b :c1690; + :b :c1691; + :b :c1692; + :b :c1693; + :b :c1694; + :b :c1695; + :b :c1696; + :b :c1697; + :b :c1698; + :b :c1699; + :b :c1700; + :b :c1701; + :b :c1702; + :b :c1703; + :b :c1704; + :b :c1705; + :b :c1706; + :b :c1707; + :b :c1708; + :b :c1709; + :b :c1710; + :b :c1711; + :b :c1712; + :b :c1713; + :b :c1714; + :b :c1715; + :b :c1716; + :b :c1717; + :b :c1718; + :b :c1719; + :b :c1720; + :b :c1721; + :b :c1722; + :b :c1723; + :b :c1724; + :b :c1725; + :b :c1726; + :b :c1727; + :b :c1728; + :b :c1729; + :b :c1730; + :b :c1731; + :b :c1732; + :b :c1733; + :b :c1734; + :b :c1735; + :b :c1736; + :b :c1737; + :b :c1738; + :b :c1739; + :b :c1740; + :b :c1741; + :b :c1742; + :b :c1743; + :b :c1744; + :b :c1745; + :b :c1746; + :b :c1747; + :b :c1748; + :b :c1749; + :b :c1750; + :b :c1751; + :b :c1752; + :b :c1753; + :b :c1754; + :b :c1755; + :b :c1756; + :b :c1757; + :b :c1758; + :b :c1759; + :b :c1760; + :b :c1761; + :b :c1762; + :b :c1763; + :b :c1764; + :b :c1765; + :b :c1766; + :b :c1767; + :b :c1768; + :b :c1769; + :b :c1770; + :b :c1771; + :b :c1772; + :b :c1773; + :b :c1774; + :b :c1775; + :b :c1776; + :b :c1777; + :b :c1778; + :b :c1779; + :b :c1780; + :b :c1781; + :b :c1782; + :b :c1783; + :b :c1784; + :b :c1785; + :b :c1786; + :b :c1787; + :b :c1788; + :b :c1789; + :b :c1790; + :b :c1791; + :b :c1792; + :b :c1793; + :b :c1794; + :b :c1795; + :b :c1796; + :b :c1797; + :b :c1798; + :b :c1799; + :b :c1800; + :b :c1801; + :b :c1802; + :b :c1803; + :b :c1804; + :b :c1805; + :b :c1806; + :b :c1807; + :b :c1808; + :b :c1809; + :b :c1810; + :b :c1811; + :b :c1812; + :b :c1813; + :b :c1814; + :b :c1815; + :b :c1816; + :b :c1817; + :b :c1818; + :b :c1819; + :b :c1820; + :b :c1821; + :b :c1822; + :b :c1823; + :b :c1824; + :b :c1825; + :b :c1826; + :b :c1827; + :b :c1828; + :b :c1829; + :b :c1830; + :b :c1831; + :b :c1832; + :b :c1833; + :b :c1834; + :b :c1835; + :b :c1836; + :b :c1837; + :b :c1838; + :b :c1839; + :b :c1840; + :b :c1841; + :b :c1842; + :b :c1843; + :b :c1844; + :b :c1845; + :b :c1846; + :b :c1847; + :b :c1848; + :b :c1849; + :b :c1850; + :b :c1851; + :b :c1852; + :b :c1853; + :b :c1854; + :b :c1855; + :b :c1856; + :b :c1857; + :b :c1858; + :b :c1859; + :b :c1860; + :b :c1861; + :b :c1862; + :b :c1863; + :b :c1864; + :b :c1865; + :b :c1866; + :b :c1867; + :b :c1868; + :b :c1869; + :b :c1870; + :b :c1871; + :b :c1872; + :b :c1873; + :b :c1874; + :b :c1875; + :b :c1876; + :b :c1877; + :b :c1878; + :b :c1879; + :b :c1880; + :b :c1881; + :b :c1882; + :b :c1883; + :b :c1884; + :b :c1885; + :b :c1886; + :b :c1887; + :b :c1888; + :b :c1889; + :b :c1890; + :b :c1891; + :b :c1892; + :b :c1893; + :b :c1894; + :b :c1895; + :b :c1896; + :b :c1897; + :b :c1898; + :b :c1899; + :b :c1900; + :b :c1901; + :b :c1902; + :b :c1903; + :b :c1904; + :b :c1905; + :b :c1906; + :b :c1907; + :b :c1908; + :b :c1909; + :b :c1910; + :b :c1911; + :b :c1912; + :b :c1913; + :b :c1914; + :b :c1915; + :b :c1916; + :b :c1917; + :b :c1918; + :b :c1919; + :b :c1920; + :b :c1921; + :b :c1922; + :b :c1923; + :b :c1924; + :b :c1925; + :b :c1926; + :b :c1927; + :b :c1928; + :b :c1929; + :b :c1930; + :b :c1931; + :b :c1932; + :b :c1933; + :b :c1934; + :b :c1935; + :b :c1936; + :b :c1937; + :b :c1938; + :b :c1939; + :b :c1940; + :b :c1941; + :b :c1942; + :b :c1943; + :b :c1944; + :b :c1945; + :b :c1946; + :b :c1947; + :b :c1948; + :b :c1949; + :b :c1950; + :b :c1951; + :b :c1952; + :b :c1953; + :b :c1954; + :b :c1955; + :b :c1956; + :b :c1957; + :b :c1958; + :b :c1959; + :b :c1960; + :b :c1961; + :b :c1962; + :b :c1963; + :b :c1964; + :b :c1965; + :b :c1966; + :b :c1967; + :b :c1968; + :b :c1969; + :b :c1970; + :b :c1971; + :b :c1972; + :b :c1973; + :b :c1974; + :b :c1975; + :b :c1976; + :b :c1977; + :b :c1978; + :b :c1979; + :b :c1980; + :b :c1981; + :b :c1982; + :b :c1983; + :b :c1984; + :b :c1985; + :b :c1986; + :b :c1987; + :b :c1988; + :b :c1989; + :b :c1990; + :b :c1991; + :b :c1992; + :b :c1993; + :b :c1994; + :b :c1995; + :b :c1996; + :b :c1997; + :b :c1998; + :b :c1999; + :b :c2000; + :b :c2001; + :b :c2002; + :b :c2003; + :b :c2004; + :b :c2005; + :b :c2006; + :b :c2007; + :b :c2008; + :b :c2009; + :b :c2010; + :b :c2011; + :b :c2012; + :b :c2013; + :b :c2014; + :b :c2015; + :b :c2016; + :b :c2017; + :b :c2018; + :b :c2019; + :b :c2020; + :b :c2021; + :b :c2022; + :b :c2023; + :b :c2024; + :b :c2025; + :b :c2026; + :b :c2027; + :b :c2028; + :b :c2029; + :b :c2030; + :b :c2031; + :b :c2032; + :b :c2033; + :b :c2034; + :b :c2035; + :b :c2036; + :b :c2037; + :b :c2038; + :b :c2039; + :b :c2040; + :b :c2041; + :b :c2042; + :b :c2043; + :b :c2044; + :b :c2045; + :b :c2046; + :b :c2047; + :b :c2048; + :b :c2049; + :b :c2050; + :b :c2051; + :b :c2052; + :b :c2053; + :b :c2054; + :b :c2055; + :b :c2056; + :b :c2057; + :b :c2058; + :b :c2059; + :b :c2060; + :b :c2061; + :b :c2062; + :b :c2063; + :b :c2064; + :b :c2065; + :b :c2066; + :b :c2067; + :b :c2068; + :b :c2069; + :b :c2070; + :b :c2071; + :b :c2072; + :b :c2073; + :b :c2074; + :b :c2075; + :b :c2076; + :b :c2077; + :b :c2078; + :b :c2079; + :b :c2080; + :b :c2081; + :b :c2082; + :b :c2083; + :b :c2084; + :b :c2085; + :b :c2086; + :b :c2087; + :b :c2088; + :b :c2089; + :b :c2090; + :b :c2091; + :b :c2092; + :b :c2093; + :b :c2094; + :b :c2095; + :b :c2096; + :b :c2097; + :b :c2098; + :b :c2099; + :b :c2100; + :b :c2101; + :b :c2102; + :b :c2103; + :b :c2104; + :b :c2105; + :b :c2106; + :b :c2107; + :b :c2108; + :b :c2109; + :b :c2110; + :b :c2111; + :b :c2112; + :b :c2113; + :b :c2114; + :b :c2115; + :b :c2116; + :b :c2117; + :b :c2118; + :b :c2119; + :b :c2120; + :b :c2121; + :b :c2122; + :b :c2123; + :b :c2124; + :b :c2125; + :b :c2126; + :b :c2127; + :b :c2128; + :b :c2129; + :b :c2130; + :b :c2131; + :b :c2132; + :b :c2133; + :b :c2134; + :b :c2135; + :b :c2136; + :b :c2137; + :b :c2138; + :b :c2139; + :b :c2140; + :b :c2141; + :b :c2142; + :b :c2143; + :b :c2144; + :b :c2145; + :b :c2146; + :b :c2147; + :b :c2148; + :b :c2149; + :b :c2150; + :b :c2151; + :b :c2152; + :b :c2153; + :b :c2154; + :b :c2155; + :b :c2156; + :b :c2157; + :b :c2158; + :b :c2159; + :b :c2160; + :b :c2161; + :b :c2162; + :b :c2163; + :b :c2164; + :b :c2165; + :b :c2166; + :b :c2167; + :b :c2168; + :b :c2169; + :b :c2170; + :b :c2171; + :b :c2172; + :b :c2173; + :b :c2174; + :b :c2175; + :b :c2176; + :b :c2177; + :b :c2178; + :b :c2179; + :b :c2180; + :b :c2181; + :b :c2182; + :b :c2183; + :b :c2184; + :b :c2185; + :b :c2186; + :b :c2187; + :b :c2188; + :b :c2189; + :b :c2190; + :b :c2191; + :b :c2192; + :b :c2193; + :b :c2194; + :b :c2195; + :b :c2196; + :b :c2197; + :b :c2198; + :b :c2199; + :b :c2200; + :b :c2201; + :b :c2202; + :b :c2203; + :b :c2204; + :b :c2205; + :b :c2206; + :b :c2207; + :b :c2208; + :b :c2209; + :b :c2210; + :b :c2211; + :b :c2212; + :b :c2213; + :b :c2214; + :b :c2215; + :b :c2216; + :b :c2217; + :b :c2218; + :b :c2219; + :b :c2220; + :b :c2221; + :b :c2222; + :b :c2223; + :b :c2224; + :b :c2225; + :b :c2226; + :b :c2227; + :b :c2228; + :b :c2229; + :b :c2230; + :b :c2231; + :b :c2232; + :b :c2233; + :b :c2234; + :b :c2235; + :b :c2236; + :b :c2237; + :b :c2238; + :b :c2239; + :b :c2240; + :b :c2241; + :b :c2242; + :b :c2243; + :b :c2244; + :b :c2245; + :b :c2246; + :b :c2247; + :b :c2248; + :b :c2249; + :b :c2250; + :b :c2251; + :b :c2252; + :b :c2253; + :b :c2254; + :b :c2255; + :b :c2256; + :b :c2257; + :b :c2258; + :b :c2259; + :b :c2260; + :b :c2261; + :b :c2262; + :b :c2263; + :b :c2264; + :b :c2265; + :b :c2266; + :b :c2267; + :b :c2268; + :b :c2269; + :b :c2270; + :b :c2271; + :b :c2272; + :b :c2273; + :b :c2274; + :b :c2275; + :b :c2276; + :b :c2277; + :b :c2278; + :b :c2279; + :b :c2280; + :b :c2281; + :b :c2282; + :b :c2283; + :b :c2284; + :b :c2285; + :b :c2286; + :b :c2287; + :b :c2288; + :b :c2289; + :b :c2290; + :b :c2291; + :b :c2292; + :b :c2293; + :b :c2294; + :b :c2295; + :b :c2296; + :b :c2297; + :b :c2298; + :b :c2299; + :b :c2300; + :b :c2301; + :b :c2302; + :b :c2303; + :b :c2304; + :b :c2305; + :b :c2306; + :b :c2307; + :b :c2308; + :b :c2309; + :b :c2310; + :b :c2311; + :b :c2312; + :b :c2313; + :b :c2314; + :b :c2315; + :b :c2316; + :b :c2317; + :b :c2318; + :b :c2319; + :b :c2320; + :b :c2321; + :b :c2322; + :b :c2323; + :b :c2324; + :b :c2325; + :b :c2326; + :b :c2327; + :b :c2328; + :b :c2329; + :b :c2330; + :b :c2331; + :b :c2332; + :b :c2333; + :b :c2334; + :b :c2335; + :b :c2336; + :b :c2337; + :b :c2338; + :b :c2339; + :b :c2340; + :b :c2341; + :b :c2342; + :b :c2343; + :b :c2344; + :b :c2345; + :b :c2346; + :b :c2347; + :b :c2348; + :b :c2349; + :b :c2350; + :b :c2351; + :b :c2352; + :b :c2353; + :b :c2354; + :b :c2355; + :b :c2356; + :b :c2357; + :b :c2358; + :b :c2359; + :b :c2360; + :b :c2361; + :b :c2362; + :b :c2363; + :b :c2364; + :b :c2365; + :b :c2366; + :b :c2367; + :b :c2368; + :b :c2369; + :b :c2370; + :b :c2371; + :b :c2372; + :b :c2373; + :b :c2374; + :b :c2375; + :b :c2376; + :b :c2377; + :b :c2378; + :b :c2379; + :b :c2380; + :b :c2381; + :b :c2382; + :b :c2383; + :b :c2384; + :b :c2385; + :b :c2386; + :b :c2387; + :b :c2388; + :b :c2389; + :b :c2390; + :b :c2391; + :b :c2392; + :b :c2393; + :b :c2394; + :b :c2395; + :b :c2396; + :b :c2397; + :b :c2398; + :b :c2399; + :b :c2400; + :b :c2401; + :b :c2402; + :b :c2403; + :b :c2404; + :b :c2405; + :b :c2406; + :b :c2407; + :b :c2408; + :b :c2409; + :b :c2410; + :b :c2411; + :b :c2412; + :b :c2413; + :b :c2414; + :b :c2415; + :b :c2416; + :b :c2417; + :b :c2418; + :b :c2419; + :b :c2420; + :b :c2421; + :b :c2422; + :b :c2423; + :b :c2424; + :b :c2425; + :b :c2426; + :b :c2427; + :b :c2428; + :b :c2429; + :b :c2430; + :b :c2431; + :b :c2432; + :b :c2433; + :b :c2434; + :b :c2435; + :b :c2436; + :b :c2437; + :b :c2438; + :b :c2439; + :b :c2440; + :b :c2441; + :b :c2442; + :b :c2443; + :b :c2444; + :b :c2445; + :b :c2446; + :b :c2447; + :b :c2448; + :b :c2449; + :b :c2450; + :b :c2451; + :b :c2452; + :b :c2453; + :b :c2454; + :b :c2455; + :b :c2456; + :b :c2457; + :b :c2458; + :b :c2459; + :b :c2460; + :b :c2461; + :b :c2462; + :b :c2463; + :b :c2464; + :b :c2465; + :b :c2466; + :b :c2467; + :b :c2468; + :b :c2469; + :b :c2470; + :b :c2471; + :b :c2472; + :b :c2473; + :b :c2474; + :b :c2475; + :b :c2476; + :b :c2477; + :b :c2478; + :b :c2479; + :b :c2480; + :b :c2481; + :b :c2482; + :b :c2483; + :b :c2484; + :b :c2485; + :b :c2486; + :b :c2487; + :b :c2488; + :b :c2489; + :b :c2490; + :b :c2491; + :b :c2492; + :b :c2493; + :b :c2494; + :b :c2495; + :b :c2496; + :b :c2497; + :b :c2498; + :b :c2499; + :b :c2500; + :b :c2501; + :b :c2502; + :b :c2503; + :b :c2504; + :b :c2505; + :b :c2506; + :b :c2507; + :b :c2508; + :b :c2509; + :b :c2510; + :b :c2511; + :b :c2512; + :b :c2513; + :b :c2514; + :b :c2515; + :b :c2516; + :b :c2517; + :b :c2518; + :b :c2519; + :b :c2520; + :b :c2521; + :b :c2522; + :b :c2523; + :b :c2524; + :b :c2525; + :b :c2526; + :b :c2527; + :b :c2528; + :b :c2529; + :b :c2530; + :b :c2531; + :b :c2532; + :b :c2533; + :b :c2534; + :b :c2535; + :b :c2536; + :b :c2537; + :b :c2538; + :b :c2539; + :b :c2540; + :b :c2541; + :b :c2542; + :b :c2543; + :b :c2544; + :b :c2545; + :b :c2546; + :b :c2547; + :b :c2548; + :b :c2549; + :b :c2550; + :b :c2551; + :b :c2552; + :b :c2553; + :b :c2554; + :b :c2555; + :b :c2556; + :b :c2557; + :b :c2558; + :b :c2559; + :b :c2560; + :b :c2561; + :b :c2562; + :b :c2563; + :b :c2564; + :b :c2565; + :b :c2566; + :b :c2567; + :b :c2568; + :b :c2569; + :b :c2570; + :b :c2571; + :b :c2572; + :b :c2573; + :b :c2574; + :b :c2575; + :b :c2576; + :b :c2577; + :b :c2578; + :b :c2579; + :b :c2580; + :b :c2581; + :b :c2582; + :b :c2583; + :b :c2584; + :b :c2585; + :b :c2586; + :b :c2587; + :b :c2588; + :b :c2589; + :b :c2590; + :b :c2591; + :b :c2592; + :b :c2593; + :b :c2594; + :b :c2595; + :b :c2596; + :b :c2597; + :b :c2598; + :b :c2599; + :b :c2600; + :b :c2601; + :b :c2602; + :b :c2603; + :b :c2604; + :b :c2605; + :b :c2606; + :b :c2607; + :b :c2608; + :b :c2609; + :b :c2610; + :b :c2611; + :b :c2612; + :b :c2613; + :b :c2614; + :b :c2615; + :b :c2616; + :b :c2617; + :b :c2618; + :b :c2619; + :b :c2620; + :b :c2621; + :b :c2622; + :b :c2623; + :b :c2624; + :b :c2625; + :b :c2626; + :b :c2627; + :b :c2628; + :b :c2629; + :b :c2630; + :b :c2631; + :b :c2632; + :b :c2633; + :b :c2634; + :b :c2635; + :b :c2636; + :b :c2637; + :b :c2638; + :b :c2639; + :b :c2640; + :b :c2641; + :b :c2642; + :b :c2643; + :b :c2644; + :b :c2645; + :b :c2646; + :b :c2647; + :b :c2648; + :b :c2649; + :b :c2650; + :b :c2651; + :b :c2652; + :b :c2653; + :b :c2654; + :b :c2655; + :b :c2656; + :b :c2657; + :b :c2658; + :b :c2659; + :b :c2660; + :b :c2661; + :b :c2662; + :b :c2663; + :b :c2664; + :b :c2665; + :b :c2666; + :b :c2667; + :b :c2668; + :b :c2669; + :b :c2670; + :b :c2671; + :b :c2672; + :b :c2673; + :b :c2674; + :b :c2675; + :b :c2676; + :b :c2677; + :b :c2678; + :b :c2679; + :b :c2680; + :b :c2681; + :b :c2682; + :b :c2683; + :b :c2684; + :b :c2685; + :b :c2686; + :b :c2687; + :b :c2688; + :b :c2689; + :b :c2690; + :b :c2691; + :b :c2692; + :b :c2693; + :b :c2694; + :b :c2695; + :b :c2696; + :b :c2697; + :b :c2698; + :b :c2699; + :b :c2700; + :b :c2701; + :b :c2702; + :b :c2703; + :b :c2704; + :b :c2705; + :b :c2706; + :b :c2707; + :b :c2708; + :b :c2709; + :b :c2710; + :b :c2711; + :b :c2712; + :b :c2713; + :b :c2714; + :b :c2715; + :b :c2716; + :b :c2717; + :b :c2718; + :b :c2719; + :b :c2720; + :b :c2721; + :b :c2722; + :b :c2723; + :b :c2724; + :b :c2725; + :b :c2726; + :b :c2727; + :b :c2728; + :b :c2729; + :b :c2730; + :b :c2731; + :b :c2732; + :b :c2733; + :b :c2734; + :b :c2735; + :b :c2736; + :b :c2737; + :b :c2738; + :b :c2739; + :b :c2740; + :b :c2741; + :b :c2742; + :b :c2743; + :b :c2744; + :b :c2745; + :b :c2746; + :b :c2747; + :b :c2748; + :b :c2749; + :b :c2750; + :b :c2751; + :b :c2752; + :b :c2753; + :b :c2754; + :b :c2755; + :b :c2756; + :b :c2757; + :b :c2758; + :b :c2759; + :b :c2760; + :b :c2761; + :b :c2762; + :b :c2763; + :b :c2764; + :b :c2765; + :b :c2766; + :b :c2767; + :b :c2768; + :b :c2769; + :b :c2770; + :b :c2771; + :b :c2772; + :b :c2773; + :b :c2774; + :b :c2775; + :b :c2776; + :b :c2777; + :b :c2778; + :b :c2779; + :b :c2780; + :b :c2781; + :b :c2782; + :b :c2783; + :b :c2784; + :b :c2785; + :b :c2786; + :b :c2787; + :b :c2788; + :b :c2789; + :b :c2790; + :b :c2791; + :b :c2792; + :b :c2793; + :b :c2794; + :b :c2795; + :b :c2796; + :b :c2797; + :b :c2798; + :b :c2799; + :b :c2800; + :b :c2801; + :b :c2802; + :b :c2803; + :b :c2804; + :b :c2805; + :b :c2806; + :b :c2807; + :b :c2808; + :b :c2809; + :b :c2810; + :b :c2811; + :b :c2812; + :b :c2813; + :b :c2814; + :b :c2815; + :b :c2816; + :b :c2817; + :b :c2818; + :b :c2819; + :b :c2820; + :b :c2821; + :b :c2822; + :b :c2823; + :b :c2824; + :b :c2825; + :b :c2826; + :b :c2827; + :b :c2828; + :b :c2829; + :b :c2830; + :b :c2831; + :b :c2832; + :b :c2833; + :b :c2834; + :b :c2835; + :b :c2836; + :b :c2837; + :b :c2838; + :b :c2839; + :b :c2840; + :b :c2841; + :b :c2842; + :b :c2843; + :b :c2844; + :b :c2845; + :b :c2846; + :b :c2847; + :b :c2848; + :b :c2849; + :b :c2850; + :b :c2851; + :b :c2852; + :b :c2853; + :b :c2854; + :b :c2855; + :b :c2856; + :b :c2857; + :b :c2858; + :b :c2859; + :b :c2860; + :b :c2861; + :b :c2862; + :b :c2863; + :b :c2864; + :b :c2865; + :b :c2866; + :b :c2867; + :b :c2868; + :b :c2869; + :b :c2870; + :b :c2871; + :b :c2872; + :b :c2873; + :b :c2874; + :b :c2875; + :b :c2876; + :b :c2877; + :b :c2878; + :b :c2879; + :b :c2880; + :b :c2881; + :b :c2882; + :b :c2883; + :b :c2884; + :b :c2885; + :b :c2886; + :b :c2887; + :b :c2888; + :b :c2889; + :b :c2890; + :b :c2891; + :b :c2892; + :b :c2893; + :b :c2894; + :b :c2895; + :b :c2896; + :b :c2897; + :b :c2898; + :b :c2899; + :b :c2900; + :b :c2901; + :b :c2902; + :b :c2903; + :b :c2904; + :b :c2905; + :b :c2906; + :b :c2907; + :b :c2908; + :b :c2909; + :b :c2910; + :b :c2911; + :b :c2912; + :b :c2913; + :b :c2914; + :b :c2915; + :b :c2916; + :b :c2917; + :b :c2918; + :b :c2919; + :b :c2920; + :b :c2921; + :b :c2922; + :b :c2923; + :b :c2924; + :b :c2925; + :b :c2926; + :b :c2927; + :b :c2928; + :b :c2929; + :b :c2930; + :b :c2931; + :b :c2932; + :b :c2933; + :b :c2934; + :b :c2935; + :b :c2936; + :b :c2937; + :b :c2938; + :b :c2939; + :b :c2940; + :b :c2941; + :b :c2942; + :b :c2943; + :b :c2944; + :b :c2945; + :b :c2946; + :b :c2947; + :b :c2948; + :b :c2949; + :b :c2950; + :b :c2951; + :b :c2952; + :b :c2953; + :b :c2954; + :b :c2955; + :b :c2956; + :b :c2957; + :b :c2958; + :b :c2959; + :b :c2960; + :b :c2961; + :b :c2962; + :b :c2963; + :b :c2964; + :b :c2965; + :b :c2966; + :b :c2967; + :b :c2968; + :b :c2969; + :b :c2970; + :b :c2971; + :b :c2972; + :b :c2973; + :b :c2974; + :b :c2975; + :b :c2976; + :b :c2977; + :b :c2978; + :b :c2979; + :b :c2980; + :b :c2981; + :b :c2982; + :b :c2983; + :b :c2984; + :b :c2985; + :b :c2986; + :b :c2987; + :b :c2988; + :b :c2989; + :b :c2990; + :b :c2991; + :b :c2992; + :b :c2993; + :b :c2994; + :b :c2995; + :b :c2996; + :b :c2997; + :b :c2998; + :b :c2999; + :b :c3000; + :b :c3001; + :b :c3002; + :b :c3003; + :b :c3004; + :b :c3005; + :b :c3006; + :b :c3007; + :b :c3008; + :b :c3009; + :b :c3010; + :b :c3011; + :b :c3012; + :b :c3013; + :b :c3014; + :b :c3015; + :b :c3016; + :b :c3017; + :b :c3018; + :b :c3019; + :b :c3020; + :b :c3021; + :b :c3022; + :b :c3023; + :b :c3024; + :b :c3025; + :b :c3026; + :b :c3027; + :b :c3028; + :b :c3029; + :b :c3030; + :b :c3031; + :b :c3032; + :b :c3033; + :b :c3034; + :b :c3035; + :b :c3036; + :b :c3037; + :b :c3038; + :b :c3039; + :b :c3040; + :b :c3041; + :b :c3042; + :b :c3043; + :b :c3044; + :b :c3045; + :b :c3046; + :b :c3047; + :b :c3048; + :b :c3049; + :b :c3050; + :b :c3051; + :b :c3052; + :b :c3053; + :b :c3054; + :b :c3055; + :b :c3056; + :b :c3057; + :b :c3058; + :b :c3059; + :b :c3060; + :b :c3061; + :b :c3062; + :b :c3063; + :b :c3064; + :b :c3065; + :b :c3066; + :b :c3067; + :b :c3068; + :b :c3069; + :b :c3070; + :b :c3071; + :b :c3072; + :b :c3073; + :b :c3074; + :b :c3075; + :b :c3076; + :b :c3077; + :b :c3078; + :b :c3079; + :b :c3080; + :b :c3081; + :b :c3082; + :b :c3083; + :b :c3084; + :b :c3085; + :b :c3086; + :b :c3087; + :b :c3088; + :b :c3089; + :b :c3090; + :b :c3091; + :b :c3092; + :b :c3093; + :b :c3094; + :b :c3095; + :b :c3096; + :b :c3097; + :b :c3098; + :b :c3099; + :b :c3100; + :b :c3101; + :b :c3102; + :b :c3103; + :b :c3104; + :b :c3105; + :b :c3106; + :b :c3107; + :b :c3108; + :b :c3109; + :b :c3110; + :b :c3111; + :b :c3112; + :b :c3113; + :b :c3114; + :b :c3115; + :b :c3116; + :b :c3117; + :b :c3118; + :b :c3119; + :b :c3120; + :b :c3121; + :b :c3122; + :b :c3123; + :b :c3124; + :b :c3125; + :b :c3126; + :b :c3127; + :b :c3128; + :b :c3129; + :b :c3130; + :b :c3131; + :b :c3132; + :b :c3133; + :b :c3134; + :b :c3135; + :b :c3136; + :b :c3137; + :b :c3138; + :b :c3139; + :b :c3140; + :b :c3141; + :b :c3142; + :b :c3143; + :b :c3144; + :b :c3145; + :b :c3146; + :b :c3147; + :b :c3148; + :b :c3149; + :b :c3150; + :b :c3151; + :b :c3152; + :b :c3153; + :b :c3154; + :b :c3155; + :b :c3156; + :b :c3157; + :b :c3158; + :b :c3159; + :b :c3160; + :b :c3161; + :b :c3162; + :b :c3163; + :b :c3164; + :b :c3165; + :b :c3166; + :b :c3167; + :b :c3168; + :b :c3169; + :b :c3170; + :b :c3171; + :b :c3172; + :b :c3173; + :b :c3174; + :b :c3175; + :b :c3176; + :b :c3177; + :b :c3178; + :b :c3179; + :b :c3180; + :b :c3181; + :b :c3182; + :b :c3183; + :b :c3184; + :b :c3185; + :b :c3186; + :b :c3187; + :b :c3188; + :b :c3189; + :b :c3190; + :b :c3191; + :b :c3192; + :b :c3193; + :b :c3194; + :b :c3195; + :b :c3196; + :b :c3197; + :b :c3198; + :b :c3199; + :b :c3200; + :b :c3201; + :b :c3202; + :b :c3203; + :b :c3204; + :b :c3205; + :b :c3206; + :b :c3207; + :b :c3208; + :b :c3209; + :b :c3210; + :b :c3211; + :b :c3212; + :b :c3213; + :b :c3214; + :b :c3215; + :b :c3216; + :b :c3217; + :b :c3218; + :b :c3219; + :b :c3220; + :b :c3221; + :b :c3222; + :b :c3223; + :b :c3224; + :b :c3225; + :b :c3226; + :b :c3227; + :b :c3228; + :b :c3229; + :b :c3230; + :b :c3231; + :b :c3232; + :b :c3233; + :b :c3234; + :b :c3235; + :b :c3236; + :b :c3237; + :b :c3238; + :b :c3239; + :b :c3240; + :b :c3241; + :b :c3242; + :b :c3243; + :b :c3244; + :b :c3245; + :b :c3246; + :b :c3247; + :b :c3248; + :b :c3249; + :b :c3250; + :b :c3251; + :b :c3252; + :b :c3253; + :b :c3254; + :b :c3255; + :b :c3256; + :b :c3257; + :b :c3258; + :b :c3259; + :b :c3260; + :b :c3261; + :b :c3262; + :b :c3263; + :b :c3264; + :b :c3265; + :b :c3266; + :b :c3267; + :b :c3268; + :b :c3269; + :b :c3270; + :b :c3271; + :b :c3272; + :b :c3273; + :b :c3274; + :b :c3275; + :b :c3276; + :b :c3277; + :b :c3278; + :b :c3279; + :b :c3280; + :b :c3281; + :b :c3282; + :b :c3283; + :b :c3284; + :b :c3285; + :b :c3286; + :b :c3287; + :b :c3288; + :b :c3289; + :b :c3290; + :b :c3291; + :b :c3292; + :b :c3293; + :b :c3294; + :b :c3295; + :b :c3296; + :b :c3297; + :b :c3298; + :b :c3299; + :b :c3300; + :b :c3301; + :b :c3302; + :b :c3303; + :b :c3304; + :b :c3305; + :b :c3306; + :b :c3307; + :b :c3308; + :b :c3309; + :b :c3310; + :b :c3311; + :b :c3312; + :b :c3313; + :b :c3314; + :b :c3315; + :b :c3316; + :b :c3317; + :b :c3318; + :b :c3319; + :b :c3320; + :b :c3321; + :b :c3322; + :b :c3323; + :b :c3324; + :b :c3325; + :b :c3326; + :b :c3327; + :b :c3328; + :b :c3329; + :b :c3330; + :b :c3331; + :b :c3332; + :b :c3333; + :b :c3334; + :b :c3335; + :b :c3336; + :b :c3337; + :b :c3338; + :b :c3339; + :b :c3340; + :b :c3341; + :b :c3342; + :b :c3343; + :b :c3344; + :b :c3345; + :b :c3346; + :b :c3347; + :b :c3348; + :b :c3349; + :b :c3350; + :b :c3351; + :b :c3352; + :b :c3353; + :b :c3354; + :b :c3355; + :b :c3356; + :b :c3357; + :b :c3358; + :b :c3359; + :b :c3360; + :b :c3361; + :b :c3362; + :b :c3363; + :b :c3364; + :b :c3365; + :b :c3366; + :b :c3367; + :b :c3368; + :b :c3369; + :b :c3370; + :b :c3371; + :b :c3372; + :b :c3373; + :b :c3374; + :b :c3375; + :b :c3376; + :b :c3377; + :b :c3378; + :b :c3379; + :b :c3380; + :b :c3381; + :b :c3382; + :b :c3383; + :b :c3384; + :b :c3385; + :b :c3386; + :b :c3387; + :b :c3388; + :b :c3389; + :b :c3390; + :b :c3391; + :b :c3392; + :b :c3393; + :b :c3394; + :b :c3395; + :b :c3396; + :b :c3397; + :b :c3398; + :b :c3399; + :b :c3400; + :b :c3401; + :b :c3402; + :b :c3403; + :b :c3404; + :b :c3405; + :b :c3406; + :b :c3407; + :b :c3408; + :b :c3409; + :b :c3410; + :b :c3411; + :b :c3412; + :b :c3413; + :b :c3414; + :b :c3415; + :b :c3416; + :b :c3417; + :b :c3418; + :b :c3419; + :b :c3420; + :b :c3421; + :b :c3422; + :b :c3423; + :b :c3424; + :b :c3425; + :b :c3426; + :b :c3427; + :b :c3428; + :b :c3429; + :b :c3430; + :b :c3431; + :b :c3432; + :b :c3433; + :b :c3434; + :b :c3435; + :b :c3436; + :b :c3437; + :b :c3438; + :b :c3439; + :b :c3440; + :b :c3441; + :b :c3442; + :b :c3443; + :b :c3444; + :b :c3445; + :b :c3446; + :b :c3447; + :b :c3448; + :b :c3449; + :b :c3450; + :b :c3451; + :b :c3452; + :b :c3453; + :b :c3454; + :b :c3455; + :b :c3456; + :b :c3457; + :b :c3458; + :b :c3459; + :b :c3460; + :b :c3461; + :b :c3462; + :b :c3463; + :b :c3464; + :b :c3465; + :b :c3466; + :b :c3467; + :b :c3468; + :b :c3469; + :b :c3470; + :b :c3471; + :b :c3472; + :b :c3473; + :b :c3474; + :b :c3475; + :b :c3476; + :b :c3477; + :b :c3478; + :b :c3479; + :b :c3480; + :b :c3481; + :b :c3482; + :b :c3483; + :b :c3484; + :b :c3485; + :b :c3486; + :b :c3487; + :b :c3488; + :b :c3489; + :b :c3490; + :b :c3491; + :b :c3492; + :b :c3493; + :b :c3494; + :b :c3495; + :b :c3496; + :b :c3497; + :b :c3498; + :b :c3499; + :b :c3500; + :b :c3501; + :b :c3502; + :b :c3503; + :b :c3504; + :b :c3505; + :b :c3506; + :b :c3507; + :b :c3508; + :b :c3509; + :b :c3510; + :b :c3511; + :b :c3512; + :b :c3513; + :b :c3514; + :b :c3515; + :b :c3516; + :b :c3517; + :b :c3518; + :b :c3519; + :b :c3520; + :b :c3521; + :b :c3522; + :b :c3523; + :b :c3524; + :b :c3525; + :b :c3526; + :b :c3527; + :b :c3528; + :b :c3529; + :b :c3530; + :b :c3531; + :b :c3532; + :b :c3533; + :b :c3534; + :b :c3535; + :b :c3536; + :b :c3537; + :b :c3538; + :b :c3539; + :b :c3540; + :b :c3541; + :b :c3542; + :b :c3543; + :b :c3544; + :b :c3545; + :b :c3546; + :b :c3547; + :b :c3548; + :b :c3549; + :b :c3550; + :b :c3551; + :b :c3552; + :b :c3553; + :b :c3554; + :b :c3555; + :b :c3556; + :b :c3557; + :b :c3558; + :b :c3559; + :b :c3560; + :b :c3561; + :b :c3562; + :b :c3563; + :b :c3564; + :b :c3565; + :b :c3566; + :b :c3567; + :b :c3568; + :b :c3569; + :b :c3570; + :b :c3571; + :b :c3572; + :b :c3573; + :b :c3574; + :b :c3575; + :b :c3576; + :b :c3577; + :b :c3578; + :b :c3579; + :b :c3580; + :b :c3581; + :b :c3582; + :b :c3583; + :b :c3584; + :b :c3585; + :b :c3586; + :b :c3587; + :b :c3588; + :b :c3589; + :b :c3590; + :b :c3591; + :b :c3592; + :b :c3593; + :b :c3594; + :b :c3595; + :b :c3596; + :b :c3597; + :b :c3598; + :b :c3599; + :b :c3600; + :b :c3601; + :b :c3602; + :b :c3603; + :b :c3604; + :b :c3605; + :b :c3606; + :b :c3607; + :b :c3608; + :b :c3609; + :b :c3610; + :b :c3611; + :b :c3612; + :b :c3613; + :b :c3614; + :b :c3615; + :b :c3616; + :b :c3617; + :b :c3618; + :b :c3619; + :b :c3620; + :b :c3621; + :b :c3622; + :b :c3623; + :b :c3624; + :b :c3625; + :b :c3626; + :b :c3627; + :b :c3628; + :b :c3629; + :b :c3630; + :b :c3631; + :b :c3632; + :b :c3633; + :b :c3634; + :b :c3635; + :b :c3636; + :b :c3637; + :b :c3638; + :b :c3639; + :b :c3640; + :b :c3641; + :b :c3642; + :b :c3643; + :b :c3644; + :b :c3645; + :b :c3646; + :b :c3647; + :b :c3648; + :b :c3649; + :b :c3650; + :b :c3651; + :b :c3652; + :b :c3653; + :b :c3654; + :b :c3655; + :b :c3656; + :b :c3657; + :b :c3658; + :b :c3659; + :b :c3660; + :b :c3661; + :b :c3662; + :b :c3663; + :b :c3664; + :b :c3665; + :b :c3666; + :b :c3667; + :b :c3668; + :b :c3669; + :b :c3670; + :b :c3671; + :b :c3672; + :b :c3673; + :b :c3674; + :b :c3675; + :b :c3676; + :b :c3677; + :b :c3678; + :b :c3679; + :b :c3680; + :b :c3681; + :b :c3682; + :b :c3683; + :b :c3684; + :b :c3685; + :b :c3686; + :b :c3687; + :b :c3688; + :b :c3689; + :b :c3690; + :b :c3691; + :b :c3692; + :b :c3693; + :b :c3694; + :b :c3695; + :b :c3696; + :b :c3697; + :b :c3698; + :b :c3699; + :b :c3700; + :b :c3701; + :b :c3702; + :b :c3703; + :b :c3704; + :b :c3705; + :b :c3706; + :b :c3707; + :b :c3708; + :b :c3709; + :b :c3710; + :b :c3711; + :b :c3712; + :b :c3713; + :b :c3714; + :b :c3715; + :b :c3716; + :b :c3717; + :b :c3718; + :b :c3719; + :b :c3720; + :b :c3721; + :b :c3722; + :b :c3723; + :b :c3724; + :b :c3725; + :b :c3726; + :b :c3727; + :b :c3728; + :b :c3729; + :b :c3730; + :b :c3731; + :b :c3732; + :b :c3733; + :b :c3734; + :b :c3735; + :b :c3736; + :b :c3737; + :b :c3738; + :b :c3739; + :b :c3740; + :b :c3741; + :b :c3742; + :b :c3743; + :b :c3744; + :b :c3745; + :b :c3746; + :b :c3747; + :b :c3748; + :b :c3749; + :b :c3750; + :b :c3751; + :b :c3752; + :b :c3753; + :b :c3754; + :b :c3755; + :b :c3756; + :b :c3757; + :b :c3758; + :b :c3759; + :b :c3760; + :b :c3761; + :b :c3762; + :b :c3763; + :b :c3764; + :b :c3765; + :b :c3766; + :b :c3767; + :b :c3768; + :b :c3769; + :b :c3770; + :b :c3771; + :b :c3772; + :b :c3773; + :b :c3774; + :b :c3775; + :b :c3776; + :b :c3777; + :b :c3778; + :b :c3779; + :b :c3780; + :b :c3781; + :b :c3782; + :b :c3783; + :b :c3784; + :b :c3785; + :b :c3786; + :b :c3787; + :b :c3788; + :b :c3789; + :b :c3790; + :b :c3791; + :b :c3792; + :b :c3793; + :b :c3794; + :b :c3795; + :b :c3796; + :b :c3797; + :b :c3798; + :b :c3799; + :b :c3800; + :b :c3801; + :b :c3802; + :b :c3803; + :b :c3804; + :b :c3805; + :b :c3806; + :b :c3807; + :b :c3808; + :b :c3809; + :b :c3810; + :b :c3811; + :b :c3812; + :b :c3813; + :b :c3814; + :b :c3815; + :b :c3816; + :b :c3817; + :b :c3818; + :b :c3819; + :b :c3820; + :b :c3821; + :b :c3822; + :b :c3823; + :b :c3824; + :b :c3825; + :b :c3826; + :b :c3827; + :b :c3828; + :b :c3829; + :b :c3830; + :b :c3831; + :b :c3832; + :b :c3833; + :b :c3834; + :b :c3835; + :b :c3836; + :b :c3837; + :b :c3838; + :b :c3839; + :b :c3840; + :b :c3841; + :b :c3842; + :b :c3843; + :b :c3844; + :b :c3845; + :b :c3846; + :b :c3847; + :b :c3848; + :b :c3849; + :b :c3850; + :b :c3851; + :b :c3852; + :b :c3853; + :b :c3854; + :b :c3855; + :b :c3856; + :b :c3857; + :b :c3858; + :b :c3859; + :b :c3860; + :b :c3861; + :b :c3862; + :b :c3863; + :b :c3864; + :b :c3865; + :b :c3866; + :b :c3867; + :b :c3868; + :b :c3869; + :b :c3870; + :b :c3871; + :b :c3872; + :b :c3873; + :b :c3874; + :b :c3875; + :b :c3876; + :b :c3877; + :b :c3878; + :b :c3879; + :b :c3880; + :b :c3881; + :b :c3882; + :b :c3883; + :b :c3884; + :b :c3885; + :b :c3886; + :b :c3887; + :b :c3888; + :b :c3889; + :b :c3890; + :b :c3891; + :b :c3892; + :b :c3893; + :b :c3894; + :b :c3895; + :b :c3896; + :b :c3897; + :b :c3898; + :b :c3899; + :b :c3900; + :b :c3901; + :b :c3902; + :b :c3903; + :b :c3904; + :b :c3905; + :b :c3906; + :b :c3907; + :b :c3908; + :b :c3909; + :b :c3910; + :b :c3911; + :b :c3912; + :b :c3913; + :b :c3914; + :b :c3915; + :b :c3916; + :b :c3917; + :b :c3918; + :b :c3919; + :b :c3920; + :b :c3921; + :b :c3922; + :b :c3923; + :b :c3924; + :b :c3925; + :b :c3926; + :b :c3927; + :b :c3928; + :b :c3929; + :b :c3930; + :b :c3931; + :b :c3932; + :b :c3933; + :b :c3934; + :b :c3935; + :b :c3936; + :b :c3937; + :b :c3938; + :b :c3939; + :b :c3940; + :b :c3941; + :b :c3942; + :b :c3943; + :b :c3944; + :b :c3945; + :b :c3946; + :b :c3947; + :b :c3948; + :b :c3949; + :b :c3950; + :b :c3951; + :b :c3952; + :b :c3953; + :b :c3954; + :b :c3955; + :b :c3956; + :b :c3957; + :b :c3958; + :b :c3959; + :b :c3960; + :b :c3961; + :b :c3962; + :b :c3963; + :b :c3964; + :b :c3965; + :b :c3966; + :b :c3967; + :b :c3968; + :b :c3969; + :b :c3970; + :b :c3971; + :b :c3972; + :b :c3973; + :b :c3974; + :b :c3975; + :b :c3976; + :b :c3977; + :b :c3978; + :b :c3979; + :b :c3980; + :b :c3981; + :b :c3982; + :b :c3983; + :b :c3984; + :b :c3985; + :b :c3986; + :b :c3987; + :b :c3988; + :b :c3989; + :b :c3990; + :b :c3991; + :b :c3992; + :b :c3993; + :b :c3994; + :b :c3995; + :b :c3996; + :b :c3997; + :b :c3998; + :b :c3999; + :b :c4000; + :b :c4001; + :b :c4002; + :b :c4003; + :b :c4004; + :b :c4005; + :b :c4006; + :b :c4007; + :b :c4008; + :b :c4009; + :b :c4010; + :b :c4011; + :b :c4012; + :b :c4013; + :b :c4014; + :b :c4015; + :b :c4016; + :b :c4017; + :b :c4018; + :b :c4019; + :b :c4020; + :b :c4021; + :b :c4022; + :b :c4023; + :b :c4024; + :b :c4025; + :b :c4026; + :b :c4027; + :b :c4028; + :b :c4029; + :b :c4030; + :b :c4031; + :b :c4032; + :b :c4033; + :b :c4034; + :b :c4035; + :b :c4036; + :b :c4037; + :b :c4038; + :b :c4039; + :b :c4040; + :b :c4041; + :b :c4042; + :b :c4043; + :b :c4044; + :b :c4045; + :b :c4046; + :b :c4047; + :b :c4048; + :b :c4049; + :b :c4050; + :b :c4051; + :b :c4052; + :b :c4053; + :b :c4054; + :b :c4055; + :b :c4056; + :b :c4057; + :b :c4058; + :b :c4059; + :b :c4060; + :b :c4061; + :b :c4062; + :b :c4063; + :b :c4064; + :b :c4065; + :b :c4066; + :b :c4067; + :b :c4068; + :b :c4069; + :b :c4070; + :b :c4071; + :b :c4072; + :b :c4073; + :b :c4074; + :b :c4075; + :b :c4076; + :b :c4077; + :b :c4078; + :b :c4079; + :b :c4080; + :b :c4081; + :b :c4082; + :b :c4083; + :b :c4084; + :b :c4085; + :b :c4086; + :b :c4087; + :b :c4088; + :b :c4089; + :b :c4090; + :b :c4091; + :b :c4092; + :b :c4093; + :b :c4094; + :b :c4095; + :b :c4096; + :b :c4097; + :b :c4098; + :b :c4099; + :b :c4100; + :b :c4101; + :b :c4102; + :b :c4103; + :b :c4104; + :b :c4105; + :b :c4106; + :b :c4107; + :b :c4108; + :b :c4109; + :b :c4110; + :b :c4111; + :b :c4112; + :b :c4113; + :b :c4114; + :b :c4115; + :b :c4116; + :b :c4117; + :b :c4118; + :b :c4119; + :b :c4120; + :b :c4121; + :b :c4122; + :b :c4123; + :b :c4124; + :b :c4125; + :b :c4126; + :b :c4127; + :b :c4128; + :b :c4129; + :b :c4130; + :b :c4131; + :b :c4132; + :b :c4133; + :b :c4134; + :b :c4135; + :b :c4136; + :b :c4137; + :b :c4138; + :b :c4139; + :b :c4140; + :b :c4141; + :b :c4142; + :b :c4143; + :b :c4144; + :b :c4145; + :b :c4146; + :b :c4147; + :b :c4148; + :b :c4149; + :b :c4150; + :b :c4151; + :b :c4152; + :b :c4153; + :b :c4154; + :b :c4155; + :b :c4156; + :b :c4157; + :b :c4158; + :b :c4159; + :b :c4160; + :b :c4161; + :b :c4162; + :b :c4163; + :b :c4164; + :b :c4165; + :b :c4166; + :b :c4167; + :b :c4168; + :b :c4169; + :b :c4170; + :b :c4171; + :b :c4172; + :b :c4173; + :b :c4174; + :b :c4175; + :b :c4176; + :b :c4177; + :b :c4178; + :b :c4179; + :b :c4180; + :b :c4181; + :b :c4182; + :b :c4183; + :b :c4184; + :b :c4185; + :b :c4186; + :b :c4187; + :b :c4188; + :b :c4189; + :b :c4190; + :b :c4191; + :b :c4192; + :b :c4193; + :b :c4194; + :b :c4195; + :b :c4196; + :b :c4197; + :b :c4198; + :b :c4199; + :b :c4200; + :b :c4201; + :b :c4202; + :b :c4203; + :b :c4204; + :b :c4205; + :b :c4206; + :b :c4207; + :b :c4208; + :b :c4209; + :b :c4210; + :b :c4211; + :b :c4212; + :b :c4213; + :b :c4214; + :b :c4215; + :b :c4216; + :b :c4217; + :b :c4218; + :b :c4219; + :b :c4220; + :b :c4221; + :b :c4222; + :b :c4223; + :b :c4224; + :b :c4225; + :b :c4226; + :b :c4227; + :b :c4228; + :b :c4229; + :b :c4230; + :b :c4231; + :b :c4232; + :b :c4233; + :b :c4234; + :b :c4235; + :b :c4236; + :b :c4237; + :b :c4238; + :b :c4239; + :b :c4240; + :b :c4241; + :b :c4242; + :b :c4243; + :b :c4244; + :b :c4245; + :b :c4246; + :b :c4247; + :b :c4248; + :b :c4249; + :b :c4250; + :b :c4251; + :b :c4252; + :b :c4253; + :b :c4254; + :b :c4255; + :b :c4256; + :b :c4257; + :b :c4258; + :b :c4259; + :b :c4260; + :b :c4261; + :b :c4262; + :b :c4263; + :b :c4264; + :b :c4265; + :b :c4266; + :b :c4267; + :b :c4268; + :b :c4269; + :b :c4270; + :b :c4271; + :b :c4272; + :b :c4273; + :b :c4274; + :b :c4275; + :b :c4276; + :b :c4277; + :b :c4278; + :b :c4279; + :b :c4280; + :b :c4281; + :b :c4282; + :b :c4283; + :b :c4284; + :b :c4285; + :b :c4286; + :b :c4287; + :b :c4288; + :b :c4289; + :b :c4290; + :b :c4291; + :b :c4292; + :b :c4293; + :b :c4294; + :b :c4295; + :b :c4296; + :b :c4297; + :b :c4298; + :b :c4299; + :b :c4300; + :b :c4301; + :b :c4302; + :b :c4303; + :b :c4304; + :b :c4305; + :b :c4306; + :b :c4307; + :b :c4308; + :b :c4309; + :b :c4310; + :b :c4311; + :b :c4312; + :b :c4313; + :b :c4314; + :b :c4315; + :b :c4316; + :b :c4317; + :b :c4318; + :b :c4319; + :b :c4320; + :b :c4321; + :b :c4322; + :b :c4323; + :b :c4324; + :b :c4325; + :b :c4326; + :b :c4327; + :b :c4328; + :b :c4329; + :b :c4330; + :b :c4331; + :b :c4332; + :b :c4333; + :b :c4334; + :b :c4335; + :b :c4336; + :b :c4337; + :b :c4338; + :b :c4339; + :b :c4340; + :b :c4341; + :b :c4342; + :b :c4343; + :b :c4344; + :b :c4345; + :b :c4346; + :b :c4347; + :b :c4348; + :b :c4349; + :b :c4350; + :b :c4351; + :b :c4352; + :b :c4353; + :b :c4354; + :b :c4355; + :b :c4356; + :b :c4357; + :b :c4358; + :b :c4359; + :b :c4360; + :b :c4361; + :b :c4362; + :b :c4363; + :b :c4364; + :b :c4365; + :b :c4366; + :b :c4367; + :b :c4368; + :b :c4369; + :b :c4370; + :b :c4371; + :b :c4372; + :b :c4373; + :b :c4374; + :b :c4375; + :b :c4376; + :b :c4377; + :b :c4378; + :b :c4379; + :b :c4380; + :b :c4381; + :b :c4382; + :b :c4383; + :b :c4384; + :b :c4385; + :b :c4386; + :b :c4387; + :b :c4388; + :b :c4389; + :b :c4390; + :b :c4391; + :b :c4392; + :b :c4393; + :b :c4394; + :b :c4395; + :b :c4396; + :b :c4397; + :b :c4398; + :b :c4399; + :b :c4400; + :b :c4401; + :b :c4402; + :b :c4403; + :b :c4404; + :b :c4405; + :b :c4406; + :b :c4407; + :b :c4408; + :b :c4409; + :b :c4410; + :b :c4411; + :b :c4412; + :b :c4413; + :b :c4414; + :b :c4415; + :b :c4416; + :b :c4417; + :b :c4418; + :b :c4419; + :b :c4420; + :b :c4421; + :b :c4422; + :b :c4423; + :b :c4424; + :b :c4425; + :b :c4426; + :b :c4427; + :b :c4428; + :b :c4429; + :b :c4430; + :b :c4431; + :b :c4432; + :b :c4433; + :b :c4434; + :b :c4435; + :b :c4436; + :b :c4437; + :b :c4438; + :b :c4439; + :b :c4440; + :b :c4441; + :b :c4442; + :b :c4443; + :b :c4444; + :b :c4445; + :b :c4446; + :b :c4447; + :b :c4448; + :b :c4449; + :b :c4450; + :b :c4451; + :b :c4452; + :b :c4453; + :b :c4454; + :b :c4455; + :b :c4456; + :b :c4457; + :b :c4458; + :b :c4459; + :b :c4460; + :b :c4461; + :b :c4462; + :b :c4463; + :b :c4464; + :b :c4465; + :b :c4466; + :b :c4467; + :b :c4468; + :b :c4469; + :b :c4470; + :b :c4471; + :b :c4472; + :b :c4473; + :b :c4474; + :b :c4475; + :b :c4476; + :b :c4477; + :b :c4478; + :b :c4479; + :b :c4480; + :b :c4481; + :b :c4482; + :b :c4483; + :b :c4484; + :b :c4485; + :b :c4486; + :b :c4487; + :b :c4488; + :b :c4489; + :b :c4490; + :b :c4491; + :b :c4492; + :b :c4493; + :b :c4494; + :b :c4495; + :b :c4496; + :b :c4497; + :b :c4498; + :b :c4499; + :b :c4500; + :b :c4501; + :b :c4502; + :b :c4503; + :b :c4504; + :b :c4505; + :b :c4506; + :b :c4507; + :b :c4508; + :b :c4509; + :b :c4510; + :b :c4511; + :b :c4512; + :b :c4513; + :b :c4514; + :b :c4515; + :b :c4516; + :b :c4517; + :b :c4518; + :b :c4519; + :b :c4520; + :b :c4521; + :b :c4522; + :b :c4523; + :b :c4524; + :b :c4525; + :b :c4526; + :b :c4527; + :b :c4528; + :b :c4529; + :b :c4530; + :b :c4531; + :b :c4532; + :b :c4533; + :b :c4534; + :b :c4535; + :b :c4536; + :b :c4537; + :b :c4538; + :b :c4539; + :b :c4540; + :b :c4541; + :b :c4542; + :b :c4543; + :b :c4544; + :b :c4545; + :b :c4546; + :b :c4547; + :b :c4548; + :b :c4549; + :b :c4550; + :b :c4551; + :b :c4552; + :b :c4553; + :b :c4554; + :b :c4555; + :b :c4556; + :b :c4557; + :b :c4558; + :b :c4559; + :b :c4560; + :b :c4561; + :b :c4562; + :b :c4563; + :b :c4564; + :b :c4565; + :b :c4566; + :b :c4567; + :b :c4568; + :b :c4569; + :b :c4570; + :b :c4571; + :b :c4572; + :b :c4573; + :b :c4574; + :b :c4575; + :b :c4576; + :b :c4577; + :b :c4578; + :b :c4579; + :b :c4580; + :b :c4581; + :b :c4582; + :b :c4583; + :b :c4584; + :b :c4585; + :b :c4586; + :b :c4587; + :b :c4588; + :b :c4589; + :b :c4590; + :b :c4591; + :b :c4592; + :b :c4593; + :b :c4594; + :b :c4595; + :b :c4596; + :b :c4597; + :b :c4598; + :b :c4599; + :b :c4600; + :b :c4601; + :b :c4602; + :b :c4603; + :b :c4604; + :b :c4605; + :b :c4606; + :b :c4607; + :b :c4608; + :b :c4609; + :b :c4610; + :b :c4611; + :b :c4612; + :b :c4613; + :b :c4614; + :b :c4615; + :b :c4616; + :b :c4617; + :b :c4618; + :b :c4619; + :b :c4620; + :b :c4621; + :b :c4622; + :b :c4623; + :b :c4624; + :b :c4625; + :b :c4626; + :b :c4627; + :b :c4628; + :b :c4629; + :b :c4630; + :b :c4631; + :b :c4632; + :b :c4633; + :b :c4634; + :b :c4635; + :b :c4636; + :b :c4637; + :b :c4638; + :b :c4639; + :b :c4640; + :b :c4641; + :b :c4642; + :b :c4643; + :b :c4644; + :b :c4645; + :b :c4646; + :b :c4647; + :b :c4648; + :b :c4649; + :b :c4650; + :b :c4651; + :b :c4652; + :b :c4653; + :b :c4654; + :b :c4655; + :b :c4656; + :b :c4657; + :b :c4658; + :b :c4659; + :b :c4660; + :b :c4661; + :b :c4662; + :b :c4663; + :b :c4664; + :b :c4665; + :b :c4666; + :b :c4667; + :b :c4668; + :b :c4669; + :b :c4670; + :b :c4671; + :b :c4672; + :b :c4673; + :b :c4674; + :b :c4675; + :b :c4676; + :b :c4677; + :b :c4678; + :b :c4679; + :b :c4680; + :b :c4681; + :b :c4682; + :b :c4683; + :b :c4684; + :b :c4685; + :b :c4686; + :b :c4687; + :b :c4688; + :b :c4689; + :b :c4690; + :b :c4691; + :b :c4692; + :b :c4693; + :b :c4694; + :b :c4695; + :b :c4696; + :b :c4697; + :b :c4698; + :b :c4699; + :b :c4700; + :b :c4701; + :b :c4702; + :b :c4703; + :b :c4704; + :b :c4705; + :b :c4706; + :b :c4707; + :b :c4708; + :b :c4709; + :b :c4710; + :b :c4711; + :b :c4712; + :b :c4713; + :b :c4714; + :b :c4715; + :b :c4716; + :b :c4717; + :b :c4718; + :b :c4719; + :b :c4720; + :b :c4721; + :b :c4722; + :b :c4723; + :b :c4724; + :b :c4725; + :b :c4726; + :b :c4727; + :b :c4728; + :b :c4729; + :b :c4730; + :b :c4731; + :b :c4732; + :b :c4733; + :b :c4734; + :b :c4735; + :b :c4736; + :b :c4737; + :b :c4738; + :b :c4739; + :b :c4740; + :b :c4741; + :b :c4742; + :b :c4743; + :b :c4744; + :b :c4745; + :b :c4746; + :b :c4747; + :b :c4748; + :b :c4749; + :b :c4750; + :b :c4751; + :b :c4752; + :b :c4753; + :b :c4754; + :b :c4755; + :b :c4756; + :b :c4757; + :b :c4758; + :b :c4759; + :b :c4760; + :b :c4761; + :b :c4762; + :b :c4763; + :b :c4764; + :b :c4765; + :b :c4766; + :b :c4767; + :b :c4768; + :b :c4769; + :b :c4770; + :b :c4771; + :b :c4772; + :b :c4773; + :b :c4774; + :b :c4775; + :b :c4776; + :b :c4777; + :b :c4778; + :b :c4779; + :b :c4780; + :b :c4781; + :b :c4782; + :b :c4783; + :b :c4784; + :b :c4785; + :b :c4786; + :b :c4787; + :b :c4788; + :b :c4789; + :b :c4790; + :b :c4791; + :b :c4792; + :b :c4793; + :b :c4794; + :b :c4795; + :b :c4796; + :b :c4797; + :b :c4798; + :b :c4799; + :b :c4800; + :b :c4801; + :b :c4802; + :b :c4803; + :b :c4804; + :b :c4805; + :b :c4806; + :b :c4807; + :b :c4808; + :b :c4809; + :b :c4810; + :b :c4811; + :b :c4812; + :b :c4813; + :b :c4814; + :b :c4815; + :b :c4816; + :b :c4817; + :b :c4818; + :b :c4819; + :b :c4820; + :b :c4821; + :b :c4822; + :b :c4823; + :b :c4824; + :b :c4825; + :b :c4826; + :b :c4827; + :b :c4828; + :b :c4829; + :b :c4830; + :b :c4831; + :b :c4832; + :b :c4833; + :b :c4834; + :b :c4835; + :b :c4836; + :b :c4837; + :b :c4838; + :b :c4839; + :b :c4840; + :b :c4841; + :b :c4842; + :b :c4843; + :b :c4844; + :b :c4845; + :b :c4846; + :b :c4847; + :b :c4848; + :b :c4849; + :b :c4850; + :b :c4851; + :b :c4852; + :b :c4853; + :b :c4854; + :b :c4855; + :b :c4856; + :b :c4857; + :b :c4858; + :b :c4859; + :b :c4860; + :b :c4861; + :b :c4862; + :b :c4863; + :b :c4864; + :b :c4865; + :b :c4866; + :b :c4867; + :b :c4868; + :b :c4869; + :b :c4870; + :b :c4871; + :b :c4872; + :b :c4873; + :b :c4874; + :b :c4875; + :b :c4876; + :b :c4877; + :b :c4878; + :b :c4879; + :b :c4880; + :b :c4881; + :b :c4882; + :b :c4883; + :b :c4884; + :b :c4885; + :b :c4886; + :b :c4887; + :b :c4888; + :b :c4889; + :b :c4890; + :b :c4891; + :b :c4892; + :b :c4893; + :b :c4894; + :b :c4895; + :b :c4896; + :b :c4897; + :b :c4898; + :b :c4899; + :b :c4900; + :b :c4901; + :b :c4902; + :b :c4903; + :b :c4904; + :b :c4905; + :b :c4906; + :b :c4907; + :b :c4908; + :b :c4909; + :b :c4910; + :b :c4911; + :b :c4912; + :b :c4913; + :b :c4914; + :b :c4915; + :b :c4916; + :b :c4917; + :b :c4918; + :b :c4919; + :b :c4920; + :b :c4921; + :b :c4922; + :b :c4923; + :b :c4924; + :b :c4925; + :b :c4926; + :b :c4927; + :b :c4928; + :b :c4929; + :b :c4930; + :b :c4931; + :b :c4932; + :b :c4933; + :b :c4934; + :b :c4935; + :b :c4936; + :b :c4937; + :b :c4938; + :b :c4939; + :b :c4940; + :b :c4941; + :b :c4942; + :b :c4943; + :b :c4944; + :b :c4945; + :b :c4946; + :b :c4947; + :b :c4948; + :b :c4949; + :b :c4950; + :b :c4951; + :b :c4952; + :b :c4953; + :b :c4954; + :b :c4955; + :b :c4956; + :b :c4957; + :b :c4958; + :b :c4959; + :b :c4960; + :b :c4961; + :b :c4962; + :b :c4963; + :b :c4964; + :b :c4965; + :b :c4966; + :b :c4967; + :b :c4968; + :b :c4969; + :b :c4970; + :b :c4971; + :b :c4972; + :b :c4973; + :b :c4974; + :b :c4975; + :b :c4976; + :b :c4977; + :b :c4978; + :b :c4979; + :b :c4980; + :b :c4981; + :b :c4982; + :b :c4983; + :b :c4984; + :b :c4985; + :b :c4986; + :b :c4987; + :b :c4988; + :b :c4989; + :b :c4990; + :b :c4991; + :b :c4992; + :b :c4993; + :b :c4994; + :b :c4995; + :b :c4996; + :b :c4997; + :b :c4998; + :b :c4999; + :b :c5000; + :b :c5001; + :b :c5002; + :b :c5003; + :b :c5004; + :b :c5005; + :b :c5006; + :b :c5007; + :b :c5008; + :b :c5009; + :b :c5010; + :b :c5011; + :b :c5012; + :b :c5013; + :b :c5014; + :b :c5015; + :b :c5016; + :b :c5017; + :b :c5018; + :b :c5019; + :b :c5020; + :b :c5021; + :b :c5022; + :b :c5023; + :b :c5024; + :b :c5025; + :b :c5026; + :b :c5027; + :b :c5028; + :b :c5029; + :b :c5030; + :b :c5031; + :b :c5032; + :b :c5033; + :b :c5034; + :b :c5035; + :b :c5036; + :b :c5037; + :b :c5038; + :b :c5039; + :b :c5040; + :b :c5041; + :b :c5042; + :b :c5043; + :b :c5044; + :b :c5045; + :b :c5046; + :b :c5047; + :b :c5048; + :b :c5049; + :b :c5050; + :b :c5051; + :b :c5052; + :b :c5053; + :b :c5054; + :b :c5055; + :b :c5056; + :b :c5057; + :b :c5058; + :b :c5059; + :b :c5060; + :b :c5061; + :b :c5062; + :b :c5063; + :b :c5064; + :b :c5065; + :b :c5066; + :b :c5067; + :b :c5068; + :b :c5069; + :b :c5070; + :b :c5071; + :b :c5072; + :b :c5073; + :b :c5074; + :b :c5075; + :b :c5076; + :b :c5077; + :b :c5078; + :b :c5079; + :b :c5080; + :b :c5081; + :b :c5082; + :b :c5083; + :b :c5084; + :b :c5085; + :b :c5086; + :b :c5087; + :b :c5088; + :b :c5089; + :b :c5090; + :b :c5091; + :b :c5092; + :b :c5093; + :b :c5094; + :b :c5095; + :b :c5096; + :b :c5097; + :b :c5098; + :b :c5099; + :b :c5100; + :b :c5101; + :b :c5102; + :b :c5103; + :b :c5104; + :b :c5105; + :b :c5106; + :b :c5107; + :b :c5108; + :b :c5109; + :b :c5110; + :b :c5111; + :b :c5112; + :b :c5113; + :b :c5114; + :b :c5115; + :b :c5116; + :b :c5117; + :b :c5118; + :b :c5119; + :b :c5120; + :b :c5121; + :b :c5122; + :b :c5123; + :b :c5124; + :b :c5125; + :b :c5126; + :b :c5127; + :b :c5128; + :b :c5129; + :b :c5130; + :b :c5131; + :b :c5132; + :b :c5133; + :b :c5134; + :b :c5135; + :b :c5136; + :b :c5137; + :b :c5138; + :b :c5139; + :b :c5140; + :b :c5141; + :b :c5142; + :b :c5143; + :b :c5144; + :b :c5145; + :b :c5146; + :b :c5147; + :b :c5148; + :b :c5149; + :b :c5150; + :b :c5151; + :b :c5152; + :b :c5153; + :b :c5154; + :b :c5155; + :b :c5156; + :b :c5157; + :b :c5158; + :b :c5159; + :b :c5160; + :b :c5161; + :b :c5162; + :b :c5163; + :b :c5164; + :b :c5165; + :b :c5166; + :b :c5167; + :b :c5168; + :b :c5169; + :b :c5170; + :b :c5171; + :b :c5172; + :b :c5173; + :b :c5174; + :b :c5175; + :b :c5176; + :b :c5177; + :b :c5178; + :b :c5179; + :b :c5180; + :b :c5181; + :b :c5182; + :b :c5183; + :b :c5184; + :b :c5185; + :b :c5186; + :b :c5187; + :b :c5188; + :b :c5189; + :b :c5190; + :b :c5191; + :b :c5192; + :b :c5193; + :b :c5194; + :b :c5195; + :b :c5196; + :b :c5197; + :b :c5198; + :b :c5199; + :b :c5200; + :b :c5201; + :b :c5202; + :b :c5203; + :b :c5204; + :b :c5205; + :b :c5206; + :b :c5207; + :b :c5208; + :b :c5209; + :b :c5210; + :b :c5211; + :b :c5212; + :b :c5213; + :b :c5214; + :b :c5215; + :b :c5216; + :b :c5217; + :b :c5218; + :b :c5219; + :b :c5220; + :b :c5221; + :b :c5222; + :b :c5223; + :b :c5224; + :b :c5225; + :b :c5226; + :b :c5227; + :b :c5228; + :b :c5229; + :b :c5230; + :b :c5231; + :b :c5232; + :b :c5233; + :b :c5234; + :b :c5235; + :b :c5236; + :b :c5237; + :b :c5238; + :b :c5239; + :b :c5240; + :b :c5241; + :b :c5242; + :b :c5243; + :b :c5244; + :b :c5245; + :b :c5246; + :b :c5247; + :b :c5248; + :b :c5249; + :b :c5250; + :b :c5251; + :b :c5252; + :b :c5253; + :b :c5254; + :b :c5255; + :b :c5256; + :b :c5257; + :b :c5258; + :b :c5259; + :b :c5260; + :b :c5261; + :b :c5262; + :b :c5263; + :b :c5264; + :b :c5265; + :b :c5266; + :b :c5267; + :b :c5268; + :b :c5269; + :b :c5270; + :b :c5271; + :b :c5272; + :b :c5273; + :b :c5274; + :b :c5275; + :b :c5276; + :b :c5277; + :b :c5278; + :b :c5279; + :b :c5280; + :b :c5281; + :b :c5282; + :b :c5283; + :b :c5284; + :b :c5285; + :b :c5286; + :b :c5287; + :b :c5288; + :b :c5289; + :b :c5290; + :b :c5291; + :b :c5292; + :b :c5293; + :b :c5294; + :b :c5295; + :b :c5296; + :b :c5297; + :b :c5298; + :b :c5299; + :b :c5300; + :b :c5301; + :b :c5302; + :b :c5303; + :b :c5304; + :b :c5305; + :b :c5306; + :b :c5307; + :b :c5308; + :b :c5309; + :b :c5310; + :b :c5311; + :b :c5312; + :b :c5313; + :b :c5314; + :b :c5315; + :b :c5316; + :b :c5317; + :b :c5318; + :b :c5319; + :b :c5320; + :b :c5321; + :b :c5322; + :b :c5323; + :b :c5324; + :b :c5325; + :b :c5326; + :b :c5327; + :b :c5328; + :b :c5329; + :b :c5330; + :b :c5331; + :b :c5332; + :b :c5333; + :b :c5334; + :b :c5335; + :b :c5336; + :b :c5337; + :b :c5338; + :b :c5339; + :b :c5340; + :b :c5341; + :b :c5342; + :b :c5343; + :b :c5344; + :b :c5345; + :b :c5346; + :b :c5347; + :b :c5348; + :b :c5349; + :b :c5350; + :b :c5351; + :b :c5352; + :b :c5353; + :b :c5354; + :b :c5355; + :b :c5356; + :b :c5357; + :b :c5358; + :b :c5359; + :b :c5360; + :b :c5361; + :b :c5362; + :b :c5363; + :b :c5364; + :b :c5365; + :b :c5366; + :b :c5367; + :b :c5368; + :b :c5369; + :b :c5370; + :b :c5371; + :b :c5372; + :b :c5373; + :b :c5374; + :b :c5375; + :b :c5376; + :b :c5377; + :b :c5378; + :b :c5379; + :b :c5380; + :b :c5381; + :b :c5382; + :b :c5383; + :b :c5384; + :b :c5385; + :b :c5386; + :b :c5387; + :b :c5388; + :b :c5389; + :b :c5390; + :b :c5391; + :b :c5392; + :b :c5393; + :b :c5394; + :b :c5395; + :b :c5396; + :b :c5397; + :b :c5398; + :b :c5399; + :b :c5400; + :b :c5401; + :b :c5402; + :b :c5403; + :b :c5404; + :b :c5405; + :b :c5406; + :b :c5407; + :b :c5408; + :b :c5409; + :b :c5410; + :b :c5411; + :b :c5412; + :b :c5413; + :b :c5414; + :b :c5415; + :b :c5416; + :b :c5417; + :b :c5418; + :b :c5419; + :b :c5420; + :b :c5421; + :b :c5422; + :b :c5423; + :b :c5424; + :b :c5425; + :b :c5426; + :b :c5427; + :b :c5428; + :b :c5429; + :b :c5430; + :b :c5431; + :b :c5432; + :b :c5433; + :b :c5434; + :b :c5435; + :b :c5436; + :b :c5437; + :b :c5438; + :b :c5439; + :b :c5440; + :b :c5441; + :b :c5442; + :b :c5443; + :b :c5444; + :b :c5445; + :b :c5446; + :b :c5447; + :b :c5448; + :b :c5449; + :b :c5450; + :b :c5451; + :b :c5452; + :b :c5453; + :b :c5454; + :b :c5455; + :b :c5456; + :b :c5457; + :b :c5458; + :b :c5459; + :b :c5460; + :b :c5461; + :b :c5462; + :b :c5463; + :b :c5464; + :b :c5465; + :b :c5466; + :b :c5467; + :b :c5468; + :b :c5469; + :b :c5470; + :b :c5471; + :b :c5472; + :b :c5473; + :b :c5474; + :b :c5475; + :b :c5476; + :b :c5477; + :b :c5478; + :b :c5479; + :b :c5480; + :b :c5481; + :b :c5482; + :b :c5483; + :b :c5484; + :b :c5485; + :b :c5486; + :b :c5487; + :b :c5488; + :b :c5489; + :b :c5490; + :b :c5491; + :b :c5492; + :b :c5493; + :b :c5494; + :b :c5495; + :b :c5496; + :b :c5497; + :b :c5498; + :b :c5499; + :b :c5500; + :b :c5501; + :b :c5502; + :b :c5503; + :b :c5504; + :b :c5505; + :b :c5506; + :b :c5507; + :b :c5508; + :b :c5509; + :b :c5510; + :b :c5511; + :b :c5512; + :b :c5513; + :b :c5514; + :b :c5515; + :b :c5516; + :b :c5517; + :b :c5518; + :b :c5519; + :b :c5520; + :b :c5521; + :b :c5522; + :b :c5523; + :b :c5524; + :b :c5525; + :b :c5526; + :b :c5527; + :b :c5528; + :b :c5529; + :b :c5530; + :b :c5531; + :b :c5532; + :b :c5533; + :b :c5534; + :b :c5535; + :b :c5536; + :b :c5537; + :b :c5538; + :b :c5539; + :b :c5540; + :b :c5541; + :b :c5542; + :b :c5543; + :b :c5544; + :b :c5545; + :b :c5546; + :b :c5547; + :b :c5548; + :b :c5549; + :b :c5550; + :b :c5551; + :b :c5552; + :b :c5553; + :b :c5554; + :b :c5555; + :b :c5556; + :b :c5557; + :b :c5558; + :b :c5559; + :b :c5560; + :b :c5561; + :b :c5562; + :b :c5563; + :b :c5564; + :b :c5565; + :b :c5566; + :b :c5567; + :b :c5568; + :b :c5569; + :b :c5570; + :b :c5571; + :b :c5572; + :b :c5573; + :b :c5574; + :b :c5575; + :b :c5576; + :b :c5577; + :b :c5578; + :b :c5579; + :b :c5580; + :b :c5581; + :b :c5582; + :b :c5583; + :b :c5584; + :b :c5585; + :b :c5586; + :b :c5587; + :b :c5588; + :b :c5589; + :b :c5590; + :b :c5591; + :b :c5592; + :b :c5593; + :b :c5594; + :b :c5595; + :b :c5596; + :b :c5597; + :b :c5598; + :b :c5599; + :b :c5600; + :b :c5601; + :b :c5602; + :b :c5603; + :b :c5604; + :b :c5605; + :b :c5606; + :b :c5607; + :b :c5608; + :b :c5609; + :b :c5610; + :b :c5611; + :b :c5612; + :b :c5613; + :b :c5614; + :b :c5615; + :b :c5616; + :b :c5617; + :b :c5618; + :b :c5619; + :b :c5620; + :b :c5621; + :b :c5622; + :b :c5623; + :b :c5624; + :b :c5625; + :b :c5626; + :b :c5627; + :b :c5628; + :b :c5629; + :b :c5630; + :b :c5631; + :b :c5632; + :b :c5633; + :b :c5634; + :b :c5635; + :b :c5636; + :b :c5637; + :b :c5638; + :b :c5639; + :b :c5640; + :b :c5641; + :b :c5642; + :b :c5643; + :b :c5644; + :b :c5645; + :b :c5646; + :b :c5647; + :b :c5648; + :b :c5649; + :b :c5650; + :b :c5651; + :b :c5652; + :b :c5653; + :b :c5654; + :b :c5655; + :b :c5656; + :b :c5657; + :b :c5658; + :b :c5659; + :b :c5660; + :b :c5661; + :b :c5662; + :b :c5663; + :b :c5664; + :b :c5665; + :b :c5666; + :b :c5667; + :b :c5668; + :b :c5669; + :b :c5670; + :b :c5671; + :b :c5672; + :b :c5673; + :b :c5674; + :b :c5675; + :b :c5676; + :b :c5677; + :b :c5678; + :b :c5679; + :b :c5680; + :b :c5681; + :b :c5682; + :b :c5683; + :b :c5684; + :b :c5685; + :b :c5686; + :b :c5687; + :b :c5688; + :b :c5689; + :b :c5690; + :b :c5691; + :b :c5692; + :b :c5693; + :b :c5694; + :b :c5695; + :b :c5696; + :b :c5697; + :b :c5698; + :b :c5699; + :b :c5700; + :b :c5701; + :b :c5702; + :b :c5703; + :b :c5704; + :b :c5705; + :b :c5706; + :b :c5707; + :b :c5708; + :b :c5709; + :b :c5710; + :b :c5711; + :b :c5712; + :b :c5713; + :b :c5714; + :b :c5715; + :b :c5716; + :b :c5717; + :b :c5718; + :b :c5719; + :b :c5720; + :b :c5721; + :b :c5722; + :b :c5723; + :b :c5724; + :b :c5725; + :b :c5726; + :b :c5727; + :b :c5728; + :b :c5729; + :b :c5730; + :b :c5731; + :b :c5732; + :b :c5733; + :b :c5734; + :b :c5735; + :b :c5736; + :b :c5737; + :b :c5738; + :b :c5739; + :b :c5740; + :b :c5741; + :b :c5742; + :b :c5743; + :b :c5744; + :b :c5745; + :b :c5746; + :b :c5747; + :b :c5748; + :b :c5749; + :b :c5750; + :b :c5751; + :b :c5752; + :b :c5753; + :b :c5754; + :b :c5755; + :b :c5756; + :b :c5757; + :b :c5758; + :b :c5759; + :b :c5760; + :b :c5761; + :b :c5762; + :b :c5763; + :b :c5764; + :b :c5765; + :b :c5766; + :b :c5767; + :b :c5768; + :b :c5769; + :b :c5770; + :b :c5771; + :b :c5772; + :b :c5773; + :b :c5774; + :b :c5775; + :b :c5776; + :b :c5777; + :b :c5778; + :b :c5779; + :b :c5780; + :b :c5781; + :b :c5782; + :b :c5783; + :b :c5784; + :b :c5785; + :b :c5786; + :b :c5787; + :b :c5788; + :b :c5789; + :b :c5790; + :b :c5791; + :b :c5792; + :b :c5793; + :b :c5794; + :b :c5795; + :b :c5796; + :b :c5797; + :b :c5798; + :b :c5799; + :b :c5800; + :b :c5801; + :b :c5802; + :b :c5803; + :b :c5804; + :b :c5805; + :b :c5806; + :b :c5807; + :b :c5808; + :b :c5809; + :b :c5810; + :b :c5811; + :b :c5812; + :b :c5813; + :b :c5814; + :b :c5815; + :b :c5816; + :b :c5817; + :b :c5818; + :b :c5819; + :b :c5820; + :b :c5821; + :b :c5822; + :b :c5823; + :b :c5824; + :b :c5825; + :b :c5826; + :b :c5827; + :b :c5828; + :b :c5829; + :b :c5830; + :b :c5831; + :b :c5832; + :b :c5833; + :b :c5834; + :b :c5835; + :b :c5836; + :b :c5837; + :b :c5838; + :b :c5839; + :b :c5840; + :b :c5841; + :b :c5842; + :b :c5843; + :b :c5844; + :b :c5845; + :b :c5846; + :b :c5847; + :b :c5848; + :b :c5849; + :b :c5850; + :b :c5851; + :b :c5852; + :b :c5853; + :b :c5854; + :b :c5855; + :b :c5856; + :b :c5857; + :b :c5858; + :b :c5859; + :b :c5860; + :b :c5861; + :b :c5862; + :b :c5863; + :b :c5864; + :b :c5865; + :b :c5866; + :b :c5867; + :b :c5868; + :b :c5869; + :b :c5870; + :b :c5871; + :b :c5872; + :b :c5873; + :b :c5874; + :b :c5875; + :b :c5876; + :b :c5877; + :b :c5878; + :b :c5879; + :b :c5880; + :b :c5881; + :b :c5882; + :b :c5883; + :b :c5884; + :b :c5885; + :b :c5886; + :b :c5887; + :b :c5888; + :b :c5889; + :b :c5890; + :b :c5891; + :b :c5892; + :b :c5893; + :b :c5894; + :b :c5895; + :b :c5896; + :b :c5897; + :b :c5898; + :b :c5899; + :b :c5900; + :b :c5901; + :b :c5902; + :b :c5903; + :b :c5904; + :b :c5905; + :b :c5906; + :b :c5907; + :b :c5908; + :b :c5909; + :b :c5910; + :b :c5911; + :b :c5912; + :b :c5913; + :b :c5914; + :b :c5915; + :b :c5916; + :b :c5917; + :b :c5918; + :b :c5919; + :b :c5920; + :b :c5921; + :b :c5922; + :b :c5923; + :b :c5924; + :b :c5925; + :b :c5926; + :b :c5927; + :b :c5928; + :b :c5929; + :b :c5930; + :b :c5931; + :b :c5932; + :b :c5933; + :b :c5934; + :b :c5935; + :b :c5936; + :b :c5937; + :b :c5938; + :b :c5939; + :b :c5940; + :b :c5941; + :b :c5942; + :b :c5943; + :b :c5944; + :b :c5945; + :b :c5946; + :b :c5947; + :b :c5948; + :b :c5949; + :b :c5950; + :b :c5951; + :b :c5952; + :b :c5953; + :b :c5954; + :b :c5955; + :b :c5956; + :b :c5957; + :b :c5958; + :b :c5959; + :b :c5960; + :b :c5961; + :b :c5962; + :b :c5963; + :b :c5964; + :b :c5965; + :b :c5966; + :b :c5967; + :b :c5968; + :b :c5969; + :b :c5970; + :b :c5971; + :b :c5972; + :b :c5973; + :b :c5974; + :b :c5975; + :b :c5976; + :b :c5977; + :b :c5978; + :b :c5979; + :b :c5980; + :b :c5981; + :b :c5982; + :b :c5983; + :b :c5984; + :b :c5985; + :b :c5986; + :b :c5987; + :b :c5988; + :b :c5989; + :b :c5990; + :b :c5991; + :b :c5992; + :b :c5993; + :b :c5994; + :b :c5995; + :b :c5996; + :b :c5997; + :b :c5998; + :b :c5999; + :b :c6000; + :b :c6001; + :b :c6002; + :b :c6003; + :b :c6004; + :b :c6005; + :b :c6006; + :b :c6007; + :b :c6008; + :b :c6009; + :b :c6010; + :b :c6011; + :b :c6012; + :b :c6013; + :b :c6014; + :b :c6015; + :b :c6016; + :b :c6017; + :b :c6018; + :b :c6019; + :b :c6020; + :b :c6021; + :b :c6022; + :b :c6023; + :b :c6024; + :b :c6025; + :b :c6026; + :b :c6027; + :b :c6028; + :b :c6029; + :b :c6030; + :b :c6031; + :b :c6032; + :b :c6033; + :b :c6034; + :b :c6035; + :b :c6036; + :b :c6037; + :b :c6038; + :b :c6039; + :b :c6040; + :b :c6041; + :b :c6042; + :b :c6043; + :b :c6044; + :b :c6045; + :b :c6046; + :b :c6047; + :b :c6048; + :b :c6049; + :b :c6050; + :b :c6051; + :b :c6052; + :b :c6053; + :b :c6054; + :b :c6055; + :b :c6056; + :b :c6057; + :b :c6058; + :b :c6059; + :b :c6060; + :b :c6061; + :b :c6062; + :b :c6063; + :b :c6064; + :b :c6065; + :b :c6066; + :b :c6067; + :b :c6068; + :b :c6069; + :b :c6070; + :b :c6071; + :b :c6072; + :b :c6073; + :b :c6074; + :b :c6075; + :b :c6076; + :b :c6077; + :b :c6078; + :b :c6079; + :b :c6080; + :b :c6081; + :b :c6082; + :b :c6083; + :b :c6084; + :b :c6085; + :b :c6086; + :b :c6087; + :b :c6088; + :b :c6089; + :b :c6090; + :b :c6091; + :b :c6092; + :b :c6093; + :b :c6094; + :b :c6095; + :b :c6096; + :b :c6097; + :b :c6098; + :b :c6099; + :b :c6100; + :b :c6101; + :b :c6102; + :b :c6103; + :b :c6104; + :b :c6105; + :b :c6106; + :b :c6107; + :b :c6108; + :b :c6109; + :b :c6110; + :b :c6111; + :b :c6112; + :b :c6113; + :b :c6114; + :b :c6115; + :b :c6116; + :b :c6117; + :b :c6118; + :b :c6119; + :b :c6120; + :b :c6121; + :b :c6122; + :b :c6123; + :b :c6124; + :b :c6125; + :b :c6126; + :b :c6127; + :b :c6128; + :b :c6129; + :b :c6130; + :b :c6131; + :b :c6132; + :b :c6133; + :b :c6134; + :b :c6135; + :b :c6136; + :b :c6137; + :b :c6138; + :b :c6139; + :b :c6140; + :b :c6141; + :b :c6142; + :b :c6143; + :b :c6144; + :b :c6145; + :b :c6146; + :b :c6147; + :b :c6148; + :b :c6149; + :b :c6150; + :b :c6151; + :b :c6152; + :b :c6153; + :b :c6154; + :b :c6155; + :b :c6156; + :b :c6157; + :b :c6158; + :b :c6159; + :b :c6160; + :b :c6161; + :b :c6162; + :b :c6163; + :b :c6164; + :b :c6165; + :b :c6166; + :b :c6167; + :b :c6168; + :b :c6169; + :b :c6170; + :b :c6171; + :b :c6172; + :b :c6173; + :b :c6174; + :b :c6175; + :b :c6176; + :b :c6177; + :b :c6178; + :b :c6179; + :b :c6180; + :b :c6181; + :b :c6182; + :b :c6183; + :b :c6184; + :b :c6185; + :b :c6186; + :b :c6187; + :b :c6188; + :b :c6189; + :b :c6190; + :b :c6191; + :b :c6192; + :b :c6193; + :b :c6194; + :b :c6195; + :b :c6196; + :b :c6197; + :b :c6198; + :b :c6199; + :b :c6200; + :b :c6201; + :b :c6202; + :b :c6203; + :b :c6204; + :b :c6205; + :b :c6206; + :b :c6207; + :b :c6208; + :b :c6209; + :b :c6210; + :b :c6211; + :b :c6212; + :b :c6213; + :b :c6214; + :b :c6215; + :b :c6216; + :b :c6217; + :b :c6218; + :b :c6219; + :b :c6220; + :b :c6221; + :b :c6222; + :b :c6223; + :b :c6224; + :b :c6225; + :b :c6226; + :b :c6227; + :b :c6228; + :b :c6229; + :b :c6230; + :b :c6231; + :b :c6232; + :b :c6233; + :b :c6234; + :b :c6235; + :b :c6236; + :b :c6237; + :b :c6238; + :b :c6239; + :b :c6240; + :b :c6241; + :b :c6242; + :b :c6243; + :b :c6244; + :b :c6245; + :b :c6246; + :b :c6247; + :b :c6248; + :b :c6249; + :b :c6250; + :b :c6251; + :b :c6252; + :b :c6253; + :b :c6254; + :b :c6255; + :b :c6256; + :b :c6257; + :b :c6258; + :b :c6259; + :b :c6260; + :b :c6261; + :b :c6262; + :b :c6263; + :b :c6264; + :b :c6265; + :b :c6266; + :b :c6267; + :b :c6268; + :b :c6269; + :b :c6270; + :b :c6271; + :b :c6272; + :b :c6273; + :b :c6274; + :b :c6275; + :b :c6276; + :b :c6277; + :b :c6278; + :b :c6279; + :b :c6280; + :b :c6281; + :b :c6282; + :b :c6283; + :b :c6284; + :b :c6285; + :b :c6286; + :b :c6287; + :b :c6288; + :b :c6289; + :b :c6290; + :b :c6291; + :b :c6292; + :b :c6293; + :b :c6294; + :b :c6295; + :b :c6296; + :b :c6297; + :b :c6298; + :b :c6299; + :b :c6300; + :b :c6301; + :b :c6302; + :b :c6303; + :b :c6304; + :b :c6305; + :b :c6306; + :b :c6307; + :b :c6308; + :b :c6309; + :b :c6310; + :b :c6311; + :b :c6312; + :b :c6313; + :b :c6314; + :b :c6315; + :b :c6316; + :b :c6317; + :b :c6318; + :b :c6319; + :b :c6320; + :b :c6321; + :b :c6322; + :b :c6323; + :b :c6324; + :b :c6325; + :b :c6326; + :b :c6327; + :b :c6328; + :b :c6329; + :b :c6330; + :b :c6331; + :b :c6332; + :b :c6333; + :b :c6334; + :b :c6335; + :b :c6336; + :b :c6337; + :b :c6338; + :b :c6339; + :b :c6340; + :b :c6341; + :b :c6342; + :b :c6343; + :b :c6344; + :b :c6345; + :b :c6346; + :b :c6347; + :b :c6348; + :b :c6349; + :b :c6350; + :b :c6351; + :b :c6352; + :b :c6353; + :b :c6354; + :b :c6355; + :b :c6356; + :b :c6357; + :b :c6358; + :b :c6359; + :b :c6360; + :b :c6361; + :b :c6362; + :b :c6363; + :b :c6364; + :b :c6365; + :b :c6366; + :b :c6367; + :b :c6368; + :b :c6369; + :b :c6370; + :b :c6371; + :b :c6372; + :b :c6373; + :b :c6374; + :b :c6375; + :b :c6376; + :b :c6377; + :b :c6378; + :b :c6379; + :b :c6380; + :b :c6381; + :b :c6382; + :b :c6383; + :b :c6384; + :b :c6385; + :b :c6386; + :b :c6387; + :b :c6388; + :b :c6389; + :b :c6390; + :b :c6391; + :b :c6392; + :b :c6393; + :b :c6394; + :b :c6395; + :b :c6396; + :b :c6397; + :b :c6398; + :b :c6399; + :b :c6400; + :b :c6401; + :b :c6402; + :b :c6403; + :b :c6404; + :b :c6405; + :b :c6406; + :b :c6407; + :b :c6408; + :b :c6409; + :b :c6410; + :b :c6411; + :b :c6412; + :b :c6413; + :b :c6414; + :b :c6415; + :b :c6416; + :b :c6417; + :b :c6418; + :b :c6419; + :b :c6420; + :b :c6421; + :b :c6422; + :b :c6423; + :b :c6424; + :b :c6425; + :b :c6426; + :b :c6427; + :b :c6428; + :b :c6429; + :b :c6430; + :b :c6431; + :b :c6432; + :b :c6433; + :b :c6434; + :b :c6435; + :b :c6436; + :b :c6437; + :b :c6438; + :b :c6439; + :b :c6440; + :b :c6441; + :b :c6442; + :b :c6443; + :b :c6444; + :b :c6445; + :b :c6446; + :b :c6447; + :b :c6448; + :b :c6449; + :b :c6450; + :b :c6451; + :b :c6452; + :b :c6453; + :b :c6454; + :b :c6455; + :b :c6456; + :b :c6457; + :b :c6458; + :b :c6459; + :b :c6460; + :b :c6461; + :b :c6462; + :b :c6463; + :b :c6464; + :b :c6465; + :b :c6466; + :b :c6467; + :b :c6468; + :b :c6469; + :b :c6470; + :b :c6471; + :b :c6472; + :b :c6473; + :b :c6474; + :b :c6475; + :b :c6476; + :b :c6477; + :b :c6478; + :b :c6479; + :b :c6480; + :b :c6481; + :b :c6482; + :b :c6483; + :b :c6484; + :b :c6485; + :b :c6486; + :b :c6487; + :b :c6488; + :b :c6489; + :b :c6490; + :b :c6491; + :b :c6492; + :b :c6493; + :b :c6494; + :b :c6495; + :b :c6496; + :b :c6497; + :b :c6498; + :b :c6499; + :b :c6500; + :b :c6501; + :b :c6502; + :b :c6503; + :b :c6504; + :b :c6505; + :b :c6506; + :b :c6507; + :b :c6508; + :b :c6509; + :b :c6510; + :b :c6511; + :b :c6512; + :b :c6513; + :b :c6514; + :b :c6515; + :b :c6516; + :b :c6517; + :b :c6518; + :b :c6519; + :b :c6520; + :b :c6521; + :b :c6522; + :b :c6523; + :b :c6524; + :b :c6525; + :b :c6526; + :b :c6527; + :b :c6528; + :b :c6529; + :b :c6530; + :b :c6531; + :b :c6532; + :b :c6533; + :b :c6534; + :b :c6535; + :b :c6536; + :b :c6537; + :b :c6538; + :b :c6539; + :b :c6540; + :b :c6541; + :b :c6542; + :b :c6543; + :b :c6544; + :b :c6545; + :b :c6546; + :b :c6547; + :b :c6548; + :b :c6549; + :b :c6550; + :b :c6551; + :b :c6552; + :b :c6553; + :b :c6554; + :b :c6555; + :b :c6556; + :b :c6557; + :b :c6558; + :b :c6559; + :b :c6560; + :b :c6561; + :b :c6562; + :b :c6563; + :b :c6564; + :b :c6565; + :b :c6566; + :b :c6567; + :b :c6568; + :b :c6569; + :b :c6570; + :b :c6571; + :b :c6572; + :b :c6573; + :b :c6574; + :b :c6575; + :b :c6576; + :b :c6577; + :b :c6578; + :b :c6579; + :b :c6580; + :b :c6581; + :b :c6582; + :b :c6583; + :b :c6584; + :b :c6585; + :b :c6586; + :b :c6587; + :b :c6588; + :b :c6589; + :b :c6590; + :b :c6591; + :b :c6592; + :b :c6593; + :b :c6594; + :b :c6595; + :b :c6596; + :b :c6597; + :b :c6598; + :b :c6599; + :b :c6600; + :b :c6601; + :b :c6602; + :b :c6603; + :b :c6604; + :b :c6605; + :b :c6606; + :b :c6607; + :b :c6608; + :b :c6609; + :b :c6610; + :b :c6611; + :b :c6612; + :b :c6613; + :b :c6614; + :b :c6615; + :b :c6616; + :b :c6617; + :b :c6618; + :b :c6619; + :b :c6620; + :b :c6621; + :b :c6622; + :b :c6623; + :b :c6624; + :b :c6625; + :b :c6626; + :b :c6627; + :b :c6628; + :b :c6629; + :b :c6630; + :b :c6631; + :b :c6632; + :b :c6633; + :b :c6634; + :b :c6635; + :b :c6636; + :b :c6637; + :b :c6638; + :b :c6639; + :b :c6640; + :b :c6641; + :b :c6642; + :b :c6643; + :b :c6644; + :b :c6645; + :b :c6646; + :b :c6647; + :b :c6648; + :b :c6649; + :b :c6650; + :b :c6651; + :b :c6652; + :b :c6653; + :b :c6654; + :b :c6655; + :b :c6656; + :b :c6657; + :b :c6658; + :b :c6659; + :b :c6660; + :b :c6661; + :b :c6662; + :b :c6663; + :b :c6664; + :b :c6665; + :b :c6666; + :b :c6667; + :b :c6668; + :b :c6669; + :b :c6670; + :b :c6671; + :b :c6672; + :b :c6673; + :b :c6674; + :b :c6675; + :b :c6676; + :b :c6677; + :b :c6678; + :b :c6679; + :b :c6680; + :b :c6681; + :b :c6682; + :b :c6683; + :b :c6684; + :b :c6685; + :b :c6686; + :b :c6687; + :b :c6688; + :b :c6689; + :b :c6690; + :b :c6691; + :b :c6692; + :b :c6693; + :b :c6694; + :b :c6695; + :b :c6696; + :b :c6697; + :b :c6698; + :b :c6699; + :b :c6700; + :b :c6701; + :b :c6702; + :b :c6703; + :b :c6704; + :b :c6705; + :b :c6706; + :b :c6707; + :b :c6708; + :b :c6709; + :b :c6710; + :b :c6711; + :b :c6712; + :b :c6713; + :b :c6714; + :b :c6715; + :b :c6716; + :b :c6717; + :b :c6718; + :b :c6719; + :b :c6720; + :b :c6721; + :b :c6722; + :b :c6723; + :b :c6724; + :b :c6725; + :b :c6726; + :b :c6727; + :b :c6728; + :b :c6729; + :b :c6730; + :b :c6731; + :b :c6732; + :b :c6733; + :b :c6734; + :b :c6735; + :b :c6736; + :b :c6737; + :b :c6738; + :b :c6739; + :b :c6740; + :b :c6741; + :b :c6742; + :b :c6743; + :b :c6744; + :b :c6745; + :b :c6746; + :b :c6747; + :b :c6748; + :b :c6749; + :b :c6750; + :b :c6751; + :b :c6752; + :b :c6753; + :b :c6754; + :b :c6755; + :b :c6756; + :b :c6757; + :b :c6758; + :b :c6759; + :b :c6760; + :b :c6761; + :b :c6762; + :b :c6763; + :b :c6764; + :b :c6765; + :b :c6766; + :b :c6767; + :b :c6768; + :b :c6769; + :b :c6770; + :b :c6771; + :b :c6772; + :b :c6773; + :b :c6774; + :b :c6775; + :b :c6776; + :b :c6777; + :b :c6778; + :b :c6779; + :b :c6780; + :b :c6781; + :b :c6782; + :b :c6783; + :b :c6784; + :b :c6785; + :b :c6786; + :b :c6787; + :b :c6788; + :b :c6789; + :b :c6790; + :b :c6791; + :b :c6792; + :b :c6793; + :b :c6794; + :b :c6795; + :b :c6796; + :b :c6797; + :b :c6798; + :b :c6799; + :b :c6800; + :b :c6801; + :b :c6802; + :b :c6803; + :b :c6804; + :b :c6805; + :b :c6806; + :b :c6807; + :b :c6808; + :b :c6809; + :b :c6810; + :b :c6811; + :b :c6812; + :b :c6813; + :b :c6814; + :b :c6815; + :b :c6816; + :b :c6817; + :b :c6818; + :b :c6819; + :b :c6820; + :b :c6821; + :b :c6822; + :b :c6823; + :b :c6824; + :b :c6825; + :b :c6826; + :b :c6827; + :b :c6828; + :b :c6829; + :b :c6830; + :b :c6831; + :b :c6832; + :b :c6833; + :b :c6834; + :b :c6835; + :b :c6836; + :b :c6837; + :b :c6838; + :b :c6839; + :b :c6840; + :b :c6841; + :b :c6842; + :b :c6843; + :b :c6844; + :b :c6845; + :b :c6846; + :b :c6847; + :b :c6848; + :b :c6849; + :b :c6850; + :b :c6851; + :b :c6852; + :b :c6853; + :b :c6854; + :b :c6855; + :b :c6856; + :b :c6857; + :b :c6858; + :b :c6859; + :b :c6860; + :b :c6861; + :b :c6862; + :b :c6863; + :b :c6864; + :b :c6865; + :b :c6866; + :b :c6867; + :b :c6868; + :b :c6869; + :b :c6870; + :b :c6871; + :b :c6872; + :b :c6873; + :b :c6874; + :b :c6875; + :b :c6876; + :b :c6877; + :b :c6878; + :b :c6879; + :b :c6880; + :b :c6881; + :b :c6882; + :b :c6883; + :b :c6884; + :b :c6885; + :b :c6886; + :b :c6887; + :b :c6888; + :b :c6889; + :b :c6890; + :b :c6891; + :b :c6892; + :b :c6893; + :b :c6894; + :b :c6895; + :b :c6896; + :b :c6897; + :b :c6898; + :b :c6899; + :b :c6900; + :b :c6901; + :b :c6902; + :b :c6903; + :b :c6904; + :b :c6905; + :b :c6906; + :b :c6907; + :b :c6908; + :b :c6909; + :b :c6910; + :b :c6911; + :b :c6912; + :b :c6913; + :b :c6914; + :b :c6915; + :b :c6916; + :b :c6917; + :b :c6918; + :b :c6919; + :b :c6920; + :b :c6921; + :b :c6922; + :b :c6923; + :b :c6924; + :b :c6925; + :b :c6926; + :b :c6927; + :b :c6928; + :b :c6929; + :b :c6930; + :b :c6931; + :b :c6932; + :b :c6933; + :b :c6934; + :b :c6935; + :b :c6936; + :b :c6937; + :b :c6938; + :b :c6939; + :b :c6940; + :b :c6941; + :b :c6942; + :b :c6943; + :b :c6944; + :b :c6945; + :b :c6946; + :b :c6947; + :b :c6948; + :b :c6949; + :b :c6950; + :b :c6951; + :b :c6952; + :b :c6953; + :b :c6954; + :b :c6955; + :b :c6956; + :b :c6957; + :b :c6958; + :b :c6959; + :b :c6960; + :b :c6961; + :b :c6962; + :b :c6963; + :b :c6964; + :b :c6965; + :b :c6966; + :b :c6967; + :b :c6968; + :b :c6969; + :b :c6970; + :b :c6971; + :b :c6972; + :b :c6973; + :b :c6974; + :b :c6975; + :b :c6976; + :b :c6977; + :b :c6978; + :b :c6979; + :b :c6980; + :b :c6981; + :b :c6982; + :b :c6983; + :b :c6984; + :b :c6985; + :b :c6986; + :b :c6987; + :b :c6988; + :b :c6989; + :b :c6990; + :b :c6991; + :b :c6992; + :b :c6993; + :b :c6994; + :b :c6995; + :b :c6996; + :b :c6997; + :b :c6998; + :b :c6999; + :b :c7000; + :b :c7001; + :b :c7002; + :b :c7003; + :b :c7004; + :b :c7005; + :b :c7006; + :b :c7007; + :b :c7008; + :b :c7009; + :b :c7010; + :b :c7011; + :b :c7012; + :b :c7013; + :b :c7014; + :b :c7015; + :b :c7016; + :b :c7017; + :b :c7018; + :b :c7019; + :b :c7020; + :b :c7021; + :b :c7022; + :b :c7023; + :b :c7024; + :b :c7025; + :b :c7026; + :b :c7027; + :b :c7028; + :b :c7029; + :b :c7030; + :b :c7031; + :b :c7032; + :b :c7033; + :b :c7034; + :b :c7035; + :b :c7036; + :b :c7037; + :b :c7038; + :b :c7039; + :b :c7040; + :b :c7041; + :b :c7042; + :b :c7043; + :b :c7044; + :b :c7045; + :b :c7046; + :b :c7047; + :b :c7048; + :b :c7049; + :b :c7050; + :b :c7051; + :b :c7052; + :b :c7053; + :b :c7054; + :b :c7055; + :b :c7056; + :b :c7057; + :b :c7058; + :b :c7059; + :b :c7060; + :b :c7061; + :b :c7062; + :b :c7063; + :b :c7064; + :b :c7065; + :b :c7066; + :b :c7067; + :b :c7068; + :b :c7069; + :b :c7070; + :b :c7071; + :b :c7072; + :b :c7073; + :b :c7074; + :b :c7075; + :b :c7076; + :b :c7077; + :b :c7078; + :b :c7079; + :b :c7080; + :b :c7081; + :b :c7082; + :b :c7083; + :b :c7084; + :b :c7085; + :b :c7086; + :b :c7087; + :b :c7088; + :b :c7089; + :b :c7090; + :b :c7091; + :b :c7092; + :b :c7093; + :b :c7094; + :b :c7095; + :b :c7096; + :b :c7097; + :b :c7098; + :b :c7099; + :b :c7100; + :b :c7101; + :b :c7102; + :b :c7103; + :b :c7104; + :b :c7105; + :b :c7106; + :b :c7107; + :b :c7108; + :b :c7109; + :b :c7110; + :b :c7111; + :b :c7112; + :b :c7113; + :b :c7114; + :b :c7115; + :b :c7116; + :b :c7117; + :b :c7118; + :b :c7119; + :b :c7120; + :b :c7121; + :b :c7122; + :b :c7123; + :b :c7124; + :b :c7125; + :b :c7126; + :b :c7127; + :b :c7128; + :b :c7129; + :b :c7130; + :b :c7131; + :b :c7132; + :b :c7133; + :b :c7134; + :b :c7135; + :b :c7136; + :b :c7137; + :b :c7138; + :b :c7139; + :b :c7140; + :b :c7141; + :b :c7142; + :b :c7143; + :b :c7144; + :b :c7145; + :b :c7146; + :b :c7147; + :b :c7148; + :b :c7149; + :b :c7150; + :b :c7151; + :b :c7152; + :b :c7153; + :b :c7154; + :b :c7155; + :b :c7156; + :b :c7157; + :b :c7158; + :b :c7159; + :b :c7160; + :b :c7161; + :b :c7162; + :b :c7163; + :b :c7164; + :b :c7165; + :b :c7166; + :b :c7167; + :b :c7168; + :b :c7169; + :b :c7170; + :b :c7171; + :b :c7172; + :b :c7173; + :b :c7174; + :b :c7175; + :b :c7176; + :b :c7177; + :b :c7178; + :b :c7179; + :b :c7180; + :b :c7181; + :b :c7182; + :b :c7183; + :b :c7184; + :b :c7185; + :b :c7186; + :b :c7187; + :b :c7188; + :b :c7189; + :b :c7190; + :b :c7191; + :b :c7192; + :b :c7193; + :b :c7194; + :b :c7195; + :b :c7196; + :b :c7197; + :b :c7198; + :b :c7199; + :b :c7200; + :b :c7201; + :b :c7202; + :b :c7203; + :b :c7204; + :b :c7205; + :b :c7206; + :b :c7207; + :b :c7208; + :b :c7209; + :b :c7210; + :b :c7211; + :b :c7212; + :b :c7213; + :b :c7214; + :b :c7215; + :b :c7216; + :b :c7217; + :b :c7218; + :b :c7219; + :b :c7220; + :b :c7221; + :b :c7222; + :b :c7223; + :b :c7224; + :b :c7225; + :b :c7226; + :b :c7227; + :b :c7228; + :b :c7229; + :b :c7230; + :b :c7231; + :b :c7232; + :b :c7233; + :b :c7234; + :b :c7235; + :b :c7236; + :b :c7237; + :b :c7238; + :b :c7239; + :b :c7240; + :b :c7241; + :b :c7242; + :b :c7243; + :b :c7244; + :b :c7245; + :b :c7246; + :b :c7247; + :b :c7248; + :b :c7249; + :b :c7250; + :b :c7251; + :b :c7252; + :b :c7253; + :b :c7254; + :b :c7255; + :b :c7256; + :b :c7257; + :b :c7258; + :b :c7259; + :b :c7260; + :b :c7261; + :b :c7262; + :b :c7263; + :b :c7264; + :b :c7265; + :b :c7266; + :b :c7267; + :b :c7268; + :b :c7269; + :b :c7270; + :b :c7271; + :b :c7272; + :b :c7273; + :b :c7274; + :b :c7275; + :b :c7276; + :b :c7277; + :b :c7278; + :b :c7279; + :b :c7280; + :b :c7281; + :b :c7282; + :b :c7283; + :b :c7284; + :b :c7285; + :b :c7286; + :b :c7287; + :b :c7288; + :b :c7289; + :b :c7290; + :b :c7291; + :b :c7292; + :b :c7293; + :b :c7294; + :b :c7295; + :b :c7296; + :b :c7297; + :b :c7298; + :b :c7299; + :b :c7300; + :b :c7301; + :b :c7302; + :b :c7303; + :b :c7304; + :b :c7305; + :b :c7306; + :b :c7307; + :b :c7308; + :b :c7309; + :b :c7310; + :b :c7311; + :b :c7312; + :b :c7313; + :b :c7314; + :b :c7315; + :b :c7316; + :b :c7317; + :b :c7318; + :b :c7319; + :b :c7320; + :b :c7321; + :b :c7322; + :b :c7323; + :b :c7324; + :b :c7325; + :b :c7326; + :b :c7327; + :b :c7328; + :b :c7329; + :b :c7330; + :b :c7331; + :b :c7332; + :b :c7333; + :b :c7334; + :b :c7335; + :b :c7336; + :b :c7337; + :b :c7338; + :b :c7339; + :b :c7340; + :b :c7341; + :b :c7342; + :b :c7343; + :b :c7344; + :b :c7345; + :b :c7346; + :b :c7347; + :b :c7348; + :b :c7349; + :b :c7350; + :b :c7351; + :b :c7352; + :b :c7353; + :b :c7354; + :b :c7355; + :b :c7356; + :b :c7357; + :b :c7358; + :b :c7359; + :b :c7360; + :b :c7361; + :b :c7362; + :b :c7363; + :b :c7364; + :b :c7365; + :b :c7366; + :b :c7367; + :b :c7368; + :b :c7369; + :b :c7370; + :b :c7371; + :b :c7372; + :b :c7373; + :b :c7374; + :b :c7375; + :b :c7376; + :b :c7377; + :b :c7378; + :b :c7379; + :b :c7380; + :b :c7381; + :b :c7382; + :b :c7383; + :b :c7384; + :b :c7385; + :b :c7386; + :b :c7387; + :b :c7388; + :b :c7389; + :b :c7390; + :b :c7391; + :b :c7392; + :b :c7393; + :b :c7394; + :b :c7395; + :b :c7396; + :b :c7397; + :b :c7398; + :b :c7399; + :b :c7400; + :b :c7401; + :b :c7402; + :b :c7403; + :b :c7404; + :b :c7405; + :b :c7406; + :b :c7407; + :b :c7408; + :b :c7409; + :b :c7410; + :b :c7411; + :b :c7412; + :b :c7413; + :b :c7414; + :b :c7415; + :b :c7416; + :b :c7417; + :b :c7418; + :b :c7419; + :b :c7420; + :b :c7421; + :b :c7422; + :b :c7423; + :b :c7424; + :b :c7425; + :b :c7426; + :b :c7427; + :b :c7428; + :b :c7429; + :b :c7430; + :b :c7431; + :b :c7432; + :b :c7433; + :b :c7434; + :b :c7435; + :b :c7436; + :b :c7437; + :b :c7438; + :b :c7439; + :b :c7440; + :b :c7441; + :b :c7442; + :b :c7443; + :b :c7444; + :b :c7445; + :b :c7446; + :b :c7447; + :b :c7448; + :b :c7449; + :b :c7450; + :b :c7451; + :b :c7452; + :b :c7453; + :b :c7454; + :b :c7455; + :b :c7456; + :b :c7457; + :b :c7458; + :b :c7459; + :b :c7460; + :b :c7461; + :b :c7462; + :b :c7463; + :b :c7464; + :b :c7465; + :b :c7466; + :b :c7467; + :b :c7468; + :b :c7469; + :b :c7470; + :b :c7471; + :b :c7472; + :b :c7473; + :b :c7474; + :b :c7475; + :b :c7476; + :b :c7477; + :b :c7478; + :b :c7479; + :b :c7480; + :b :c7481; + :b :c7482; + :b :c7483; + :b :c7484; + :b :c7485; + :b :c7486; + :b :c7487; + :b :c7488; + :b :c7489; + :b :c7490; + :b :c7491; + :b :c7492; + :b :c7493; + :b :c7494; + :b :c7495; + :b :c7496; + :b :c7497; + :b :c7498; + :b :c7499; + :b :c7500; + :b :c7501; + :b :c7502; + :b :c7503; + :b :c7504; + :b :c7505; + :b :c7506; + :b :c7507; + :b :c7508; + :b :c7509; + :b :c7510; + :b :c7511; + :b :c7512; + :b :c7513; + :b :c7514; + :b :c7515; + :b :c7516; + :b :c7517; + :b :c7518; + :b :c7519; + :b :c7520; + :b :c7521; + :b :c7522; + :b :c7523; + :b :c7524; + :b :c7525; + :b :c7526; + :b :c7527; + :b :c7528; + :b :c7529; + :b :c7530; + :b :c7531; + :b :c7532; + :b :c7533; + :b :c7534; + :b :c7535; + :b :c7536; + :b :c7537; + :b :c7538; + :b :c7539; + :b :c7540; + :b :c7541; + :b :c7542; + :b :c7543; + :b :c7544; + :b :c7545; + :b :c7546; + :b :c7547; + :b :c7548; + :b :c7549; + :b :c7550; + :b :c7551; + :b :c7552; + :b :c7553; + :b :c7554; + :b :c7555; + :b :c7556; + :b :c7557; + :b :c7558; + :b :c7559; + :b :c7560; + :b :c7561; + :b :c7562; + :b :c7563; + :b :c7564; + :b :c7565; + :b :c7566; + :b :c7567; + :b :c7568; + :b :c7569; + :b :c7570; + :b :c7571; + :b :c7572; + :b :c7573; + :b :c7574; + :b :c7575; + :b :c7576; + :b :c7577; + :b :c7578; + :b :c7579; + :b :c7580; + :b :c7581; + :b :c7582; + :b :c7583; + :b :c7584; + :b :c7585; + :b :c7586; + :b :c7587; + :b :c7588; + :b :c7589; + :b :c7590; + :b :c7591; + :b :c7592; + :b :c7593; + :b :c7594; + :b :c7595; + :b :c7596; + :b :c7597; + :b :c7598; + :b :c7599; + :b :c7600; + :b :c7601; + :b :c7602; + :b :c7603; + :b :c7604; + :b :c7605; + :b :c7606; + :b :c7607; + :b :c7608; + :b :c7609; + :b :c7610; + :b :c7611; + :b :c7612; + :b :c7613; + :b :c7614; + :b :c7615; + :b :c7616; + :b :c7617; + :b :c7618; + :b :c7619; + :b :c7620; + :b :c7621; + :b :c7622; + :b :c7623; + :b :c7624; + :b :c7625; + :b :c7626; + :b :c7627; + :b :c7628; + :b :c7629; + :b :c7630; + :b :c7631; + :b :c7632; + :b :c7633; + :b :c7634; + :b :c7635; + :b :c7636; + :b :c7637; + :b :c7638; + :b :c7639; + :b :c7640; + :b :c7641; + :b :c7642; + :b :c7643; + :b :c7644; + :b :c7645; + :b :c7646; + :b :c7647; + :b :c7648; + :b :c7649; + :b :c7650; + :b :c7651; + :b :c7652; + :b :c7653; + :b :c7654; + :b :c7655; + :b :c7656; + :b :c7657; + :b :c7658; + :b :c7659; + :b :c7660; + :b :c7661; + :b :c7662; + :b :c7663; + :b :c7664; + :b :c7665; + :b :c7666; + :b :c7667; + :b :c7668; + :b :c7669; + :b :c7670; + :b :c7671; + :b :c7672; + :b :c7673; + :b :c7674; + :b :c7675; + :b :c7676; + :b :c7677; + :b :c7678; + :b :c7679; + :b :c7680; + :b :c7681; + :b :c7682; + :b :c7683; + :b :c7684; + :b :c7685; + :b :c7686; + :b :c7687; + :b :c7688; + :b :c7689; + :b :c7690; + :b :c7691; + :b :c7692; + :b :c7693; + :b :c7694; + :b :c7695; + :b :c7696; + :b :c7697; + :b :c7698; + :b :c7699; + :b :c7700; + :b :c7701; + :b :c7702; + :b :c7703; + :b :c7704; + :b :c7705; + :b :c7706; + :b :c7707; + :b :c7708; + :b :c7709; + :b :c7710; + :b :c7711; + :b :c7712; + :b :c7713; + :b :c7714; + :b :c7715; + :b :c7716; + :b :c7717; + :b :c7718; + :b :c7719; + :b :c7720; + :b :c7721; + :b :c7722; + :b :c7723; + :b :c7724; + :b :c7725; + :b :c7726; + :b :c7727; + :b :c7728; + :b :c7729; + :b :c7730; + :b :c7731; + :b :c7732; + :b :c7733; + :b :c7734; + :b :c7735; + :b :c7736; + :b :c7737; + :b :c7738; + :b :c7739; + :b :c7740; + :b :c7741; + :b :c7742; + :b :c7743; + :b :c7744; + :b :c7745; + :b :c7746; + :b :c7747; + :b :c7748; + :b :c7749; + :b :c7750; + :b :c7751; + :b :c7752; + :b :c7753; + :b :c7754; + :b :c7755; + :b :c7756; + :b :c7757; + :b :c7758; + :b :c7759; + :b :c7760; + :b :c7761; + :b :c7762; + :b :c7763; + :b :c7764; + :b :c7765; + :b :c7766; + :b :c7767; + :b :c7768; + :b :c7769; + :b :c7770; + :b :c7771; + :b :c7772; + :b :c7773; + :b :c7774; + :b :c7775; + :b :c7776; + :b :c7777; + :b :c7778; + :b :c7779; + :b :c7780; + :b :c7781; + :b :c7782; + :b :c7783; + :b :c7784; + :b :c7785; + :b :c7786; + :b :c7787; + :b :c7788; + :b :c7789; + :b :c7790; + :b :c7791; + :b :c7792; + :b :c7793; + :b :c7794; + :b :c7795; + :b :c7796; + :b :c7797; + :b :c7798; + :b :c7799; + :b :c7800; + :b :c7801; + :b :c7802; + :b :c7803; + :b :c7804; + :b :c7805; + :b :c7806; + :b :c7807; + :b :c7808; + :b :c7809; + :b :c7810; + :b :c7811; + :b :c7812; + :b :c7813; + :b :c7814; + :b :c7815; + :b :c7816; + :b :c7817; + :b :c7818; + :b :c7819; + :b :c7820; + :b :c7821; + :b :c7822; + :b :c7823; + :b :c7824; + :b :c7825; + :b :c7826; + :b :c7827; + :b :c7828; + :b :c7829; + :b :c7830; + :b :c7831; + :b :c7832; + :b :c7833; + :b :c7834; + :b :c7835; + :b :c7836; + :b :c7837; + :b :c7838; + :b :c7839; + :b :c7840; + :b :c7841; + :b :c7842; + :b :c7843; + :b :c7844; + :b :c7845; + :b :c7846; + :b :c7847; + :b :c7848; + :b :c7849; + :b :c7850; + :b :c7851; + :b :c7852; + :b :c7853; + :b :c7854; + :b :c7855; + :b :c7856; + :b :c7857; + :b :c7858; + :b :c7859; + :b :c7860; + :b :c7861; + :b :c7862; + :b :c7863; + :b :c7864; + :b :c7865; + :b :c7866; + :b :c7867; + :b :c7868; + :b :c7869; + :b :c7870; + :b :c7871; + :b :c7872; + :b :c7873; + :b :c7874; + :b :c7875; + :b :c7876; + :b :c7877; + :b :c7878; + :b :c7879; + :b :c7880; + :b :c7881; + :b :c7882; + :b :c7883; + :b :c7884; + :b :c7885; + :b :c7886; + :b :c7887; + :b :c7888; + :b :c7889; + :b :c7890; + :b :c7891; + :b :c7892; + :b :c7893; + :b :c7894; + :b :c7895; + :b :c7896; + :b :c7897; + :b :c7898; + :b :c7899; + :b :c7900; + :b :c7901; + :b :c7902; + :b :c7903; + :b :c7904; + :b :c7905; + :b :c7906; + :b :c7907; + :b :c7908; + :b :c7909; + :b :c7910; + :b :c7911; + :b :c7912; + :b :c7913; + :b :c7914; + :b :c7915; + :b :c7916; + :b :c7917; + :b :c7918; + :b :c7919; + :b :c7920; + :b :c7921; + :b :c7922; + :b :c7923; + :b :c7924; + :b :c7925; + :b :c7926; + :b :c7927; + :b :c7928; + :b :c7929; + :b :c7930; + :b :c7931; + :b :c7932; + :b :c7933; + :b :c7934; + :b :c7935; + :b :c7936; + :b :c7937; + :b :c7938; + :b :c7939; + :b :c7940; + :b :c7941; + :b :c7942; + :b :c7943; + :b :c7944; + :b :c7945; + :b :c7946; + :b :c7947; + :b :c7948; + :b :c7949; + :b :c7950; + :b :c7951; + :b :c7952; + :b :c7953; + :b :c7954; + :b :c7955; + :b :c7956; + :b :c7957; + :b :c7958; + :b :c7959; + :b :c7960; + :b :c7961; + :b :c7962; + :b :c7963; + :b :c7964; + :b :c7965; + :b :c7966; + :b :c7967; + :b :c7968; + :b :c7969; + :b :c7970; + :b :c7971; + :b :c7972; + :b :c7973; + :b :c7974; + :b :c7975; + :b :c7976; + :b :c7977; + :b :c7978; + :b :c7979; + :b :c7980; + :b :c7981; + :b :c7982; + :b :c7983; + :b :c7984; + :b :c7985; + :b :c7986; + :b :c7987; + :b :c7988; + :b :c7989; + :b :c7990; + :b :c7991; + :b :c7992; + :b :c7993; + :b :c7994; + :b :c7995; + :b :c7996; + :b :c7997; + :b :c7998; + :b :c7999; + :b :c8000; + :b :c8001; + :b :c8002; + :b :c8003; + :b :c8004; + :b :c8005; + :b :c8006; + :b :c8007; + :b :c8008; + :b :c8009; + :b :c8010; + :b :c8011; + :b :c8012; + :b :c8013; + :b :c8014; + :b :c8015; + :b :c8016; + :b :c8017; + :b :c8018; + :b :c8019; + :b :c8020; + :b :c8021; + :b :c8022; + :b :c8023; + :b :c8024; + :b :c8025; + :b :c8026; + :b :c8027; + :b :c8028; + :b :c8029; + :b :c8030; + :b :c8031; + :b :c8032; + :b :c8033; + :b :c8034; + :b :c8035; + :b :c8036; + :b :c8037; + :b :c8038; + :b :c8039; + :b :c8040; + :b :c8041; + :b :c8042; + :b :c8043; + :b :c8044; + :b :c8045; + :b :c8046; + :b :c8047; + :b :c8048; + :b :c8049; + :b :c8050; + :b :c8051; + :b :c8052; + :b :c8053; + :b :c8054; + :b :c8055; + :b :c8056; + :b :c8057; + :b :c8058; + :b :c8059; + :b :c8060; + :b :c8061; + :b :c8062; + :b :c8063; + :b :c8064; + :b :c8065; + :b :c8066; + :b :c8067; + :b :c8068; + :b :c8069; + :b :c8070; + :b :c8071; + :b :c8072; + :b :c8073; + :b :c8074; + :b :c8075; + :b :c8076; + :b :c8077; + :b :c8078; + :b :c8079; + :b :c8080; + :b :c8081; + :b :c8082; + :b :c8083; + :b :c8084; + :b :c8085; + :b :c8086; + :b :c8087; + :b :c8088; + :b :c8089; + :b :c8090; + :b :c8091; + :b :c8092; + :b :c8093; + :b :c8094; + :b :c8095; + :b :c8096; + :b :c8097; + :b :c8098; + :b :c8099; + :b :c8100; + :b :c8101; + :b :c8102; + :b :c8103; + :b :c8104; + :b :c8105; + :b :c8106; + :b :c8107; + :b :c8108; + :b :c8109; + :b :c8110; + :b :c8111; + :b :c8112; + :b :c8113; + :b :c8114; + :b :c8115; + :b :c8116; + :b :c8117; + :b :c8118; + :b :c8119; + :b :c8120; + :b :c8121; + :b :c8122; + :b :c8123; + :b :c8124; + :b :c8125; + :b :c8126; + :b :c8127; + :b :c8128; + :b :c8129; + :b :c8130; + :b :c8131; + :b :c8132; + :b :c8133; + :b :c8134; + :b :c8135; + :b :c8136; + :b :c8137; + :b :c8138; + :b :c8139; + :b :c8140; + :b :c8141; + :b :c8142; + :b :c8143; + :b :c8144; + :b :c8145; + :b :c8146; + :b :c8147; + :b :c8148; + :b :c8149; + :b :c8150; + :b :c8151; + :b :c8152; + :b :c8153; + :b :c8154; + :b :c8155; + :b :c8156; + :b :c8157; + :b :c8158; + :b :c8159; + :b :c8160; + :b :c8161; + :b :c8162; + :b :c8163; + :b :c8164; + :b :c8165; + :b :c8166; + :b :c8167; + :b :c8168; + :b :c8169; + :b :c8170; + :b :c8171; + :b :c8172; + :b :c8173; + :b :c8174; + :b :c8175; + :b :c8176; + :b :c8177; + :b :c8178; + :b :c8179; + :b :c8180; + :b :c8181; + :b :c8182; + :b :c8183; + :b :c8184; + :b :c8185; + :b :c8186; + :b :c8187; + :b :c8188; + :b :c8189; + :b :c8190; + :b :c8191; + :b :c8192; + :b :c8193; + :b :c8194; + :b :c8195; + :b :c8196; + :b :c8197; + :b :c8198; + :b :c8199; + :b :c8200; + :b :c8201; + :b :c8202; + :b :c8203; + :b :c8204; + :b :c8205; + :b :c8206; + :b :c8207; + :b :c8208; + :b :c8209; + :b :c8210; + :b :c8211; + :b :c8212; + :b :c8213; + :b :c8214; + :b :c8215; + :b :c8216; + :b :c8217; + :b :c8218; + :b :c8219; + :b :c8220; + :b :c8221; + :b :c8222; + :b :c8223; + :b :c8224; + :b :c8225; + :b :c8226; + :b :c8227; + :b :c8228; + :b :c8229; + :b :c8230; + :b :c8231; + :b :c8232; + :b :c8233; + :b :c8234; + :b :c8235; + :b :c8236; + :b :c8237; + :b :c8238; + :b :c8239; + :b :c8240; + :b :c8241; + :b :c8242; + :b :c8243; + :b :c8244; + :b :c8245; + :b :c8246; + :b :c8247; + :b :c8248; + :b :c8249; + :b :c8250; + :b :c8251; + :b :c8252; + :b :c8253; + :b :c8254; + :b :c8255; + :b :c8256; + :b :c8257; + :b :c8258; + :b :c8259; + :b :c8260; + :b :c8261; + :b :c8262; + :b :c8263; + :b :c8264; + :b :c8265; + :b :c8266; + :b :c8267; + :b :c8268; + :b :c8269; + :b :c8270; + :b :c8271; + :b :c8272; + :b :c8273; + :b :c8274; + :b :c8275; + :b :c8276; + :b :c8277; + :b :c8278; + :b :c8279; + :b :c8280; + :b :c8281; + :b :c8282; + :b :c8283; + :b :c8284; + :b :c8285; + :b :c8286; + :b :c8287; + :b :c8288; + :b :c8289; + :b :c8290; + :b :c8291; + :b :c8292; + :b :c8293; + :b :c8294; + :b :c8295; + :b :c8296; + :b :c8297; + :b :c8298; + :b :c8299; + :b :c8300; + :b :c8301; + :b :c8302; + :b :c8303; + :b :c8304; + :b :c8305; + :b :c8306; + :b :c8307; + :b :c8308; + :b :c8309; + :b :c8310; + :b :c8311; + :b :c8312; + :b :c8313; + :b :c8314; + :b :c8315; + :b :c8316; + :b :c8317; + :b :c8318; + :b :c8319; + :b :c8320; + :b :c8321; + :b :c8322; + :b :c8323; + :b :c8324; + :b :c8325; + :b :c8326; + :b :c8327; + :b :c8328; + :b :c8329; + :b :c8330; + :b :c8331; + :b :c8332; + :b :c8333; + :b :c8334; + :b :c8335; + :b :c8336; + :b :c8337; + :b :c8338; + :b :c8339; + :b :c8340; + :b :c8341; + :b :c8342; + :b :c8343; + :b :c8344; + :b :c8345; + :b :c8346; + :b :c8347; + :b :c8348; + :b :c8349; + :b :c8350; + :b :c8351; + :b :c8352; + :b :c8353; + :b :c8354; + :b :c8355; + :b :c8356; + :b :c8357; + :b :c8358; + :b :c8359; + :b :c8360; + :b :c8361; + :b :c8362; + :b :c8363; + :b :c8364; + :b :c8365; + :b :c8366; + :b :c8367; + :b :c8368; + :b :c8369; + :b :c8370; + :b :c8371; + :b :c8372; + :b :c8373; + :b :c8374; + :b :c8375; + :b :c8376; + :b :c8377; + :b :c8378; + :b :c8379; + :b :c8380; + :b :c8381; + :b :c8382; + :b :c8383; + :b :c8384; + :b :c8385; + :b :c8386; + :b :c8387; + :b :c8388; + :b :c8389; + :b :c8390; + :b :c8391; + :b :c8392; + :b :c8393; + :b :c8394; + :b :c8395; + :b :c8396; + :b :c8397; + :b :c8398; + :b :c8399; + :b :c8400; + :b :c8401; + :b :c8402; + :b :c8403; + :b :c8404; + :b :c8405; + :b :c8406; + :b :c8407; + :b :c8408; + :b :c8409; + :b :c8410; + :b :c8411; + :b :c8412; + :b :c8413; + :b :c8414; + :b :c8415; + :b :c8416; + :b :c8417; + :b :c8418; + :b :c8419; + :b :c8420; + :b :c8421; + :b :c8422; + :b :c8423; + :b :c8424; + :b :c8425; + :b :c8426; + :b :c8427; + :b :c8428; + :b :c8429; + :b :c8430; + :b :c8431; + :b :c8432; + :b :c8433; + :b :c8434; + :b :c8435; + :b :c8436; + :b :c8437; + :b :c8438; + :b :c8439; + :b :c8440; + :b :c8441; + :b :c8442; + :b :c8443; + :b :c8444; + :b :c8445; + :b :c8446; + :b :c8447; + :b :c8448; + :b :c8449; + :b :c8450; + :b :c8451; + :b :c8452; + :b :c8453; + :b :c8454; + :b :c8455; + :b :c8456; + :b :c8457; + :b :c8458; + :b :c8459; + :b :c8460; + :b :c8461; + :b :c8462; + :b :c8463; + :b :c8464; + :b :c8465; + :b :c8466; + :b :c8467; + :b :c8468; + :b :c8469; + :b :c8470; + :b :c8471; + :b :c8472; + :b :c8473; + :b :c8474; + :b :c8475; + :b :c8476; + :b :c8477; + :b :c8478; + :b :c8479; + :b :c8480; + :b :c8481; + :b :c8482; + :b :c8483; + :b :c8484; + :b :c8485; + :b :c8486; + :b :c8487; + :b :c8488; + :b :c8489; + :b :c8490; + :b :c8491; + :b :c8492; + :b :c8493; + :b :c8494; + :b :c8495; + :b :c8496; + :b :c8497; + :b :c8498; + :b :c8499; + :b :c8500; + :b :c8501; + :b :c8502; + :b :c8503; + :b :c8504; + :b :c8505; + :b :c8506; + :b :c8507; + :b :c8508; + :b :c8509; + :b :c8510; + :b :c8511; + :b :c8512; + :b :c8513; + :b :c8514; + :b :c8515; + :b :c8516; + :b :c8517; + :b :c8518; + :b :c8519; + :b :c8520; + :b :c8521; + :b :c8522; + :b :c8523; + :b :c8524; + :b :c8525; + :b :c8526; + :b :c8527; + :b :c8528; + :b :c8529; + :b :c8530; + :b :c8531; + :b :c8532; + :b :c8533; + :b :c8534; + :b :c8535; + :b :c8536; + :b :c8537; + :b :c8538; + :b :c8539; + :b :c8540; + :b :c8541; + :b :c8542; + :b :c8543; + :b :c8544; + :b :c8545; + :b :c8546; + :b :c8547; + :b :c8548; + :b :c8549; + :b :c8550; + :b :c8551; + :b :c8552; + :b :c8553; + :b :c8554; + :b :c8555; + :b :c8556; + :b :c8557; + :b :c8558; + :b :c8559; + :b :c8560; + :b :c8561; + :b :c8562; + :b :c8563; + :b :c8564; + :b :c8565; + :b :c8566; + :b :c8567; + :b :c8568; + :b :c8569; + :b :c8570; + :b :c8571; + :b :c8572; + :b :c8573; + :b :c8574; + :b :c8575; + :b :c8576; + :b :c8577; + :b :c8578; + :b :c8579; + :b :c8580; + :b :c8581; + :b :c8582; + :b :c8583; + :b :c8584; + :b :c8585; + :b :c8586; + :b :c8587; + :b :c8588; + :b :c8589; + :b :c8590; + :b :c8591; + :b :c8592; + :b :c8593; + :b :c8594; + :b :c8595; + :b :c8596; + :b :c8597; + :b :c8598; + :b :c8599; + :b :c8600; + :b :c8601; + :b :c8602; + :b :c8603; + :b :c8604; + :b :c8605; + :b :c8606; + :b :c8607; + :b :c8608; + :b :c8609; + :b :c8610; + :b :c8611; + :b :c8612; + :b :c8613; + :b :c8614; + :b :c8615; + :b :c8616; + :b :c8617; + :b :c8618; + :b :c8619; + :b :c8620; + :b :c8621; + :b :c8622; + :b :c8623; + :b :c8624; + :b :c8625; + :b :c8626; + :b :c8627; + :b :c8628; + :b :c8629; + :b :c8630; + :b :c8631; + :b :c8632; + :b :c8633; + :b :c8634; + :b :c8635; + :b :c8636; + :b :c8637; + :b :c8638; + :b :c8639; + :b :c8640; + :b :c8641; + :b :c8642; + :b :c8643; + :b :c8644; + :b :c8645; + :b :c8646; + :b :c8647; + :b :c8648; + :b :c8649; + :b :c8650; + :b :c8651; + :b :c8652; + :b :c8653; + :b :c8654; + :b :c8655; + :b :c8656; + :b :c8657; + :b :c8658; + :b :c8659; + :b :c8660; + :b :c8661; + :b :c8662; + :b :c8663; + :b :c8664; + :b :c8665; + :b :c8666; + :b :c8667; + :b :c8668; + :b :c8669; + :b :c8670; + :b :c8671; + :b :c8672; + :b :c8673; + :b :c8674; + :b :c8675; + :b :c8676; + :b :c8677; + :b :c8678; + :b :c8679; + :b :c8680; + :b :c8681; + :b :c8682; + :b :c8683; + :b :c8684; + :b :c8685; + :b :c8686; + :b :c8687; + :b :c8688; + :b :c8689; + :b :c8690; + :b :c8691; + :b :c8692; + :b :c8693; + :b :c8694; + :b :c8695; + :b :c8696; + :b :c8697; + :b :c8698; + :b :c8699; + :b :c8700; + :b :c8701; + :b :c8702; + :b :c8703; + :b :c8704; + :b :c8705; + :b :c8706; + :b :c8707; + :b :c8708; + :b :c8709; + :b :c8710; + :b :c8711; + :b :c8712; + :b :c8713; + :b :c8714; + :b :c8715; + :b :c8716; + :b :c8717; + :b :c8718; + :b :c8719; + :b :c8720; + :b :c8721; + :b :c8722; + :b :c8723; + :b :c8724; + :b :c8725; + :b :c8726; + :b :c8727; + :b :c8728; + :b :c8729; + :b :c8730; + :b :c8731; + :b :c8732; + :b :c8733; + :b :c8734; + :b :c8735; + :b :c8736; + :b :c8737; + :b :c8738; + :b :c8739; + :b :c8740; + :b :c8741; + :b :c8742; + :b :c8743; + :b :c8744; + :b :c8745; + :b :c8746; + :b :c8747; + :b :c8748; + :b :c8749; + :b :c8750; + :b :c8751; + :b :c8752; + :b :c8753; + :b :c8754; + :b :c8755; + :b :c8756; + :b :c8757; + :b :c8758; + :b :c8759; + :b :c8760; + :b :c8761; + :b :c8762; + :b :c8763; + :b :c8764; + :b :c8765; + :b :c8766; + :b :c8767; + :b :c8768; + :b :c8769; + :b :c8770; + :b :c8771; + :b :c8772; + :b :c8773; + :b :c8774; + :b :c8775; + :b :c8776; + :b :c8777; + :b :c8778; + :b :c8779; + :b :c8780; + :b :c8781; + :b :c8782; + :b :c8783; + :b :c8784; + :b :c8785; + :b :c8786; + :b :c8787; + :b :c8788; + :b :c8789; + :b :c8790; + :b :c8791; + :b :c8792; + :b :c8793; + :b :c8794; + :b :c8795; + :b :c8796; + :b :c8797; + :b :c8798; + :b :c8799; + :b :c8800; + :b :c8801; + :b :c8802; + :b :c8803; + :b :c8804; + :b :c8805; + :b :c8806; + :b :c8807; + :b :c8808; + :b :c8809; + :b :c8810; + :b :c8811; + :b :c8812; + :b :c8813; + :b :c8814; + :b :c8815; + :b :c8816; + :b :c8817; + :b :c8818; + :b :c8819; + :b :c8820; + :b :c8821; + :b :c8822; + :b :c8823; + :b :c8824; + :b :c8825; + :b :c8826; + :b :c8827; + :b :c8828; + :b :c8829; + :b :c8830; + :b :c8831; + :b :c8832; + :b :c8833; + :b :c8834; + :b :c8835; + :b :c8836; + :b :c8837; + :b :c8838; + :b :c8839; + :b :c8840; + :b :c8841; + :b :c8842; + :b :c8843; + :b :c8844; + :b :c8845; + :b :c8846; + :b :c8847; + :b :c8848; + :b :c8849; + :b :c8850; + :b :c8851; + :b :c8852; + :b :c8853; + :b :c8854; + :b :c8855; + :b :c8856; + :b :c8857; + :b :c8858; + :b :c8859; + :b :c8860; + :b :c8861; + :b :c8862; + :b :c8863; + :b :c8864; + :b :c8865; + :b :c8866; + :b :c8867; + :b :c8868; + :b :c8869; + :b :c8870; + :b :c8871; + :b :c8872; + :b :c8873; + :b :c8874; + :b :c8875; + :b :c8876; + :b :c8877; + :b :c8878; + :b :c8879; + :b :c8880; + :b :c8881; + :b :c8882; + :b :c8883; + :b :c8884; + :b :c8885; + :b :c8886; + :b :c8887; + :b :c8888; + :b :c8889; + :b :c8890; + :b :c8891; + :b :c8892; + :b :c8893; + :b :c8894; + :b :c8895; + :b :c8896; + :b :c8897; + :b :c8898; + :b :c8899; + :b :c8900; + :b :c8901; + :b :c8902; + :b :c8903; + :b :c8904; + :b :c8905; + :b :c8906; + :b :c8907; + :b :c8908; + :b :c8909; + :b :c8910; + :b :c8911; + :b :c8912; + :b :c8913; + :b :c8914; + :b :c8915; + :b :c8916; + :b :c8917; + :b :c8918; + :b :c8919; + :b :c8920; + :b :c8921; + :b :c8922; + :b :c8923; + :b :c8924; + :b :c8925; + :b :c8926; + :b :c8927; + :b :c8928; + :b :c8929; + :b :c8930; + :b :c8931; + :b :c8932; + :b :c8933; + :b :c8934; + :b :c8935; + :b :c8936; + :b :c8937; + :b :c8938; + :b :c8939; + :b :c8940; + :b :c8941; + :b :c8942; + :b :c8943; + :b :c8944; + :b :c8945; + :b :c8946; + :b :c8947; + :b :c8948; + :b :c8949; + :b :c8950; + :b :c8951; + :b :c8952; + :b :c8953; + :b :c8954; + :b :c8955; + :b :c8956; + :b :c8957; + :b :c8958; + :b :c8959; + :b :c8960; + :b :c8961; + :b :c8962; + :b :c8963; + :b :c8964; + :b :c8965; + :b :c8966; + :b :c8967; + :b :c8968; + :b :c8969; + :b :c8970; + :b :c8971; + :b :c8972; + :b :c8973; + :b :c8974; + :b :c8975; + :b :c8976; + :b :c8977; + :b :c8978; + :b :c8979; + :b :c8980; + :b :c8981; + :b :c8982; + :b :c8983; + :b :c8984; + :b :c8985; + :b :c8986; + :b :c8987; + :b :c8988; + :b :c8989; + :b :c8990; + :b :c8991; + :b :c8992; + :b :c8993; + :b :c8994; + :b :c8995; + :b :c8996; + :b :c8997; + :b :c8998; + :b :c8999; + :b :c9000; + :b :c9001; + :b :c9002; + :b :c9003; + :b :c9004; + :b :c9005; + :b :c9006; + :b :c9007; + :b :c9008; + :b :c9009; + :b :c9010; + :b :c9011; + :b :c9012; + :b :c9013; + :b :c9014; + :b :c9015; + :b :c9016; + :b :c9017; + :b :c9018; + :b :c9019; + :b :c9020; + :b :c9021; + :b :c9022; + :b :c9023; + :b :c9024; + :b :c9025; + :b :c9026; + :b :c9027; + :b :c9028; + :b :c9029; + :b :c9030; + :b :c9031; + :b :c9032; + :b :c9033; + :b :c9034; + :b :c9035; + :b :c9036; + :b :c9037; + :b :c9038; + :b :c9039; + :b :c9040; + :b :c9041; + :b :c9042; + :b :c9043; + :b :c9044; + :b :c9045; + :b :c9046; + :b :c9047; + :b :c9048; + :b :c9049; + :b :c9050; + :b :c9051; + :b :c9052; + :b :c9053; + :b :c9054; + :b :c9055; + :b :c9056; + :b :c9057; + :b :c9058; + :b :c9059; + :b :c9060; + :b :c9061; + :b :c9062; + :b :c9063; + :b :c9064; + :b :c9065; + :b :c9066; + :b :c9067; + :b :c9068; + :b :c9069; + :b :c9070; + :b :c9071; + :b :c9072; + :b :c9073; + :b :c9074; + :b :c9075; + :b :c9076; + :b :c9077; + :b :c9078; + :b :c9079; + :b :c9080; + :b :c9081; + :b :c9082; + :b :c9083; + :b :c9084; + :b :c9085; + :b :c9086; + :b :c9087; + :b :c9088; + :b :c9089; + :b :c9090; + :b :c9091; + :b :c9092; + :b :c9093; + :b :c9094; + :b :c9095; + :b :c9096; + :b :c9097; + :b :c9098; + :b :c9099; + :b :c9100; + :b :c9101; + :b :c9102; + :b :c9103; + :b :c9104; + :b :c9105; + :b :c9106; + :b :c9107; + :b :c9108; + :b :c9109; + :b :c9110; + :b :c9111; + :b :c9112; + :b :c9113; + :b :c9114; + :b :c9115; + :b :c9116; + :b :c9117; + :b :c9118; + :b :c9119; + :b :c9120; + :b :c9121; + :b :c9122; + :b :c9123; + :b :c9124; + :b :c9125; + :b :c9126; + :b :c9127; + :b :c9128; + :b :c9129; + :b :c9130; + :b :c9131; + :b :c9132; + :b :c9133; + :b :c9134; + :b :c9135; + :b :c9136; + :b :c9137; + :b :c9138; + :b :c9139; + :b :c9140; + :b :c9141; + :b :c9142; + :b :c9143; + :b :c9144; + :b :c9145; + :b :c9146; + :b :c9147; + :b :c9148; + :b :c9149; + :b :c9150; + :b :c9151; + :b :c9152; + :b :c9153; + :b :c9154; + :b :c9155; + :b :c9156; + :b :c9157; + :b :c9158; + :b :c9159; + :b :c9160; + :b :c9161; + :b :c9162; + :b :c9163; + :b :c9164; + :b :c9165; + :b :c9166; + :b :c9167; + :b :c9168; + :b :c9169; + :b :c9170; + :b :c9171; + :b :c9172; + :b :c9173; + :b :c9174; + :b :c9175; + :b :c9176; + :b :c9177; + :b :c9178; + :b :c9179; + :b :c9180; + :b :c9181; + :b :c9182; + :b :c9183; + :b :c9184; + :b :c9185; + :b :c9186; + :b :c9187; + :b :c9188; + :b :c9189; + :b :c9190; + :b :c9191; + :b :c9192; + :b :c9193; + :b :c9194; + :b :c9195; + :b :c9196; + :b :c9197; + :b :c9198; + :b :c9199; + :b :c9200; + :b :c9201; + :b :c9202; + :b :c9203; + :b :c9204; + :b :c9205; + :b :c9206; + :b :c9207; + :b :c9208; + :b :c9209; + :b :c9210; + :b :c9211; + :b :c9212; + :b :c9213; + :b :c9214; + :b :c9215; + :b :c9216; + :b :c9217; + :b :c9218; + :b :c9219; + :b :c9220; + :b :c9221; + :b :c9222; + :b :c9223; + :b :c9224; + :b :c9225; + :b :c9226; + :b :c9227; + :b :c9228; + :b :c9229; + :b :c9230; + :b :c9231; + :b :c9232; + :b :c9233; + :b :c9234; + :b :c9235; + :b :c9236; + :b :c9237; + :b :c9238; + :b :c9239; + :b :c9240; + :b :c9241; + :b :c9242; + :b :c9243; + :b :c9244; + :b :c9245; + :b :c9246; + :b :c9247; + :b :c9248; + :b :c9249; + :b :c9250; + :b :c9251; + :b :c9252; + :b :c9253; + :b :c9254; + :b :c9255; + :b :c9256; + :b :c9257; + :b :c9258; + :b :c9259; + :b :c9260; + :b :c9261; + :b :c9262; + :b :c9263; + :b :c9264; + :b :c9265; + :b :c9266; + :b :c9267; + :b :c9268; + :b :c9269; + :b :c9270; + :b :c9271; + :b :c9272; + :b :c9273; + :b :c9274; + :b :c9275; + :b :c9276; + :b :c9277; + :b :c9278; + :b :c9279; + :b :c9280; + :b :c9281; + :b :c9282; + :b :c9283; + :b :c9284; + :b :c9285; + :b :c9286; + :b :c9287; + :b :c9288; + :b :c9289; + :b :c9290; + :b :c9291; + :b :c9292; + :b :c9293; + :b :c9294; + :b :c9295; + :b :c9296; + :b :c9297; + :b :c9298; + :b :c9299; + :b :c9300; + :b :c9301; + :b :c9302; + :b :c9303; + :b :c9304; + :b :c9305; + :b :c9306; + :b :c9307; + :b :c9308; + :b :c9309; + :b :c9310; + :b :c9311; + :b :c9312; + :b :c9313; + :b :c9314; + :b :c9315; + :b :c9316; + :b :c9317; + :b :c9318; + :b :c9319; + :b :c9320; + :b :c9321; + :b :c9322; + :b :c9323; + :b :c9324; + :b :c9325; + :b :c9326; + :b :c9327; + :b :c9328; + :b :c9329; + :b :c9330; + :b :c9331; + :b :c9332; + :b :c9333; + :b :c9334; + :b :c9335; + :b :c9336; + :b :c9337; + :b :c9338; + :b :c9339; + :b :c9340; + :b :c9341; + :b :c9342; + :b :c9343; + :b :c9344; + :b :c9345; + :b :c9346; + :b :c9347; + :b :c9348; + :b :c9349; + :b :c9350; + :b :c9351; + :b :c9352; + :b :c9353; + :b :c9354; + :b :c9355; + :b :c9356; + :b :c9357; + :b :c9358; + :b :c9359; + :b :c9360; + :b :c9361; + :b :c9362; + :b :c9363; + :b :c9364; + :b :c9365; + :b :c9366; + :b :c9367; + :b :c9368; + :b :c9369; + :b :c9370; + :b :c9371; + :b :c9372; + :b :c9373; + :b :c9374; + :b :c9375; + :b :c9376; + :b :c9377; + :b :c9378; + :b :c9379; + :b :c9380; + :b :c9381; + :b :c9382; + :b :c9383; + :b :c9384; + :b :c9385; + :b :c9386; + :b :c9387; + :b :c9388; + :b :c9389; + :b :c9390; + :b :c9391; + :b :c9392; + :b :c9393; + :b :c9394; + :b :c9395; + :b :c9396; + :b :c9397; + :b :c9398; + :b :c9399; + :b :c9400; + :b :c9401; + :b :c9402; + :b :c9403; + :b :c9404; + :b :c9405; + :b :c9406; + :b :c9407; + :b :c9408; + :b :c9409; + :b :c9410; + :b :c9411; + :b :c9412; + :b :c9413; + :b :c9414; + :b :c9415; + :b :c9416; + :b :c9417; + :b :c9418; + :b :c9419; + :b :c9420; + :b :c9421; + :b :c9422; + :b :c9423; + :b :c9424; + :b :c9425; + :b :c9426; + :b :c9427; + :b :c9428; + :b :c9429; + :b :c9430; + :b :c9431; + :b :c9432; + :b :c9433; + :b :c9434; + :b :c9435; + :b :c9436; + :b :c9437; + :b :c9438; + :b :c9439; + :b :c9440; + :b :c9441; + :b :c9442; + :b :c9443; + :b :c9444; + :b :c9445; + :b :c9446; + :b :c9447; + :b :c9448; + :b :c9449; + :b :c9450; + :b :c9451; + :b :c9452; + :b :c9453; + :b :c9454; + :b :c9455; + :b :c9456; + :b :c9457; + :b :c9458; + :b :c9459; + :b :c9460; + :b :c9461; + :b :c9462; + :b :c9463; + :b :c9464; + :b :c9465; + :b :c9466; + :b :c9467; + :b :c9468; + :b :c9469; + :b :c9470; + :b :c9471; + :b :c9472; + :b :c9473; + :b :c9474; + :b :c9475; + :b :c9476; + :b :c9477; + :b :c9478; + :b :c9479; + :b :c9480; + :b :c9481; + :b :c9482; + :b :c9483; + :b :c9484; + :b :c9485; + :b :c9486; + :b :c9487; + :b :c9488; + :b :c9489; + :b :c9490; + :b :c9491; + :b :c9492; + :b :c9493; + :b :c9494; + :b :c9495; + :b :c9496; + :b :c9497; + :b :c9498; + :b :c9499; + :b :c9500; + :b :c9501; + :b :c9502; + :b :c9503; + :b :c9504; + :b :c9505; + :b :c9506; + :b :c9507; + :b :c9508; + :b :c9509; + :b :c9510; + :b :c9511; + :b :c9512; + :b :c9513; + :b :c9514; + :b :c9515; + :b :c9516; + :b :c9517; + :b :c9518; + :b :c9519; + :b :c9520; + :b :c9521; + :b :c9522; + :b :c9523; + :b :c9524; + :b :c9525; + :b :c9526; + :b :c9527; + :b :c9528; + :b :c9529; + :b :c9530; + :b :c9531; + :b :c9532; + :b :c9533; + :b :c9534; + :b :c9535; + :b :c9536; + :b :c9537; + :b :c9538; + :b :c9539; + :b :c9540; + :b :c9541; + :b :c9542; + :b :c9543; + :b :c9544; + :b :c9545; + :b :c9546; + :b :c9547; + :b :c9548; + :b :c9549; + :b :c9550; + :b :c9551; + :b :c9552; + :b :c9553; + :b :c9554; + :b :c9555; + :b :c9556; + :b :c9557; + :b :c9558; + :b :c9559; + :b :c9560; + :b :c9561; + :b :c9562; + :b :c9563; + :b :c9564; + :b :c9565; + :b :c9566; + :b :c9567; + :b :c9568; + :b :c9569; + :b :c9570; + :b :c9571; + :b :c9572; + :b :c9573; + :b :c9574; + :b :c9575; + :b :c9576; + :b :c9577; + :b :c9578; + :b :c9579; + :b :c9580; + :b :c9581; + :b :c9582; + :b :c9583; + :b :c9584; + :b :c9585; + :b :c9586; + :b :c9587; + :b :c9588; + :b :c9589; + :b :c9590; + :b :c9591; + :b :c9592; + :b :c9593; + :b :c9594; + :b :c9595; + :b :c9596; + :b :c9597; + :b :c9598; + :b :c9599; + :b :c9600; + :b :c9601; + :b :c9602; + :b :c9603; + :b :c9604; + :b :c9605; + :b :c9606; + :b :c9607; + :b :c9608; + :b :c9609; + :b :c9610; + :b :c9611; + :b :c9612; + :b :c9613; + :b :c9614; + :b :c9615; + :b :c9616; + :b :c9617; + :b :c9618; + :b :c9619; + :b :c9620; + :b :c9621; + :b :c9622; + :b :c9623; + :b :c9624; + :b :c9625; + :b :c9626; + :b :c9627; + :b :c9628; + :b :c9629; + :b :c9630; + :b :c9631; + :b :c9632; + :b :c9633; + :b :c9634; + :b :c9635; + :b :c9636; + :b :c9637; + :b :c9638; + :b :c9639; + :b :c9640; + :b :c9641; + :b :c9642; + :b :c9643; + :b :c9644; + :b :c9645; + :b :c9646; + :b :c9647; + :b :c9648; + :b :c9649; + :b :c9650; + :b :c9651; + :b :c9652; + :b :c9653; + :b :c9654; + :b :c9655; + :b :c9656; + :b :c9657; + :b :c9658; + :b :c9659; + :b :c9660; + :b :c9661; + :b :c9662; + :b :c9663; + :b :c9664; + :b :c9665; + :b :c9666; + :b :c9667; + :b :c9668; + :b :c9669; + :b :c9670; + :b :c9671; + :b :c9672; + :b :c9673; + :b :c9674; + :b :c9675; + :b :c9676; + :b :c9677; + :b :c9678; + :b :c9679; + :b :c9680; + :b :c9681; + :b :c9682; + :b :c9683; + :b :c9684; + :b :c9685; + :b :c9686; + :b :c9687; + :b :c9688; + :b :c9689; + :b :c9690; + :b :c9691; + :b :c9692; + :b :c9693; + :b :c9694; + :b :c9695; + :b :c9696; + :b :c9697; + :b :c9698; + :b :c9699; + :b :c9700; + :b :c9701; + :b :c9702; + :b :c9703; + :b :c9704; + :b :c9705; + :b :c9706; + :b :c9707; + :b :c9708; + :b :c9709; + :b :c9710; + :b :c9711; + :b :c9712; + :b :c9713; + :b :c9714; + :b :c9715; + :b :c9716; + :b :c9717; + :b :c9718; + :b :c9719; + :b :c9720; + :b :c9721; + :b :c9722; + :b :c9723; + :b :c9724; + :b :c9725; + :b :c9726; + :b :c9727; + :b :c9728; + :b :c9729; + :b :c9730; + :b :c9731; + :b :c9732; + :b :c9733; + :b :c9734; + :b :c9735; + :b :c9736; + :b :c9737; + :b :c9738; + :b :c9739; + :b :c9740; + :b :c9741; + :b :c9742; + :b :c9743; + :b :c9744; + :b :c9745; + :b :c9746; + :b :c9747; + :b :c9748; + :b :c9749; + :b :c9750; + :b :c9751; + :b :c9752; + :b :c9753; + :b :c9754; + :b :c9755; + :b :c9756; + :b :c9757; + :b :c9758; + :b :c9759; + :b :c9760; + :b :c9761; + :b :c9762; + :b :c9763; + :b :c9764; + :b :c9765; + :b :c9766; + :b :c9767; + :b :c9768; + :b :c9769; + :b :c9770; + :b :c9771; + :b :c9772; + :b :c9773; + :b :c9774; + :b :c9775; + :b :c9776; + :b :c9777; + :b :c9778; + :b :c9779; + :b :c9780; + :b :c9781; + :b :c9782; + :b :c9783; + :b :c9784; + :b :c9785; + :b :c9786; + :b :c9787; + :b :c9788; + :b :c9789; + :b :c9790; + :b :c9791; + :b :c9792; + :b :c9793; + :b :c9794; + :b :c9795; + :b :c9796; + :b :c9797; + :b :c9798; + :b :c9799; + :b :c9800; + :b :c9801; + :b :c9802; + :b :c9803; + :b :c9804; + :b :c9805; + :b :c9806; + :b :c9807; + :b :c9808; + :b :c9809; + :b :c9810; + :b :c9811; + :b :c9812; + :b :c9813; + :b :c9814; + :b :c9815; + :b :c9816; + :b :c9817; + :b :c9818; + :b :c9819; + :b :c9820; + :b :c9821; + :b :c9822; + :b :c9823; + :b :c9824; + :b :c9825; + :b :c9826; + :b :c9827; + :b :c9828; + :b :c9829; + :b :c9830; + :b :c9831; + :b :c9832; + :b :c9833; + :b :c9834; + :b :c9835; + :b :c9836; + :b :c9837; + :b :c9838; + :b :c9839; + :b :c9840; + :b :c9841; + :b :c9842; + :b :c9843; + :b :c9844; + :b :c9845; + :b :c9846; + :b :c9847; + :b :c9848; + :b :c9849; + :b :c9850; + :b :c9851; + :b :c9852; + :b :c9853; + :b :c9854; + :b :c9855; + :b :c9856; + :b :c9857; + :b :c9858; + :b :c9859; + :b :c9860; + :b :c9861; + :b :c9862; + :b :c9863; + :b :c9864; + :b :c9865; + :b :c9866; + :b :c9867; + :b :c9868; + :b :c9869; + :b :c9870; + :b :c9871; + :b :c9872; + :b :c9873; + :b :c9874; + :b :c9875; + :b :c9876; + :b :c9877; + :b :c9878; + :b :c9879; + :b :c9880; + :b :c9881; + :b :c9882; + :b :c9883; + :b :c9884; + :b :c9885; + :b :c9886; + :b :c9887; + :b :c9888; + :b :c9889; + :b :c9890; + :b :c9891; + :b :c9892; + :b :c9893; + :b :c9894; + :b :c9895; + :b :c9896; + :b :c9897; + :b :c9898; + :b :c9899; + :b :c9900; + :b :c9901; + :b :c9902; + :b :c9903; + :b :c9904; + :b :c9905; + :b :c9906; + :b :c9907; + :b :c9908; + :b :c9909; + :b :c9910; + :b :c9911; + :b :c9912; + :b :c9913; + :b :c9914; + :b :c9915; + :b :c9916; + :b :c9917; + :b :c9918; + :b :c9919; + :b :c9920; + :b :c9921; + :b :c9922; + :b :c9923; + :b :c9924; + :b :c9925; + :b :c9926; + :b :c9927; + :b :c9928; + :b :c9929; + :b :c9930; + :b :c9931; + :b :c9932; + :b :c9933; + :b :c9934; + :b :c9935; + :b :c9936; + :b :c9937; + :b :c9938; + :b :c9939; + :b :c9940; + :b :c9941; + :b :c9942; + :b :c9943; + :b :c9944; + :b :c9945; + :b :c9946; + :b :c9947; + :b :c9948; + :b :c9949; + :b :c9950; + :b :c9951; + :b :c9952; + :b :c9953; + :b :c9954; + :b :c9955; + :b :c9956; + :b :c9957; + :b :c9958; + :b :c9959; + :b :c9960; + :b :c9961; + :b :c9962; + :b :c9963; + :b :c9964; + :b :c9965; + :b :c9966; + :b :c9967; + :b :c9968; + :b :c9969; + :b :c9970; + :b :c9971; + :b :c9972; + :b :c9973; + :b :c9974; + :b :c9975; + :b :c9976; + :b :c9977; + :b :c9978; + :b :c9979; + :b :c9980; + :b :c9981; + :b :c9982; + :b :c9983; + :b :c9984; + :b :c9985; + :b :c9986; + :b :c9987; + :b :c9988; + :b :c9989; + :b :c9990; + :b :c9991; + :b :c9992; + :b :c9993; + :b :c9994; + :b :c9995; + :b :c9996; + :b :c9997; + :b :c9998; + :b :c9999; + :b :c10000 . diff --git a/testsuite/serd-tests/good/test-18.nt b/testsuite/serd-tests/good/test-18.nt new file mode 100644 index 00000000..4ff4b95b --- /dev/null +++ b/testsuite/serd-tests/good/test-18.nt @@ -0,0 +1,2 @@ + "\nthis \ris a \U00015678long\t\nliteral\uABCD\n" . + "\tThis \uABCDis\r \U00015678another\n\none\n" . diff --git a/testsuite/serd-tests/good/test-18.ttl b/testsuite/serd-tests/good/test-18.ttl new file mode 100644 index 00000000..1adfa451 --- /dev/null +++ b/testsuite/serd-tests/good/test-18.ttl @@ -0,0 +1,9 @@ +@prefix : . + +:a :b """\nthis \ris a \U00015678long\t +literal\uABCD +""" . + +:d :e """\tThis \uABCDis\r \U00015678another\n +one +""" . diff --git a/testsuite/serd-tests/good/test-30.nt b/testsuite/serd-tests/good/test-30.nt new file mode 100644 index 00000000..f400cba6 --- /dev/null +++ b/testsuite/serd-tests/good/test-30.nt @@ -0,0 +1,5 @@ + . + . + . + . + . diff --git a/testsuite/serd-tests/good/test-30.ttl b/testsuite/serd-tests/good/test-30.ttl new file mode 100644 index 00000000..9aff060f --- /dev/null +++ b/testsuite/serd-tests/good/test-30.ttl @@ -0,0 +1,12 @@ +# In-scope base URI is http://drobilla.net/sw/serd/test/good/ at this point + . +@base . +# In-scope base URI is http://example.org/ns/ at this point + . +@base . +# In-scope base URI is http://example.org/ns/foo/ at this point + . +@prefix : . +:a4 :b4 :c4 . +@prefix : . +:a5 :b5 :c5 . diff --git a/testsuite/serd-tests/good/test-a-without-whitespace.nt b/testsuite/serd-tests/good/test-a-without-whitespace.nt new file mode 100644 index 00000000..27c5e7da --- /dev/null +++ b/testsuite/serd-tests/good/test-a-without-whitespace.nt @@ -0,0 +1,6 @@ +_:b1 . +_:b2 _:b3 . +_:b4 . + . + _:b5 . + . diff --git a/testsuite/serd-tests/good/test-a-without-whitespace.ttl b/testsuite/serd-tests/good/test-a-without-whitespace.ttl new file mode 100644 index 00000000..2d6db000 --- /dev/null +++ b/testsuite/serd-tests/good/test-a-without-whitespace.ttl @@ -0,0 +1,6 @@ +[a]. +[a[]]. +[a()]. +a. +a[]. +a(). diff --git a/testsuite/serd-tests/good/test-backspace.nt b/testsuite/serd-tests/good/test-backspace.nt new file mode 100644 index 00000000..3dfffa38 --- /dev/null +++ b/testsuite/serd-tests/good/test-backspace.nt @@ -0,0 +1,3 @@ + "\u0008" . + "\u0008" . + "\n\u0008\n" . diff --git a/testsuite/serd-tests/good/test-backspace.ttl b/testsuite/serd-tests/good/test-backspace.ttl new file mode 100644 index 00000000..1e33ba95 --- /dev/null +++ b/testsuite/serd-tests/good/test-backspace.ttl @@ -0,0 +1,5 @@ + "\u0008" . + "" . + """ + +""" . \ No newline at end of file diff --git a/testsuite/serd-tests/good/test-base-nopath.nt b/testsuite/serd-tests/good/test-base-nopath.nt new file mode 100644 index 00000000..8628a30b --- /dev/null +++ b/testsuite/serd-tests/good/test-base-nopath.nt @@ -0,0 +1 @@ + . diff --git a/testsuite/serd-tests/good/test-base-nopath.ttl b/testsuite/serd-tests/good/test-base-nopath.ttl new file mode 100644 index 00000000..8735936a --- /dev/null +++ b/testsuite/serd-tests/good/test-base-nopath.ttl @@ -0,0 +1,3 @@ +@base . + +

. diff --git a/testsuite/serd-tests/good/test-base-query.nt b/testsuite/serd-tests/good/test-base-query.nt new file mode 100644 index 00000000..d40c2f6b --- /dev/null +++ b/testsuite/serd-tests/good/test-base-query.nt @@ -0,0 +1 @@ + . diff --git a/testsuite/serd-tests/good/test-base-query.ttl b/testsuite/serd-tests/good/test-base-query.ttl new file mode 100644 index 00000000..77638817 --- /dev/null +++ b/testsuite/serd-tests/good/test-base-query.ttl @@ -0,0 +1,3 @@ +@base . + +<> a . \ No newline at end of file diff --git a/testsuite/serd-tests/good/test-blank-cont.nt b/testsuite/serd-tests/good/test-blank-cont.nt new file mode 100644 index 00000000..a4b44e7d --- /dev/null +++ b/testsuite/serd-tests/good/test-blank-cont.nt @@ -0,0 +1,4 @@ + _:b1 . +_:b1 . + _:b2 . +_:b2 . diff --git a/testsuite/serd-tests/good/test-blank-cont.ttl b/testsuite/serd-tests/good/test-blank-cont.ttl new file mode 100644 index 00000000..b161cdc4 --- /dev/null +++ b/testsuite/serd-tests/good/test-blank-cont.ttl @@ -0,0 +1,4 @@ +@prefix : . + +:subj :pred [ a :FirstThing ] ; + :pred [ a :SecondThing ] . diff --git a/testsuite/serd-tests/good/test-blank-in-list.nt b/testsuite/serd-tests/good/test-blank-in-list.nt new file mode 100644 index 00000000..36bac6d7 --- /dev/null +++ b/testsuite/serd-tests/good/test-blank-in-list.nt @@ -0,0 +1,4 @@ + _:b1 . +_:b1 _:b2 . +_:b2 . +_:b1 . diff --git a/testsuite/serd-tests/good/test-blank-in-list.ttl b/testsuite/serd-tests/good/test-blank-in-list.ttl new file mode 100644 index 00000000..5c0b0766 --- /dev/null +++ b/testsuite/serd-tests/good/test-blank-in-list.ttl @@ -0,0 +1,2 @@ +@prefix : . +:a :b ( [ :c :d ] ) . diff --git a/testsuite/serd-tests/good/test-blank-node-statement.nt b/testsuite/serd-tests/good/test-blank-node-statement.nt new file mode 100644 index 00000000..3280b28d --- /dev/null +++ b/testsuite/serd-tests/good/test-blank-node-statement.nt @@ -0,0 +1 @@ +_:b1 . diff --git a/testsuite/serd-tests/good/test-blank-node-statement.ttl b/testsuite/serd-tests/good/test-blank-node-statement.ttl new file mode 100644 index 00000000..c84f8487 --- /dev/null +++ b/testsuite/serd-tests/good/test-blank-node-statement.ttl @@ -0,0 +1,3 @@ +[ + +] . diff --git a/testsuite/serd-tests/good/test-blankdot.nt b/testsuite/serd-tests/good/test-blankdot.nt new file mode 100644 index 00000000..9d4b769b --- /dev/null +++ b/testsuite/serd-tests/good/test-blankdot.nt @@ -0,0 +1 @@ + _:c . diff --git a/testsuite/serd-tests/good/test-blankdot.ttl b/testsuite/serd-tests/good/test-blankdot.ttl new file mode 100644 index 00000000..a6ee7234 --- /dev/null +++ b/testsuite/serd-tests/good/test-blankdot.ttl @@ -0,0 +1 @@ + _:c. diff --git a/testsuite/serd-tests/good/test-bom.nt b/testsuite/serd-tests/good/test-bom.nt new file mode 100644 index 00000000..aea1655b --- /dev/null +++ b/testsuite/serd-tests/good/test-bom.nt @@ -0,0 +1 @@ + . diff --git a/testsuite/serd-tests/good/test-bom.ttl b/testsuite/serd-tests/good/test-bom.ttl new file mode 100644 index 00000000..8d6534e1 --- /dev/null +++ b/testsuite/serd-tests/good/test-bom.ttl @@ -0,0 +1,3 @@ +# This file starts with a UTF-8 Byte Order Mark + + a . \ No newline at end of file diff --git a/testsuite/serd-tests/good/test-cr.nt b/testsuite/serd-tests/good/test-cr.nt new file mode 100644 index 00000000..aea1655b --- /dev/null +++ b/testsuite/serd-tests/good/test-cr.nt @@ -0,0 +1 @@ + . diff --git a/testsuite/serd-tests/good/test-cr.ttl b/testsuite/serd-tests/good/test-cr.ttl new file mode 100644 index 00000000..5410648a --- /dev/null +++ b/testsuite/serd-tests/good/test-cr.ttl @@ -0,0 +1,2 @@ +#Test a . + diff --git a/testsuite/serd-tests/good/test-delete.nt b/testsuite/serd-tests/good/test-delete.nt new file mode 100644 index 00000000..41ac8062 --- /dev/null +++ b/testsuite/serd-tests/good/test-delete.nt @@ -0,0 +1,2 @@ + "\u007F" . + "\u007F" . diff --git a/testsuite/serd-tests/good/test-delete.ttl b/testsuite/serd-tests/good/test-delete.ttl new file mode 100644 index 00000000..4bc97060 --- /dev/null +++ b/testsuite/serd-tests/good/test-delete.ttl @@ -0,0 +1,2 @@ + "\u007F" . + "" . diff --git a/testsuite/serd-tests/good/test-digit-start-pname.nt b/testsuite/serd-tests/good/test-digit-start-pname.nt new file mode 100644 index 00000000..66eab7db --- /dev/null +++ b/testsuite/serd-tests/good/test-digit-start-pname.nt @@ -0,0 +1 @@ + . diff --git a/testsuite/serd-tests/good/test-digit-start-pname.ttl b/testsuite/serd-tests/good/test-digit-start-pname.ttl new file mode 100644 index 00000000..6ca8ade8 --- /dev/null +++ b/testsuite/serd-tests/good/test-digit-start-pname.ttl @@ -0,0 +1,3 @@ +@prefix eg: . + +eg:1thing a eg:Thing . \ No newline at end of file diff --git a/testsuite/serd-tests/good/test-empty-path-base.nt b/testsuite/serd-tests/good/test-empty-path-base.nt new file mode 100644 index 00000000..9ad50617 --- /dev/null +++ b/testsuite/serd-tests/good/test-empty-path-base.nt @@ -0,0 +1 @@ + . diff --git a/testsuite/serd-tests/good/test-empty-path-base.ttl b/testsuite/serd-tests/good/test-empty-path-base.ttl new file mode 100644 index 00000000..8410553a --- /dev/null +++ b/testsuite/serd-tests/good/test-empty-path-base.ttl @@ -0,0 +1,3 @@ +@base . + + a . diff --git a/testsuite/serd-tests/good/test-empty.nt b/testsuite/serd-tests/good/test-empty.nt new file mode 100644 index 00000000..e69de29b diff --git a/testsuite/serd-tests/good/test-empty.ttl b/testsuite/serd-tests/good/test-empty.ttl new file mode 100644 index 00000000..e69de29b diff --git a/testsuite/serd-tests/good/test-eof-at-page-end.nt b/testsuite/serd-tests/good/test-eof-at-page-end.nt new file mode 100644 index 00000000..64d24586 --- /dev/null +++ b/testsuite/serd-tests/good/test-eof-at-page-end.nt @@ -0,0 +1 @@ + "\n0123456789012345678901234567890123456789\n\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567890123456789012345678901234567890123456789\n01234567" . diff --git a/testsuite/serd-tests/good/test-eof-at-page-end.ttl b/testsuite/serd-tests/good/test-eof-at-page-end.ttl new file mode 100644 index 00000000..d6d9af26 --- /dev/null +++ b/testsuite/serd-tests/good/test-eof-at-page-end.ttl @@ -0,0 +1,85 @@ + """ +0123456789012345678901234567890123456789 + +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 + +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 + +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 + +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567890123456789012345678901234567890123456789 +01234567""" . \ No newline at end of file diff --git a/testsuite/serd-tests/good/test-escapes.nt b/testsuite/serd-tests/good/test-escapes.nt new file mode 100644 index 00000000..e30222e7 --- /dev/null +++ b/testsuite/serd-tests/good/test-escapes.nt @@ -0,0 +1,2 @@ + "\\\r\n\t" . + . diff --git a/testsuite/serd-tests/good/test-escapes.ttl b/testsuite/serd-tests/good/test-escapes.ttl new file mode 100644 index 00000000..e30222e7 --- /dev/null +++ b/testsuite/serd-tests/good/test-escapes.ttl @@ -0,0 +1,2 @@ + "\\\r\n\t" . + . diff --git a/testsuite/serd-tests/good/test-ext-namedblank-iri.nt b/testsuite/serd-tests/good/test-ext-namedblank-iri.nt new file mode 100644 index 00000000..26075a73 --- /dev/null +++ b/testsuite/serd-tests/good/test-ext-namedblank-iri.nt @@ -0,0 +1,2 @@ + . + "o" . diff --git a/testsuite/serd-tests/good/test-ext-namedblank-iri.ttl b/testsuite/serd-tests/good/test-ext-namedblank-iri.ttl new file mode 100644 index 00000000..28fb32ab --- /dev/null +++ b/testsuite/serd-tests/good/test-ext-namedblank-iri.ttl @@ -0,0 +1,3 @@ +@prefix eg: . + +eg:s eg:p [ == ; eg:name "o" ] . diff --git a/testsuite/serd-tests/good/test-ext-namedblank-prefix.nt b/testsuite/serd-tests/good/test-ext-namedblank-prefix.nt new file mode 100644 index 00000000..9a2710fb --- /dev/null +++ b/testsuite/serd-tests/good/test-ext-namedblank-prefix.nt @@ -0,0 +1,2 @@ + . + "o" . diff --git a/testsuite/serd-tests/good/test-ext-namedblank-prefix.ttl b/testsuite/serd-tests/good/test-ext-namedblank-prefix.ttl new file mode 100644 index 00000000..7de79655 --- /dev/null +++ b/testsuite/serd-tests/good/test-ext-namedblank-prefix.ttl @@ -0,0 +1,3 @@ +@prefix eg: . + +eg:s eg:p [ == eg:o ; eg:name "o" ] . diff --git a/testsuite/serd-tests/good/test-form-feed.nt b/testsuite/serd-tests/good/test-form-feed.nt new file mode 100644 index 00000000..05478971 --- /dev/null +++ b/testsuite/serd-tests/good/test-form-feed.nt @@ -0,0 +1,2 @@ + "\u000C" . + "\u000C" . diff --git a/testsuite/serd-tests/good/test-form-feed.ttl b/testsuite/serd-tests/good/test-form-feed.ttl new file mode 100644 index 00000000..78921159 --- /dev/null +++ b/testsuite/serd-tests/good/test-form-feed.ttl @@ -0,0 +1,2 @@ + "\u000C" . + " " . diff --git a/testsuite/serd-tests/good/test-id.nt b/testsuite/serd-tests/good/test-id.nt new file mode 100644 index 00000000..b4b11069 --- /dev/null +++ b/testsuite/serd-tests/good/test-id.nt @@ -0,0 +1,2 @@ + _:b1 . +_:B1 . diff --git a/testsuite/serd-tests/good/test-id.ttl b/testsuite/serd-tests/good/test-id.ttl new file mode 100644 index 00000000..cfd0e9ca --- /dev/null +++ b/testsuite/serd-tests/good/test-id.ttl @@ -0,0 +1,4 @@ +@prefix : . + +:c :d [] . +_:b1 :a :b . diff --git a/testsuite/serd-tests/good/test-lang.nt b/testsuite/serd-tests/good/test-lang.nt new file mode 100644 index 00000000..c36dda42 --- /dev/null +++ b/testsuite/serd-tests/good/test-lang.nt @@ -0,0 +1,4 @@ + "Hello"@en . + "Howdy"@en-us . + "Bonjour"@fr . + "Guten Tag"@de-latn-de . diff --git a/testsuite/serd-tests/good/test-lang.ttl b/testsuite/serd-tests/good/test-lang.ttl new file mode 100644 index 00000000..f7c2727f --- /dev/null +++ b/testsuite/serd-tests/good/test-lang.ttl @@ -0,0 +1,6 @@ +@prefix : . + +:thing :greeting "Hello"@en ; + :greeting "Howdy"@en-us ; + :greeting "Bonjour"@fr ; + :greeting "Guten Tag"@de-latn-de . diff --git a/testsuite/serd-tests/good/test-list-in-blank.nt b/testsuite/serd-tests/good/test-list-in-blank.nt new file mode 100644 index 00000000..108363f2 --- /dev/null +++ b/testsuite/serd-tests/good/test-list-in-blank.nt @@ -0,0 +1,6 @@ + _:b1 . +_:b1 _:b2 . +_:b2 "apple" . +_:b2 _:b3 . +_:b3 "banana" . +_:b3 . diff --git a/testsuite/serd-tests/good/test-list-in-blank.ttl b/testsuite/serd-tests/good/test-list-in-blank.ttl new file mode 100644 index 00000000..bdf75fce --- /dev/null +++ b/testsuite/serd-tests/good/test-list-in-blank.ttl @@ -0,0 +1,2 @@ +@prefix : . +:a :b [ :c ( "apple" "banana" ) ] . diff --git a/testsuite/serd-tests/good/test-list-subject.nt b/testsuite/serd-tests/good/test-list-subject.nt new file mode 100644 index 00000000..e4f9c40c --- /dev/null +++ b/testsuite/serd-tests/good/test-list-subject.nt @@ -0,0 +1,6 @@ + . +_:b1 "apple" . +_:b1 _:b2 . +_:b2 "banana" . +_:b2 . +_:b1 . diff --git a/testsuite/serd-tests/good/test-list-subject.ttl b/testsuite/serd-tests/good/test-list-subject.ttl new file mode 100644 index 00000000..5cd81dfb --- /dev/null +++ b/testsuite/serd-tests/good/test-list-subject.ttl @@ -0,0 +1,2 @@ +() a . +("apple" "banana") a . diff --git a/testsuite/serd-tests/good/test-list.nt b/testsuite/serd-tests/good/test-list.nt new file mode 100644 index 00000000..3a27a5a6 --- /dev/null +++ b/testsuite/serd-tests/good/test-list.nt @@ -0,0 +1 @@ + . diff --git a/testsuite/serd-tests/good/test-list.ttl b/testsuite/serd-tests/good/test-list.ttl new file mode 100644 index 00000000..7f4c7699 --- /dev/null +++ b/testsuite/serd-tests/good/test-list.ttl @@ -0,0 +1 @@ +() a . \ No newline at end of file diff --git a/testsuite/serd-tests/good/test-local-name-ends-with-dot.ttl b/testsuite/serd-tests/good/test-local-name-ends-with-dot.ttl new file mode 100644 index 00000000..e426876d --- /dev/null +++ b/testsuite/serd-tests/good/test-local-name-ends-with-dot.ttl @@ -0,0 +1,3 @@ +@prefix eg: . + +eg:s eg:p eg:foo\. . diff --git a/testsuite/serd-tests/good/test-long-string.nt b/testsuite/serd-tests/good/test-long-string.nt new file mode 100644 index 00000000..ba6be49c --- /dev/null +++ b/testsuite/serd-tests/good/test-long-string.nt @@ -0,0 +1 @@ + "This is a\nlong string\nwith\n tabs\nand spaces." . diff --git a/testsuite/serd-tests/good/test-long-string.ttl b/testsuite/serd-tests/good/test-long-string.ttl new file mode 100644 index 00000000..7da44b4b --- /dev/null +++ b/testsuite/serd-tests/good/test-long-string.ttl @@ -0,0 +1,7 @@ +@prefix : . + +:thing :greeting """This is a +long string +with + tabs +and spaces.""" . \ No newline at end of file diff --git a/testsuite/serd-tests/good/test-no-spaces.nt b/testsuite/serd-tests/good/test-no-spaces.nt new file mode 100644 index 00000000..3619a5f0 --- /dev/null +++ b/testsuite/serd-tests/good/test-no-spaces.nt @@ -0,0 +1,4 @@ + . + . + . + . diff --git a/testsuite/serd-tests/good/test-no-spaces.ttl b/testsuite/serd-tests/good/test-no-spaces.ttl new file mode 100644 index 00000000..88171e71 --- /dev/null +++ b/testsuite/serd-tests/good/test-no-spaces.ttl @@ -0,0 +1,3 @@ +@prefix eg: . +eg:s eg:p eg:o;eg:p2 eg:o2,eg:o3 . +eg:s a . \ No newline at end of file diff --git a/testsuite/serd-tests/good/test-non-curie-uri.nt b/testsuite/serd-tests/good/test-non-curie-uri.nt new file mode 100644 index 00000000..502a8754 --- /dev/null +++ b/testsuite/serd-tests/good/test-non-curie-uri.nt @@ -0,0 +1 @@ + . diff --git a/testsuite/serd-tests/good/test-non-curie-uri.ttl b/testsuite/serd-tests/good/test-non-curie-uri.ttl new file mode 100644 index 00000000..40734514 --- /dev/null +++ b/testsuite/serd-tests/good/test-non-curie-uri.ttl @@ -0,0 +1,3 @@ +@prefix eg: . + +eg:s eg:p . diff --git a/testsuite/serd-tests/good/test-num.nt b/testsuite/serd-tests/good/test-num.nt new file mode 100644 index 00000000..54263d46 --- /dev/null +++ b/testsuite/serd-tests/good/test-num.nt @@ -0,0 +1,10 @@ + "0.1"^^ . + "+0.2"^^ . + "-0.3"^^ . + ".4"^^ . + "+.5"^^ . + "-.6"^^ . + "1.58490e-05"^^ . + "1.58490e+05"^^ . + "1.58490e05"^^ . + "1.58490E05"^^ . diff --git a/testsuite/serd-tests/good/test-num.ttl b/testsuite/serd-tests/good/test-num.ttl new file mode 100644 index 00000000..68ad290b --- /dev/null +++ b/testsuite/serd-tests/good/test-num.ttl @@ -0,0 +1,12 @@ +@prefix eg: . + +eg:thing eg:num 0.1 . +eg:thing eg:num +0.2 . +eg:thing eg:num -0.3 . +eg:thing eg:num .4 . +eg:thing eg:num +.5 . +eg:thing eg:num -.6 . +eg:thing eg:num 1.58490e-05 . +eg:thing eg:num 1.58490e+05 . +eg:thing eg:num 1.58490e05 . +eg:thing eg:num 1.58490E05 . diff --git a/testsuite/serd-tests/good/test-out-of-range-unicode.nt b/testsuite/serd-tests/good/test-out-of-range-unicode.nt new file mode 100644 index 00000000..5def9e31 --- /dev/null +++ b/testsuite/serd-tests/good/test-out-of-range-unicode.nt @@ -0,0 +1 @@ + "\uFFFD" . diff --git a/testsuite/serd-tests/good/test-out-of-range-unicode.ttl b/testsuite/serd-tests/good/test-out-of-range-unicode.ttl new file mode 100644 index 00000000..7e64785a --- /dev/null +++ b/testsuite/serd-tests/good/test-out-of-range-unicode.ttl @@ -0,0 +1 @@ + "\U00110000" . diff --git a/testsuite/serd-tests/good/test-prefix.nt b/testsuite/serd-tests/good/test-prefix.nt new file mode 100644 index 00000000..bcfdd4b6 --- /dev/null +++ b/testsuite/serd-tests/good/test-prefix.nt @@ -0,0 +1,6 @@ + . + . + . + . + . + . diff --git a/testsuite/serd-tests/good/test-prefix.ttl b/testsuite/serd-tests/good/test-prefix.ttl new file mode 100644 index 00000000..f79896cb --- /dev/null +++ b/testsuite/serd-tests/good/test-prefix.ttl @@ -0,0 +1,8 @@ +@prefix eg: . + + a . + a . + a . + a . + a . + a . diff --git a/testsuite/serd-tests/good/test-pretty.nt b/testsuite/serd-tests/good/test-pretty.nt new file mode 100644 index 00000000..9251563a --- /dev/null +++ b/testsuite/serd-tests/good/test-pretty.nt @@ -0,0 +1,46 @@ + . +_:b1 . + . +_:b2 _:b3 . +_:b4 "apple" . +_:b4 _:b5 . +_:b5 "banana" . +_:b5 _:b6 . +_:b6 "pear" . +_:b6 . +_:b4 . +_:b7 _:b8 . +_:b8 . +_:b8 _:b9 . +_:b9 . +_:b9 . +_:b7 _:b10 . +_:b10 _:b11 . +_:b11 . +_:b11 _:b12 . +_:b12 . +_:b12 . +_:b10 . +_:b7 . +_:b13 _:b14 . +_:b14 "apple" . +_:b14 _:b15 . +_:b15 "banana" . +_:b15 _:b16 . +_:b16 "pear" . +_:b16 . +_:b17 . +_:b17 . +_:b17 . +_:b18 _:b19 . +_:b19 . +_:b19 . +_:b18 _:b20 . +_:b20 . +_:b21 _:b22 . +_:b22 _:b23 . +_:b23 . +_:b22 _:b24 . +_:b24 _:b25 . +_:b25 . +_:b24 . diff --git a/testsuite/serd-tests/good/test-pretty.ttl b/testsuite/serd-tests/good/test-pretty.ttl new file mode 100644 index 00000000..4eb7204f --- /dev/null +++ b/testsuite/serd-tests/good/test-pretty.ttl @@ -0,0 +1,44 @@ +@prefix : . + +() :isA :List . + +[] :isA :Blank . + +() :sameAs () . + +[] :sameAs [] . + +( + "apple" + "banana" + "pear" +) a :List . + +( + (:a :b) + (:c :d) +) a :List . + +[] + :list ( + "apple" + "banana" + "pear" + ) . + +[] + :a :b , :c , :d . + +[] + :a [ + :b :c ; + :d :e ; + ] , [ + :f :g + ] . + +[] + :list ( + [ a :Apple ] + [ a :Banana ] + ) . \ No newline at end of file diff --git a/testsuite/serd-tests/good/test-rel.nt b/testsuite/serd-tests/good/test-rel.nt new file mode 100644 index 00000000..343ce19c --- /dev/null +++ b/testsuite/serd-tests/good/test-rel.nt @@ -0,0 +1,6 @@ + . + . + . + . + . + . diff --git a/testsuite/serd-tests/good/test-rel.ttl b/testsuite/serd-tests/good/test-rel.ttl new file mode 100644 index 00000000..4752be70 --- /dev/null +++ b/testsuite/serd-tests/good/test-rel.ttl @@ -0,0 +1,6 @@ + a . + a . + a . + a . + a . + a . diff --git a/testsuite/serd-tests/good/test-semi-dot.nt b/testsuite/serd-tests/good/test-semi-dot.nt new file mode 100644 index 00000000..aea1655b --- /dev/null +++ b/testsuite/serd-tests/good/test-semi-dot.nt @@ -0,0 +1 @@ + . diff --git a/testsuite/serd-tests/good/test-semi-dot.ttl b/testsuite/serd-tests/good/test-semi-dot.ttl new file mode 100644 index 00000000..6d4b4146 --- /dev/null +++ b/testsuite/serd-tests/good/test-semi-dot.ttl @@ -0,0 +1 @@ + a ; . \ No newline at end of file diff --git a/testsuite/serd-tests/good/test-several-eaten-dots.nq b/testsuite/serd-tests/good/test-several-eaten-dots.nq new file mode 100644 index 00000000..5cafa59f --- /dev/null +++ b/testsuite/serd-tests/good/test-several-eaten-dots.nq @@ -0,0 +1,3 @@ + . + . + . diff --git a/testsuite/serd-tests/good/test-several-eaten-dots.trig b/testsuite/serd-tests/good/test-several-eaten-dots.trig new file mode 100644 index 00000000..bab135ed --- /dev/null +++ b/testsuite/serd-tests/good/test-several-eaten-dots.trig @@ -0,0 +1,6 @@ +prefix : + +:g { + :s :p1 :o1. + :s :p1 :o2; :p2 :o3. +} \ No newline at end of file diff --git a/testsuite/serd-tests/good/test-uri-escape.nt b/testsuite/serd-tests/good/test-uri-escape.nt new file mode 100644 index 00000000..320e7c33 --- /dev/null +++ b/testsuite/serd-tests/good/test-uri-escape.nt @@ -0,0 +1 @@ + . diff --git a/testsuite/serd-tests/good/test-uri-escape.ttl b/testsuite/serd-tests/good/test-uri-escape.ttl new file mode 100644 index 00000000..320e7c33 --- /dev/null +++ b/testsuite/serd-tests/good/test-uri-escape.ttl @@ -0,0 +1 @@ + . diff --git a/testsuite/serd-tests/good/test-uri.nt b/testsuite/serd-tests/good/test-uri.nt new file mode 100644 index 00000000..1744963c --- /dev/null +++ b/testsuite/serd-tests/good/test-uri.nt @@ -0,0 +1,48 @@ + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . diff --git a/testsuite/serd-tests/good/test-uri.ttl b/testsuite/serd-tests/good/test-uri.ttl new file mode 100644 index 00000000..b6a8d967 --- /dev/null +++ b/testsuite/serd-tests/good/test-uri.ttl @@ -0,0 +1,71 @@ +# http://tools.ietf.org/html/rfc3986#section-5.4 + +@base . + +@prefix owl: . + +# 5.4.1. Normal Examples + owl:sameAs . + owl:sameAs . +<./g> owl:sameAs . + owl:sameAs . + owl:sameAs . + owl:sameAs . + owl:sameAs . + owl:sameAs . +<#s> owl:sameAs . + owl:sameAs . + owl:sameAs . +<;x> owl:sameAs . + owl:sameAs . + owl:sameAs . +<> owl:sameAs . +<.> owl:sameAs . +<./> owl:sameAs . +<..> owl:sameAs . +<../> owl:sameAs . +<../g> owl:sameAs . +<../..> owl:sameAs . +<../../> owl:sameAs . +<../../g> owl:sameAs . + +# 5.4.2. Abnormal Examples +<../../../g> owl:sameAs . +<../../../../g> owl:sameAs . + owl:sameAs . + owl:sameAs . + owl:sameAs . +<.g> owl:sameAs . + owl:sameAs . +<..g> owl:sameAs . +<./../g> owl:sameAs . +# Serd only resolves the leading components +#<./g/.> owl:sameAs . +# owl:sameAs . +# owl:sameAs . +# owl:sameAs . +# owl:sameAs . + owl:sameAs . + owl:sameAs . + owl:sameAs . + owl:sameAs . + +# Additional tests for Serd + owl:sameAs . + owl:sameAs . + owl:sameAs . + owl:sameAs . + owl:sameAs . +<#afragment> owl:sameAs . +<../../../../../../> owl:sameAs . + owl:sameAs . + owl:sameAs . + owl:sameAs . + +@base . + + owl:sameAs . + +@base . + + owl:sameAs . diff --git a/testsuite/serd-tests/good/test-utf8-uri.nt b/testsuite/serd-tests/good/test-utf8-uri.nt new file mode 100644 index 00000000..b8a73a88 --- /dev/null +++ b/testsuite/serd-tests/good/test-utf8-uri.nt @@ -0,0 +1 @@ + . diff --git a/testsuite/serd-tests/good/test-utf8-uri.ttl b/testsuite/serd-tests/good/test-utf8-uri.ttl new file mode 100644 index 00000000..51f26ff2 --- /dev/null +++ b/testsuite/serd-tests/good/test-utf8-uri.ttl @@ -0,0 +1 @@ + a . \ No newline at end of file diff --git a/testsuite/src/files.rs b/testsuite/src/files.rs index 0d744b1f..59653814 100644 --- a/testsuite/src/files.rs +++ b/testsuite/src/files.rs @@ -1,6 +1,8 @@ use anyhow::{anyhow, bail, Result}; use oxigraph::io::{DatasetFormat, DatasetParser, GraphFormat, GraphParser}; use oxigraph::model::{Dataset, Graph}; +use oxttl::n3::N3Quad; +use oxttl::N3Parser; use std::fs::File; use std::io::{BufRead, BufReader, Read}; use std::path::PathBuf; @@ -18,6 +20,8 @@ pub fn read_file(url: &str) -> Result { "https://github.com/oxigraph/oxigraph/tests/", "oxigraph-tests/", ) + } else if url.starts_with("http://drobilla.net/sw/serd/test/") { + url.replace("http://drobilla.net/sw/serd/test/", "serd-tests/") } else { bail!("Not supported url for file: {url}") }); @@ -30,17 +34,33 @@ pub fn read_file_to_string(url: &str) -> Result { Ok(buf) } -pub fn load_to_graph(url: &str, graph: &mut Graph, format: GraphFormat) -> Result<()> { +pub fn load_to_graph( + url: &str, + graph: &mut Graph, + format: GraphFormat, + ignore_errors: bool, +) -> Result<()> { let parser = GraphParser::from_format(format).with_base_iri(url)?; - for t in parser.read_triples(read_file(url)?)? { - graph.insert(&t?); + for t in parser.read_triples(read_file(url)?) { + match t { + Ok(t) => { + graph.insert(&t); + } + Err(e) => { + if ignore_errors { + continue; + } else { + return Err(e.into()); + } + } + } } Ok(()) } -pub fn load_graph(url: &str, format: GraphFormat) -> Result { +pub fn load_graph(url: &str, format: GraphFormat, ignore_errors: bool) -> Result { let mut graph = Graph::new(); - load_to_graph(url, &mut graph, format)?; + load_to_graph(url, &mut graph, format, ignore_errors)?; Ok(graph) } @@ -50,17 +70,33 @@ pub fn guess_graph_format(url: &str) -> Result { .ok_or_else(|| anyhow!("Serialization type not found for {url}")) } -pub fn load_to_dataset(url: &str, dataset: &mut Dataset, format: DatasetFormat) -> Result<()> { +pub fn load_to_dataset( + url: &str, + dataset: &mut Dataset, + format: DatasetFormat, + ignore_errors: bool, +) -> Result<()> { let parser = DatasetParser::from_format(format).with_base_iri(url)?; - for q in parser.read_quads(read_file(url)?)? { - dataset.insert(&q?); + for q in parser.read_quads(read_file(url)?) { + match q { + Ok(q) => { + dataset.insert(&q); + } + Err(e) => { + if ignore_errors { + continue; + } else { + return Err(e.into()); + } + } + } } Ok(()) } -pub fn load_dataset(url: &str, format: DatasetFormat) -> Result { +pub fn load_dataset(url: &str, format: DatasetFormat, ignore_errors: bool) -> Result { let mut dataset = Dataset::new(); - load_to_dataset(url, &mut dataset, format)?; + load_to_dataset(url, &mut dataset, format, ignore_errors)?; Ok(dataset) } @@ -69,3 +105,24 @@ pub fn guess_dataset_format(url: &str) -> Result { .and_then(|(_, extension)| DatasetFormat::from_extension(extension)) .ok_or_else(|| anyhow!("Serialization type not found for {url}")) } + +pub fn load_n3(url: &str, ignore_errors: bool) -> Result> { + let mut quads = Vec::new(); + for q in N3Parser::new() + .with_base_iri(url)? + .with_prefix("", format!("{url}#"))? + .parse_from_read(read_file(url)?) + { + match q { + Ok(q) => quads.push(q), + Err(e) => { + if ignore_errors { + continue; + } else { + return Err(e.into()); + } + } + } + } + Ok(quads) +} diff --git a/testsuite/src/lib.rs b/testsuite/src/lib.rs index 3ebc2669..28bdada1 100644 --- a/testsuite/src/lib.rs +++ b/testsuite/src/lib.rs @@ -7,3 +7,35 @@ pub mod parser_evaluator; pub mod report; pub mod sparql_evaluator; mod vocab; + +use crate::evaluator::TestEvaluator; +use crate::manifest::TestManifest; +use crate::parser_evaluator::register_parser_tests; +use crate::sparql_evaluator::register_sparql_tests; +use anyhow::Result; + +pub fn check_testsuite(manifest_url: &str, ignored_tests: &[&str]) -> Result<()> { + let mut evaluator = TestEvaluator::default(); + register_parser_tests(&mut evaluator); + register_sparql_tests(&mut evaluator); + + let manifest = TestManifest::new([manifest_url]); + let results = evaluator.evaluate(manifest)?; + + let mut errors = Vec::default(); + for result in results { + if let Err(error) = &result.outcome { + if !ignored_tests.contains(&result.test.as_str()) { + errors.push(format!("{}: failed with error {}", result.test, error)) + } + } + } + + assert!( + errors.is_empty(), + "{} failing tests:\n{}\n", + errors.len(), + errors.join("\n") + ); + Ok(()) +} diff --git a/testsuite/src/manifest.rs b/testsuite/src/manifest.rs index 1b93f244..7563b5c3 100644 --- a/testsuite/src/manifest.rs +++ b/testsuite/src/manifest.rs @@ -279,7 +279,7 @@ impl TestManifest { return Ok(None); }; self.graph.clear(); - load_to_graph(&url, &mut self.graph, guess_graph_format(&url)?)?; + load_to_graph(&url, &mut self.graph, guess_graph_format(&url)?, false)?; let manifests = self .graph diff --git a/testsuite/src/parser_evaluator.rs b/testsuite/src/parser_evaluator.rs index 3ea3f91b..8e1a818c 100644 --- a/testsuite/src/parser_evaluator.rs +++ b/testsuite/src/parser_evaluator.rs @@ -1,9 +1,11 @@ use crate::evaluator::TestEvaluator; -use crate::files::{guess_dataset_format, guess_graph_format, load_dataset, load_graph}; +use crate::files::{guess_dataset_format, guess_graph_format, load_dataset, load_graph, load_n3}; use crate::manifest::Test; use crate::report::{dataset_diff, graph_diff}; use anyhow::{anyhow, bail, Result}; use oxigraph::io::{DatasetFormat, GraphFormat}; +use oxigraph::model::{BlankNode, Dataset, Quad}; +use oxttl::n3::{N3Quad, N3Term}; pub fn register_parser_tests(evaluator: &mut TestEvaluator) { evaluator.register( @@ -21,6 +23,10 @@ pub fn register_parser_tests(evaluator: &mut TestEvaluator) { evaluator.register("http://www.w3.org/ns/rdftest#TestTrigPositiveSyntax", |t| { evaluate_positive_dataset_syntax_test(t, DatasetFormat::TriG) }); + evaluator.register( + "https://w3c.github.io/N3/tests/test.n3#TestN3PositiveSyntax", + evaluate_positive_n3_syntax_test, + ); evaluator.register( "http://www.w3.org/ns/rdftest#TestNTriplesNegativeSyntax", |t| evaluate_negative_graph_syntax_test(t, GraphFormat::NTriples), @@ -39,14 +45,21 @@ pub fn register_parser_tests(evaluator: &mut TestEvaluator) { evaluator.register("http://www.w3.org/ns/rdftest#TestXMLNegativeSyntax", |t| { evaluate_negative_graph_syntax_test(t, GraphFormat::RdfXml) }); + evaluator.register( + "https://w3c.github.io/N3/tests/test.n3#TestN3NegativeSyntax", + evaluate_negative_n3_syntax_test, + ); evaluator.register("http://www.w3.org/ns/rdftest#TestTurtleEval", |t| { - evaluate_graph_eval_test(t, GraphFormat::Turtle) + evaluate_graph_eval_test(t, GraphFormat::Turtle, false) }); evaluator.register("http://www.w3.org/ns/rdftest#TestTrigEval", |t| { - evaluate_dataset_eval_test(t, DatasetFormat::TriG) + evaluate_dataset_eval_test(t, DatasetFormat::TriG, false) }); evaluator.register("http://www.w3.org/ns/rdftest#TestXMLEval", |t| { - evaluate_graph_eval_test(t, GraphFormat::RdfXml) + evaluate_graph_eval_test(t, GraphFormat::RdfXml, false) + }); + evaluator.register("https://w3c.github.io/N3/tests/test.n3#TestN3Eval", |t| { + evaluate_n3_eval_test(t, false) }); evaluator.register("http://www.w3.org/ns/rdftest#TestTurtleNegativeEval", |t| { evaluate_negative_graph_syntax_test(t, GraphFormat::Turtle) @@ -54,6 +67,18 @@ pub fn register_parser_tests(evaluator: &mut TestEvaluator) { evaluator.register("http://www.w3.org/ns/rdftest#TestTrigNegativeEval", |t| { evaluate_negative_dataset_syntax_test(t, DatasetFormat::TriG) }); + evaluator.register( + "https://github.com/oxigraph/oxigraph/tests#TestNTripleRecovery", + |t| evaluate_graph_eval_test(t, GraphFormat::NTriples, true), + ); + evaluator.register( + "https://github.com/oxigraph/oxigraph/tests#TestTurtleRecovery", + |t| evaluate_graph_eval_test(t, GraphFormat::Turtle, true), + ); + evaluator.register( + "https://github.com/oxigraph/oxigraph/tests#TestN3Recovery", + |t| evaluate_n3_eval_test(t, true), + ); } fn evaluate_positive_graph_syntax_test(test: &Test, format: GraphFormat) -> Result<()> { @@ -61,7 +86,7 @@ fn evaluate_positive_graph_syntax_test(test: &Test, format: GraphFormat) -> Resu .action .as_deref() .ok_or_else(|| anyhow!("No action found for test {test}"))?; - load_graph(action, format).map_err(|e| anyhow!("Parse error: {e}"))?; + load_graph(action, format, false).map_err(|e| anyhow!("Parse error: {e}"))?; Ok(()) } @@ -70,7 +95,16 @@ fn evaluate_positive_dataset_syntax_test(test: &Test, format: DatasetFormat) -> .action .as_deref() .ok_or_else(|| anyhow!("No action found for test {test}"))?; - load_dataset(action, format).map_err(|e| anyhow!("Parse error: {e}"))?; + load_dataset(action, format, false).map_err(|e| anyhow!("Parse error: {e}"))?; + Ok(()) +} + +fn evaluate_positive_n3_syntax_test(test: &Test) -> Result<()> { + let action = test + .action + .as_deref() + .ok_or_else(|| anyhow!("No action found for test {test}"))?; + load_n3(action, false).map_err(|e| anyhow!("Parse error: {e}"))?; Ok(()) } @@ -79,7 +113,7 @@ fn evaluate_negative_graph_syntax_test(test: &Test, format: GraphFormat) -> Resu .action .as_deref() .ok_or_else(|| anyhow!("No action found for test {test}"))?; - match load_graph(action, format) { + match load_graph(action, format, false) { Ok(_) => bail!("File parsed without errors even if it should not"), Err(_) => Ok(()), } @@ -90,25 +124,36 @@ fn evaluate_negative_dataset_syntax_test(test: &Test, format: DatasetFormat) -> .action .as_deref() .ok_or_else(|| anyhow!("No action found for test {test}"))?; - match load_dataset(action, format) { + match load_dataset(action, format, false) { Ok(_) => bail!("File parsed without errors even if it should not"), Err(_) => Ok(()), } } -fn evaluate_graph_eval_test(test: &Test, format: GraphFormat) -> Result<()> { +fn evaluate_negative_n3_syntax_test(test: &Test) -> Result<()> { let action = test .action .as_deref() .ok_or_else(|| anyhow!("No action found for test {test}"))?; - let mut actual_graph = - load_graph(action, format).map_err(|e| anyhow!("Parse error on file {action}: {e}"))?; + match load_n3(action, false) { + Ok(_) => bail!("File parsed without errors even if it should not"), + Err(_) => Ok(()), + } +} + +fn evaluate_graph_eval_test(test: &Test, format: GraphFormat, ignore_errors: bool) -> Result<()> { + let action = test + .action + .as_deref() + .ok_or_else(|| anyhow!("No action found for test {test}"))?; + let mut actual_graph = load_graph(action, format, ignore_errors) + .map_err(|e| anyhow!("Parse error on file {action}: {e}"))?; actual_graph.canonicalize(); let results = test .result .as_ref() .ok_or_else(|| anyhow!("No tests result found"))?; - let mut expected_graph = load_graph(results, guess_graph_format(results)?) + let mut expected_graph = load_graph(results, guess_graph_format(results)?, false) .map_err(|e| anyhow!("Parse error on file {results}: {e}"))?; expected_graph.canonicalize(); if expected_graph == actual_graph { @@ -121,19 +166,23 @@ fn evaluate_graph_eval_test(test: &Test, format: GraphFormat) -> Result<()> { } } -fn evaluate_dataset_eval_test(test: &Test, format: DatasetFormat) -> Result<()> { +fn evaluate_dataset_eval_test( + test: &Test, + format: DatasetFormat, + ignore_errors: bool, +) -> Result<()> { let action = test .action .as_deref() .ok_or_else(|| anyhow!("No action found for test {test}"))?; - let mut actual_dataset = - load_dataset(action, format).map_err(|e| anyhow!("Parse error on file {action}: {e}"))?; + let mut actual_dataset = load_dataset(action, format, ignore_errors) + .map_err(|e| anyhow!("Parse error on file {action}: {e}"))?; actual_dataset.canonicalize(); let results = test .result .as_ref() .ok_or_else(|| anyhow!("No tests result found"))?; - let mut expected_dataset = load_dataset(results, guess_dataset_format(results)?) + let mut expected_dataset = load_dataset(results, guess_dataset_format(results)?, false) .map_err(|e| anyhow!("Parse error on file {results}: {e}"))?; expected_dataset.canonicalize(); if expected_dataset == actual_dataset { @@ -145,3 +194,59 @@ fn evaluate_dataset_eval_test(test: &Test, format: DatasetFormat) -> Result<()> ) } } + +fn evaluate_n3_eval_test(test: &Test, ignore_errors: bool) -> Result<()> { + let action = test + .action + .as_deref() + .ok_or_else(|| anyhow!("No action found for test {test}"))?; + let mut actual_dataset = n3_to_dataset( + load_n3(action, ignore_errors).map_err(|e| anyhow!("Parse error on file {action}: {e}"))?, + ); + actual_dataset.canonicalize(); + let results = test + .result + .as_ref() + .ok_or_else(|| anyhow!("No tests result found"))?; + let mut expected_dataset = n3_to_dataset( + load_n3(results, false).map_err(|e| anyhow!("Parse error on file {results}: {e}"))?, + ); + expected_dataset.canonicalize(); + if expected_dataset == actual_dataset { + Ok(()) + } else { + bail!( + "The two files are not isomorphic. Diff:\n{}", + dataset_diff(&expected_dataset, &actual_dataset) + ) + } +} + +fn n3_to_dataset(quads: Vec) -> Dataset { + quads + .into_iter() + .filter_map(|q| { + Some(Quad { + subject: match q.subject { + N3Term::NamedNode(n) => n.into(), + N3Term::BlankNode(n) => n.into(), + N3Term::Triple(n) => n.into(), + N3Term::Literal(_) => return None, + N3Term::Variable(v) => BlankNode::new_unchecked(v.into_string()).into(), + }, + predicate: match q.predicate { + N3Term::NamedNode(n) => n, + _ => return None, + }, + object: match q.object { + N3Term::NamedNode(n) => n.into(), + N3Term::BlankNode(n) => n.into(), + N3Term::Triple(n) => n.into(), + N3Term::Literal(n) => n.into(), + N3Term::Variable(v) => BlankNode::new_unchecked(v.into_string()).into(), + }, + graph_name: q.graph_name, + }) + }) + .collect() +} diff --git a/testsuite/src/sparql_evaluator.rs b/testsuite/src/sparql_evaluator.rs index dc68dff4..e72feaa7 100644 --- a/testsuite/src/sparql_evaluator.rs +++ b/testsuite/src/sparql_evaluator.rs @@ -298,7 +298,7 @@ fn load_sparql_query_result(url: &str) -> Result { { StaticQueryResults::from_query_results(QueryResults::read(read_file(url)?, format)?, false) } else { - StaticQueryResults::from_graph(&load_graph(url, guess_graph_format(url)?)?) + StaticQueryResults::from_graph(&load_graph(url, guess_graph_format(url)?, false)?) } } diff --git a/testsuite/tests/oxigraph.rs b/testsuite/tests/oxigraph.rs index a37bfab2..a5bc7e0f 100644 --- a/testsuite/tests/oxigraph.rs +++ b/testsuite/tests/oxigraph.rs @@ -1,41 +1,43 @@ use anyhow::Result; -use oxigraph_testsuite::evaluator::TestEvaluator; -use oxigraph_testsuite::manifest::TestManifest; -use oxigraph_testsuite::sparql_evaluator::register_sparql_tests; +use oxigraph_testsuite::check_testsuite; -fn run_testsuite(manifest_url: &str) -> Result<()> { - let mut evaluator = TestEvaluator::default(); - register_sparql_tests(&mut evaluator); - let manifest = TestManifest::new([manifest_url]); - let results = evaluator.evaluate(manifest)?; - - let mut errors = Vec::default(); - for result in results { - if let Err(error) = &result.outcome { - errors.push(format!("{}: failed with error {}", result.test, error)) - } - } +#[cfg(not(windows))] // Tests don't like git auto "\r\n" on Windows +#[test] +fn oxigraph_parser_testsuite() -> Result<()> { + check_testsuite( + "https://github.com/oxigraph/oxigraph/tests/parser/manifest.ttl", + &[], + ) +} - assert!( - errors.is_empty(), - "{} failing tests:\n{}\n", - errors.len(), - errors.join("\n") - ); - Ok(()) +#[test] +fn oxigraph_parser_recovery_testsuite() -> Result<()> { + check_testsuite( + "https://github.com/oxigraph/oxigraph/tests/parser-recovery/manifest.ttl", + &[], + ) } #[test] fn oxigraph_sparql_testsuite() -> Result<()> { - run_testsuite("https://github.com/oxigraph/oxigraph/tests/sparql/manifest.ttl") + check_testsuite( + "https://github.com/oxigraph/oxigraph/tests/sparql/manifest.ttl", + &[], + ) } #[test] fn oxigraph_sparql_results_testsuite() -> Result<()> { - run_testsuite("https://github.com/oxigraph/oxigraph/tests/sparql-results/manifest.ttl") + check_testsuite( + "https://github.com/oxigraph/oxigraph/tests/sparql-results/manifest.ttl", + &[], + ) } #[test] fn oxigraph_optimizer_testsuite() -> Result<()> { - run_testsuite("https://github.com/oxigraph/oxigraph/tests/sparql-optimization/manifest.ttl") + check_testsuite( + "https://github.com/oxigraph/oxigraph/tests/sparql-optimization/manifest.ttl", + &[], + ) } diff --git a/testsuite/tests/parser.rs b/testsuite/tests/parser.rs index 5a28e37f..e9458256 100644 --- a/testsuite/tests/parser.rs +++ b/testsuite/tests/parser.rs @@ -1,78 +1,93 @@ use anyhow::Result; -use oxigraph_testsuite::evaluator::TestEvaluator; -use oxigraph_testsuite::manifest::TestManifest; -use oxigraph_testsuite::parser_evaluator::register_parser_tests; - -fn run_testsuite(manifest_url: &str) -> Result<()> { - let mut evaluator = TestEvaluator::default(); - register_parser_tests(&mut evaluator); - let manifest = TestManifest::new(vec![manifest_url]); - let results = evaluator.evaluate(manifest)?; - - let mut errors = Vec::default(); - for result in results { - if let Err(error) = &result.outcome { - errors.push(format!("{}: failed with error {}", result.test, error)) - } - } - - assert!( - errors.is_empty(), - "{} failing tests:\n{}\n", - errors.len(), - errors.join("\n") - ); - Ok(()) -} +use oxigraph_testsuite::check_testsuite; #[test] fn ntriples_w3c_testsuite() -> Result<()> { - run_testsuite("http://w3c.github.io/rdf-tests/ntriples/manifest.ttl") + check_testsuite("http://w3c.github.io/rdf-tests/ntriples/manifest.ttl", &[]) } #[test] fn nquads_w3c_testsuite() -> Result<()> { - run_testsuite("http://w3c.github.io/rdf-tests/nquads/manifest.ttl") + check_testsuite("http://w3c.github.io/rdf-tests/nquads/manifest.ttl", &[]) } -#[cfg(not(target_os = "windows"))] // Tests don't like git auto "\r\n" on Windows +#[cfg(not(windows))] // Tests don't like git auto "\r\n" on Windows #[test] fn turtle_w3c_testsuite() -> Result<()> { - run_testsuite("http://w3c.github.io/rdf-tests/turtle/manifest.ttl") + check_testsuite("http://w3c.github.io/rdf-tests/turtle/manifest.ttl", &[]) } -#[cfg(not(target_os = "windows"))] // Tests don't like git auto "\r\n" on Windows +#[cfg(not(windows))] // Tests don't like git auto "\r\n" on Windows #[test] fn trig_w3c_testsuite() -> Result<()> { - run_testsuite("http://w3c.github.io/rdf-tests/trig/manifest.ttl") + check_testsuite("http://w3c.github.io/rdf-tests/trig/manifest.ttl", &[]) +} + +#[test] +fn n3_parser_testsuite() -> Result<()> { + check_testsuite( + "https://w3c.github.io/N3/tests/N3Tests/manifest-parser.ttl", + &[], + ) +} +#[test] +fn n3_extended_testsuite() -> Result<()> { + check_testsuite( + "https://w3c.github.io/N3/tests/N3Tests/manifest-extended.ttl", + &[], + ) +} + +#[cfg(not(windows))] // Tests don't like git auto "\r\n" on Windows +#[test] +fn n3_turtle_testsuite() -> Result<()> { + check_testsuite( + "https://w3c.github.io/N3/tests/TurtleTests/manifest.ttl", + &[], + ) } #[test] fn rdf_xml_w3c_testsuite() -> Result<()> { - run_testsuite("http://www.w3.org/2013/RDFXMLTests/manifest.ttl") + check_testsuite("http://www.w3.org/2013/RDFXMLTests/manifest.ttl", &[]) } #[test] fn ntriples_star_w3c_testsuite() -> Result<()> { - run_testsuite("https://w3c.github.io/rdf-star/tests/nt/syntax/manifest.ttl") + check_testsuite( + "https://w3c.github.io/rdf-star/tests/nt/syntax/manifest.ttl", + &[], + ) } #[test] fn turtle_star_syntax_w3c_testsuite() -> Result<()> { - run_testsuite("https://w3c.github.io/rdf-star/tests/turtle/syntax/manifest.ttl") + check_testsuite( + "https://w3c.github.io/rdf-star/tests/turtle/syntax/manifest.ttl", + &[], + ) } #[test] fn turtle_star_eval_w3c_testsuite() -> Result<()> { - run_testsuite("https://w3c.github.io/rdf-star/tests/turtle/eval/manifest.ttl") + check_testsuite( + "https://w3c.github.io/rdf-star/tests/turtle/eval/manifest.ttl", + &[], + ) } #[test] fn trig_star_syntax_w3c_testsuite() -> Result<()> { - run_testsuite("https://w3c.github.io/rdf-star/tests/trig/syntax/manifest.ttl") + check_testsuite( + "https://w3c.github.io/rdf-star/tests/trig/syntax/manifest.ttl", + &[], + ) } #[test] fn trig_star_eval_w3c_testsuite() -> Result<()> { - run_testsuite("https://w3c.github.io/rdf-star/tests/trig/eval/manifest.ttl") + check_testsuite( + "https://w3c.github.io/rdf-star/tests/trig/eval/manifest.ttl", + &[], + ) } diff --git a/testsuite/tests/serd.rs b/testsuite/tests/serd.rs new file mode 100644 index 00000000..ae40d752 --- /dev/null +++ b/testsuite/tests/serd.rs @@ -0,0 +1,13 @@ +use anyhow::Result; +use oxigraph_testsuite::check_testsuite; + +#[cfg(not(windows))] // Tests don't like git auto "\r\n" on Windows +#[test] +fn serd_good_testsuite() -> Result<()> { + check_testsuite("http://drobilla.net/sw/serd/test/good/manifest.ttl", &[]) +} + +#[test] +fn serd_bad_testsuite() -> Result<()> { + check_testsuite("http://drobilla.net/sw/serd/test/bad/manifest.ttl", &[]) +} diff --git a/testsuite/tests/sparql.rs b/testsuite/tests/sparql.rs index d058bd71..74e8c437 100644 --- a/testsuite/tests/sparql.rs +++ b/testsuite/tests/sparql.rs @@ -1,35 +1,9 @@ use anyhow::Result; -use oxigraph_testsuite::evaluator::TestEvaluator; -use oxigraph_testsuite::manifest::TestManifest; -use oxigraph_testsuite::sparql_evaluator::register_sparql_tests; - -fn run_testsuite(manifest_url: &str, ignored_tests: &[&str]) -> Result<()> { - let mut evaluator = TestEvaluator::default(); - register_sparql_tests(&mut evaluator); - let manifest = TestManifest::new(vec![manifest_url]); - let results = evaluator.evaluate(manifest)?; - - let mut errors = Vec::default(); - for result in results { - if let Err(error) = &result.outcome { - if !ignored_tests.contains(&result.test.as_str()) { - errors.push(format!("{}: failed with error {}", result.test, error)) - } - } - } - - assert!( - errors.is_empty(), - "{} failing tests:\n{}\n", - errors.len(), - errors.join("\n") - ); - Ok(()) -} +use oxigraph_testsuite::check_testsuite; #[test] fn sparql10_w3c_query_syntax_testsuite() -> Result<()> { - run_testsuite( + check_testsuite( "https://w3c.github.io/rdf-tests/sparql/sparql10/manifest-syntax.ttl", &[ "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/syntax-sparql3/manifest#syn-bad-26", // tokenizer @@ -39,7 +13,7 @@ fn sparql10_w3c_query_syntax_testsuite() -> Result<()> { #[test] fn sparql10_w3c_query_evaluation_testsuite() -> Result<()> { - run_testsuite("https://w3c.github.io/rdf-tests/sparql/sparql10/manifest-evaluation.ttl", &[ + check_testsuite("https://w3c.github.io/rdf-tests/sparql/sparql10/manifest-evaluation.ttl", &[ //Multiple writing of the same xsd:integer. Our system does strong normalization. "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/distinct/manifest#distinct-1", "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/distinct/manifest#distinct-9", @@ -74,7 +48,7 @@ fn sparql10_w3c_query_evaluation_testsuite() -> Result<()> { #[test] fn sparql11_query_w3c_evaluation_testsuite() -> Result<()> { - run_testsuite( + check_testsuite( "https://w3c.github.io/rdf-tests/sparql/sparql11/manifest-sparql11-query.ttl", &[ //BNODE() scope is currently wrong @@ -87,7 +61,7 @@ fn sparql11_query_w3c_evaluation_testsuite() -> Result<()> { #[test] fn sparql11_federation_w3c_evaluation_testsuite() -> Result<()> { - run_testsuite( + check_testsuite( "https://w3c.github.io/rdf-tests/sparql/sparql11/manifest-sparql11-fed.ttl", &[ // Problem during service evaluation order @@ -98,7 +72,7 @@ fn sparql11_federation_w3c_evaluation_testsuite() -> Result<()> { #[test] fn sparql11_update_w3c_evaluation_testsuite() -> Result<()> { - run_testsuite( + check_testsuite( "https://w3c.github.io/rdf-tests/sparql/sparql11/manifest-sparql11-update.ttl", &[ // We allow multiple INSERT DATA with the same blank nodes @@ -109,7 +83,7 @@ fn sparql11_update_w3c_evaluation_testsuite() -> Result<()> { #[test] fn sparql11_json_w3c_evaluation_testsuite() -> Result<()> { - run_testsuite( + check_testsuite( "https://w3c.github.io/rdf-tests/sparql/sparql11/json-res/manifest.ttl", &[], ) @@ -117,7 +91,7 @@ fn sparql11_json_w3c_evaluation_testsuite() -> Result<()> { #[test] fn sparql11_tsv_w3c_evaluation_testsuite() -> Result<()> { - run_testsuite( + check_testsuite( "https://w3c.github.io/rdf-tests/sparql/sparql11/csv-tsv-res/manifest.ttl", &[ // We do not run CSVResultFormatTest tests yet @@ -130,7 +104,7 @@ fn sparql11_tsv_w3c_evaluation_testsuite() -> Result<()> { #[test] fn sparql_star_syntax_testsuite() -> Result<()> { - run_testsuite( + check_testsuite( "https://w3c.github.io/rdf-star/tests/sparql/syntax/manifest.ttl", &[], ) @@ -138,7 +112,7 @@ fn sparql_star_syntax_testsuite() -> Result<()> { #[test] fn sparql_star_eval_testsuite() -> Result<()> { - run_testsuite( + check_testsuite( "https://w3c.github.io/rdf-star/tests/sparql/eval/manifest.ttl", &[], ) From 43e6ce87f87b0ae6eed64dcc00ae9fcd01bdb3c8 Mon Sep 17 00:00:00 2001 From: Dan Yamamoto Date: Mon, 24 Apr 2023 10:59:40 +0000 Subject: [PATCH 017/217] OxRDF: Add extra literal escaping to generate canonical N-Triples and N-Quads --- lib/oxrdf/src/literal.rs | 72 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/lib/oxrdf/src/literal.rs b/lib/oxrdf/src/literal.rs index 6891ba15..159a507c 100644 --- a/lib/oxrdf/src/literal.rs +++ b/lib/oxrdf/src/literal.rs @@ -620,10 +620,41 @@ pub fn print_quoted_str(string: &str, f: &mut impl Write) -> fmt::Result { f.write_char('"')?; for c in string.chars() { match c { + '\0' => f.write_str("\\u0000"), + '\u{01}' => f.write_str("\\u0001"), + '\u{02}' => f.write_str("\\u0002"), + '\u{03}' => f.write_str("\\u0003"), + '\u{04}' => f.write_str("\\u0004"), + '\u{05}' => f.write_str("\\u0005"), + '\u{06}' => f.write_str("\\u0006"), + '\u{07}' => f.write_str("\\u0007"), + '\u{08}' => f.write_str("\\b"), + '\t' => f.write_str("\\t"), '\n' => f.write_str("\\n"), + '\u{0b}' => f.write_str("\\u000B"), + '\u{0c}' => f.write_str("\\f"), '\r' => f.write_str("\\r"), + '\u{0e}' => f.write_str("\\u000E"), + '\u{0f}' => f.write_str("\\u000F"), + '\u{10}' => f.write_str("\\u0010"), + '\u{11}' => f.write_str("\\u0011"), + '\u{12}' => f.write_str("\\u0012"), + '\u{13}' => f.write_str("\\u0013"), + '\u{14}' => f.write_str("\\u0014"), + '\u{15}' => f.write_str("\\u0015"), + '\u{16}' => f.write_str("\\u0016"), + '\u{17}' => f.write_str("\\u0017"), + '\u{18}' => f.write_str("\\u0018"), + '\u{19}' => f.write_str("\\u0019"), + '\u{1a}' => f.write_str("\\u001A"), + '\u{1b}' => f.write_str("\\u001B"), + '\u{1c}' => f.write_str("\\u001C"), + '\u{1d}' => f.write_str("\\u001D"), + '\u{1e}' => f.write_str("\\u001E"), + '\u{1f}' => f.write_str("\\u001F"), '"' => f.write_str("\\\""), '\\' => f.write_str("\\\\"), + '\u{7f}' => f.write_str("\\u007F"), c => f.write_char(c), }?; } @@ -633,6 +664,7 @@ pub fn print_quoted_str(string: &str, f: &mut impl Write) -> fmt::Result { #[cfg(test)] mod tests { use super::*; + use std::str::FromStr; #[test] fn test_simple_literal_equality() { @@ -663,4 +695,44 @@ mod tests { assert_eq!("NaN", Literal::from(f32::NAN).value()); assert_eq!("NaN", Literal::from(f64::NAN).value()); } + + #[test] + fn test_canoincal_escaping() { + assert_eq!( + Literal::from_str(r#""\u0000\u0001\u0002\u0003\u0004\u0005\u0006\u0007\u0008\u0009\u000a\u000b\u000c\u000d\u000e\u000f""#).unwrap().to_string(), + r###""\u0000\u0001\u0002\u0003\u0004\u0005\u0006\u0007\b\t\n\u000B\f\r\u000E\u000F""### + ); + assert_eq!( + Literal::from_str(r#""\u0010\u0011\u0012\u0013\u0014\u0015\u0016\u0017\u0018\u0019\u001a\u001b\u001c\u001d\u001e\u001f""#).unwrap().to_string(), + r###""\u0010\u0011\u0012\u0013\u0014\u0015\u0016\u0017\u0018\u0019\u001A\u001B\u001C\u001D\u001E\u001F""### + ); + assert_eq!( + Literal::from_str(r#""\u0020\u0021\u0022\u0023\u0024\u0025\u0026\u0027\u0028\u0029\u002a\u002b\u002c\u002d\u002e\u002f""#).unwrap().to_string(), + r###"" !\"#$%&'()*+,-./""### + ); + assert_eq!( + Literal::from_str(r#""\u0030\u0031\u0032\u0033\u0034\u0035\u0036\u0037\u0038\u0039\u003a\u003b\u003c\u003d\u003e\u003f""#).unwrap().to_string(), + r###""0123456789:;<=>?""### + ); + assert_eq!( + Literal::from_str(r#""\u0040\u0041\u0042\u0043\u0044\u0045\u0046\u0047\u0048\u0049\u004a\u004b\u004c\u004d\u004e\u004f""#).unwrap().to_string(), + r###""@ABCDEFGHIJKLMNO""### + ); + assert_eq!( + Literal::from_str(r#""\u0050\u0051\u0052\u0053\u0054\u0055\u0056\u0057\u0058\u0059\u005a\u005b\u005c\u005d\u005e\u005f""#).unwrap().to_string(), + r###""PQRSTUVWXYZ[\\]^_""### + ); + assert_eq!( + Literal::from_str(r#""\u0060\u0061\u0062\u0063\u0064\u0065\u0066\u0067\u0068\u0069\u006a\u006b\u006c\u006d\u006e\u006f""#).unwrap().to_string(), + r###""`abcdefghijklmno""### + ); + assert_eq!( + Literal::from_str(r#""\u0070\u0071\u0072\u0073\u0074\u0075\u0076\u0077\u0078\u0079\u007a\u007b\u007c\u007d\u007e\u007f""#).unwrap().to_string(), + r###""pqrstuvwxyz{|}~\u007F""### + ); + assert_eq!( + Literal::from_str(r#""\u0080\u0081\u0082\u0083\u0084\u0085\u0086\u0087\u0088\u0089\u008a\u008b\u008c\u008d\u008e\u008f""#).unwrap().to_string(), + "\"\u{80}\u{81}\u{82}\u{83}\u{84}\u{85}\u{86}\u{87}\u{88}\u{89}\u{8a}\u{8b}\u{8c}\u{8d}\u{8e}\u{8f}\"" + ); + } } From 5085a60a87f794aebc099a9ab5c4faa03f16af94 Mon Sep 17 00:00:00 2001 From: Dan Yamamoto Date: Thu, 27 Apr 2023 07:11:05 +0000 Subject: [PATCH 018/217] Apply sugestions from code review Co-authored-by: Tpt --- lib/oxrdf/src/literal.rs | 29 +---------------------------- 1 file changed, 1 insertion(+), 28 deletions(-) diff --git a/lib/oxrdf/src/literal.rs b/lib/oxrdf/src/literal.rs index 159a507c..607c3a69 100644 --- a/lib/oxrdf/src/literal.rs +++ b/lib/oxrdf/src/literal.rs @@ -620,41 +620,14 @@ pub fn print_quoted_str(string: &str, f: &mut impl Write) -> fmt::Result { f.write_char('"')?; for c in string.chars() { match c { - '\0' => f.write_str("\\u0000"), - '\u{01}' => f.write_str("\\u0001"), - '\u{02}' => f.write_str("\\u0002"), - '\u{03}' => f.write_str("\\u0003"), - '\u{04}' => f.write_str("\\u0004"), - '\u{05}' => f.write_str("\\u0005"), - '\u{06}' => f.write_str("\\u0006"), - '\u{07}' => f.write_str("\\u0007"), '\u{08}' => f.write_str("\\b"), '\t' => f.write_str("\\t"), '\n' => f.write_str("\\n"), - '\u{0b}' => f.write_str("\\u000B"), '\u{0c}' => f.write_str("\\f"), '\r' => f.write_str("\\r"), - '\u{0e}' => f.write_str("\\u000E"), - '\u{0f}' => f.write_str("\\u000F"), - '\u{10}' => f.write_str("\\u0010"), - '\u{11}' => f.write_str("\\u0011"), - '\u{12}' => f.write_str("\\u0012"), - '\u{13}' => f.write_str("\\u0013"), - '\u{14}' => f.write_str("\\u0014"), - '\u{15}' => f.write_str("\\u0015"), - '\u{16}' => f.write_str("\\u0016"), - '\u{17}' => f.write_str("\\u0017"), - '\u{18}' => f.write_str("\\u0018"), - '\u{19}' => f.write_str("\\u0019"), - '\u{1a}' => f.write_str("\\u001A"), - '\u{1b}' => f.write_str("\\u001B"), - '\u{1c}' => f.write_str("\\u001C"), - '\u{1d}' => f.write_str("\\u001D"), - '\u{1e}' => f.write_str("\\u001E"), - '\u{1f}' => f.write_str("\\u001F"), '"' => f.write_str("\\\""), '\\' => f.write_str("\\\\"), - '\u{7f}' => f.write_str("\\u007F"), + '\0'..='\u{1f}' | '\u{7f}' => write!(f, "\\u{:04X}", u32::from(c)), c => f.write_char(c), }?; } From cb9922379c9df23f92913dd248d9ea8b83efd62d Mon Sep 17 00:00:00 2001 From: Tpt Date: Thu, 15 Jun 2023 21:07:20 +0200 Subject: [PATCH 019/217] Uses "let else" syntax where relevant --- lib/oxttl/src/toolkit/parser.rs | 7 +++--- lib/oxttl/src/trig.rs | 6 ++--- lib/sparesults/src/csv.rs | 6 ++--- lib/src/storage/backend/fallback.rs | 39 +++++++++++++---------------- lib/src/storage/backend/rocksdb.rs | 32 +++++++++++------------ testsuite/src/manifest.rs | 8 ++---- 6 files changed, 41 insertions(+), 57 deletions(-) diff --git a/lib/oxttl/src/toolkit/parser.rs b/lib/oxttl/src/toolkit/parser.rs index 10965818..a02afec4 100644 --- a/lib/oxttl/src/toolkit/parser.rs +++ b/lib/oxttl/src/toolkit/parser.rs @@ -99,11 +99,10 @@ impl Parser { } } if self.lexer.is_end() { - if let Some(state) = self.state.take() { - state.recognize_end(&mut self.results, &mut self.errors) - } else { + let Some(state) = self.state.take() else { return None; - } + }; + state.recognize_end(&mut self.results, &mut self.errors) } else { return None; } diff --git a/lib/oxttl/src/trig.rs b/lib/oxttl/src/trig.rs index 692d6543..1860c56e 100644 --- a/lib/oxttl/src/trig.rs +++ b/lib/oxttl/src/trig.rs @@ -570,11 +570,9 @@ fn is_turtle_decimal(value: &str) -> bool { while value.first().map_or(false, u8::is_ascii_digit) { value = &value[1..]; } - if let Some(v) = value.strip_prefix(b".") { - value = v; - } else { + let Some(value) = value.strip_prefix(b".") else { return false; - } + }; !value.is_empty() && value.iter().all(u8::is_ascii_digit) } diff --git a/lib/sparesults/src/csv.rs b/lib/sparesults/src/csv.rs index 7d737e5b..759aea26 100644 --- a/lib/sparesults/src/csv.rs +++ b/lib/sparesults/src/csv.rs @@ -230,11 +230,9 @@ fn is_turtle_decimal(value: &str) -> bool { while value.first().map_or(false, u8::is_ascii_digit) { value = &value[1..]; } - if let Some(v) = value.strip_prefix(b".") { - value = v; - } else { + let Some(value) = value.strip_prefix(b".") else { return false; - } + }; !value.is_empty() && value.iter().all(u8::is_ascii_digit) } diff --git a/lib/src/storage/backend/fallback.rs b/lib/src/storage/backend/fallback.rs index 1c156111..6bdd5673 100644 --- a/lib/src/storage/backend/fallback.rs +++ b/lib/src/storage/backend/fallback.rs @@ -129,9 +129,7 @@ impl Reader { let data: Vec<_> = match &self.0 { InnerReader::Simple(reader) => { let trees = reader.read().unwrap(); - let tree = if let Some(tree) = trees.get(column_family) { - tree - } else { + let Some(tree) = trees.get(column_family) else { return Ok(Iter { iter: Vec::new().into_iter(), current: None, @@ -147,28 +145,25 @@ impl Reader { } } InnerReader::Transaction(reader) => { - if let Some(reader) = reader.upgrade() { - let trees = (*reader).borrow(); - let tree = if let Some(tree) = trees.get(column_family) { - tree - } else { - return Ok(Iter { - iter: Vec::new().into_iter(), - current: None, - }); - }; - if prefix.is_empty() { - tree.iter().map(|(k, v)| (k.clone(), v.clone())).collect() - } else { - tree.range(prefix.to_vec()..) - .take_while(|(k, _)| k.starts_with(prefix)) - .map(|(k, v)| (k.clone(), v.clone())) - .collect() - } - } else { + let Some(reader) = reader.upgrade() else { return Err(StorageError::Other( "The transaction is already ended".into(), )); + }; + let trees = (*reader).borrow(); + let Some(tree) = trees.get(column_family) else { + return Ok(Iter { + iter: Vec::new().into_iter(), + current: None, + }); + }; + if prefix.is_empty() { + tree.iter().map(|(k, v)| (k.clone(), v.clone())).collect() + } else { + tree.range(prefix.to_vec()..) + .take_while(|(k, _)| k.starts_with(prefix)) + .map(|(k, v)| (k.clone(), v.clone())) + .collect() } } }; diff --git a/lib/src/storage/backend/rocksdb.rs b/lib/src/storage/backend/rocksdb.rs index 81577551..0e9fe986 100644 --- a/lib/src/storage/backend/rocksdb.rs +++ b/lib/src/storage/backend/rocksdb.rs @@ -941,19 +941,18 @@ impl Reader { )) } InnerReader::Transaction(inner) => { - if let Some(inner) = inner.upgrade() { - ffi_result!(rocksdb_transaction_get_pinned_cf_with_status( - *inner, - self.options, - column_family.0, - key.as_ptr().cast(), - key.len() - )) - } else { - return Err(StorageError::Other( - "The transaction is already ended".into(), - )); - } + let Some(inner) = inner.upgrade() else { + return Err(StorageError::Other( + "The transaction is already ended".into(), + )); + }; + ffi_result!(rocksdb_transaction_get_pinned_cf_with_status( + *inner, + self.options, + column_family.0, + key.as_ptr().cast(), + key.len() + )) } InnerReader::PlainDb(inner) => { ffi_result!(rocksdb_get_pinned_cf_with_status( @@ -1022,13 +1021,12 @@ impl Reader { rocksdb_transactiondb_create_iterator_cf(inner.db.db, options, column_family.0) } InnerReader::Transaction(inner) => { - if let Some(inner) = inner.upgrade() { - rocksdb_transaction_create_iterator_cf(*inner, options, column_family.0) - } else { + let Some(inner) = inner.upgrade() else { return Err(StorageError::Other( "The transaction is already ended".into(), )); - } + }; + rocksdb_transaction_create_iterator_cf(*inner, options, column_family.0) } InnerReader::PlainDb(inner) => { rocksdb_create_iterator_cf(inner.db, options, column_family.0) diff --git a/testsuite/src/manifest.rs b/testsuite/src/manifest.rs index 7563b5c3..f854b95e 100644 --- a/testsuite/src/manifest.rs +++ b/testsuite/src/manifest.rs @@ -84,9 +84,7 @@ impl TestManifest { fn next_test(&mut self) -> Result> { loop { - let test_node = if let Some(test_node) = self.tests_to_do.pop_front() { - test_node - } else { + let Some(test_node) = self.tests_to_do.pop_front() else { return Ok(None); }; let test_node = if let Term::NamedNode(test_node) = test_node { @@ -273,9 +271,7 @@ impl TestManifest { } fn load_next_manifest(&mut self) -> Result> { - let url = if let Some(url) = self.manifests_to_do.pop_front() { - url - } else { + let Some(url) = self.manifests_to_do.pop_front() else { return Ok(None); }; self.graph.clear(); From 86f14ce96f6c691d838aa1d662bc62dccfca69ce Mon Sep 17 00:00:00 2001 From: Tpt Date: Sat, 24 Jun 2023 15:30:16 +0200 Subject: [PATCH 020/217] Improves oxttl documentation --- Cargo.lock | 2 +- lib/Cargo.toml | 2 +- lib/oxttl/Cargo.toml | 6 ++--- lib/oxttl/README.md | 54 +++++++++++++++++++++++++++++++++++++++ lib/oxttl/src/lib.rs | 6 +++++ lib/oxttl/src/n3.rs | 21 ++++++--------- lib/oxttl/src/nquads.rs | 44 +++++++++++++------------------ lib/oxttl/src/ntriples.rs | 44 +++++++++++++------------------ lib/oxttl/src/trig.rs | 44 +++++++++++++------------------ lib/oxttl/src/turtle.rs | 44 +++++++++++++------------------ 10 files changed, 145 insertions(+), 122 deletions(-) create mode 100644 lib/oxttl/README.md diff --git a/Cargo.lock b/Cargo.lock index 3ab7ed54..affef9d4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1049,7 +1049,7 @@ dependencies = [ [[package]] name = "oxttl" -version = "0.1.0" +version = "0.1.0-alpha.1-dev" dependencies = [ "memchr", "oxilangtag", diff --git a/lib/Cargo.toml b/lib/Cargo.toml index 224cf7b4..9c3fb465 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -39,7 +39,7 @@ lazy_static = "1" json-event-parser = "0.1" oxrdf = { version = "0.2.0-alpha.1-dev", path = "oxrdf", features = ["rdf-star", "oxsdatatypes"] } oxsdatatypes = { version = "0.2.0-alpha.1-dev", path="oxsdatatypes" } -oxttl = { version = "0.1.0" , path = "oxttl", features = ["rdf-star"] } +oxttl = { version = "0.1.0-alpha.1-dev" , path = "oxttl", features = ["rdf-star"] } spargebra = { version = "0.3.0-alpha.1-dev", path = "spargebra", features = ["rdf-star", "sep-0002", "sep-0006"] } sparopt = { version = "0.1.0-alpha.1-dev", path="sparopt", features = ["rdf-star", "sep-0002", "sep-0006"] } sparesults = { version = "0.2.0-alpha.1-dev", path = "sparesults", features = ["rdf-star"] } diff --git a/lib/oxttl/Cargo.toml b/lib/oxttl/Cargo.toml index bc8105c3..09d2798d 100644 --- a/lib/oxttl/Cargo.toml +++ b/lib/oxttl/Cargo.toml @@ -1,14 +1,14 @@ [package] name = "oxttl" -version = "0.1.0" +version = "0.1.0-alpha.1-dev" authors = ["Tpt "] license = "MIT OR Apache-2.0" readme = "README.md" -keywords = ["SPARQL"] +keywords = ["N-Triples", "N-Quads", "Turtle", "TriG", "N3", "RDF"] repository = "https://github.com/oxigraph/oxigraph/tree/master/lib/oxttl" homepage = "https://oxigraph.org/" description = """ -N-Triples parser +Parser for languages related to RDF Turtle (N-Triples, N-Quads, Turtle, TriG and N3) """ edition = "2021" rust-version = "1.65" diff --git a/lib/oxttl/README.md b/lib/oxttl/README.md new file mode 100644 index 00000000..e99dd2fc --- /dev/null +++ b/lib/oxttl/README.md @@ -0,0 +1,54 @@ +OxTTL +===== + +[![Latest Version](https://img.shields.io/crates/v/oxttl.svg)](https://crates.io/crates/oxttl) +[![Released API docs](https://docs.rs/oxttl/badge.svg)](https://docs.rs/oxttl) +[![Crates.io downloads](https://img.shields.io/crates/d/oxttl)](https://crates.io/crates/oxttl) +[![actions status](https://github.com/oxigraph/oxigraph/workflows/build/badge.svg)](https://github.com/oxigraph/oxigraph/actions) +[![Gitter](https://badges.gitter.im/oxigraph/community.svg)](https://gitter.im/oxigraph/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) + +Oxttl is a set of parsers and serializers for [Turtle](https://www.w3.org/TR/turtle/), [TriG](https://www.w3.org/TR/trig/), [N-Triples](https://www.w3.org/TR/n-triples/), [N-Quads](https://www.w3.org/TR/n-quads/) and [N3](https://w3c.github.io/N3/spec/). + +Support for [SPARQL-star](https://w3c.github.io/rdf-star/cg-spec/2021-12-17.html) is also available behind the `rdf-star`feature for all languages but N3 ([Turtle-star](https://w3c.github.io/rdf-star/cg-spec/2021-12-17.html#turtle-star), [TriG-star](https://w3c.github.io/rdf-star/cg-spec/2021-12-17.html#trig-star), [N-Triples-star](https://w3c.github.io/rdf-star/cg-spec/2021-12-17.html#n-triples-star) and [N-Quads-star](https://w3c.github.io/rdf-star/cg-spec/2021-12-17.html#n-quads-star)) + +It is designed as a low level parser compatible with both synchronous and asynchronous I/O. + +Usage example counting the number of people in a Turtle file: +```rust +use oxrdf::{NamedNodeRef, vocab::rdf}; +use oxttl::TurtleParser; + +let file = b"@base . +@prefix schema: . + a schema:Person ; + schema:name \"Foo\" . + a schema:Person ; + schema:name \"Bar\" ."; + +let schema_person = NamedNodeRef::new("http://schema.org/Person").unwrap(); +let mut count = 0; +for triple in TurtleParser::new().parse_from_read(file.as_ref()) { + let triple = triple.unwrap(); + if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { + count += 1; + } +} +assert_eq!(2, count); +``` + + +## License + +This project is licensed under either of + +* Apache License, Version 2.0, ([LICENSE-APACHE](../LICENSE-APACHE) or + ``) +* MIT license ([LICENSE-MIT](../LICENSE-MIT) or + ``) + +at your option. + + +### Contribution + +Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in Oxigraph by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. diff --git a/lib/oxttl/src/lib.rs b/lib/oxttl/src/lib.rs index e0d1ce90..11bafc29 100644 --- a/lib/oxttl/src/lib.rs +++ b/lib/oxttl/src/lib.rs @@ -1,3 +1,9 @@ +#![doc = include_str!("../README.md")] +#![doc(test(attr(deny(warnings))))] +#![cfg_attr(docsrs, feature(doc_auto_cfg))] +#![doc(html_favicon_url = "https://raw.githubusercontent.com/oxigraph/oxigraph/main/logo.svg")] +#![doc(html_logo_url = "https://raw.githubusercontent.com/oxigraph/oxigraph/main/logo.svg")] + mod lexer; mod line_formats; pub mod n3; diff --git a/lib/oxttl/src/n3.rs b/lib/oxttl/src/n3.rs index 9722e500..3bd7e3d6 100644 --- a/lib/oxttl/src/n3.rs +++ b/lib/oxttl/src/n3.rs @@ -177,8 +177,7 @@ impl From for N3Quad { /// /// Count the number of people: /// ``` -/// use oxrdf::NamedNode; -/// use oxttl::ParseError; +/// use oxrdf::{NamedNode, vocab::rdf}; /// use oxttl::n3::{N3Parser, N3Term}; /// /// let file = b"@base . @@ -188,7 +187,7 @@ impl From for N3Quad { /// a schema:Person ; /// schema:name \"Bar\" ."; /// -/// let rdf_type = N3Term::NamedNode(NamedNode::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?); +/// let rdf_type = N3Term::NamedNode(rdf::TYPE.into_owned()); /// let schema_person = N3Term::NamedNode(NamedNode::new("http://schema.org/Person")?); /// let mut count = 0; /// for triple in N3Parser::new().parse_from_read(file.as_ref()) { @@ -235,7 +234,6 @@ impl N3Parser { /// Count the number of people: /// ``` /// use oxrdf::NamedNode; - /// use oxttl::ParseError; /// use oxttl::n3::{N3Parser, N3Term}; /// /// let file = b"@base . @@ -267,8 +265,7 @@ impl N3Parser { /// /// Count the number of people: /// ``` - /// use oxrdf::NamedNode; - /// use oxttl::ParseError; + /// use oxrdf::{NamedNode, vocab::rdf}; /// use oxttl::n3::{N3Parser, N3Term}; /// /// let file: [&[u8]; 5] = [b"@base ", @@ -278,7 +275,7 @@ impl N3Parser { /// b" a schema:Person ; schema:name \"Bar\" ." /// ]; /// - /// let rdf_type = N3Term::NamedNode(NamedNode::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?); + /// let rdf_type = N3Term::NamedNode(rdf::TYPE.into_owned()); /// let schema_person = N3Term::NamedNode(NamedNode::new("http://schema.org/Person")?); /// let mut count = 0; /// let mut parser = N3Parser::new().parse(); @@ -312,8 +309,7 @@ impl N3Parser { /// /// Count the number of people: /// ``` -/// use oxrdf::NamedNode; -/// use oxttl::ParseError; +/// use oxrdf::{NamedNode, vocab::rdf}; /// use oxttl::n3::{N3Parser, N3Term}; /// /// let file = b"@base . @@ -323,7 +319,7 @@ impl N3Parser { /// a schema:Person ; /// schema:name \"Bar\" ."; /// -/// let rdf_type = N3Term::NamedNode(NamedNode::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?); +/// let rdf_type = N3Term::NamedNode(rdf::TYPE.into_owned()); /// let schema_person = N3Term::NamedNode(NamedNode::new("http://schema.org/Person")?); /// let mut count = 0; /// for triple in N3Parser::new().parse_from_read(file.as_ref()) { @@ -351,8 +347,7 @@ impl Iterator for FromReadN3Reader { /// /// Count the number of people: /// ``` -/// use oxrdf::NamedNode; -/// use oxttl::ParseError; +/// use oxrdf::{NamedNode, vocab::rdf}; /// use oxttl::n3::{N3Parser, N3Term}; /// /// let file: [&[u8]; 5] = [b"@base ", @@ -362,7 +357,7 @@ impl Iterator for FromReadN3Reader { /// b" a schema:Person ; schema:name \"Bar\" ." /// ]; /// -/// let rdf_type = N3Term::NamedNode(NamedNode::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?); +/// let rdf_type = N3Term::NamedNode(rdf::TYPE.into_owned()); /// let schema_person = N3Term::NamedNode(NamedNode::new("http://schema.org/Person")?); /// let mut count = 0; /// let mut parser = N3Parser::new().parse(); diff --git a/lib/oxttl/src/nquads.rs b/lib/oxttl/src/nquads.rs index c3357a38..ce8c3a2a 100644 --- a/lib/oxttl/src/nquads.rs +++ b/lib/oxttl/src/nquads.rs @@ -11,20 +11,19 @@ use std::io::{self, Read, Write}; /// /// Count the number of people: /// ``` -/// use oxrdf::NamedNodeRef; -/// use oxttl::{NQuadsParser, ParseError}; +/// use oxrdf::{NamedNodeRef, vocab::rdf}; +/// use oxttl::NQuadsParser; /// /// let file = b" . /// \"Foo\" . /// . /// \"Bar\" ."; /// -/// let rdf_type = NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?; /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; /// let mut count = 0; /// for quad in NQuadsParser::new().parse_from_read(file.as_ref()) { /// let quad = quad?; -/// if quad.predicate == rdf_type && quad.object == schema_person.into() { +/// if quad.predicate == rdf::TYPE && quad.object == schema_person.into() { /// count += 1; /// } /// } @@ -57,20 +56,19 @@ impl NQuadsParser { /// /// Count the number of people: /// ``` - /// use oxrdf::NamedNodeRef; - /// use oxttl::{NQuadsParser, ParseError}; + /// use oxrdf::{NamedNodeRef, vocab::rdf}; + /// use oxttl::NQuadsParser; /// /// let file = b" . /// \"Foo\" . /// . /// \"Bar\" ."; /// - /// let rdf_type = NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?; /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; /// let mut count = 0; /// for quad in NQuadsParser::new().parse_from_read(file.as_ref()) { /// let quad = quad?; - /// if quad.predicate == rdf_type && quad.object == schema_person.into() { + /// if quad.predicate == rdf::TYPE && quad.object == schema_person.into() { /// count += 1; /// } /// } @@ -87,8 +85,8 @@ impl NQuadsParser { /// /// Count the number of people: /// ``` - /// use oxrdf::NamedNodeRef; - /// use oxttl::{NQuadsParser, ParseError}; + /// use oxrdf::{NamedNodeRef, vocab::rdf}; + /// use oxttl::NQuadsParser; /// /// let file: [&[u8]; 4] = [ /// b" .\n", @@ -97,7 +95,6 @@ impl NQuadsParser { /// b" \"Bar\" .\n" /// ]; /// - /// let rdf_type = NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?; /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; /// let mut count = 0; /// let mut parser = NQuadsParser::new().parse(); @@ -112,7 +109,7 @@ impl NQuadsParser { /// // We read as many quads from the parser as possible /// while let Some(quad) = parser.read_next() { /// let quad = quad?; - /// if quad.predicate == rdf_type && quad.object == schema_person.into() { + /// if quad.predicate == rdf::TYPE && quad.object == schema_person.into() { /// count += 1; /// } /// } @@ -136,20 +133,19 @@ impl NQuadsParser { /// /// Count the number of people: /// ``` -/// use oxrdf::NamedNodeRef; -/// use oxttl::{NQuadsParser, ParseError}; +/// use oxrdf::{NamedNodeRef, vocab::rdf}; +/// use oxttl::NQuadsParser; /// /// let file = b" . /// \"Foo\" . /// . /// \"Bar\" ."; /// -/// let rdf_type = NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?; /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; /// let mut count = 0; /// for quad in NQuadsParser::new().parse_from_read(file.as_ref()) { /// let quad = quad?; -/// if quad.predicate == rdf_type && quad.object == schema_person.into() { +/// if quad.predicate == rdf::TYPE && quad.object == schema_person.into() { /// count += 1; /// } /// } @@ -172,8 +168,8 @@ impl Iterator for FromReadNQuadsReader { /// /// Count the number of people: /// ``` -/// use oxrdf::NamedNodeRef; -/// use oxttl::{NQuadsParser, ParseError}; +/// use oxrdf::{NamedNodeRef, vocab::rdf}; +/// use oxttl::NQuadsParser; /// /// let file: [&[u8]; 4] = [ /// b" .\n", @@ -182,7 +178,6 @@ impl Iterator for FromReadNQuadsReader { /// b" \"Bar\" .\n" /// ]; /// -/// let rdf_type = NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?; /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; /// let mut count = 0; /// let mut parser = NQuadsParser::new().parse(); @@ -197,7 +192,7 @@ impl Iterator for FromReadNQuadsReader { /// // We read as many quads from the parser as possible /// while let Some(quad) = parser.read_next() { /// let quad = quad?; -/// if quad.predicate == rdf_type && quad.object == schema_person.into() { +/// if quad.predicate == rdf::TYPE && quad.object == schema_person.into() { /// count += 1; /// } /// } @@ -244,8 +239,7 @@ impl LowLevelNQuadsReader { /// use oxrdf::{NamedNodeRef, QuadRef}; /// use oxttl::NQuadsSerializer; /// -/// let mut buf = Vec::new(); -/// let mut writer = NQuadsSerializer::new().serialize_to_write(buf); +/// let mut writer = NQuadsSerializer::new().serialize_to_write(Vec::new()); /// writer.write_quad(QuadRef::new( /// NamedNodeRef::new("http://example.com#me")?, /// NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?, @@ -274,8 +268,7 @@ impl NQuadsSerializer { /// use oxrdf::{NamedNodeRef, QuadRef}; /// use oxttl::NQuadsSerializer; /// - /// let mut buf = Vec::new(); - /// let mut writer = NQuadsSerializer::new().serialize_to_write(buf); + /// let mut writer = NQuadsSerializer::new().serialize_to_write(Vec::new()); /// writer.write_quad(QuadRef::new( /// NamedNodeRef::new("http://example.com#me")?, /// NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?, @@ -327,8 +320,7 @@ impl NQuadsSerializer { /// use oxrdf::{NamedNodeRef, QuadRef}; /// use oxttl::NQuadsSerializer; /// -/// let mut buf = Vec::new(); -/// let mut writer = NQuadsSerializer::new().serialize_to_write(buf); +/// let mut writer = NQuadsSerializer::new().serialize_to_write(Vec::new()); /// writer.write_quad(QuadRef::new( /// NamedNodeRef::new("http://example.com#me")?, /// NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?, diff --git a/lib/oxttl/src/ntriples.rs b/lib/oxttl/src/ntriples.rs index fb6692fc..daaa2d5c 100644 --- a/lib/oxttl/src/ntriples.rs +++ b/lib/oxttl/src/ntriples.rs @@ -12,20 +12,19 @@ use std::io::{self, Read, Write}; /// /// Count the number of people: /// ``` -/// use oxrdf::NamedNodeRef; -/// use oxttl::{NTriplesParser, ParseError}; +/// use oxrdf::{NamedNodeRef, vocab::rdf}; +/// use oxttl::NTriplesParser; /// /// let file = b" . /// \"Foo\" . /// . /// \"Bar\" ."; /// -/// let rdf_type = NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?; /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; /// let mut count = 0; /// for triple in NTriplesParser::new().parse_from_read(file.as_ref()) { /// let triple = triple?; -/// if triple.predicate == rdf_type && triple.object == schema_person.into() { +/// if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { /// count += 1; /// } /// } @@ -58,20 +57,19 @@ impl NTriplesParser { /// /// Count the number of people: /// ``` - /// use oxrdf::NamedNodeRef; - /// use oxttl::{NTriplesParser, ParseError}; + /// use oxrdf::{NamedNodeRef, vocab::rdf}; + /// use oxttl::NTriplesParser; /// /// let file = b" . /// \"Foo\" . /// . /// \"Bar\" ."; /// - /// let rdf_type = NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?; /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; /// let mut count = 0; /// for triple in NTriplesParser::new().parse_from_read(file.as_ref()) { /// let triple = triple?; - /// if triple.predicate == rdf_type && triple.object == schema_person.into() { + /// if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { /// count += 1; /// } /// } @@ -88,8 +86,8 @@ impl NTriplesParser { /// /// Count the number of people: /// ``` - /// use oxrdf::NamedNodeRef; - /// use oxttl::{NTriplesParser, ParseError}; + /// use oxrdf::{NamedNodeRef, vocab::rdf}; + /// use oxttl::NTriplesParser; /// /// let file: [&[u8]; 4] = [ /// b" .\n", @@ -98,7 +96,6 @@ impl NTriplesParser { /// b" \"Bar\" .\n" /// ]; /// - /// let rdf_type = NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?; /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; /// let mut count = 0; /// let mut parser = NTriplesParser::new().parse(); @@ -113,7 +110,7 @@ impl NTriplesParser { /// // We read as many triples from the parser as possible /// while let Some(triple) = parser.read_next() { /// let triple = triple?; - /// if triple.predicate == rdf_type && triple.object == schema_person.into() { + /// if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { /// count += 1; /// } /// } @@ -137,20 +134,19 @@ impl NTriplesParser { /// /// Count the number of people: /// ``` -/// use oxrdf::NamedNodeRef; -/// use oxttl::{NTriplesParser, ParseError}; +/// use oxrdf::{NamedNodeRef, vocab::rdf}; +/// use oxttl::NTriplesParser; /// /// let file = b" . /// \"Foo\" . /// . /// \"Bar\" ."; /// -/// let rdf_type = NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?; /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; /// let mut count = 0; /// for triple in NTriplesParser::new().parse_from_read(file.as_ref()) { /// let triple = triple?; -/// if triple.predicate == rdf_type && triple.object == schema_person.into() { +/// if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { /// count += 1; /// } /// } @@ -173,8 +169,8 @@ impl Iterator for FromReadNTriplesReader { /// /// Count the number of people: /// ``` -/// use oxrdf::NamedNodeRef; -/// use oxttl::{NTriplesParser, ParseError}; +/// use oxrdf::{NamedNodeRef, vocab::rdf}; +/// use oxttl::NTriplesParser; /// /// let file: [&[u8]; 4] = [ /// b" .\n", @@ -183,7 +179,6 @@ impl Iterator for FromReadNTriplesReader { /// b" \"Bar\" .\n" /// ]; /// -/// let rdf_type = NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?; /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; /// let mut count = 0; /// let mut parser = NTriplesParser::new().parse(); @@ -198,7 +193,7 @@ impl Iterator for FromReadNTriplesReader { /// // We read as many triples from the parser as possible /// while let Some(triple) = parser.read_next() { /// let triple = triple?; -/// if triple.predicate == rdf_type && triple.object == schema_person.into() { +/// if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { /// count += 1; /// } /// } @@ -245,8 +240,7 @@ impl LowLevelNTriplesReader { /// use oxrdf::{NamedNodeRef, TripleRef}; /// use oxttl::NTriplesSerializer; /// -/// let mut buf = Vec::new(); -/// let mut writer = NTriplesSerializer::new().serialize_to_write(buf); +/// let mut writer = NTriplesSerializer::new().serialize_to_write(Vec::new()); /// writer.write_triple(TripleRef::new( /// NamedNodeRef::new("http://example.com#me")?, /// NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?, @@ -274,8 +268,7 @@ impl NTriplesSerializer { /// use oxrdf::{NamedNodeRef, TripleRef}; /// use oxttl::NTriplesSerializer; /// - /// let mut buf = Vec::new(); - /// let mut writer = NTriplesSerializer::new().serialize_to_write(buf); + /// let mut writer = NTriplesSerializer::new().serialize_to_write(Vec::new()); /// writer.write_triple(TripleRef::new( /// NamedNodeRef::new("http://example.com#me")?, /// NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?, @@ -325,8 +318,7 @@ impl NTriplesSerializer { /// use oxrdf::{NamedNodeRef, TripleRef}; /// use oxttl::NTriplesSerializer; /// -/// let mut buf = Vec::new(); -/// let mut writer = NTriplesSerializer::new().serialize_to_write(buf); +/// let mut writer = NTriplesSerializer::new().serialize_to_write(Vec::new()); /// writer.write_triple(TripleRef::new( /// NamedNodeRef::new("http://example.com#me")?, /// NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?, diff --git a/lib/oxttl/src/trig.rs b/lib/oxttl/src/trig.rs index 1860c56e..311b8f96 100644 --- a/lib/oxttl/src/trig.rs +++ b/lib/oxttl/src/trig.rs @@ -14,8 +14,8 @@ use std::io::{self, Read, Write}; /// /// Count the number of people: /// ``` -/// use oxrdf::NamedNodeRef; -/// use oxttl::{TriGParser, ParseError}; +/// use oxrdf::{NamedNodeRef, vocab::rdf}; +/// use oxttl::TriGParser; /// /// let file = b"@base . /// @prefix schema: . @@ -24,12 +24,11 @@ use std::io::{self, Read, Write}; /// a schema:Person ; /// schema:name \"Bar\" ."; /// -/// let rdf_type = NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?; /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; /// let mut count = 0; /// for quad in TriGParser::new().parse_from_read(file.as_ref()) { /// let quad = quad?; -/// if quad.predicate == rdf_type && quad.object == schema_person.into() { +/// if quad.predicate == rdf::TYPE && quad.object == schema_person.into() { /// count += 1; /// } /// } @@ -81,8 +80,8 @@ impl TriGParser { /// /// Count the number of people: /// ``` - /// use oxrdf::NamedNodeRef; - /// use oxttl::{TriGParser, ParseError}; + /// use oxrdf::{NamedNodeRef, vocab::rdf}; + /// use oxttl::TriGParser; /// /// let file = b"@base . /// @prefix schema: . @@ -91,12 +90,11 @@ impl TriGParser { /// a schema:Person ; /// schema:name \"Bar\" ."; /// - /// let rdf_type = NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?; /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; /// let mut count = 0; /// for quad in TriGParser::new().parse_from_read(file.as_ref()) { /// let quad = quad?; - /// if quad.predicate == rdf_type && quad.object == schema_person.into() { + /// if quad.predicate == rdf::TYPE && quad.object == schema_person.into() { /// count += 1; /// } /// } @@ -113,8 +111,8 @@ impl TriGParser { /// /// Count the number of people: /// ``` - /// use oxrdf::NamedNodeRef; - /// use oxttl::{TriGParser, ParseError}; + /// use oxrdf::{NamedNodeRef, vocab::rdf}; + /// use oxttl::TriGParser; /// /// let file: [&[u8]; 5] = [b"@base ", /// b". @prefix schema: .", @@ -123,7 +121,6 @@ impl TriGParser { /// b" a schema:Person ; schema:name \"Bar\" ." /// ]; /// - /// let rdf_type = NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?; /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; /// let mut count = 0; /// let mut parser = TriGParser::new().parse(); @@ -138,7 +135,7 @@ impl TriGParser { /// // We read as many quads from the parser as possible /// while let Some(quad) = parser.read_next() { /// let quad = quad?; - /// if quad.predicate == rdf_type && quad.object == schema_person.into() { + /// if quad.predicate == rdf::TYPE && quad.object == schema_person.into() { /// count += 1; /// } /// } @@ -163,8 +160,8 @@ impl TriGParser { /// /// Count the number of people: /// ``` -/// use oxrdf::NamedNodeRef; -/// use oxttl::{TriGParser, ParseError}; +/// use oxrdf::{NamedNodeRef, vocab::rdf}; +/// use oxttl::TriGParser; /// /// let file = b"@base . /// @prefix schema: . @@ -173,12 +170,11 @@ impl TriGParser { /// a schema:Person ; /// schema:name \"Bar\" ."; /// -/// let rdf_type = NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?; /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; /// let mut count = 0; /// for quad in TriGParser::new().parse_from_read(file.as_ref()) { /// let quad = quad?; -/// if quad.predicate == rdf_type && quad.object == schema_person.into() { +/// if quad.predicate == rdf::TYPE && quad.object == schema_person.into() { /// count += 1; /// } /// } @@ -201,8 +197,8 @@ impl Iterator for FromReadTriGReader { /// /// Count the number of people: /// ``` -/// use oxrdf::NamedNodeRef; -/// use oxttl::{TriGParser, ParseError}; +/// use oxrdf::{NamedNodeRef, vocab::rdf}; +/// use oxttl::TriGParser; /// /// let file: [&[u8]; 5] = [b"@base ", /// b". @prefix schema: .", @@ -211,7 +207,6 @@ impl Iterator for FromReadTriGReader { /// b" a schema:Person ; schema:name \"Bar\" ." /// ]; /// -/// let rdf_type = NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?; /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; /// let mut count = 0; /// let mut parser = TriGParser::new().parse(); @@ -226,7 +221,7 @@ impl Iterator for FromReadTriGReader { /// // We read as many quads from the parser as possible /// while let Some(quad) = parser.read_next() { /// let quad = quad?; -/// if quad.predicate == rdf_type && quad.object == schema_person.into() { +/// if quad.predicate == rdf::TYPE && quad.object == schema_person.into() { /// count += 1; /// } /// } @@ -273,8 +268,7 @@ impl LowLevelTriGReader { /// use oxrdf::{NamedNodeRef, QuadRef}; /// use oxttl::TriGSerializer; /// -/// let mut buf = Vec::new(); -/// let mut writer = TriGSerializer::new().serialize_to_write(buf); +/// let mut writer = TriGSerializer::new().serialize_to_write(Vec::new()); /// writer.write_quad(QuadRef::new( /// NamedNodeRef::new("http://example.com#me")?, /// NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?, @@ -303,8 +297,7 @@ impl TriGSerializer { /// use oxrdf::{NamedNodeRef, QuadRef}; /// use oxttl::TriGSerializer; /// - /// let mut buf = Vec::new(); - /// let mut writer = TriGSerializer::new().serialize_to_write(buf); + /// let mut writer = TriGSerializer::new().serialize_to_write(Vec::new()); /// writer.write_quad(QuadRef::new( /// NamedNodeRef::new("http://example.com#me")?, /// NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?, @@ -360,8 +353,7 @@ impl TriGSerializer { /// use oxrdf::{NamedNodeRef, QuadRef}; /// use oxttl::TriGSerializer; /// -/// let mut buf = Vec::new(); -/// let mut writer = TriGSerializer::new().serialize_to_write(buf); +/// let mut writer = TriGSerializer::new().serialize_to_write(Vec::new()); /// writer.write_quad(QuadRef::new( /// NamedNodeRef::new("http://example.com#me")?, /// NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?, diff --git a/lib/oxttl/src/turtle.rs b/lib/oxttl/src/turtle.rs index 275cec28..65967613 100644 --- a/lib/oxttl/src/turtle.rs +++ b/lib/oxttl/src/turtle.rs @@ -15,8 +15,8 @@ use std::io::{self, Read, Write}; /// /// Count the number of people: /// ``` -/// use oxrdf::NamedNodeRef; -/// use oxttl::{TurtleParser, ParseError}; +/// use oxrdf::{NamedNodeRef, vocab::rdf}; +/// use oxttl::TurtleParser; /// /// let file = b"@base . /// @prefix schema: . @@ -25,12 +25,11 @@ use std::io::{self, Read, Write}; /// a schema:Person ; /// schema:name \"Bar\" ."; /// -/// let rdf_type = NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?; /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; /// let mut count = 0; /// for triple in TurtleParser::new().parse_from_read(file.as_ref()) { /// let triple = triple?; -/// if triple.predicate == rdf_type && triple.object == schema_person.into() { +/// if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { /// count += 1; /// } /// } @@ -82,8 +81,8 @@ impl TurtleParser { /// /// Count the number of people: /// ``` - /// use oxrdf::NamedNodeRef; - /// use oxttl::{TurtleParser, ParseError}; + /// use oxrdf::{NamedNodeRef, vocab::rdf}; + /// use oxttl::TurtleParser; /// /// let file = b"@base . /// @prefix schema: . @@ -92,12 +91,11 @@ impl TurtleParser { /// a schema:Person ; /// schema:name \"Bar\" ."; /// - /// let rdf_type = NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?; /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; /// let mut count = 0; /// for triple in TurtleParser::new().parse_from_read(file.as_ref()) { /// let triple = triple?; - /// if triple.predicate == rdf_type && triple.object == schema_person.into() { + /// if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { /// count += 1; /// } /// } @@ -114,8 +112,8 @@ impl TurtleParser { /// /// Count the number of people: /// ``` - /// use oxrdf::NamedNodeRef; - /// use oxttl::{TurtleParser, ParseError}; + /// use oxrdf::{NamedNodeRef, vocab::rdf}; + /// use oxttl::TurtleParser; /// /// let file: [&[u8]; 5] = [b"@base ", /// b". @prefix schema: .", @@ -124,7 +122,6 @@ impl TurtleParser { /// b" a schema:Person ; schema:name \"Bar\" ." /// ]; /// - /// let rdf_type = NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?; /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; /// let mut count = 0; /// let mut parser = TurtleParser::new().parse(); @@ -139,7 +136,7 @@ impl TurtleParser { /// // We read as many triples from the parser as possible /// while let Some(triple) = parser.read_next() { /// let triple = triple?; - /// if triple.predicate == rdf_type && triple.object == schema_person.into() { + /// if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { /// count += 1; /// } /// } @@ -164,8 +161,8 @@ impl TurtleParser { /// /// Count the number of people: /// ``` -/// use oxrdf::NamedNodeRef; -/// use oxttl::{TurtleParser, ParseError}; +/// use oxrdf::{NamedNodeRef, vocab::rdf}; +/// use oxttl::TurtleParser; /// /// let file = b"@base . /// @prefix schema: . @@ -174,12 +171,11 @@ impl TurtleParser { /// a schema:Person ; /// schema:name \"Bar\" ."; /// -/// let rdf_type = NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?; /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; /// let mut count = 0; /// for triple in TurtleParser::new().parse_from_read(file.as_ref()) { /// let triple = triple?; -/// if triple.predicate == rdf_type && triple.object == schema_person.into() { +/// if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { /// count += 1; /// } /// } @@ -202,8 +198,8 @@ impl Iterator for FromReadTurtleReader { /// /// Count the number of people: /// ``` -/// use oxrdf::NamedNodeRef; -/// use oxttl::{TurtleParser, ParseError}; +/// use oxrdf::{NamedNodeRef, vocab::rdf}; +/// use oxttl::TurtleParser; /// /// let file: [&[u8]; 5] = [b"@base ", /// b". @prefix schema: .", @@ -212,7 +208,6 @@ impl Iterator for FromReadTurtleReader { /// b" a schema:Person ; schema:name \"Bar\" ." /// ]; /// -/// let rdf_type = NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?; /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; /// let mut count = 0; /// let mut parser = TurtleParser::new().parse(); @@ -227,7 +222,7 @@ impl Iterator for FromReadTurtleReader { /// // We read as many triples from the parser as possible /// while let Some(triple) = parser.read_next() { /// let triple = triple?; -/// if triple.predicate == rdf_type && triple.object == schema_person.into() { +/// if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { /// count += 1; /// } /// } @@ -274,8 +269,7 @@ impl LowLevelTurtleReader { /// use oxrdf::{NamedNodeRef, TripleRef}; /// use oxttl::TurtleSerializer; /// -/// let mut buf = Vec::new(); -/// let mut writer = TurtleSerializer::new().serialize_to_write(buf); +/// let mut writer = TurtleSerializer::new().serialize_to_write(Vec::new()); /// writer.write_triple(TripleRef::new( /// NamedNodeRef::new("http://example.com#me")?, /// NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?, @@ -305,8 +299,7 @@ impl TurtleSerializer { /// use oxrdf::{NamedNodeRef, TripleRef}; /// use oxttl::TurtleSerializer; /// - /// let mut buf = Vec::new(); - /// let mut writer = TurtleSerializer::new().serialize_to_write(buf); + /// let mut writer = TurtleSerializer::new().serialize_to_write(Vec::new()); /// writer.write_triple(TripleRef::new( /// NamedNodeRef::new("http://example.com#me")?, /// NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?, @@ -357,8 +350,7 @@ impl TurtleSerializer { /// use oxrdf::{NamedNodeRef, TripleRef}; /// use oxttl::TurtleSerializer; /// -/// let mut buf = Vec::new(); -/// let mut writer = TurtleSerializer::new().serialize_to_write(buf); +/// let mut writer = TurtleSerializer::new().serialize_to_write(Vec::new()); /// writer.write_triple(TripleRef::new( /// NamedNodeRef::new("http://example.com#me")?, /// NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?, From 001b6e07b7f1b9671930bef6d2f34a328319e823 Mon Sep 17 00:00:00 2001 From: Tpt Date: Fri, 23 Jun 2023 17:17:40 +0200 Subject: [PATCH 021/217] Enforces "return self not must use" lint --- lib/oxsdatatypes/src/boolean.rs | 1 + lib/oxsdatatypes/src/date_time.rs | 102 ++++++++++++++++++++++++++++++ lib/oxsdatatypes/src/decimal.rs | 17 +++++ lib/oxsdatatypes/src/double.rs | 9 +++ lib/oxsdatatypes/src/duration.rs | 15 +++++ lib/oxsdatatypes/src/float.rs | 9 +++ lib/oxsdatatypes/src/integer.rs | 13 ++++ lib/oxsdatatypes/src/lib.rs | 1 - lib/src/lib.rs | 1 - lib/src/storage/mod.rs | 3 + lib/src/store.rs | 4 ++ 11 files changed, 173 insertions(+), 2 deletions(-) diff --git a/lib/oxsdatatypes/src/boolean.rs b/lib/oxsdatatypes/src/boolean.rs index 7ce5d977..de29debc 100644 --- a/lib/oxsdatatypes/src/boolean.rs +++ b/lib/oxsdatatypes/src/boolean.rs @@ -14,6 +14,7 @@ pub struct Boolean { impl Boolean { /// Checks if the two values are [identical](https://www.w3.org/TR/xmlschema11-2/#identity). #[inline] + #[must_use] pub fn is_identical_with(self, other: Self) -> bool { self == other } diff --git a/lib/oxsdatatypes/src/date_time.rs b/lib/oxsdatatypes/src/date_time.rs index 8d005a46..b5cc8d9e 100644 --- a/lib/oxsdatatypes/src/date_time.rs +++ b/lib/oxsdatatypes/src/date_time.rs @@ -52,6 +52,7 @@ impl DateTime { } #[inline] + #[must_use] pub fn from_be_bytes(bytes: [u8; 18]) -> Self { Self { timestamp: Timestamp::from_be_bytes(bytes), @@ -60,47 +61,55 @@ impl DateTime { /// [fn:year-from-dateTime](https://www.w3.org/TR/xpath-functions-31/#func-year-from-dateTime) #[inline] + #[must_use] pub fn year(self) -> i64 { self.timestamp.year() } /// [fn:month-from-dateTime](https://www.w3.org/TR/xpath-functions-31/#func-month-from-dateTime) #[inline] + #[must_use] pub fn month(self) -> u8 { self.timestamp.month() } /// [fn:day-from-dateTime](https://www.w3.org/TR/xpath-functions-31/#func-day-from-dateTime) #[inline] + #[must_use] pub fn day(self) -> u8 { self.timestamp.day() } /// [fn:hour-from-dateTime](https://www.w3.org/TR/xpath-functions-31/#func-hours-from-dateTime) #[inline] + #[must_use] pub fn hour(self) -> u8 { self.timestamp.hour() } /// [fn:minute-from-dateTime](https://www.w3.org/TR/xpath-functions-31/#func-minutes-from-dateTime) #[inline] + #[must_use] pub fn minute(self) -> u8 { self.timestamp.minute() } /// [fn:second-from-dateTime](https://www.w3.org/TR/xpath-functions-31/#func-seconds-from-dateTime) #[inline] + #[must_use] pub fn second(self) -> Decimal { self.timestamp.second() } /// [fn:timezone-from-dateTime](https://www.w3.org/TR/xpath-functions-31/#func-timezone-from-dateTime) #[inline] + #[must_use] pub fn timezone(self) -> Option { Some(self.timezone_offset()?.into()) } #[inline] + #[must_use] pub fn timezone_offset(self) -> Option { self.timestamp.timezone_offset() } @@ -119,18 +128,21 @@ impl DateTime { } #[inline] + #[must_use] pub fn to_be_bytes(self) -> [u8; 18] { self.timestamp.to_be_bytes() } /// [op:subtract-dateTimes](https://www.w3.org/TR/xpath-functions-31/#func-subtract-dateTimes) #[inline] + #[must_use] pub fn checked_sub(self, rhs: impl Into) -> Option { self.timestamp.checked_sub(rhs.into().timestamp) } /// [op:add-yearMonthDuration-to-dateTime](https://www.w3.org/TR/xpath-functions-31/#func-add-yearMonthDuration-to-dateTime) #[inline] + #[must_use] pub fn checked_add_year_month_duration( self, rhs: impl Into, @@ -140,6 +152,7 @@ impl DateTime { /// [op:add-dayTimeDuration-to-dateTime](https://www.w3.org/TR/xpath-functions-31/#func-add-dayTimeDuration-to-dateTime) #[inline] + #[must_use] pub fn checked_add_day_time_duration(self, rhs: impl Into) -> Option { let rhs = rhs.into(); Some(Self { @@ -149,6 +162,7 @@ impl DateTime { /// [op:add-yearMonthDuration-to-dateTime](https://www.w3.org/TR/xpath-functions-31/#func-add-yearMonthDuration-to-dateTime) and [op:add-dayTimeDuration-to-dateTime](https://www.w3.org/TR/xpath-functions-31/#func-add-dayTimeDuration-to-dateTime) #[inline] + #[must_use] pub fn checked_add_duration(self, rhs: impl Into) -> Option { let rhs = rhs.into(); if let Ok(rhs) = DayTimeDuration::try_from(rhs) { @@ -163,6 +177,7 @@ impl DateTime { /// [op:subtract-yearMonthDuration-from-dateTime](https://www.w3.org/TR/xpath-functions-31/#func-subtract-yearMonthDuration-from-dateTime) #[inline] + #[must_use] pub fn checked_sub_year_month_duration( self, rhs: impl Into, @@ -172,6 +187,7 @@ impl DateTime { /// [op:subtract-dayTimeDuration-from-dateTime](https://www.w3.org/TR/xpath-functions-31/#func-subtract-dayTimeDuration-from-dateTime) #[inline] + #[must_use] pub fn checked_sub_day_time_duration(self, rhs: impl Into) -> Option { let rhs = rhs.into(); Some(Self { @@ -181,6 +197,7 @@ impl DateTime { /// [op:subtract-yearMonthDuration-from-dateTime](https://www.w3.org/TR/xpath-functions-31/#func-subtract-yearMonthDuration-from-dateTime) and [op:subtract-dayTimeDuration-from-dateTime](https://www.w3.org/TR/xpath-functions-31/#func-subtract-dayTimeDuration-from-dateTime) #[inline] + #[must_use] pub fn checked_sub_duration(self, rhs: impl Into) -> Option { let rhs = rhs.into(); if let Ok(rhs) = DayTimeDuration::try_from(rhs) { @@ -198,6 +215,7 @@ impl DateTime { /// [fn:adjust-dateTime-to-timezone](https://www.w3.org/TR/xpath-functions-31/#func-adjust-dateTime-to-timezone) #[inline] + #[must_use] pub fn adjust(self, timezone_offset: Option) -> Option { Some(Self { timestamp: self.timestamp.adjust(timezone_offset)?, @@ -206,6 +224,7 @@ impl DateTime { /// Checks if the two values are [identical](https://www.w3.org/TR/xmlschema11-2/#identity). #[inline] + #[must_use] pub fn is_identical_with(self, other: Self) -> bool { self.timestamp.is_identical_with(other.timestamp) } @@ -300,6 +319,7 @@ impl Time { } #[inline] + #[must_use] pub fn from_be_bytes(bytes: [u8; 18]) -> Self { Self { timestamp: Timestamp::from_be_bytes(bytes), @@ -314,52 +334,61 @@ impl Time { /// [fn:hour-from-time](https://www.w3.org/TR/xpath-functions-31/#func-hours-from-time) #[inline] + #[must_use] pub fn hour(self) -> u8 { self.timestamp.hour() } /// [fn:minute-from-time](https://www.w3.org/TR/xpath-functions-31/#func-minutes-from-time) #[inline] + #[must_use] pub fn minute(self) -> u8 { self.timestamp.minute() } /// [fn:second-from-time](https://www.w3.org/TR/xpath-functions-31/#func-seconds-from-time) #[inline] + #[must_use] pub fn second(self) -> Decimal { self.timestamp.second() } /// [fn:timezone-from-time](https://www.w3.org/TR/xpath-functions-31/#func-timezone-from-time) #[inline] + #[must_use] pub fn timezone(self) -> Option { Some(self.timezone_offset()?.into()) } #[inline] + #[must_use] pub fn timezone_offset(self) -> Option { self.timestamp.timezone_offset() } #[inline] + #[must_use] pub fn to_be_bytes(self) -> [u8; 18] { self.timestamp.to_be_bytes() } /// [op:subtract-times](https://www.w3.org/TR/xpath-functions-31/#func-subtract-times) #[inline] + #[must_use] pub fn checked_sub(self, rhs: impl Into) -> Option { self.timestamp.checked_sub(rhs.into().timestamp) } /// [op:add-dayTimeDuration-to-time](https://www.w3.org/TR/xpath-functions-31/#func-add-dayTimeDuration-to-time) #[inline] + #[must_use] pub fn checked_add_day_time_duration(self, rhs: impl Into) -> Option { self.checked_add_duration(Duration::from(rhs.into())) } /// [op:add-dayTimeDuration-to-time](https://www.w3.org/TR/xpath-functions-31/#func-add-dayTimeDuration-to-time) #[inline] + #[must_use] pub fn checked_add_duration(self, rhs: impl Into) -> Option { DateTime::new( 1972, @@ -378,12 +407,14 @@ impl Time { /// [op:subtract-dayTimeDuration-from-time](https://www.w3.org/TR/xpath-functions-31/#func-subtract-dayTimeDuration-from-time) #[inline] + #[must_use] pub fn checked_sub_day_time_duration(self, rhs: impl Into) -> Option { self.checked_sub_duration(Duration::from(rhs.into())) } /// [op:subtract-dayTimeDuration-from-time](https://www.w3.org/TR/xpath-functions-31/#func-subtract-dayTimeDuration-from-time) #[inline] + #[must_use] pub fn checked_sub_duration(self, rhs: impl Into) -> Option { DateTime::new( 1972, @@ -402,6 +433,7 @@ impl Time { // [fn:adjust-time-to-timezone](https://www.w3.org/TR/xpath-functions-31/#func-adjust-time-to-timezone) #[inline] + #[must_use] pub fn adjust(self, timezone_offset: Option) -> Option { DateTime::new( 1972, @@ -420,6 +452,7 @@ impl Time { /// Checks if the two values are [identical](https://www.w3.org/TR/xmlschema11-2/#identity). #[inline] + #[must_use] pub fn is_identical_with(self, other: Self) -> bool { self.timestamp.is_identical_with(other.timestamp) } @@ -501,6 +534,7 @@ impl Date { } #[inline] + #[must_use] pub fn from_be_bytes(bytes: [u8; 18]) -> Self { Self { timestamp: Timestamp::from_be_bytes(bytes), @@ -515,46 +549,54 @@ impl Date { /// [fn:year-from-date](https://www.w3.org/TR/xpath-functions-31/#func-year-from-date) #[inline] + #[must_use] pub fn year(self) -> i64 { self.timestamp.year() } /// [fn:month-from-date](https://www.w3.org/TR/xpath-functions-31/#func-month-from-date) #[inline] + #[must_use] pub fn month(self) -> u8 { self.timestamp.month() } /// [fn:day-from-date](https://www.w3.org/TR/xpath-functions-31/#func-day-from-date) #[inline] + #[must_use] pub fn day(self) -> u8 { self.timestamp.day() } /// [fn:timezone-from-date](https://www.w3.org/TR/xpath-functions-31/#func-timezone-from-date) #[inline] + #[must_use] pub fn timezone(self) -> Option { Some(self.timezone_offset()?.into()) } #[inline] + #[must_use] pub fn timezone_offset(self) -> Option { self.timestamp.timezone_offset() } #[inline] + #[must_use] pub fn to_be_bytes(self) -> [u8; 18] { self.timestamp.to_be_bytes() } /// [op:subtract-dates](https://www.w3.org/TR/xpath-functions-31/#func-subtract-dates) #[inline] + #[must_use] pub fn checked_sub(self, rhs: impl Into) -> Option { self.timestamp.checked_sub(rhs.into().timestamp) } /// [op:add-yearMonthDuration-to-date](https://www.w3.org/TR/xpath-functions-31/#func-add-yearMonthDuration-to-date) #[inline] + #[must_use] pub fn checked_add_year_month_duration( self, rhs: impl Into, @@ -564,12 +606,14 @@ impl Date { /// [op:add-dayTimeDuration-to-dateTime](https://www.w3.org/TR/xpath-functions-31/#func-add-dayTimeDuration-to-date) #[inline] + #[must_use] pub fn checked_add_day_time_duration(self, rhs: impl Into) -> Option { self.checked_add_duration(Duration::from(rhs.into())) } /// [op:add-yearMonthDuration-to-date](https://www.w3.org/TR/xpath-functions-31/#func-add-yearMonthDuration-to-date) and [op:add-dayTimeDuration-to-dateTime](https://www.w3.org/TR/xpath-functions-31/#func-add-dayTimeDuration-to-date) #[inline] + #[must_use] pub fn checked_add_duration(self, rhs: impl Into) -> Option { DateTime::try_from(self) .ok()? @@ -580,6 +624,7 @@ impl Date { /// [op:subtract-yearMonthDuration-from-date](https://www.w3.org/TR/xpath-functions-31/#func-subtract-yearMonthDuration-from-date) #[inline] + #[must_use] pub fn checked_sub_year_month_duration( self, rhs: impl Into, @@ -589,12 +634,14 @@ impl Date { /// [op:subtract-dayTimeDuration-from-date](https://www.w3.org/TR/xpath-functions-31/#func-subtract-dayTimeDuration-from-date) #[inline] + #[must_use] pub fn checked_sub_day_time_duration(self, rhs: impl Into) -> Option { self.checked_sub_duration(Duration::from(rhs.into())) } /// [op:subtract-yearMonthDuration-from-date](https://www.w3.org/TR/xpath-functions-31/#func-subtract-yearMonthDuration-from-date) and [op:subtract-dayTimeDuration-from-date](https://www.w3.org/TR/xpath-functions-31/#func-subtract-dayTimeDuration-from-date) #[inline] + #[must_use] pub fn checked_sub_duration(self, rhs: impl Into) -> Option { DateTime::try_from(self) .ok()? @@ -605,6 +652,7 @@ impl Date { // [fn:adjust-date-to-timezone](https://www.w3.org/TR/xpath-functions-31/#func-adjust-date-to-timezone) #[inline] + #[must_use] pub fn adjust(self, timezone_offset: Option) -> Option { DateTime::new( self.year(), @@ -623,6 +671,7 @@ impl Date { /// Checks if the two values are [identical](https://www.w3.org/TR/xmlschema11-2/#identity). #[inline] + #[must_use] pub fn is_identical_with(self, other: Self) -> bool { self.timestamp.is_identical_with(other.timestamp) } @@ -696,6 +745,7 @@ impl GYearMonth { } #[inline] + #[must_use] pub fn from_be_bytes(bytes: [u8; 18]) -> Self { Self { timestamp: Timestamp::from_be_bytes(bytes), @@ -703,26 +753,31 @@ impl GYearMonth { } #[inline] + #[must_use] pub fn year(self) -> i64 { self.timestamp.year() } #[inline] + #[must_use] pub fn month(self) -> u8 { self.timestamp.month() } #[inline] + #[must_use] pub fn timezone(self) -> Option { Some(self.timezone_offset()?.into()) } #[inline] + #[must_use] pub fn timezone_offset(self) -> Option { self.timestamp.timezone_offset() } #[inline] + #[must_use] pub fn adjust(self, timezone_offset: Option) -> Option { Some(Self { timestamp: self.timestamp.adjust(timezone_offset)?, @@ -730,12 +785,14 @@ impl GYearMonth { } #[inline] + #[must_use] pub fn to_be_bytes(self) -> [u8; 18] { self.timestamp.to_be_bytes() } /// Checks if the two values are [identical](https://www.w3.org/TR/xmlschema11-2/#identity). #[inline] + #[must_use] pub fn is_identical_with(self, other: Self) -> bool { self.timestamp.is_identical_with(other.timestamp) } @@ -817,6 +874,7 @@ impl GYear { } #[inline] + #[must_use] pub fn from_be_bytes(bytes: [u8; 18]) -> Self { Self { timestamp: Timestamp::from_be_bytes(bytes), @@ -824,21 +882,25 @@ impl GYear { } #[inline] + #[must_use] pub fn year(self) -> i64 { self.timestamp.year() } #[inline] + #[must_use] pub fn timezone(self) -> Option { Some(self.timezone_offset()?.into()) } #[inline] + #[must_use] pub fn timezone_offset(self) -> Option { self.timestamp.timezone_offset() } #[inline] + #[must_use] pub fn adjust(self, timezone_offset: Option) -> Option { Some(Self { timestamp: self.timestamp.adjust(timezone_offset)?, @@ -846,12 +908,14 @@ impl GYear { } #[inline] + #[must_use] pub fn to_be_bytes(self) -> [u8; 18] { self.timestamp.to_be_bytes() } /// Checks if the two values are [identical](https://www.w3.org/TR/xmlschema11-2/#identity). #[inline] + #[must_use] pub fn is_identical_with(self, other: Self) -> bool { self.timestamp.is_identical_with(other.timestamp) } @@ -939,6 +1003,7 @@ impl GMonthDay { } #[inline] + #[must_use] pub fn from_be_bytes(bytes: [u8; 18]) -> Self { Self { timestamp: Timestamp::from_be_bytes(bytes), @@ -946,26 +1011,31 @@ impl GMonthDay { } #[inline] + #[must_use] pub fn month(&self) -> u8 { self.timestamp.month() } #[inline] + #[must_use] pub fn day(&self) -> u8 { self.timestamp.day() } #[inline] + #[must_use] pub fn timezone(&self) -> Option { Some(self.timezone_offset()?.into()) } #[inline] + #[must_use] pub fn timezone_offset(&self) -> Option { self.timestamp.timezone_offset() } #[inline] + #[must_use] pub fn adjust(&self, timezone_offset: Option) -> Option { Some(Self { timestamp: self.timestamp.adjust(timezone_offset)?, @@ -973,12 +1043,14 @@ impl GMonthDay { } #[inline] + #[must_use] pub fn to_be_bytes(self) -> [u8; 18] { self.timestamp.to_be_bytes() } /// Checks if the two values are [identical](https://www.w3.org/TR/xmlschema11-2/#identity). #[inline] + #[must_use] pub fn is_identical_with(self, other: Self) -> bool { self.timestamp.is_identical_with(other.timestamp) } @@ -1056,6 +1128,7 @@ impl GMonth { } #[inline] + #[must_use] pub fn from_be_bytes(bytes: [u8; 18]) -> Self { Self { timestamp: Timestamp::from_be_bytes(bytes), @@ -1063,21 +1136,25 @@ impl GMonth { } #[inline] + #[must_use] pub fn month(&self) -> u8 { self.timestamp.month() } #[inline] + #[must_use] pub fn timezone(&self) -> Option { Some(self.timezone_offset()?.into()) } #[inline] + #[must_use] pub fn timezone_offset(&self) -> Option { self.timestamp.timezone_offset() } #[inline] + #[must_use] pub fn adjust(&self, timezone_offset: Option) -> Option { Some(Self { timestamp: self.timestamp.adjust(timezone_offset)?, @@ -1085,12 +1162,14 @@ impl GMonth { } #[inline] + #[must_use] pub fn to_be_bytes(self) -> [u8; 18] { self.timestamp.to_be_bytes() } /// Checks if the two values are [identical](https://www.w3.org/TR/xmlschema11-2/#identity). #[inline] + #[must_use] pub fn is_identical_with(self, other: Self) -> bool { self.timestamp.is_identical_with(other.timestamp) } @@ -1182,6 +1261,7 @@ impl GDay { } #[inline] + #[must_use] pub fn from_be_bytes(bytes: [u8; 18]) -> Self { Self { timestamp: Timestamp::from_be_bytes(bytes), @@ -1189,21 +1269,25 @@ impl GDay { } #[inline] + #[must_use] pub fn day(&self) -> u8 { self.timestamp.day() } #[inline] + #[must_use] pub fn timezone(&self) -> Option { Some(self.timezone_offset()?.into()) } #[inline] + #[must_use] pub fn timezone_offset(&self) -> Option { self.timestamp.timezone_offset() } #[inline] + #[must_use] pub fn adjust(&self, timezone_offset: Option) -> Option { Some(Self { timestamp: self.timestamp.adjust(timezone_offset)?, @@ -1211,12 +1295,14 @@ impl GDay { } #[inline] + #[must_use] pub fn to_be_bytes(self) -> [u8; 18] { self.timestamp.to_be_bytes() } /// Checks if the two values are [identical](https://www.w3.org/TR/xmlschema11-2/#identity). #[inline] + #[must_use] pub fn is_identical_with(self, other: Self) -> bool { self.timestamp.is_identical_with(other.timestamp) } @@ -1293,6 +1379,7 @@ impl TimezoneOffset { } #[inline] + #[must_use] pub fn from_be_bytes(bytes: [u8; 2]) -> Self { Self { offset: i16::from_be_bytes(bytes), @@ -1300,6 +1387,7 @@ impl TimezoneOffset { } #[inline] + #[must_use] pub fn to_be_bytes(self) -> [u8; 2] { self.offset.to_be_bytes() } @@ -1486,6 +1574,7 @@ impl Timestamp { #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] #[inline] + #[must_use] fn year_month_day(&self) -> (i64, u8, u8) { let mut days = (self.value.as_i128() + i128::from(self.timezone_offset.unwrap_or(TimezoneOffset::UTC).offset) * 60) @@ -1537,18 +1626,21 @@ impl Timestamp { } #[inline] + #[must_use] fn year(&self) -> i64 { let (year, _, _) = self.year_month_day(); year } #[inline] + #[must_use] fn month(&self) -> u8 { let (_, month, _) = self.year_month_day(); month } #[inline] + #[must_use] fn day(&self) -> u8 { let (_, _, day) = self.year_month_day(); day @@ -1556,6 +1648,7 @@ impl Timestamp { #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] #[inline] + #[must_use] fn hour(&self) -> u8 { (((self.value.as_i128() + i128::from(self.timezone_offset.unwrap_or(TimezoneOffset::UTC).offset) * 60) @@ -1565,6 +1658,7 @@ impl Timestamp { #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] #[inline] + #[must_use] fn minute(&self) -> u8 { (((self.value.as_i128() + i128::from(self.timezone_offset.unwrap_or(TimezoneOffset::UTC).offset) * 60) @@ -1573,16 +1667,19 @@ impl Timestamp { } #[inline] + #[must_use] fn second(&self) -> Decimal { self.value.checked_rem_euclid(60).unwrap().abs() } #[inline] + #[must_use] const fn timezone_offset(&self) -> Option { self.timezone_offset } #[inline] + #[must_use] fn checked_add_seconds(&self, seconds: impl Into) -> Option { Some(Self { value: self.value.checked_add(seconds.into())?, @@ -1591,6 +1688,7 @@ impl Timestamp { } #[inline] + #[must_use] fn checked_sub(&self, rhs: Self) -> Option { match (self.timezone_offset, rhs.timezone_offset) { (Some(_), Some(_)) | (None, None) => { @@ -1601,6 +1699,7 @@ impl Timestamp { } #[inline] + #[must_use] fn checked_sub_seconds(&self, seconds: Decimal) -> Option { Some(Self { value: self.value.checked_sub(seconds)?, @@ -1609,6 +1708,7 @@ impl Timestamp { } #[inline] + #[must_use] fn adjust(&self, timezone_offset: Option) -> Option { Some(if let Some(from_timezone) = self.timezone_offset { if let Some(to_timezone) = timezone_offset { @@ -1638,6 +1738,7 @@ impl Timestamp { } #[inline] + #[must_use] fn to_be_bytes(self) -> [u8; 18] { let mut bytes = [0; 18]; bytes[0..16].copy_from_slice(&self.value.to_be_bytes()); @@ -1650,6 +1751,7 @@ impl Timestamp { /// Checks if the two values are [identical](https://www.w3.org/TR/xmlschema11-2/#identity). #[inline] + #[must_use] pub fn is_identical_with(self, other: Self) -> bool { self.value == other.value && self.timezone_offset == other.timezone_offset } diff --git a/lib/oxsdatatypes/src/decimal.rs b/lib/oxsdatatypes/src/decimal.rs index 412ca2b8..f3880dcb 100644 --- a/lib/oxsdatatypes/src/decimal.rs +++ b/lib/oxsdatatypes/src/decimal.rs @@ -33,6 +33,7 @@ impl Decimal { } #[inline] + #[must_use] pub fn from_be_bytes(bytes: [u8; 16]) -> Self { Self { value: i128::from_be_bytes(bytes), @@ -40,12 +41,14 @@ impl Decimal { } #[inline] + #[must_use] pub fn to_be_bytes(self) -> [u8; 16] { self.value.to_be_bytes() } /// [op:numeric-add](https://www.w3.org/TR/xpath-functions-31/#func-numeric-add) #[inline] + #[must_use] pub fn checked_add(self, rhs: impl Into) -> Option { Some(Self { value: self.value.checked_add(rhs.into().value)?, @@ -54,6 +57,7 @@ impl Decimal { /// [op:numeric-subtract](https://www.w3.org/TR/xpath-functions-31/#func-numeric-subtract) #[inline] + #[must_use] pub fn checked_sub(self, rhs: impl Into) -> Option { Some(Self { value: self.value.checked_sub(rhs.into().value)?, @@ -62,6 +66,7 @@ impl Decimal { /// [op:numeric-multiply](https://www.w3.org/TR/xpath-functions-31/#func-numeric-multiply) #[inline] + #[must_use] pub fn checked_mul(self, rhs: impl Into) -> Option { // Idea: we shift right as much as possible to keep as much precision as possible // Do the multiplication and do the required left shift @@ -94,6 +99,7 @@ impl Decimal { /// [op:numeric-divide](https://www.w3.org/TR/xpath-functions-31/#func-numeric-divide) #[inline] + #[must_use] pub fn checked_div(self, rhs: impl Into) -> Option { // Idea: we shift the dividend left as much as possible to keep as much precision as possible // And we shift right the divisor as much as possible @@ -127,6 +133,7 @@ impl Decimal { /// [op:numeric-mod](https://www.w3.org/TR/xpath-functions-31/#func-numeric-mod) #[inline] + #[must_use] pub fn checked_rem(self, rhs: impl Into) -> Option { Some(Self { value: self.value.checked_rem(rhs.into().value)?, @@ -134,6 +141,7 @@ impl Decimal { } #[inline] + #[must_use] pub fn checked_rem_euclid(self, rhs: impl Into) -> Option { Some(Self { value: self.value.checked_rem_euclid(rhs.into().value)?, @@ -142,6 +150,7 @@ impl Decimal { /// [op:numeric-unary-minus](https://www.w3.org/TR/xpath-functions-31/#func-numeric-unary-minus) #[inline] + #[must_use] pub fn checked_neg(self) -> Option { Some(Self { value: self.value.checked_neg()?, @@ -150,6 +159,7 @@ impl Decimal { /// [fn:abs](https://www.w3.org/TR/xpath-functions-31/#func-abs) #[inline] + #[must_use] pub const fn abs(self) -> Self { Self { value: self.value.abs(), @@ -158,6 +168,7 @@ impl Decimal { /// [fn:round](https://www.w3.org/TR/xpath-functions-31/#func-round) #[inline] + #[must_use] pub fn round(self) -> Self { let value = self.value / DECIMAL_PART_POW_MINUS_ONE; Self { @@ -171,6 +182,7 @@ impl Decimal { /// [fn:ceiling](https://www.w3.org/TR/xpath-functions-31/#func-ceiling) #[inline] + #[must_use] pub fn ceil(self) -> Self { Self { value: if self.value >= 0 && self.value % DECIMAL_PART_POW != 0 { @@ -183,6 +195,7 @@ impl Decimal { /// [fn:floor](https://www.w3.org/TR/xpath-functions-31/#func-floor) #[inline] + #[must_use] pub fn floor(self) -> Self { Self { value: if self.value >= 0 || self.value % DECIMAL_PART_POW == 0 { @@ -194,22 +207,26 @@ impl Decimal { } #[inline] + #[must_use] pub const fn is_negative(self) -> bool { self.value < 0 } #[inline] + #[must_use] pub const fn is_positive(self) -> bool { self.value > 0 } /// Checks if the two values are [identical](https://www.w3.org/TR/xmlschema11-2/#identity). #[inline] + #[must_use] pub fn is_identical_with(self, other: Self) -> bool { self == other } #[inline] + #[must_use] pub(super) const fn as_i128(self) -> i128 { self.value / DECIMAL_PART_POW } diff --git a/lib/oxsdatatypes/src/double.rs b/lib/oxsdatatypes/src/double.rs index 6768c772..b392d9cf 100644 --- a/lib/oxsdatatypes/src/double.rs +++ b/lib/oxsdatatypes/src/double.rs @@ -18,6 +18,7 @@ pub struct Double { impl Double { #[inline] + #[must_use] pub fn from_be_bytes(bytes: [u8; 8]) -> Self { Self { value: f64::from_be_bytes(bytes), @@ -25,46 +26,54 @@ impl Double { } #[inline] + #[must_use] pub fn to_be_bytes(self) -> [u8; 8] { self.value.to_be_bytes() } /// [fn:abs](https://www.w3.org/TR/xpath-functions-31/#func-abs) #[inline] + #[must_use] pub fn abs(self) -> Self { self.value.abs().into() } /// [fn:ceiling](https://www.w3.org/TR/xpath-functions-31/#func-ceiling) #[inline] + #[must_use] pub fn ceil(self) -> Self { self.value.ceil().into() } /// [fn:floor](https://www.w3.org/TR/xpath-functions-31/#func-floor) #[inline] + #[must_use] pub fn floor(self) -> Self { self.value.floor().into() } /// [fn:round](https://www.w3.org/TR/xpath-functions-31/#func-round) #[inline] + #[must_use] pub fn round(self) -> Self { self.value.round().into() } #[inline] + #[must_use] pub fn is_nan(self) -> bool { self.value.is_nan() } #[inline] + #[must_use] pub fn is_finite(self) -> bool { self.value.is_finite() } /// Checks if the two values are [identical](https://www.w3.org/TR/xmlschema11-2/#identity). #[inline] + #[must_use] pub fn is_identical_with(self, other: Self) -> bool { self.value.to_ne_bytes() == other.value.to_ne_bytes() } diff --git a/lib/oxsdatatypes/src/duration.rs b/lib/oxsdatatypes/src/duration.rs index 29167121..e2d9de95 100644 --- a/lib/oxsdatatypes/src/duration.rs +++ b/lib/oxsdatatypes/src/duration.rs @@ -17,6 +17,7 @@ pub struct Duration { impl Duration { #[inline] + #[must_use] pub fn new(months: impl Into, seconds: impl Into) -> Self { Self { year_month: YearMonthDuration::new(months), @@ -25,6 +26,7 @@ impl Duration { } #[inline] + #[must_use] pub fn from_be_bytes(bytes: [u8; 24]) -> Self { Self { year_month: YearMonthDuration::from_be_bytes(bytes[0..8].try_into().unwrap()), @@ -34,51 +36,60 @@ impl Duration { /// [fn:years-from-duration](https://www.w3.org/TR/xpath-functions-31/#func-years-from-duration) #[inline] + #[must_use] pub fn years(self) -> i64 { self.year_month.years() } /// [fn:months-from-duration](https://www.w3.org/TR/xpath-functions-31/#func-months-from-duration) #[inline] + #[must_use] pub fn months(self) -> i64 { self.year_month.months() } /// [fn:days-from-duration](https://www.w3.org/TR/xpath-functions-31/#func-days-from-duration) #[inline] + #[must_use] pub fn days(self) -> i64 { self.day_time.days() } /// [fn:hours-from-duration](https://www.w3.org/TR/xpath-functions-31/#func-hours-from-duration) #[inline] + #[must_use] pub fn hours(self) -> i64 { self.day_time.hours() } /// [fn:minutes-from-duration](https://www.w3.org/TR/xpath-functions-31/#func-minutes-from-duration) #[inline] + #[must_use] pub fn minutes(self) -> i64 { self.day_time.minutes() } /// [fn:seconds-from-duration](https://www.w3.org/TR/xpath-functions-31/#func-seconds-from-duration) #[inline] + #[must_use] pub fn seconds(self) -> Decimal { self.day_time.seconds() } #[inline] + #[must_use] pub(super) const fn all_months(self) -> i64 { self.year_month.all_months() } #[inline] + #[must_use] pub(super) const fn all_seconds(self) -> Decimal { self.day_time.all_seconds() } #[inline] + #[must_use] pub fn to_be_bytes(self) -> [u8; 24] { let mut bytes = [0; 24]; bytes[0..8].copy_from_slice(&self.year_month.to_be_bytes()); @@ -88,6 +99,7 @@ impl Duration { /// [op:add-yearMonthDurations](https://www.w3.org/TR/xpath-functions-31/#func-add-yearMonthDurations) and [op:add-dayTimeDurations](https://www.w3.org/TR/xpath-functions-31/#func-add-dayTimeDurations) #[inline] + #[must_use] pub fn checked_add(self, rhs: impl Into) -> Option { let rhs = rhs.into(); Some(Self { @@ -98,6 +110,7 @@ impl Duration { /// [op:subtract-yearMonthDurations](https://www.w3.org/TR/xpath-functions-31/#func-subtract-yearMonthDurations) and [op:subtract-dayTimeDurations](https://www.w3.org/TR/xpath-functions-31/#func-subtract-dayTimeDurations) #[inline] + #[must_use] pub fn checked_sub(self, rhs: impl Into) -> Option { let rhs = rhs.into(); Some(Self { @@ -107,6 +120,7 @@ impl Duration { } #[inline] + #[must_use] pub fn checked_neg(self) -> Option { Some(Self { year_month: self.year_month.checked_neg()?, @@ -116,6 +130,7 @@ impl Duration { /// Checks if the two values are [identical](https://www.w3.org/TR/xmlschema11-2/#identity). #[inline] + #[must_use] pub fn is_identical_with(self, other: Self) -> bool { self == other } diff --git a/lib/oxsdatatypes/src/float.rs b/lib/oxsdatatypes/src/float.rs index 8c8602d3..4de94913 100644 --- a/lib/oxsdatatypes/src/float.rs +++ b/lib/oxsdatatypes/src/float.rs @@ -18,6 +18,7 @@ pub struct Float { impl Float { #[inline] + #[must_use] pub fn from_be_bytes(bytes: [u8; 4]) -> Self { Self { value: f32::from_be_bytes(bytes), @@ -25,46 +26,54 @@ impl Float { } #[inline] + #[must_use] pub fn to_be_bytes(self) -> [u8; 4] { self.value.to_be_bytes() } /// [fn:abs](https://www.w3.org/TR/xpath-functions-31/#func-abs) #[inline] + #[must_use] pub fn abs(self) -> Self { self.value.abs().into() } /// [fn:ceiling](https://www.w3.org/TR/xpath-functions-31/#func-ceiling) #[inline] + #[must_use] pub fn ceil(self) -> Self { self.value.ceil().into() } /// [fn:floor](https://www.w3.org/TR/xpath-functions-31/#func-floor) #[inline] + #[must_use] pub fn floor(self) -> Self { self.value.floor().into() } /// [fn:round](https://www.w3.org/TR/xpath-functions-31/#func-round) #[inline] + #[must_use] pub fn round(self) -> Self { self.value.round().into() } #[inline] + #[must_use] pub fn is_nan(self) -> bool { self.value.is_nan() } #[inline] + #[must_use] pub fn is_finite(self) -> bool { self.value.is_finite() } /// Checks if the two values are [identical](https://www.w3.org/TR/xmlschema11-2/#identity). #[inline] + #[must_use] pub fn is_identical_with(self, other: Self) -> bool { self.value.to_ne_bytes() == other.value.to_ne_bytes() } diff --git a/lib/oxsdatatypes/src/integer.rs b/lib/oxsdatatypes/src/integer.rs index 957a4b24..f376a57d 100644 --- a/lib/oxsdatatypes/src/integer.rs +++ b/lib/oxsdatatypes/src/integer.rs @@ -14,6 +14,7 @@ pub struct Integer { impl Integer { #[inline] + #[must_use] pub fn from_be_bytes(bytes: [u8; 8]) -> Self { Self { value: i64::from_be_bytes(bytes), @@ -21,12 +22,14 @@ impl Integer { } #[inline] + #[must_use] pub fn to_be_bytes(self) -> [u8; 8] { self.value.to_be_bytes() } /// [op:numeric-add](https://www.w3.org/TR/xpath-functions-31/#func-numeric-add) #[inline] + #[must_use] pub fn checked_add(self, rhs: impl Into) -> Option { Some(Self { value: self.value.checked_add(rhs.into().value)?, @@ -35,6 +38,7 @@ impl Integer { /// [op:numeric-subtract](https://www.w3.org/TR/xpath-functions-31/#func-numeric-subtract) #[inline] + #[must_use] pub fn checked_sub(self, rhs: impl Into) -> Option { Some(Self { value: self.value.checked_sub(rhs.into().value)?, @@ -43,6 +47,7 @@ impl Integer { /// [op:numeric-multiply](https://www.w3.org/TR/xpath-functions-31/#func-numeric-multiply) #[inline] + #[must_use] pub fn checked_mul(self, rhs: impl Into) -> Option { Some(Self { value: self.value.checked_mul(rhs.into().value)?, @@ -51,6 +56,7 @@ impl Integer { /// [op:numeric-integer-divide](https://www.w3.org/TR/xpath-functions-31/#func-numeric-integer-divide) #[inline] + #[must_use] pub fn checked_div(self, rhs: impl Into) -> Option { Some(Self { value: self.value.checked_div(rhs.into().value)?, @@ -59,6 +65,7 @@ impl Integer { /// [op:numeric-mod](https://www.w3.org/TR/xpath-functions-31/#func-numeric-mod) #[inline] + #[must_use] pub fn checked_rem(self, rhs: impl Into) -> Option { Some(Self { value: self.value.checked_rem(rhs.into().value)?, @@ -66,6 +73,7 @@ impl Integer { } #[inline] + #[must_use] pub fn checked_rem_euclid(self, rhs: impl Into) -> Option { Some(Self { value: self.value.checked_rem_euclid(rhs.into().value)?, @@ -74,6 +82,7 @@ impl Integer { /// [op:numeric-unary-minus](https://www.w3.org/TR/xpath-functions-31/#func-numeric-unary-minus) #[inline] + #[must_use] pub fn checked_neg(self) -> Option { Some(Self { value: self.value.checked_neg()?, @@ -82,6 +91,7 @@ impl Integer { /// [fn:abs](https://www.w3.org/TR/xpath-functions-31/#func-abs) #[inline] + #[must_use] pub const fn abs(self) -> Self { Self { value: self.value.abs(), @@ -89,17 +99,20 @@ impl Integer { } #[inline] + #[must_use] pub const fn is_negative(self) -> bool { self.value < 0 } #[inline] + #[must_use] pub const fn is_positive(self) -> bool { self.value > 0 } /// Checks if the two values are [identical](https://www.w3.org/TR/xmlschema11-2/#identity). #[inline] + #[must_use] pub fn is_identical_with(self, other: Self) -> bool { self == other } diff --git a/lib/oxsdatatypes/src/lib.rs b/lib/oxsdatatypes/src/lib.rs index 67737b13..6e1cd28f 100644 --- a/lib/oxsdatatypes/src/lib.rs +++ b/lib/oxsdatatypes/src/lib.rs @@ -3,7 +3,6 @@ #![cfg_attr(docsrs, feature(doc_auto_cfg))] #![doc(html_favicon_url = "https://raw.githubusercontent.com/oxigraph/oxigraph/main/logo.svg")] #![doc(html_logo_url = "https://raw.githubusercontent.com/oxigraph/oxigraph/main/logo.svg")] -#![allow(clippy::return_self_not_must_use)] mod boolean; mod date_time; diff --git a/lib/src/lib.rs b/lib/src/lib.rs index 29ef24ae..d5578fb2 100644 --- a/lib/src/lib.rs +++ b/lib/src/lib.rs @@ -3,7 +3,6 @@ #![doc(html_logo_url = "https://raw.githubusercontent.com/oxigraph/oxigraph/main/logo.svg")] #![cfg_attr(docsrs, feature(doc_auto_cfg))] #![doc(test(attr(deny(warnings))))] -#![allow(clippy::return_self_not_must_use)] pub mod io; pub mod sparql; diff --git a/lib/src/storage/mod.rs b/lib/src/storage/mod.rs index f1c81414..6591ac47 100644 --- a/lib/src/storage/mod.rs +++ b/lib/src/storage/mod.rs @@ -1212,16 +1212,19 @@ impl StorageBulkLoader { } } + #[must_use] pub fn set_num_threads(mut self, num_threads: usize) -> Self { self.num_threads = Some(num_threads); self } + #[must_use] pub fn set_max_memory_size_in_megabytes(mut self, max_memory_size: usize) -> Self { self.max_memory_size = Some(max_memory_size); self } + #[must_use] pub fn on_progress(mut self, callback: impl Fn(u64) + 'static) -> Self { self.hooks.push(Box::new(callback)); self diff --git a/lib/src/store.rs b/lib/src/store.rs index fc6a83f1..beb8cfd1 100644 --- a/lib/src/store.rs +++ b/lib/src/store.rs @@ -1390,6 +1390,7 @@ impl BulkLoader { /// This number must be at last 2 (one for parsing and one for loading). /// /// The default value is 2. + #[must_use] pub fn set_num_threads(mut self, num_threads: usize) -> Self { self.storage = self.storage.set_num_threads(num_threads); self @@ -1404,6 +1405,7 @@ impl BulkLoader { /// (for example if the data contains very long IRIs or literals). /// /// By default, a target 2GB per used thread is used. + #[must_use] pub fn set_max_memory_size_in_megabytes(mut self, max_memory_size: usize) -> Self { self.storage = self .storage @@ -1412,6 +1414,7 @@ impl BulkLoader { } /// Adds a `callback` evaluated from time to time with the number of loaded triples. + #[must_use] pub fn on_progress(mut self, callback: impl Fn(u64) + 'static) -> Self { self.storage = self.storage.on_progress(callback); self @@ -1421,6 +1424,7 @@ impl BulkLoader { /// by returning `Ok` or fail by returning `Err`. /// /// By default the parsing fails. + #[must_use] pub fn on_parse_error( mut self, callback: impl Fn(ParseError) -> Result<(), ParseError> + 'static, From 98ac08998440037ccfdea5510786e6f748c8598d Mon Sep 17 00:00:00 2001 From: Tpt Date: Sun, 25 Jun 2023 13:30:26 +0200 Subject: [PATCH 022/217] Adds TryFrom to std::time::Duration --- lib/oxsdatatypes/src/duration.rs | 36 ++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/lib/oxsdatatypes/src/duration.rs b/lib/oxsdatatypes/src/duration.rs index e2d9de95..4815eb16 100644 --- a/lib/oxsdatatypes/src/duration.rs +++ b/lib/oxsdatatypes/src/duration.rs @@ -522,6 +522,34 @@ impl TryFrom for DayTimeDuration { } } +impl TryFrom for StdDuration { + type Error = DecimalOverflowError; + + #[inline] + fn try_from(value: DayTimeDuration) -> Result { + if value.seconds.is_negative() { + return Err(DecimalOverflowError); + } + let secs = value.seconds.floor(); + let nanos = value + .seconds + .checked_sub(secs) + .ok_or(DecimalOverflowError)? + .checked_mul(1_000_000_000) + .ok_or(DecimalOverflowError)? + .floor(); + Ok(StdDuration::new( + secs.as_i128() + .try_into() + .map_err(|_| DecimalOverflowError)?, + nanos + .as_i128() + .try_into() + .map_err(|_| DecimalOverflowError)?, + )) + } +} + impl FromStr for DayTimeDuration { type Err = XsdParseError; @@ -647,6 +675,14 @@ mod tests { ); } + #[test] + fn to_std() -> Result<(), XsdParseError> { + let duration = StdDuration::try_from(DayTimeDuration::from_str("PT10.00000001S")?).unwrap(); + assert_eq!(duration.as_secs(), 10); + assert_eq!(duration.subsec_nanos(), 10); + Ok(()) + } + #[test] fn equals() -> Result<(), XsdParseError> { assert_eq!( From 94986a0d28381565c9766a92af9c92f36df18ea2 Mon Sep 17 00:00:00 2001 From: Thomas Date: Tue, 4 Jul 2023 17:29:43 +0200 Subject: [PATCH 023/217] Fixes a testsuite typo (cherry picked from commit b69e0d38f68edb2a5cb07adebbe17ef9898d52b5) --- testsuite/oxigraph-tests/sparql-results/manifest.ttl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testsuite/oxigraph-tests/sparql-results/manifest.ttl b/testsuite/oxigraph-tests/sparql-results/manifest.ttl index 55b4f7df..9993b63a 100644 --- a/testsuite/oxigraph-tests/sparql-results/manifest.ttl +++ b/testsuite/oxigraph-tests/sparql-results/manifest.ttl @@ -5,7 +5,7 @@ @prefix ox: . <> rdf:type mf:Manifest ; - rdfs:label "Oxigraph SPARQL resutls tests" ; + rdfs:label "Oxigraph SPARQL results tests" ; mf:entries ( :results_json_duplicated_variables From 69d8ce6b4e87e5d8df48aa63c62bb27e276d3746 Mon Sep 17 00:00:00 2001 From: Tpt Date: Sat, 24 Jun 2023 15:32:50 +0200 Subject: [PATCH 024/217] Migrates RDF/XML parser from Rio --- .clusterfuzzlite/build.sh | 4 +- .github/workflows/tests.yml | 6 +- Cargo.lock | 29 +- Cargo.toml | 1 + fuzz/Cargo.toml | 5 + fuzz/fuzz_targets/rdf_xml.rs | 37 ++ lib/Cargo.toml | 3 +- lib/oxrdf/src/blank_node.rs | 9 +- lib/oxrdfxml/Cargo.toml | 23 + lib/oxrdfxml/README.md | 52 ++ lib/oxrdfxml/src/error.rs | 107 ++++ lib/oxrdfxml/src/lib.rs | 14 + lib/oxrdfxml/src/parser.rs | 1081 ++++++++++++++++++++++++++++++++ lib/oxrdfxml/src/serializer.rs | 229 +++++++ lib/oxrdfxml/src/utils.rs | 26 + lib/sparesults/Cargo.toml | 2 +- lib/sparopt/src/lib.rs | 6 + lib/src/io/error.rs | 2 +- lib/src/io/read.rs | 113 +--- lib/src/io/write.rs | 71 +-- lib/src/sparql/model.rs | 2 +- lib/src/store.rs | 2 +- python/src/io.rs | 4 +- server/src/main.rs | 17 +- 24 files changed, 1657 insertions(+), 188 deletions(-) create mode 100644 fuzz/fuzz_targets/rdf_xml.rs create mode 100644 lib/oxrdfxml/Cargo.toml create mode 100644 lib/oxrdfxml/README.md create mode 100644 lib/oxrdfxml/src/error.rs create mode 100644 lib/oxrdfxml/src/lib.rs create mode 100644 lib/oxrdfxml/src/parser.rs create mode 100644 lib/oxrdfxml/src/serializer.rs create mode 100644 lib/oxrdfxml/src/utils.rs diff --git a/.clusterfuzzlite/build.sh b/.clusterfuzzlite/build.sh index 698dd187..44671ce3 100755 --- a/.clusterfuzzlite/build.sh +++ b/.clusterfuzzlite/build.sh @@ -15,7 +15,7 @@ function build_seed_corpus() { cd "$SRC"/oxigraph cargo fuzz build -O --debug-assertions -for TARGET in sparql_eval sparql_results_json sparql_results_tsv n3 nquads trig # sparql_results_xml https://github.com/tafia/quick-xml/issues/608 +for TARGET in sparql_eval sparql_results_json sparql_results_tsv sparql_results_xml n3 nquads trig rdf_xml do cp fuzz/target/x86_64-unknown-linux-gnu/release/$TARGET "$OUT"/ done @@ -25,4 +25,4 @@ build_seed_corpus sparql_results_xml srx build_seed_corpus n3 n3 build_seed_corpus nquads nq build_seed_corpus trig trig - +build_seed_corpus rdf_xml rdf diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 106fba84..7098e3fd 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -32,6 +32,8 @@ jobs: working-directory: ./lib/oxsdatatypes - run: cargo clippy working-directory: ./lib/oxrdf + - run: cargo clippy + working-directory: ./lib/oxrdfxml - run: cargo clippy working-directory: ./lib/oxttl - run: cargo clippy @@ -76,6 +78,8 @@ jobs: working-directory: ./lib/oxsdatatypes - run: cargo clippy -- -D warnings -D clippy::all working-directory: ./lib/oxrdf + - run: cargo clippy -- -D warnings -D clippy::all + working-directory: ./lib/oxrdfxml - run: cargo clippy -- -D warnings -D clippy::all working-directory: ./lib/oxttl - run: cargo clippy -- -D warnings -D clippy::all @@ -127,7 +131,7 @@ jobs: - run: rustup update - uses: Swatinem/rust-cache@v2 - run: cargo install cargo-semver-checks || true - - run: cargo semver-checks check-release --exclude oxrocksdb-sys --exclude oxigraph_js --exclude pyoxigraph --exclude oxigraph_testsuite --exclude oxigraph_server --exclude oxttl --exclude sparopt + - run: cargo semver-checks check-release --exclude oxrocksdb-sys --exclude oxigraph_js --exclude pyoxigraph --exclude oxigraph_testsuite --exclude oxigraph_server --exclude oxrdfxml --exclude oxttl --exclude sparopt test_linux: runs-on: ubuntu-latest diff --git a/Cargo.lock b/Cargo.lock index affef9d4..d2568bac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -946,13 +946,12 @@ dependencies = [ "oxilangtag", "oxiri", "oxrdf", + "oxrdfxml", "oxrocksdb-sys", "oxsdatatypes", "oxttl", "rand", "regex", - "rio_api", - "rio_xml", "sha-1", "sha2", "siphasher", @@ -1031,6 +1030,16 @@ dependencies = [ "rand", ] +[[package]] +name = "oxrdfxml" +version = "0.1.0-alpha.1-dev" +dependencies = [ + "oxilangtag", + "oxiri", + "oxrdf", + "quick-xml", +] + [[package]] name = "oxrocksdb-sys" version = "0.4.0-alpha.1-dev" @@ -1279,9 +1288,9 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.28.2" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ce5e73202a820a31f8a0ee32ada5e21029c81fd9e3ebf668a40832e4219d9d1" +checksum = "81b9228215d82c7b61490fec1de287136b5de6f5700f6e58ea9ad61a7964ca51" dependencies = [ "memchr", ] @@ -1411,18 +1420,6 @@ dependencies = [ "rio_api", ] -[[package]] -name = "rio_xml" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2edda57b877119dc326c612ba822e3ca1ee22bfc86781a4e9dc0884756b58c3" -dependencies = [ - "oxilangtag", - "oxiri", - "quick-xml", - "rio_api", -] - [[package]] name = "rustc-hash" version = "1.1.0" diff --git a/Cargo.toml b/Cargo.toml index 0da9b1c9..0ad8536d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,7 @@ members = [ "js", "lib", "lib/oxrdf", + "lib/oxrdfxml", "lib/oxsdatatypes", "lib/oxttl", "lib/spargebra", diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index 216ba902..8bef5528 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -13,6 +13,7 @@ anyhow = "1" lazy_static = "1" libfuzzer-sys = "0.4" oxttl = { path = "../lib/oxttl", features = ["rdf-star"] } +oxrdfxml = { path = "../lib/oxrdfxml" } spargebra = { path = "../lib/spargebra", features = ["rdf-star", "sep-0006"] } sparesults = { path = "../lib/sparesults", features = ["rdf-star"] } sparql-smith = { path = "../lib/sparql-smith", features = ["sep-0006"] } @@ -32,6 +33,10 @@ path = "fuzz_targets/nquads.rs" name = "n3" path = "fuzz_targets/n3.rs" +[[bin]] +name = "rdf_xml" +path = "fuzz_targets/rdf_xml.rs" + [[bin]] name = "sparql_eval" path = "fuzz_targets/sparql_eval.rs" diff --git a/fuzz/fuzz_targets/rdf_xml.rs b/fuzz/fuzz_targets/rdf_xml.rs new file mode 100644 index 00000000..ae0cb6b1 --- /dev/null +++ b/fuzz/fuzz_targets/rdf_xml.rs @@ -0,0 +1,37 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; +use oxrdfxml::{RdfXmlParser, RdfXmlSerializer}; + +fuzz_target!(|data: &[u8]| { + // We parse + let mut triples = Vec::new(); + for triple in RdfXmlParser::new().parse_from_read(data) { + if let Ok(triple) = triple { + triples.push(triple); + } + } + + // We serialize + let mut writer = RdfXmlSerializer::new().serialize_to_write(Vec::new()); + for triple in &triples { + writer.write_triple(triple).unwrap(); + } + let new_serialization = writer.finish().unwrap(); + + // We parse the serialization + let new_triples = RdfXmlParser::new() + .parse_from_read(new_serialization.as_slice()) + .collect::, _>>() + .map_err(|e| { + format!( + "Error on {:?} from {triples:?} based on {:?}: {e}", + String::from_utf8_lossy(&new_serialization), + String::from_utf8_lossy(data) + ) + }) + .unwrap(); + + // We check the roundtrip has not changed anything + assert_eq!(new_triples, triples); +}); diff --git a/lib/Cargo.toml b/lib/Cargo.toml index 9c3fb465..f4d3cae1 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -31,13 +31,12 @@ digest = "0.10" regex = "1" oxilangtag = "0.1" oxiri = "0.2" -rio_api = "0.8" -rio_xml = "0.8" hex = "0.4" siphasher = "0.3" lazy_static = "1" json-event-parser = "0.1" oxrdf = { version = "0.2.0-alpha.1-dev", path = "oxrdf", features = ["rdf-star", "oxsdatatypes"] } +oxrdfxml = { version = "0.1.0-alpha.1-dev", path = "oxrdfxml" } oxsdatatypes = { version = "0.2.0-alpha.1-dev", path="oxsdatatypes" } oxttl = { version = "0.1.0-alpha.1-dev" , path = "oxttl", features = ["rdf-star"] } spargebra = { version = "0.3.0-alpha.1-dev", path = "spargebra", features = ["rdf-star", "sep-0002", "sep-0006"] } diff --git a/lib/oxrdf/src/blank_node.rs b/lib/oxrdf/src/blank_node.rs index 0b485beb..bfd27231 100644 --- a/lib/oxrdf/src/blank_node.rs +++ b/lib/oxrdf/src/blank_node.rs @@ -111,7 +111,14 @@ impl Default for BlankNode { /// Builds a new RDF [blank node](https://www.w3.org/TR/rdf11-concepts/#dfn-blank-node) with a unique id. #[inline] fn default() -> Self { - Self::new_from_unique_id(random::()) + // We ensure the ID does not start with a number to be also valid with RDF/XML + loop { + let id = random(); + let str = IdStr::new(id); + if matches!(str.as_str().as_bytes().first(), Some(b'a'..=b'f')) { + return Self(BlankNodeContent::Anonymous { id, str }); + } + } } } diff --git a/lib/oxrdfxml/Cargo.toml b/lib/oxrdfxml/Cargo.toml new file mode 100644 index 00000000..25e19c72 --- /dev/null +++ b/lib/oxrdfxml/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "oxrdfxml" +version = "0.1.0-alpha.1-dev" +authors = ["Tpt "] +license = "MIT OR Apache-2.0" +readme = "README.md" +keywords = ["RDF/XML", "RDF"] +repository = "https://github.com/oxigraph/oxigraph/tree/master/lib/oxrdfxml" +homepage = "https://oxigraph.org/" +description = """ +Parser for the RDF/XML language +""" +edition = "2021" +rust-version = "1.65" + +[dependencies] +oxrdf = { version = "0.2.0-alpha.1-dev", path = "../oxrdf" } +oxilangtag = "0.1" +oxiri = "0.2" +quick-xml = "0.29" + +[package.metadata.docs.rs] +all-features = true diff --git a/lib/oxrdfxml/README.md b/lib/oxrdfxml/README.md new file mode 100644 index 00000000..66f9f563 --- /dev/null +++ b/lib/oxrdfxml/README.md @@ -0,0 +1,52 @@ +OxRDF/XML +========= + +[![Latest Version](https://img.shields.io/crates/v/oxrdfxml.svg)](https://crates.io/crates/oxrdfxml) +[![Released API docs](https://docs.rs/oxrdfxml/badge.svg)](https://docs.rs/oxrdfxml) +[![Crates.io downloads](https://img.shields.io/crates/d/oxrdfxml)](https://crates.io/crates/oxrdfxml) +[![actions status](https://github.com/oxigraph/oxigraph/workflows/build/badge.svg)](https://github.com/oxigraph/oxigraph/actions) +[![Gitter](https://badges.gitter.im/oxigraph/community.svg)](https://gitter.im/oxigraph/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) + +OxRdfXml is a parser and serializer for [RDF/XML](https://www.w3.org/TR/rdf-syntax-grammar/). + +Usage example counting the number of people in a RDF/XML file: +```rust +use oxrdf::{NamedNodeRef, vocab::rdf}; +use oxrdfxml::RdfXmlParser; + +let file = b" + + + + Foo + + +"; + +let schema_person = NamedNodeRef::new("http://schema.org/Person").unwrap(); +let mut count = 0; +for triple in RdfXmlParser::new().parse_from_read(file.as_ref()) { + let triple = triple.unwrap(); + if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { + count += 1; + } +} +assert_eq!(2, count); +``` + + +## License + +This project is licensed under either of + +* Apache License, Version 2.0, ([LICENSE-APACHE](../LICENSE-APACHE) or + ``) +* MIT license ([LICENSE-MIT](../LICENSE-MIT) or + ``) + +at your option. + + +### Contribution + +Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in Oxigraph by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. diff --git a/lib/oxrdfxml/src/error.rs b/lib/oxrdfxml/src/error.rs new file mode 100644 index 00000000..413d3c4a --- /dev/null +++ b/lib/oxrdfxml/src/error.rs @@ -0,0 +1,107 @@ +use oxilangtag::LanguageTagParseError; +use oxiri::IriParseError; +use std::error::Error; +use std::sync::Arc; +use std::{fmt, io}; + +/// Error that might be returned during parsing. +/// +/// It might wrap an IO error or be a parsing error. +#[derive(Debug)] +pub struct RdfXmlError { + pub(crate) kind: RdfXmlErrorKind, +} + +#[derive(Debug)] +pub(crate) enum RdfXmlErrorKind { + Xml(quick_xml::Error), + XmlAttribute(quick_xml::events::attributes::AttrError), + InvalidIri { + iri: String, + error: IriParseError, + }, + InvalidLanguageTag { + tag: String, + error: LanguageTagParseError, + }, + Other(String), +} + +impl RdfXmlError { + pub(crate) fn msg(msg: impl Into) -> Self { + Self { + kind: RdfXmlErrorKind::Other(msg.into()), + } + } +} + +impl fmt::Display for RdfXmlError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match &self.kind { + RdfXmlErrorKind::Xml(error) => error.fmt(f), + RdfXmlErrorKind::XmlAttribute(error) => error.fmt(f), + RdfXmlErrorKind::InvalidIri { iri, error } => { + write!(f, "error while parsing IRI '{}': {}", iri, error) + } + RdfXmlErrorKind::InvalidLanguageTag { tag, error } => { + write!(f, "error while parsing language tag '{}': {}", tag, error) + } + RdfXmlErrorKind::Other(message) => write!(f, "{}", message), + } + } +} + +impl Error for RdfXmlError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match &self.kind { + RdfXmlErrorKind::Xml(error) => Some(error), + RdfXmlErrorKind::XmlAttribute(error) => Some(error), + RdfXmlErrorKind::InvalidIri { error, .. } => Some(error), + RdfXmlErrorKind::InvalidLanguageTag { error, .. } => Some(error), + RdfXmlErrorKind::Other(_) => None, + } + } +} + +impl From for RdfXmlError { + fn from(error: quick_xml::Error) -> Self { + Self { + kind: RdfXmlErrorKind::Xml(error), + } + } +} + +impl From for RdfXmlError { + fn from(error: quick_xml::events::attributes::AttrError) -> Self { + Self { + kind: RdfXmlErrorKind::XmlAttribute(error), + } + } +} + +impl From for RdfXmlError { + fn from(error: io::Error) -> Self { + Self { + kind: RdfXmlErrorKind::Xml(quick_xml::Error::Io(Arc::new(error))), + } + } +} + +impl From for io::Error { + fn from(error: RdfXmlError) -> Self { + match error.kind { + RdfXmlErrorKind::Xml(error) => match error { + quick_xml::Error::Io(error) => match Arc::try_unwrap(error) { + Ok(error) => error, + Err(error) => io::Error::new(error.kind(), error), + }, + quick_xml::Error::UnexpectedEof(error) => { + io::Error::new(io::ErrorKind::UnexpectedEof, error) + } + error => io::Error::new(io::ErrorKind::InvalidData, error), + }, + RdfXmlErrorKind::Other(error) => io::Error::new(io::ErrorKind::InvalidData, error), + _ => io::Error::new(io::ErrorKind::InvalidData, error), + } + } +} diff --git a/lib/oxrdfxml/src/lib.rs b/lib/oxrdfxml/src/lib.rs new file mode 100644 index 00000000..e04e0d86 --- /dev/null +++ b/lib/oxrdfxml/src/lib.rs @@ -0,0 +1,14 @@ +#![doc = include_str!("../README.md")] +#![doc(test(attr(deny(warnings))))] +#![cfg_attr(docsrs, feature(doc_auto_cfg))] +#![doc(html_favicon_url = "https://raw.githubusercontent.com/oxigraph/oxigraph/main/logo.svg")] +#![doc(html_logo_url = "https://raw.githubusercontent.com/oxigraph/oxigraph/main/logo.svg")] + +mod error; +mod parser; +mod serializer; +mod utils; + +pub use crate::serializer::{RdfXmlSerializer, ToWriteRdfXmlWriter}; +pub use error::RdfXmlError; +pub use parser::{FromReadRdfXmlReader, RdfXmlParser}; diff --git a/lib/oxrdfxml/src/parser.rs b/lib/oxrdfxml/src/parser.rs new file mode 100644 index 00000000..3493fe54 --- /dev/null +++ b/lib/oxrdfxml/src/parser.rs @@ -0,0 +1,1081 @@ +use crate::error::{RdfXmlError, RdfXmlErrorKind}; +use crate::utils::*; +use oxilangtag::LanguageTag; +use oxiri::{Iri, IriParseError}; +use oxrdf::vocab::rdf; +use oxrdf::{BlankNode, Literal, NamedNode, Subject, Term, Triple}; +use quick_xml::escape::unescape_with; +use quick_xml::events::attributes::Attribute; +use quick_xml::events::*; +use quick_xml::name::{LocalName, QName, ResolveResult}; +use quick_xml::{NsReader, Writer}; +use std::collections::{HashMap, HashSet}; +use std::io::{BufRead, BufReader, Read}; +use std::str; + +/// A [RDF/XML](https://www.w3.org/TR/rdf-syntax-grammar/) streaming parser. +/// +/// It reads the file in streaming. +/// It does not keep data in memory except a stack for handling nested XML tags, and a set of all +/// seen `rdf:ID`s to detect duplicate ids and fail according to the specification. +/// +/// Its performances are not optimized yet and hopefully could be significantly enhanced by reducing the +/// number of allocations and copies done by the parser. +/// +/// Count the number of people: +/// ``` +/// use oxrdf::{NamedNodeRef, vocab::rdf}; +/// use oxrdfxml::RdfXmlParser; +/// +/// let file = b" +/// +/// +/// +/// Foo +/// +/// +/// "; +/// +/// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; +/// let mut count = 0; +/// for triple in RdfXmlParser::new().parse_from_read(file.as_ref()) { +/// let triple = triple?; +/// if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { +/// count += 1; +/// } +/// } +/// assert_eq!(2, count); +/// # Result::<_,Box>::Ok(()) +/// ``` +#[derive(Default)] +pub struct RdfXmlParser { + base: Option>, +} + +impl RdfXmlParser { + /// Builds a new [`RdfXmlParser`]. + #[inline] + pub fn new() -> Self { + Self::default() + } + + #[inline] + pub fn with_base_iri(mut self, base_iri: impl Into) -> Result { + self.base = Some(Iri::parse(base_iri.into())?); + Ok(self) + } + + /// Parses a RDF/XML file from a [`Read`] implementation. + /// + /// Count the number of people: + /// ``` + /// use oxrdf::{NamedNodeRef, vocab::rdf}; + /// use oxrdfxml::RdfXmlParser; + /// + /// let file = b" + /// + /// + /// + /// Foo + /// + /// + /// "; + /// + /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; + /// let mut count = 0; + /// for triple in RdfXmlParser::new().parse_from_read(file.as_ref()) { + /// let triple = triple?; + /// if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { + /// count += 1; + /// } + /// } + /// assert_eq!(2, count); + /// # Result::<_,Box>::Ok(()) + /// ``` + pub fn parse_from_read(&self, read: R) -> FromReadRdfXmlReader { + let mut reader = NsReader::from_reader(BufReader::new(read)); + reader.expand_empty_elements(true); + FromReadRdfXmlReader { + results: Vec::new(), + reader: RdfXmlReader { + reader, + state: vec![RdfXmlState::Doc { + base_iri: self.base.clone(), + }], + custom_entities: HashMap::default(), + in_literal_depth: 0, + known_rdf_id: HashSet::default(), + is_end: false, + }, + reader_buffer: Vec::default(), + } + } +} + +/// Parses a RDF/XML file from a [`Read`] implementation. Can be built using [`RdfXmlParser::parse_from_read`]. +/// +/// Count the number of people: +/// ``` +/// use oxrdf::{NamedNodeRef, vocab::rdf}; +/// use oxrdfxml::RdfXmlParser; +/// +/// let file = b" +/// +/// +/// +/// Foo +/// +/// +/// "; +/// +/// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; +/// let mut count = 0; +/// for triple in RdfXmlParser::new().parse_from_read(file.as_ref()) { +/// let triple = triple?; +/// if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { +/// count += 1; +/// } +/// } +/// assert_eq!(2, count); +/// # Result::<_,Box>::Ok(()) +/// ``` +pub struct FromReadRdfXmlReader { + results: Vec, + reader: RdfXmlReader>, + reader_buffer: Vec, +} + +impl Iterator for FromReadRdfXmlReader { + type Item = Result; + + fn next(&mut self) -> Option> { + loop { + if let Some(triple) = self.results.pop() { + return Some(Ok(triple)); + } else if self.reader.is_end { + return None; + } + if let Err(e) = self.parse_step() { + return Some(Err(e)); + } + } + } +} + +impl FromReadRdfXmlReader { + /// The current byte position in the input data. + pub fn buffer_position(&self) -> usize { + self.reader.reader.buffer_position() + } + + fn parse_step(&mut self) -> Result<(), RdfXmlError> { + self.reader_buffer.clear(); + let event = self + .reader + .reader + .read_event_into(&mut self.reader_buffer)?; + self.reader.parse_event(event, &mut self.results) + } +} + +const RDF_ABOUT: &str = "http://www.w3.org/1999/02/22-rdf-syntax-ns#about"; +const RDF_ABOUT_EACH: &str = "http://www.w3.org/1999/02/22-rdf-syntax-ns#aboutEach"; +const RDF_ABOUT_EACH_PREFIX: &str = "http://www.w3.org/1999/02/22-rdf-syntax-ns#aboutEachPrefix"; +const RDF_BAG_ID: &str = "http://www.w3.org/1999/02/22-rdf-syntax-ns#bagID"; +const RDF_DATATYPE: &str = "http://www.w3.org/1999/02/22-rdf-syntax-ns#datatype"; +const RDF_DESCRIPTION: &str = "http://www.w3.org/1999/02/22-rdf-syntax-ns#Description"; +const RDF_ID: &str = "http://www.w3.org/1999/02/22-rdf-syntax-ns#ID"; +const RDF_LI: &str = "http://www.w3.org/1999/02/22-rdf-syntax-ns#li"; +const RDF_NODE_ID: &str = "http://www.w3.org/1999/02/22-rdf-syntax-ns#nodeID"; +const RDF_PARSE_TYPE: &str = "http://www.w3.org/1999/02/22-rdf-syntax-ns#parseType"; +const RDF_RDF: &str = "http://www.w3.org/1999/02/22-rdf-syntax-ns#RDF"; +const RDF_RESOURCE: &str = "http://www.w3.org/1999/02/22-rdf-syntax-ns#resource"; + +const RESERVED_RDF_ELEMENTS: [&str; 11] = [ + RDF_ABOUT, + RDF_ABOUT_EACH, + RDF_ABOUT_EACH_PREFIX, + RDF_BAG_ID, + RDF_DATATYPE, + RDF_ID, + RDF_LI, + RDF_NODE_ID, + RDF_PARSE_TYPE, + RDF_RDF, + RDF_RESOURCE, +]; +const RESERVED_RDF_ATTRIBUTES: [&str; 5] = [ + RDF_ABOUT_EACH, + RDF_ABOUT_EACH_PREFIX, + RDF_LI, + RDF_RDF, + RDF_RESOURCE, +]; + +#[derive(Clone, Debug)] +enum NodeOrText { + Node(Subject), + Text(String), +} + +enum RdfXmlState { + Doc { + base_iri: Option>, + }, + Rdf { + base_iri: Option>, + language: Option, + }, + NodeElt { + base_iri: Option>, + language: Option, + subject: Subject, + li_counter: u64, + }, + PropertyElt { + //Resource, Literal or Empty property element + iri: NamedNode, + base_iri: Option>, + language: Option, + subject: Subject, + object: Option, + id_attr: Option, + datatype_attr: Option, + }, + ParseTypeCollectionPropertyElt { + iri: NamedNode, + base_iri: Option>, + language: Option, + subject: Subject, + objects: Vec, + id_attr: Option, + }, + ParseTypeLiteralPropertyElt { + iri: NamedNode, + base_iri: Option>, + language: Option, + subject: Subject, + writer: Writer>, + id_attr: Option, + emit: bool, //false for parseTypeOtherPropertyElt support + }, +} + +impl RdfXmlState { + fn base_iri(&self) -> Option<&Iri> { + match self { + RdfXmlState::Doc { base_iri, .. } + | RdfXmlState::Rdf { base_iri, .. } + | RdfXmlState::NodeElt { base_iri, .. } + | RdfXmlState::PropertyElt { base_iri, .. } + | RdfXmlState::ParseTypeCollectionPropertyElt { base_iri, .. } + | RdfXmlState::ParseTypeLiteralPropertyElt { base_iri, .. } => base_iri.as_ref(), + } + } + + fn language(&self) -> Option<&String> { + match self { + RdfXmlState::Doc { .. } => None, + RdfXmlState::Rdf { language, .. } + | RdfXmlState::NodeElt { language, .. } + | RdfXmlState::PropertyElt { language, .. } + | RdfXmlState::ParseTypeCollectionPropertyElt { language, .. } + | RdfXmlState::ParseTypeLiteralPropertyElt { language, .. } => language.as_ref(), + } + } +} + +struct RdfXmlReader { + reader: NsReader, + state: Vec, + custom_entities: HashMap, + in_literal_depth: usize, + known_rdf_id: HashSet, + is_end: bool, +} + +impl RdfXmlReader { + fn parse_event(&mut self, event: Event, results: &mut Vec) -> Result<(), RdfXmlError> { + match event { + Event::Start(event) => self.parse_start_event(&event, results), + Event::End(event) => self.parse_end_event(&event, results), + Event::Empty(_) => unreachable!("The expand_empty_elements option must be enabled"), + Event::Text(event) => self.parse_text_event(&event), + Event::CData(event) => self.parse_text_event(&event.escape()?), + Event::Comment(_) | Event::PI(_) => Ok(()), + Event::Decl(decl) => { + if let Some(encoding) = decl.encoding() { + if !is_utf8(&encoding?) { + return Err(RdfXmlError::msg( + "Only UTF-8 is supported by the RDF/XML parser", + )); + } + } + Ok(()) + } + Event::DocType(dt) => self.parse_doctype(&dt), + Event::Eof => { + self.is_end = true; + Ok(()) + } + } + } + + fn parse_doctype(&mut self, dt: &BytesText<'_>) -> Result<(), RdfXmlError> { + // we extract entities + for input in self + .reader + .decoder() + .decode(dt.as_ref())? + .split('<') + .skip(1) + { + if let Some(input) = input.strip_prefix("!ENTITY") { + let input = input.trim_start().strip_prefix('%').unwrap_or(input); + let (entity_name, input) = input.trim_start().split_once(|c: char| c.is_ascii_whitespace()).ok_or_else(|| { + RdfXmlError::msg( + "').ok_or_else(|| { + RdfXmlError::msg("") + })?; + + // Resolves custom entities within the current entity definition. + let entity_value = unescape_with(entity_value, |e| self.resolve_entity(e)) + .map_err(quick_xml::Error::from)?; + self.custom_entities + .insert(entity_name.to_owned(), entity_value.to_string()); + } + } + Ok(()) + } + + fn parse_start_event( + &mut self, + event: &BytesStart<'_>, + results: &mut Vec, + ) -> Result<(), RdfXmlError> { + #[derive(PartialEq, Eq)] + enum RdfXmlParseType { + Default, + Collection, + Literal, + Resource, + Other, + } + + #[derive(PartialEq, Eq)] + enum RdfXmlNextProduction { + Rdf, + NodeElt, + PropertyElt { subject: Subject }, + } + + //Literal case + if let Some(RdfXmlState::ParseTypeLiteralPropertyElt { writer, .. }) = self.state.last_mut() + { + let mut clean_event = BytesStart::new( + self.reader + .decoder() + .decode(event.name().as_ref())? + .to_string(), + ); + for attr in event.attributes() { + clean_event.push_attribute(attr?); + } + writer.write_event(Event::Start(clean_event))?; + self.in_literal_depth += 1; + return Ok(()); + } + + let tag_name = self.resolve_tag_name(event.name())?; + + //We read attributes + let (mut language, mut base_iri) = if let Some(current_state) = self.state.last() { + ( + current_state.language().cloned(), + current_state.base_iri().cloned(), + ) + } else { + (None, None) + }; + + let mut id_attr = None; + let mut node_id_attr = None; + let mut about_attr = None; + let mut property_attrs = Vec::default(); + let mut resource_attr = None; + let mut datatype_attr = None; + let mut parse_type = RdfXmlParseType::Default; + let mut type_attr = None; + + for attribute in event.attributes() { + let attribute = attribute?; + if attribute.key.as_ref().starts_with(b"xml") { + if attribute.key.as_ref() == b"xml:lang" { + let tag = self.convert_attribute(&attribute)?; + language = Some( + LanguageTag::parse(tag.to_ascii_lowercase()) + .map_err(|error| RdfXmlError { + kind: RdfXmlErrorKind::InvalidLanguageTag { tag, error }, + })? + .into_inner(), + ); + } else if attribute.key.as_ref() == b"xml:base" { + let iri = self.convert_attribute(&attribute)?; + base_iri = Some(Iri::parse(iri.clone()).map_err(|error| RdfXmlError { + kind: RdfXmlErrorKind::InvalidIri { iri, error }, + })?) + } else { + // We ignore other xml attributes + } + } else { + let attribute_url = self.resolve_attribute_name(attribute.key)?; + if *attribute_url == *RDF_ID { + let mut id = self.convert_attribute(&attribute)?; + if !is_nc_name(&id) { + return Err(RdfXmlError::msg(format!( + "{} is not a valid rdf:ID value", + &id + ))); + } + id.insert(0, '#'); + id_attr = Some(id); + } else if *attribute_url == *RDF_BAG_ID { + let bag_id = self.convert_attribute(&attribute)?; + if !is_nc_name(&bag_id) { + return Err(RdfXmlError::msg(format!( + "{} is not a valid rdf:bagID value", + &bag_id + ))); + } + } else if *attribute_url == *RDF_NODE_ID { + let id = self.convert_attribute(&attribute)?; + if !is_nc_name(&id) { + return Err(RdfXmlError::msg(format!( + "{} is not a valid rdf:nodeID value", + &id + ))); + } + node_id_attr = Some(BlankNode::new_unchecked(id)); + } else if *attribute_url == *RDF_ABOUT { + about_attr = Some(attribute); + } else if *attribute_url == *RDF_RESOURCE { + resource_attr = Some(attribute); + } else if *attribute_url == *RDF_DATATYPE { + datatype_attr = Some(attribute); + } else if *attribute_url == *RDF_PARSE_TYPE { + parse_type = match attribute.value.as_ref() { + b"Collection" => RdfXmlParseType::Collection, + b"Literal" => RdfXmlParseType::Literal, + b"Resource" => RdfXmlParseType::Resource, + _ => RdfXmlParseType::Other, + }; + } else if attribute_url == rdf::TYPE.as_str() { + type_attr = Some(attribute); + } else if RESERVED_RDF_ATTRIBUTES.contains(&&*attribute_url) { + return Err(RdfXmlError::msg(format!( + "{} is not a valid attribute", + &attribute_url + ))); + } else { + property_attrs.push(( + NamedNode::new(attribute_url.clone()).map_err(|error| RdfXmlError { + kind: RdfXmlErrorKind::InvalidIri { + iri: attribute_url, + error, + }, + })?, + self.convert_attribute(&attribute)?, + )); + } + } + } + + //Parsing with the base URI + let id_attr = match id_attr { + Some(iri) => { + let iri = resolve(&base_iri, iri)?; + if self.known_rdf_id.contains(iri.as_str()) { + return Err(RdfXmlError::msg(format!( + "{} has already been used as rdf:ID value", + &iri + ))); + } + self.known_rdf_id.insert(iri.as_str().into()); + Some(iri) + } + None => None, + }; + let about_attr = match about_attr { + Some(attr) => Some(self.convert_iri_attribute(&base_iri, &attr)?), + None => None, + }; + let resource_attr = match resource_attr { + Some(attr) => Some(self.convert_iri_attribute(&base_iri, &attr)?), + None => None, + }; + let datatype_attr = match datatype_attr { + Some(attr) => Some(self.convert_iri_attribute(&base_iri, &attr)?), + None => None, + }; + let type_attr = match type_attr { + Some(attr) => Some(self.convert_iri_attribute(&base_iri, &attr)?), + None => None, + }; + + let expected_production = match self.state.last() { + Some(RdfXmlState::Doc { .. }) => RdfXmlNextProduction::Rdf, + Some( + RdfXmlState::Rdf { .. } + | RdfXmlState::PropertyElt { .. } + | RdfXmlState::ParseTypeCollectionPropertyElt { .. }, + ) => RdfXmlNextProduction::NodeElt, + Some(RdfXmlState::NodeElt { subject, .. }) => RdfXmlNextProduction::PropertyElt { + subject: subject.clone(), + }, + Some(RdfXmlState::ParseTypeLiteralPropertyElt { .. }) => { + panic!("ParseTypeLiteralPropertyElt production children should never be considered as a RDF/XML content") + } + None => { + return Err(RdfXmlError::msg( + "No state in the stack: the XML is not balanced", + )); + } + }; + + let new_state = match expected_production { + RdfXmlNextProduction::Rdf => { + if *tag_name == *RDF_RDF { + RdfXmlState::Rdf { base_iri, language } + } else if RESERVED_RDF_ELEMENTS.contains(&&*tag_name) { + return Err(RdfXmlError::msg(format!( + "Invalid node element tag name: {}", + &tag_name + ))); + } else { + Self::build_node_elt( + NamedNode::new(tag_name.clone()).map_err(|error| RdfXmlError { + kind: RdfXmlErrorKind::InvalidIri { + iri: tag_name, + error, + }, + })?, + base_iri, + language, + id_attr, + node_id_attr, + about_attr, + type_attr, + property_attrs, + results, + )? + } + } + RdfXmlNextProduction::NodeElt => { + if RESERVED_RDF_ELEMENTS.contains(&&*tag_name) { + return Err(RdfXmlError::msg(format!( + "Invalid property element tag name: {}", + &tag_name + ))); + } + Self::build_node_elt( + NamedNode::new(tag_name.clone()).map_err(|error| RdfXmlError { + kind: RdfXmlErrorKind::InvalidIri { + iri: tag_name, + error, + }, + })?, + base_iri, + language, + id_attr, + node_id_attr, + about_attr, + type_attr, + property_attrs, + results, + )? + } + RdfXmlNextProduction::PropertyElt { subject } => { + let iri = if *tag_name == *RDF_LI { + let Some(RdfXmlState::NodeElt { li_counter, .. }) = self.state.last_mut() else { + return Err(RdfXmlError::msg(format!( + "Invalid property element tag name: {}", + &tag_name + ))); + }; + *li_counter += 1; + NamedNode::new_unchecked(format!( + "http://www.w3.org/1999/02/22-rdf-syntax-ns#_{}", + li_counter + )) + } else if RESERVED_RDF_ELEMENTS.contains(&&*tag_name) + || *tag_name == *RDF_DESCRIPTION + { + return Err(RdfXmlError::msg(format!( + "Invalid property element tag name: {}", + &tag_name + ))); + } else { + NamedNode::new(tag_name.clone()).map_err(|error| RdfXmlError { + kind: RdfXmlErrorKind::InvalidIri { + iri: tag_name, + error, + }, + })? + }; + match parse_type { + RdfXmlParseType::Default => { + if resource_attr.is_some() + || node_id_attr.is_some() + || !property_attrs.is_empty() + { + let object = match (resource_attr, node_id_attr) + { + (Some(resource_attr), None) => Subject::from(resource_attr), + (None, Some(node_id_attr)) => node_id_attr.into(), + (None, None) => BlankNode::default().into(), + (Some(_), Some(_)) => return Err(RdfXmlError::msg("Not both rdf:resource and rdf:nodeID could be set at the same time")) + }; + Self::emit_property_attrs(&object, property_attrs, &language, results); + if let Some(type_attr) = type_attr { + results.push(Triple::new(object.clone(), rdf::TYPE, type_attr)); + } + RdfXmlState::PropertyElt { + iri, + base_iri, + language, + subject, + object: Some(NodeOrText::Node(object)), + id_attr, + datatype_attr, + } + } else { + RdfXmlState::PropertyElt { + iri, + base_iri, + language, + subject, + object: None, + id_attr, + datatype_attr, + } + } + } + RdfXmlParseType::Literal => RdfXmlState::ParseTypeLiteralPropertyElt { + iri, + base_iri, + language, + subject, + writer: Writer::new(Vec::default()), + id_attr, + emit: true, + }, + RdfXmlParseType::Resource => Self::build_parse_type_resource_property_elt( + iri, base_iri, language, subject, id_attr, results, + ), + RdfXmlParseType::Collection => RdfXmlState::ParseTypeCollectionPropertyElt { + iri, + base_iri, + language, + subject, + objects: Vec::default(), + id_attr, + }, + RdfXmlParseType::Other => RdfXmlState::ParseTypeLiteralPropertyElt { + iri, + base_iri, + language, + subject, + writer: Writer::new(Vec::default()), + id_attr, + emit: false, + }, + } + } + }; + self.state.push(new_state); + Ok(()) + } + + fn parse_end_event( + &mut self, + event: &BytesEnd<'_>, + results: &mut Vec, + ) -> Result<(), RdfXmlError> { + //Literal case + if self.in_literal_depth > 0 { + if let Some(RdfXmlState::ParseTypeLiteralPropertyElt { writer, .. }) = + self.state.last_mut() + { + writer.write_event(Event::End(BytesEnd::new( + self.reader.decoder().decode(event.name().as_ref())?, + )))?; + self.in_literal_depth -= 1; + return Ok(()); + } + } + + if let Some(current_state) = self.state.pop() { + self.end_state(current_state, results)?; + } + Ok(()) + } + + fn parse_text_event(&mut self, event: &BytesText<'_>) -> Result<(), RdfXmlError> { + let text = event.unescape_with(|e| self.resolve_entity(e))?.to_string(); + match self.state.last_mut() { + Some(RdfXmlState::PropertyElt { object, .. }) => { + if !event.iter().copied().all(is_whitespace) { + *object = Some(NodeOrText::Text(text)); + } + Ok(()) + } + Some(RdfXmlState::ParseTypeLiteralPropertyElt { writer, .. }) => { + writer.write_event(Event::Text(BytesText::new(&text)))?; + Ok(()) + } + _ => { + if event.iter().copied().all(is_whitespace) { + Ok(()) + } else { + Err(RdfXmlError::msg(format!( + "Unexpected text event: '{}'", + text + ))) + } + } + } + } + + fn resolve_tag_name(&self, qname: QName<'_>) -> Result { + let (namespace, local_name) = self.reader.resolve_element(qname); + self.resolve_ns_name(namespace, local_name) + } + + fn resolve_attribute_name(&self, qname: QName<'_>) -> Result { + let (namespace, local_name) = self.reader.resolve_attribute(qname); + self.resolve_ns_name(namespace, local_name) + } + + fn resolve_ns_name( + &self, + namespace: ResolveResult, + local_name: LocalName<'_>, + ) -> Result { + match namespace { + ResolveResult::Bound(ns) => { + let mut value = Vec::with_capacity(ns.as_ref().len() + local_name.as_ref().len()); + value.extend_from_slice(ns.as_ref()); + value.extend_from_slice(local_name.as_ref()); + Ok(unescape_with(&self.reader.decoder().decode(&value)?, |e| { + self.resolve_entity(e) + }) + .map_err(quick_xml::Error::from)? + .to_string()) + } + ResolveResult::Unbound => { + Err(RdfXmlError::msg("XML namespaces are required in RDF/XML")) + } + ResolveResult::Unknown(v) => Err(RdfXmlError::msg(format!( + "Unknown prefix {}:", + self.reader.decoder().decode(&v)? + ))), + } + } + + #[allow(clippy::too_many_arguments)] + fn build_node_elt( + iri: NamedNode, + base_iri: Option>, + language: Option, + id_attr: Option, + node_id_attr: Option, + about_attr: Option, + type_attr: Option, + property_attrs: Vec<(NamedNode, String)>, + results: &mut Vec, + ) -> Result { + let subject = match (id_attr, node_id_attr, about_attr) { + (Some(id_attr), None, None) => Subject::from(id_attr), + (None, Some(node_id_attr), None) => node_id_attr.into(), + (None, None, Some(about_attr)) => about_attr.into(), + (None, None, None) => BlankNode::default().into(), + (Some(_), Some(_), _) => { + return Err(RdfXmlError::msg( + "Not both rdf:ID and rdf:nodeID could be set at the same time", + )) + } + (_, Some(_), Some(_)) => { + return Err(RdfXmlError::msg( + "Not both rdf:nodeID and rdf:resource could be set at the same time", + )) + } + (Some(_), _, Some(_)) => { + return Err(RdfXmlError::msg( + "Not both rdf:ID and rdf:resource could be set at the same time", + )) + } + }; + + Self::emit_property_attrs(&subject, property_attrs, &language, results); + + if let Some(type_attr) = type_attr { + results.push(Triple::new(subject.clone(), rdf::TYPE, type_attr)); + } + + if iri != *RDF_DESCRIPTION { + results.push(Triple::new(subject.clone(), rdf::TYPE, iri)); + } + Ok(RdfXmlState::NodeElt { + base_iri, + language, + subject, + li_counter: 0, + }) + } + + fn build_parse_type_resource_property_elt( + iri: NamedNode, + base_iri: Option>, + language: Option, + subject: Subject, + id_attr: Option, + results: &mut Vec, + ) -> RdfXmlState { + let object = BlankNode::default(); + let triple = Triple::new(subject, iri, object.clone()); + if let Some(id_attr) = id_attr { + Self::reify(triple.clone(), id_attr, results); + } + results.push(triple); + RdfXmlState::NodeElt { + base_iri, + language, + subject: object.into(), + li_counter: 0, + } + } + + fn end_state( + &mut self, + state: RdfXmlState, + results: &mut Vec, + ) -> Result<(), RdfXmlError> { + match state { + RdfXmlState::PropertyElt { + iri, + language, + subject, + id_attr, + datatype_attr, + object, + .. + } => { + let object = match object { + Some(NodeOrText::Node(node)) => Term::from(node), + Some(NodeOrText::Text(text)) => { + Self::new_literal(text, language, datatype_attr).into() + } + None => Self::new_literal(String::new(), language, datatype_attr).into(), + }; + let triple = Triple::new(subject, iri, object); + if let Some(id_attr) = id_attr { + Self::reify(triple.clone(), id_attr, results); + } + results.push(triple); + } + RdfXmlState::ParseTypeCollectionPropertyElt { + iri, + subject, + id_attr, + objects, + .. + } => { + let mut current_node = Subject::from(rdf::NIL); + for object in objects.into_iter().rev() { + let subject = Subject::from(BlankNode::default()); + results.push(Triple::new(subject.clone(), rdf::FIRST, object)); + results.push(Triple::new(subject.clone(), rdf::REST, current_node)); + current_node = subject; + } + let triple = Triple::new(subject, iri, current_node); + if let Some(id_attr) = id_attr { + Self::reify(triple.clone(), id_attr, results); + } + results.push(triple); + } + RdfXmlState::ParseTypeLiteralPropertyElt { + iri, + subject, + id_attr, + writer, + emit, + .. + } => { + if emit { + let object = writer.into_inner(); + if object.is_empty() { + return Err(RdfXmlError::msg(format!( + "No value found for rdf:XMLLiteral value of property {}", + iri + ))); + } + let triple = Triple::new( + subject, + iri, + Literal::new_typed_literal( + str::from_utf8(&object).map_err(|_| { + RdfXmlError::msg("The XML literal is not in valid UTF-8".to_owned()) + })?, + rdf::XML_LITERAL, + ), + ); + if let Some(id_attr) = id_attr { + Self::reify(triple.clone(), id_attr, results); + } + results.push(triple); + } + } + RdfXmlState::NodeElt { subject, .. } => match self.state.last_mut() { + Some(RdfXmlState::PropertyElt { object, .. }) => { + *object = Some(NodeOrText::Node(subject)) + } + Some(RdfXmlState::ParseTypeCollectionPropertyElt { objects, .. }) => { + objects.push(subject) + } + _ => (), + }, + _ => (), + } + Ok(()) + } + + fn new_literal( + value: String, + language: Option, + datatype: Option, + ) -> Literal { + if let Some(datatype) = datatype { + Literal::new_typed_literal(value, datatype) + } else if let Some(language) = language { + Literal::new_language_tagged_literal_unchecked(value, language) + } else { + Literal::new_simple_literal(value) + } + } + + fn reify(triple: Triple, statement_id: NamedNode, results: &mut Vec) { + results.push(Triple::new(statement_id.clone(), rdf::TYPE, rdf::STATEMENT)); + results.push(Triple::new( + statement_id.clone(), + rdf::SUBJECT, + triple.subject, + )); + results.push(Triple::new( + statement_id.clone(), + rdf::PREDICATE, + triple.predicate, + )); + results.push(Triple::new(statement_id, rdf::OBJECT, triple.object)); + } + + fn emit_property_attrs( + subject: &Subject, + literal_attributes: Vec<(NamedNode, String)>, + language: &Option, + results: &mut Vec, + ) { + for (literal_predicate, literal_value) in literal_attributes { + results.push(Triple::new( + subject.clone(), + literal_predicate, + if let Some(language) = language.clone() { + Literal::new_language_tagged_literal_unchecked(literal_value, language) + } else { + Literal::new_simple_literal(literal_value) + }, + )); + } + } + + fn convert_attribute(&self, attribute: &Attribute) -> Result { + Ok(attribute + .decode_and_unescape_value_with(&self.reader, |e| self.resolve_entity(e))? + .to_string()) + } + + fn convert_iri_attribute( + &self, + base_iri: &Option>, + attribute: &Attribute<'_>, + ) -> Result { + resolve(base_iri, self.convert_attribute(attribute)?) + } + + fn resolve_entity(&self, e: &str) -> Option<&str> { + self.custom_entities.get(e).map(String::as_str) + } +} + +fn resolve(base_iri: &Option>, relative_iri: String) -> Result { + if let Some(base_iri) = base_iri { + Ok(NamedNode::new_unchecked( + base_iri + .resolve(&relative_iri) + .map_err(|error| RdfXmlError { + kind: RdfXmlErrorKind::InvalidIri { + iri: relative_iri, + error, + }, + })? + .into_inner(), + )) + } else { + NamedNode::new(relative_iri.clone()).map_err(|error| RdfXmlError { + kind: RdfXmlErrorKind::InvalidIri { + iri: relative_iri, + error, + }, + }) + } +} + +fn is_nc_name(name: &str) -> bool { + // Name - (Char* ':' Char*) + is_name(name) && name.chars().all(|c| c != ':') +} + +fn is_name(name: &str) -> bool { + // NameStartChar (NameChar)* + let mut c = name.chars(); + if !c.next().map_or(false, is_name_start_char) { + return false; + } + c.all(is_name_char) +} + +fn is_whitespace(c: u8) -> bool { + matches!(c, b' ' | b'\t' | b'\n' | b'\r') +} + +fn is_utf8(encoding: &[u8]) -> bool { + matches!( + encoding.to_ascii_lowercase().as_slice(), + b"unicode-1-1-utf-8" + | b"unicode11utf8" + | b"unicode20utf8" + | b"utf-8" + | b"utf8" + | b"x-unicode20utf8" + ) +} diff --git a/lib/oxrdfxml/src/serializer.rs b/lib/oxrdfxml/src/serializer.rs new file mode 100644 index 00000000..03de4fb9 --- /dev/null +++ b/lib/oxrdfxml/src/serializer.rs @@ -0,0 +1,229 @@ +use crate::utils::*; +use oxrdf::{Subject, SubjectRef, TermRef, TripleRef}; +use quick_xml::events::*; +use quick_xml::Writer; +use std::io; +use std::io::Write; +use std::sync::Arc; + +/// A [RDF/XML](https://www.w3.org/TR/rdf-syntax-grammar/) serializer. +/// +/// ``` +/// use oxrdf::{NamedNodeRef, TripleRef}; +/// use oxrdfxml::RdfXmlSerializer; +/// +/// let mut writer = RdfXmlSerializer::new().serialize_to_write(Vec::new()); +/// writer.write_triple(TripleRef::new( +/// NamedNodeRef::new("http://example.com#me")?, +/// NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?, +/// NamedNodeRef::new("http://schema.org/Person")?, +/// ))?; +/// assert_eq!( +/// b"\n\n\t\n\t\t\n\t\n", +/// writer.finish()?.as_slice() +/// ); +/// # Result::<_,Box>::Ok(()) +/// ``` +#[derive(Default)] +pub struct RdfXmlSerializer; + +impl RdfXmlSerializer { + /// Builds a new [`RdfXmlSerializer`]. + #[inline] + pub fn new() -> Self { + Self + } + + /// Writes a RdfXml file to a [`Write`] implementation. + /// + /// ``` + /// use oxrdf::{NamedNodeRef, TripleRef}; + /// use oxrdfxml::RdfXmlSerializer; + /// + /// let mut writer = RdfXmlSerializer::new().serialize_to_write(Vec::new()); + /// writer.write_triple(TripleRef::new( + /// NamedNodeRef::new("http://example.com#me")?, + /// NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?, + /// NamedNodeRef::new("http://schema.org/Person")?, + /// ))?; + /// assert_eq!( + /// b"\n\n\t\n\t\t\n\t\n", + /// writer.finish()?.as_slice() + /// ); + /// # Result::<_,Box>::Ok(()) + /// ``` + #[allow(clippy::unused_self)] + pub fn serialize_to_write(&self, write: W) -> ToWriteRdfXmlWriter { + ToWriteRdfXmlWriter { + writer: Writer::new_with_indent(write, b'\t', 1), + current_subject: None, + } + } +} + +/// Writes a RDF/XML file to a [`Write`] implementation. Can be built using [`RdfXmlSerializer::serialize_to_write`]. +/// +/// ``` +/// use oxrdf::{NamedNodeRef, TripleRef}; +/// use oxrdfxml::RdfXmlSerializer; +/// +/// let mut writer = RdfXmlSerializer::new().serialize_to_write(Vec::new()); +/// writer.write_triple(TripleRef::new( +/// NamedNodeRef::new("http://example.com#me")?, +/// NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?, +/// NamedNodeRef::new("http://schema.org/Person")?, +/// ))?; +/// assert_eq!( +/// b"\n\n\t\n\t\t\n\t\n", +/// writer.finish()?.as_slice() +/// ); +/// # Result::<_,Box>::Ok(()) +/// ``` +pub struct ToWriteRdfXmlWriter { + writer: Writer, + current_subject: Option, +} + +impl ToWriteRdfXmlWriter { + /// Writes an extra triple. + #[allow(clippy::match_wildcard_for_single_variants, unreachable_patterns)] + pub fn write_triple<'a>(&mut self, t: impl Into>) -> io::Result<()> { + if self.current_subject.is_none() { + self.write_start()?; + } + + let triple = t.into(); + // We open a new rdf:Description if useful + if self.current_subject.as_ref().map(Subject::as_ref) != Some(triple.subject) { + if self.current_subject.is_some() { + self.writer + .write_event(Event::End(BytesEnd::new("rdf:Description"))) + .map_err(map_err)?; + } + + let mut description_open = BytesStart::new("rdf:Description"); + match triple.subject { + SubjectRef::NamedNode(node) => { + description_open.push_attribute(("rdf:about", node.as_str())) + } + SubjectRef::BlankNode(node) => { + description_open.push_attribute(("rdf:nodeID", node.as_str())) + } + _ => { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "RDF/XML only supports named or blank subject", + )) + } + } + self.writer + .write_event(Event::Start(description_open)) + .map_err(map_err)?; + } + + let (prop_prefix, prop_value) = split_iri(triple.predicate.as_str()); + let (prop_qname, prop_xmlns) = if prop_value.is_empty() { + ("prop:", ("xmlns:prop", prop_prefix)) + } else { + (prop_value, ("xmlns", prop_prefix)) + }; + let property_element = self.writer.create_element(prop_qname); + let property_element = property_element.with_attribute(prop_xmlns); + + match triple.object { + TermRef::NamedNode(node) => property_element + .with_attribute(("rdf:resource", node.as_str())) + .write_empty(), + TermRef::BlankNode(node) => property_element + .with_attribute(("rdf:nodeID", node.as_str())) + .write_empty(), + TermRef::Literal(literal) => { + let property_element = if let Some(language) = literal.language() { + property_element.with_attribute(("xml:lang", language)) + } else if !literal.is_plain() { + property_element.with_attribute(("rdf:datatype", literal.datatype().as_str())) + } else { + property_element + }; + property_element.write_text_content(BytesText::new(literal.value())) + } + _ => { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "RDF/XML only supports named, blank or literal object", + )) + } + } + .map_err(map_err)?; + self.current_subject = Some(triple.subject.into_owned()); + Ok(()) + } + + pub fn write_start(&mut self) -> io::Result<()> { + // We open the file + self.writer + .write_event(Event::Decl(BytesDecl::new("1.0", Some("UTF-8"), None))) + .map_err(map_err)?; + let mut rdf_open = BytesStart::new("rdf:RDF"); + rdf_open.push_attribute(("xmlns:rdf", "http://www.w3.org/1999/02/22-rdf-syntax-ns#")); + self.writer + .write_event(Event::Start(rdf_open)) + .map_err(map_err) + } + + /// Ends the write process and returns the underlying [`Write`]. + pub fn finish(mut self) -> io::Result { + if self.current_subject.is_some() { + self.writer + .write_event(Event::End(BytesEnd::new("rdf:Description"))) + .map_err(map_err)?; + } else { + self.write_start()?; + } + self.writer + .write_event(Event::End(BytesEnd::new("rdf:RDF"))) + .map_err(map_err)?; + Ok(self.writer.into_inner()) + } +} + +fn map_err(error: quick_xml::Error) -> io::Error { + if let quick_xml::Error::Io(error) = error { + match Arc::try_unwrap(error) { + Ok(error) => error, + Err(error) => io::Error::new(error.kind(), error), + } + } else { + io::Error::new(io::ErrorKind::Other, error) + } +} + +fn split_iri(iri: &str) -> (&str, &str) { + if let Some(position_base) = iri.rfind(|c| !is_name_char(c) || c == ':') { + if let Some(position_add) = iri[position_base..].find(|c| is_name_start_char(c) && c != ':') + { + ( + &iri[..position_base + position_add], + &iri[position_base + position_add..], + ) + } else { + (iri, "") + } + } else { + (iri, "") + } +} + +#[test] +fn test_split_iri() { + assert_eq!( + split_iri("http://schema.org/Person"), + ("http://schema.org/", "Person") + ); + assert_eq!(split_iri("http://schema.org/"), ("http://schema.org/", "")); + assert_eq!( + split_iri("http://schema.org#foo"), + ("http://schema.org#", "foo") + ); + assert_eq!(split_iri("urn:isbn:foo"), ("urn:isbn:", "foo")); +} diff --git a/lib/oxrdfxml/src/utils.rs b/lib/oxrdfxml/src/utils.rs new file mode 100644 index 00000000..b8fb2447 --- /dev/null +++ b/lib/oxrdfxml/src/utils.rs @@ -0,0 +1,26 @@ +pub fn is_name_start_char(c: char) -> bool { + // ":" | [A-Z] | "_" | [a-z] | [#xC0-#xD6] | [#xD8-#xF6] | [#xF8-#x2FF] | [#x370-#x37D] | [#x37F-#x1FFF] | [#x200C-#x200D] | [#x2070-#x218F] | [#x2C00-#x2FEF] | [#x3001-#xD7FF] | [#xF900-#xFDCF] | [#xFDF0-#xFFFD] | [#x10000-#xEFFFF] + matches!(c, + ':' + | 'A'..='Z' + | '_' + | 'a'..='z' + | '\u{C0}'..='\u{D6}' + | '\u{D8}'..='\u{F6}' + | '\u{F8}'..='\u{2FF}' + | '\u{370}'..='\u{37D}' + | '\u{37F}'..='\u{1FFF}' + | '\u{200C}'..='\u{200D}' + | '\u{2070}'..='\u{218F}' + | '\u{2C00}'..='\u{2FEF}' + | '\u{3001}'..='\u{D7FF}' + | '\u{F900}'..='\u{FDCF}' + | '\u{FDF0}'..='\u{FFFD}' + | '\u{10000}'..='\u{EFFFF}') +} + +pub fn is_name_char(c: char) -> bool { + // NameStartChar | "-" | "." | [0-9] | #xB7 | [#x0300-#x036F] | [#x203F-#x2040] + is_name_start_char(c) + || matches!(c, '-' | '.' | '0'..='9' | '\u{B7}' | '\u{0300}'..='\u{036F}' | '\u{203F}'..='\u{2040}') +} diff --git a/lib/sparesults/Cargo.toml b/lib/sparesults/Cargo.toml index baddda71..29973e23 100644 --- a/lib/sparesults/Cargo.toml +++ b/lib/sparesults/Cargo.toml @@ -20,7 +20,7 @@ rdf-star = ["oxrdf/rdf-star"] [dependencies] json-event-parser = "0.1" oxrdf = { version = "0.2.0-alpha.1-dev", path="../oxrdf" } -quick-xml = "0.28" +quick-xml = "0.29" [package.metadata.docs.rs] all-features = true diff --git a/lib/sparopt/src/lib.rs b/lib/sparopt/src/lib.rs index d6f62207..2182ff1e 100644 --- a/lib/sparopt/src/lib.rs +++ b/lib/sparopt/src/lib.rs @@ -1,3 +1,9 @@ +#![doc = include_str!("../README.md")] +#![doc(test(attr(deny(warnings))))] +#![cfg_attr(docsrs, feature(doc_auto_cfg))] +#![doc(html_favicon_url = "https://raw.githubusercontent.com/oxigraph/oxigraph/main/logo.svg")] +#![doc(html_logo_url = "https://raw.githubusercontent.com/oxigraph/oxigraph/main/logo.svg")] + pub use crate::optimizer::Optimizer; pub mod algebra; diff --git a/lib/src/io/error.rs b/lib/src/io/error.rs index 54696363..6d49148d 100644 --- a/lib/src/io/error.rs +++ b/lib/src/io/error.rs @@ -1,5 +1,5 @@ use oxiri::IriParseError; -use rio_xml::RdfXmlError; +use oxrdfxml::RdfXmlError; use std::error::Error; use std::{fmt, io}; diff --git a/lib/src/io/read.rs b/lib/src/io/read.rs index 3b2a107b..9489a168 100644 --- a/lib/src/io/read.rs +++ b/lib/src/io/read.rs @@ -3,14 +3,12 @@ pub use crate::io::error::{ParseError, SyntaxError}; use crate::io::{DatasetFormat, GraphFormat}; use crate::model::*; -use oxiri::{Iri, IriParseError}; +use oxiri::IriParseError; +use oxrdfxml::{FromReadRdfXmlReader, RdfXmlParser}; use oxttl::nquads::{FromReadNQuadsReader, NQuadsParser}; use oxttl::ntriples::{FromReadNTriplesReader, NTriplesParser}; use oxttl::trig::{FromReadTriGReader, TriGParser}; use oxttl::turtle::{FromReadTurtleReader, TurtleParser}; -use rio_api::model as rio; -use rio_api::parser::TriplesParser; -use rio_xml::RdfXmlParser; use std::collections::HashMap; use std::io::BufRead; @@ -40,7 +38,7 @@ pub struct GraphParser { enum GraphParserKind { NTriples(NTriplesParser), Turtle(TurtleParser), - RdfXml { base_iri: Option> }, + RdfXml(RdfXmlParser), } impl GraphParser { @@ -55,7 +53,7 @@ impl GraphParser { GraphFormat::Turtle => { GraphParserKind::Turtle(TurtleParser::new().with_quoted_triples()) } - GraphFormat::RdfXml => GraphParserKind::RdfXml { base_iri: None }, + GraphFormat::RdfXml => GraphParserKind::RdfXml(RdfXmlParser::new()), }, } } @@ -80,9 +78,7 @@ impl GraphParser { inner: match self.inner { GraphParserKind::NTriples(p) => GraphParserKind::NTriples(p), GraphParserKind::Turtle(p) => GraphParserKind::Turtle(p.with_base_iri(base_iri)?), - GraphParserKind::RdfXml { .. } => GraphParserKind::RdfXml { - base_iri: Some(Iri::parse(base_iri.into())?), - }, + GraphParserKind::RdfXml(p) => GraphParserKind::RdfXml(p.with_base_iri(base_iri)?), }, }) } @@ -96,11 +92,8 @@ impl GraphParser { TripleReaderKind::NTriples(p.parse_from_read(reader)) } GraphParserKind::Turtle(p) => TripleReaderKind::Turtle(p.parse_from_read(reader)), - GraphParserKind::RdfXml { base_iri } => { - TripleReaderKind::RdfXml(RdfXmlParser::new(reader, base_iri.clone())) - } + GraphParserKind::RdfXml(p) => TripleReaderKind::RdfXml(p.parse_from_read(reader)), }, - buffer: Vec::new(), } } } @@ -124,48 +117,33 @@ impl GraphParser { pub struct TripleReader { mapper: BlankNodeMapper, parser: TripleReaderKind, - buffer: Vec, } #[allow(clippy::large_enum_variant)] enum TripleReaderKind { NTriples(FromReadNTriplesReader), Turtle(FromReadTurtleReader), - RdfXml(RdfXmlParser), + RdfXml(FromReadRdfXmlReader), } impl Iterator for TripleReader { type Item = Result; fn next(&mut self) -> Option> { - loop { - if let Some(r) = self.buffer.pop() { - return Some(Ok(r)); - } - - return Some(match &mut self.parser { - TripleReaderKind::NTriples(parser) => match parser.next()? { - Ok(triple) => Ok(self.mapper.triple(triple)), - Err(e) => Err(e.into()), - }, - TripleReaderKind::Turtle(parser) => match parser.next()? { - Ok(triple) => Ok(self.mapper.triple(triple)), - Err(e) => Err(e.into()), - }, - TripleReaderKind::RdfXml(parser) => { - if parser.is_end() { - return None; - } else if let Err(e) = parser.parse_step(&mut |t| { - self.buffer.push(self.mapper.triple(RioMapper::triple(&t))); - Ok(()) - }) { - Err(e) - } else { - continue; - } - } - }); - } + Some(match &mut self.parser { + TripleReaderKind::NTriples(parser) => match parser.next()? { + Ok(triple) => Ok(self.mapper.triple(triple)), + Err(e) => Err(e.into()), + }, + TripleReaderKind::Turtle(parser) => match parser.next()? { + Ok(triple) => Ok(self.mapper.triple(triple)), + Err(e) => Err(e.into()), + }, + TripleReaderKind::RdfXml(parser) => match parser.next()? { + Ok(triple) => Ok(self.mapper.triple(triple)), + Err(e) => Err(e.into()), + }, + }) } } @@ -291,55 +269,6 @@ impl Iterator for QuadReader { } } -struct RioMapper; - -impl<'a> RioMapper { - fn named_node(node: rio::NamedNode<'a>) -> NamedNode { - NamedNode::new_unchecked(node.iri) - } - - fn blank_node(node: rio::BlankNode<'a>) -> BlankNode { - BlankNode::new_unchecked(node.id) - } - - fn literal(literal: rio::Literal<'a>) -> Literal { - match literal { - rio::Literal::Simple { value } => Literal::new_simple_literal(value), - rio::Literal::LanguageTaggedString { value, language } => { - Literal::new_language_tagged_literal_unchecked(value, language) - } - rio::Literal::Typed { value, datatype } => { - Literal::new_typed_literal(value, Self::named_node(datatype)) - } - } - } - - fn subject(node: rio::Subject<'a>) -> Subject { - match node { - rio::Subject::NamedNode(node) => Self::named_node(node).into(), - rio::Subject::BlankNode(node) => Self::blank_node(node).into(), - rio::Subject::Triple(triple) => Self::triple(triple).into(), - } - } - - fn term(node: rio::Term<'a>) -> Term { - match node { - rio::Term::NamedNode(node) => Self::named_node(node).into(), - rio::Term::BlankNode(node) => Self::blank_node(node).into(), - rio::Term::Literal(literal) => Self::literal(literal).into(), - rio::Term::Triple(triple) => Self::triple(triple).into(), - } - } - - fn triple(triple: &rio::Triple<'a>) -> Triple { - Triple { - subject: Self::subject(triple.subject), - predicate: Self::named_node(triple.predicate), - object: Self::term(triple.object), - } - } -} - #[derive(Default)] struct BlankNodeMapper { bnode_map: HashMap, diff --git a/lib/src/io/write.rs b/lib/src/io/write.rs index 051ea202..1661343f 100644 --- a/lib/src/io/write.rs +++ b/lib/src/io/write.rs @@ -2,13 +2,11 @@ use crate::io::{DatasetFormat, GraphFormat}; use crate::model::*; +use oxrdfxml::{RdfXmlSerializer, ToWriteRdfXmlWriter}; use oxttl::nquads::{NQuadsSerializer, ToWriteNQuadsWriter}; use oxttl::ntriples::{NTriplesSerializer, ToWriteNTriplesWriter}; use oxttl::trig::{ToWriteTriGWriter, TriGSerializer}; use oxttl::turtle::{ToWriteTurtleWriter, TurtleSerializer}; -use rio_api::formatter::TriplesFormatter; -use rio_api::model as rio; -use rio_xml::RdfXmlFormatter; use std::io::{self, Write}; /// A serializer for RDF graph serialization formats. @@ -23,7 +21,7 @@ use std::io::{self, Write}; /// use oxigraph::model::*; /// /// let mut buffer = Vec::new(); -/// let mut writer = GraphSerializer::from_format(GraphFormat::NTriples).triple_writer(&mut buffer)?; +/// let mut writer = GraphSerializer::from_format(GraphFormat::NTriples).triple_writer(&mut buffer); /// writer.write(&Triple { /// subject: NamedNode::new("http://example.com/s")?.into(), /// predicate: NamedNode::new("http://example.com/p")?, @@ -46,8 +44,8 @@ impl GraphSerializer { } /// Returns a [`TripleWriter`] allowing writing triples into the given [`Write`] implementation - pub fn triple_writer(&self, writer: W) -> io::Result> { - Ok(TripleWriter { + pub fn triple_writer(&self, writer: W) -> TripleWriter { + TripleWriter { formatter: match self.format { GraphFormat::NTriples => { TripleWriterKind::NTriples(NTriplesSerializer::new().serialize_to_write(writer)) @@ -55,9 +53,11 @@ impl GraphSerializer { GraphFormat::Turtle => { TripleWriterKind::Turtle(TurtleSerializer::new().serialize_to_write(writer)) } - GraphFormat::RdfXml => TripleWriterKind::RdfXml(RdfXmlFormatter::new(writer)?), + GraphFormat::RdfXml => { + TripleWriterKind::RdfXml(RdfXmlSerializer::new().serialize_to_write(writer)) + } }, - }) + } } } @@ -71,7 +71,7 @@ impl GraphSerializer { /// use oxigraph::model::*; /// /// let mut buffer = Vec::new(); -/// let mut writer = GraphSerializer::from_format(GraphFormat::NTriples).triple_writer(&mut buffer)?; +/// let mut writer = GraphSerializer::from_format(GraphFormat::NTriples).triple_writer(&mut buffer); /// writer.write(&Triple { /// subject: NamedNode::new("http://example.com/s")?.into(), /// predicate: NamedNode::new("http://example.com/p")?, @@ -90,7 +90,7 @@ pub struct TripleWriter { enum TripleWriterKind { NTriples(ToWriteNTriplesWriter), Turtle(ToWriteTurtleWriter), - RdfXml(RdfXmlFormatter), + RdfXml(ToWriteRdfXmlWriter), } impl TripleWriter { @@ -99,54 +99,7 @@ impl TripleWriter { match &mut self.formatter { TripleWriterKind::NTriples(writer) => writer.write_triple(triple), TripleWriterKind::Turtle(writer) => writer.write_triple(triple), - TripleWriterKind::RdfXml(formatter) => { - let triple = triple.into(); - formatter.format(&rio::Triple { - subject: match triple.subject { - SubjectRef::NamedNode(node) => rio::NamedNode { iri: node.as_str() }.into(), - SubjectRef::BlankNode(node) => rio::BlankNode { id: node.as_str() }.into(), - SubjectRef::Triple(_) => { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "RDF/XML does not support RDF-star yet", - )) - } - }, - predicate: rio::NamedNode { - iri: triple.predicate.as_str(), - }, - object: match triple.object { - TermRef::NamedNode(node) => rio::NamedNode { iri: node.as_str() }.into(), - TermRef::BlankNode(node) => rio::BlankNode { id: node.as_str() }.into(), - TermRef::Literal(literal) => if literal.is_plain() { - if let Some(language) = literal.language() { - rio::Literal::LanguageTaggedString { - value: literal.value(), - language, - } - } else { - rio::Literal::Simple { - value: literal.value(), - } - } - } else { - rio::Literal::Typed { - value: literal.value(), - datatype: rio::NamedNode { - iri: literal.datatype().as_str(), - }, - } - } - .into(), - TermRef::Triple(_) => { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "RDF/XML does not support RDF-star yet", - )) - } - }, - }) - } + TripleWriterKind::RdfXml(writer) => writer.write_triple(triple), } } @@ -155,7 +108,7 @@ impl TripleWriter { match self.formatter { TripleWriterKind::NTriples(writer) => writer.finish().flush(), TripleWriterKind::Turtle(writer) => writer.finish()?.flush(), - TripleWriterKind::RdfXml(formatter) => formatter.finish()?.flush(), //TODO: remove flush when the next version of Rio is going to be released + TripleWriterKind::RdfXml(formatter) => formatter.finish()?.flush(), } } } diff --git a/lib/src/sparql/model.rs b/lib/src/sparql/model.rs index b5b1a650..1aec94a1 100644 --- a/lib/src/sparql/model.rs +++ b/lib/src/sparql/model.rs @@ -115,7 +115,7 @@ impl QueryResults { format: GraphFormat, ) -> Result<(), EvaluationError> { if let Self::Graph(triples) = self { - let mut writer = GraphSerializer::from_format(format).triple_writer(write)?; + let mut writer = GraphSerializer::from_format(format).triple_writer(write); for triple in triples { writer.write(&triple?)?; } diff --git a/lib/src/store.rs b/lib/src/store.rs index beb8cfd1..c174bcb1 100644 --- a/lib/src/store.rs +++ b/lib/src/store.rs @@ -616,7 +616,7 @@ impl Store { format: GraphFormat, from_graph_name: impl Into>, ) -> Result<(), SerializerError> { - let mut writer = GraphSerializer::from_format(format).triple_writer(writer)?; + let mut writer = GraphSerializer::from_format(format).triple_writer(writer); for quad in self.quads_for_pattern(None, None, None, Some(from_graph_name.into())) { writer.write(quad?.as_ref())?; } diff --git a/python/src/io.rs b/python/src/io.rs index 35bb00eb..382a8c79 100644 --- a/python/src/io.rs +++ b/python/src/io.rs @@ -125,9 +125,7 @@ pub fn serialize(input: &PyAny, output: PyObject, mime_type: &str, py: Python<'_ PyWritable::from_data(output) }; if let Some(graph_format) = GraphFormat::from_media_type(mime_type) { - let mut writer = GraphSerializer::from_format(graph_format) - .triple_writer(output) - .map_err(map_io_err)?; + let mut writer = GraphSerializer::from_format(graph_format).triple_writer(output); for i in input.iter()? { writer .write(&*i?.extract::>()?) diff --git a/server/src/main.rs b/server/src/main.rs index 41c77ba8..7474cae4 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -512,9 +512,11 @@ pub fn main() -> anyhow::Result<()> { } writer.finish()?; } else { - let stdout = stdout(); // Not needed in Rust 1.61 let mut writer = QueryResultsSerializer::from_format(format) - .solutions_writer(stdout.lock(), solutions.variables().to_vec())?; + .solutions_writer( + stdout().lock(), + solutions.variables().to_vec(), + )?; for solution in solutions { writer.write(&solution?)?; } @@ -570,15 +572,14 @@ pub fn main() -> anyhow::Result<()> { }; if let Some(results_file) = results_file { let mut writer = GraphSerializer::from_format(format) - .triple_writer(BufWriter::new(File::create(results_file)?))?; + .triple_writer(BufWriter::new(File::create(results_file)?)); for triple in triples { writer.write(triple?.as_ref())?; } writer.finish()?; } else { - let stdout = stdout(); // Not needed in Rust 1.61 - let mut writer = GraphSerializer::from_format(format) - .triple_writer(stdout.lock())?; + let mut writer = + GraphSerializer::from_format(format).triple_writer(stdout().lock()); for triple in triples { writer.write(triple?.as_ref())?; } @@ -926,7 +927,7 @@ fn handle_request( ReadForWrite::build_response( move |w| { Ok(( - GraphSerializer::from_format(format).triple_writer(w)?, + GraphSerializer::from_format(format).triple_writer(w), triples, )) }, @@ -1232,7 +1233,7 @@ fn evaluate_sparql_query( ReadForWrite::build_response( move |w| { Ok(( - GraphSerializer::from_format(format).triple_writer(w)?, + GraphSerializer::from_format(format).triple_writer(w), triples, )) }, From f6c8358b24d79ba97bdac37fb3fb50c0e44478ae Mon Sep 17 00:00:00 2001 From: Tpt Date: Fri, 7 Jul 2023 21:41:13 +0200 Subject: [PATCH 025/217] Refactor parsers error types Uses SyntaxError and ParseError everywhere --- lib/oxrdfxml/src/error.rs | 170 +++++++++++++++++++++----------- lib/oxrdfxml/src/lib.rs | 2 +- lib/oxrdfxml/src/parser.rs | 155 +++++++++++++++-------------- lib/oxttl/src/lib.rs | 2 +- lib/oxttl/src/n3.rs | 10 +- lib/oxttl/src/nquads.rs | 8 +- lib/oxttl/src/ntriples.rs | 8 +- lib/oxttl/src/toolkit/lexer.rs | 1 + lib/oxttl/src/toolkit/mod.rs | 2 +- lib/oxttl/src/toolkit/parser.rs | 71 ++++++++----- lib/oxttl/src/trig.rs | 8 +- lib/oxttl/src/turtle.rs | 8 +- lib/sparesults/src/error.rs | 11 ++- lib/sparesults/src/xml.rs | 6 +- lib/src/io/error.rs | 48 ++++----- 15 files changed, 302 insertions(+), 208 deletions(-) diff --git a/lib/oxrdfxml/src/error.rs b/lib/oxrdfxml/src/error.rs index 413d3c4a..1382844e 100644 --- a/lib/oxrdfxml/src/error.rs +++ b/lib/oxrdfxml/src/error.rs @@ -4,16 +4,91 @@ use std::error::Error; use std::sync::Arc; use std::{fmt, io}; -/// Error that might be returned during parsing. -/// -/// It might wrap an IO error or be a parsing error. +/// Error returned during RDF/XML parsing. #[derive(Debug)] -pub struct RdfXmlError { - pub(crate) kind: RdfXmlErrorKind, +pub enum ParseError { + /// I/O error during parsing (file not found...). + Io(io::Error), + /// An error in the file syntax. + Syntax(SyntaxError), +} + +impl fmt::Display for ParseError { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Io(e) => e.fmt(f), + Self::Syntax(e) => e.fmt(f), + } + } +} + +impl Error for ParseError { + #[inline] + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Io(e) => Some(e), + Self::Syntax(e) => Some(e), + } + } +} + +impl From for ParseError { + #[inline] + fn from(error: io::Error) -> Self { + Self::Io(error) + } +} + +impl From for ParseError { + #[inline] + fn from(error: SyntaxError) -> Self { + Self::Syntax(error) + } +} + +impl From for io::Error { + #[inline] + fn from(error: ParseError) -> Self { + match error { + ParseError::Io(error) => error, + ParseError::Syntax(error) => error.into(), + } + } +} + +impl From for ParseError { + #[inline] + fn from(error: quick_xml::Error) -> Self { + match error { + quick_xml::Error::Io(error) => Self::Io(match Arc::try_unwrap(error) { + Ok(error) => error, + Err(error) => io::Error::new(error.kind(), error), + }), + error => Self::Syntax(SyntaxError { + inner: SyntaxErrorKind::Xml(error), + }), + } + } +} + +impl From for ParseError { + #[inline] + fn from(error: quick_xml::events::attributes::AttrError) -> Self { + Self::Syntax(SyntaxError { + inner: SyntaxErrorKind::XmlAttribute(error), + }) + } +} + +/// An error in the syntax of the parsed file. +#[derive(Debug)] +pub struct SyntaxError { + pub(crate) inner: SyntaxErrorKind, } #[derive(Debug)] -pub(crate) enum RdfXmlErrorKind { +pub enum SyntaxErrorKind { Xml(quick_xml::Error), XmlAttribute(quick_xml::events::attributes::AttrError), InvalidIri { @@ -24,84 +99,67 @@ pub(crate) enum RdfXmlErrorKind { tag: String, error: LanguageTagParseError, }, - Other(String), + Msg { + msg: String, + }, } -impl RdfXmlError { +impl SyntaxError { + /// Builds an error from a printable error message. + #[inline] pub(crate) fn msg(msg: impl Into) -> Self { Self { - kind: RdfXmlErrorKind::Other(msg.into()), + inner: SyntaxErrorKind::Msg { msg: msg.into() }, } } } -impl fmt::Display for RdfXmlError { +impl fmt::Display for SyntaxError { + #[inline] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match &self.kind { - RdfXmlErrorKind::Xml(error) => error.fmt(f), - RdfXmlErrorKind::XmlAttribute(error) => error.fmt(f), - RdfXmlErrorKind::InvalidIri { iri, error } => { + match &self.inner { + SyntaxErrorKind::Xml(error) => error.fmt(f), + SyntaxErrorKind::XmlAttribute(error) => error.fmt(f), + SyntaxErrorKind::InvalidIri { iri, error } => { write!(f, "error while parsing IRI '{}': {}", iri, error) } - RdfXmlErrorKind::InvalidLanguageTag { tag, error } => { + SyntaxErrorKind::InvalidLanguageTag { tag, error } => { write!(f, "error while parsing language tag '{}': {}", tag, error) } - RdfXmlErrorKind::Other(message) => write!(f, "{}", message), + SyntaxErrorKind::Msg { msg } => f.write_str(msg), } } } -impl Error for RdfXmlError { +impl Error for SyntaxError { + #[inline] fn source(&self) -> Option<&(dyn Error + 'static)> { - match &self.kind { - RdfXmlErrorKind::Xml(error) => Some(error), - RdfXmlErrorKind::XmlAttribute(error) => Some(error), - RdfXmlErrorKind::InvalidIri { error, .. } => Some(error), - RdfXmlErrorKind::InvalidLanguageTag { error, .. } => Some(error), - RdfXmlErrorKind::Other(_) => None, - } - } -} - -impl From for RdfXmlError { - fn from(error: quick_xml::Error) -> Self { - Self { - kind: RdfXmlErrorKind::Xml(error), - } - } -} - -impl From for RdfXmlError { - fn from(error: quick_xml::events::attributes::AttrError) -> Self { - Self { - kind: RdfXmlErrorKind::XmlAttribute(error), - } - } -} - -impl From for RdfXmlError { - fn from(error: io::Error) -> Self { - Self { - kind: RdfXmlErrorKind::Xml(quick_xml::Error::Io(Arc::new(error))), + match &self.inner { + SyntaxErrorKind::Xml(error) => Some(error), + SyntaxErrorKind::XmlAttribute(error) => Some(error), + SyntaxErrorKind::InvalidIri { error, .. } => Some(error), + SyntaxErrorKind::InvalidLanguageTag { error, .. } => Some(error), + SyntaxErrorKind::Msg { .. } => None, } } } -impl From for io::Error { - fn from(error: RdfXmlError) -> Self { - match error.kind { - RdfXmlErrorKind::Xml(error) => match error { +impl From for io::Error { + #[inline] + fn from(error: SyntaxError) -> Self { + match error.inner { + SyntaxErrorKind::Xml(error) => match error { quick_xml::Error::Io(error) => match Arc::try_unwrap(error) { Ok(error) => error, - Err(error) => io::Error::new(error.kind(), error), + Err(error) => Self::new(error.kind(), error), }, quick_xml::Error::UnexpectedEof(error) => { - io::Error::new(io::ErrorKind::UnexpectedEof, error) + Self::new(io::ErrorKind::UnexpectedEof, error) } - error => io::Error::new(io::ErrorKind::InvalidData, error), + error => Self::new(io::ErrorKind::InvalidData, error), }, - RdfXmlErrorKind::Other(error) => io::Error::new(io::ErrorKind::InvalidData, error), - _ => io::Error::new(io::ErrorKind::InvalidData, error), + SyntaxErrorKind::Msg { msg } => Self::new(io::ErrorKind::InvalidData, msg), + _ => Self::new(io::ErrorKind::InvalidData, error), } } } diff --git a/lib/oxrdfxml/src/lib.rs b/lib/oxrdfxml/src/lib.rs index e04e0d86..e07e31c8 100644 --- a/lib/oxrdfxml/src/lib.rs +++ b/lib/oxrdfxml/src/lib.rs @@ -10,5 +10,5 @@ mod serializer; mod utils; pub use crate::serializer::{RdfXmlSerializer, ToWriteRdfXmlWriter}; -pub use error::RdfXmlError; +pub use error::{ParseError, SyntaxError}; pub use parser::{FromReadRdfXmlReader, RdfXmlParser}; diff --git a/lib/oxrdfxml/src/parser.rs b/lib/oxrdfxml/src/parser.rs index 3493fe54..a726a7d0 100644 --- a/lib/oxrdfxml/src/parser.rs +++ b/lib/oxrdfxml/src/parser.rs @@ -1,4 +1,4 @@ -use crate::error::{RdfXmlError, RdfXmlErrorKind}; +use crate::error::{ParseError, SyntaxError, SyntaxErrorKind}; use crate::utils::*; use oxilangtag::LanguageTag; use oxiri::{Iri, IriParseError}; @@ -146,9 +146,9 @@ pub struct FromReadRdfXmlReader { } impl Iterator for FromReadRdfXmlReader { - type Item = Result; + type Item = Result; - fn next(&mut self) -> Option> { + fn next(&mut self) -> Option> { loop { if let Some(triple) = self.results.pop() { return Some(Ok(triple)); @@ -168,7 +168,7 @@ impl FromReadRdfXmlReader { self.reader.reader.buffer_position() } - fn parse_step(&mut self) -> Result<(), RdfXmlError> { + fn parse_step(&mut self) -> Result<(), ParseError> { self.reader_buffer.clear(); let event = self .reader @@ -295,7 +295,7 @@ struct RdfXmlReader { } impl RdfXmlReader { - fn parse_event(&mut self, event: Event, results: &mut Vec) -> Result<(), RdfXmlError> { + fn parse_event(&mut self, event: Event, results: &mut Vec) -> Result<(), ParseError> { match event { Event::Start(event) => self.parse_start_event(&event, results), Event::End(event) => self.parse_end_event(&event, results), @@ -306,9 +306,10 @@ impl RdfXmlReader { Event::Decl(decl) => { if let Some(encoding) = decl.encoding() { if !is_utf8(&encoding?) { - return Err(RdfXmlError::msg( + return Err(SyntaxError::msg( "Only UTF-8 is supported by the RDF/XML parser", - )); + ) + .into()); } } Ok(()) @@ -321,7 +322,7 @@ impl RdfXmlReader { } } - fn parse_doctype(&mut self, dt: &BytesText<'_>) -> Result<(), RdfXmlError> { + fn parse_doctype(&mut self, dt: &BytesText<'_>) -> Result<(), ParseError> { // we extract entities for input in self .reader @@ -333,20 +334,20 @@ impl RdfXmlReader { if let Some(input) = input.strip_prefix("!ENTITY") { let input = input.trim_start().strip_prefix('%').unwrap_or(input); let (entity_name, input) = input.trim_start().split_once(|c: char| c.is_ascii_whitespace()).ok_or_else(|| { - RdfXmlError::msg( + SyntaxError::msg( "').ok_or_else(|| { - RdfXmlError::msg("") + SyntaxError::msg("") })?; // Resolves custom entities within the current entity definition. @@ -363,7 +364,7 @@ impl RdfXmlReader { &mut self, event: &BytesStart<'_>, results: &mut Vec, - ) -> Result<(), RdfXmlError> { + ) -> Result<(), ParseError> { #[derive(PartialEq, Eq)] enum RdfXmlParseType { Default, @@ -425,15 +426,15 @@ impl RdfXmlReader { let tag = self.convert_attribute(&attribute)?; language = Some( LanguageTag::parse(tag.to_ascii_lowercase()) - .map_err(|error| RdfXmlError { - kind: RdfXmlErrorKind::InvalidLanguageTag { tag, error }, + .map_err(|error| SyntaxError { + inner: SyntaxErrorKind::InvalidLanguageTag { tag, error }, })? .into_inner(), ); } else if attribute.key.as_ref() == b"xml:base" { let iri = self.convert_attribute(&attribute)?; - base_iri = Some(Iri::parse(iri.clone()).map_err(|error| RdfXmlError { - kind: RdfXmlErrorKind::InvalidIri { iri, error }, + base_iri = Some(Iri::parse(iri.clone()).map_err(|error| SyntaxError { + inner: SyntaxErrorKind::InvalidIri { iri, error }, })?) } else { // We ignore other xml attributes @@ -443,28 +444,31 @@ impl RdfXmlReader { if *attribute_url == *RDF_ID { let mut id = self.convert_attribute(&attribute)?; if !is_nc_name(&id) { - return Err(RdfXmlError::msg(format!( + return Err(SyntaxError::msg(format!( "{} is not a valid rdf:ID value", &id - ))); + )) + .into()); } id.insert(0, '#'); id_attr = Some(id); } else if *attribute_url == *RDF_BAG_ID { let bag_id = self.convert_attribute(&attribute)?; if !is_nc_name(&bag_id) { - return Err(RdfXmlError::msg(format!( + return Err(SyntaxError::msg(format!( "{} is not a valid rdf:bagID value", &bag_id - ))); + )) + .into()); } } else if *attribute_url == *RDF_NODE_ID { let id = self.convert_attribute(&attribute)?; if !is_nc_name(&id) { - return Err(RdfXmlError::msg(format!( + return Err(SyntaxError::msg(format!( "{} is not a valid rdf:nodeID value", &id - ))); + )) + .into()); } node_id_attr = Some(BlankNode::new_unchecked(id)); } else if *attribute_url == *RDF_ABOUT { @@ -483,14 +487,15 @@ impl RdfXmlReader { } else if attribute_url == rdf::TYPE.as_str() { type_attr = Some(attribute); } else if RESERVED_RDF_ATTRIBUTES.contains(&&*attribute_url) { - return Err(RdfXmlError::msg(format!( + return Err(SyntaxError::msg(format!( "{} is not a valid attribute", &attribute_url - ))); + )) + .into()); } else { property_attrs.push(( - NamedNode::new(attribute_url.clone()).map_err(|error| RdfXmlError { - kind: RdfXmlErrorKind::InvalidIri { + NamedNode::new(attribute_url.clone()).map_err(|error| SyntaxError { + inner: SyntaxErrorKind::InvalidIri { iri: attribute_url, error, }, @@ -506,10 +511,11 @@ impl RdfXmlReader { Some(iri) => { let iri = resolve(&base_iri, iri)?; if self.known_rdf_id.contains(iri.as_str()) { - return Err(RdfXmlError::msg(format!( + return Err(SyntaxError::msg(format!( "{} has already been used as rdf:ID value", &iri - ))); + )) + .into()); } self.known_rdf_id.insert(iri.as_str().into()); Some(iri) @@ -547,9 +553,9 @@ impl RdfXmlReader { panic!("ParseTypeLiteralPropertyElt production children should never be considered as a RDF/XML content") } None => { - return Err(RdfXmlError::msg( - "No state in the stack: the XML is not balanced", - )); + return Err( + SyntaxError::msg("No state in the stack: the XML is not balanced").into(), + ); } }; @@ -558,14 +564,15 @@ impl RdfXmlReader { if *tag_name == *RDF_RDF { RdfXmlState::Rdf { base_iri, language } } else if RESERVED_RDF_ELEMENTS.contains(&&*tag_name) { - return Err(RdfXmlError::msg(format!( + return Err(SyntaxError::msg(format!( "Invalid node element tag name: {}", &tag_name - ))); + )) + .into()); } else { Self::build_node_elt( - NamedNode::new(tag_name.clone()).map_err(|error| RdfXmlError { - kind: RdfXmlErrorKind::InvalidIri { + NamedNode::new(tag_name.clone()).map_err(|error| SyntaxError { + inner: SyntaxErrorKind::InvalidIri { iri: tag_name, error, }, @@ -583,14 +590,15 @@ impl RdfXmlReader { } RdfXmlNextProduction::NodeElt => { if RESERVED_RDF_ELEMENTS.contains(&&*tag_name) { - return Err(RdfXmlError::msg(format!( + return Err(SyntaxError::msg(format!( "Invalid property element tag name: {}", &tag_name - ))); + )) + .into()); } Self::build_node_elt( - NamedNode::new(tag_name.clone()).map_err(|error| RdfXmlError { - kind: RdfXmlErrorKind::InvalidIri { + NamedNode::new(tag_name.clone()).map_err(|error| SyntaxError { + inner: SyntaxErrorKind::InvalidIri { iri: tag_name, error, }, @@ -608,10 +616,10 @@ impl RdfXmlReader { RdfXmlNextProduction::PropertyElt { subject } => { let iri = if *tag_name == *RDF_LI { let Some(RdfXmlState::NodeElt { li_counter, .. }) = self.state.last_mut() else { - return Err(RdfXmlError::msg(format!( + return Err(SyntaxError::msg(format!( "Invalid property element tag name: {}", &tag_name - ))); + )).into()); }; *li_counter += 1; NamedNode::new_unchecked(format!( @@ -621,13 +629,14 @@ impl RdfXmlReader { } else if RESERVED_RDF_ELEMENTS.contains(&&*tag_name) || *tag_name == *RDF_DESCRIPTION { - return Err(RdfXmlError::msg(format!( + return Err(SyntaxError::msg(format!( "Invalid property element tag name: {}", &tag_name - ))); + )) + .into()); } else { - NamedNode::new(tag_name.clone()).map_err(|error| RdfXmlError { - kind: RdfXmlErrorKind::InvalidIri { + NamedNode::new(tag_name.clone()).map_err(|error| SyntaxError { + inner: SyntaxErrorKind::InvalidIri { iri: tag_name, error, }, @@ -644,7 +653,7 @@ impl RdfXmlReader { (Some(resource_attr), None) => Subject::from(resource_attr), (None, Some(node_id_attr)) => node_id_attr.into(), (None, None) => BlankNode::default().into(), - (Some(_), Some(_)) => return Err(RdfXmlError::msg("Not both rdf:resource and rdf:nodeID could be set at the same time")) + (Some(_), Some(_)) => return Err(SyntaxError::msg("Not both rdf:resource and rdf:nodeID could be set at the same time").into()) }; Self::emit_property_attrs(&object, property_attrs, &language, results); if let Some(type_attr) = type_attr { @@ -711,7 +720,7 @@ impl RdfXmlReader { &mut self, event: &BytesEnd<'_>, results: &mut Vec, - ) -> Result<(), RdfXmlError> { + ) -> Result<(), ParseError> { //Literal case if self.in_literal_depth > 0 { if let Some(RdfXmlState::ParseTypeLiteralPropertyElt { writer, .. }) = @@ -731,7 +740,7 @@ impl RdfXmlReader { Ok(()) } - fn parse_text_event(&mut self, event: &BytesText<'_>) -> Result<(), RdfXmlError> { + fn parse_text_event(&mut self, event: &BytesText<'_>) -> Result<(), ParseError> { let text = event.unescape_with(|e| self.resolve_entity(e))?.to_string(); match self.state.last_mut() { Some(RdfXmlState::PropertyElt { object, .. }) => { @@ -748,21 +757,18 @@ impl RdfXmlReader { if event.iter().copied().all(is_whitespace) { Ok(()) } else { - Err(RdfXmlError::msg(format!( - "Unexpected text event: '{}'", - text - ))) + Err(SyntaxError::msg(format!("Unexpected text event: '{}'", text)).into()) } } } } - fn resolve_tag_name(&self, qname: QName<'_>) -> Result { + fn resolve_tag_name(&self, qname: QName<'_>) -> Result { let (namespace, local_name) = self.reader.resolve_element(qname); self.resolve_ns_name(namespace, local_name) } - fn resolve_attribute_name(&self, qname: QName<'_>) -> Result { + fn resolve_attribute_name(&self, qname: QName<'_>) -> Result { let (namespace, local_name) = self.reader.resolve_attribute(qname); self.resolve_ns_name(namespace, local_name) } @@ -771,7 +777,7 @@ impl RdfXmlReader { &self, namespace: ResolveResult, local_name: LocalName<'_>, - ) -> Result { + ) -> Result { match namespace { ResolveResult::Bound(ns) => { let mut value = Vec::with_capacity(ns.as_ref().len() + local_name.as_ref().len()); @@ -784,12 +790,13 @@ impl RdfXmlReader { .to_string()) } ResolveResult::Unbound => { - Err(RdfXmlError::msg("XML namespaces are required in RDF/XML")) + Err(SyntaxError::msg("XML namespaces are required in RDF/XML").into()) } - ResolveResult::Unknown(v) => Err(RdfXmlError::msg(format!( + ResolveResult::Unknown(v) => Err(SyntaxError::msg(format!( "Unknown prefix {}:", self.reader.decoder().decode(&v)? - ))), + )) + .into()), } } @@ -804,24 +811,24 @@ impl RdfXmlReader { type_attr: Option, property_attrs: Vec<(NamedNode, String)>, results: &mut Vec, - ) -> Result { + ) -> Result { let subject = match (id_attr, node_id_attr, about_attr) { (Some(id_attr), None, None) => Subject::from(id_attr), (None, Some(node_id_attr), None) => node_id_attr.into(), (None, None, Some(about_attr)) => about_attr.into(), (None, None, None) => BlankNode::default().into(), (Some(_), Some(_), _) => { - return Err(RdfXmlError::msg( + return Err(SyntaxError::msg( "Not both rdf:ID and rdf:nodeID could be set at the same time", )) } (_, Some(_), Some(_)) => { - return Err(RdfXmlError::msg( + return Err(SyntaxError::msg( "Not both rdf:nodeID and rdf:resource could be set at the same time", )) } (Some(_), _, Some(_)) => { - return Err(RdfXmlError::msg( + return Err(SyntaxError::msg( "Not both rdf:ID and rdf:resource could be set at the same time", )) } @@ -870,7 +877,7 @@ impl RdfXmlReader { &mut self, state: RdfXmlState, results: &mut Vec, - ) -> Result<(), RdfXmlError> { + ) -> Result<(), SyntaxError> { match state { RdfXmlState::PropertyElt { iri, @@ -925,7 +932,7 @@ impl RdfXmlReader { if emit { let object = writer.into_inner(); if object.is_empty() { - return Err(RdfXmlError::msg(format!( + return Err(SyntaxError::msg(format!( "No value found for rdf:XMLLiteral value of property {}", iri ))); @@ -935,7 +942,7 @@ impl RdfXmlReader { iri, Literal::new_typed_literal( str::from_utf8(&object).map_err(|_| { - RdfXmlError::msg("The XML literal is not in valid UTF-8".to_owned()) + SyntaxError::msg("The XML literal is not in valid UTF-8".to_owned()) })?, rdf::XML_LITERAL, ), @@ -1008,7 +1015,7 @@ impl RdfXmlReader { } } - fn convert_attribute(&self, attribute: &Attribute) -> Result { + fn convert_attribute(&self, attribute: &Attribute) -> Result { Ok(attribute .decode_and_unescape_value_with(&self.reader, |e| self.resolve_entity(e))? .to_string()) @@ -1018,8 +1025,8 @@ impl RdfXmlReader { &self, base_iri: &Option>, attribute: &Attribute<'_>, - ) -> Result { - resolve(base_iri, self.convert_attribute(attribute)?) + ) -> Result { + Ok(resolve(base_iri, self.convert_attribute(attribute)?)?) } fn resolve_entity(&self, e: &str) -> Option<&str> { @@ -1027,13 +1034,13 @@ impl RdfXmlReader { } } -fn resolve(base_iri: &Option>, relative_iri: String) -> Result { +fn resolve(base_iri: &Option>, relative_iri: String) -> Result { if let Some(base_iri) = base_iri { Ok(NamedNode::new_unchecked( base_iri .resolve(&relative_iri) - .map_err(|error| RdfXmlError { - kind: RdfXmlErrorKind::InvalidIri { + .map_err(|error| SyntaxError { + inner: SyntaxErrorKind::InvalidIri { iri: relative_iri, error, }, @@ -1041,8 +1048,8 @@ fn resolve(base_iri: &Option>, relative_iri: String) -> Result { } impl Iterator for FromReadN3Reader { - type Item = Result; + type Item = Result; - fn next(&mut self) -> Option> { + fn next(&mut self) -> Option> { self.inner.next() } } @@ -406,7 +406,7 @@ impl LowLevelN3Reader { /// /// Returns [`None`] if the parsing is finished or more data is required. /// If it is the case more data should be fed using [`extend_from_slice`](Self::extend_from_slice). - pub fn read_next(&mut self) -> Option> { + pub fn read_next(&mut self) -> Option> { self.parser.read_next() } } diff --git a/lib/oxttl/src/nquads.rs b/lib/oxttl/src/nquads.rs index ce8c3a2a..4414d3fc 100644 --- a/lib/oxttl/src/nquads.rs +++ b/lib/oxttl/src/nquads.rs @@ -1,7 +1,7 @@ //! A [N-Quads](https://www.w3.org/TR/n-quads/) streaming parser implemented by [`NQuadsParser`]. use crate::line_formats::NQuadsRecognizer; -use crate::toolkit::{FromReadIterator, ParseError, ParseOrIoError, Parser}; +use crate::toolkit::{FromReadIterator, ParseError, Parser, SyntaxError}; use oxrdf::{Quad, QuadRef}; use std::io::{self, Read, Write}; @@ -157,9 +157,9 @@ pub struct FromReadNQuadsReader { } impl Iterator for FromReadNQuadsReader { - type Item = Result; + type Item = Result; - fn next(&mut self) -> Option> { + fn next(&mut self) -> Option> { self.inner.next() } } @@ -226,7 +226,7 @@ impl LowLevelNQuadsReader { /// /// Returns [`None`] if the parsing is finished or more data is required. /// If it is the case more data should be fed using [`extend_from_slice`](Self::extend_from_slice). - pub fn read_next(&mut self) -> Option> { + pub fn read_next(&mut self) -> Option> { self.parser.read_next() } } diff --git a/lib/oxttl/src/ntriples.rs b/lib/oxttl/src/ntriples.rs index daaa2d5c..cac8cd22 100644 --- a/lib/oxttl/src/ntriples.rs +++ b/lib/oxttl/src/ntriples.rs @@ -2,7 +2,7 @@ //! and a serializer implemented by [`NTriplesSerializer`]. use crate::line_formats::NQuadsRecognizer; -use crate::toolkit::{FromReadIterator, ParseError, ParseOrIoError, Parser}; +use crate::toolkit::{FromReadIterator, ParseError, Parser, SyntaxError}; use oxrdf::{Triple, TripleRef}; use std::io::{self, Read, Write}; @@ -158,9 +158,9 @@ pub struct FromReadNTriplesReader { } impl Iterator for FromReadNTriplesReader { - type Item = Result; + type Item = Result; - fn next(&mut self) -> Option> { + fn next(&mut self) -> Option> { Some(self.inner.next()?.map(Into::into)) } } @@ -227,7 +227,7 @@ impl LowLevelNTriplesReader { /// /// Returns [`None`] if the parsing is finished or more data is required. /// If it is the case more data should be fed using [`extend_from_slice`](Self::extend_from_slice). - pub fn read_next(&mut self) -> Option> { + pub fn read_next(&mut self) -> Option> { Some(self.parser.read_next()?.map(Into::into)) } } diff --git a/lib/oxttl/src/toolkit/lexer.rs b/lib/oxttl/src/toolkit/lexer.rs index 26906386..3510ab28 100644 --- a/lib/oxttl/src/toolkit/lexer.rs +++ b/lib/oxttl/src/toolkit/lexer.rs @@ -92,6 +92,7 @@ impl Lexer { self.end = self.data.len(); } + #[inline] pub fn end(&mut self) { self.is_ending = true; } diff --git a/lib/oxttl/src/toolkit/mod.rs b/lib/oxttl/src/toolkit/mod.rs index 986504f9..39f9d40d 100644 --- a/lib/oxttl/src/toolkit/mod.rs +++ b/lib/oxttl/src/toolkit/mod.rs @@ -7,5 +7,5 @@ mod parser; pub use self::lexer::{Lexer, LexerError, TokenRecognizer, TokenRecognizerError}; pub use self::parser::{ - FromReadIterator, ParseError, ParseOrIoError, Parser, RuleRecognizer, RuleRecognizerError, + FromReadIterator, ParseError, Parser, RuleRecognizer, RuleRecognizerError, SyntaxError, }; diff --git a/lib/oxttl/src/toolkit/parser.rs b/lib/oxttl/src/toolkit/parser.rs index a02afec4..44c01d5a 100644 --- a/lib/oxttl/src/toolkit/parser.rs +++ b/lib/oxttl/src/toolkit/parser.rs @@ -60,18 +60,20 @@ impl Parser { self.lexer.extend_from_slice(other) } + #[inline] pub fn end(&mut self) { self.lexer.end() } + #[inline] pub fn is_end(&self) -> bool { self.state.is_none() && self.results.is_empty() && self.errors.is_empty() } - pub fn read_next(&mut self) -> Option> { + pub fn read_next(&mut self) -> Option> { loop { if let Some(error) = self.errors.pop() { - return Some(Err(ParseError { + return Some(Err(SyntaxError { position: self.position.clone(), message: error.message, })); @@ -114,33 +116,37 @@ impl Parser { } } -/// An error from parsing. +/// An error in the syntax of the parsed file. /// /// It is composed of a message and a byte range in the input. #[derive(Debug)] -pub struct ParseError { +pub struct SyntaxError { position: Range, message: String, } -impl ParseError { +impl SyntaxError { /// The invalid byte range in the input. + #[inline] pub fn position(&self) -> Range { self.position.clone() } /// The error message. + #[inline] pub fn message(&self) -> &str { &self.message } /// Converts this error to an error message. + #[inline] pub fn into_message(self) -> String { self.message } } -impl fmt::Display for ParseError { +impl fmt::Display for SyntaxError { + #[inline] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { if self.position.start + 1 == self.position.end { write!( @@ -158,15 +164,17 @@ impl fmt::Display for ParseError { } } -impl Error for ParseError {} +impl Error for SyntaxError {} -impl From for io::Error { - fn from(error: ParseError) -> Self { +impl From for io::Error { + #[inline] + fn from(error: SyntaxError) -> Self { io::Error::new(io::ErrorKind::InvalidData, error) } } -impl From for ParseError { +impl From for SyntaxError { + #[inline] fn from(e: LexerError) -> Self { Self { position: e.position(), @@ -175,48 +183,57 @@ impl From for ParseError { } } -/// The union of [`ParseError`] and [`std::io::Error`]. +/// A parsing error. +/// +/// It is the union of [`SyntaxError`] and [`std::io::Error`]. #[derive(Debug)] -pub enum ParseOrIoError { - Parse(ParseError), +pub enum ParseError { + /// I/O error during parsing (file not found...). Io(io::Error), + /// An error in the file syntax. + Syntax(SyntaxError), } -impl fmt::Display for ParseOrIoError { +impl fmt::Display for ParseError { + #[inline] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - Self::Parse(e) => e.fmt(f), Self::Io(e) => e.fmt(f), + Self::Syntax(e) => e.fmt(f), } } } -impl Error for ParseOrIoError { +impl Error for ParseError { + #[inline] fn source(&self) -> Option<&(dyn Error + 'static)> { Some(match self { - Self::Parse(e) => e, Self::Io(e) => e, + Self::Syntax(e) => e, }) } } -impl From for ParseOrIoError { - fn from(error: ParseError) -> Self { - Self::Parse(error) +impl From for ParseError { + #[inline] + fn from(error: SyntaxError) -> Self { + Self::Syntax(error) } } -impl From for ParseOrIoError { +impl From for ParseError { + #[inline] fn from(error: io::Error) -> Self { Self::Io(error) } } -impl From for io::Error { - fn from(error: ParseOrIoError) -> Self { +impl From for io::Error { + #[inline] + fn from(error: ParseError) -> Self { match error { - ParseOrIoError::Parse(e) => e.into(), - ParseOrIoError::Io(e) => e, + ParseError::Syntax(e) => e.into(), + ParseError::Io(e) => e, } } } @@ -227,12 +244,12 @@ pub struct FromReadIterator { } impl Iterator for FromReadIterator { - type Item = Result; + type Item = Result; fn next(&mut self) -> Option { while !self.parser.is_end() { if let Some(result) = self.parser.read_next() { - return Some(result.map_err(ParseOrIoError::Parse)); + return Some(result.map_err(ParseError::Syntax)); } if let Err(e) = self.parser.lexer.extend_from_read(&mut self.read) { return Some(Err(e.into())); diff --git a/lib/oxttl/src/trig.rs b/lib/oxttl/src/trig.rs index 311b8f96..62fc331f 100644 --- a/lib/oxttl/src/trig.rs +++ b/lib/oxttl/src/trig.rs @@ -1,7 +1,7 @@ //! A [TriG](https://www.w3.org/TR/trig/) streaming parser implemented by [`TriGParser`]. use crate::terse::TriGRecognizer; -use crate::toolkit::{FromReadIterator, ParseError, ParseOrIoError, Parser}; +use crate::toolkit::{FromReadIterator, ParseError, Parser, SyntaxError}; use oxiri::{Iri, IriParseError}; use oxrdf::{vocab::xsd, GraphName, NamedNode, Quad, QuadRef, Subject, TermRef}; use std::collections::HashMap; @@ -186,9 +186,9 @@ pub struct FromReadTriGReader { } impl Iterator for FromReadTriGReader { - type Item = Result; + type Item = Result; - fn next(&mut self) -> Option> { + fn next(&mut self) -> Option> { self.inner.next() } } @@ -255,7 +255,7 @@ impl LowLevelTriGReader { /// /// Returns [`None`] if the parsing is finished or more data is required. /// If it is the case more data should be fed using [`extend_from_slice`](Self::extend_from_slice). - pub fn read_next(&mut self) -> Option> { + pub fn read_next(&mut self) -> Option> { self.parser.read_next() } } diff --git a/lib/oxttl/src/turtle.rs b/lib/oxttl/src/turtle.rs index 65967613..3b5639c0 100644 --- a/lib/oxttl/src/turtle.rs +++ b/lib/oxttl/src/turtle.rs @@ -1,7 +1,7 @@ //! A [Turtle](https://www.w3.org/TR/turtle/) streaming parser implemented by [`TurtleParser`]. use crate::terse::TriGRecognizer; -use crate::toolkit::{FromReadIterator, ParseError, ParseOrIoError, Parser}; +use crate::toolkit::{FromReadIterator, ParseError, Parser, SyntaxError}; use crate::trig::{LowLevelTriGWriter, ToWriteTriGWriter}; use crate::TriGSerializer; use oxiri::{Iri, IriParseError}; @@ -187,9 +187,9 @@ pub struct FromReadTurtleReader { } impl Iterator for FromReadTurtleReader { - type Item = Result; + type Item = Result; - fn next(&mut self) -> Option> { + fn next(&mut self) -> Option> { Some(self.inner.next()?.map(Into::into)) } } @@ -256,7 +256,7 @@ impl LowLevelTurtleReader { /// /// Returns [`None`] if the parsing is finished or more data is required. /// If it is the case more data should be fed using [`extend_from_slice`](Self::extend_from_slice). - pub fn read_next(&mut self) -> Option> { + pub fn read_next(&mut self) -> Option> { Some(self.parser.read_next()?.map(Into::into)) } } diff --git a/lib/sparesults/src/error.rs b/lib/sparesults/src/error.rs index 24ed05fa..fbb45728 100644 --- a/lib/sparesults/src/error.rs +++ b/lib/sparesults/src/error.rs @@ -1,5 +1,6 @@ use oxrdf::TermParseError; use std::error::Error; +use std::sync::Arc; use std::{fmt, io}; /// Error returned during SPARQL result formats format parsing. @@ -59,7 +60,10 @@ impl From for ParseError { #[inline] fn from(error: quick_xml::Error) -> Self { match error { - quick_xml::Error::Io(error) => Self::Io(io::Error::new(error.kind(), error)), + quick_xml::Error::Io(error) => Self::Io(match Arc::try_unwrap(error) { + Ok(error) => error, + Err(error) => io::Error::new(error.kind(), error), + }), error => Self::Syntax(SyntaxError { inner: SyntaxErrorKind::Xml(error), }), @@ -117,7 +121,10 @@ impl From for io::Error { fn from(error: SyntaxError) -> Self { match error.inner { SyntaxErrorKind::Xml(error) => match error { - quick_xml::Error::Io(error) => Self::new(error.kind(), error), + quick_xml::Error::Io(error) => match Arc::try_unwrap(error) { + Ok(error) => error, + Err(error) => Self::new(error.kind(), error), + }, quick_xml::Error::UnexpectedEof(error) => { Self::new(io::ErrorKind::UnexpectedEof, error) } diff --git a/lib/sparesults/src/xml.rs b/lib/sparesults/src/xml.rs index dbc17242..f7e0bb41 100644 --- a/lib/sparesults/src/xml.rs +++ b/lib/sparesults/src/xml.rs @@ -10,6 +10,7 @@ use std::borrow::Cow; use std::collections::BTreeMap; use std::io::{self, BufRead, Write}; use std::str; +use std::sync::Arc; pub fn write_boolean_xml_result(sink: W, value: bool) -> io::Result { do_write_boolean_xml_result(sink, value).map_err(map_xml_error) @@ -604,7 +605,10 @@ fn decode<'a, T>( fn map_xml_error(error: quick_xml::Error) -> io::Error { match error { - quick_xml::Error::Io(error) => io::Error::new(error.kind(), error), + quick_xml::Error::Io(error) => match Arc::try_unwrap(error) { + Ok(error) => error, + Err(error) => io::Error::new(error.kind(), error), + }, quick_xml::Error::UnexpectedEof(_) => io::Error::new(io::ErrorKind::UnexpectedEof, error), _ => io::Error::new(io::ErrorKind::InvalidData, error), } diff --git a/lib/src/io/error.rs b/lib/src/io/error.rs index 6d49148d..bea22d8e 100644 --- a/lib/src/io/error.rs +++ b/lib/src/io/error.rs @@ -1,5 +1,4 @@ use oxiri::IriParseError; -use oxrdfxml::RdfXmlError; use std::error::Error; use std::{fmt, io}; @@ -44,39 +43,40 @@ impl Error for ParseError { } } -impl From for ParseError { +impl From for SyntaxError { #[inline] - fn from(error: oxttl::ParseError) -> Self { - Self::Syntax(SyntaxError { + fn from(error: oxttl::SyntaxError) -> Self { + SyntaxError { inner: SyntaxErrorKind::Turtle(error), - }) + } } } -impl From for ParseError { +impl From for ParseError { #[inline] - fn from(error: oxttl::ParseOrIoError) -> Self { + fn from(error: oxttl::ParseError) -> Self { match error { - oxttl::ParseOrIoError::Parse(e) => e.into(), - oxttl::ParseOrIoError::Io(e) => e.into(), + oxttl::ParseError::Syntax(e) => Self::Syntax(e.into()), + oxttl::ParseError::Io(e) => Self::Io(e), } } } -#[allow(clippy::fallible_impl_from)] -impl From for ParseError { +impl From for SyntaxError { #[inline] - fn from(error: RdfXmlError) -> Self { - let error = io::Error::from(error); - if error.get_ref().map_or( - false, - <(dyn Error + Send + Sync + 'static)>::is::, - ) { - Self::Syntax(SyntaxError { - inner: SyntaxErrorKind::RdfXml(*error.into_inner().unwrap().downcast().unwrap()), - }) - } else { - Self::Io(error) + fn from(error: oxrdfxml::SyntaxError) -> Self { + SyntaxError { + inner: SyntaxErrorKind::RdfXml(error), + } + } +} + +impl From for ParseError { + #[inline] + fn from(error: oxrdfxml::ParseError) -> Self { + match error { + oxrdfxml::ParseError::Syntax(e) => Self::Syntax(e.into()), + oxrdfxml::ParseError::Io(e) => Self::Io(e), } } } @@ -113,8 +113,8 @@ pub struct SyntaxError { #[derive(Debug)] enum SyntaxErrorKind { - Turtle(oxttl::ParseError), - RdfXml(RdfXmlError), + Turtle(oxttl::SyntaxError), + RdfXml(oxrdfxml::SyntaxError), InvalidBaseIri { iri: String, error: IriParseError }, } From db7fab0f2016b58757bb0ead3b487fca22c8c01c Mon Sep 17 00:00:00 2001 From: Tpt Date: Sat, 8 Jul 2023 14:07:43 +0200 Subject: [PATCH 026/217] Run Clippy on more configurations --- .github/workflows/tests.yml | 19 ++++++++++++++++++- lib/oxrdf/src/literal.rs | 16 ++++++++-------- python/src/lib.rs | 6 ++++++ python/src/model.rs | 2 +- testsuite/src/files.rs | 12 +++--------- 5 files changed, 36 insertions(+), 19 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7098e3fd..e752ed0b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -42,7 +42,16 @@ jobs: working-directory: ./lib/spargebra - run: cargo clippy working-directory: ./lib/sparopt + - run: cargo clippy + working-directory: ./lib - run: cargo clippy --all-targets --all-features + working-directory: ./lib + - run: cargo clippy + working-directory: ./python + - run: cargo clippy + working-directory: ./server + - run: cargo clippy + working-directory: ./testsuite clippy_wasm_js: runs-on: ubuntu-latest @@ -88,8 +97,16 @@ jobs: working-directory: ./lib/spargebra - run: cargo clippy -- -D warnings -D clippy::all working-directory: ./lib/sparopt - - run: cargo clippy --all-targets -- -D warnings -D clippy::all + - run: cargo clippy -- -D warnings -D clippy::all + working-directory: ./lib + - run: cargo clippy --all-targets --all-features -- -D warnings -D clippy::all + working-directory: ./lib + - run: cargo clippy -- -D warnings -D clippy::all + working-directory: ./python + - run: cargo clippy -- -D warnings -D clippy::all working-directory: ./server + - run: cargo clippy -- -D warnings -D clippy::all + working-directory: ./testsuite clippy_msv_wasm_js: runs-on: ubuntu-latest diff --git a/lib/oxrdf/src/literal.rs b/lib/oxrdf/src/literal.rs index 607c3a69..de37adc8 100644 --- a/lib/oxrdf/src/literal.rs +++ b/lib/oxrdf/src/literal.rs @@ -673,35 +673,35 @@ mod tests { fn test_canoincal_escaping() { assert_eq!( Literal::from_str(r#""\u0000\u0001\u0002\u0003\u0004\u0005\u0006\u0007\u0008\u0009\u000a\u000b\u000c\u000d\u000e\u000f""#).unwrap().to_string(), - r###""\u0000\u0001\u0002\u0003\u0004\u0005\u0006\u0007\b\t\n\u000B\f\r\u000E\u000F""### + r#""\u0000\u0001\u0002\u0003\u0004\u0005\u0006\u0007\b\t\n\u000B\f\r\u000E\u000F""# ); assert_eq!( Literal::from_str(r#""\u0010\u0011\u0012\u0013\u0014\u0015\u0016\u0017\u0018\u0019\u001a\u001b\u001c\u001d\u001e\u001f""#).unwrap().to_string(), - r###""\u0010\u0011\u0012\u0013\u0014\u0015\u0016\u0017\u0018\u0019\u001A\u001B\u001C\u001D\u001E\u001F""### + r#""\u0010\u0011\u0012\u0013\u0014\u0015\u0016\u0017\u0018\u0019\u001A\u001B\u001C\u001D\u001E\u001F""# ); assert_eq!( Literal::from_str(r#""\u0020\u0021\u0022\u0023\u0024\u0025\u0026\u0027\u0028\u0029\u002a\u002b\u002c\u002d\u002e\u002f""#).unwrap().to_string(), - r###"" !\"#$%&'()*+,-./""### + r##"" !\"#$%&'()*+,-./""## ); assert_eq!( Literal::from_str(r#""\u0030\u0031\u0032\u0033\u0034\u0035\u0036\u0037\u0038\u0039\u003a\u003b\u003c\u003d\u003e\u003f""#).unwrap().to_string(), - r###""0123456789:;<=>?""### + r#""0123456789:;<=>?""# ); assert_eq!( Literal::from_str(r#""\u0040\u0041\u0042\u0043\u0044\u0045\u0046\u0047\u0048\u0049\u004a\u004b\u004c\u004d\u004e\u004f""#).unwrap().to_string(), - r###""@ABCDEFGHIJKLMNO""### + r#""@ABCDEFGHIJKLMNO""# ); assert_eq!( Literal::from_str(r#""\u0050\u0051\u0052\u0053\u0054\u0055\u0056\u0057\u0058\u0059\u005a\u005b\u005c\u005d\u005e\u005f""#).unwrap().to_string(), - r###""PQRSTUVWXYZ[\\]^_""### + r#""PQRSTUVWXYZ[\\]^_""# ); assert_eq!( Literal::from_str(r#""\u0060\u0061\u0062\u0063\u0064\u0065\u0066\u0067\u0068\u0069\u006a\u006b\u006c\u006d\u006e\u006f""#).unwrap().to_string(), - r###""`abcdefghijklmno""### + r#""`abcdefghijklmno""# ); assert_eq!( Literal::from_str(r#""\u0070\u0071\u0072\u0073\u0074\u0075\u0076\u0077\u0078\u0079\u007a\u007b\u007c\u007d\u007e\u007f""#).unwrap().to_string(), - r###""pqrstuvwxyz{|}~\u007F""### + r#""pqrstuvwxyz{|}~\u007F""# ); assert_eq!( Literal::from_str(r#""\u0080\u0081\u0082\u0083\u0084\u0085\u0086\u0087\u0088\u0089\u008a\u008b\u008c\u008d\u008e\u008f""#).unwrap().to_string(), diff --git a/python/src/lib.rs b/python/src/lib.rs index 170d78b8..736cbfc9 100644 --- a/python/src/lib.rs +++ b/python/src/lib.rs @@ -1,3 +1,9 @@ +#![allow( + clippy::used_underscore_binding, + clippy::unused_self, + clippy::trivially_copy_pass_by_ref +)] + mod io; mod model; mod sparql; diff --git a/python/src/model.rs b/python/src/model.rs index 9bebf1db..e2694c61 100644 --- a/python/src/model.rs +++ b/python/src/model.rs @@ -426,7 +426,7 @@ impl PyLiteral { /// The RDF `default graph name `_. #[pyclass(frozen, name = "DefaultGraph", module = "pyoxigraph")] #[derive(Eq, PartialEq, Debug, Clone, Copy, Hash)] -pub struct PyDefaultGraph {} +pub struct PyDefaultGraph; impl From for GraphName { fn from(_: PyDefaultGraph) -> Self { diff --git a/testsuite/src/files.rs b/testsuite/src/files.rs index 59653814..f7101e9d 100644 --- a/testsuite/src/files.rs +++ b/testsuite/src/files.rs @@ -47,9 +47,7 @@ pub fn load_to_graph( graph.insert(&t); } Err(e) => { - if ignore_errors { - continue; - } else { + if !ignore_errors { return Err(e.into()); } } @@ -83,9 +81,7 @@ pub fn load_to_dataset( dataset.insert(&q); } Err(e) => { - if ignore_errors { - continue; - } else { + if !ignore_errors { return Err(e.into()); } } @@ -116,9 +112,7 @@ pub fn load_n3(url: &str, ignore_errors: bool) -> Result> { match q { Ok(q) => quads.push(q), Err(e) => { - if ignore_errors { - continue; - } else { + if !ignore_errors { return Err(e.into()); } } From c8e718ed2dfb8e289ca51a79a66a0446e6df6992 Mon Sep 17 00:00:00 2001 From: Tpt Date: Sat, 8 Jul 2023 15:09:41 +0200 Subject: [PATCH 027/217] Properly document features in docs.rs --- lib/Cargo.toml | 7 ++++--- lib/oxrdf/Cargo.toml | 1 + lib/oxrdfxml/Cargo.toml | 1 + lib/oxsdatatypes/Cargo.toml | 1 + lib/oxttl/Cargo.toml | 1 + lib/sparesults/Cargo.toml | 1 + lib/spargebra/Cargo.toml | 1 + lib/sparopt/Cargo.toml | 1 + lib/src/lib.rs | 4 ++-- 9 files changed, 13 insertions(+), 5 deletions(-) diff --git a/lib/Cargo.toml b/lib/Cargo.toml index f4d3cae1..1ea58475 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -14,9 +14,6 @@ a SPARQL database and RDF toolkit edition = "2021" rust-version = "1.65" -[package.metadata.docs.rs] -all-features = true - [features] default = [] http_client = ["oxhttp", "oxhttp/rustls"] @@ -57,6 +54,10 @@ criterion = "0.5" oxhttp = "0.1" zstd = "0.12" +[package.metadata.docs.rs] +all-features = true +rustdoc-args = ["--cfg", "docsrs"] + [[bench]] name = "store" harness = false diff --git a/lib/oxrdf/Cargo.toml b/lib/oxrdf/Cargo.toml index 9e429d7d..344d46fe 100644 --- a/lib/oxrdf/Cargo.toml +++ b/lib/oxrdf/Cargo.toml @@ -25,3 +25,4 @@ oxsdatatypes = { version = "0.2.0-alpha.1-dev", path="../oxsdatatypes", optional [package.metadata.docs.rs] all-features = true +rustdoc-args = ["--cfg", "docsrs"] \ No newline at end of file diff --git a/lib/oxrdfxml/Cargo.toml b/lib/oxrdfxml/Cargo.toml index 25e19c72..3d012e17 100644 --- a/lib/oxrdfxml/Cargo.toml +++ b/lib/oxrdfxml/Cargo.toml @@ -21,3 +21,4 @@ quick-xml = "0.29" [package.metadata.docs.rs] all-features = true +rustdoc-args = ["--cfg", "docsrs"] diff --git a/lib/oxsdatatypes/Cargo.toml b/lib/oxsdatatypes/Cargo.toml index 86c7d292..1535a325 100644 --- a/lib/oxsdatatypes/Cargo.toml +++ b/lib/oxsdatatypes/Cargo.toml @@ -18,3 +18,4 @@ js-sys = "0.3" [package.metadata.docs.rs] all-features = true +rustdoc-args = ["--cfg", "docsrs"] diff --git a/lib/oxttl/Cargo.toml b/lib/oxttl/Cargo.toml index 09d2798d..8717a9b6 100644 --- a/lib/oxttl/Cargo.toml +++ b/lib/oxttl/Cargo.toml @@ -25,3 +25,4 @@ oxilangtag = "0.1" [package.metadata.docs.rs] all-features = true +rustdoc-args = ["--cfg", "docsrs"] diff --git a/lib/sparesults/Cargo.toml b/lib/sparesults/Cargo.toml index 29973e23..c345b2a0 100644 --- a/lib/sparesults/Cargo.toml +++ b/lib/sparesults/Cargo.toml @@ -24,3 +24,4 @@ quick-xml = "0.29" [package.metadata.docs.rs] all-features = true +rustdoc-args = ["--cfg", "docsrs"] diff --git a/lib/spargebra/Cargo.toml b/lib/spargebra/Cargo.toml index 5fa672b5..d13d8037 100644 --- a/lib/spargebra/Cargo.toml +++ b/lib/spargebra/Cargo.toml @@ -28,3 +28,4 @@ oxrdf = { version = "0.2.0-alpha.1-dev", path="../oxrdf" } [package.metadata.docs.rs] all-features = true +rustdoc-args = ["--cfg", "docsrs"] diff --git a/lib/sparopt/Cargo.toml b/lib/sparopt/Cargo.toml index 3e30b5ac..06934952 100644 --- a/lib/sparopt/Cargo.toml +++ b/lib/sparopt/Cargo.toml @@ -26,3 +26,4 @@ spargebra = { version = "0.3.0-alpha.1-dev", path="../spargebra" } [package.metadata.docs.rs] all-features = true +rustdoc-args = ["--cfg", "docsrs"] diff --git a/lib/src/lib.rs b/lib/src/lib.rs index d5578fb2..1ccbe197 100644 --- a/lib/src/lib.rs +++ b/lib/src/lib.rs @@ -1,8 +1,8 @@ #![doc = include_str!("../README.md")] +#![doc(test(attr(deny(warnings))))] +#![cfg_attr(docsrs, feature(doc_auto_cfg))] #![doc(html_favicon_url = "https://raw.githubusercontent.com/oxigraph/oxigraph/main/logo.svg")] #![doc(html_logo_url = "https://raw.githubusercontent.com/oxigraph/oxigraph/main/logo.svg")] -#![cfg_attr(docsrs, feature(doc_auto_cfg))] -#![doc(test(attr(deny(warnings))))] pub mod io; pub mod sparql; From 24a1dd2556e664f226354a0265e21cfa80b85dc5 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 10 Jul 2023 14:27:33 +0200 Subject: [PATCH 028/217] Applies some recent Clippy lints --- lib/spargebra/src/query.rs | 2 +- lib/src/sparql/plan.rs | 2 +- testsuite/benches/parser.rs | 32 ++++++++++++++++--------------- testsuite/src/sparql_evaluator.rs | 2 +- 4 files changed, 20 insertions(+), 18 deletions(-) diff --git a/lib/spargebra/src/query.rs b/lib/spargebra/src/query.rs index 9f927a8d..6cb7e57b 100644 --- a/lib/spargebra/src/query.rs +++ b/lib/spargebra/src/query.rs @@ -210,7 +210,7 @@ impl fmt::Display for Query { writeln!(f, "BASE <{base_iri}>")?; } write!(f, "CONSTRUCT {{ ")?; - for triple in template.iter() { + for triple in template { write!(f, "{triple} . ")?; } write!(f, "}}")?; diff --git a/lib/src/sparql/plan.rs b/lib/src/sparql/plan.rs index af51e7a4..905e4da0 100644 --- a/lib/src/sparql/plan.rs +++ b/lib/src/sparql/plan.rs @@ -155,7 +155,7 @@ impl PlanNode { child.lookup_used_variables(callback); } Self::Union { children } => { - for child in children.iter() { + for child in children { child.lookup_used_variables(callback); } } diff --git a/testsuite/benches/parser.rs b/testsuite/benches/parser.rs index 63638647..ed7205eb 100644 --- a/testsuite/benches/parser.rs +++ b/testsuite/benches/parser.rs @@ -1,3 +1,5 @@ +#![allow(clippy::print_stderr)] + use anyhow::Result; use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; use oxigraph_testsuite::files::read_file; @@ -40,7 +42,7 @@ fn parse_bench( c: &mut Criterion, parser_name: &str, data_name: &str, - data: Vec, + data: &[u8], bench: impl Fn(&[u8]), ) { let mut group = c.benchmark_group(parser_name); @@ -51,7 +53,7 @@ fn parse_bench( group.finish(); } -fn parse_oxttl_ntriples(c: &mut Criterion, name: &str, data: Vec) { +fn parse_oxttl_ntriples(c: &mut Criterion, name: &str, data: &[u8]) { parse_bench(c, "oxttl ntriples", name, data, |data| { let mut parser = oxttl::NTriplesParser::new().parse(); parser.extend_from_slice(data); @@ -62,7 +64,7 @@ fn parse_oxttl_ntriples(c: &mut Criterion, name: &str, data: Vec) { }); } -fn parse_oxttl_turtle(c: &mut Criterion, name: &str, data: Vec) { +fn parse_oxttl_turtle(c: &mut Criterion, name: &str, data: &[u8]) { parse_bench(c, "oxttl turtle", name, data, |data| { let mut parser = oxttl::TurtleParser::new().parse(); parser.extend_from_slice(data); @@ -73,25 +75,25 @@ fn parse_oxttl_turtle(c: &mut Criterion, name: &str, data: Vec) { }); } -fn parse_rio_ntriples(c: &mut Criterion, name: &str, data: Vec) { +fn parse_rio_ntriples(c: &mut Criterion, name: &str, data: &[u8]) { parse_bench(c, "rio ntriples", name, data, |data| { let mut count: u64 = 0; NTriplesParser::new(data) - .parse_all(&mut |_| { + .parse_all::(&mut |_| { count += 1; - Ok(()) as Result<(), TurtleError> + Ok(()) }) .unwrap(); }); } -fn parse_rio_turtle(c: &mut Criterion, name: &str, data: Vec) { +fn parse_rio_turtle(c: &mut Criterion, name: &str, data: &[u8]) { parse_bench(c, "rio turtle", name, data, |data| { let mut count: u64 = 0; TurtleParser::new(data, None) - .parse_all(&mut |_| { + .parse_all::(&mut |_| { count += 1; - Ok(()) as Result<(), TurtleError> + Ok(()) }) .unwrap(); }); @@ -101,7 +103,7 @@ fn bench_parse_oxttl_ntriples_with_ntriples(c: &mut Criterion) { parse_oxttl_ntriples( c, "ntriples", - match ntriples_test_data() { + &match ntriples_test_data() { Ok(d) => d, Err(e) => { eprintln!("{e}"); @@ -115,7 +117,7 @@ fn bench_parse_oxttl_ntriples_with_turtle(c: &mut Criterion) { parse_oxttl_turtle( c, "ntriples", - match ntriples_test_data() { + &match ntriples_test_data() { Ok(d) => d, Err(e) => { eprintln!("{e}"); @@ -129,7 +131,7 @@ fn bench_parse_oxttl_turtle_with_turtle(c: &mut Criterion) { parse_oxttl_turtle( c, "turtle", - match turtle_test_data() { + &match turtle_test_data() { Ok(d) => d, Err(e) => { eprintln!("{e}"); @@ -143,7 +145,7 @@ fn bench_parse_rio_ntriples_with_ntriples(c: &mut Criterion) { parse_rio_ntriples( c, "ntriples", - match ntriples_test_data() { + &match ntriples_test_data() { Ok(d) => d, Err(e) => { eprintln!("{e}"); @@ -157,7 +159,7 @@ fn bench_parse_rio_ntriples_with_turtle(c: &mut Criterion) { parse_rio_turtle( c, "ntriples", - match ntriples_test_data() { + &match ntriples_test_data() { Ok(d) => d, Err(e) => { eprintln!("{e}"); @@ -171,7 +173,7 @@ fn bench_parse_rio_turtle_with_turtle(c: &mut Criterion) { parse_rio_turtle( c, "turtle", - match turtle_test_data() { + &match turtle_test_data() { Ok(d) => d, Err(e) => { eprintln!("{e}"); diff --git a/testsuite/src/sparql_evaluator.rs b/testsuite/src/sparql_evaluator.rs index e72feaa7..da4f0347 100644 --- a/testsuite/src/sparql_evaluator.rs +++ b/testsuite/src/sparql_evaluator.rs @@ -501,7 +501,7 @@ impl StaticQueryResults { fn from_graph(graph: &Graph) -> Result { // Hack to normalize literals let store = Store::new()?; - for t in graph.iter() { + for t in graph { store .insert(t.in_graph(GraphNameRef::DefaultGraph)) .unwrap(); From 501f9ce6f93697be88ae730085baeb7440384dc4 Mon Sep 17 00:00:00 2001 From: Thomas Date: Fri, 7 Jul 2023 14:57:26 +0200 Subject: [PATCH 029/217] Makes profiler independent from query plan --- lib/src/sparql/eval.rs | 219 +++++++++++++++++++++++++++++++-------- lib/src/sparql/mod.rs | 13 ++- lib/src/sparql/plan.rs | 144 +------------------------ lib/src/sparql/update.rs | 2 +- 4 files changed, 186 insertions(+), 192 deletions(-) diff --git a/lib/src/sparql/eval.rs b/lib/src/sparql/eval.rs index 8821fb1e..8ab0d8ba 100644 --- a/lib/src/sparql/eval.rs +++ b/lib/src/sparql/eval.rs @@ -9,6 +9,7 @@ use crate::sparql::service::ServiceHandler; use crate::storage::numeric_encoder::*; use crate::storage::small_string::SmallString; use digest::Digest; +use json_event_parser::{JsonEvent, JsonWriter}; use md5::Md5; use oxilangtag::LanguageTag; use oxiri::Iri; @@ -27,10 +28,10 @@ use std::hash::{Hash, Hasher}; use std::iter::Iterator; use std::iter::{empty, once}; use std::rc::Rc; -use std::str; use std::time::Duration as StdDuration; #[cfg(not(all(target_family = "wasm", target_os = "unknown")))] use std::time::Instant; +use std::{fmt, io, str}; const REGEX_SIZE_LIMIT: usize = 1_000_000; @@ -68,9 +69,9 @@ impl SimpleEvaluator { #[allow(clippy::rc_buffer)] pub fn evaluate_select_plan( &self, - plan: Rc, + plan: &PlanNode, variables: Rc>, - ) -> (QueryResults, Rc) { + ) -> (QueryResults, Rc) { let (eval, stats) = self.plan_evaluator(plan); ( QueryResults::Solutions(decode_bindings( @@ -84,8 +85,8 @@ impl SimpleEvaluator { pub fn evaluate_ask_plan( &self, - plan: Rc, - ) -> (Result, Rc) { + plan: &PlanNode, + ) -> (Result, Rc) { let from = EncodedTuple::with_capacity(plan.used_variables().len()); let (eval, stats) = self.plan_evaluator(plan); ( @@ -100,9 +101,9 @@ impl SimpleEvaluator { pub fn evaluate_construct_plan( &self, - plan: Rc, + plan: &PlanNode, template: Vec, - ) -> (QueryResults, Rc) { + ) -> (QueryResults, Rc) { let from = EncodedTuple::with_capacity(plan.used_variables().len()); let (eval, stats) = self.plan_evaluator(plan); ( @@ -119,10 +120,7 @@ impl SimpleEvaluator { ) } - pub fn evaluate_describe_plan( - &self, - plan: Rc, - ) -> (QueryResults, Rc) { + pub fn evaluate_describe_plan(&self, plan: &PlanNode) -> (QueryResults, Rc) { let from = EncodedTuple::with_capacity(plan.used_variables().len()); let (eval, stats) = self.plan_evaluator(plan); ( @@ -139,13 +137,13 @@ impl SimpleEvaluator { pub fn plan_evaluator( &self, - node: Rc, + node: &PlanNode, ) -> ( Rc EncodedTuplesIterator>, - Rc, + Rc, ) { let mut stat_children = Vec::new(); - let mut evaluator: Rc EncodedTuplesIterator> = match node.as_ref() { + let mut evaluator: Rc EncodedTuplesIterator> = match node { PlanNode::StaticBindings { encoded_tuples, .. } => { let tuples = encoded_tuples.clone(); Rc::new(move |from| { @@ -388,9 +386,9 @@ impl SimpleEvaluator { .intersection(&build_child.always_bound_variables()) .copied() .collect(); - let (probe, probe_stats) = self.plan_evaluator(Rc::clone(probe_child)); + let (probe, probe_stats) = self.plan_evaluator(probe_child); stat_children.push(probe_stats); - let (build, build_stats) = self.plan_evaluator(Rc::clone(build_child)); + let (build, build_stats) = self.plan_evaluator(build_child); stat_children.push(build_stats); if join_keys.is_empty() { // Cartesian product @@ -434,9 +432,9 @@ impl SimpleEvaluator { } } PlanNode::ForLoopJoin { left, right } => { - let (left, left_stats) = self.plan_evaluator(Rc::clone(left)); + let (left, left_stats) = self.plan_evaluator(left); stat_children.push(left_stats); - let (right, right_stats) = self.plan_evaluator(Rc::clone(right)); + let (right, right_stats) = self.plan_evaluator(right); stat_children.push(right_stats); Rc::new(move |from| { let right = Rc::clone(&right); @@ -452,9 +450,9 @@ impl SimpleEvaluator { .intersection(&right.always_bound_variables()) .copied() .collect(); - let (left, left_stats) = self.plan_evaluator(Rc::clone(left)); + let (left, left_stats) = self.plan_evaluator(left); stat_children.push(left_stats); - let (right, right_stats) = self.plan_evaluator(Rc::clone(right)); + let (right, right_stats) = self.plan_evaluator(right); stat_children.push(right_stats); if join_keys.is_empty() { Rc::new(move |from| { @@ -495,9 +493,9 @@ impl SimpleEvaluator { .intersection(&right.always_bound_variables()) .copied() .collect(); - let (left, left_stats) = self.plan_evaluator(Rc::clone(left)); + let (left, left_stats) = self.plan_evaluator(left); stat_children.push(left_stats); - let (right, right_stats) = self.plan_evaluator(Rc::clone(right)); + let (right, right_stats) = self.plan_evaluator(right); stat_children.push(right_stats); let expression = self.expression_evaluator(expression, &mut stat_children); // Real hash join @@ -520,9 +518,9 @@ impl SimpleEvaluator { }) } PlanNode::ForLoopLeftJoin { left, right } => { - let (left, left_stats) = self.plan_evaluator(Rc::clone(left)); + let (left, left_stats) = self.plan_evaluator(left); stat_children.push(left_stats); - let (right, right_stats) = self.plan_evaluator(Rc::clone(right)); + let (right, right_stats) = self.plan_evaluator(right); stat_children.push(right_stats); Rc::new(move |from| { Box::new(ForLoopLeftJoinIterator { @@ -533,7 +531,7 @@ impl SimpleEvaluator { }) } PlanNode::Filter { child, expression } => { - let (child, child_stats) = self.plan_evaluator(Rc::clone(child)); + let (child, child_stats) = self.plan_evaluator(child); stat_children.push(child_stats); let expression = self.expression_evaluator(expression, &mut stat_children); Rc::new(move |from| { @@ -552,7 +550,7 @@ impl SimpleEvaluator { let children: Vec<_> = children .iter() .map(|child| { - let (child, child_stats) = self.plan_evaluator(Rc::clone(child)); + let (child, child_stats) = self.plan_evaluator(child); stat_children.push(child_stats); child }) @@ -571,7 +569,7 @@ impl SimpleEvaluator { variable, expression, } => { - let (child, child_stats) = self.plan_evaluator(Rc::clone(child)); + let (child, child_stats) = self.plan_evaluator(child); stat_children.push(child_stats); let position = variable.encoded; let expression = self.expression_evaluator(expression, &mut stat_children); @@ -587,7 +585,7 @@ impl SimpleEvaluator { }) } PlanNode::Sort { child, by } => { - let (child, child_stats) = self.plan_evaluator(Rc::clone(child)); + let (child, child_stats) = self.plan_evaluator(child); stat_children.push(child_stats); let by: Vec<_> = by .iter() @@ -645,12 +643,12 @@ impl SimpleEvaluator { }) } PlanNode::HashDeduplicate { child } => { - let (child, child_stats) = self.plan_evaluator(Rc::clone(child)); + let (child, child_stats) = self.plan_evaluator(child); stat_children.push(child_stats); Rc::new(move |from| Box::new(hash_deduplicate(child(from)))) } PlanNode::Reduced { child } => { - let (child, child_stats) = self.plan_evaluator(Rc::clone(child)); + let (child, child_stats) = self.plan_evaluator(child); stat_children.push(child_stats); Rc::new(move |from| { Box::new(ConsecutiveDeduplication { @@ -660,19 +658,19 @@ impl SimpleEvaluator { }) } PlanNode::Skip { child, count } => { - let (child, child_stats) = self.plan_evaluator(Rc::clone(child)); + let (child, child_stats) = self.plan_evaluator(child); stat_children.push(child_stats); let count = *count; Rc::new(move |from| Box::new(child(from).skip(count))) } PlanNode::Limit { child, count } => { - let (child, child_stats) = self.plan_evaluator(Rc::clone(child)); + let (child, child_stats) = self.plan_evaluator(child); stat_children.push(child_stats); let count = *count; Rc::new(move |from| Box::new(child(from).take(count))) } PlanNode::Project { child, mapping } => { - let (child, child_stats) = self.plan_evaluator(Rc::clone(child)); + let (child, child_stats) = self.plan_evaluator(child); stat_children.push(child_stats); let mapping = Rc::clone(mapping); Rc::new(move |from| { @@ -712,7 +710,7 @@ impl SimpleEvaluator { key_variables, aggregates, } => { - let (child, child_stats) = self.plan_evaluator(Rc::clone(child)); + let (child, child_stats) = self.plan_evaluator(child); stat_children.push(child_stats); let key_variables = Rc::clone(key_variables); let aggregate_input_expressions: Vec<_> = aggregates @@ -806,11 +804,11 @@ impl SimpleEvaluator { }) } }; - let stats = Rc::new(PlanNodeWithStats { - node, + let stats = Rc::new(EvalNodeWithStats { + label: eval_node_label(node), children: stat_children, exec_count: Cell::new(0), - exec_duration: Cell::new(std::time::Duration::from_secs(0)), + exec_duration: Cell::new(StdDuration::from_secs(0)), }); if self.run_stats { let stats = Rc::clone(&stats); @@ -920,7 +918,7 @@ impl SimpleEvaluator { fn expression_evaluator( &self, expression: &PlanExpression, - stat_children: &mut Vec>, + stat_children: &mut Vec>, ) -> Rc Option> { match expression { PlanExpression::NamedNode(t) => { @@ -936,7 +934,7 @@ impl SimpleEvaluator { Rc::new(move |tuple| tuple.get(v).cloned()) } PlanExpression::Exists(plan) => { - let (eval, stats) = self.plan_evaluator(Rc::clone(plan)); + let (eval, stats) = self.plan_evaluator(plan); stat_children.push(stats); Rc::new(move |tuple| Some(eval(tuple.clone()).next().is_some().into())) } @@ -2086,7 +2084,7 @@ impl SimpleEvaluator { fn hash( &self, arg: &PlanExpression, - stat_children: &mut Vec>, + stat_children: &mut Vec>, ) -> Rc Option> { let arg = self.expression_evaluator(arg, stat_children); let dataset = Rc::clone(&self.dataset); @@ -4719,7 +4717,7 @@ impl Extend for EncodedTupleSet { struct StatsIterator { inner: EncodedTuplesIterator, - stats: Rc, + stats: Rc, } impl Iterator for StatsIterator { @@ -4738,6 +4736,145 @@ impl Iterator for StatsIterator { } } +pub struct EvalNodeWithStats { + pub label: String, + pub children: Vec>, + pub exec_count: Cell, + pub exec_duration: Cell, +} + +impl EvalNodeWithStats { + pub fn json_node( + &self, + writer: &mut JsonWriter, + with_stats: bool, + ) -> io::Result<()> { + writer.write_event(JsonEvent::StartObject)?; + writer.write_event(JsonEvent::ObjectKey("name"))?; + writer.write_event(JsonEvent::String(&self.label))?; + if with_stats { + writer.write_event(JsonEvent::ObjectKey("number of results"))?; + writer.write_event(JsonEvent::Number(&self.exec_count.get().to_string()))?; + writer.write_event(JsonEvent::ObjectKey("duration in seconds"))?; + writer.write_event(JsonEvent::Number( + &self.exec_duration.get().as_secs_f32().to_string(), + ))?; + } + writer.write_event(JsonEvent::ObjectKey("children"))?; + writer.write_event(JsonEvent::StartArray)?; + for child in &self.children { + child.json_node(writer, with_stats)?; + } + writer.write_event(JsonEvent::EndArray)?; + writer.write_event(JsonEvent::EndObject) + } +} + +impl fmt::Debug for EvalNodeWithStats { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut obj = f.debug_struct("Node"); + obj.field("name", &self.label); + if self.exec_duration.get() > StdDuration::default() { + obj.field("number of results", &self.exec_count.get()); + obj.field("duration in seconds", &self.exec_duration.get()); + } + if !self.children.is_empty() { + obj.field("children", &self.children); + } + obj.finish() + } +} + +fn eval_node_label(node: &PlanNode) -> String { + match node { + PlanNode::Aggregate { + key_variables, + aggregates, + .. + } => format!( + "Aggregate({})", + key_variables + .iter() + .map(ToString::to_string) + .chain(aggregates.iter().map(|(agg, v)| format!("{agg} -> {v}"))) + .collect::>() + .join(", ") + ), + PlanNode::AntiJoin { .. } => "AntiJoin".to_owned(), + PlanNode::Extend { + expression, + variable, + .. + } => format!("Extend({expression} -> {variable})"), + PlanNode::Filter { expression, .. } => format!("Filter({expression})"), + PlanNode::ForLoopJoin { .. } => "ForLoopJoin".to_owned(), + PlanNode::ForLoopLeftJoin { .. } => "ForLoopLeftJoin".to_owned(), + PlanNode::HashDeduplicate { .. } => "HashDeduplicate".to_owned(), + PlanNode::HashJoin { .. } => "HashJoin".to_owned(), + PlanNode::HashLeftJoin { expression, .. } => format!("HashLeftJoin({expression})"), + PlanNode::Limit { count, .. } => format!("Limit({count})"), + PlanNode::PathPattern { + subject, + path, + object, + graph_name, + } => format!("PathPattern({subject} {path} {object} {graph_name})"), + PlanNode::Project { mapping, .. } => { + format!( + "Project({})", + mapping + .iter() + .map(|(f, t)| if f.plain == t.plain { + f.to_string() + } else { + format!("{f} -> {t}") + }) + .collect::>() + .join(", ") + ) + } + PlanNode::QuadPattern { + subject, + predicate, + object, + graph_name, + } => format!("QuadPattern({subject} {predicate} {object} {graph_name})"), + PlanNode::Reduced { .. } => "Reduced".to_owned(), + PlanNode::Service { + service_name, + silent, + .. + } => { + if *silent { + format!("SilentService({service_name})") + } else { + format!("Service({service_name})") + } + } + PlanNode::Skip { count, .. } => format!("Skip({count})"), + PlanNode::Sort { by, .. } => { + format!( + "Sort({})", + by.iter() + .map(ToString::to_string) + .collect::>() + .join(", ") + ) + } + PlanNode::StaticBindings { variables, .. } => { + format!( + "StaticBindings({})", + variables + .iter() + .map(ToString::to_string) + .collect::>() + .join(", ") + ) + } + PlanNode::Union { .. } => "Union".to_owned(), + } +} + #[cfg(all(target_family = "wasm", target_os = "unknown"))] pub struct Timer { timestamp_ms: f64, diff --git a/lib/src/sparql/mod.rs b/lib/src/sparql/mod.rs index 9612d514..e66e114b 100644 --- a/lib/src/sparql/mod.rs +++ b/lib/src/sparql/mod.rs @@ -17,9 +17,8 @@ use crate::model::{NamedNode, Term}; pub use crate::sparql::algebra::{Query, QueryDataset, Update}; use crate::sparql::dataset::DatasetView; pub use crate::sparql::error::{EvaluationError, QueryError}; -use crate::sparql::eval::{SimpleEvaluator, Timer}; +use crate::sparql::eval::{EvalNodeWithStats, SimpleEvaluator, Timer}; pub use crate::sparql::model::{QueryResults, QuerySolution, QuerySolutionIter, QueryTripleIter}; -use crate::sparql::plan::PlanNodeWithStats; use crate::sparql::plan_builder::PlanBuilder; pub use crate::sparql::service::ServiceHandler; use crate::sparql::service::{EmptyServiceHandler, ErrorConversionServiceHandler}; @@ -63,7 +62,7 @@ pub(crate) fn evaluate_query( Rc::new(options.custom_functions), run_stats, ) - .evaluate_select_plan(Rc::new(plan), Rc::new(variables)); + .evaluate_select_plan(&plan, Rc::new(variables)); (Ok(results), explanation, planning_duration) } spargebra::Query::Ask { @@ -84,7 +83,7 @@ pub(crate) fn evaluate_query( Rc::new(options.custom_functions), run_stats, ) - .evaluate_ask_plan(Rc::new(plan)); + .evaluate_ask_plan(&plan); (results, explanation, planning_duration) } spargebra::Query::Construct { @@ -114,7 +113,7 @@ pub(crate) fn evaluate_query( Rc::new(options.custom_functions), run_stats, ) - .evaluate_construct_plan(Rc::new(plan), construct); + .evaluate_construct_plan(&plan, construct); (Ok(results), explanation, planning_duration) } spargebra::Query::Describe { @@ -135,7 +134,7 @@ pub(crate) fn evaluate_query( Rc::new(options.custom_functions), run_stats, ) - .evaluate_describe_plan(Rc::new(plan)); + .evaluate_describe_plan(&plan); (Ok(results), explanation, planning_duration) } }; @@ -284,7 +283,7 @@ impl From for UpdateOptions { /// The explanation of a query. #[derive(Clone)] pub struct QueryExplanation { - inner: Rc, + inner: Rc, with_stats: bool, parsing_duration: Option, planning_duration: Duration, diff --git a/lib/src/sparql/plan.rs b/lib/src/sparql/plan.rs index 905e4da0..a447fb01 100644 --- a/lib/src/sparql/plan.rs +++ b/lib/src/sparql/plan.rs @@ -1,17 +1,14 @@ use crate::model::{BlankNode, Literal, NamedNode, Term, Triple}; use crate::sparql::Variable; use crate::storage::numeric_encoder::EncodedTerm; -use json_event_parser::{JsonEvent, JsonWriter}; use regex::Regex; use spargebra::algebra::GraphPattern; use spargebra::term::GroundTerm; -use std::cell::Cell; use std::cmp::max; use std::collections::btree_map::Entry; use std::collections::{BTreeMap, BTreeSet}; +use std::fmt; use std::rc::Rc; -use std::time::Duration; -use std::{fmt, io}; #[derive(Debug, Clone)] pub enum PlanNode { @@ -1022,142 +1019,3 @@ impl IntoIterator for EncodedTuple { self.inner.into_iter() } } - -pub struct PlanNodeWithStats { - pub node: Rc, - pub children: Vec>, - pub exec_count: Cell, - pub exec_duration: Cell, -} - -impl PlanNodeWithStats { - pub fn json_node( - &self, - writer: &mut JsonWriter, - with_stats: bool, - ) -> io::Result<()> { - writer.write_event(JsonEvent::StartObject)?; - writer.write_event(JsonEvent::ObjectKey("name"))?; - writer.write_event(JsonEvent::String(&self.node_label()))?; - if with_stats { - writer.write_event(JsonEvent::ObjectKey("number of results"))?; - writer.write_event(JsonEvent::Number(&self.exec_count.get().to_string()))?; - writer.write_event(JsonEvent::ObjectKey("duration in seconds"))?; - writer.write_event(JsonEvent::Number( - &self.exec_duration.get().as_secs_f32().to_string(), - ))?; - } - writer.write_event(JsonEvent::ObjectKey("children"))?; - writer.write_event(JsonEvent::StartArray)?; - for child in &self.children { - child.json_node(writer, with_stats)?; - } - writer.write_event(JsonEvent::EndArray)?; - writer.write_event(JsonEvent::EndObject) - } - - fn node_label(&self) -> String { - match self.node.as_ref() { - PlanNode::Aggregate { - key_variables, - aggregates, - .. - } => format!( - "Aggregate({})", - key_variables - .iter() - .map(ToString::to_string) - .chain(aggregates.iter().map(|(agg, v)| format!("{agg} -> {v}"))) - .collect::>() - .join(", ") - ), - PlanNode::AntiJoin { .. } => "AntiJoin".to_owned(), - PlanNode::Extend { - expression, - variable, - .. - } => format!("Extend({expression} -> {variable})"), - PlanNode::Filter { expression, .. } => format!("Filter({expression})"), - PlanNode::ForLoopJoin { .. } => "ForLoopJoin".to_owned(), - PlanNode::ForLoopLeftJoin { .. } => "ForLoopLeftJoin".to_owned(), - PlanNode::HashDeduplicate { .. } => "HashDeduplicate".to_owned(), - PlanNode::HashJoin { .. } => "HashJoin".to_owned(), - PlanNode::HashLeftJoin { expression, .. } => format!("HashLeftJoin({expression})"), - PlanNode::Limit { count, .. } => format!("Limit({count})"), - PlanNode::PathPattern { - subject, - path, - object, - graph_name, - } => format!("PathPattern({subject} {path} {object} {graph_name})"), - PlanNode::Project { mapping, .. } => { - format!( - "Project({})", - mapping - .iter() - .map(|(f, t)| if f.plain == t.plain { - f.to_string() - } else { - format!("{f} -> {t}") - }) - .collect::>() - .join(", ") - ) - } - PlanNode::QuadPattern { - subject, - predicate, - object, - graph_name, - } => format!("QuadPattern({subject} {predicate} {object} {graph_name})"), - PlanNode::Reduced { .. } => "Reduced".to_owned(), - PlanNode::Service { - service_name, - silent, - .. - } => { - if *silent { - format!("SilentService({service_name})") - } else { - format!("Service({service_name})") - } - } - PlanNode::Skip { count, .. } => format!("Skip({count})"), - PlanNode::Sort { by, .. } => { - format!( - "Sort({})", - by.iter() - .map(ToString::to_string) - .collect::>() - .join(", ") - ) - } - PlanNode::StaticBindings { variables, .. } => { - format!( - "StaticBindings({})", - variables - .iter() - .map(ToString::to_string) - .collect::>() - .join(", ") - ) - } - PlanNode::Union { .. } => "Union".to_owned(), - } - } -} - -impl fmt::Debug for PlanNodeWithStats { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let mut obj = f.debug_struct("Node"); - obj.field("name", &self.node_label()); - if self.exec_duration.get() > Duration::default() { - obj.field("number of results", &self.exec_count.get()); - obj.field("duration in seconds", &self.exec_duration.get()); - } - if !self.children.is_empty() { - obj.field("children", &self.children); - } - obj.finish() - } -} diff --git a/lib/src/sparql/update.rs b/lib/src/sparql/update.rs index 3c05c8f5..3091172f 100644 --- a/lib/src/sparql/update.rs +++ b/lib/src/sparql/update.rs @@ -140,7 +140,7 @@ impl<'a, 'b: 'a> SimpleUpdateEvaluator<'a, 'b> { false, ); let mut bnodes = HashMap::new(); - let (eval, _) = evaluator.plan_evaluator(Rc::new(plan)); + let (eval, _) = evaluator.plan_evaluator(&plan); let tuples = eval(EncodedTuple::with_capacity(variables.len())).collect::, _>>()?; //TODO: would be much better to stream for tuple in tuples { From cdabe528478bca5b1d869af06db33d33e5b4267d Mon Sep 17 00:00:00 2001 From: Tpt Date: Tue, 27 Jun 2023 21:23:18 +0200 Subject: [PATCH 030/217] RDF I/O: adds basic Tokio support --- Cargo.lock | 39 ++++++ lib/oxrdfxml/Cargo.toml | 8 ++ lib/oxrdfxml/src/lib.rs | 6 +- lib/oxrdfxml/src/parser.rs | 149 +++++++++++++++++++--- lib/oxrdfxml/src/serializer.rs | 217 +++++++++++++++++++++++++------- lib/oxttl/Cargo.toml | 5 + lib/oxttl/src/n3.rs | 89 +++++++++++++ lib/oxttl/src/nquads.rs | 162 ++++++++++++++++++++++++ lib/oxttl/src/ntriples.rs | 160 +++++++++++++++++++++++ lib/oxttl/src/toolkit/lexer.rs | 31 +++++ lib/oxttl/src/toolkit/mod.rs | 2 + lib/oxttl/src/toolkit/parser.rs | 36 ++++++ lib/oxttl/src/trig.rs | 169 +++++++++++++++++++++++++ lib/oxttl/src/turtle.rs | 164 +++++++++++++++++++++++- 14 files changed, 1172 insertions(+), 65 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d2568bac..ab40b360 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -194,6 +194,12 @@ version = "3.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" +[[package]] +name = "bytes" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" + [[package]] name = "cast" version = "0.3.0" @@ -1038,6 +1044,7 @@ dependencies = [ "oxiri", "oxrdf", "quick-xml", + "tokio", ] [[package]] @@ -1064,6 +1071,7 @@ dependencies = [ "oxilangtag", "oxiri", "oxrdf", + "tokio", ] [[package]] @@ -1128,6 +1136,12 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" +[[package]] +name = "pin-project-lite" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" + [[package]] name = "pkg-config" version = "0.3.27" @@ -1293,6 +1307,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81b9228215d82c7b61490fec1de287136b5de6f5700f6e58ea9ad61a7964ca51" dependencies = [ "memchr", + "tokio", ] [[package]] @@ -1785,6 +1800,30 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +[[package]] +name = "tokio" +version = "1.28.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94d7b1cfd2aa4011f2de74c2c4c63665e27a71006b0a192dcd2710272e73dfa2" +dependencies = [ + "autocfg", + "bytes", + "pin-project-lite", + "tokio-macros", + "windows-sys 0.48.0", +] + +[[package]] +name = "tokio-macros" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.18", +] + [[package]] name = "typenum" version = "1.16.0" diff --git a/lib/oxrdfxml/Cargo.toml b/lib/oxrdfxml/Cargo.toml index 3d012e17..895159b5 100644 --- a/lib/oxrdfxml/Cargo.toml +++ b/lib/oxrdfxml/Cargo.toml @@ -13,11 +13,19 @@ Parser for the RDF/XML language edition = "2021" rust-version = "1.65" +[features] +default = [] +async-tokio = ["dep:tokio", "quick-xml/async-tokio"] + [dependencies] oxrdf = { version = "0.2.0-alpha.1-dev", path = "../oxrdf" } oxilangtag = "0.1" oxiri = "0.2" quick-xml = "0.29" +tokio = { version = "1", optional = true, features = ["io-util"] } + +[dev-dependencies] +tokio = { version = "1", features = ["rt", "macros"] } [package.metadata.docs.rs] all-features = true diff --git a/lib/oxrdfxml/src/lib.rs b/lib/oxrdfxml/src/lib.rs index e07e31c8..bcc87308 100644 --- a/lib/oxrdfxml/src/lib.rs +++ b/lib/oxrdfxml/src/lib.rs @@ -9,6 +9,10 @@ mod parser; mod serializer; mod utils; -pub use crate::serializer::{RdfXmlSerializer, ToWriteRdfXmlWriter}; pub use error::{ParseError, SyntaxError}; +#[cfg(feature = "async-tokio")] +pub use parser::FromTokioAsyncReadRdfXmlReader; pub use parser::{FromReadRdfXmlReader, RdfXmlParser}; +#[cfg(feature = "async-tokio")] +pub use serializer::ToTokioAsyncWriteRdfXmlWriter; +pub use serializer::{RdfXmlSerializer, ToWriteRdfXmlWriter}; diff --git a/lib/oxrdfxml/src/parser.rs b/lib/oxrdfxml/src/parser.rs index a726a7d0..999becad 100644 --- a/lib/oxrdfxml/src/parser.rs +++ b/lib/oxrdfxml/src/parser.rs @@ -10,8 +10,10 @@ use quick_xml::events::*; use quick_xml::name::{LocalName, QName, ResolveResult}; use quick_xml::{NsReader, Writer}; use std::collections::{HashMap, HashSet}; -use std::io::{BufRead, BufReader, Read}; +use std::io::{BufReader, Read}; use std::str; +#[cfg(feature = "async-tokio")] +use tokio::io::{AsyncRead, BufReader as AsyncBufReader}; /// A [RDF/XML](https://www.w3.org/TR/rdf-syntax-grammar/) streaming parser. /// @@ -93,23 +95,70 @@ impl RdfXmlParser { /// # Result::<_,Box>::Ok(()) /// ``` pub fn parse_from_read(&self, read: R) -> FromReadRdfXmlReader { - let mut reader = NsReader::from_reader(BufReader::new(read)); - reader.expand_empty_elements(true); FromReadRdfXmlReader { results: Vec::new(), - reader: RdfXmlReader { - reader, - state: vec![RdfXmlState::Doc { - base_iri: self.base.clone(), - }], - custom_entities: HashMap::default(), - in_literal_depth: 0, - known_rdf_id: HashSet::default(), - is_end: false, - }, + reader: self.parse(BufReader::new(read)), + reader_buffer: Vec::default(), + } + } + + /// Parses a RDF/XML file from a [`AsyncRead`] implementation. + /// + /// Count the number of people: + /// ``` + /// use oxrdf::{NamedNodeRef, vocab::rdf}; + /// use oxrdfxml::{RdfXmlParser, ParseError}; + /// + /// #[tokio::main(flavor = "current_thread")] + /// async fn main() -> Result<(), ParseError> { + /// let file = b" + /// + /// + /// + /// Foo + /// + /// + /// "; + /// + /// let schema_person = NamedNodeRef::new_unchecked("http://schema.org/Person"); + /// let mut count = 0; + /// let mut parser = RdfXmlParser::new().parse_from_tokio_async_read(file.as_ref()); + /// while let Some(triple) = parser.next().await { + /// let triple = triple?; + /// if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { + /// count += 1; + /// } + /// } + /// assert_eq!(2, count); + /// Ok(()) + /// } + /// ``` + #[cfg(feature = "async-tokio")] + pub fn parse_from_tokio_async_read( + &self, + read: R, + ) -> FromTokioAsyncReadRdfXmlReader { + FromTokioAsyncReadRdfXmlReader { + results: Vec::new(), + reader: self.parse(AsyncBufReader::new(read)), reader_buffer: Vec::default(), } } + + fn parse(&self, reader: T) -> RdfXmlReader { + let mut reader = NsReader::from_reader(reader); + reader.expand_empty_elements(true); + RdfXmlReader { + reader, + state: vec![RdfXmlState::Doc { + base_iri: self.base.clone(), + }], + custom_entities: HashMap::default(), + in_literal_depth: 0, + known_rdf_id: HashSet::default(), + is_end: false, + } + } } /// Parses a RDF/XML file from a [`Read`] implementation. Can be built using [`RdfXmlParser::parse_from_read`]. @@ -178,6 +227,76 @@ impl FromReadRdfXmlReader { } } +/// Parses a RDF/XML file from a [`AsyncRead`] implementation. Can be built using [`RdfXmlParser::parse_from_tokio_async_read`]. +/// +/// Count the number of people: +/// ``` +/// use oxrdf::{NamedNodeRef, vocab::rdf}; +/// use oxrdfxml::{RdfXmlParser, ParseError}; +/// +/// #[tokio::main(flavor = "current_thread")] +/// async fn main() -> Result<(), ParseError> { +/// let file = b" +/// +/// +/// +/// Foo +/// +/// +/// "; +/// +/// let schema_person = NamedNodeRef::new_unchecked("http://schema.org/Person"); +/// let mut count = 0; +/// let mut parser = RdfXmlParser::new().parse_from_tokio_async_read(file.as_ref()); +/// while let Some(triple) = parser.next().await { +/// let triple = triple?; +/// if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { +/// count += 1; +/// } +/// } +/// assert_eq!(2, count); +/// Ok(()) +/// } +/// ``` +#[cfg(feature = "async-tokio")] +pub struct FromTokioAsyncReadRdfXmlReader { + results: Vec, + reader: RdfXmlReader>, + reader_buffer: Vec, +} + +#[cfg(feature = "async-tokio")] +impl FromTokioAsyncReadRdfXmlReader { + /// Reads the next triple or returns `None` if the file is finished. + pub async fn next(&mut self) -> Option> { + loop { + if let Some(triple) = self.results.pop() { + return Some(Ok(triple)); + } else if self.reader.is_end { + return None; + } + if let Err(e) = self.parse_step().await { + return Some(Err(e)); + } + } + } + + /// The current byte position in the input data. + pub fn buffer_position(&self) -> usize { + self.reader.reader.buffer_position() + } + + async fn parse_step(&mut self) -> Result<(), ParseError> { + self.reader_buffer.clear(); + let event = self + .reader + .reader + .read_event_into_async(&mut self.reader_buffer) + .await?; + self.reader.parse_event(event, &mut self.results) + } +} + const RDF_ABOUT: &str = "http://www.w3.org/1999/02/22-rdf-syntax-ns#about"; const RDF_ABOUT_EACH: &str = "http://www.w3.org/1999/02/22-rdf-syntax-ns#aboutEach"; const RDF_ABOUT_EACH_PREFIX: &str = "http://www.w3.org/1999/02/22-rdf-syntax-ns#aboutEachPrefix"; @@ -285,7 +404,7 @@ impl RdfXmlState { } } -struct RdfXmlReader { +struct RdfXmlReader { reader: NsReader, state: Vec, custom_entities: HashMap, @@ -294,7 +413,7 @@ struct RdfXmlReader { is_end: bool, } -impl RdfXmlReader { +impl RdfXmlReader { fn parse_event(&mut self, event: Event, results: &mut Vec) -> Result<(), ParseError> { match event { Event::Start(event) => self.parse_start_event(&event, results), diff --git a/lib/oxrdfxml/src/serializer.rs b/lib/oxrdfxml/src/serializer.rs index 03de4fb9..51802af5 100644 --- a/lib/oxrdfxml/src/serializer.rs +++ b/lib/oxrdfxml/src/serializer.rs @@ -5,6 +5,8 @@ use quick_xml::Writer; use std::io; use std::io::Write; use std::sync::Arc; +#[cfg(feature = "async-tokio")] +use tokio::io::AsyncWrite; /// A [RDF/XML](https://www.w3.org/TR/rdf-syntax-grammar/) serializer. /// @@ -34,7 +36,9 @@ impl RdfXmlSerializer { Self } - /// Writes a RdfXml file to a [`Write`] implementation. + /// Writes a RDF/XML file to a [`Write`] implementation. + /// + /// This writer does unbuffered writes. /// /// ``` /// use oxrdf::{NamedNodeRef, TripleRef}; @@ -56,7 +60,47 @@ impl RdfXmlSerializer { pub fn serialize_to_write(&self, write: W) -> ToWriteRdfXmlWriter { ToWriteRdfXmlWriter { writer: Writer::new_with_indent(write, b'\t', 1), - current_subject: None, + inner: InnerRdfXmlWriter { + current_subject: None, + }, + } + } + + /// Writes a RDF/XML file to a [`AsyncWrite`] implementation. + /// + /// This writer does unbuffered writes. + /// + /// ``` + /// use oxrdf::{NamedNodeRef, TripleRef}; + /// use oxrdfxml::RdfXmlSerializer; + /// use std::io::Result; + /// + /// #[tokio::main(flavor = "current_thread")] + /// async fn main() -> Result<()> { + /// let mut writer = RdfXmlSerializer::new().serialize_to_tokio_async_write(Vec::new()); + /// writer.write_triple(TripleRef::new( + /// NamedNodeRef::new_unchecked("http://example.com#me"), + /// NamedNodeRef::new_unchecked("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), + /// NamedNodeRef::new_unchecked("http://schema.org/Person"), + /// )).await?; + /// assert_eq!( + /// b"\n\n\t\n\t\t\n\t\n", + /// writer.finish().await?.as_slice() + /// ); + /// Ok(()) + /// } + /// ``` + #[allow(clippy::unused_self)] + #[cfg(feature = "async-tokio")] + pub fn serialize_to_tokio_async_write( + &self, + write: W, + ) -> ToTokioAsyncWriteRdfXmlWriter { + ToTokioAsyncWriteRdfXmlWriter { + writer: Writer::new_with_indent(write, b'\t', 1), + inner: InnerRdfXmlWriter { + current_subject: None, + }, } } } @@ -81,24 +125,111 @@ impl RdfXmlSerializer { /// ``` pub struct ToWriteRdfXmlWriter { writer: Writer, - current_subject: Option, + inner: InnerRdfXmlWriter, } impl ToWriteRdfXmlWriter { /// Writes an extra triple. #[allow(clippy::match_wildcard_for_single_variants, unreachable_patterns)] pub fn write_triple<'a>(&mut self, t: impl Into>) -> io::Result<()> { + let mut buffer = Vec::new(); + self.inner.write_triple(t, &mut buffer)?; + self.flush_buffer(&mut buffer) + } + + /// Ends the write process and returns the underlying [`Write`]. + pub fn finish(mut self) -> io::Result { + let mut buffer = Vec::new(); + self.inner.finish(&mut buffer); + self.flush_buffer(&mut buffer)?; + Ok(self.writer.into_inner()) + } + + fn flush_buffer(&mut self, buffer: &mut Vec>) -> io::Result<()> { + for event in buffer.drain(0..) { + self.writer.write_event(event).map_err(map_err)?; + } + Ok(()) + } +} + +/// Writes a RDF/XML file to a [`AsyncWrite`] implementation. Can be built using [`RdfXmlSerializer::serialize_to_tokio_async_write`]. +/// +/// ``` +/// use oxrdf::{NamedNodeRef, TripleRef}; +/// use oxrdfxml::RdfXmlSerializer; +/// use std::io::Result; +/// +/// #[tokio::main(flavor = "current_thread")] +/// async fn main() -> Result<()> { +/// let mut writer = RdfXmlSerializer::new().serialize_to_tokio_async_write(Vec::new()); +/// writer.write_triple(TripleRef::new( +/// NamedNodeRef::new_unchecked("http://example.com#me"), +/// NamedNodeRef::new_unchecked("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), +/// NamedNodeRef::new_unchecked("http://schema.org/Person"), +/// )).await?; +/// assert_eq!( +/// b"\n\n\t\n\t\t\n\t\n", +/// writer.finish().await?.as_slice() +/// ); +/// Ok(()) +/// } +/// ``` +#[cfg(feature = "async-tokio")] +pub struct ToTokioAsyncWriteRdfXmlWriter { + writer: Writer, + inner: InnerRdfXmlWriter, +} + +#[cfg(feature = "async-tokio")] +impl ToTokioAsyncWriteRdfXmlWriter { + /// Writes an extra triple. + #[allow(clippy::match_wildcard_for_single_variants, unreachable_patterns)] + pub async fn write_triple<'a>(&mut self, t: impl Into>) -> io::Result<()> { + let mut buffer = Vec::new(); + self.inner.write_triple(t, &mut buffer)?; + self.flush_buffer(&mut buffer).await + } + + /// Ends the write process and returns the underlying [`Write`]. + pub async fn finish(mut self) -> io::Result { + let mut buffer = Vec::new(); + self.inner.finish(&mut buffer); + self.flush_buffer(&mut buffer).await?; + Ok(self.writer.into_inner()) + } + + async fn flush_buffer(&mut self, buffer: &mut Vec>) -> io::Result<()> { + for event in buffer.drain(0..) { + self.writer + .write_event_async(event) + .await + .map_err(map_err)?; + } + Ok(()) + } +} + +pub struct InnerRdfXmlWriter { + current_subject: Option, +} + +impl InnerRdfXmlWriter { + #[allow(clippy::match_wildcard_for_single_variants, unreachable_patterns)] + fn write_triple<'a>( + &mut self, + t: impl Into>, + output: &mut Vec>, + ) -> io::Result<()> { if self.current_subject.is_none() { - self.write_start()?; + Self::write_start(output); } let triple = t.into(); // We open a new rdf:Description if useful if self.current_subject.as_ref().map(Subject::as_ref) != Some(triple.subject) { if self.current_subject.is_some() { - self.writer - .write_event(Event::End(BytesEnd::new("rdf:Description"))) - .map_err(map_err)?; + output.push(Event::End(BytesEnd::new("rdf:Description"))); } let mut description_open = BytesStart::new("rdf:Description"); @@ -116,10 +247,9 @@ impl ToWriteRdfXmlWriter { )) } } - self.writer - .write_event(Event::Start(description_open)) - .map_err(map_err)?; + output.push(Event::Start(description_open)); } + self.current_subject = Some(triple.subject.into_owned()); let (prop_prefix, prop_value) = split_iri(triple.predicate.as_str()); let (prop_qname, prop_xmlns) = if prop_value.is_empty() { @@ -127,25 +257,24 @@ impl ToWriteRdfXmlWriter { } else { (prop_value, ("xmlns", prop_prefix)) }; - let property_element = self.writer.create_element(prop_qname); - let property_element = property_element.with_attribute(prop_xmlns); - - match triple.object { - TermRef::NamedNode(node) => property_element - .with_attribute(("rdf:resource", node.as_str())) - .write_empty(), - TermRef::BlankNode(node) => property_element - .with_attribute(("rdf:nodeID", node.as_str())) - .write_empty(), + let mut property_open = BytesStart::new(prop_qname); + property_open.push_attribute(prop_xmlns); + let content = match triple.object { + TermRef::NamedNode(node) => { + property_open.push_attribute(("rdf:resource", node.as_str())); + None + } + TermRef::BlankNode(node) => { + property_open.push_attribute(("rdf:nodeID", node.as_str())); + None + } TermRef::Literal(literal) => { - let property_element = if let Some(language) = literal.language() { - property_element.with_attribute(("xml:lang", language)) + if let Some(language) = literal.language() { + property_open.push_attribute(("xml:lang", language)); } else if !literal.is_plain() { - property_element.with_attribute(("rdf:datatype", literal.datatype().as_str())) - } else { - property_element - }; - property_element.write_text_content(BytesText::new(literal.value())) + property_open.push_attribute(("rdf:datatype", literal.datatype().as_str())); + } + Some(literal.value()) } _ => { return Err(io::Error::new( @@ -153,37 +282,31 @@ impl ToWriteRdfXmlWriter { "RDF/XML only supports named, blank or literal object", )) } + }; + if let Some(content) = content { + output.push(Event::Start(property_open)); + output.push(Event::Text(BytesText::new(content))); + output.push(Event::End(BytesEnd::new(prop_qname))); + } else { + output.push(Event::Empty(property_open)); } - .map_err(map_err)?; - self.current_subject = Some(triple.subject.into_owned()); Ok(()) } - pub fn write_start(&mut self) -> io::Result<()> { - // We open the file - self.writer - .write_event(Event::Decl(BytesDecl::new("1.0", Some("UTF-8"), None))) - .map_err(map_err)?; + fn write_start(output: &mut Vec>) { + output.push(Event::Decl(BytesDecl::new("1.0", Some("UTF-8"), None))); let mut rdf_open = BytesStart::new("rdf:RDF"); rdf_open.push_attribute(("xmlns:rdf", "http://www.w3.org/1999/02/22-rdf-syntax-ns#")); - self.writer - .write_event(Event::Start(rdf_open)) - .map_err(map_err) + output.push(Event::Start(rdf_open)) } - /// Ends the write process and returns the underlying [`Write`]. - pub fn finish(mut self) -> io::Result { + fn finish(&self, output: &mut Vec>) { if self.current_subject.is_some() { - self.writer - .write_event(Event::End(BytesEnd::new("rdf:Description"))) - .map_err(map_err)?; + output.push(Event::End(BytesEnd::new("rdf:Description"))); } else { - self.write_start()?; + Self::write_start(output); } - self.writer - .write_event(Event::End(BytesEnd::new("rdf:RDF"))) - .map_err(map_err)?; - Ok(self.writer.into_inner()) + output.push(Event::End(BytesEnd::new("rdf:RDF"))); } } diff --git a/lib/oxttl/Cargo.toml b/lib/oxttl/Cargo.toml index 8717a9b6..06871c96 100644 --- a/lib/oxttl/Cargo.toml +++ b/lib/oxttl/Cargo.toml @@ -16,12 +16,17 @@ rust-version = "1.65" [features] default = [] rdf-star = ["oxrdf/rdf-star"] +async-tokio = ["dep:tokio"] [dependencies] memchr = "2" oxrdf = { version = "0.2.0-alpha.1-dev", path = "../oxrdf" } oxiri = "0.2" oxilangtag = "0.1" +tokio = { version = "1", optional = true, features = ["io-util"] } + +[dev-dependencies] +tokio = { version = "1", features = ["rt", "macros"] } [package.metadata.docs.rs] all-features = true diff --git a/lib/oxttl/src/n3.rs b/lib/oxttl/src/n3.rs index 4a537ef7..106263a2 100644 --- a/lib/oxttl/src/n3.rs +++ b/lib/oxttl/src/n3.rs @@ -1,6 +1,8 @@ //! A [N3](https://w3c.github.io/N3/spec/) streaming parser implemented by [`N3Parser`]. use crate::lexer::{resolve_local_name, N3Lexer, N3LexerMode, N3LexerOptions, N3Token}; +#[cfg(feature = "async-tokio")] +use crate::toolkit::FromTokioAsyncReadIterator; use crate::toolkit::{ FromReadIterator, Lexer, Parser, RuleRecognizer, RuleRecognizerError, SyntaxError, }; @@ -16,6 +18,8 @@ use oxrdf::{ use std::collections::HashMap; use std::fmt; use std::io::Read; +#[cfg(feature = "async-tokio")] +use tokio::io::AsyncRead; /// A N3 term i.e. a RDF `Term` or a `Variable`. #[derive(Eq, PartialEq, Debug, Clone, Hash)] @@ -261,6 +265,47 @@ impl N3Parser { } } + /// Parses a N3 file from a [`AsyncRead`] implementation. + /// + /// Count the number of people: + /// ``` + /// use oxrdf::{NamedNode, vocab::rdf}; + /// use oxttl::n3::{N3Parser, N3Term}; + /// use oxttl::ParseError; + /// + /// #[tokio::main(flavor = "current_thread")] + /// async fn main() -> Result<(), ParseError> { + /// let file = b"@base . + /// @prefix schema: . + /// a schema:Person ; + /// schema:name \"Foo\" . + /// a schema:Person ; + /// schema:name \"Bar\" ."; + /// + /// let rdf_type = N3Term::NamedNode(rdf::TYPE.into_owned()); + /// let schema_person = N3Term::NamedNode(NamedNode::new_unchecked("http://schema.org/Person")); + /// let mut count = 0; + /// let mut parser = N3Parser::new().parse_from_tokio_async_read(file.as_ref()); + /// while let Some(triple) = parser.next().await { + /// let triple = triple?; + /// if triple.predicate == rdf_type && triple.object == schema_person { + /// count += 1; + /// } + /// } + /// assert_eq!(2, count); + /// Ok(()) + /// } + /// ``` + #[cfg(feature = "async-tokio")] + pub fn parse_from_tokio_async_read( + &self, + read: R, + ) -> FromTokioAsyncReadN3Reader { + FromTokioAsyncReadN3Reader { + inner: self.parse().parser.parse_from_tokio_async_read(read), + } + } + /// Allows to parse a N3 file by using a low-level API. /// /// Count the number of people: @@ -343,6 +388,50 @@ impl Iterator for FromReadN3Reader { } } +/// Parses a N3 file from a [`AsyncRead`] implementation. Can be built using [`N3Parser::parse_from_tokio_async_read`]. +/// +/// Count the number of people: +/// ``` +/// use oxrdf::{NamedNode, vocab::rdf}; +/// use oxttl::n3::{N3Parser, N3Term}; +/// use oxttl::ParseError; +/// +/// #[tokio::main(flavor = "current_thread")] +/// async fn main() -> Result<(), ParseError> { +/// let file = b"@base . +/// @prefix schema: . +/// a schema:Person ; +/// schema:name \"Foo\" . +/// a schema:Person ; +/// schema:name \"Bar\" ."; +/// +/// let rdf_type = N3Term::NamedNode(rdf::TYPE.into_owned()); +/// let schema_person = N3Term::NamedNode(NamedNode::new_unchecked("http://schema.org/Person")); +/// let mut count = 0; +/// let mut parser = N3Parser::new().parse_from_tokio_async_read(file.as_ref()); +/// while let Some(triple) = parser.next().await { +/// let triple = triple?; +/// if triple.predicate == rdf_type && triple.object == schema_person { +/// count += 1; +/// } +/// } +/// assert_eq!(2, count); +/// Ok(()) +/// } +/// ``` +#[cfg(feature = "async-tokio")] +pub struct FromTokioAsyncReadN3Reader { + inner: FromTokioAsyncReadIterator, +} + +#[cfg(feature = "async-tokio")] +impl FromTokioAsyncReadN3Reader { + /// Reads the next triple or returns `None` if the file is finished. + pub async fn next(&mut self) -> Option> { + Some(self.inner.next().await?.map(Into::into)) + } +} + /// Parses a N3 file by using a low-level API. Can be built using [`N3Parser::parse`]. /// /// Count the number of people: diff --git a/lib/oxttl/src/nquads.rs b/lib/oxttl/src/nquads.rs index 4414d3fc..5cd27dae 100644 --- a/lib/oxttl/src/nquads.rs +++ b/lib/oxttl/src/nquads.rs @@ -1,9 +1,13 @@ //! A [N-Quads](https://www.w3.org/TR/n-quads/) streaming parser implemented by [`NQuadsParser`]. use crate::line_formats::NQuadsRecognizer; +#[cfg(feature = "async-tokio")] +use crate::toolkit::FromTokioAsyncReadIterator; use crate::toolkit::{FromReadIterator, ParseError, Parser, SyntaxError}; use oxrdf::{Quad, QuadRef}; use std::io::{self, Read, Write}; +#[cfg(feature = "async-tokio")] +use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; /// A [N-Quads](https://www.w3.org/TR/n-quads/) streaming parser. /// @@ -81,6 +85,43 @@ impl NQuadsParser { } } + /// Parses a N-Quads file from a [`AsyncRead`] implementation. + /// + /// Count the number of people: + /// ``` + /// use oxrdf::{NamedNodeRef, vocab::rdf}; + /// use oxttl::{ParseError, NQuadsParser}; + /// + /// #[tokio::main(flavor = "current_thread")] + /// async fn main() -> Result<(), ParseError> { + /// let file = b" . + /// \"Foo\" . + /// . + /// \"Bar\" ."; + /// + /// let schema_person = NamedNodeRef::new_unchecked("http://schema.org/Person"); + /// let mut count = 0; + /// let mut parser = NQuadsParser::new().parse_from_tokio_async_read(file.as_ref()); + /// while let Some(triple) = parser.next().await { + /// let triple = triple?; + /// if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { + /// count += 1; + /// } + /// } + /// assert_eq!(2, count); + /// Ok(()) + /// } + /// ``` + #[cfg(feature = "async-tokio")] + pub fn parse_from_tokio_async_read( + &self, + read: R, + ) -> FromTokioAsyncReadNQuadsReader { + FromTokioAsyncReadNQuadsReader { + inner: self.parse().parser.parse_from_tokio_async_read(read), + } + } + /// Allows to parse a N-Quads file by using a low-level API. /// /// Count the number of people: @@ -164,6 +205,46 @@ impl Iterator for FromReadNQuadsReader { } } +/// Parses a N-Quads file from a [`AsyncRead`] implementation. Can be built using [`NQuadsParser::parse_from_tokio_async_read`]. +/// +/// Count the number of people: +/// ``` +/// use oxrdf::{NamedNodeRef, vocab::rdf}; +/// use oxttl::{ParseError, NQuadsParser}; +/// +/// #[tokio::main(flavor = "current_thread")] +/// async fn main() -> Result<(), ParseError> { +/// let file = b" . +/// \"Foo\" . +/// . +/// \"Bar\" ."; +/// +/// let schema_person = NamedNodeRef::new_unchecked("http://schema.org/Person"); +/// let mut count = 0; +/// let mut parser = NQuadsParser::new().parse_from_tokio_async_read(file.as_ref()); +/// while let Some(triple) = parser.next().await { +/// let triple = triple?; +/// if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { +/// count += 1; +/// } +/// } +/// assert_eq!(2, count); +/// Ok(()) +/// } +/// ``` +#[cfg(feature = "async-tokio")] +pub struct FromTokioAsyncReadNQuadsReader { + inner: FromTokioAsyncReadIterator, +} + +#[cfg(feature = "async-tokio")] +impl FromTokioAsyncReadNQuadsReader { + /// Reads the next triple or returns `None` if the file is finished. + pub async fn next(&mut self) -> Option> { + Some(self.inner.next().await?.map(Into::into)) + } +} + /// Parses a N-Quads file by using a low-level API. Can be built using [`NQuadsParser::parse`]. /// /// Count the number of people: @@ -288,6 +369,41 @@ impl NQuadsSerializer { } } + /// Writes a N-Quads file to a [`AsyncWrite`] implementation. + /// + /// ``` + /// use oxrdf::{NamedNodeRef, QuadRef}; + /// use oxttl::NQuadsSerializer; + /// use std::io::Result; + /// + /// #[tokio::main(flavor = "current_thread")] + /// async fn main() -> Result<()> { + /// let mut writer = NQuadsSerializer::new().serialize_to_tokio_async_write(Vec::new()); + /// writer.write_quad(QuadRef::new( + /// NamedNodeRef::new_unchecked("http://example.com#me"), + /// NamedNodeRef::new_unchecked("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), + /// NamedNodeRef::new_unchecked("http://schema.org/Person"), + /// NamedNodeRef::new_unchecked("http://example.com"), + /// )).await?; + /// assert_eq!( + /// b" .\n", + /// writer.finish().as_slice() + /// ); + /// Ok(()) + /// } + /// ``` + #[cfg(feature = "async-tokio")] + pub fn serialize_to_tokio_async_write( + &self, + write: W, + ) -> ToTokioAsyncWriteNQuadsWriter { + ToTokioAsyncWriteNQuadsWriter { + write, + writer: self.serialize(), + buffer: Vec::new(), + } + } + /// Builds a low-level N-Quads writer. /// /// ``` @@ -350,6 +466,52 @@ impl ToWriteNQuadsWriter { } } +/// Writes a N-Quads file to a [`AsyncWrite`] implementation. Can be built using [`NQuadsSerializer::serialize_to_tokio_async_write`]. +/// +/// ``` +/// use oxrdf::{NamedNodeRef, QuadRef}; +/// use oxttl::NQuadsSerializer; +/// use std::io::Result; +/// +/// #[tokio::main(flavor = "current_thread")] +/// async fn main() -> Result<()> { +/// let mut writer = NQuadsSerializer::new().serialize_to_tokio_async_write(Vec::new()); +/// writer.write_quad(QuadRef::new( +/// NamedNodeRef::new_unchecked("http://example.com#me"), +/// NamedNodeRef::new_unchecked("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), +/// NamedNodeRef::new_unchecked("http://schema.org/Person"), +/// NamedNodeRef::new_unchecked("http://example.com"), +/// )).await?; +/// assert_eq!( +/// b" .\n", +/// writer.finish().as_slice() +/// ); +/// Ok(()) +/// } +/// ``` +#[cfg(feature = "async-tokio")] +pub struct ToTokioAsyncWriteNQuadsWriter { + write: W, + writer: LowLevelNQuadsWriter, + buffer: Vec, +} + +#[cfg(feature = "async-tokio")] +impl ToTokioAsyncWriteNQuadsWriter { + /// Writes an extra quad. + pub async fn write_quad<'a>(&mut self, q: impl Into>) -> io::Result<()> { + self.writer.write_quad(q, &mut self.buffer)?; + self.write.write_all(&self.buffer).await?; + self.buffer.clear(); + Ok(()) + } + + /// Ends the write process and returns the underlying [`Write`]. + pub fn finish(self) -> W { + self.write + } +} + /// Writes a N-Quads file by using a low-level API. Can be built using [`NQuadsSerializer::serialize`]. /// /// ``` diff --git a/lib/oxttl/src/ntriples.rs b/lib/oxttl/src/ntriples.rs index cac8cd22..2278db81 100644 --- a/lib/oxttl/src/ntriples.rs +++ b/lib/oxttl/src/ntriples.rs @@ -2,9 +2,13 @@ //! and a serializer implemented by [`NTriplesSerializer`]. use crate::line_formats::NQuadsRecognizer; +#[cfg(feature = "async-tokio")] +use crate::toolkit::FromTokioAsyncReadIterator; use crate::toolkit::{FromReadIterator, ParseError, Parser, SyntaxError}; use oxrdf::{Triple, TripleRef}; use std::io::{self, Read, Write}; +#[cfg(feature = "async-tokio")] +use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; /// A [N-Triples](https://www.w3.org/TR/n-triples/) streaming parser. /// @@ -82,6 +86,43 @@ impl NTriplesParser { } } + /// Parses a N-Triples file from a [`AsyncRead`] implementation. + /// + /// Count the number of people: + /// ``` + /// use oxrdf::{NamedNodeRef, vocab::rdf}; + /// use oxttl::{ParseError, NTriplesParser}; + /// + /// #[tokio::main(flavor = "current_thread")] + /// async fn main() -> Result<(), ParseError> { + /// let file = b" . + /// \"Foo\" . + /// . + /// \"Bar\" ."; + /// + /// let schema_person = NamedNodeRef::new_unchecked("http://schema.org/Person"); + /// let mut count = 0; + /// let mut parser = NTriplesParser::new().parse_from_tokio_async_read(file.as_ref()); + /// while let Some(triple) = parser.next().await { + /// let triple = triple?; + /// if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { + /// count += 1; + /// } + /// } + /// assert_eq!(2, count); + /// Ok(()) + /// } + /// ``` + #[cfg(feature = "async-tokio")] + pub fn parse_from_tokio_async_read( + &self, + read: R, + ) -> FromTokioAsyncReadNTriplesReader { + FromTokioAsyncReadNTriplesReader { + inner: self.parse().parser.parse_from_tokio_async_read(read), + } + } + /// Allows to parse a N-Triples file by using a low-level API. /// /// Count the number of people: @@ -165,6 +206,46 @@ impl Iterator for FromReadNTriplesReader { } } +/// Parses a N-Triples file from a [`AsyncRead`] implementation. Can be built using [`NTriplesParser::parse_from_tokio_async_read`]. +/// +/// Count the number of people: +/// ``` +/// use oxrdf::{NamedNodeRef, vocab::rdf}; +/// use oxttl::{ParseError, NTriplesParser}; +/// +/// #[tokio::main(flavor = "current_thread")] +/// async fn main() -> Result<(), ParseError> { +/// let file = b" . +/// \"Foo\" . +/// . +/// \"Bar\" ."; +/// +/// let schema_person = NamedNodeRef::new_unchecked("http://schema.org/Person"); +/// let mut count = 0; +/// let mut parser = NTriplesParser::new().parse_from_tokio_async_read(file.as_ref()); +/// while let Some(triple) = parser.next().await { +/// let triple = triple?; +/// if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { +/// count += 1; +/// } +/// } +/// assert_eq!(2, count); +/// Ok(()) +/// } +/// ``` +#[cfg(feature = "async-tokio")] +pub struct FromTokioAsyncReadNTriplesReader { + inner: FromTokioAsyncReadIterator, +} + +#[cfg(feature = "async-tokio")] +impl FromTokioAsyncReadNTriplesReader { + /// Reads the next triple or returns `None` if the file is finished. + pub async fn next(&mut self) -> Option> { + Some(self.inner.next().await?.map(Into::into)) + } +} + /// Parses a N-Triples file by using a low-level API. Can be built using [`NTriplesParser::parse`]. /// /// Count the number of people: @@ -287,6 +368,40 @@ impl NTriplesSerializer { } } + /// Writes a N-Triples file to a [`AsyncWrite`] implementation. + /// + /// ``` + /// use oxrdf::{NamedNodeRef, TripleRef}; + /// use oxttl::NTriplesSerializer; + /// use std::io::Result; + /// + /// #[tokio::main(flavor = "current_thread")] + /// async fn main() -> Result<()> { + /// let mut writer = NTriplesSerializer::new().serialize_to_tokio_async_write(Vec::new()); + /// writer.write_triple(TripleRef::new( + /// NamedNodeRef::new_unchecked("http://example.com#me"), + /// NamedNodeRef::new_unchecked("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), + /// NamedNodeRef::new_unchecked("http://schema.org/Person"), + /// )).await?; + /// assert_eq!( + /// b" .\n", + /// writer.finish().as_slice() + /// ); + /// Ok(()) + /// } + /// ``` + #[cfg(feature = "async-tokio")] + pub fn serialize_to_tokio_async_write( + &self, + write: W, + ) -> ToTokioAsyncWriteNTriplesWriter { + ToTokioAsyncWriteNTriplesWriter { + write, + writer: self.serialize(), + buffer: Vec::new(), + } + } + /// Builds a low-level N-Triples writer. /// /// ``` @@ -347,6 +462,51 @@ impl ToWriteNTriplesWriter { } } +/// Writes a N-Triples file to a [`AsyncWrite`] implementation. Can be built using [`NTriplesSerializer::serialize_to_tokio_async_write`]. +/// +/// ``` +/// use oxrdf::{NamedNodeRef, TripleRef}; +/// use oxttl::NTriplesSerializer; +/// use std::io::Result; +/// +/// #[tokio::main(flavor = "current_thread")] +/// async fn main() -> Result<()> { +/// let mut writer = NTriplesSerializer::new().serialize_to_tokio_async_write(Vec::new()); +/// writer.write_triple(TripleRef::new( +/// NamedNodeRef::new_unchecked("http://example.com#me"), +/// NamedNodeRef::new_unchecked("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), +/// NamedNodeRef::new_unchecked("http://schema.org/Person") +/// )).await?; +/// assert_eq!( +/// b" .\n", +/// writer.finish().as_slice() +/// ); +/// Ok(()) +/// } +/// ``` +#[cfg(feature = "async-tokio")] +pub struct ToTokioAsyncWriteNTriplesWriter { + write: W, + writer: LowLevelNTriplesWriter, + buffer: Vec, +} + +#[cfg(feature = "async-tokio")] +impl ToTokioAsyncWriteNTriplesWriter { + /// Writes an extra triple. + pub async fn write_triple<'a>(&mut self, t: impl Into>) -> io::Result<()> { + self.writer.write_triple(t, &mut self.buffer)?; + self.write.write_all(&self.buffer).await?; + self.buffer.clear(); + Ok(()) + } + + /// Ends the write process and returns the underlying [`Write`]. + pub fn finish(self) -> W { + self.write + } +} + /// Writes a N-Triples file by using a low-level API. Can be built using [`NTriplesSerializer::serialize`]. /// /// ``` diff --git a/lib/oxttl/src/toolkit/lexer.rs b/lib/oxttl/src/toolkit/lexer.rs index 3510ab28..664e35fe 100644 --- a/lib/oxttl/src/toolkit/lexer.rs +++ b/lib/oxttl/src/toolkit/lexer.rs @@ -3,6 +3,8 @@ use std::error::Error; use std::fmt; use std::io::{self, Read}; use std::ops::{Range, RangeInclusive}; +#[cfg(feature = "async-tokio")] +use tokio::io::{AsyncRead, AsyncReadExt}; pub trait TokenRecognizer { type Token<'a> @@ -122,6 +124,35 @@ impl Lexer { Ok(()) } + #[cfg(feature = "async-tokio")] + pub async fn extend_from_tokio_async_read( + &mut self, + read: &mut (impl AsyncRead + Unpin), + ) -> io::Result<()> { + self.shrink_if_useful(); + let min_end = self.end + self.min_buffer_size; + if min_end > self.max_buffer_size { + return Err(io::Error::new( + io::ErrorKind::OutOfMemory, + format!( + "The buffer maximal size is {} < {min_end}", + self.max_buffer_size + ), + )); + } + if self.data.len() < min_end { + self.data.resize(min_end, 0); + } + if self.data.len() < self.data.capacity() { + // We keep extending to have as much space as available without reallocation + self.data.resize(self.data.capacity(), 0); + } + let read = read.read(&mut self.data[self.end..]).await?; + self.end += read; + self.is_ending = read == 0; + Ok(()) + } + pub fn read_next( &mut self, options: &R::Options, diff --git a/lib/oxttl/src/toolkit/mod.rs b/lib/oxttl/src/toolkit/mod.rs index 39f9d40d..300b9c2c 100644 --- a/lib/oxttl/src/toolkit/mod.rs +++ b/lib/oxttl/src/toolkit/mod.rs @@ -6,6 +6,8 @@ mod lexer; mod parser; pub use self::lexer::{Lexer, LexerError, TokenRecognizer, TokenRecognizerError}; +#[cfg(feature = "async-tokio")] +pub use self::parser::FromTokioAsyncReadIterator; pub use self::parser::{ FromReadIterator, ParseError, Parser, RuleRecognizer, RuleRecognizerError, SyntaxError, }; diff --git a/lib/oxttl/src/toolkit/parser.rs b/lib/oxttl/src/toolkit/parser.rs index 44c01d5a..c5808199 100644 --- a/lib/oxttl/src/toolkit/parser.rs +++ b/lib/oxttl/src/toolkit/parser.rs @@ -4,6 +4,8 @@ use std::error::Error; use std::io::Read; use std::ops::Range; use std::{fmt, io}; +#[cfg(feature = "async-tokio")] +use tokio::io::AsyncRead; pub trait RuleRecognizer: Sized { type TokenRecognizer: TokenRecognizer; @@ -114,6 +116,14 @@ impl Parser { pub fn parse_from_read(self, read: R) -> FromReadIterator { FromReadIterator { read, parser: self } } + + #[cfg(feature = "async-tokio")] + pub fn parse_from_tokio_async_read( + self, + read: R, + ) -> FromTokioAsyncReadIterator { + FromTokioAsyncReadIterator { read, parser: self } + } } /// An error in the syntax of the parsed file. @@ -258,3 +268,29 @@ impl Iterator for FromReadIterator { None } } + +#[cfg(feature = "async-tokio")] +pub struct FromTokioAsyncReadIterator { + read: R, + parser: Parser, +} + +#[cfg(feature = "async-tokio")] +impl FromTokioAsyncReadIterator { + pub async fn next(&mut self) -> Option> { + while !self.parser.is_end() { + if let Some(result) = self.parser.read_next() { + return Some(result.map_err(ParseError::Syntax)); + } + if let Err(e) = self + .parser + .lexer + .extend_from_tokio_async_read(&mut self.read) + .await + { + return Some(Err(e.into())); + } + } + None + } +} diff --git a/lib/oxttl/src/trig.rs b/lib/oxttl/src/trig.rs index 62fc331f..5ad21402 100644 --- a/lib/oxttl/src/trig.rs +++ b/lib/oxttl/src/trig.rs @@ -1,12 +1,16 @@ //! A [TriG](https://www.w3.org/TR/trig/) streaming parser implemented by [`TriGParser`]. use crate::terse::TriGRecognizer; +#[cfg(feature = "async-tokio")] +use crate::toolkit::FromTokioAsyncReadIterator; use crate::toolkit::{FromReadIterator, ParseError, Parser, SyntaxError}; use oxiri::{Iri, IriParseError}; use oxrdf::{vocab::xsd, GraphName, NamedNode, Quad, QuadRef, Subject, TermRef}; use std::collections::HashMap; use std::fmt; use std::io::{self, Read, Write}; +#[cfg(feature = "async-tokio")] +use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; /// A [TriG](https://www.w3.org/TR/trig/) streaming parser. /// @@ -107,6 +111,45 @@ impl TriGParser { } } + /// Parses a TriG file from a [`AsyncRead`] implementation. + /// + /// Count the number of people: + /// ``` + /// use oxrdf::{NamedNodeRef, vocab::rdf}; + /// use oxttl::{ParseError, TriGParser}; + /// + /// #[tokio::main(flavor = "current_thread")] + /// async fn main() -> Result<(), ParseError> { + /// let file = b"@base . + /// @prefix schema: . + /// a schema:Person ; + /// schema:name \"Foo\" . + /// a schema:Person ; + /// schema:name \"Bar\" ."; + /// + /// let schema_person = NamedNodeRef::new_unchecked("http://schema.org/Person"); + /// let mut count = 0; + /// let mut parser = TriGParser::new().parse_from_tokio_async_read(file.as_ref()); + /// while let Some(triple) = parser.next().await { + /// let triple = triple?; + /// if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { + /// count += 1; + /// } + /// } + /// assert_eq!(2, count); + /// Ok(()) + /// } + /// ``` + #[cfg(feature = "async-tokio")] + pub fn parse_from_tokio_async_read( + &self, + read: R, + ) -> FromTokioAsyncReadTriGReader { + FromTokioAsyncReadTriGReader { + inner: self.parse().parser.parse_from_tokio_async_read(read), + } + } + /// Allows to parse a TriG file by using a low-level API. /// /// Count the number of people: @@ -193,6 +236,48 @@ impl Iterator for FromReadTriGReader { } } +/// Parses a TriG file from a [`AsyncRead`] implementation. Can be built using [`TriGParser::parse_from_tokio_async_read`]. +/// +/// Count the number of people: +/// ``` +/// use oxrdf::{NamedNodeRef, vocab::rdf}; +/// use oxttl::{ParseError, TriGParser}; +/// +/// #[tokio::main(flavor = "current_thread")] +/// async fn main() -> Result<(), ParseError> { +/// let file = b"@base . +/// @prefix schema: . +/// a schema:Person ; +/// schema:name \"Foo\" . +/// a schema:Person ; +/// schema:name \"Bar\" ."; +/// +/// let schema_person = NamedNodeRef::new_unchecked("http://schema.org/Person"); +/// let mut count = 0; +/// let mut parser = TriGParser::new().parse_from_tokio_async_read(file.as_ref()); +/// while let Some(triple) = parser.next().await { +/// let triple = triple?; +/// if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { +/// count += 1; +/// } +/// } +/// assert_eq!(2, count); +/// Ok(()) +/// } +/// ``` +#[cfg(feature = "async-tokio")] +pub struct FromTokioAsyncReadTriGReader { + inner: FromTokioAsyncReadIterator, +} + +#[cfg(feature = "async-tokio")] +impl FromTokioAsyncReadTriGReader { + /// Reads the next triple or returns `None` if the file is finished. + pub async fn next(&mut self) -> Option> { + Some(self.inner.next().await?.map(Into::into)) + } +} + /// Parses a TriG file by using a low-level API. Can be built using [`TriGParser::parse`]. /// /// Count the number of people: @@ -317,6 +402,41 @@ impl TriGSerializer { } } + /// Writes a TriG file to a [`AsyncWrite`] implementation. + /// + /// ``` + /// use oxrdf::{NamedNodeRef, QuadRef}; + /// use oxttl::TriGSerializer; + /// use std::io::Result; + /// + /// #[tokio::main(flavor = "current_thread")] + /// async fn main() -> Result<()> { + /// let mut writer = TriGSerializer::new().serialize_to_tokio_async_write(Vec::new()); + /// writer.write_quad(QuadRef::new( + /// NamedNodeRef::new_unchecked("http://example.com#me"), + /// NamedNodeRef::new_unchecked("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), + /// NamedNodeRef::new_unchecked("http://schema.org/Person"), + /// NamedNodeRef::new_unchecked("http://example.com"), + /// )).await?; + /// assert_eq!( + /// b" {\n\t .\n}\n", + /// writer.finish().await?.as_slice() + /// ); + /// Ok(()) + /// } + /// ``` + #[cfg(feature = "async-tokio")] + pub fn serialize_to_tokio_async_write( + &self, + write: W, + ) -> ToTokioAsyncWriteTriGWriter { + ToTokioAsyncWriteTriGWriter { + write, + writer: self.serialize(), + buffer: Vec::new(), + } + } + /// Builds a low-level TriG writer. /// /// ``` @@ -384,6 +504,55 @@ impl ToWriteTriGWriter { } } +/// Writes a TriG file to a [`AsyncWrite`] implementation. Can be built using [`TriGSerializer::serialize_to_tokio_async_write`]. +/// +/// ``` +/// use oxrdf::{NamedNodeRef, QuadRef}; +/// use oxttl::TriGSerializer; +/// use std::io::Result; +/// +/// #[tokio::main(flavor = "current_thread")] +/// async fn main() -> Result<()> { +/// let mut writer = TriGSerializer::new().serialize_to_tokio_async_write(Vec::new()); +/// writer.write_quad(QuadRef::new( +/// NamedNodeRef::new_unchecked("http://example.com#me"), +/// NamedNodeRef::new_unchecked("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), +/// NamedNodeRef::new_unchecked("http://schema.org/Person"), +/// NamedNodeRef::new_unchecked("http://example.com"), +/// )).await?; +/// assert_eq!( +/// b" {\n\t .\n}\n", +/// writer.finish().await?.as_slice() +/// ); +/// Ok(()) +/// } +/// ``` +#[cfg(feature = "async-tokio")] +pub struct ToTokioAsyncWriteTriGWriter { + write: W, + writer: LowLevelTriGWriter, + buffer: Vec, +} + +#[cfg(feature = "async-tokio")] +impl ToTokioAsyncWriteTriGWriter { + /// Writes an extra quad. + pub async fn write_quad<'a>(&mut self, q: impl Into>) -> io::Result<()> { + self.writer.write_quad(q, &mut self.buffer)?; + self.write.write_all(&self.buffer).await?; + self.buffer.clear(); + Ok(()) + } + + /// Ends the write process and returns the underlying [`Write`]. + pub async fn finish(mut self) -> io::Result { + self.writer.finish(&mut self.buffer)?; + self.write.write_all(&self.buffer).await?; + self.buffer.clear(); + Ok(self.write) + } +} + /// Writes a TriG file by using a low-level API. Can be built using [`TriGSerializer::serialize`]. /// /// ``` diff --git a/lib/oxttl/src/turtle.rs b/lib/oxttl/src/turtle.rs index 3b5639c0..0c875b8a 100644 --- a/lib/oxttl/src/turtle.rs +++ b/lib/oxttl/src/turtle.rs @@ -1,13 +1,18 @@ //! A [Turtle](https://www.w3.org/TR/turtle/) streaming parser implemented by [`TurtleParser`]. use crate::terse::TriGRecognizer; +#[cfg(feature = "async-tokio")] +use crate::toolkit::FromTokioAsyncReadIterator; use crate::toolkit::{FromReadIterator, ParseError, Parser, SyntaxError}; -use crate::trig::{LowLevelTriGWriter, ToWriteTriGWriter}; -use crate::TriGSerializer; +#[cfg(feature = "async-tokio")] +use crate::trig::ToTokioAsyncWriteTriGWriter; +use crate::trig::{LowLevelTriGWriter, ToWriteTriGWriter, TriGSerializer}; use oxiri::{Iri, IriParseError}; use oxrdf::{GraphNameRef, Triple, TripleRef}; use std::collections::HashMap; use std::io::{self, Read, Write}; +#[cfg(feature = "async-tokio")] +use tokio::io::{AsyncRead, AsyncWrite}; /// A [Turtle](https://www.w3.org/TR/turtle/) streaming parser. /// @@ -108,6 +113,45 @@ impl TurtleParser { } } + /// Parses a Turtle file from a [`AsyncRead`] implementation. + /// + /// Count the number of people: + /// ``` + /// use oxrdf::{NamedNodeRef, vocab::rdf}; + /// use oxttl::{ParseError, TurtleParser}; + /// + /// #[tokio::main(flavor = "current_thread")] + /// async fn main() -> Result<(), ParseError> { + /// let file = b"@base . + /// @prefix schema: . + /// a schema:Person ; + /// schema:name \"Foo\" . + /// a schema:Person ; + /// schema:name \"Bar\" ."; + /// + /// let schema_person = NamedNodeRef::new_unchecked("http://schema.org/Person"); + /// let mut count = 0; + /// let mut parser = TurtleParser::new().parse_from_tokio_async_read(file.as_ref()); + /// while let Some(triple) = parser.next().await { + /// let triple = triple?; + /// if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { + /// count += 1; + /// } + /// } + /// assert_eq!(2, count); + /// Ok(()) + /// } + /// ``` + #[cfg(feature = "async-tokio")] + pub fn parse_from_tokio_async_read( + &self, + read: R, + ) -> FromTokioAsyncReadTurtleReader { + FromTokioAsyncReadTurtleReader { + inner: self.parse().parser.parse_from_tokio_async_read(read), + } + } + /// Allows to parse a Turtle file by using a low-level API. /// /// Count the number of people: @@ -194,6 +238,48 @@ impl Iterator for FromReadTurtleReader { } } +/// Parses a Turtle file from a [`AsyncRead`] implementation. Can be built using [`TurtleParser::parse_from_tokio_async_read`]. +/// +/// Count the number of people: +/// ``` +/// use oxrdf::{NamedNodeRef, vocab::rdf}; +/// use oxttl::{ParseError, TurtleParser}; +/// +/// #[tokio::main(flavor = "current_thread")] +/// async fn main() -> Result<(), ParseError> { +/// let file = b"@base . +/// @prefix schema: . +/// a schema:Person ; +/// schema:name \"Foo\" . +/// a schema:Person ; +/// schema:name \"Bar\" ."; +/// +/// let schema_person = NamedNodeRef::new_unchecked("http://schema.org/Person"); +/// let mut count = 0; +/// let mut parser = TurtleParser::new().parse_from_tokio_async_read(file.as_ref()); +/// while let Some(triple) = parser.next().await { +/// let triple = triple?; +/// if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { +/// count += 1; +/// } +/// } +/// assert_eq!(2, count); +/// Ok(()) +/// } +/// ``` +#[cfg(feature = "async-tokio")] +pub struct FromTokioAsyncReadTurtleReader { + inner: FromTokioAsyncReadIterator, +} + +#[cfg(feature = "async-tokio")] +impl FromTokioAsyncReadTurtleReader { + /// Reads the next triple or returns `None` if the file is finished. + pub async fn next(&mut self) -> Option> { + Some(self.inner.next().await?.map(Into::into)) + } +} + /// Parses a Turtle file by using a low-level API. Can be built using [`TurtleParser::parse`]. /// /// Count the number of people: @@ -317,6 +403,38 @@ impl TurtleSerializer { } } + /// Writes a Turtle file to a [`AsyncWrite`] implementation. + /// + /// ``` + /// use oxrdf::{NamedNodeRef, TripleRef}; + /// use oxttl::TurtleSerializer; + /// use std::io::Result; + /// + /// #[tokio::main(flavor = "current_thread")] + /// async fn main() -> Result<()> { + /// let mut writer = TurtleSerializer::new().serialize_to_tokio_async_write(Vec::new()); + /// writer.write_triple(TripleRef::new( + /// NamedNodeRef::new_unchecked("http://example.com#me"), + /// NamedNodeRef::new_unchecked("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), + /// NamedNodeRef::new_unchecked("http://schema.org/Person"), + /// )).await?; + /// assert_eq!( + /// b" .\n", + /// writer.finish().await?.as_slice() + /// ); + /// Ok(()) + /// } + /// ``` + #[cfg(feature = "async-tokio")] + pub fn serialize_to_tokio_async_write( + &self, + write: W, + ) -> ToTokioAsyncWriteTurtleWriter { + ToTokioAsyncWriteTurtleWriter { + inner: self.inner.serialize_to_tokio_async_write(write), + } + } + /// Builds a low-level Turtle writer. /// /// ``` @@ -379,6 +497,48 @@ impl ToWriteTurtleWriter { } } +/// Writes a Turtle file to a [`AsyncWrite`] implementation. Can be built using [`TurtleSerializer::serialize_to_tokio_async_write`]. +/// +/// ``` +/// use oxrdf::{NamedNodeRef, TripleRef}; +/// use oxttl::TurtleSerializer; +/// use std::io::Result; +/// +/// #[tokio::main(flavor = "current_thread")] +/// async fn main() -> Result<()> { +/// let mut writer = TurtleSerializer::new().serialize_to_tokio_async_write(Vec::new()); +/// writer.write_triple(TripleRef::new( +/// NamedNodeRef::new_unchecked("http://example.com#me"), +/// NamedNodeRef::new_unchecked("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), +/// NamedNodeRef::new_unchecked("http://schema.org/Person") +/// )).await?; +/// assert_eq!( +/// b" .\n", +/// writer.finish().await?.as_slice() +/// ); +/// Ok(()) +/// } +/// ``` +#[cfg(feature = "async-tokio")] +pub struct ToTokioAsyncWriteTurtleWriter { + inner: ToTokioAsyncWriteTriGWriter, +} + +#[cfg(feature = "async-tokio")] +impl ToTokioAsyncWriteTurtleWriter { + /// Writes an extra triple. + pub async fn write_triple<'a>(&mut self, t: impl Into>) -> io::Result<()> { + self.inner + .write_quad(t.into().in_graph(GraphNameRef::DefaultGraph)) + .await + } + + /// Ends the write process and returns the underlying [`Write`]. + pub async fn finish(self) -> io::Result { + self.inner.finish().await + } +} + /// Writes a Turtle file by using a low-level API. Can be built using [`TurtleSerializer::serialize`]. /// /// ``` From c31ba0e82388b44f53b1ebbef918eaecbbe61d20 Mon Sep 17 00:00:00 2001 From: Thomas Date: Fri, 7 Jul 2023 17:13:03 +0200 Subject: [PATCH 031/217] Makes sparopt figure out good join keys --- lib/sparopt/src/algebra.rs | 62 ++++++++-- lib/sparopt/src/optimizer.rs | 191 +++++++++++++++++++----------- lib/sparopt/src/type_inference.rs | 11 +- lib/src/sparql/eval.rs | 22 +--- lib/src/sparql/plan.rs | 139 ++-------------------- lib/src/sparql/plan_builder.rs | 38 ++++-- 6 files changed, 231 insertions(+), 232 deletions(-) diff --git a/lib/sparopt/src/algebra.rs b/lib/sparopt/src/algebra.rs index 65c4c618..c7d7c82a 100644 --- a/lib/sparopt/src/algebra.rs +++ b/lib/sparopt/src/algebra.rs @@ -660,6 +660,7 @@ pub enum GraphPattern { left: Box, right: Box, expression: Expression, + algorithm: LeftJoinAlgorithm, }, /// Lateral join i.e. evaluate right for all result row of left #[cfg(feature = "sep-0006")] @@ -678,7 +679,11 @@ pub enum GraphPattern { expression: Expression, }, /// [Minus](https://www.w3.org/TR/sparql11-query/#defn_algMinus). - Minus { left: Box, right: Box }, + Minus { + left: Box, + right: Box, + algorithm: MinusAlgorithm, + }, /// A table used to provide inline values Values { variables: Vec, @@ -784,7 +789,12 @@ impl GraphPattern { } } - pub fn left_join(left: Self, right: Self, expression: Expression) -> Self { + pub fn left_join( + left: Self, + right: Self, + expression: Expression, + algorithm: LeftJoinAlgorithm, + ) -> Self { let expression_ebv = expression.effective_boolean_value(); if left.is_empty() || right.is_empty() @@ -801,10 +811,11 @@ impl GraphPattern { } else { expression }, + algorithm, } } - pub fn minus(left: Self, right: Self) -> Self { + pub fn minus(left: Self, right: Self, algorithm: MinusAlgorithm) -> Self { if left.is_empty() { return Self::empty(); } @@ -814,6 +825,7 @@ impl GraphPattern { Self::Minus { left: Box::new(left), right: Box::new(right), + algorithm, } } @@ -1046,7 +1058,7 @@ impl GraphPattern { child.lookup_used_variables(callback); } } - Self::Join { left, right, .. } | Self::Minus { left, right } => { + Self::Join { left, right, .. } | Self::Minus { left, right, .. } => { left.lookup_used_variables(callback); right.lookup_used_variables(callback); } @@ -1059,6 +1071,7 @@ impl GraphPattern { left, right, expression, + .. } => { expression.lookup_used_variables(callback); left.lookup_used_variables(callback); @@ -1148,6 +1161,7 @@ impl GraphPattern { || true.into(), |e| Expression::from_sparql_algebra(e, graph_name), ), + algorithm: LeftJoinAlgorithm::default(), }, #[cfg(feature = "sep-0006")] AlGraphPattern::Lateral { left, right } => Self::Lateral { @@ -1179,6 +1193,7 @@ impl GraphPattern { AlGraphPattern::Minus { left, right } => Self::Minus { left: Box::new(Self::from_sparql_algebra(left, graph_name, blank_nodes)), right: Box::new(Self::from_sparql_algebra(right, graph_name, blank_nodes)), + algorithm: MinusAlgorithm::default(), }, AlGraphPattern::Values { variables, @@ -1365,6 +1380,7 @@ impl From<&GraphPattern> for AlGraphPattern { left, right, expression, + .. } => { let empty_expr = if let Expression::Literal(l) = expression { l.datatype() == xsd::BOOLEAN && l.value() == "true" @@ -1418,7 +1434,7 @@ impl From<&GraphPattern> for AlGraphPattern { expression: expression.into(), variable: variable.clone(), }, - GraphPattern::Minus { left, right } => Self::Minus { + GraphPattern::Minus { left, right, .. } => Self::Minus { left: Box::new(left.as_ref().into()), right: Box::new(right.as_ref().into()), }, @@ -1478,14 +1494,44 @@ impl From<&GraphPattern> for AlGraphPattern { } /// The join algorithm used (c.f. [`GraphPattern::Join`]). -#[derive(Eq, PartialEq, Debug, Clone, Copy, Hash)] +#[derive(Eq, PartialEq, Debug, Clone, Hash)] pub enum JoinAlgorithm { - HashBuildLeftProbeRight, + HashBuildLeftProbeRight { keys: Vec }, } impl Default for JoinAlgorithm { fn default() -> Self { - Self::HashBuildLeftProbeRight + Self::HashBuildLeftProbeRight { + keys: Vec::default(), + } + } +} + +/// The left join algorithm used (c.f. [`GraphPattern::LeftJoin`]). +#[derive(Eq, PartialEq, Debug, Clone, Hash)] +pub enum LeftJoinAlgorithm { + HashBuildRightProbeLeft { keys: Vec }, +} + +impl Default for LeftJoinAlgorithm { + fn default() -> Self { + Self::HashBuildRightProbeLeft { + keys: Vec::default(), + } + } +} + +/// The left join algorithm used (c.f. [`GraphPattern::Minus`]). +#[derive(Eq, PartialEq, Debug, Clone, Hash)] +pub enum MinusAlgorithm { + HashBuildRightProbeLeft { keys: Vec }, +} + +impl Default for MinusAlgorithm { + fn default() -> Self { + Self::HashBuildRightProbeLeft { + keys: Vec::default(), + } } } diff --git a/lib/sparopt/src/optimizer.rs b/lib/sparopt/src/optimizer.rs index 91ff65d0..a141bd63 100644 --- a/lib/sparopt/src/optimizer.rs +++ b/lib/sparopt/src/optimizer.rs @@ -1,7 +1,10 @@ -use crate::algebra::{Expression, GraphPattern, JoinAlgorithm, OrderExpression}; +use crate::algebra::{ + Expression, GraphPattern, JoinAlgorithm, LeftJoinAlgorithm, MinusAlgorithm, OrderExpression, +}; use crate::type_inference::{ infer_expression_type, infer_graph_pattern_types, VariableType, VariableTypes, }; +use oxrdf::Variable; use spargebra::algebra::PropertyPathExpression; use spargebra::term::{GroundTermPattern, NamedNodePattern}; use std::cmp::{max, min}; @@ -53,6 +56,7 @@ impl Optimizer { left, right, expression, + algorithm, } => { let left = Self::normalize_pattern(*left, input_types); let right = Self::normalize_pattern(*right, input_types); @@ -62,6 +66,7 @@ impl Optimizer { left, right, Self::normalize_expression(expression, &inner_types), + algorithm, ) } #[cfg(feature = "sep-0006")] @@ -103,9 +108,14 @@ impl Optimizer { GraphPattern::extend(inner, variable, expression) } } - GraphPattern::Minus { left, right } => GraphPattern::minus( + GraphPattern::Minus { + left, + right, + algorithm, + } => GraphPattern::minus( Self::normalize_pattern(*left, input_types), Self::normalize_pattern(*right, input_types), + algorithm, ), GraphPattern::Values { variables, @@ -336,6 +346,7 @@ impl Optimizer { left, right, expression, + algorithm, } => { let left_types = infer_graph_pattern_types(&left, input_types.clone()); let right_types = infer_graph_pattern_types(&right, input_types.clone()); @@ -364,13 +375,19 @@ impl Optimizer { Self::push_filters(*left, left_filters, input_types), Self::push_filters(*right, right_filters, input_types), expression, + algorithm, ), Expression::and_all(final_filters), ) } - GraphPattern::Minus { left, right } => GraphPattern::minus( + GraphPattern::Minus { + left, + right, + algorithm, + } => GraphPattern::minus( Self::push_filters(*left, filters, input_types), Self::push_filters(*right, Vec::new(), input_types), + algorithm, ), GraphPattern::Extend { inner, @@ -503,11 +520,7 @@ impl Optimizer { .enumerate() .filter_map(|(i, v)| v.then(|| i)) .filter(|i| { - count_common_variables( - &output_types, - &to_reorder_types[*i], - input_types, - ) > 0 + has_common_variables(&output_types, &to_reorder_types[*i], input_types) }) .min_by_key(|i| { // Estimation of the join cost @@ -527,10 +540,14 @@ impl Optimizer { } else { estimate_join_cost( &output, - &output_types, &to_reorder[*i], - &to_reorder_types[*i], - JoinAlgorithm::HashBuildLeftProbeRight, + &JoinAlgorithm::HashBuildLeftProbeRight { + keys: join_key_variables( + &output_types, + &to_reorder_types[*i], + input_types, + ), + }, input_types, ) } @@ -547,7 +564,13 @@ impl Optimizer { GraphPattern::join( output, next, - JoinAlgorithm::HashBuildLeftProbeRight, + JoinAlgorithm::HashBuildLeftProbeRight { + keys: join_key_variables( + &output_types, + &to_reorder_types[next_id], + input_types, + ), + }, ) }; } @@ -556,7 +579,13 @@ impl Optimizer { output = GraphPattern::join( output, next, - JoinAlgorithm::HashBuildLeftProbeRight, + JoinAlgorithm::HashBuildLeftProbeRight { + keys: join_key_variables( + &output_types, + &to_reorder_types[next_id], + input_types, + ), + }, ); } output_types.intersect_with(to_reorder_types[next_id].clone()); @@ -566,12 +595,25 @@ impl Optimizer { output_cartesian_product_joins .into_iter() .reduce(|left, right| { + let keys = join_key_variables( + &infer_graph_pattern_types(&left, input_types.clone()), + &infer_graph_pattern_types(&right, input_types.clone()), + input_types, + ); if estimate_graph_pattern_size(&left, input_types) <= estimate_graph_pattern_size(&right, input_types) { - GraphPattern::join(left, right, JoinAlgorithm::HashBuildLeftProbeRight) + GraphPattern::join( + left, + right, + JoinAlgorithm::HashBuildLeftProbeRight { keys }, + ) } else { - GraphPattern::join(right, left, JoinAlgorithm::HashBuildLeftProbeRight) + GraphPattern::join( + right, + left, + JoinAlgorithm::HashBuildLeftProbeRight { keys }, + ) } }) .unwrap() @@ -588,15 +630,16 @@ impl Optimizer { left, right, expression, + .. } => { let left = Self::reorder_joins(*left, input_types); + let left_types = infer_graph_pattern_types(&left, input_types.clone()); let right = Self::reorder_joins(*right, input_types); + let right_types = infer_graph_pattern_types(&right, input_types.clone()); #[cfg(feature = "sep-0006")] { - let left_types = infer_graph_pattern_types(&left, input_types.clone()); - let right_types = infer_graph_pattern_types(&right, input_types.clone()); if is_fit_for_for_loop_join(&right, input_types, &left_types) - && count_common_variables(&left_types, &right_types, input_types) > 0 + && has_common_variables(&left_types, &right_types, input_types) { return GraphPattern::lateral( left, @@ -604,16 +647,33 @@ impl Optimizer { GraphPattern::empty_singleton(), right, expression, + LeftJoinAlgorithm::HashBuildRightProbeLeft { keys: Vec::new() }, ), ); } } - GraphPattern::left_join(left, right, expression) + GraphPattern::left_join( + left, + right, + expression, + LeftJoinAlgorithm::HashBuildRightProbeLeft { + keys: join_key_variables(&left_types, &right_types, input_types), + }, + ) + } + GraphPattern::Minus { left, right, .. } => { + let left = Self::reorder_joins(*left, input_types); + let left_types = infer_graph_pattern_types(&left, input_types.clone()); + let right = Self::reorder_joins(*right, input_types); + let right_types = infer_graph_pattern_types(&right, input_types.clone()); + GraphPattern::minus( + left, + right, + MinusAlgorithm::HashBuildRightProbeLeft { + keys: join_key_variables(&left_types, &right_types, input_types), + }, + ) } - GraphPattern::Minus { left, right } => GraphPattern::minus( - Self::reorder_joins(*left, input_types), - Self::reorder_joins(*right, input_types), - ), GraphPattern::Extend { inner, expression, @@ -685,6 +745,7 @@ fn is_fit_for_for_loop_join( left, right, expression, + .. } => { if !is_fit_for_for_loop_join(left, global_input_types, entry_types) { return false; @@ -802,17 +863,28 @@ fn is_expression_fit_for_for_loop_join( } } -fn count_common_variables( +fn has_common_variables( left: &VariableTypes, right: &VariableTypes, input_types: &VariableTypes, -) -> usize { +) -> bool { // TODO: we should be smart and count as shared variables FILTER(?a = ?b) + left.iter().any(|(variable, left_type)| { + !left_type.undef && !right.get(variable).undef && input_types.get(variable).undef + }) +} + +fn join_key_variables( + left: &VariableTypes, + right: &VariableTypes, + input_types: &VariableTypes, +) -> Vec { left.iter() .filter(|(variable, left_type)| { !left_type.undef && !right.get(variable).undef && input_types.get(variable).undef }) - .count() + .map(|(variable, _)| variable.clone()) + .collect() } fn estimate_graph_pattern_size(pattern: &GraphPattern, input_types: &VariableTypes) -> usize { @@ -842,35 +914,26 @@ fn estimate_graph_pattern_size(pattern: &GraphPattern, input_types: &VariableTyp left, right, algorithm, - } => { - let left_types = infer_graph_pattern_types(left, input_types.clone()); - let right_types = infer_graph_pattern_types(right, input_types.clone()); - estimate_join_cost( - left, - &left_types, - right, - &right_types, - *algorithm, - input_types, - ) - } - GraphPattern::LeftJoin { left, right, .. } => { - let left_size = estimate_graph_pattern_size(left, input_types); - let left_types = infer_graph_pattern_types(left, input_types.clone()); - let right_types = infer_graph_pattern_types(right, input_types.clone()); - max( - left_size, - left_size - .saturating_mul(estimate_graph_pattern_size(right, &right_types)) - .saturating_div( - 1_000_usize.saturating_pow( - count_common_variables(&left_types, &right_types, input_types) - .try_into() - .unwrap(), - ), - ), - ) - } + } => estimate_join_cost(left, right, algorithm, input_types), + GraphPattern::LeftJoin { + left, + right, + algorithm, + .. + } => match algorithm { + LeftJoinAlgorithm::HashBuildRightProbeLeft { keys } => { + let left_size = estimate_graph_pattern_size(left, input_types); + max( + left_size, + left_size + .saturating_mul(estimate_graph_pattern_size( + right, + &infer_graph_pattern_types(right, input_types.clone()), + )) + .saturating_div(1_000_usize.saturating_pow(keys.len().try_into().unwrap())), + ) + } + }, #[cfg(feature = "sep-0006")] GraphPattern::Lateral { left, right } => estimate_lateral_cost( left, @@ -908,22 +971,16 @@ fn estimate_graph_pattern_size(pattern: &GraphPattern, input_types: &VariableTyp fn estimate_join_cost( left: &GraphPattern, - left_types: &VariableTypes, right: &GraphPattern, - right_types: &VariableTypes, - algorithm: JoinAlgorithm, + algorithm: &JoinAlgorithm, input_types: &VariableTypes, ) -> usize { match algorithm { - JoinAlgorithm::HashBuildLeftProbeRight => estimate_graph_pattern_size(left, input_types) - .saturating_mul(estimate_graph_pattern_size(right, input_types)) - .saturating_div( - 1_000_usize.saturating_pow( - count_common_variables(left_types, right_types, input_types) - .try_into() - .unwrap(), - ), - ), + JoinAlgorithm::HashBuildLeftProbeRight { keys } => { + estimate_graph_pattern_size(left, input_types) + .saturating_mul(estimate_graph_pattern_size(right, input_types)) + .saturating_div(1_000_usize.saturating_pow(keys.len().try_into().unwrap())) + } } } fn estimate_lateral_cost( diff --git a/lib/sparopt/src/type_inference.rs b/lib/sparopt/src/type_inference.rs index 421fd756..03960adb 100644 --- a/lib/sparopt/src/type_inference.rs +++ b/lib/sparopt/src/type_inference.rs @@ -124,11 +124,20 @@ pub fn infer_graph_pattern_types( } types } - GraphPattern::Service { name, inner, .. } => { + GraphPattern::Service { + name, + inner, + silent, + } => { + let parent_types = types.clone(); let mut types = infer_graph_pattern_types(inner, types); if let NamedNodePattern::Variable(v) = name { types.intersect_variable_with(v.clone(), VariableType::NAMED_NODE) } + if *silent { + // On failure, single empty solution + types.union_with(parent_types); + } types } } diff --git a/lib/src/sparql/eval.rs b/lib/src/sparql/eval.rs index 8ab0d8ba..06372e25 100644 --- a/lib/src/sparql/eval.rs +++ b/lib/src/sparql/eval.rs @@ -380,12 +380,9 @@ impl SimpleEvaluator { PlanNode::HashJoin { probe_child, build_child, + keys, } => { - let join_keys: Vec<_> = probe_child - .always_bound_variables() - .intersection(&build_child.always_bound_variables()) - .copied() - .collect(); + let join_keys = keys.iter().map(|v| v.encoded).collect::>(); let (probe, probe_stats) = self.plan_evaluator(probe_child); stat_children.push(probe_stats); let (build, build_stats) = self.plan_evaluator(build_child); @@ -444,12 +441,8 @@ impl SimpleEvaluator { })) }) } - PlanNode::AntiJoin { left, right } => { - let join_keys: Vec<_> = left - .always_bound_variables() - .intersection(&right.always_bound_variables()) - .copied() - .collect(); + PlanNode::AntiJoin { left, right, keys } => { + let join_keys = keys.iter().map(|v| v.encoded).collect::>(); let (left, left_stats) = self.plan_evaluator(left); stat_children.push(left_stats); let (right, right_stats) = self.plan_evaluator(right); @@ -487,12 +480,9 @@ impl SimpleEvaluator { left, right, expression, + keys, } => { - let join_keys: Vec<_> = left - .always_bound_variables() - .intersection(&right.always_bound_variables()) - .copied() - .collect(); + let join_keys = keys.iter().map(|v| v.encoded).collect::>(); let (left, left_stats) = self.plan_evaluator(left); stat_children.push(left_stats); let (right, right_stats) = self.plan_evaluator(right); diff --git a/lib/src/sparql/plan.rs b/lib/src/sparql/plan.rs index a447fb01..fd9fb989 100644 --- a/lib/src/sparql/plan.rs +++ b/lib/src/sparql/plan.rs @@ -4,9 +4,7 @@ use crate::storage::numeric_encoder::EncodedTerm; use regex::Regex; use spargebra::algebra::GraphPattern; use spargebra::term::GroundTerm; -use std::cmp::max; -use std::collections::btree_map::Entry; -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::BTreeSet; use std::fmt; use std::rc::Rc; @@ -40,6 +38,7 @@ pub enum PlanNode { HashJoin { probe_child: Rc, build_child: Rc, + keys: Vec, }, /// Right nested in left loop ForLoopJoin { @@ -50,6 +49,7 @@ pub enum PlanNode { AntiJoin { left: Rc, right: Rc, + keys: Vec, }, Filter { child: Rc, @@ -63,6 +63,7 @@ pub enum PlanNode { left: Rc, right: Rc, expression: Box, + keys: Vec, }, /// right nested in left loop ForLoopLeftJoin { @@ -159,9 +160,10 @@ impl PlanNode { Self::HashJoin { probe_child: left, build_child: right, + .. } | Self::ForLoopJoin { left, right, .. } - | Self::AntiJoin { left, right } + | Self::AntiJoin { left, right, .. } | Self::ForLoopLeftJoin { left, right, .. } => { left.lookup_used_variables(callback); right.lookup_used_variables(callback); @@ -170,6 +172,7 @@ impl PlanNode { left, right, expression, + .. } => { left.lookup_used_variables(callback); right.lookup_used_variables(callback); @@ -219,134 +222,6 @@ impl PlanNode { } } } - - /// Returns subset of the set of variables that are always bound in the result set - /// - /// (subset because this function is not perfect yet) - pub fn always_bound_variables(&self) -> BTreeSet { - let mut set = BTreeSet::default(); - self.lookup_always_bound_variables(&mut |v| { - set.insert(v); - }); - set - } - - pub fn lookup_always_bound_variables(&self, callback: &mut impl FnMut(usize)) { - match self { - Self::StaticBindings { encoded_tuples, .. } => { - let mut variables = BTreeMap::default(); // value true iff always bound - let max_tuple_length = encoded_tuples - .iter() - .map(EncodedTuple::capacity) - .fold(0, max); - for tuple in encoded_tuples { - for key in 0..max_tuple_length { - match variables.entry(key) { - Entry::Vacant(e) => { - e.insert(tuple.contains(key)); - } - Entry::Occupied(mut e) => { - if !tuple.contains(key) { - e.insert(false); - } - } - } - } - } - for (k, v) in variables { - if v { - callback(k); - } - } - } - Self::QuadPattern { - subject, - predicate, - object, - graph_name, - } => { - subject.lookup_variables(callback); - predicate.lookup_variables(callback); - object.lookup_variables(callback); - graph_name.lookup_variables(callback); - } - Self::PathPattern { - subject, - object, - graph_name, - .. - } => { - subject.lookup_variables(callback); - object.lookup_variables(callback); - graph_name.lookup_variables(callback); - } - Self::Filter { child, .. } => { - //TODO: have a look at the expression to know if it filters out unbound variables - child.lookup_always_bound_variables(callback); - } - Self::Union { children } => { - if let Some(vars) = children - .iter() - .map(|c| c.always_bound_variables()) - .reduce(|a, b| a.intersection(&b).copied().collect()) - { - for v in vars { - callback(v); - } - } - } - Self::HashJoin { - probe_child: left, - build_child: right, - } - | Self::ForLoopJoin { left, right, .. } => { - left.lookup_always_bound_variables(callback); - right.lookup_always_bound_variables(callback); - } - Self::AntiJoin { left, .. } - | Self::HashLeftJoin { left, .. } - | Self::ForLoopLeftJoin { left, .. } => { - left.lookup_always_bound_variables(callback); - } - Self::Extend { - child, - variable, - expression, - } => { - if matches!( - expression.as_ref(), - PlanExpression::NamedNode(_) | PlanExpression::Literal(_) - ) { - // TODO: more cases? - callback(variable.encoded); - } - child.lookup_always_bound_variables(callback); - } - Self::Sort { child, .. } - | Self::HashDeduplicate { child } - | Self::Reduced { child } - | Self::Skip { child, .. } - | Self::Limit { child, .. } => child.lookup_always_bound_variables(callback), - Self::Service { child, silent, .. } => { - if *silent { - // none, might return a null tuple - } else { - child.lookup_always_bound_variables(callback) - } - } - Self::Project { mapping, child } => { - let child_bound = child.always_bound_variables(); - for (child_i, output_i) in mapping.iter() { - if child_bound.contains(&child_i.encoded) { - callback(output_i.encoded); - } - } - } - Self::Aggregate { .. } => { - //TODO - } - } - } } #[derive(Debug, Clone)] diff --git a/lib/src/sparql/plan_builder.rs b/lib/src/sparql/plan_builder.rs index ec93bb7f..12faeb6c 100644 --- a/lib/src/sparql/plan_builder.rs +++ b/lib/src/sparql/plan_builder.rs @@ -106,25 +106,37 @@ impl<'a> PlanBuilder<'a> { right, algorithm, } => match algorithm { - JoinAlgorithm::HashBuildLeftProbeRight => PlanNode::HashJoin { + JoinAlgorithm::HashBuildLeftProbeRight { keys } => PlanNode::HashJoin { build_child: Rc::new(self.build_for_graph_pattern(left, variables)?), probe_child: Rc::new(self.build_for_graph_pattern(right, variables)?), + keys: keys + .iter() + .map(|v| build_plan_variable(variables, v)) + .collect(), }, }, GraphPattern::LeftJoin { left, right, expression, - } => PlanNode::HashLeftJoin { - left: Rc::new(self.build_for_graph_pattern(left, variables)?), - right: Rc::new(self.build_for_graph_pattern(right, variables)?), - expression: Box::new(self.build_for_expression(expression, variables)?), + algorithm, + } => match algorithm { + LeftJoinAlgorithm::HashBuildRightProbeLeft { keys } => PlanNode::HashLeftJoin { + left: Rc::new(self.build_for_graph_pattern(left, variables)?), + right: Rc::new(self.build_for_graph_pattern(right, variables)?), + expression: Box::new(self.build_for_expression(expression, variables)?), + keys: keys + .iter() + .map(|v| build_plan_variable(variables, v)) + .collect(), + }, }, GraphPattern::Lateral { left, right } => { if let GraphPattern::LeftJoin { left: nested_left, right: nested_right, expression, + .. } = right.as_ref() { if nested_left.is_empty_singleton() { @@ -167,9 +179,19 @@ impl<'a> PlanBuilder<'a> { variable: build_plan_variable(variables, variable), expression: Box::new(self.build_for_expression(expression, variables)?), }, - GraphPattern::Minus { left, right } => PlanNode::AntiJoin { - left: Rc::new(self.build_for_graph_pattern(left, variables)?), - right: Rc::new(self.build_for_graph_pattern(right, variables)?), + GraphPattern::Minus { + left, + right, + algorithm, + } => match algorithm { + MinusAlgorithm::HashBuildRightProbeLeft { keys } => PlanNode::AntiJoin { + left: Rc::new(self.build_for_graph_pattern(left, variables)?), + right: Rc::new(self.build_for_graph_pattern(right, variables)?), + keys: keys + .iter() + .map(|v| build_plan_variable(variables, v)) + .collect(), + }, }, GraphPattern::Service { name, From 8e770fbb5dde12bcd8ce72cc390329f110ab509d Mon Sep 17 00:00:00 2001 From: Thomas Date: Fri, 7 Jul 2023 17:02:47 +0200 Subject: [PATCH 032/217] SPARQL: Removes intermediate query plan representation Covered by the optimizer plan --- js/src/store.rs | 2 +- lib/src/sparql/eval.rs | 3737 ++++++++++++++++++++------------ lib/src/sparql/mod.rs | 69 +- lib/src/sparql/plan.rs | 896 -------- lib/src/sparql/plan_builder.rs | 1109 ---------- lib/src/sparql/update.rs | 21 +- 6 files changed, 2397 insertions(+), 3437 deletions(-) delete mode 100644 lib/src/sparql/plan.rs delete mode 100644 lib/src/sparql/plan_builder.rs diff --git a/js/src/store.rs b/js/src/store.rs index 12d4a039..9fce396d 100644 --- a/js/src/store.rs +++ b/js/src/store.rs @@ -26,7 +26,7 @@ impl JsStore { store: Store::new().map_err(to_err)?, }; if let Some(quads) = quads { - for quad in quads.iter() { + for quad in &*quads { store.add(quad)?; } } diff --git a/lib/src/sparql/eval.rs b/lib/src/sparql/eval.rs index 06372e25..82f0e06f 100644 --- a/lib/src/sparql/eval.rs +++ b/lib/src/sparql/eval.rs @@ -4,7 +4,6 @@ use crate::sparql::algebra::{Query, QueryDataset}; use crate::sparql::dataset::DatasetView; use crate::sparql::error::EvaluationError; use crate::sparql::model::*; -use crate::sparql::plan::*; use crate::sparql::service::ServiceHandler; use crate::storage::numeric_encoder::*; use crate::storage::small_string::SmallString; @@ -13,13 +12,21 @@ use json_event_parser::{JsonEvent, JsonWriter}; use md5::Md5; use oxilangtag::LanguageTag; use oxiri::Iri; -use oxrdf::Variable; +use oxrdf::{TermRef, Variable}; use oxsdatatypes::*; use rand::random; use regex::{Regex, RegexBuilder}; use sha1::Sha1; use sha2::{Sha256, Sha384, Sha512}; -use spargebra::algebra::GraphPattern; +use spargebra::algebra::{Function, PropertyPathExpression}; +use spargebra::term::{ + GroundSubject, GroundTerm, GroundTermPattern, GroundTriple, NamedNodePattern, TermPattern, + TriplePattern, +}; +use sparopt::algebra::{ + AggregateExpression, Expression, GraphPattern, JoinAlgorithm, LeftJoinAlgorithm, + MinusAlgorithm, OrderExpression, +}; use std::cell::Cell; use std::cmp::Ordering; use std::collections::hash_map::DefaultHasher; @@ -35,6 +42,85 @@ use std::{fmt, io, str}; const REGEX_SIZE_LIMIT: usize = 1_000_000; +#[derive(Eq, PartialEq, Debug, Clone, Hash)] +pub struct EncodedTuple { + inner: Vec>, +} + +impl EncodedTuple { + pub fn with_capacity(capacity: usize) -> Self { + Self { + inner: Vec::with_capacity(capacity), + } + } + + pub fn capacity(&self) -> usize { + self.inner.capacity() + } + + pub fn contains(&self, index: usize) -> bool { + self.inner.get(index).map_or(false, Option::is_some) + } + + pub fn get(&self, index: usize) -> Option<&EncodedTerm> { + self.inner.get(index).unwrap_or(&None).as_ref() + } + + pub fn iter(&self) -> impl Iterator> + '_ { + self.inner.iter().cloned() + } + + pub fn set(&mut self, index: usize, value: EncodedTerm) { + if self.inner.len() <= index { + self.inner.resize(index + 1, None); + } + self.inner[index] = Some(value); + } + + pub fn combine_with(&self, other: &Self) -> Option { + if self.inner.len() < other.inner.len() { + let mut result = other.inner.clone(); + for (key, self_value) in self.inner.iter().enumerate() { + if let Some(self_value) = self_value { + match &other.inner[key] { + Some(other_value) => { + if self_value != other_value { + return None; + } + } + None => result[key] = Some(self_value.clone()), + } + } + } + Some(Self { inner: result }) + } else { + let mut result = self.inner.clone(); + for (key, other_value) in other.inner.iter().enumerate() { + if let Some(other_value) = other_value { + match &self.inner[key] { + Some(self_value) => { + if self_value != other_value { + return None; + } + } + None => result[key] = Some(other_value.clone()), + } + } + } + Some(Self { inner: result }) + } + } +} + +impl IntoIterator for EncodedTuple { + type Item = Option; + type IntoIter = std::vec::IntoIter>; + + fn into_iter(self) -> Self::IntoIter { + self.inner.into_iter() + } +} + type EncodedTuplesIterator = Box>>; type CustomFunctionRegistry = HashMap Option>>; @@ -67,28 +153,27 @@ impl SimpleEvaluator { } #[allow(clippy::rc_buffer)] - pub fn evaluate_select_plan( - &self, - plan: &PlanNode, - variables: Rc>, - ) -> (QueryResults, Rc) { - let (eval, stats) = self.plan_evaluator(plan); + pub fn evaluate_select(&self, pattern: &GraphPattern) -> (QueryResults, Rc) { + let mut variables = Vec::new(); + let (eval, stats) = self.graph_pattern_evaluator(pattern, &mut variables); + let from = EncodedTuple::with_capacity(variables.len()); ( QueryResults::Solutions(decode_bindings( Rc::clone(&self.dataset), - eval(EncodedTuple::with_capacity(variables.len())), - variables, + eval(from), + Rc::from(variables), )), stats, ) } - pub fn evaluate_ask_plan( + pub fn evaluate_ask( &self, - plan: &PlanNode, + pattern: &GraphPattern, ) -> (Result, Rc) { - let from = EncodedTuple::with_capacity(plan.used_variables().len()); - let (eval, stats) = self.plan_evaluator(plan); + let mut variables = Vec::new(); + let (eval, stats) = self.graph_pattern_evaluator(pattern, &mut variables); + let from = EncodedTuple::with_capacity(variables.len()); ( match eval(from).next() { Some(Ok(_)) => Ok(QueryResults::Boolean(true)), @@ -99,13 +184,32 @@ impl SimpleEvaluator { ) } - pub fn evaluate_construct_plan( + pub fn evaluate_construct( &self, - plan: &PlanNode, - template: Vec, + pattern: &GraphPattern, + template: &[TriplePattern], ) -> (QueryResults, Rc) { - let from = EncodedTuple::with_capacity(plan.used_variables().len()); - let (eval, stats) = self.plan_evaluator(plan); + let mut variables = Vec::new(); + let (eval, stats) = self.graph_pattern_evaluator(pattern, &mut variables); + let mut bnodes = Vec::new(); + let template = template + .iter() + .map(|t| TripleTemplate { + subject: self.template_value_from_term_or_variable( + &t.subject, + &mut variables, + &mut bnodes, + ), + predicate: self + .template_value_from_named_node_or_variable(&t.predicate, &mut variables), + object: self.template_value_from_term_or_variable( + &t.object, + &mut variables, + &mut bnodes, + ), + }) + .collect(); + let from = EncodedTuple::with_capacity(variables.len()); ( QueryResults::Graph(QueryTripleIter { iter: Box::new(ConstructIterator { @@ -120,9 +224,13 @@ impl SimpleEvaluator { ) } - pub fn evaluate_describe_plan(&self, plan: &PlanNode) -> (QueryResults, Rc) { - let from = EncodedTuple::with_capacity(plan.used_variables().len()); - let (eval, stats) = self.plan_evaluator(plan); + pub fn evaluate_describe( + &self, + pattern: &GraphPattern, + ) -> (QueryResults, Rc) { + let mut variables = Vec::new(); + let (eval, stats) = self.graph_pattern_evaluator(pattern, &mut variables); + let from = EncodedTuple::with_capacity(variables.len()); ( QueryResults::Graph(QueryTripleIter { iter: Box::new(DescribeIterator { @@ -135,20 +243,77 @@ impl SimpleEvaluator { ) } - pub fn plan_evaluator( + pub fn graph_pattern_evaluator( &self, - node: &PlanNode, + pattern: &GraphPattern, + encoded_variables: &mut Vec, ) -> ( Rc EncodedTuplesIterator>, Rc, ) { let mut stat_children = Vec::new(); - let mut evaluator: Rc EncodedTuplesIterator> = match node { - PlanNode::StaticBindings { encoded_tuples, .. } => { - let tuples = encoded_tuples.clone(); + let mut evaluator = + self.build_graph_pattern_evaluator(pattern, encoded_variables, &mut stat_children); + let stats = Rc::new(EvalNodeWithStats { + label: eval_node_label(pattern), + children: stat_children, + exec_count: Cell::new(0), + exec_duration: Cell::new(StdDuration::from_secs(0)), + }); + if self.run_stats { + let stats = Rc::clone(&stats); + evaluator = Rc::new(move |tuple| { + let start = Timer::now(); + let inner = evaluator(tuple); + stats + .exec_duration + .set(stats.exec_duration.get() + start.elapsed()); + Box::new(StatsIterator { + inner, + stats: Rc::clone(&stats), + }) + }) + } + (evaluator, stats) + } + + fn build_graph_pattern_evaluator( + &self, + pattern: &GraphPattern, + encoded_variables: &mut Vec, + stat_children: &mut Vec>, + ) -> Rc EncodedTuplesIterator> { + match pattern { + GraphPattern::Values { + variables, + bindings, + } => { + let encoding = variables + .iter() + .map(|v| encode_variable(encoded_variables, v)) + .collect::>(); + let encoded_tuples = bindings + .iter() + .map(|row| { + let mut result = EncodedTuple::with_capacity(variables.len()); + for (key, value) in row.iter().enumerate() { + if let Some(term) = value { + result.set( + encoding[key], + match term { + GroundTerm::NamedNode(node) => self.encode_term(node), + GroundTerm::Literal(literal) => self.encode_term(literal), + GroundTerm::Triple(triple) => self.encode_triple(triple), + }, + ); + } + } + result + }) + .collect::>(); Rc::new(move |from| { Box::new( - tuples + encoded_tuples .iter() .filter_map(move |t| Some(Ok(t.combine_with(&from)?))) .collect::>() @@ -156,17 +321,18 @@ impl SimpleEvaluator { ) }) } - PlanNode::Service { - variables, + GraphPattern::Service { + name, + inner, silent, - service_name, - graph_pattern, - .. } => { - let variables = Rc::clone(variables); let silent = *silent; - let service_name = service_name.clone(); - let graph_pattern = Rc::clone(graph_pattern); + let service_name = + TupleSelector::from_named_node_pattern(name, encoded_variables, &self.dataset); + let _ = + self.build_graph_pattern_evaluator(inner, encoded_variables, &mut Vec::new()); // We call recursively to fill "encoded_variables" + let graph_pattern = spargebra::algebra::GraphPattern::from(inner.as_ref()); + let variables = Rc::from(encoded_variables.as_slice()); let eval = self.clone(); Rc::new(move |from| { match eval.evaluate_service( @@ -190,23 +356,39 @@ impl SimpleEvaluator { } }) } - PlanNode::QuadPattern { + GraphPattern::QuadPattern { subject, predicate, object, graph_name, } => { - let subject = TupleSelector::from(subject); - let predicate = TupleSelector::from(predicate); - let object = TupleSelector::from(object); - let graph_name = TupleSelector::from(graph_name); + let subject = TupleSelector::from_ground_term_pattern( + subject, + encoded_variables, + &self.dataset, + ); + let predicate = TupleSelector::from_named_node_pattern( + predicate, + encoded_variables, + &self.dataset, + ); + let object = TupleSelector::from_ground_term_pattern( + object, + encoded_variables, + &self.dataset, + ); + let graph_name = TupleSelector::from_graph_name_pattern( + graph_name, + encoded_variables, + &self.dataset, + ); let dataset = Rc::clone(&self.dataset); Rc::new(move |from| { let iter = dataset.encoded_quads_for_pattern( - get_pattern_value(&subject, &from).as_ref(), - get_pattern_value(&predicate, &from).as_ref(), - get_pattern_value(&object, &from).as_ref(), - get_pattern_value(&graph_name, &from).as_ref(), + subject.get_pattern_value(&from).as_ref(), + predicate.get_pattern_value(&from).as_ref(), + object.get_pattern_value(&from).as_ref(), + graph_name.get_pattern_value(&from).as_ref(), ); let subject = subject.clone(); let predicate = predicate.clone(); @@ -225,21 +407,34 @@ impl SimpleEvaluator { })) }) } - PlanNode::PathPattern { + GraphPattern::Path { subject, path, object, graph_name, } => { - let subject = TupleSelector::from(subject); - let path = Rc::clone(path); - let object = TupleSelector::from(object); - let graph_name = TupleSelector::from(graph_name); + let subject = TupleSelector::from_ground_term_pattern( + subject, + encoded_variables, + &self.dataset, + ); + let path = self.encode_property_path(path); + + let object = TupleSelector::from_ground_term_pattern( + object, + encoded_variables, + &self.dataset, + ); + let graph_name = TupleSelector::from_graph_name_pattern( + graph_name, + encoded_variables, + &self.dataset, + ); let dataset = Rc::clone(&self.dataset); Rc::new(move |from| { - let input_subject = get_pattern_value(&subject, &from); - let input_object = get_pattern_value(&object, &from); - let input_graph_name = get_pattern_value(&graph_name, &from); + let input_subject = subject.get_pattern_value(&from); + let input_object = object.get_pattern_value(&from); + let input_graph_name = graph_name.get_pattern_value(&from); let path_eval = PathEvaluator { dataset: Rc::clone(&dataset), }; @@ -377,61 +572,95 @@ impl SimpleEvaluator { } }) } - PlanNode::HashJoin { - probe_child, - build_child, - keys, + GraphPattern::Join { + left, + right, + algorithm, } => { - let join_keys = keys.iter().map(|v| v.encoded).collect::>(); - let (probe, probe_stats) = self.plan_evaluator(probe_child); - stat_children.push(probe_stats); - let (build, build_stats) = self.plan_evaluator(build_child); - stat_children.push(build_stats); - if join_keys.is_empty() { - // Cartesian product - Rc::new(move |from| { - let mut errors = Vec::default(); - let build_values = build(from.clone()) - .filter_map(|result| match result { - Ok(result) => Some(result), - Err(error) => { - errors.push(Err(error)); - None - } + let (left, left_stats) = self.graph_pattern_evaluator(left, encoded_variables); + stat_children.push(left_stats); + let (right, right_stats) = self.graph_pattern_evaluator(right, encoded_variables); + stat_children.push(right_stats); + + match algorithm { + JoinAlgorithm::HashBuildLeftProbeRight { keys } => { + let build = left; + let probe = right; + if keys.is_empty() { + // Cartesian product + Rc::new(move |from| { + let mut errors = Vec::default(); + let build_values = build(from.clone()) + .filter_map(|result| match result { + Ok(result) => Some(result), + Err(error) => { + errors.push(Err(error)); + None + } + }) + .collect::>(); + Box::new(CartesianProductJoinIterator { + probe_iter: probe(from), + built: build_values, + buffered_results: errors, + }) }) - .collect::>(); - Box::new(CartesianProductJoinIterator { - probe_iter: probe(from), - built: build_values, - buffered_results: errors, - }) - }) - } else { - // Real hash join - Rc::new(move |from| { - let mut errors = Vec::default(); - let mut built_values = EncodedTupleSet::new(join_keys.clone()); - built_values.extend(build(from.clone()).filter_map( - |result| match result { - Ok(result) => Some(result), - Err(error) => { - errors.push(Err(error)); - None - } - }, - )); - Box::new(HashJoinIterator { - probe_iter: probe(from), - built: built_values, - buffered_results: errors, - }) - }) + } else { + // Real hash join + let keys = keys + .iter() + .map(|v| encode_variable(encoded_variables, v)) + .collect::>(); + Rc::new(move |from| { + let mut errors = Vec::default(); + let mut built_values = EncodedTupleSet::new(keys.clone()); + built_values.extend(build(from.clone()).filter_map(|result| { + match result { + Ok(result) => Some(result), + Err(error) => { + errors.push(Err(error)); + None + } + } + })); + Box::new(HashJoinIterator { + probe_iter: probe(from), + built: built_values, + buffered_results: errors, + }) + }) + } + } } } - PlanNode::ForLoopJoin { left, right } => { - let (left, left_stats) = self.plan_evaluator(left); + GraphPattern::Lateral { left, right } => { + let (left, left_stats) = self.graph_pattern_evaluator(left, encoded_variables); stat_children.push(left_stats); - let (right, right_stats) = self.plan_evaluator(right); + + if let GraphPattern::LeftJoin { + left: nested_left, + right: nested_right, + expression, + .. + } = right.as_ref() + { + if nested_left.is_empty_singleton() { + // We are in a ForLoopLeftJoin + let right = + GraphPattern::filter(nested_right.as_ref().clone(), expression.clone()); + let (right, right_stats) = + self.graph_pattern_evaluator(&right, encoded_variables); + stat_children.push(right_stats); + return Rc::new(move |from| { + Box::new(ForLoopLeftJoinIterator { + right_evaluator: Rc::clone(&right), + left_iter: left(from), + current_right: Box::new(empty()), + }) + }); + } + } + let (right, right_stats) = self.graph_pattern_evaluator(right, encoded_variables); stat_children.push(right_stats); Rc::new(move |from| { let right = Rc::clone(&right); @@ -441,89 +670,108 @@ impl SimpleEvaluator { })) }) } - PlanNode::AntiJoin { left, right, keys } => { - let join_keys = keys.iter().map(|v| v.encoded).collect::>(); - let (left, left_stats) = self.plan_evaluator(left); + GraphPattern::Minus { + left, + right, + algorithm, + } => { + let (left, left_stats) = self.graph_pattern_evaluator(left, encoded_variables); stat_children.push(left_stats); - let (right, right_stats) = self.plan_evaluator(right); + let (right, right_stats) = self.graph_pattern_evaluator(right, encoded_variables); stat_children.push(right_stats); - if join_keys.is_empty() { - Rc::new(move |from| { - let right: Vec<_> = right(from.clone()).filter_map(Result::ok).collect(); - Box::new(left(from).filter(move |left_tuple| { - if let Ok(left_tuple) = left_tuple { - !right.iter().any(|right_tuple| { - are_compatible_and_not_disjointed(left_tuple, right_tuple) - }) - } else { - true - } - })) - }) - } else { - Rc::new(move |from| { - let mut right_values = EncodedTupleSet::new(join_keys.clone()); - right_values.extend(right(from.clone()).filter_map(Result::ok)); - Box::new(left(from).filter(move |left_tuple| { - if let Ok(left_tuple) = left_tuple { - !right_values.get(left_tuple).iter().any(|right_tuple| { - are_compatible_and_not_disjointed(left_tuple, right_tuple) - }) - } else { - true - } - })) - }) + + match algorithm { + MinusAlgorithm::HashBuildRightProbeLeft { keys } => { + if keys.is_empty() { + Rc::new(move |from| { + let right: Vec<_> = + right(from.clone()).filter_map(Result::ok).collect(); + Box::new(left(from).filter(move |left_tuple| { + if let Ok(left_tuple) = left_tuple { + !right.iter().any(|right_tuple| { + are_compatible_and_not_disjointed( + left_tuple, + right_tuple, + ) + }) + } else { + true + } + })) + }) + } else { + let keys = keys + .iter() + .map(|v| encode_variable(encoded_variables, v)) + .collect::>(); + Rc::new(move |from| { + let mut right_values = EncodedTupleSet::new(keys.clone()); + right_values.extend(right(from.clone()).filter_map(Result::ok)); + Box::new(left(from).filter(move |left_tuple| { + if let Ok(left_tuple) = left_tuple { + !right_values.get(left_tuple).iter().any(|right_tuple| { + are_compatible_and_not_disjointed( + left_tuple, + right_tuple, + ) + }) + } else { + true + } + })) + }) + } + } } } - PlanNode::HashLeftJoin { + GraphPattern::LeftJoin { left, right, expression, - keys, + algorithm, } => { - let join_keys = keys.iter().map(|v| v.encoded).collect::>(); - let (left, left_stats) = self.plan_evaluator(left); - stat_children.push(left_stats); - let (right, right_stats) = self.plan_evaluator(right); - stat_children.push(right_stats); - let expression = self.expression_evaluator(expression, &mut stat_children); - // Real hash join - Rc::new(move |from| { - let mut errors = Vec::default(); - let mut right_values = EncodedTupleSet::new(join_keys.clone()); - right_values.extend(right(from.clone()).filter_map(|result| match result { - Ok(result) => Some(result), - Err(error) => { - errors.push(Err(error)); - None - } - })); - Box::new(HashLeftJoinIterator { - left_iter: left(from), - right: right_values, - buffered_results: errors, - expression: Rc::clone(&expression), - }) - }) - } - PlanNode::ForLoopLeftJoin { left, right } => { - let (left, left_stats) = self.plan_evaluator(left); + let (left, left_stats) = self.graph_pattern_evaluator(left, encoded_variables); stat_children.push(left_stats); - let (right, right_stats) = self.plan_evaluator(right); + let (right, right_stats) = self.graph_pattern_evaluator(right, encoded_variables); stat_children.push(right_stats); - Rc::new(move |from| { - Box::new(ForLoopLeftJoinIterator { - right_evaluator: Rc::clone(&right), - left_iter: left(from), - current_right: Box::new(empty()), - }) - }) + let expression = + self.expression_evaluator(expression, encoded_variables, stat_children); + + match algorithm { + LeftJoinAlgorithm::HashBuildRightProbeLeft { keys } => { + // Real hash join + let keys = keys + .iter() + .map(|v| encode_variable(encoded_variables, v)) + .collect::>(); + Rc::new(move |from| { + let mut errors = Vec::default(); + let mut right_values = EncodedTupleSet::new(keys.clone()); + right_values.extend(right(from.clone()).filter_map( + |result| match result { + Ok(result) => Some(result), + Err(error) => { + errors.push(Err(error)); + None + } + }, + )); + Box::new(HashLeftJoinIterator { + left_iter: left(from), + right: right_values, + buffered_results: errors, + expression: Rc::clone(&expression), + }) + }) + } + } } - PlanNode::Filter { child, expression } => { - let (child, child_stats) = self.plan_evaluator(child); + GraphPattern::Filter { inner, expression } => { + let (child, child_stats) = self.graph_pattern_evaluator(inner, encoded_variables); stat_children.push(child_stats); - let expression = self.expression_evaluator(expression, &mut stat_children); + let expression = + self.expression_evaluator(expression, encoded_variables, stat_children); + Rc::new(move |from| { let expression = Rc::clone(&expression); Box::new(child(from).filter(move |tuple| { @@ -536,15 +784,17 @@ impl SimpleEvaluator { })) }) } - PlanNode::Union { children } => { - let children: Vec<_> = children + GraphPattern::Union { inner } => { + let children = inner .iter() .map(|child| { - let (child, child_stats) = self.plan_evaluator(child); + let (child, child_stats) = + self.graph_pattern_evaluator(child, encoded_variables); stat_children.push(child_stats); child }) - .collect(); + .collect::>(); + Rc::new(move |from| { Box::new(UnionIterator { plans: children.clone(), @@ -554,15 +804,17 @@ impl SimpleEvaluator { }) }) } - PlanNode::Extend { - child, + GraphPattern::Extend { + inner, variable, expression, } => { - let (child, child_stats) = self.plan_evaluator(child); + let (child, child_stats) = self.graph_pattern_evaluator(inner, encoded_variables); stat_children.push(child_stats); - let position = variable.encoded; - let expression = self.expression_evaluator(expression, &mut stat_children); + + let position = encode_variable(encoded_variables, variable); + let expression = + self.expression_evaluator(expression, encoded_variables, stat_children); Rc::new(move |from| { let expression = Rc::clone(&expression); Box::new(child(from).map(move |tuple| { @@ -574,20 +826,20 @@ impl SimpleEvaluator { })) }) } - PlanNode::Sort { child, by } => { - let (child, child_stats) = self.plan_evaluator(child); + GraphPattern::OrderBy { inner, expression } => { + let (child, child_stats) = self.graph_pattern_evaluator(inner, encoded_variables); stat_children.push(child_stats); - let by: Vec<_> = by + let by = expression .iter() .map(|comp| match comp { - Comparator::Asc(expression) => ComparatorFunction::Asc( - self.expression_evaluator(expression, &mut stat_children), + OrderExpression::Asc(expression) => ComparatorFunction::Asc( + self.expression_evaluator(expression, encoded_variables, stat_children), ), - Comparator::Desc(expression) => ComparatorFunction::Desc( - self.expression_evaluator(expression, &mut stat_children), + OrderExpression::Desc(expression) => ComparatorFunction::Desc( + self.expression_evaluator(expression, encoded_variables, stat_children), ), }) - .collect(); + .collect::>(); let dataset = Rc::clone(&self.dataset); Rc::new(move |from| { let mut errors = Vec::default(); @@ -632,13 +884,13 @@ impl SimpleEvaluator { Box::new(errors.into_iter().chain(values.into_iter().map(Ok))) }) } - PlanNode::HashDeduplicate { child } => { - let (child, child_stats) = self.plan_evaluator(child); + GraphPattern::Distinct { inner } => { + let (child, child_stats) = self.graph_pattern_evaluator(inner, encoded_variables); stat_children.push(child_stats); Rc::new(move |from| Box::new(hash_deduplicate(child(from)))) } - PlanNode::Reduced { child } => { - let (child, child_stats) = self.plan_evaluator(child); + GraphPattern::Reduced { inner } => { + let (child, child_stats) = self.graph_pattern_evaluator(inner, encoded_variables); stat_children.push(child_stats); Rc::new(move |from| { Box::new(ConsecutiveDeduplication { @@ -647,44 +899,56 @@ impl SimpleEvaluator { }) }) } - PlanNode::Skip { child, count } => { - let (child, child_stats) = self.plan_evaluator(child); - stat_children.push(child_stats); - let count = *count; - Rc::new(move |from| Box::new(child(from).skip(count))) - } - PlanNode::Limit { child, count } => { - let (child, child_stats) = self.plan_evaluator(child); + GraphPattern::Slice { + inner, + start, + length, + } => { + let (mut child, child_stats) = + self.graph_pattern_evaluator(inner, encoded_variables); stat_children.push(child_stats); - let count = *count; - Rc::new(move |from| Box::new(child(from).take(count))) + let start = *start; + if start > 0 { + child = Rc::new(move |from| Box::new(child(from).skip(start))); + } + if let Some(length) = *length { + child = Rc::new(move |from| Box::new(child(from).take(length))); + } + child } - PlanNode::Project { child, mapping } => { - let (child, child_stats) = self.plan_evaluator(child); + GraphPattern::Project { inner, variables } => { + let mut inner_encoded_variables = variables.clone(); + let (child, child_stats) = + self.graph_pattern_evaluator(inner, &mut inner_encoded_variables); stat_children.push(child_stats); - let mapping = Rc::clone(mapping); + let mapping = variables + .iter() + .enumerate() + .map(|(new_variable, variable)| { + (new_variable, encode_variable(encoded_variables, variable)) + }) + .collect::>(); Rc::new(move |from| { let mapping = Rc::clone(&mapping); let mut input_tuple = EncodedTuple::with_capacity(mapping.len()); - for (input_key, output_key) in mapping.iter() { - if let Some(value) = from.get(output_key.encoded) { - input_tuple.set(input_key.encoded, value.clone()); + for (input_key, output_key) in &*mapping { + if let Some(value) = from.get(*output_key) { + input_tuple.set(*input_key, value.clone()); } } Box::new(child(input_tuple).filter_map(move |tuple| { match tuple { Ok(tuple) => { let mut output_tuple = from.clone(); - for (input_key, output_key) in mapping.iter() { - if let Some(value) = tuple.get(input_key.encoded) { - if let Some(existing_value) = - output_tuple.get(output_key.encoded) + for (input_key, output_key) in &*mapping { + if let Some(value) = tuple.get(*input_key) { + if let Some(existing_value) = output_tuple.get(*output_key) { if existing_value != value { return None; // Conflict } } else { - output_tuple.set(output_key.encoded, value.clone()); + output_tuple.set(*output_key, value.clone()); } } } @@ -695,35 +959,42 @@ impl SimpleEvaluator { })) }) } - PlanNode::Aggregate { - child, - key_variables, + GraphPattern::Group { + inner, aggregates, + variables, } => { - let (child, child_stats) = self.plan_evaluator(child); + let (child, child_stats) = self.graph_pattern_evaluator(inner, encoded_variables); stat_children.push(child_stats); - let key_variables = Rc::clone(key_variables); - let aggregate_input_expressions: Vec<_> = aggregates + let key_variables = variables .iter() - .map(|(aggregate, _)| { - aggregate - .parameter - .as_ref() - .map(|p| self.expression_evaluator(p, &mut stat_children)) - }) - .collect(); - let accumulator_builders: Vec<_> = aggregates + .map(|k| encode_variable(encoded_variables, k)) + .collect::>(); + let aggregate_input_expressions = aggregates .iter() - .map(|(aggregate, _)| { - Self::accumulator_builder( - &self.dataset, - &aggregate.function, - aggregate.distinct, - ) + .map(|(_, expression)| match expression { + AggregateExpression::Count { expr, .. } => expr.as_ref().map(|e| { + self.expression_evaluator(e, encoded_variables, stat_children) + }), + AggregateExpression::Sum { expr, .. } + | AggregateExpression::Avg { expr, .. } + | AggregateExpression::Min { expr, .. } + | AggregateExpression::Max { expr, .. } + | AggregateExpression::GroupConcat { expr, .. } + | AggregateExpression::Sample { expr, .. } + | AggregateExpression::Custom { expr, .. } => { + Some(self.expression_evaluator(expr, encoded_variables, stat_children)) + } }) - .collect(); - let accumulator_variables: Vec<_> = - aggregates.iter().map(|(_, var)| var.encoded).collect(); + .collect::>(); + let accumulator_builders = aggregates + .iter() + .map(|(_, aggregate)| Self::accumulator_builder(&self.dataset, aggregate)) + .collect::>(); + let accumulator_variables = aggregates + .iter() + .map(|(variable, _)| encode_variable(encoded_variables, variable)) + .collect::>(); Rc::new(move |from| { let tuple_size = from.capacity(); let key_variables = Rc::clone(&key_variables); @@ -749,7 +1020,7 @@ impl SimpleEvaluator { //TODO avoid copy for key? let key = key_variables .iter() - .map(|v| tuple.get(v.encoded).cloned()) + .map(|v| tuple.get(*v).cloned()) .collect(); let key_accumulators = @@ -777,7 +1048,7 @@ impl SimpleEvaluator { let mut result = EncodedTuple::with_capacity(tuple_size); for (variable, value) in key_variables.iter().zip(key) { if let Some(value) = value { - result.set(variable.encoded, value); + result.set(*variable, value); } } for (accumulator, variable) in @@ -793,38 +1064,18 @@ impl SimpleEvaluator { ) }) } - }; - let stats = Rc::new(EvalNodeWithStats { - label: eval_node_label(node), - children: stat_children, - exec_count: Cell::new(0), - exec_duration: Cell::new(StdDuration::from_secs(0)), - }); - if self.run_stats { - let stats = Rc::clone(&stats); - evaluator = Rc::new(move |tuple| { - let start = Timer::now(); - let inner = evaluator(tuple); - stats - .exec_duration - .set(stats.exec_duration.get() + start.elapsed()); - Box::new(StatsIterator { - inner, - stats: Rc::clone(&stats), - }) - }) } - (evaluator, stats) } fn evaluate_service( &self, - service_name: &PatternValue, - graph_pattern: &GraphPattern, + service_name: &TupleSelector, + graph_pattern: &spargebra::algebra::GraphPattern, variables: Rc<[Variable]>, from: &EncodedTuple, ) -> Result { - let service_name = get_pattern_value(&service_name.into(), from) + let service_name = service_name + .get_pattern_value(from) .ok_or_else(|| EvaluationError::msg("The SERVICE name is not bound"))?; if let QueryResults::Solutions(iter) = self.service_handler.handle( self.dataset.decode_named_node(&service_name)?, @@ -849,44 +1100,47 @@ impl SimpleEvaluator { #[allow(clippy::redundant_closure)] // False positive in 1.60 fn accumulator_builder( dataset: &Rc, - function: &PlanAggregationFunction, - distinct: bool, + expression: &AggregateExpression, ) -> Box Box> { - match function { - PlanAggregationFunction::Count => { - if distinct { + match expression { + AggregateExpression::Count { distinct, .. } => { + if *distinct { Box::new(|| Box::new(DistinctAccumulator::new(CountAccumulator::default()))) } else { Box::new(|| Box::::default()) } } - PlanAggregationFunction::Sum => { - if distinct { + AggregateExpression::Sum { distinct, .. } => { + if *distinct { Box::new(|| Box::new(DistinctAccumulator::new(SumAccumulator::default()))) } else { Box::new(|| Box::::default()) } } - PlanAggregationFunction::Min => { + AggregateExpression::Min { .. } => { let dataset = Rc::clone(dataset); Box::new(move || Box::new(MinAccumulator::new(Rc::clone(&dataset)))) } // DISTINCT does not make sense with min - PlanAggregationFunction::Max => { + AggregateExpression::Max { .. } => { let dataset = Rc::clone(dataset); Box::new(move || Box::new(MaxAccumulator::new(Rc::clone(&dataset)))) } // DISTINCT does not make sense with max - PlanAggregationFunction::Avg => { - if distinct { + AggregateExpression::Avg { distinct, .. } => { + if *distinct { Box::new(|| Box::new(DistinctAccumulator::new(AvgAccumulator::default()))) } else { Box::new(|| Box::::default()) } } - PlanAggregationFunction::Sample => Box::new(|| Box::::default()), // DISTINCT does not make sense with sample - PlanAggregationFunction::GroupConcat { separator } => { + AggregateExpression::Sample { .. } => Box::new(|| Box::::default()), // DISTINCT does not make sense with sample + AggregateExpression::GroupConcat { + distinct, + separator, + .. + } => { let dataset = Rc::clone(dataset); - let separator = Rc::clone(separator); - if distinct { + let separator = Rc::from(separator.as_deref().unwrap_or(" ")); + if *distinct { Box::new(move || { Box::new(DistinctAccumulator::new(GroupConcatAccumulator::new( Rc::clone(&dataset), @@ -902,40 +1156,46 @@ impl SimpleEvaluator { }) } } + AggregateExpression::Custom { .. } => Box::new(|| Box::new(FailingAccumulator)), } } fn expression_evaluator( &self, - expression: &PlanExpression, + expression: &Expression, + encoded_variables: &mut Vec, stat_children: &mut Vec>, ) -> Rc Option> { match expression { - PlanExpression::NamedNode(t) => { - let t = t.encoded.clone(); + Expression::NamedNode(t) => { + let t = self.encode_term(t); Rc::new(move |_| Some(t.clone())) } - PlanExpression::Literal(t) => { - let t = t.encoded.clone(); + Expression::Literal(t) => { + let t = self.encode_term(t); Rc::new(move |_| Some(t.clone())) } - PlanExpression::Variable(v) => { - let v = v.encoded; + Expression::Variable(v) => { + let v = encode_variable(encoded_variables, v); Rc::new(move |tuple| tuple.get(v).cloned()) } - PlanExpression::Exists(plan) => { - let (eval, stats) = self.plan_evaluator(plan); + Expression::Bound(v) => { + let v = encode_variable(encoded_variables, v); + Rc::new(move |tuple| Some(tuple.contains(v).into())) + } + Expression::Exists(plan) => { + let (eval, stats) = self.graph_pattern_evaluator(plan, encoded_variables); stat_children.push(stats); Rc::new(move |tuple| Some(eval(tuple.clone()).next().is_some().into())) } - PlanExpression::Or(inner) => { + Expression::Or(inner) => { let children = inner .iter() - .map(|i| self.expression_evaluator(i, stat_children)) + .map(|i| self.expression_evaluator(i, encoded_variables, stat_children)) .collect::>(); Rc::new(move |tuple| { let mut error = false; - for child in children.iter() { + for child in &*children { match child(tuple).and_then(|v| to_bool(&v)) { Some(true) => return Some(true.into()), Some(false) => continue, @@ -949,14 +1209,14 @@ impl SimpleEvaluator { } }) } - PlanExpression::And(inner) => { + Expression::And(inner) => { let children = inner .iter() - .map(|i| self.expression_evaluator(i, stat_children)) + .map(|i| self.expression_evaluator(i, encoded_variables, stat_children)) .collect::>(); Rc::new(move |tuple| { let mut error = false; - for child in children.iter() { + for child in &*children { match child(tuple).and_then(|v| to_bool(&v)) { Some(true) => continue, Some(false) => return Some(false.into()), @@ -970,14 +1230,19 @@ impl SimpleEvaluator { } }) } - PlanExpression::Equal(a, b) => { - let a = self.expression_evaluator(a, stat_children); - let b = self.expression_evaluator(b, stat_children); + Expression::Equal(a, b) => { + let a = self.expression_evaluator(a, encoded_variables, stat_children); + let b = self.expression_evaluator(b, encoded_variables, stat_children); Rc::new(move |tuple| equals(&a(tuple)?, &b(tuple)?).map(Into::into)) } - PlanExpression::Greater(a, b) => { - let a = self.expression_evaluator(a, stat_children); - let b = self.expression_evaluator(b, stat_children); + Expression::SameTerm(a, b) => { + let a = self.expression_evaluator(a, encoded_variables, stat_children); + let b = self.expression_evaluator(b, encoded_variables, stat_children); + Rc::new(move |tuple| Some((a(tuple)? == b(tuple)?).into())) + } + Expression::Greater(a, b) => { + let a = self.expression_evaluator(a, encoded_variables, stat_children); + let b = self.expression_evaluator(b, encoded_variables, stat_children); let dataset = Rc::clone(&self.dataset); Rc::new(move |tuple| { Some( @@ -986,9 +1251,9 @@ impl SimpleEvaluator { ) }) } - PlanExpression::GreaterOrEqual(a, b) => { - let a = self.expression_evaluator(a, stat_children); - let b = self.expression_evaluator(b, stat_children); + Expression::GreaterOrEqual(a, b) => { + let a = self.expression_evaluator(a, encoded_variables, stat_children); + let b = self.expression_evaluator(b, encoded_variables, stat_children); let dataset = Rc::clone(&self.dataset); Rc::new(move |tuple| { Some( @@ -1000,17 +1265,17 @@ impl SimpleEvaluator { ) }) } - PlanExpression::Less(a, b) => { - let a = self.expression_evaluator(a, stat_children); - let b = self.expression_evaluator(b, stat_children); + Expression::Less(a, b) => { + let a = self.expression_evaluator(a, encoded_variables, stat_children); + let b = self.expression_evaluator(b, encoded_variables, stat_children); let dataset = Rc::clone(&self.dataset); Rc::new(move |tuple| { Some((partial_cmp(&dataset, &a(tuple)?, &b(tuple)?)? == Ordering::Less).into()) }) } - PlanExpression::LessOrEqual(a, b) => { - let a = self.expression_evaluator(a, stat_children); - let b = self.expression_evaluator(b, stat_children); + Expression::LessOrEqual(a, b) => { + let a = self.expression_evaluator(a, encoded_variables, stat_children); + let b = self.expression_evaluator(b, encoded_variables, stat_children); let dataset = Rc::clone(&self.dataset); Rc::new(move |tuple| { Some( @@ -1022,9 +1287,9 @@ impl SimpleEvaluator { ) }) } - PlanExpression::Add(a, b) => { - let a = self.expression_evaluator(a, stat_children); - let b = self.expression_evaluator(b, stat_children); + Expression::Add(a, b) => { + let a = self.expression_evaluator(a, encoded_variables, stat_children); + let b = self.expression_evaluator(b, encoded_variables, stat_children); Rc::new( move |tuple| match NumericBinaryOperands::new(a(tuple)?, b(tuple)?)? { NumericBinaryOperands::Float(v1, v2) => Some((v1 + v2).into()), @@ -1068,9 +1333,9 @@ impl SimpleEvaluator { }, ) } - PlanExpression::Subtract(a, b) => { - let a = self.expression_evaluator(a, stat_children); - let b = self.expression_evaluator(b, stat_children); + Expression::Subtract(a, b) => { + let a = self.expression_evaluator(a, encoded_variables, stat_children); + let b = self.expression_evaluator(b, encoded_variables, stat_children); Rc::new(move |tuple| { Some(match NumericBinaryOperands::new(a(tuple)?, b(tuple)?)? { NumericBinaryOperands::Float(v1, v2) => (v1 - v2).into(), @@ -1114,9 +1379,9 @@ impl SimpleEvaluator { }) }) } - PlanExpression::Multiply(a, b) => { - let a = self.expression_evaluator(a, stat_children); - let b = self.expression_evaluator(b, stat_children); + Expression::Multiply(a, b) => { + let a = self.expression_evaluator(a, encoded_variables, stat_children); + let b = self.expression_evaluator(b, encoded_variables, stat_children); Rc::new( move |tuple| match NumericBinaryOperands::new(a(tuple)?, b(tuple)?)? { NumericBinaryOperands::Float(v1, v2) => Some((v1 * v2).into()), @@ -1127,9 +1392,9 @@ impl SimpleEvaluator { }, ) } - PlanExpression::Divide(a, b) => { - let a = self.expression_evaluator(a, stat_children); - let b = self.expression_evaluator(b, stat_children); + Expression::Divide(a, b) => { + let a = self.expression_evaluator(a, encoded_variables, stat_children); + let b = self.expression_evaluator(b, encoded_variables, stat_children); Rc::new( move |tuple| match NumericBinaryOperands::new(a(tuple)?, b(tuple)?)? { NumericBinaryOperands::Float(v1, v2) => Some((v1 / v2).into()), @@ -1142,8 +1407,8 @@ impl SimpleEvaluator { }, ) } - PlanExpression::UnaryPlus(e) => { - let e = self.expression_evaluator(e, stat_children); + Expression::UnaryPlus(e) => { + let e = self.expression_evaluator(e, encoded_variables, stat_children); Rc::new(move |tuple| match e(tuple)? { EncodedTerm::FloatLiteral(value) => Some(value.into()), EncodedTerm::DoubleLiteral(value) => Some(value.into()), @@ -1155,8 +1420,8 @@ impl SimpleEvaluator { _ => None, }) } - PlanExpression::UnaryMinus(e) => { - let e = self.expression_evaluator(e, stat_children); + Expression::UnaryMinus(e) => { + let e = self.expression_evaluator(e, encoded_variables, stat_children); Rc::new(move |tuple| match e(tuple)? { EncodedTerm::FloatLiteral(value) => Some((-value).into()), EncodedTerm::DoubleLiteral(value) => Some((-value).into()), @@ -1170,902 +1435,1333 @@ impl SimpleEvaluator { _ => None, }) } - PlanExpression::Not(e) => { - let e = self.expression_evaluator(e, stat_children); + Expression::Not(e) => { + let e = self.expression_evaluator(e, encoded_variables, stat_children); Rc::new(move |tuple| to_bool(&e(tuple)?).map(|v| (!v).into())) } - PlanExpression::Str(e) | PlanExpression::StringCast(e) => { - let e = self.expression_evaluator(e, stat_children); - let dataset = Rc::clone(&self.dataset); - Rc::new(move |tuple| { - Some(build_string_literal_from_id(to_string_id( - &dataset, - &e(tuple)?, - )?)) - }) - } - PlanExpression::Lang(e) => { - let e = self.expression_evaluator(e, stat_children); - let dataset = Rc::clone(&self.dataset); - Rc::new(move |tuple| match e(tuple)? { - EncodedTerm::SmallSmallLangStringLiteral { language, .. } - | EncodedTerm::BigSmallLangStringLiteral { language, .. } => { - Some(build_string_literal_from_id(language.into())) - } - EncodedTerm::SmallBigLangStringLiteral { language_id, .. } - | EncodedTerm::BigBigLangStringLiteral { language_id, .. } => { - Some(build_string_literal_from_id(language_id.into())) - } - e if e.is_literal() => Some(build_string_literal(&dataset, "")), - _ => None, - }) - } - PlanExpression::LangMatches(language_tag, language_range) => { - let language_tag = self.expression_evaluator(language_tag, stat_children); - let language_range = self.expression_evaluator(language_range, stat_children); - let dataset = Rc::clone(&self.dataset); + Expression::Coalesce(l) => { + let l: Vec<_> = l + .iter() + .map(|e| self.expression_evaluator(e, encoded_variables, stat_children)) + .collect(); Rc::new(move |tuple| { - let mut language_tag = to_simple_string(&dataset, &language_tag(tuple)?)?; - language_tag.make_ascii_lowercase(); - let mut language_range = to_simple_string(&dataset, &language_range(tuple)?)?; - language_range.make_ascii_lowercase(); - Some( - if &*language_range == "*" { - !language_tag.is_empty() - } else { - !ZipLongest::new(language_range.split('-'), language_tag.split('-')) - .any(|parts| match parts { - (Some(range_subtag), Some(language_subtag)) => { - range_subtag != language_subtag - } - (Some(_), None) => true, - (None, _) => false, - }) + for e in &l { + if let Some(result) = e(tuple) { + return Some(result); } - .into(), - ) + } + None }) } - PlanExpression::Datatype(e) => { - let e = self.expression_evaluator(e, stat_children); - let dataset = Rc::clone(&self.dataset); - Rc::new(move |tuple| datatype(&dataset, &e(tuple)?)) - } - PlanExpression::Bound(v) => { - let v = v.encoded; - Rc::new(move |tuple| Some(tuple.contains(v).into())) - } - PlanExpression::Iri(e) => { - let e = self.expression_evaluator(e, stat_children); - let dataset = Rc::clone(&self.dataset); - let base_iri = self.base_iri.clone(); + Expression::If(a, b, c) => { + let a = self.expression_evaluator(a, encoded_variables, stat_children); + let b = self.expression_evaluator(b, encoded_variables, stat_children); + let c = self.expression_evaluator(c, encoded_variables, stat_children); Rc::new(move |tuple| { - let e = e(tuple)?; - if e.is_named_node() { - Some(e) + if to_bool(&a(tuple)?)? { + b(tuple) } else { - let iri = to_simple_string(&dataset, &e)?; - Some(build_named_node( - &dataset, - &if let Some(base_iri) = &base_iri { - base_iri.resolve(&iri) - } else { - Iri::parse(iri) - } - .ok()? - .into_inner(), - )) + c(tuple) } }) } - PlanExpression::BNode(id) => match id { - Some(id) => { - let id = self.expression_evaluator(id, stat_children); - let dataset = Rc::clone(&self.dataset); - Rc::new(move |tuple| { - Some( - dataset.encode_term( - BlankNode::new(to_simple_string(&dataset, &id(tuple)?)?) - .ok()? - .as_ref(), - ), - ) - }) - } - None => Rc::new(|_| { - Some(EncodedTerm::NumericalBlankNode { - id: random::(), - }) - }), - }, - PlanExpression::Rand => Rc::new(|_| Some(random::().into())), - PlanExpression::Abs(e) => { - let e = self.expression_evaluator(e, stat_children); - Rc::new(move |tuple| match e(tuple)? { - EncodedTerm::IntegerLiteral(value) => Some(value.abs().into()), - EncodedTerm::DecimalLiteral(value) => Some(value.abs().into()), - EncodedTerm::FloatLiteral(value) => Some(value.abs().into()), - EncodedTerm::DoubleLiteral(value) => Some(value.abs().into()), - _ => None, - }) - } - PlanExpression::Ceil(e) => { - let e = self.expression_evaluator(e, stat_children); - Rc::new(move |tuple| match e(tuple)? { - EncodedTerm::IntegerLiteral(value) => Some(value.into()), - EncodedTerm::DecimalLiteral(value) => Some(value.ceil().into()), - EncodedTerm::FloatLiteral(value) => Some(value.ceil().into()), - EncodedTerm::DoubleLiteral(value) => Some(value.ceil().into()), - _ => None, - }) - } - PlanExpression::Floor(e) => { - let e = self.expression_evaluator(e, stat_children); - Rc::new(move |tuple| match e(tuple)? { - EncodedTerm::IntegerLiteral(value) => Some(value.into()), - EncodedTerm::DecimalLiteral(value) => Some(value.floor().into()), - EncodedTerm::FloatLiteral(value) => Some(value.floor().into()), - EncodedTerm::DoubleLiteral(value) => Some(value.floor().into()), - _ => None, - }) - } - PlanExpression::Round(e) => { - let e = self.expression_evaluator(e, stat_children); - Rc::new(move |tuple| match e(tuple)? { - EncodedTerm::IntegerLiteral(value) => Some(value.into()), - EncodedTerm::DecimalLiteral(value) => Some(value.round().into()), - EncodedTerm::FloatLiteral(value) => Some(value.round().into()), - EncodedTerm::DoubleLiteral(value) => Some(value.round().into()), - _ => None, - }) - } - PlanExpression::Concat(l) => { - let l: Vec<_> = l - .iter() - .map(|e| self.expression_evaluator(e, stat_children)) - .collect(); - let dataset = Rc::clone(&self.dataset); - Rc::new(move |tuple| { - let mut result = String::default(); - let mut language = None; - for e in &l { - let (value, e_language) = to_string_and_language(&dataset, &e(tuple)?)?; - if let Some(lang) = language { - if lang != e_language { - language = Some(None) + Expression::FunctionCall(function, parameters) => { + match function { + Function::Str => { + let e = self.expression_evaluator( + ¶meters[0], + encoded_variables, + stat_children, + ); + let dataset = Rc::clone(&self.dataset); + Rc::new(move |tuple| { + Some(build_string_literal_from_id(to_string_id( + &dataset, + &e(tuple)?, + )?)) + }) + } + Function::Lang => { + let e = self.expression_evaluator( + ¶meters[0], + encoded_variables, + stat_children, + ); + let dataset = Rc::clone(&self.dataset); + Rc::new(move |tuple| match e(tuple)? { + EncodedTerm::SmallSmallLangStringLiteral { language, .. } + | EncodedTerm::BigSmallLangStringLiteral { language, .. } => { + Some(build_string_literal_from_id(language.into())) } - } else { - language = Some(e_language) - } - result += &value + EncodedTerm::SmallBigLangStringLiteral { language_id, .. } + | EncodedTerm::BigBigLangStringLiteral { language_id, .. } => { + Some(build_string_literal_from_id(language_id.into())) + } + e if e.is_literal() => Some(build_string_literal(&dataset, "")), + _ => None, + }) } - Some(build_plain_literal( - &dataset, - &result, - language.and_then(|v| v), - )) - }) - } - PlanExpression::SubStr(source, starting_loc, length) => { - let source = self.expression_evaluator(source, stat_children); - let starting_loc = self.expression_evaluator(starting_loc, stat_children); - let length = length - .as_ref() - .map(|l| self.expression_evaluator(l, stat_children)); - let dataset = Rc::clone(&self.dataset); - Rc::new(move |tuple| { - let (source, language) = to_string_and_language(&dataset, &source(tuple)?)?; - - let starting_location: usize = - if let EncodedTerm::IntegerLiteral(v) = starting_loc(tuple)? { - i64::from(v).try_into().ok()? - } else { - return None; - }; - let length: Option = if let Some(length) = &length { - if let EncodedTerm::IntegerLiteral(v) = length(tuple)? { - Some(i64::from(v).try_into().ok()?) - } else { - return None; - } - } else { - None - }; - - // We want to slice on char indices, not byte indices - let mut start_iter = source - .char_indices() - .skip(starting_location.checked_sub(1)?) - .peekable(); - let result = if let Some((start_position, _)) = start_iter.peek().copied() { - if let Some(length) = length { - let mut end_iter = start_iter.skip(length).peekable(); - if let Some((end_position, _)) = end_iter.peek() { - &source[start_position..*end_position] + Function::LangMatches => { + let language_tag = self.expression_evaluator( + ¶meters[0], + encoded_variables, + stat_children, + ); + let language_range = self.expression_evaluator( + ¶meters[1], + encoded_variables, + stat_children, + ); + let dataset = Rc::clone(&self.dataset); + Rc::new(move |tuple| { + let mut language_tag = + to_simple_string(&dataset, &language_tag(tuple)?)?; + language_tag.make_ascii_lowercase(); + let mut language_range = + to_simple_string(&dataset, &language_range(tuple)?)?; + language_range.make_ascii_lowercase(); + Some( + if &*language_range == "*" { + !language_tag.is_empty() + } else { + !ZipLongest::new( + language_range.split('-'), + language_tag.split('-'), + ) + .any(|parts| match parts { + (Some(range_subtag), Some(language_subtag)) => { + range_subtag != language_subtag + } + (Some(_), None) => true, + (None, _) => false, + }) + } + .into(), + ) + }) + } + Function::Datatype => { + let e = self.expression_evaluator( + ¶meters[0], + encoded_variables, + stat_children, + ); + let dataset = Rc::clone(&self.dataset); + Rc::new(move |tuple| datatype(&dataset, &e(tuple)?)) + } + Function::Iri => { + let e = self.expression_evaluator( + ¶meters[0], + encoded_variables, + stat_children, + ); + let dataset = Rc::clone(&self.dataset); + let base_iri = self.base_iri.clone(); + Rc::new(move |tuple| { + let e = e(tuple)?; + if e.is_named_node() { + Some(e) } else { - &source[start_position..] + let iri = to_simple_string(&dataset, &e)?; + Some(build_named_node( + &dataset, + &if let Some(base_iri) = &base_iri { + base_iri.resolve(&iri) + } else { + Iri::parse(iri) + } + .ok()? + .into_inner(), + )) } - } else { - &source[start_position..] + }) + } + Function::BNode => match parameters.get(0) { + Some(id) => { + let id = + self.expression_evaluator(id, encoded_variables, stat_children); + let dataset = Rc::clone(&self.dataset); + Rc::new(move |tuple| { + Some( + dataset.encode_term( + BlankNode::new(to_simple_string(&dataset, &id(tuple)?)?) + .ok()? + .as_ref(), + ), + ) + }) } - } else { - "" - }; - Some(build_plain_literal(&dataset, result, language)) - }) - } - PlanExpression::StrLen(arg) => { - let arg = self.expression_evaluator(arg, stat_children); - let dataset = Rc::clone(&self.dataset); - Rc::new(move |tuple| { - Some( - i64::try_from(to_string(&dataset, &arg(tuple)?)?.chars().count()) - .ok()? - .into(), - ) - }) - } - PlanExpression::StaticReplace(arg, regex, replacement) => { - let arg = self.expression_evaluator(arg, stat_children); - let regex = regex.clone(); - let replacement = self.expression_evaluator(replacement, stat_children); - let dataset = Rc::clone(&self.dataset); - Rc::new(move |tuple| { - let (text, language) = to_string_and_language(&dataset, &arg(tuple)?)?; - let replacement = to_simple_string(&dataset, &replacement(tuple)?)?; - Some(build_plain_literal( - &dataset, - ®ex.replace_all(&text, replacement.as_str()), - language, - )) - }) - } - PlanExpression::DynamicReplace(arg, pattern, replacement, flags) => { - let arg = self.expression_evaluator(arg, stat_children); - let pattern = self.expression_evaluator(pattern, stat_children); - let replacement = self.expression_evaluator(replacement, stat_children); - let flags = flags - .as_ref() - .map(|flags| self.expression_evaluator(flags, stat_children)); - let dataset = Rc::clone(&self.dataset); - Rc::new(move |tuple| { - let pattern = to_simple_string(&dataset, &pattern(tuple)?)?; - let options = if let Some(flags) = &flags { - Some(to_simple_string(&dataset, &flags(tuple)?)?) - } else { - None - }; - let regex = compile_pattern(&pattern, options.as_deref())?; - let (text, language) = to_string_and_language(&dataset, &arg(tuple)?)?; - let replacement = to_simple_string(&dataset, &replacement(tuple)?)?; - Some(build_plain_literal( - &dataset, - ®ex.replace_all(&text, replacement.as_str()), - language, - )) - }) - } - PlanExpression::UCase(e) => { - let e = self.expression_evaluator(e, stat_children); - let dataset = Rc::clone(&self.dataset); - Rc::new(move |tuple| { - let (value, language) = to_string_and_language(&dataset, &e(tuple)?)?; - Some(build_plain_literal( - &dataset, - &value.to_uppercase(), - language, - )) - }) - } - PlanExpression::LCase(e) => { - let e = self.expression_evaluator(e, stat_children); - let dataset = Rc::clone(&self.dataset); - Rc::new(move |tuple| { - let (value, language) = to_string_and_language(&dataset, &e(tuple)?)?; - Some(build_plain_literal( - &dataset, - &value.to_lowercase(), - language, - )) - }) - } - PlanExpression::StrStarts(arg1, arg2) => { - let arg1 = self.expression_evaluator(arg1, stat_children); - let arg2 = self.expression_evaluator(arg2, stat_children); - let dataset = Rc::clone(&self.dataset); - Rc::new(move |tuple| { - let (arg1, arg2, _) = - to_argument_compatible_strings(&dataset, &arg1(tuple)?, &arg2(tuple)?)?; - Some(arg1.starts_with(arg2.as_str()).into()) - }) - } - PlanExpression::EncodeForUri(ltrl) => { - let ltrl = self.expression_evaluator(ltrl, stat_children); - let dataset = Rc::clone(&self.dataset); - Rc::new(move |tuple| { - let ltlr = to_string(&dataset, <rl(tuple)?)?; - let mut result = Vec::with_capacity(ltlr.len()); - for c in ltlr.bytes() { - match c { - b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => { - result.push(c) + None => Rc::new(|_| { + Some(EncodedTerm::NumericalBlankNode { + id: random::(), + }) + }), + }, + Function::Rand => Rc::new(|_| Some(random::().into())), + Function::Abs => { + let e = self.expression_evaluator( + ¶meters[0], + encoded_variables, + stat_children, + ); + Rc::new(move |tuple| match e(tuple)? { + EncodedTerm::IntegerLiteral(value) => Some(value.abs().into()), + EncodedTerm::DecimalLiteral(value) => Some(value.abs().into()), + EncodedTerm::FloatLiteral(value) => Some(value.abs().into()), + EncodedTerm::DoubleLiteral(value) => Some(value.abs().into()), + _ => None, + }) + } + Function::Ceil => { + let e = self.expression_evaluator( + ¶meters[0], + encoded_variables, + stat_children, + ); + Rc::new(move |tuple| match e(tuple)? { + EncodedTerm::IntegerLiteral(value) => Some(value.into()), + EncodedTerm::DecimalLiteral(value) => Some(value.ceil().into()), + EncodedTerm::FloatLiteral(value) => Some(value.ceil().into()), + EncodedTerm::DoubleLiteral(value) => Some(value.ceil().into()), + _ => None, + }) + } + Function::Floor => { + let e = self.expression_evaluator( + ¶meters[0], + encoded_variables, + stat_children, + ); + Rc::new(move |tuple| match e(tuple)? { + EncodedTerm::IntegerLiteral(value) => Some(value.into()), + EncodedTerm::DecimalLiteral(value) => Some(value.floor().into()), + EncodedTerm::FloatLiteral(value) => Some(value.floor().into()), + EncodedTerm::DoubleLiteral(value) => Some(value.floor().into()), + _ => None, + }) + } + Function::Round => { + let e = self.expression_evaluator( + ¶meters[0], + encoded_variables, + stat_children, + ); + Rc::new(move |tuple| match e(tuple)? { + EncodedTerm::IntegerLiteral(value) => Some(value.into()), + EncodedTerm::DecimalLiteral(value) => Some(value.round().into()), + EncodedTerm::FloatLiteral(value) => Some(value.round().into()), + EncodedTerm::DoubleLiteral(value) => Some(value.round().into()), + _ => None, + }) + } + Function::Concat => { + let l: Vec<_> = parameters + .iter() + .map(|e| self.expression_evaluator(e, encoded_variables, stat_children)) + .collect(); + let dataset = Rc::clone(&self.dataset); + Rc::new(move |tuple| { + let mut result = String::default(); + let mut language = None; + for e in &l { + let (value, e_language) = + to_string_and_language(&dataset, &e(tuple)?)?; + if let Some(lang) = language { + if lang != e_language { + language = Some(None) + } + } else { + language = Some(e_language) + } + result += &value } - _ => { - result.push(b'%'); - let high = c / 16; - let low = c % 16; - result.push(if high < 10 { - b'0' + high + Some(build_plain_literal( + &dataset, + &result, + language.and_then(|v| v), + )) + }) + } + Function::SubStr => { + let source = self.expression_evaluator( + ¶meters[0], + encoded_variables, + stat_children, + ); + let starting_loc = self.expression_evaluator( + ¶meters[1], + encoded_variables, + stat_children, + ); + let length = parameters.get(2).map(|l| { + self.expression_evaluator(l, encoded_variables, stat_children) + }); + let dataset = Rc::clone(&self.dataset); + Rc::new(move |tuple| { + let (source, language) = + to_string_and_language(&dataset, &source(tuple)?)?; + + let starting_location: usize = + if let EncodedTerm::IntegerLiteral(v) = starting_loc(tuple)? { + i64::from(v).try_into().ok()? } else { - b'A' + (high - 10) - }); - result.push(if low < 10 { - b'0' + low + return None; + }; + let length: Option = if let Some(length) = &length { + if let EncodedTerm::IntegerLiteral(v) = length(tuple)? { + Some(i64::from(v).try_into().ok()?) } else { - b'A' + (low - 10) - }); - } + return None; + } + } else { + None + }; + + // We want to slice on char indices, not byte indices + let mut start_iter = source + .char_indices() + .skip(starting_location.checked_sub(1)?) + .peekable(); + let result = + if let Some((start_position, _)) = start_iter.peek().copied() { + if let Some(length) = length { + let mut end_iter = start_iter.skip(length).peekable(); + if let Some((end_position, _)) = end_iter.peek() { + &source[start_position..*end_position] + } else { + &source[start_position..] + } + } else { + &source[start_position..] + } + } else { + "" + }; + Some(build_plain_literal(&dataset, result, language)) + }) + } + Function::StrLen => { + let arg = self.expression_evaluator( + ¶meters[0], + encoded_variables, + stat_children, + ); + let dataset = Rc::clone(&self.dataset); + Rc::new(move |tuple| { + Some( + i64::try_from(to_string(&dataset, &arg(tuple)?)?.chars().count()) + .ok()? + .into(), + ) + }) + } + Function::Replace => { + let arg = self.expression_evaluator( + ¶meters[0], + encoded_variables, + stat_children, + ); + let replacement = self.expression_evaluator( + ¶meters[2], + encoded_variables, + stat_children, + ); + let dataset = Rc::clone(&self.dataset); + if let Some(regex) = + compile_static_pattern_if_exists(¶meters[1], parameters.get(3)) + { + Rc::new(move |tuple| { + let (text, language) = + to_string_and_language(&dataset, &arg(tuple)?)?; + let replacement = to_simple_string(&dataset, &replacement(tuple)?)?; + Some(build_plain_literal( + &dataset, + ®ex.replace_all(&text, replacement.as_str()), + language, + )) + }) + } else { + let pattern = self.expression_evaluator( + ¶meters[1], + encoded_variables, + stat_children, + ); + let flags = parameters.get(3).map(|flags| { + self.expression_evaluator(flags, encoded_variables, stat_children) + }); + Rc::new(move |tuple| { + let pattern = to_simple_string(&dataset, &pattern(tuple)?)?; + let options = if let Some(flags) = &flags { + Some(to_simple_string(&dataset, &flags(tuple)?)?) + } else { + None + }; + let regex = compile_pattern(&pattern, options.as_deref())?; + let (text, language) = + to_string_and_language(&dataset, &arg(tuple)?)?; + let replacement = to_simple_string(&dataset, &replacement(tuple)?)?; + Some(build_plain_literal( + &dataset, + ®ex.replace_all(&text, replacement.as_str()), + language, + )) + }) } } - Some(build_string_literal( - &dataset, - str::from_utf8(&result).ok()?, - )) - }) - } - PlanExpression::StrEnds(arg1, arg2) => { - let arg1 = self.expression_evaluator(arg1, stat_children); - let arg2 = self.expression_evaluator(arg2, stat_children); - let dataset = Rc::clone(&self.dataset); - Rc::new(move |tuple| { - let (arg1, arg2, _) = - to_argument_compatible_strings(&dataset, &arg1(tuple)?, &arg2(tuple)?)?; - Some(arg1.ends_with(arg2.as_str()).into()) - }) - } - PlanExpression::Contains(arg1, arg2) => { - let arg1 = self.expression_evaluator(arg1, stat_children); - let arg2 = self.expression_evaluator(arg2, stat_children); - let dataset = Rc::clone(&self.dataset); - Rc::new(move |tuple| { - let (arg1, arg2, _) = - to_argument_compatible_strings(&dataset, &arg1(tuple)?, &arg2(tuple)?)?; - Some(arg1.contains(arg2.as_str()).into()) - }) - } - PlanExpression::StrBefore(arg1, arg2) => { - let arg1 = self.expression_evaluator(arg1, stat_children); - let arg2 = self.expression_evaluator(arg2, stat_children); - let dataset = Rc::clone(&self.dataset); - Rc::new(move |tuple| { - let (arg1, arg2, language) = - to_argument_compatible_strings(&dataset, &arg1(tuple)?, &arg2(tuple)?)?; - Some(if let Some(position) = arg1.find(arg2.as_str()) { - build_plain_literal(&dataset, &arg1[..position], language) - } else { - build_string_literal(&dataset, "") - }) - }) - } - PlanExpression::StrAfter(arg1, arg2) => { - let arg1 = self.expression_evaluator(arg1, stat_children); - let arg2 = self.expression_evaluator(arg2, stat_children); - let dataset = Rc::clone(&self.dataset); - Rc::new(move |tuple| { - let (arg1, arg2, language) = - to_argument_compatible_strings(&dataset, &arg1(tuple)?, &arg2(tuple)?)?; - Some(if let Some(position) = arg1.find(arg2.as_str()) { - build_plain_literal(&dataset, &arg1[position + arg2.len()..], language) - } else { - build_string_literal(&dataset, "") - }) - }) - } - PlanExpression::Year(e) => { - let e = self.expression_evaluator(e, stat_children); - Rc::new(move |tuple| match e(tuple)? { - EncodedTerm::DateTimeLiteral(date_time) => Some(date_time.year().into()), - EncodedTerm::DateLiteral(date) => Some(date.year().into()), - EncodedTerm::GYearMonthLiteral(year_month) => Some(year_month.year().into()), - EncodedTerm::GYearLiteral(year) => Some(year.year().into()), - _ => None, - }) - } - PlanExpression::Month(e) => { - let e = self.expression_evaluator(e, stat_children); - Rc::new(move |tuple| match e(tuple)? { - EncodedTerm::DateTimeLiteral(date_time) => Some(date_time.month().into()), - EncodedTerm::DateLiteral(date) => Some(date.month().into()), - EncodedTerm::GYearMonthLiteral(year_month) => Some(year_month.month().into()), - EncodedTerm::GMonthDayLiteral(month_day) => Some(month_day.month().into()), - EncodedTerm::GMonthLiteral(month) => Some(month.month().into()), - _ => None, - }) - } - PlanExpression::Day(e) => { - let e = self.expression_evaluator(e, stat_children); - Rc::new(move |tuple| match e(tuple)? { - EncodedTerm::DateTimeLiteral(date_time) => Some(date_time.day().into()), - EncodedTerm::DateLiteral(date) => Some(date.day().into()), - EncodedTerm::GMonthDayLiteral(month_day) => Some(month_day.day().into()), - EncodedTerm::GDayLiteral(day) => Some(day.day().into()), - _ => None, - }) - } - PlanExpression::Hours(e) => { - let e = self.expression_evaluator(e, stat_children); - Rc::new(move |tuple| match e(tuple)? { - EncodedTerm::DateTimeLiteral(date_time) => Some(date_time.hour().into()), - EncodedTerm::TimeLiteral(time) => Some(time.hour().into()), - _ => None, - }) - } - PlanExpression::Minutes(e) => { - let e = self.expression_evaluator(e, stat_children); - Rc::new(move |tuple| match e(tuple)? { - EncodedTerm::DateTimeLiteral(date_time) => Some(date_time.minute().into()), - EncodedTerm::TimeLiteral(time) => Some(time.minute().into()), - _ => None, - }) - } - PlanExpression::Seconds(e) => { - let e = self.expression_evaluator(e, stat_children); - Rc::new(move |tuple| match e(tuple)? { - EncodedTerm::DateTimeLiteral(date_time) => Some(date_time.second().into()), - EncodedTerm::TimeLiteral(time) => Some(time.second().into()), - _ => None, - }) - } - PlanExpression::Timezone(e) => { - let e = self.expression_evaluator(e, stat_children); - Rc::new(move |tuple| { - Some( - match e(tuple)? { - EncodedTerm::DateTimeLiteral(date_time) => date_time.timezone(), - EncodedTerm::TimeLiteral(time) => time.timezone(), - EncodedTerm::DateLiteral(date) => date.timezone(), - EncodedTerm::GYearMonthLiteral(year_month) => year_month.timezone(), - EncodedTerm::GYearLiteral(year) => year.timezone(), - EncodedTerm::GMonthDayLiteral(month_day) => month_day.timezone(), - EncodedTerm::GDayLiteral(day) => day.timezone(), - EncodedTerm::GMonthLiteral(month) => month.timezone(), + Function::UCase => { + let e = self.expression_evaluator( + ¶meters[0], + encoded_variables, + stat_children, + ); + let dataset = Rc::clone(&self.dataset); + Rc::new(move |tuple| { + let (value, language) = to_string_and_language(&dataset, &e(tuple)?)?; + Some(build_plain_literal( + &dataset, + &value.to_uppercase(), + language, + )) + }) + } + Function::LCase => { + let e = self.expression_evaluator( + ¶meters[0], + encoded_variables, + stat_children, + ); + let dataset = Rc::clone(&self.dataset); + Rc::new(move |tuple| { + let (value, language) = to_string_and_language(&dataset, &e(tuple)?)?; + Some(build_plain_literal( + &dataset, + &value.to_lowercase(), + language, + )) + }) + } + Function::StrStarts => { + let arg1 = self.expression_evaluator( + ¶meters[0], + encoded_variables, + stat_children, + ); + let arg2 = self.expression_evaluator( + ¶meters[1], + encoded_variables, + stat_children, + ); + let dataset = Rc::clone(&self.dataset); + Rc::new(move |tuple| { + let (arg1, arg2, _) = to_argument_compatible_strings( + &dataset, + &arg1(tuple)?, + &arg2(tuple)?, + )?; + Some(arg1.starts_with(arg2.as_str()).into()) + }) + } + Function::EncodeForUri => { + let ltrl = self.expression_evaluator( + ¶meters[0], + encoded_variables, + stat_children, + ); + let dataset = Rc::clone(&self.dataset); + Rc::new(move |tuple| { + let ltlr = to_string(&dataset, <rl(tuple)?)?; + let mut result = Vec::with_capacity(ltlr.len()); + for c in ltlr.bytes() { + match c { + b'A'..=b'Z' + | b'a'..=b'z' + | b'0'..=b'9' + | b'-' + | b'_' + | b'.' + | b'~' => result.push(c), + _ => { + result.push(b'%'); + let high = c / 16; + let low = c % 16; + result.push(if high < 10 { + b'0' + high + } else { + b'A' + (high - 10) + }); + result.push(if low < 10 { + b'0' + low + } else { + b'A' + (low - 10) + }); + } + } + } + Some(build_string_literal( + &dataset, + str::from_utf8(&result).ok()?, + )) + }) + } + Function::StrEnds => { + let arg1 = self.expression_evaluator( + ¶meters[0], + encoded_variables, + stat_children, + ); + let arg2 = self.expression_evaluator( + ¶meters[1], + encoded_variables, + stat_children, + ); + let dataset = Rc::clone(&self.dataset); + Rc::new(move |tuple| { + let (arg1, arg2, _) = to_argument_compatible_strings( + &dataset, + &arg1(tuple)?, + &arg2(tuple)?, + )?; + Some(arg1.ends_with(arg2.as_str()).into()) + }) + } + Function::Contains => { + let arg1 = self.expression_evaluator( + ¶meters[0], + encoded_variables, + stat_children, + ); + let arg2 = self.expression_evaluator( + ¶meters[1], + encoded_variables, + stat_children, + ); + let dataset = Rc::clone(&self.dataset); + Rc::new(move |tuple| { + let (arg1, arg2, _) = to_argument_compatible_strings( + &dataset, + &arg1(tuple)?, + &arg2(tuple)?, + )?; + Some(arg1.contains(arg2.as_str()).into()) + }) + } + Function::StrBefore => { + let arg1 = self.expression_evaluator( + ¶meters[0], + encoded_variables, + stat_children, + ); + let arg2 = self.expression_evaluator( + ¶meters[1], + encoded_variables, + stat_children, + ); + let dataset = Rc::clone(&self.dataset); + Rc::new(move |tuple| { + let (arg1, arg2, language) = to_argument_compatible_strings( + &dataset, + &arg1(tuple)?, + &arg2(tuple)?, + )?; + Some(if let Some(position) = arg1.find(arg2.as_str()) { + build_plain_literal(&dataset, &arg1[..position], language) + } else { + build_string_literal(&dataset, "") + }) + }) + } + Function::StrAfter => { + let arg1 = self.expression_evaluator( + ¶meters[0], + encoded_variables, + stat_children, + ); + let arg2 = self.expression_evaluator( + ¶meters[1], + encoded_variables, + stat_children, + ); + let dataset = Rc::clone(&self.dataset); + Rc::new(move |tuple| { + let (arg1, arg2, language) = to_argument_compatible_strings( + &dataset, + &arg1(tuple)?, + &arg2(tuple)?, + )?; + Some(if let Some(position) = arg1.find(arg2.as_str()) { + build_plain_literal( + &dataset, + &arg1[position + arg2.len()..], + language, + ) + } else { + build_string_literal(&dataset, "") + }) + }) + } + Function::Year => { + let e = self.expression_evaluator( + ¶meters[0], + encoded_variables, + stat_children, + ); + Rc::new(move |tuple| match e(tuple)? { + EncodedTerm::DateTimeLiteral(date_time) => { + Some(date_time.year().into()) + } + EncodedTerm::DateLiteral(date) => Some(date.year().into()), + EncodedTerm::GYearMonthLiteral(year_month) => { + Some(year_month.year().into()) + } + EncodedTerm::GYearLiteral(year) => Some(year.year().into()), _ => None, - }? - .into(), - ) - }) - } - PlanExpression::Tz(e) => { - let e = self.expression_evaluator(e, stat_children); - let dataset = Rc::clone(&self.dataset); - Rc::new(move |tuple| { - let timezone_offset = match e(tuple)? { - EncodedTerm::DateTimeLiteral(date_time) => date_time.timezone_offset(), - EncodedTerm::TimeLiteral(time) => time.timezone_offset(), - EncodedTerm::DateLiteral(date) => date.timezone_offset(), - EncodedTerm::GYearMonthLiteral(year_month) => year_month.timezone_offset(), - EncodedTerm::GYearLiteral(year) => year.timezone_offset(), - EncodedTerm::GMonthDayLiteral(month_day) => month_day.timezone_offset(), - EncodedTerm::GDayLiteral(day) => day.timezone_offset(), - EncodedTerm::GMonthLiteral(month) => month.timezone_offset(), - _ => return None, - }; - Some(match timezone_offset { - Some(timezone_offset) => { - build_string_literal(&dataset, &timezone_offset.to_string()) - } - None => build_string_literal(&dataset, ""), - }) - }) - } - - PlanExpression::Adjust(dt, tz) => { - let dt = self.expression_evaluator(dt, stat_children); - let tz = self.expression_evaluator(tz, stat_children); - Rc::new(move |tuple| { - let timezone_offset = Some( - match tz(tuple)? { - EncodedTerm::DayTimeDurationLiteral(tz) => TimezoneOffset::try_from(tz), - EncodedTerm::DurationLiteral(tz) => TimezoneOffset::try_from(tz), - _ => return None, - } - .ok()?, - ); - Some(match dt(tuple)? { - EncodedTerm::DateTimeLiteral(date_time) => { - date_time.adjust(timezone_offset)?.into() - } - EncodedTerm::TimeLiteral(time) => time.adjust(timezone_offset)?.into(), - EncodedTerm::DateLiteral(date) => date.adjust(timezone_offset)?.into(), - EncodedTerm::GYearMonthLiteral(year_month) => { - year_month.adjust(timezone_offset)?.into() - } - EncodedTerm::GYearLiteral(year) => year.adjust(timezone_offset)?.into(), - EncodedTerm::GMonthDayLiteral(month_day) => { - month_day.adjust(timezone_offset)?.into() - } - EncodedTerm::GDayLiteral(day) => day.adjust(timezone_offset)?.into(), - EncodedTerm::GMonthLiteral(month) => month.adjust(timezone_offset)?.into(), - _ => return None, - }) - }) - } - PlanExpression::Now => { - let now = self.now; - Rc::new(move |_| Some(now.into())) - } - PlanExpression::Uuid => { - let dataset = Rc::clone(&self.dataset); - Rc::new(move |_| { - let mut buffer = String::with_capacity(44); - buffer.push_str("urn:uuid:"); - generate_uuid(&mut buffer); - Some(build_named_node(&dataset, &buffer)) - }) - } - PlanExpression::StrUuid => { - let dataset = Rc::clone(&self.dataset); - Rc::new(move |_| { - let mut buffer = String::with_capacity(36); - generate_uuid(&mut buffer); - Some(build_string_literal(&dataset, &buffer)) - }) - } - PlanExpression::Md5(arg) => self.hash::(arg, stat_children), - PlanExpression::Sha1(arg) => self.hash::(arg, stat_children), - PlanExpression::Sha256(arg) => self.hash::(arg, stat_children), - PlanExpression::Sha384(arg) => self.hash::(arg, stat_children), - PlanExpression::Sha512(arg) => self.hash::(arg, stat_children), - PlanExpression::Coalesce(l) => { - let l: Vec<_> = l - .iter() - .map(|e| self.expression_evaluator(e, stat_children)) - .collect(); - Rc::new(move |tuple| { - for e in &l { - if let Some(result) = e(tuple) { - return Some(result); - } + }) } - None - }) - } - PlanExpression::If(a, b, c) => { - let a = self.expression_evaluator(a, stat_children); - let b = self.expression_evaluator(b, stat_children); - let c = self.expression_evaluator(c, stat_children); - Rc::new(move |tuple| { - if to_bool(&a(tuple)?)? { - b(tuple) - } else { - c(tuple) + Function::Month => { + let e = self.expression_evaluator( + ¶meters[0], + encoded_variables, + stat_children, + ); + Rc::new(move |tuple| match e(tuple)? { + EncodedTerm::DateTimeLiteral(date_time) => { + Some(date_time.month().into()) + } + EncodedTerm::DateLiteral(date) => Some(date.month().into()), + EncodedTerm::GYearMonthLiteral(year_month) => { + Some(year_month.month().into()) + } + EncodedTerm::GMonthDayLiteral(month_day) => { + Some(month_day.month().into()) + } + EncodedTerm::GMonthLiteral(month) => Some(month.month().into()), + _ => None, + }) } - }) - } - PlanExpression::StrLang(lexical_form, lang_tag) => { - let lexical_form = self.expression_evaluator(lexical_form, stat_children); - let lang_tag = self.expression_evaluator(lang_tag, stat_children); - let dataset = Rc::clone(&self.dataset); - Rc::new(move |tuple| { - Some(build_lang_string_literal_from_id( - to_simple_string_id(&lexical_form(tuple)?)?, - build_language_id(&dataset, &lang_tag(tuple)?)?, - )) - }) - } - PlanExpression::StrDt(lexical_form, datatype) => { - let lexical_form = self.expression_evaluator(lexical_form, stat_children); - let datatype = self.expression_evaluator(datatype, stat_children); - let dataset = Rc::clone(&self.dataset); - Rc::new(move |tuple| { - let value = to_simple_string(&dataset, &lexical_form(tuple)?)?; - let datatype = if let EncodedTerm::NamedNode { iri_id } = datatype(tuple)? { - dataset.get_str(&iri_id).ok()? - } else { - None - }?; - Some(dataset.encode_term(LiteralRef::new_typed_literal( - &value, - NamedNodeRef::new_unchecked(&datatype), - ))) - }) - } - PlanExpression::SameTerm(a, b) => { - let a = self.expression_evaluator(a, stat_children); - let b = self.expression_evaluator(b, stat_children); - Rc::new(move |tuple| Some((a(tuple)? == b(tuple)?).into())) - } - PlanExpression::IsIri(e) => { - let e = self.expression_evaluator(e, stat_children); - Rc::new(move |tuple| Some(e(tuple)?.is_named_node().into())) - } - PlanExpression::IsBlank(e) => { - let e = self.expression_evaluator(e, stat_children); - Rc::new(move |tuple| Some(e(tuple)?.is_blank_node().into())) - } - PlanExpression::IsLiteral(e) => { - let e = self.expression_evaluator(e, stat_children); - Rc::new(move |tuple| Some(e(tuple)?.is_literal().into())) - } - PlanExpression::IsNumeric(e) => { - let e = self.expression_evaluator(e, stat_children); - Rc::new(move |tuple| { - Some( - matches!( - e(tuple)?, - EncodedTerm::FloatLiteral(_) - | EncodedTerm::DoubleLiteral(_) - | EncodedTerm::IntegerLiteral(_) - | EncodedTerm::DecimalLiteral(_) - ) - .into(), - ) - }) - } - PlanExpression::StaticRegex(text, regex) => { - let text = self.expression_evaluator(text, stat_children); - let dataset = Rc::clone(&self.dataset); - let regex = regex.clone(); - Rc::new(move |tuple| { - let text = to_string(&dataset, &text(tuple)?)?; - Some(regex.is_match(&text).into()) - }) - } - PlanExpression::DynamicRegex(text, pattern, flags) => { - let text = self.expression_evaluator(text, stat_children); - let pattern = self.expression_evaluator(pattern, stat_children); - let flags = flags - .as_ref() - .map(|flags| self.expression_evaluator(flags, stat_children)); - let dataset = Rc::clone(&self.dataset); - Rc::new(move |tuple| { - let pattern = to_simple_string(&dataset, &pattern(tuple)?)?; - let options = if let Some(flags) = &flags { - Some(to_simple_string(&dataset, &flags(tuple)?)?) - } else { - None - }; - let regex = compile_pattern(&pattern, options.as_deref())?; - let text = to_string(&dataset, &text(tuple)?)?; - Some(regex.is_match(&text).into()) - }) - } - PlanExpression::Triple(s, p, o) => { - let s = self.expression_evaluator(s, stat_children); - let p = self.expression_evaluator(p, stat_children); - let o = self.expression_evaluator(o, stat_children); - Rc::new(move |tuple| { - let s = s(tuple)?; - let p = p(tuple)?; - let o = o(tuple)?; - (!s.is_literal() - && !s.is_default_graph() - && p.is_named_node() - && !o.is_default_graph()) - .then(|| EncodedTriple::new(s, p, o).into()) - }) - } - PlanExpression::Subject(e) => { - let e = self.expression_evaluator(e, stat_children); - Rc::new(move |tuple| { - if let EncodedTerm::Triple(t) = e(tuple)? { - Some(t.subject.clone()) - } else { - None + Function::Day => { + let e = self.expression_evaluator( + ¶meters[0], + encoded_variables, + stat_children, + ); + Rc::new(move |tuple| match e(tuple)? { + EncodedTerm::DateTimeLiteral(date_time) => Some(date_time.day().into()), + EncodedTerm::DateLiteral(date) => Some(date.day().into()), + EncodedTerm::GMonthDayLiteral(month_day) => { + Some(month_day.day().into()) + } + EncodedTerm::GDayLiteral(day) => Some(day.day().into()), + _ => None, + }) } - }) - } - PlanExpression::Predicate(e) => { - let e = self.expression_evaluator(e, stat_children); - Rc::new(move |tuple| { - if let EncodedTerm::Triple(t) = e(tuple)? { - Some(t.predicate.clone()) - } else { - None + Function::Hours => { + let e = self.expression_evaluator( + ¶meters[0], + encoded_variables, + stat_children, + ); + Rc::new(move |tuple| match e(tuple)? { + EncodedTerm::DateTimeLiteral(date_time) => { + Some(date_time.hour().into()) + } + EncodedTerm::TimeLiteral(time) => Some(time.hour().into()), + _ => None, + }) } - }) - } - PlanExpression::Object(e) => { - let e = self.expression_evaluator(e, stat_children); - Rc::new(move |tuple| { - if let EncodedTerm::Triple(t) = e(tuple)? { - Some(t.object.clone()) - } else { - None + Function::Minutes => { + let e = self.expression_evaluator( + ¶meters[0], + encoded_variables, + stat_children, + ); + Rc::new(move |tuple| match e(tuple)? { + EncodedTerm::DateTimeLiteral(date_time) => { + Some(date_time.minute().into()) + } + EncodedTerm::TimeLiteral(time) => Some(time.minute().into()), + _ => None, + }) } - }) - } - PlanExpression::IsTriple(e) => { - let e = self.expression_evaluator(e, stat_children); - Rc::new(move |tuple| Some(e(tuple)?.is_triple().into())) - } - PlanExpression::BooleanCast(e) => { - let e = self.expression_evaluator(e, stat_children); - Rc::new(move |tuple| match e(tuple)? { - EncodedTerm::BooleanLiteral(value) => Some(value.into()), - EncodedTerm::FloatLiteral(value) => Some(Boolean::from(value).into()), - EncodedTerm::DoubleLiteral(value) => Some(Boolean::from(value).into()), - EncodedTerm::IntegerLiteral(value) => Some(Boolean::from(value).into()), - EncodedTerm::DecimalLiteral(value) => Some(Boolean::from(value).into()), - EncodedTerm::SmallStringLiteral(value) => parse_boolean_str(&value), - _ => None, - }) - } - PlanExpression::DoubleCast(e) => { - let e = self.expression_evaluator(e, stat_children); - let dataset = Rc::clone(&self.dataset); - Rc::new(move |tuple| match e(tuple)? { - EncodedTerm::FloatLiteral(value) => Some(Double::from(value).into()), - EncodedTerm::DoubleLiteral(value) => Some(value.into()), - EncodedTerm::IntegerLiteral(value) => Some(Double::from(value).into()), - EncodedTerm::DecimalLiteral(value) => Some(Double::from(value).into()), - EncodedTerm::BooleanLiteral(value) => Some(Double::from(value).into()), - EncodedTerm::SmallStringLiteral(value) => parse_double_str(&value), - EncodedTerm::BigStringLiteral { value_id } => { - parse_double_str(&dataset.get_str(&value_id).ok()??) + Function::Seconds => { + let e = self.expression_evaluator( + ¶meters[0], + encoded_variables, + stat_children, + ); + Rc::new(move |tuple| match e(tuple)? { + EncodedTerm::DateTimeLiteral(date_time) => { + Some(date_time.second().into()) + } + EncodedTerm::TimeLiteral(time) => Some(time.second().into()), + _ => None, + }) } - _ => None, - }) - } - PlanExpression::FloatCast(e) => { - let e = self.expression_evaluator(e, stat_children); - let dataset = Rc::clone(&self.dataset); - Rc::new(move |tuple| match e(tuple)? { - EncodedTerm::FloatLiteral(value) => Some(value.into()), - EncodedTerm::DoubleLiteral(value) => Some(Float::from(value).into()), - EncodedTerm::IntegerLiteral(value) => Some(Float::from(value).into()), - EncodedTerm::DecimalLiteral(value) => Some(Float::from(value).into()), - EncodedTerm::BooleanLiteral(value) => Some(Float::from(value).into()), - EncodedTerm::SmallStringLiteral(value) => parse_float_str(&value), - EncodedTerm::BigStringLiteral { value_id } => { - parse_float_str(&dataset.get_str(&value_id).ok()??) + Function::Timezone => { + let e = self.expression_evaluator( + ¶meters[0], + encoded_variables, + stat_children, + ); + Rc::new(move |tuple| { + Some( + match e(tuple)? { + EncodedTerm::DateTimeLiteral(date_time) => date_time.timezone(), + EncodedTerm::TimeLiteral(time) => time.timezone(), + EncodedTerm::DateLiteral(date) => date.timezone(), + EncodedTerm::GYearMonthLiteral(year_month) => { + year_month.timezone() + } + EncodedTerm::GYearLiteral(year) => year.timezone(), + EncodedTerm::GMonthDayLiteral(month_day) => { + month_day.timezone() + } + EncodedTerm::GDayLiteral(day) => day.timezone(), + EncodedTerm::GMonthLiteral(month) => month.timezone(), + _ => None, + }? + .into(), + ) + }) } - _ => None, - }) - } - PlanExpression::IntegerCast(e) => { - let e = self.expression_evaluator(e, stat_children); - let dataset = Rc::clone(&self.dataset); - Rc::new(move |tuple| match e(tuple)? { - EncodedTerm::FloatLiteral(value) => Some(Integer::try_from(value).ok()?.into()), - EncodedTerm::DoubleLiteral(value) => { - Some(Integer::try_from(value).ok()?.into()) + Function::Tz => { + let e = self.expression_evaluator( + ¶meters[0], + encoded_variables, + stat_children, + ); + let dataset = Rc::clone(&self.dataset); + Rc::new(move |tuple| { + let timezone_offset = match e(tuple)? { + EncodedTerm::DateTimeLiteral(date_time) => { + date_time.timezone_offset() + } + EncodedTerm::TimeLiteral(time) => time.timezone_offset(), + EncodedTerm::DateLiteral(date) => date.timezone_offset(), + EncodedTerm::GYearMonthLiteral(year_month) => { + year_month.timezone_offset() + } + EncodedTerm::GYearLiteral(year) => year.timezone_offset(), + EncodedTerm::GMonthDayLiteral(month_day) => { + month_day.timezone_offset() + } + EncodedTerm::GDayLiteral(day) => day.timezone_offset(), + EncodedTerm::GMonthLiteral(month) => month.timezone_offset(), + _ => return None, + }; + Some(match timezone_offset { + Some(timezone_offset) => { + build_string_literal(&dataset, &timezone_offset.to_string()) + } + None => build_string_literal(&dataset, ""), + }) + }) } - EncodedTerm::IntegerLiteral(value) => Some(value.into()), - EncodedTerm::DecimalLiteral(value) => { - Some(Integer::try_from(value).ok()?.into()) + Function::Adjust => { + let dt = self.expression_evaluator( + ¶meters[0], + encoded_variables, + stat_children, + ); + let tz = self.expression_evaluator( + ¶meters[1], + encoded_variables, + stat_children, + ); + Rc::new(move |tuple| { + let timezone_offset = Some( + match tz(tuple)? { + EncodedTerm::DayTimeDurationLiteral(tz) => { + TimezoneOffset::try_from(tz) + } + EncodedTerm::DurationLiteral(tz) => { + TimezoneOffset::try_from(tz) + } + _ => return None, + } + .ok()?, + ); + Some(match dt(tuple)? { + EncodedTerm::DateTimeLiteral(date_time) => { + date_time.adjust(timezone_offset)?.into() + } + EncodedTerm::TimeLiteral(time) => { + time.adjust(timezone_offset)?.into() + } + EncodedTerm::DateLiteral(date) => { + date.adjust(timezone_offset)?.into() + } + EncodedTerm::GYearMonthLiteral(year_month) => { + year_month.adjust(timezone_offset)?.into() + } + EncodedTerm::GYearLiteral(year) => { + year.adjust(timezone_offset)?.into() + } + EncodedTerm::GMonthDayLiteral(month_day) => { + month_day.adjust(timezone_offset)?.into() + } + EncodedTerm::GDayLiteral(day) => { + day.adjust(timezone_offset)?.into() + } + EncodedTerm::GMonthLiteral(month) => { + month.adjust(timezone_offset)?.into() + } + _ => return None, + }) + }) } - EncodedTerm::BooleanLiteral(value) => Some(Integer::from(value).into()), - EncodedTerm::SmallStringLiteral(value) => parse_integer_str(&value), - EncodedTerm::BigStringLiteral { value_id } => { - parse_integer_str(&dataset.get_str(&value_id).ok()??) + Function::Now => { + let now = self.now; + Rc::new(move |_| Some(now.into())) } - _ => None, - }) - } - PlanExpression::DecimalCast(e) => { - let e = self.expression_evaluator(e, stat_children); - let dataset = Rc::clone(&self.dataset); - Rc::new(move |tuple| match e(tuple)? { - EncodedTerm::FloatLiteral(value) => Some(Decimal::try_from(value).ok()?.into()), - EncodedTerm::DoubleLiteral(value) => { - Some(Decimal::try_from(value).ok()?.into()) + Function::Uuid => { + let dataset = Rc::clone(&self.dataset); + Rc::new(move |_| { + let mut buffer = String::with_capacity(44); + buffer.push_str("urn:uuid:"); + generate_uuid(&mut buffer); + Some(build_named_node(&dataset, &buffer)) + }) } - EncodedTerm::IntegerLiteral(value) => { - Some(Decimal::try_from(value).ok()?.into()) + Function::StrUuid => { + let dataset = Rc::clone(&self.dataset); + Rc::new(move |_| { + let mut buffer = String::with_capacity(36); + generate_uuid(&mut buffer); + Some(build_string_literal(&dataset, &buffer)) + }) } - EncodedTerm::DecimalLiteral(value) => Some(value.into()), - EncodedTerm::BooleanLiteral(value) => Some(Decimal::from(value).into()), - EncodedTerm::SmallStringLiteral(value) => parse_decimal_str(&value), - EncodedTerm::BigStringLiteral { value_id } => { - parse_decimal_str(&dataset.get_str(&value_id).ok()??) + Function::Md5 => self.hash::(parameters, encoded_variables, stat_children), + Function::Sha1 => { + self.hash::(parameters, encoded_variables, stat_children) } - _ => None, - }) - } - PlanExpression::DateCast(e) => { - let e = self.expression_evaluator(e, stat_children); - let dataset = Rc::clone(&self.dataset); - Rc::new(move |tuple| match e(tuple)? { - EncodedTerm::DateLiteral(value) => Some(value.into()), - EncodedTerm::DateTimeLiteral(value) => Some(Date::try_from(value).ok()?.into()), - EncodedTerm::SmallStringLiteral(value) => parse_date_str(&value), - EncodedTerm::BigStringLiteral { value_id } => { - parse_date_str(&dataset.get_str(&value_id).ok()??) + Function::Sha256 => { + self.hash::(parameters, encoded_variables, stat_children) } - _ => None, - }) - } - PlanExpression::TimeCast(e) => { - let e = self.expression_evaluator(e, stat_children); - let dataset = Rc::clone(&self.dataset); - Rc::new(move |tuple| match e(tuple)? { - EncodedTerm::TimeLiteral(value) => Some(value.into()), - EncodedTerm::DateTimeLiteral(value) => Some(Time::try_from(value).ok()?.into()), - EncodedTerm::SmallStringLiteral(value) => parse_time_str(&value), - EncodedTerm::BigStringLiteral { value_id } => { - parse_time_str(&dataset.get_str(&value_id).ok()??) + Function::Sha384 => { + self.hash::(parameters, encoded_variables, stat_children) } - _ => None, - }) - } - PlanExpression::DateTimeCast(e) => { - let e = self.expression_evaluator(e, stat_children); - let dataset = Rc::clone(&self.dataset); - Rc::new(move |tuple| match e(tuple)? { - EncodedTerm::DateTimeLiteral(value) => Some(value.into()), - EncodedTerm::DateLiteral(value) => Some(DateTime::try_from(value).ok()?.into()), - EncodedTerm::SmallStringLiteral(value) => parse_date_time_str(&value), - EncodedTerm::BigStringLiteral { value_id } => { - parse_date_time_str(&dataset.get_str(&value_id).ok()??) + Function::Sha512 => { + self.hash::(parameters, encoded_variables, stat_children) } - _ => None, - }) - } - PlanExpression::DurationCast(e) => { - let e = self.expression_evaluator(e, stat_children); - let dataset = Rc::clone(&self.dataset); - Rc::new(move |tuple| match e(tuple)? { - EncodedTerm::DurationLiteral(value) => Some(value.into()), - EncodedTerm::YearMonthDurationLiteral(value) => { - Some(Duration::from(value).into()) + Function::StrLang => { + let lexical_form = self.expression_evaluator( + ¶meters[0], + encoded_variables, + stat_children, + ); + let lang_tag = self.expression_evaluator( + ¶meters[1], + encoded_variables, + stat_children, + ); + let dataset = Rc::clone(&self.dataset); + Rc::new(move |tuple| { + Some(build_lang_string_literal_from_id( + to_simple_string_id(&lexical_form(tuple)?)?, + build_language_id(&dataset, &lang_tag(tuple)?)?, + )) + }) } - EncodedTerm::DayTimeDurationLiteral(value) => { - Some(Duration::from(value).into()) + Function::StrDt => { + let lexical_form = self.expression_evaluator( + ¶meters[0], + encoded_variables, + stat_children, + ); + let datatype = self.expression_evaluator( + ¶meters[1], + encoded_variables, + stat_children, + ); + let dataset = Rc::clone(&self.dataset); + Rc::new(move |tuple| { + let value = to_simple_string(&dataset, &lexical_form(tuple)?)?; + let datatype = + if let EncodedTerm::NamedNode { iri_id } = datatype(tuple)? { + dataset.get_str(&iri_id).ok()? + } else { + None + }?; + Some(dataset.encode_term(LiteralRef::new_typed_literal( + &value, + NamedNodeRef::new_unchecked(&datatype), + ))) + }) } - EncodedTerm::SmallStringLiteral(value) => parse_duration_str(&value), - EncodedTerm::BigStringLiteral { value_id } => { - parse_duration_str(&dataset.get_str(&value_id).ok()??) + Function::IsIri => { + let e = self.expression_evaluator( + ¶meters[0], + encoded_variables, + stat_children, + ); + Rc::new(move |tuple| Some(e(tuple)?.is_named_node().into())) } - _ => None, - }) - } - PlanExpression::YearMonthDurationCast(e) => { - let e = self.expression_evaluator(e, stat_children); - let dataset = Rc::clone(&self.dataset); - Rc::new(move |tuple| match e(tuple)? { - EncodedTerm::DurationLiteral(value) => { - Some(YearMonthDuration::try_from(value).ok()?.into()) + Function::IsBlank => { + let e = self.expression_evaluator( + ¶meters[0], + encoded_variables, + stat_children, + ); + Rc::new(move |tuple| Some(e(tuple)?.is_blank_node().into())) } - EncodedTerm::YearMonthDurationLiteral(value) => Some(value.into()), - EncodedTerm::SmallStringLiteral(value) => parse_year_month_duration_str(&value), - EncodedTerm::BigStringLiteral { value_id } => { - parse_year_month_duration_str(&dataset.get_str(&value_id).ok()??) + Function::IsLiteral => { + let e = self.expression_evaluator( + ¶meters[0], + encoded_variables, + stat_children, + ); + Rc::new(move |tuple| Some(e(tuple)?.is_literal().into())) } - _ => None, - }) - } - PlanExpression::DayTimeDurationCast(e) => { - let e = self.expression_evaluator(e, stat_children); - let dataset = Rc::clone(&self.dataset); - Rc::new(move |tuple| match e(tuple)? { - EncodedTerm::DurationLiteral(value) => { - Some(DayTimeDuration::try_from(value).ok()?.into()) + Function::IsNumeric => { + let e = self.expression_evaluator( + ¶meters[0], + encoded_variables, + stat_children, + ); + Rc::new(move |tuple| { + Some( + matches!( + e(tuple)?, + EncodedTerm::FloatLiteral(_) + | EncodedTerm::DoubleLiteral(_) + | EncodedTerm::IntegerLiteral(_) + | EncodedTerm::DecimalLiteral(_) + ) + .into(), + ) + }) } - EncodedTerm::DayTimeDurationLiteral(value) => Some(value.into()), - EncodedTerm::SmallStringLiteral(value) => parse_day_time_duration_str(&value), - EncodedTerm::BigStringLiteral { value_id } => { - parse_day_time_duration_str(&dataset.get_str(&value_id).ok()??) + Function::Regex => { + let text = self.expression_evaluator( + ¶meters[0], + encoded_variables, + stat_children, + ); + let dataset = Rc::clone(&self.dataset); + if let Some(regex) = + compile_static_pattern_if_exists(¶meters[1], parameters.get(2)) + { + Rc::new(move |tuple| { + let text = to_string(&dataset, &text(tuple)?)?; + Some(regex.is_match(&text).into()) + }) + } else { + let pattern = self.expression_evaluator( + ¶meters[0], + encoded_variables, + stat_children, + ); + let flags = parameters.get(2).map(|flags| { + self.expression_evaluator(flags, encoded_variables, stat_children) + }); + Rc::new(move |tuple| { + let pattern = to_simple_string(&dataset, &pattern(tuple)?)?; + let options = if let Some(flags) = &flags { + Some(to_simple_string(&dataset, &flags(tuple)?)?) + } else { + None + }; + let regex = compile_pattern(&pattern, options.as_deref())?; + let text = to_string(&dataset, &text(tuple)?)?; + Some(regex.is_match(&text).into()) + }) + } + } + Function::Triple => { + let s = self.expression_evaluator( + ¶meters[0], + encoded_variables, + stat_children, + ); + let p = self.expression_evaluator( + ¶meters[1], + encoded_variables, + stat_children, + ); + let o = self.expression_evaluator( + ¶meters[2], + encoded_variables, + stat_children, + ); + Rc::new(move |tuple| { + let s = s(tuple)?; + let p = p(tuple)?; + let o = o(tuple)?; + (!s.is_literal() + && !s.is_default_graph() + && p.is_named_node() + && !o.is_default_graph()) + .then(|| EncodedTriple::new(s, p, o).into()) + }) + } + Function::Subject => { + let e = self.expression_evaluator( + ¶meters[0], + encoded_variables, + stat_children, + ); + Rc::new(move |tuple| { + if let EncodedTerm::Triple(t) = e(tuple)? { + Some(t.subject.clone()) + } else { + None + } + }) + } + Function::Predicate => { + let e = self.expression_evaluator( + ¶meters[0], + encoded_variables, + stat_children, + ); + Rc::new(move |tuple| { + if let EncodedTerm::Triple(t) = e(tuple)? { + Some(t.predicate.clone()) + } else { + None + } + }) + } + Function::Object => { + let e = self.expression_evaluator( + ¶meters[0], + encoded_variables, + stat_children, + ); + Rc::new(move |tuple| { + if let EncodedTerm::Triple(t) = e(tuple)? { + Some(t.object.clone()) + } else { + None + } + }) + } + Function::IsTriple => { + let e = self.expression_evaluator( + ¶meters[0], + encoded_variables, + stat_children, + ); + Rc::new(move |tuple| Some(e(tuple)?.is_triple().into())) + } + Function::Custom(function_name) => { + if let Some(function) = self.custom_functions.get(function_name).cloned() { + let args = parameters + .iter() + .map(|e| { + self.expression_evaluator(e, encoded_variables, stat_children) + }) + .collect::>(); + let dataset = Rc::clone(&self.dataset); + return Rc::new(move |tuple| { + let args = args + .iter() + .map(|f| dataset.decode_term(&f(tuple)?).ok()) + .collect::>>()?; + Some(dataset.encode_term(&function(&args)?)) + }); + } + match function_name.as_ref() { + xsd::STRING => { + let e = self.expression_evaluator( + ¶meters[0], + encoded_variables, + stat_children, + ); + let dataset = Rc::clone(&self.dataset); + Rc::new(move |tuple| { + Some(build_string_literal_from_id(to_string_id( + &dataset, + &e(tuple)?, + )?)) + }) + } + xsd::BOOLEAN => { + let e = self.expression_evaluator( + ¶meters[0], + encoded_variables, + stat_children, + ); + Rc::new(move |tuple| match e(tuple)? { + EncodedTerm::BooleanLiteral(value) => Some(value.into()), + EncodedTerm::FloatLiteral(value) => { + Some(Boolean::from(value).into()) + } + EncodedTerm::DoubleLiteral(value) => { + Some(Boolean::from(value).into()) + } + EncodedTerm::IntegerLiteral(value) => { + Some(Boolean::from(value).into()) + } + EncodedTerm::DecimalLiteral(value) => { + Some(Boolean::from(value).into()) + } + EncodedTerm::SmallStringLiteral(value) => { + parse_boolean_str(&value) + } + _ => None, + }) + } + xsd::DOUBLE => { + let e = self.expression_evaluator( + ¶meters[0], + encoded_variables, + stat_children, + ); + let dataset = Rc::clone(&self.dataset); + Rc::new(move |tuple| match e(tuple)? { + EncodedTerm::FloatLiteral(value) => { + Some(Double::from(value).into()) + } + EncodedTerm::DoubleLiteral(value) => Some(value.into()), + EncodedTerm::IntegerLiteral(value) => { + Some(Double::from(value).into()) + } + EncodedTerm::DecimalLiteral(value) => { + Some(Double::from(value).into()) + } + EncodedTerm::BooleanLiteral(value) => { + Some(Double::from(value).into()) + } + EncodedTerm::SmallStringLiteral(value) => { + parse_double_str(&value) + } + EncodedTerm::BigStringLiteral { value_id } => { + parse_double_str(&dataset.get_str(&value_id).ok()??) + } + _ => None, + }) + } + xsd::FLOAT => { + let e = self.expression_evaluator( + ¶meters[0], + encoded_variables, + stat_children, + ); + let dataset = Rc::clone(&self.dataset); + Rc::new(move |tuple| match e(tuple)? { + EncodedTerm::FloatLiteral(value) => Some(value.into()), + EncodedTerm::DoubleLiteral(value) => { + Some(Float::from(value).into()) + } + EncodedTerm::IntegerLiteral(value) => { + Some(Float::from(value).into()) + } + EncodedTerm::DecimalLiteral(value) => { + Some(Float::from(value).into()) + } + EncodedTerm::BooleanLiteral(value) => { + Some(Float::from(value).into()) + } + EncodedTerm::SmallStringLiteral(value) => { + parse_float_str(&value) + } + EncodedTerm::BigStringLiteral { value_id } => { + parse_float_str(&dataset.get_str(&value_id).ok()??) + } + _ => None, + }) + } + xsd::INTEGER => { + let e = self.expression_evaluator( + ¶meters[0], + encoded_variables, + stat_children, + ); + let dataset = Rc::clone(&self.dataset); + Rc::new(move |tuple| match e(tuple)? { + EncodedTerm::FloatLiteral(value) => { + Some(Integer::try_from(value).ok()?.into()) + } + EncodedTerm::DoubleLiteral(value) => { + Some(Integer::try_from(value).ok()?.into()) + } + EncodedTerm::IntegerLiteral(value) => Some(value.into()), + EncodedTerm::DecimalLiteral(value) => { + Some(Integer::try_from(value).ok()?.into()) + } + EncodedTerm::BooleanLiteral(value) => { + Some(Integer::from(value).into()) + } + EncodedTerm::SmallStringLiteral(value) => { + parse_integer_str(&value) + } + EncodedTerm::BigStringLiteral { value_id } => { + parse_integer_str(&dataset.get_str(&value_id).ok()??) + } + _ => None, + }) + } + xsd::DECIMAL => { + let e = self.expression_evaluator( + ¶meters[0], + encoded_variables, + stat_children, + ); + let dataset = Rc::clone(&self.dataset); + Rc::new(move |tuple| match e(tuple)? { + EncodedTerm::FloatLiteral(value) => { + Some(Decimal::try_from(value).ok()?.into()) + } + EncodedTerm::DoubleLiteral(value) => { + Some(Decimal::try_from(value).ok()?.into()) + } + EncodedTerm::IntegerLiteral(value) => { + Some(Decimal::try_from(value).ok()?.into()) + } + EncodedTerm::DecimalLiteral(value) => Some(value.into()), + EncodedTerm::BooleanLiteral(value) => { + Some(Decimal::from(value).into()) + } + EncodedTerm::SmallStringLiteral(value) => { + parse_decimal_str(&value) + } + EncodedTerm::BigStringLiteral { value_id } => { + parse_decimal_str(&dataset.get_str(&value_id).ok()??) + } + _ => None, + }) + } + xsd::DATE => { + let e = self.expression_evaluator( + ¶meters[0], + encoded_variables, + stat_children, + ); + let dataset = Rc::clone(&self.dataset); + Rc::new(move |tuple| match e(tuple)? { + EncodedTerm::DateLiteral(value) => Some(value.into()), + EncodedTerm::DateTimeLiteral(value) => { + Some(Date::try_from(value).ok()?.into()) + } + EncodedTerm::SmallStringLiteral(value) => { + parse_date_str(&value) + } + EncodedTerm::BigStringLiteral { value_id } => { + parse_date_str(&dataset.get_str(&value_id).ok()??) + } + _ => None, + }) + } + xsd::TIME => { + let e = self.expression_evaluator( + ¶meters[0], + encoded_variables, + stat_children, + ); + let dataset = Rc::clone(&self.dataset); + Rc::new(move |tuple| match e(tuple)? { + EncodedTerm::TimeLiteral(value) => Some(value.into()), + EncodedTerm::DateTimeLiteral(value) => { + Some(Time::try_from(value).ok()?.into()) + } + EncodedTerm::SmallStringLiteral(value) => { + parse_time_str(&value) + } + EncodedTerm::BigStringLiteral { value_id } => { + parse_time_str(&dataset.get_str(&value_id).ok()??) + } + _ => None, + }) + } + xsd::DATE_TIME => { + let e = self.expression_evaluator( + ¶meters[0], + encoded_variables, + stat_children, + ); + let dataset = Rc::clone(&self.dataset); + Rc::new(move |tuple| match e(tuple)? { + EncodedTerm::DateTimeLiteral(value) => Some(value.into()), + EncodedTerm::DateLiteral(value) => { + Some(DateTime::try_from(value).ok()?.into()) + } + EncodedTerm::SmallStringLiteral(value) => { + parse_date_time_str(&value) + } + EncodedTerm::BigStringLiteral { value_id } => { + parse_date_time_str(&dataset.get_str(&value_id).ok()??) + } + _ => None, + }) + } + xsd::DURATION => { + let e = self.expression_evaluator( + ¶meters[0], + encoded_variables, + stat_children, + ); + let dataset = Rc::clone(&self.dataset); + Rc::new(move |tuple| match e(tuple)? { + EncodedTerm::DurationLiteral(value) => Some(value.into()), + EncodedTerm::YearMonthDurationLiteral(value) => { + Some(Duration::from(value).into()) + } + EncodedTerm::DayTimeDurationLiteral(value) => { + Some(Duration::from(value).into()) + } + EncodedTerm::SmallStringLiteral(value) => { + parse_duration_str(&value) + } + EncodedTerm::BigStringLiteral { value_id } => { + parse_duration_str(&dataset.get_str(&value_id).ok()??) + } + _ => None, + }) + } + xsd::YEAR_MONTH_DURATION => { + let e = self.expression_evaluator( + ¶meters[0], + encoded_variables, + stat_children, + ); + let dataset = Rc::clone(&self.dataset); + Rc::new(move |tuple| match e(tuple)? { + EncodedTerm::DurationLiteral(value) => { + Some(YearMonthDuration::try_from(value).ok()?.into()) + } + EncodedTerm::YearMonthDurationLiteral(value) => { + Some(value.into()) + } + EncodedTerm::SmallStringLiteral(value) => { + parse_year_month_duration_str(&value) + } + EncodedTerm::BigStringLiteral { value_id } => { + parse_year_month_duration_str( + &dataset.get_str(&value_id).ok()??, + ) + } + _ => None, + }) + } + xsd::DAY_TIME_DURATION => { + let e = self.expression_evaluator( + ¶meters[0], + encoded_variables, + stat_children, + ); + let dataset = Rc::clone(&self.dataset); + Rc::new(move |tuple| match e(tuple)? { + EncodedTerm::DurationLiteral(value) => { + Some(DayTimeDuration::try_from(value).ok()?.into()) + } + EncodedTerm::DayTimeDurationLiteral(value) => { + Some(value.into()) + } + EncodedTerm::SmallStringLiteral(value) => { + parse_day_time_duration_str(&value) + } + EncodedTerm::BigStringLiteral { value_id } => { + parse_day_time_duration_str( + &dataset.get_str(&value_id).ok()??, + ) + } + _ => None, + }) + } + _ => Rc::new(|_| None), + } } - _ => None, - }) - } - PlanExpression::CustomFunction(function_name, args) => { - if let Some(function) = self.custom_functions.get(function_name).cloned() { - let args = args - .iter() - .map(|e| self.expression_evaluator(e, stat_children)) - .collect::>(); - let dataset = Rc::clone(&self.dataset); - Rc::new(move |tuple| { - let args = args - .iter() - .map(|f| dataset.decode_term(&f(tuple)?).ok()) - .collect::>>()?; - Some(dataset.encode_term(&function(&args)?)) - }) - } else { - Rc::new(|_| None) } } } @@ -2073,10 +2769,11 @@ impl SimpleEvaluator { fn hash( &self, - arg: &PlanExpression, + parameters: &[Expression], + encoded_variables: &mut Vec, stat_children: &mut Vec>, ) -> Rc Option> { - let arg = self.expression_evaluator(arg, stat_children); + let arg = self.expression_evaluator(¶meters[0], encoded_variables, stat_children); let dataset = Rc::clone(&self.dataset); Rc::new(move |tuple| { let input = to_simple_string(&dataset, &arg(tuple)?)?; @@ -2084,6 +2781,114 @@ impl SimpleEvaluator { Some(build_string_literal(&dataset, &hash)) }) } + + fn encode_term<'b>(&self, term: impl Into>) -> EncodedTerm { + self.dataset.encode_term(term) + } + + fn encode_triple(&self, triple: &GroundTriple) -> EncodedTerm { + EncodedTriple::new( + match &triple.subject { + GroundSubject::NamedNode(node) => self.encode_term(node), + GroundSubject::Triple(triple) => self.encode_triple(triple), + }, + self.encode_term(&triple.predicate), + match &triple.object { + GroundTerm::NamedNode(node) => self.encode_term(node), + GroundTerm::Literal(literal) => self.encode_term(literal), + GroundTerm::Triple(triple) => self.encode_triple(triple), + }, + ) + .into() + } + + fn encode_property_path(&self, path: &PropertyPathExpression) -> Rc { + Rc::new(match path { + PropertyPathExpression::NamedNode(node) => PropertyPath::Path(self.encode_term(node)), + PropertyPathExpression::Reverse(p) => { + PropertyPath::Reverse(self.encode_property_path(p)) + } + PropertyPathExpression::Sequence(a, b) => { + PropertyPath::Sequence(self.encode_property_path(a), self.encode_property_path(b)) + } + PropertyPathExpression::Alternative(a, b) => PropertyPath::Alternative( + self.encode_property_path(a), + self.encode_property_path(b), + ), + PropertyPathExpression::ZeroOrMore(p) => { + PropertyPath::ZeroOrMore(self.encode_property_path(p)) + } + PropertyPathExpression::OneOrMore(p) => { + PropertyPath::OneOrMore(self.encode_property_path(p)) + } + PropertyPathExpression::ZeroOrOne(p) => { + PropertyPath::ZeroOrOne(self.encode_property_path(p)) + } + PropertyPathExpression::NegatedPropertySet(ps) => { + PropertyPath::NegatedPropertySet(ps.iter().map(|p| self.encode_term(p)).collect()) + } + }) + } + + fn template_value_from_term_or_variable( + &self, + term_or_variable: &TermPattern, + variables: &mut Vec, + bnodes: &mut Vec, + ) -> TripleTemplateValue { + match term_or_variable { + TermPattern::Variable(variable) => { + TripleTemplateValue::Variable(encode_variable(variables, variable)) + } + TermPattern::NamedNode(node) => TripleTemplateValue::Constant(self.encode_term(node)), + TermPattern::BlankNode(bnode) => { + TripleTemplateValue::BlankNode(bnode_key(bnodes, bnode)) + } + TermPattern::Literal(literal) => { + TripleTemplateValue::Constant(self.encode_term(literal)) + } + TermPattern::Triple(triple) => match ( + self.template_value_from_term_or_variable(&triple.subject, variables, bnodes), + self.template_value_from_named_node_or_variable(&triple.predicate, variables), + self.template_value_from_term_or_variable(&triple.object, variables, bnodes), + ) { + ( + TripleTemplateValue::Constant(subject), + TripleTemplateValue::Constant(predicate), + TripleTemplateValue::Constant(object), + ) => TripleTemplateValue::Constant( + EncodedTriple { + subject, + predicate, + object, + } + .into(), + ), + (subject, predicate, object) => { + TripleTemplateValue::Triple(Box::new(TripleTemplate { + subject, + predicate, + object, + })) + } + }, + } + } + + fn template_value_from_named_node_or_variable( + &self, + named_node_or_variable: &NamedNodePattern, + variables: &mut Vec, + ) -> TripleTemplateValue { + match named_node_or_variable { + NamedNodePattern::Variable(variable) => { + TripleTemplateValue::Variable(encode_variable(variables, variable)) + } + NamedNodePattern::NamedNode(term) => { + TripleTemplateValue::Constant(self.encode_term(term)) + } + } + } } fn to_bool(term: &EncodedTerm) -> Option { @@ -2293,6 +3098,31 @@ fn to_argument_compatible_strings( (language2.is_none() || language1 == language2).then_some((value1, value2, language1)) } +fn compile_static_pattern_if_exists( + pattern: &Expression, + options: Option<&Expression>, +) -> Option { + let static_pattern = if let Expression::Literal(pattern) = pattern { + (pattern.datatype() == xsd::STRING).then(|| pattern.value()) + } else { + None + }; + let static_options = if let Some(options) = options { + if let Expression::Literal(options) = options { + (options.datatype() == xsd::STRING).then(|| Some(options.value())) + } else { + None + } + } else { + Some(None) + }; + if let (Some(static_pattern), Some(static_options)) = (static_pattern, static_options) { + compile_pattern(static_pattern, static_options) + } else { + None + } +} + pub(super) fn compile_pattern(pattern: &str, flags: Option<&str>) -> Option { let mut regex_builder = RegexBuilder::new(pattern); regex_builder.size_limit(REGEX_SIZE_LIMIT); @@ -3008,32 +3838,86 @@ enum TupleSelector { TriplePattern(Rc), } -impl From<&PatternValue> for TupleSelector { - fn from(value: &PatternValue) -> Self { - match value { - PatternValue::Constant(c) => Self::Constant(c.encoded.clone()), - PatternValue::Variable(v) => Self::Variable(v.encoded), - PatternValue::TriplePattern(p) => Self::TriplePattern(Rc::new(TripleTupleSelector { - subject: (&p.subject).into(), - predicate: (&p.predicate).into(), - object: (&p.object).into(), - })), +impl TupleSelector { + fn from_ground_term_pattern( + term_pattern: &GroundTermPattern, + variables: &mut Vec, + dataset: &DatasetView, + ) -> Self { + match term_pattern { + GroundTermPattern::Variable(variable) => { + Self::Variable(encode_variable(variables, variable)) + } + GroundTermPattern::NamedNode(term) => Self::Constant(dataset.encode_term(term)), + GroundTermPattern::Literal(term) => Self::Constant(dataset.encode_term(term)), + GroundTermPattern::Triple(triple) => { + match ( + Self::from_ground_term_pattern(&triple.subject, variables, dataset), + Self::from_named_node_pattern(&triple.predicate, variables, dataset), + Self::from_ground_term_pattern(&triple.object, variables, dataset), + ) { + ( + Self::Constant(subject), + Self::Constant(predicate), + Self::Constant(object), + ) => Self::Constant( + EncodedTriple { + subject, + predicate, + object, + } + .into(), + ), + (subject, predicate, object) => { + Self::TriplePattern(Rc::new(TripleTupleSelector { + subject, + predicate, + object, + })) + } + } + } } } -} -fn get_pattern_value(selector: &TupleSelector, tuple: &EncodedTuple) -> Option { - match selector { - TupleSelector::Constant(c) => Some(c.clone()), - TupleSelector::Variable(v) => tuple.get(*v).cloned(), - TupleSelector::TriplePattern(triple) => Some( - EncodedTriple { - subject: get_pattern_value(&triple.subject, tuple)?, - predicate: get_pattern_value(&triple.predicate, tuple)?, - object: get_pattern_value(&triple.object, tuple)?, + fn from_named_node_pattern( + named_node_pattern: &NamedNodePattern, + variables: &mut Vec, + dataset: &DatasetView, + ) -> Self { + match named_node_pattern { + NamedNodePattern::Variable(variable) => { + Self::Variable(encode_variable(variables, variable)) } - .into(), - ), + NamedNodePattern::NamedNode(term) => Self::Constant(dataset.encode_term(term)), + } + } + + fn from_graph_name_pattern( + graph_name_pattern: &Option, + variables: &mut Vec, + dataset: &DatasetView, + ) -> Self { + if let Some(graph_name_pattern) = graph_name_pattern { + Self::from_named_node_pattern(graph_name_pattern, variables, dataset) + } else { + Self::Constant(EncodedTerm::DefaultGraph) + } + } + + fn get_pattern_value(&self, tuple: &EncodedTuple) -> Option { + match self { + TupleSelector::Constant(c) => Some(c.clone()), + TupleSelector::Variable(v) => tuple.get(*v).cloned(), + TupleSelector::TriplePattern(triple) => Some( + EncodedTriple { + subject: triple.subject.get_pattern_value(tuple)?, + predicate: triple.predicate.get_pattern_value(tuple)?, + object: triple.object.get_pattern_value(tuple)?, + } + .into(), + ), + } } } @@ -3097,6 +3981,17 @@ pub fn are_compatible_and_not_disjointed(a: &EncodedTuple, b: &EncodedTuple) -> found_intersection } +pub enum PropertyPath { + Path(EncodedTerm), + Reverse(Rc), + Sequence(Rc, Rc), + Alternative(Rc, Rc), + ZeroOrMore(Rc), + OneOrMore(Rc), + ZeroOrOne(Rc), + NegatedPropertySet(Rc<[EncodedTerm]>), +} + #[derive(Clone)] struct PathEvaluator { dataset: Rc, @@ -3105,25 +4000,20 @@ struct PathEvaluator { impl PathEvaluator { fn eval_closed_in_graph( &self, - path: &PlanPropertyPath, + path: &PropertyPath, start: &EncodedTerm, end: &EncodedTerm, graph_name: &EncodedTerm, ) -> Result { Ok(match path { - PlanPropertyPath::Path(p) => self + PropertyPath::Path(p) => self .dataset - .encoded_quads_for_pattern( - Some(start), - Some(&p.encoded), - Some(end), - Some(graph_name), - ) + .encoded_quads_for_pattern(Some(start), Some(p), Some(end), Some(graph_name)) .next() .transpose()? .is_some(), - PlanPropertyPath::Reverse(p) => self.eval_closed_in_graph(p, end, start, graph_name)?, - PlanPropertyPath::Sequence(a, b) => self + PropertyPath::Reverse(p) => self.eval_closed_in_graph(p, end, start, graph_name)?, + PropertyPath::Sequence(a, b) => self .eval_from_in_graph(a, start, graph_name) .find_map(|middle| { middle @@ -3136,11 +4026,11 @@ impl PathEvaluator { }) .transpose()? .is_some(), - PlanPropertyPath::Alternative(a, b) => { + PropertyPath::Alternative(a, b) => { self.eval_closed_in_graph(a, start, end, graph_name)? || self.eval_closed_in_graph(b, start, end, graph_name)? } - PlanPropertyPath::ZeroOrMore(p) => { + PropertyPath::ZeroOrMore(p) => { if start == end { self.is_subject_or_object_in_graph(start, graph_name)? } else { @@ -3151,24 +4041,24 @@ impl PathEvaluator { )? } } - PlanPropertyPath::OneOrMore(p) => look_in_transitive_closure( + PropertyPath::OneOrMore(p) => look_in_transitive_closure( self.eval_from_in_graph(p, start, graph_name), move |e| self.eval_from_in_graph(p, &e, graph_name), end, )?, - PlanPropertyPath::ZeroOrOne(p) => { + PropertyPath::ZeroOrOne(p) => { if start == end { self.is_subject_or_object_in_graph(start, graph_name) } else { self.eval_closed_in_graph(p, start, end, graph_name) }? } - PlanPropertyPath::NegatedPropertySet(ps) => self + PropertyPath::NegatedPropertySet(ps) => self .dataset .encoded_quads_for_pattern(Some(start), None, Some(end), Some(graph_name)) .find_map(move |t| match t { Ok(t) => { - if ps.iter().any(|p| p.encoded == t.predicate) { + if ps.iter().any(|p| *p == t.predicate) { None } else { Some(Ok(())) @@ -3183,18 +4073,18 @@ impl PathEvaluator { fn eval_closed_in_unknown_graph( &self, - path: &PlanPropertyPath, + path: &PropertyPath, start: &EncodedTerm, end: &EncodedTerm, ) -> Box>> { match path { - PlanPropertyPath::Path(p) => Box::new( + PropertyPath::Path(p) => Box::new( self.dataset - .encoded_quads_for_pattern(Some(start), Some(&p.encoded), Some(end), None) + .encoded_quads_for_pattern(Some(start), Some(p), Some(end), None) .map(|t| Ok(t?.graph_name)), ), - PlanPropertyPath::Reverse(p) => self.eval_closed_in_unknown_graph(p, end, start), - PlanPropertyPath::Sequence(a, b) => { + PropertyPath::Reverse(p) => self.eval_closed_in_unknown_graph(p, end, start), + PropertyPath::Sequence(a, b) => { let eval = self.clone(); let b = Rc::clone(b); let end = end.clone(); @@ -3206,11 +4096,11 @@ impl PathEvaluator { }, )) } - PlanPropertyPath::Alternative(a, b) => Box::new(hash_deduplicate( + PropertyPath::Alternative(a, b) => Box::new(hash_deduplicate( self.eval_closed_in_unknown_graph(a, start, end) .chain(self.eval_closed_in_unknown_graph(b, start, end)), )), - PlanPropertyPath::ZeroOrMore(p) => { + PropertyPath::ZeroOrMore(p) => { let eval = self.clone(); let start2 = start.clone(); let end = end.clone(); @@ -3225,7 +4115,7 @@ impl PathEvaluator { .transpose() }) } - PlanPropertyPath::OneOrMore(p) => { + PropertyPath::OneOrMore(p) => { let eval = self.clone(); let end = end.clone(); let p = Rc::clone(p); @@ -3244,7 +4134,7 @@ impl PathEvaluator { }), ) } - PlanPropertyPath::ZeroOrOne(p) => { + PropertyPath::ZeroOrOne(p) => { if start == end { self.run_if_term_is_a_dataset_node(start, |graph_name| Some(Ok(graph_name))) } else { @@ -3259,14 +4149,14 @@ impl PathEvaluator { }) } } - PlanPropertyPath::NegatedPropertySet(ps) => { + PropertyPath::NegatedPropertySet(ps) => { let ps = Rc::clone(ps); Box::new( self.dataset .encoded_quads_for_pattern(Some(start), None, Some(end), None) .filter_map(move |t| match t { Ok(t) => { - if ps.iter().any(|p| p.encoded == t.predicate) { + if ps.iter().any(|p| *p == t.predicate) { None } else { Some(Ok(t.graph_name)) @@ -3281,23 +4171,18 @@ impl PathEvaluator { fn eval_from_in_graph( &self, - path: &PlanPropertyPath, + path: &PropertyPath, start: &EncodedTerm, graph_name: &EncodedTerm, ) -> Box>> { match path { - PlanPropertyPath::Path(p) => Box::new( + PropertyPath::Path(p) => Box::new( self.dataset - .encoded_quads_for_pattern( - Some(start), - Some(&p.encoded), - None, - Some(graph_name), - ) + .encoded_quads_for_pattern(Some(start), Some(p), None, Some(graph_name)) .map(|t| Ok(t?.object)), ), - PlanPropertyPath::Reverse(p) => self.eval_to_in_graph(p, start, graph_name), - PlanPropertyPath::Sequence(a, b) => { + PropertyPath::Reverse(p) => self.eval_to_in_graph(p, start, graph_name), + PropertyPath::Sequence(a, b) => { let eval = self.clone(); let b = Rc::clone(b); let graph_name2 = graph_name.clone(); @@ -3308,11 +4193,11 @@ impl PathEvaluator { }), ) } - PlanPropertyPath::Alternative(a, b) => Box::new(hash_deduplicate( + PropertyPath::Alternative(a, b) => Box::new(hash_deduplicate( self.eval_from_in_graph(a, start, graph_name) .chain(self.eval_from_in_graph(b, start, graph_name)), )), - PlanPropertyPath::ZeroOrMore(p) => { + PropertyPath::ZeroOrMore(p) => { self.run_if_term_is_a_graph_node(start, graph_name, || { let eval = self.clone(); let p = Rc::clone(p); @@ -3322,7 +4207,7 @@ impl PathEvaluator { }) }) } - PlanPropertyPath::OneOrMore(p) => { + PropertyPath::OneOrMore(p) => { let eval = self.clone(); let p = Rc::clone(p); let graph_name2 = graph_name.clone(); @@ -3331,7 +4216,7 @@ impl PathEvaluator { move |e| eval.eval_from_in_graph(&p, &e, &graph_name2), )) } - PlanPropertyPath::ZeroOrOne(p) => { + PropertyPath::ZeroOrOne(p) => { self.run_if_term_is_a_graph_node(start, graph_name, || { hash_deduplicate( once(Ok(start.clone())) @@ -3339,14 +4224,14 @@ impl PathEvaluator { ) }) } - PlanPropertyPath::NegatedPropertySet(ps) => { + PropertyPath::NegatedPropertySet(ps) => { let ps = Rc::clone(ps); Box::new( self.dataset .encoded_quads_for_pattern(Some(start), None, None, Some(graph_name)) .filter_map(move |t| match t { Ok(t) => { - if ps.iter().any(|p| p.encoded == t.predicate) { + if ps.iter().any(|p| *p == t.predicate) { None } else { Some(Ok(t.object)) @@ -3361,20 +4246,20 @@ impl PathEvaluator { fn eval_from_in_unknown_graph( &self, - path: &PlanPropertyPath, + path: &PropertyPath, start: &EncodedTerm, ) -> Box>> { match path { - PlanPropertyPath::Path(p) => Box::new( + PropertyPath::Path(p) => Box::new( self.dataset - .encoded_quads_for_pattern(Some(start), Some(&p.encoded), None, None) + .encoded_quads_for_pattern(Some(start), Some(p), None, None) .map(|t| { let t = t?; Ok((t.object, t.graph_name)) }), ), - PlanPropertyPath::Reverse(p) => self.eval_to_in_unknown_graph(p, start), - PlanPropertyPath::Sequence(a, b) => { + PropertyPath::Reverse(p) => self.eval_to_in_unknown_graph(p, start), + PropertyPath::Sequence(a, b) => { let eval = self.clone(); let b = Rc::clone(b); Box::new(self.eval_from_in_unknown_graph(a, start).flat_map_ok( @@ -3384,11 +4269,11 @@ impl PathEvaluator { }, )) } - PlanPropertyPath::Alternative(a, b) => Box::new(hash_deduplicate( + PropertyPath::Alternative(a, b) => Box::new(hash_deduplicate( self.eval_from_in_unknown_graph(a, start) .chain(self.eval_from_in_unknown_graph(b, start)), )), - PlanPropertyPath::ZeroOrMore(p) => { + PropertyPath::ZeroOrMore(p) => { let start2 = start.clone(); let eval = self.clone(); let p = Rc::clone(p); @@ -3402,7 +4287,7 @@ impl PathEvaluator { .map(move |e| Ok((e?, graph_name.clone()))) }) } - PlanPropertyPath::OneOrMore(p) => { + PropertyPath::OneOrMore(p) => { let eval = self.clone(); let p = Rc::clone(p); Box::new(transitive_closure( @@ -3413,7 +4298,7 @@ impl PathEvaluator { }, )) } - PlanPropertyPath::ZeroOrOne(p) => { + PropertyPath::ZeroOrOne(p) => { let eval = self.clone(); let start2 = start.clone(); let p = Rc::clone(p); @@ -3426,14 +4311,14 @@ impl PathEvaluator { .map(move |e| Ok((e?, graph_name.clone()))) }) } - PlanPropertyPath::NegatedPropertySet(ps) => { + PropertyPath::NegatedPropertySet(ps) => { let ps = Rc::clone(ps); Box::new( self.dataset .encoded_quads_for_pattern(Some(start), None, None, None) .filter_map(move |t| match t { Ok(t) => { - if ps.iter().any(|p| p.encoded == t.predicate) { + if ps.iter().any(|p| *p == t.predicate) { None } else { Some(Ok((t.object, t.graph_name))) @@ -3448,18 +4333,18 @@ impl PathEvaluator { fn eval_to_in_graph( &self, - path: &PlanPropertyPath, + path: &PropertyPath, end: &EncodedTerm, graph_name: &EncodedTerm, ) -> Box>> { match path { - PlanPropertyPath::Path(p) => Box::new( + PropertyPath::Path(p) => Box::new( self.dataset - .encoded_quads_for_pattern(None, Some(&p.encoded), Some(end), Some(graph_name)) + .encoded_quads_for_pattern(None, Some(p), Some(end), Some(graph_name)) .map(|t| Ok(t?.subject)), ), - PlanPropertyPath::Reverse(p) => self.eval_from_in_graph(p, end, graph_name), - PlanPropertyPath::Sequence(a, b) => { + PropertyPath::Reverse(p) => self.eval_from_in_graph(p, end, graph_name), + PropertyPath::Sequence(a, b) => { let eval = self.clone(); let a = Rc::clone(a); let graph_name2 = graph_name.clone(); @@ -3470,11 +4355,11 @@ impl PathEvaluator { }), ) } - PlanPropertyPath::Alternative(a, b) => Box::new(hash_deduplicate( + PropertyPath::Alternative(a, b) => Box::new(hash_deduplicate( self.eval_to_in_graph(a, end, graph_name) .chain(self.eval_to_in_graph(b, end, graph_name)), )), - PlanPropertyPath::ZeroOrMore(p) => { + PropertyPath::ZeroOrMore(p) => { self.run_if_term_is_a_graph_node(end, graph_name, || { let eval = self.clone(); let p = Rc::clone(p); @@ -3484,7 +4369,7 @@ impl PathEvaluator { }) }) } - PlanPropertyPath::OneOrMore(p) => { + PropertyPath::OneOrMore(p) => { let eval = self.clone(); let p = Rc::clone(p); let graph_name2 = graph_name.clone(); @@ -3493,21 +4378,19 @@ impl PathEvaluator { move |e| eval.eval_to_in_graph(&p, &e, &graph_name2), )) } - PlanPropertyPath::ZeroOrOne(p) => { - self.run_if_term_is_a_graph_node(end, graph_name, || { - hash_deduplicate( - once(Ok(end.clone())).chain(self.eval_to_in_graph(p, end, graph_name)), - ) - }) - } - PlanPropertyPath::NegatedPropertySet(ps) => { + PropertyPath::ZeroOrOne(p) => self.run_if_term_is_a_graph_node(end, graph_name, || { + hash_deduplicate( + once(Ok(end.clone())).chain(self.eval_to_in_graph(p, end, graph_name)), + ) + }), + PropertyPath::NegatedPropertySet(ps) => { let ps = Rc::clone(ps); Box::new( self.dataset .encoded_quads_for_pattern(None, None, Some(end), Some(graph_name)) .filter_map(move |t| match t { Ok(t) => { - if ps.iter().any(|p| p.encoded == t.predicate) { + if ps.iter().any(|p| *p == t.predicate) { None } else { Some(Ok(t.subject)) @@ -3521,20 +4404,20 @@ impl PathEvaluator { } fn eval_to_in_unknown_graph( &self, - path: &PlanPropertyPath, + path: &PropertyPath, end: &EncodedTerm, ) -> Box>> { match path { - PlanPropertyPath::Path(p) => Box::new( + PropertyPath::Path(p) => Box::new( self.dataset - .encoded_quads_for_pattern(None, Some(&p.encoded), Some(end), None) + .encoded_quads_for_pattern(None, Some(p), Some(end), None) .map(|t| { let t = t?; Ok((t.subject, t.graph_name)) }), ), - PlanPropertyPath::Reverse(p) => self.eval_from_in_unknown_graph(p, end), - PlanPropertyPath::Sequence(a, b) => { + PropertyPath::Reverse(p) => self.eval_from_in_unknown_graph(p, end), + PropertyPath::Sequence(a, b) => { let eval = self.clone(); let a = Rc::clone(a); Box::new(self.eval_to_in_unknown_graph(b, end).flat_map_ok( @@ -3544,11 +4427,11 @@ impl PathEvaluator { }, )) } - PlanPropertyPath::Alternative(a, b) => Box::new(hash_deduplicate( + PropertyPath::Alternative(a, b) => Box::new(hash_deduplicate( self.eval_to_in_unknown_graph(a, end) .chain(self.eval_to_in_unknown_graph(b, end)), )), - PlanPropertyPath::ZeroOrMore(p) => { + PropertyPath::ZeroOrMore(p) => { let end2 = end.clone(); let eval = self.clone(); let p = Rc::clone(p); @@ -3562,7 +4445,7 @@ impl PathEvaluator { .map(move |e| Ok((e?, graph_name.clone()))) }) } - PlanPropertyPath::OneOrMore(p) => { + PropertyPath::OneOrMore(p) => { let eval = self.clone(); let p = Rc::clone(p); Box::new(transitive_closure( @@ -3573,7 +4456,7 @@ impl PathEvaluator { }, )) } - PlanPropertyPath::ZeroOrOne(p) => { + PropertyPath::ZeroOrOne(p) => { let eval = self.clone(); let end2 = end.clone(); let p = Rc::clone(p); @@ -3586,14 +4469,14 @@ impl PathEvaluator { .map(move |e| Ok((e?, graph_name.clone()))) }) } - PlanPropertyPath::NegatedPropertySet(ps) => { + PropertyPath::NegatedPropertySet(ps) => { let ps = Rc::clone(ps); Box::new( self.dataset .encoded_quads_for_pattern(Some(end), None, None, None) .filter_map(move |t| match t { Ok(t) => { - if ps.iter().any(|p| p.encoded == t.predicate) { + if ps.iter().any(|p| *p == t.predicate) { None } else { Some(Ok((t.subject, t.graph_name))) @@ -3608,20 +4491,20 @@ impl PathEvaluator { fn eval_open_in_graph( &self, - path: &PlanPropertyPath, + path: &PropertyPath, graph_name: &EncodedTerm, ) -> Box>> { match path { - PlanPropertyPath::Path(p) => Box::new( + PropertyPath::Path(p) => Box::new( self.dataset - .encoded_quads_for_pattern(None, Some(&p.encoded), None, Some(graph_name)) + .encoded_quads_for_pattern(None, Some(p), None, Some(graph_name)) .map(|t| t.map(|t| (t.subject, t.object))), ), - PlanPropertyPath::Reverse(p) => Box::new( + PropertyPath::Reverse(p) => Box::new( self.eval_open_in_graph(p, graph_name) .map(|t| t.map(|(s, o)| (o, s))), ), - PlanPropertyPath::Sequence(a, b) => { + PropertyPath::Sequence(a, b) => { let eval = self.clone(); let b = Rc::clone(b); let graph_name2 = graph_name.clone(); @@ -3632,11 +4515,11 @@ impl PathEvaluator { }, )) } - PlanPropertyPath::Alternative(a, b) => Box::new(hash_deduplicate( + PropertyPath::Alternative(a, b) => Box::new(hash_deduplicate( self.eval_open_in_graph(a, graph_name) .chain(self.eval_open_in_graph(b, graph_name)), )), - PlanPropertyPath::ZeroOrMore(p) => { + PropertyPath::ZeroOrMore(p) => { let eval = self.clone(); let p = Rc::clone(p); let graph_name2 = graph_name.clone(); @@ -3648,7 +4531,7 @@ impl PathEvaluator { }, )) } - PlanPropertyPath::OneOrMore(p) => { + PropertyPath::OneOrMore(p) => { let eval = self.clone(); let p = Rc::clone(p); let graph_name2 = graph_name.clone(); @@ -3660,18 +4543,18 @@ impl PathEvaluator { }, )) } - PlanPropertyPath::ZeroOrOne(p) => Box::new(hash_deduplicate( + PropertyPath::ZeroOrOne(p) => Box::new(hash_deduplicate( self.get_subject_or_object_identity_pairs_in_graph(graph_name) .chain(self.eval_open_in_graph(p, graph_name)), )), - PlanPropertyPath::NegatedPropertySet(ps) => { + PropertyPath::NegatedPropertySet(ps) => { let ps = Rc::clone(ps); Box::new( self.dataset .encoded_quads_for_pattern(None, None, None, Some(graph_name)) .filter_map(move |t| match t { Ok(t) => { - if ps.iter().any(|p| p.encoded == t.predicate) { + if ps.iter().any(|p| *p == t.predicate) { None } else { Some(Ok((t.subject, t.object))) @@ -3686,20 +4569,20 @@ impl PathEvaluator { fn eval_open_in_unknown_graph( &self, - path: &PlanPropertyPath, + path: &PropertyPath, ) -> Box>> { match path { - PlanPropertyPath::Path(p) => Box::new( + PropertyPath::Path(p) => Box::new( self.dataset - .encoded_quads_for_pattern(None, Some(&p.encoded), None, None) + .encoded_quads_for_pattern(None, Some(p), None, None) .map(|t| t.map(|t| (t.subject, t.object, t.graph_name))), ), - PlanPropertyPath::Reverse(p) => Box::new( + PropertyPath::Reverse(p) => Box::new( self.eval_open_in_unknown_graph(p) .map(|t| t.map(|(s, o, g)| (o, s, g))), ), - PlanPropertyPath::Sequence(a, b) => { + PropertyPath::Sequence(a, b) => { let eval = self.clone(); let b = Rc::clone(b); Box::new(self.eval_open_in_unknown_graph(a).flat_map_ok( @@ -3709,11 +4592,11 @@ impl PathEvaluator { }, )) } - PlanPropertyPath::Alternative(a, b) => Box::new(hash_deduplicate( + PropertyPath::Alternative(a, b) => Box::new(hash_deduplicate( self.eval_open_in_unknown_graph(a) .chain(self.eval_open_in_unknown_graph(b)), )), - PlanPropertyPath::ZeroOrMore(p) => { + PropertyPath::ZeroOrMore(p) => { let eval = self.clone(); let p = Rc::clone(p); Box::new(transitive_closure( @@ -3724,7 +4607,7 @@ impl PathEvaluator { }, )) } - PlanPropertyPath::OneOrMore(p) => { + PropertyPath::OneOrMore(p) => { let eval = self.clone(); let p = Rc::clone(p); Box::new(transitive_closure( @@ -3735,18 +4618,18 @@ impl PathEvaluator { }, )) } - PlanPropertyPath::ZeroOrOne(p) => Box::new(hash_deduplicate( + PropertyPath::ZeroOrOne(p) => Box::new(hash_deduplicate( self.get_subject_or_object_identity_pairs_in_dataset() .chain(self.eval_open_in_unknown_graph(p)), )), - PlanPropertyPath::NegatedPropertySet(ps) => { + PropertyPath::NegatedPropertySet(ps) => { let ps = Rc::clone(ps); Box::new( self.dataset .encoded_quads_for_pattern(None, None, None, None) .filter_map(move |t| match t { Ok(t) => { - if ps.iter().any(|p| p.encoded == t.predicate) { + if ps.iter().any(|p| *p == t.predicate) { None } else { Some(Ok((t.subject, t.object, t.graph_name))) @@ -4114,19 +4997,32 @@ impl Iterator for ConstructIterator { } } +pub struct TripleTemplate { + pub subject: TripleTemplateValue, + pub predicate: TripleTemplateValue, + pub object: TripleTemplateValue, +} + +pub enum TripleTemplateValue { + Constant(EncodedTerm), + BlankNode(usize), + Variable(usize), + Triple(Box), +} + fn get_triple_template_value<'a>( selector: &'a TripleTemplateValue, tuple: &'a EncodedTuple, bnodes: &'a mut Vec, ) -> Option { match selector { - TripleTemplateValue::Constant(term) => Some(term.encoded.clone()), - TripleTemplateValue::Variable(v) => tuple.get(v.encoded).cloned(), + TripleTemplateValue::Constant(term) => Some(term.clone()), + TripleTemplateValue::Variable(v) => tuple.get(*v).cloned(), TripleTemplateValue::BlankNode(bnode) => { - if bnode.encoded >= bnodes.len() { - bnodes.resize_with(bnode.encoded + 1, new_bnode) + if *bnode >= bnodes.len() { + bnodes.resize_with(*bnode + 1, new_bnode) } - Some(bnodes[bnode.encoded].clone()) + Some(bnodes[*bnode].clone()) } TripleTemplateValue::Triple(triple) => Some( EncodedTriple { @@ -4403,7 +5299,6 @@ impl Accumulator for CountAccumulator { } } -#[derive(Debug)] struct SumAccumulator { sum: Option, } @@ -4447,7 +5342,7 @@ impl Accumulator for SumAccumulator { } } -#[derive(Debug, Default)] +#[derive(Default)] struct AvgAccumulator { sum: SumAccumulator, count: i64, @@ -4536,7 +5431,7 @@ impl Accumulator for MaxAccumulator { } } -#[derive(Debug, Default)] +#[derive(Default)] struct SampleAccumulator { value: Option, } @@ -4598,6 +5493,43 @@ impl Accumulator for GroupConcatAccumulator { } } +struct FailingAccumulator; + +impl Accumulator for FailingAccumulator { + fn add(&mut self, _: Option) {} + + fn state(&self) -> Option { + None + } +} + +fn encode_variable(variables: &mut Vec, variable: &Variable) -> usize { + if let Some(key) = slice_key(variables, variable) { + key + } else { + variables.push(variable.clone()); + variables.len() - 1 + } +} + +fn bnode_key(blank_nodes: &mut Vec, blank_node: &BlankNode) -> usize { + if let Some(key) = slice_key(blank_nodes, blank_node) { + key + } else { + blank_nodes.push(blank_node.clone()); + blank_nodes.len() - 1 + } +} + +fn slice_key(slice: &[T], element: &T) -> Option { + for (i, item) in slice.iter().enumerate() { + if item == element { + return Some(i); + } + } + None +} + fn generate_uuid(buffer: &mut String) { let mut uuid = random::().to_ne_bytes(); uuid[6] = (uuid[6] & 0x0F) | 0x40; @@ -4775,96 +5707,143 @@ impl fmt::Debug for EvalNodeWithStats { } } -fn eval_node_label(node: &PlanNode) -> String { +fn eval_node_label(node: &GraphPattern) -> String { match node { - PlanNode::Aggregate { - key_variables, - aggregates, + GraphPattern::Distinct { .. } => "Distinct(Hash)".to_owned(), + GraphPattern::Extend { + expression, + variable, .. } => format!( - "Aggregate({})", - key_variables - .iter() - .map(ToString::to_string) - .chain(aggregates.iter().map(|(agg, v)| format!("{agg} -> {v}"))) - .collect::>() - .join(", ") + "Extend({} -> {variable})", + spargebra::algebra::Expression::from(expression) + ), + GraphPattern::Filter { expression, .. } => format!( + "Filter({})", + spargebra::algebra::Expression::from(expression) ), - PlanNode::AntiJoin { .. } => "AntiJoin".to_owned(), - PlanNode::Extend { + GraphPattern::Group { + variables, + aggregates, + .. + } => { + format!( + "Aggregate({})", + format_list(variables.iter().map(ToString::to_string).chain( + aggregates.iter().map(|(v, agg)| format!( + "{} -> {v}", + spargebra::algebra::AggregateExpression::from(agg) + )) + )) + ) + } + GraphPattern::Join { algorithm, .. } => match algorithm { + JoinAlgorithm::HashBuildLeftProbeRight { keys } => format!( + "LeftJoin(HashBuildLeftProbeRight, keys = {})", + format_list(keys) + ), + }, + GraphPattern::Lateral { right, .. } => { + if let GraphPattern::LeftJoin { + left: nested_left, + expression, + .. + } = right.as_ref() + { + if nested_left.is_empty_singleton() { + // We are in a ForLoopLeftJoin + return format!( + "ForLoopLeftJoin(expression = {})", + spargebra::algebra::Expression::from(expression) + ); + } + } + "Lateral".to_owned() + } + GraphPattern::LeftJoin { + algorithm, expression, - variable, .. - } => format!("Extend({expression} -> {variable})"), - PlanNode::Filter { expression, .. } => format!("Filter({expression})"), - PlanNode::ForLoopJoin { .. } => "ForLoopJoin".to_owned(), - PlanNode::ForLoopLeftJoin { .. } => "ForLoopLeftJoin".to_owned(), - PlanNode::HashDeduplicate { .. } => "HashDeduplicate".to_owned(), - PlanNode::HashJoin { .. } => "HashJoin".to_owned(), - PlanNode::HashLeftJoin { expression, .. } => format!("HashLeftJoin({expression})"), - PlanNode::Limit { count, .. } => format!("Limit({count})"), - PlanNode::PathPattern { + } => match algorithm { + LeftJoinAlgorithm::HashBuildRightProbeLeft { keys } => format!( + "LeftJoin(HashBuildRightProbeLeft, keys = {}, expression = {})", + format_list(keys), + spargebra::algebra::Expression::from(expression) + ), + }, + GraphPattern::Minus { algorithm, .. } => match algorithm { + MinusAlgorithm::HashBuildRightProbeLeft { keys } => format!( + "AntiJoin(HashBuildRightProbeLeft, keys = {})", + format_list(keys) + ), + }, + GraphPattern::OrderBy { expression, .. } => { + format!( + "Sort({})", + format_list( + expression + .iter() + .map(spargebra::algebra::OrderExpression::from) + ) + ) + } + GraphPattern::Path { subject, path, object, graph_name, - } => format!("PathPattern({subject} {path} {object} {graph_name})"), - PlanNode::Project { mapping, .. } => { - format!( - "Project({})", - mapping - .iter() - .map(|(f, t)| if f.plain == t.plain { - f.to_string() - } else { - format!("{f} -> {t}") - }) - .collect::>() - .join(", ") - ) + } => { + if let Some(graph_name) = graph_name { + format!("Path({subject} {path} {object} {graph_name})") + } else { + format!("Path({subject} {path} {object})") + } } - PlanNode::QuadPattern { + GraphPattern::Project { variables, .. } => { + format!("Project({})", format_list(variables)) + } + GraphPattern::QuadPattern { subject, predicate, object, graph_name, - } => format!("QuadPattern({subject} {predicate} {object} {graph_name})"), - PlanNode::Reduced { .. } => "Reduced".to_owned(), - PlanNode::Service { - service_name, - silent, - .. } => { + if let Some(graph_name) = graph_name { + format!("QuadPattern({subject} {predicate} {object} {graph_name})") + } else { + format!("QuadPattern({subject} {predicate} {object})") + } + } + GraphPattern::Reduced { .. } => "Reduced".to_owned(), + GraphPattern::Service { name, silent, .. } => { if *silent { - format!("SilentService({service_name})") + format!("Service({name}, Silent)") } else { - format!("Service({service_name})") + format!("Service({name})") } } - PlanNode::Skip { count, .. } => format!("Skip({count})"), - PlanNode::Sort { by, .. } => { - format!( - "Sort({})", - by.iter() - .map(ToString::to_string) - .collect::>() - .join(", ") - ) + GraphPattern::Slice { start, length, .. } => { + if let Some(length) = length { + format!("Slice(start = {start}, length = {length})") + } else { + format!("Slice(start = {start})") + } } - PlanNode::StaticBindings { variables, .. } => { - format!( - "StaticBindings({})", - variables - .iter() - .map(ToString::to_string) - .collect::>() - .join(", ") - ) + GraphPattern::Union { .. } => "Union".to_owned(), + GraphPattern::Values { variables, .. } => { + format!("StaticBindings({})", format_list(variables)) } - PlanNode::Union { .. } => "Union".to_owned(), } } +fn format_list(values: impl IntoIterator) -> String { + values + .into_iter() + .map(|v| v.to_string()) + .collect::>() + .join(", ") +} + #[cfg(all(target_family = "wasm", target_os = "unknown"))] pub struct Timer { timestamp_ms: f64, diff --git a/lib/src/sparql/mod.rs b/lib/src/sparql/mod.rs index e66e114b..d5e7233e 100644 --- a/lib/src/sparql/mod.rs +++ b/lib/src/sparql/mod.rs @@ -8,8 +8,6 @@ mod error; mod eval; mod http; mod model; -mod plan; -mod plan_builder; mod service; mod update; @@ -19,7 +17,6 @@ use crate::sparql::dataset::DatasetView; pub use crate::sparql::error::{EvaluationError, QueryError}; use crate::sparql::eval::{EvalNodeWithStats, SimpleEvaluator, Timer}; pub use crate::sparql::model::{QueryResults, QuerySolution, QuerySolutionIter, QueryTripleIter}; -use crate::sparql::plan_builder::PlanBuilder; pub use crate::sparql::service::ServiceHandler; use crate::sparql::service::{EmptyServiceHandler, ErrorConversionServiceHandler}; pub(crate) use crate::sparql::update::evaluate_update; @@ -28,6 +25,8 @@ use json_event_parser::{JsonEvent, JsonWriter}; pub use oxrdf::{Variable, VariableNameParseError}; pub use sparesults::QueryResultsFormat; pub use spargebra::ParseError; +use sparopt::algebra::GraphPattern; +use sparopt::Optimizer; use std::collections::HashMap; use std::rc::Rc; use std::time::Duration; @@ -47,13 +46,10 @@ pub(crate) fn evaluate_query( spargebra::Query::Select { pattern, base_iri, .. } => { - let (plan, variables) = PlanBuilder::build( - &dataset, - &pattern, - true, - &options.custom_functions, - options.without_optimizations, - )?; + let mut pattern = GraphPattern::from(&pattern); + if !options.without_optimizations { + pattern = Optimizer::optimize_graph_pattern(pattern); + } let planning_duration = start_planning.elapsed(); let (results, explanation) = SimpleEvaluator::new( Rc::new(dataset), @@ -62,19 +58,18 @@ pub(crate) fn evaluate_query( Rc::new(options.custom_functions), run_stats, ) - .evaluate_select_plan(&plan, Rc::new(variables)); + .evaluate_select(&pattern); (Ok(results), explanation, planning_duration) } spargebra::Query::Ask { pattern, base_iri, .. } => { - let (plan, _) = PlanBuilder::build( - &dataset, - &pattern, - false, - &options.custom_functions, - options.without_optimizations, - )?; + let mut pattern = GraphPattern::from(&pattern); + if !options.without_optimizations { + pattern = Optimizer::optimize_graph_pattern(GraphPattern::Reduced { + inner: Box::new(pattern), + }); + } let planning_duration = start_planning.elapsed(); let (results, explanation) = SimpleEvaluator::new( Rc::new(dataset), @@ -83,7 +78,7 @@ pub(crate) fn evaluate_query( Rc::new(options.custom_functions), run_stats, ) - .evaluate_ask_plan(&plan); + .evaluate_ask(&pattern); (results, explanation, planning_duration) } spargebra::Query::Construct { @@ -92,19 +87,12 @@ pub(crate) fn evaluate_query( base_iri, .. } => { - let (plan, variables) = PlanBuilder::build( - &dataset, - &pattern, - false, - &options.custom_functions, - options.without_optimizations, - )?; - let construct = PlanBuilder::build_graph_template( - &dataset, - &template, - variables, - &options.custom_functions, - ); + let mut pattern = GraphPattern::from(&pattern); + if !options.without_optimizations { + pattern = Optimizer::optimize_graph_pattern(GraphPattern::Reduced { + inner: Box::new(pattern), + }); + } let planning_duration = start_planning.elapsed(); let (results, explanation) = SimpleEvaluator::new( Rc::new(dataset), @@ -113,19 +101,18 @@ pub(crate) fn evaluate_query( Rc::new(options.custom_functions), run_stats, ) - .evaluate_construct_plan(&plan, construct); + .evaluate_construct(&pattern, &template); (Ok(results), explanation, planning_duration) } spargebra::Query::Describe { pattern, base_iri, .. } => { - let (plan, _) = PlanBuilder::build( - &dataset, - &pattern, - false, - &options.custom_functions, - options.without_optimizations, - )?; + let mut pattern = GraphPattern::from(&pattern); + if !options.without_optimizations { + pattern = Optimizer::optimize_graph_pattern(GraphPattern::Reduced { + inner: Box::new(pattern), + }); + } let planning_duration = start_planning.elapsed(); let (results, explanation) = SimpleEvaluator::new( Rc::new(dataset), @@ -134,7 +121,7 @@ pub(crate) fn evaluate_query( Rc::new(options.custom_functions), run_stats, ) - .evaluate_describe_plan(&plan); + .evaluate_describe(&pattern); (Ok(results), explanation, planning_duration) } }; diff --git a/lib/src/sparql/plan.rs b/lib/src/sparql/plan.rs deleted file mode 100644 index fd9fb989..00000000 --- a/lib/src/sparql/plan.rs +++ /dev/null @@ -1,896 +0,0 @@ -use crate::model::{BlankNode, Literal, NamedNode, Term, Triple}; -use crate::sparql::Variable; -use crate::storage::numeric_encoder::EncodedTerm; -use regex::Regex; -use spargebra::algebra::GraphPattern; -use spargebra::term::GroundTerm; -use std::collections::BTreeSet; -use std::fmt; -use std::rc::Rc; - -#[derive(Debug, Clone)] -pub enum PlanNode { - StaticBindings { - encoded_tuples: Vec, - variables: Vec, - plain_bindings: Vec>>, - }, - Service { - service_name: PatternValue, - variables: Rc<[Variable]>, - child: Rc, - graph_pattern: Rc, - silent: bool, - }, - QuadPattern { - subject: PatternValue, - predicate: PatternValue, - object: PatternValue, - graph_name: PatternValue, - }, - PathPattern { - subject: PatternValue, - path: Rc, - object: PatternValue, - graph_name: PatternValue, - }, - /// Streams left and materializes right join - HashJoin { - probe_child: Rc, - build_child: Rc, - keys: Vec, - }, - /// Right nested in left loop - ForLoopJoin { - left: Rc, - right: Rc, - }, - /// Streams left and materializes right anti join - AntiJoin { - left: Rc, - right: Rc, - keys: Vec, - }, - Filter { - child: Rc, - expression: Box, - }, - Union { - children: Vec>, - }, - /// hash left join - HashLeftJoin { - left: Rc, - right: Rc, - expression: Box, - keys: Vec, - }, - /// right nested in left loop - ForLoopLeftJoin { - left: Rc, - right: Rc, - }, - Extend { - child: Rc, - variable: PlanVariable, - expression: Box, - }, - Sort { - child: Rc, - by: Vec, - }, - HashDeduplicate { - child: Rc, - }, - /// Removes duplicated consecutive elements - Reduced { - child: Rc, - }, - Skip { - child: Rc, - count: usize, - }, - Limit { - child: Rc, - count: usize, - }, - Project { - child: Rc, - mapping: Rc<[(PlanVariable, PlanVariable)]>, // pairs of (variable key in child, variable key in output) - }, - Aggregate { - // By definition the group by key are the range 0..key_mapping.len() - child: Rc, - key_variables: Rc<[PlanVariable]>, - aggregates: Rc<[(PlanAggregation, PlanVariable)]>, - }, -} - -impl PlanNode { - /// Returns variables that might be bound in the result set - pub fn used_variables(&self) -> BTreeSet { - let mut set = BTreeSet::default(); - self.lookup_used_variables(&mut |v| { - set.insert(v); - }); - set - } - - pub fn lookup_used_variables(&self, callback: &mut impl FnMut(usize)) { - match self { - Self::StaticBindings { encoded_tuples, .. } => { - for tuple in encoded_tuples { - for (key, value) in tuple.iter().enumerate() { - if value.is_some() { - callback(key); - } - } - } - } - Self::QuadPattern { - subject, - predicate, - object, - graph_name, - } => { - subject.lookup_variables(callback); - predicate.lookup_variables(callback); - object.lookup_variables(callback); - graph_name.lookup_variables(callback); - } - Self::PathPattern { - subject, - object, - graph_name, - .. - } => { - subject.lookup_variables(callback); - object.lookup_variables(callback); - graph_name.lookup_variables(callback); - } - Self::Filter { child, expression } => { - expression.lookup_used_variables(callback); - child.lookup_used_variables(callback); - } - Self::Union { children } => { - for child in children { - child.lookup_used_variables(callback); - } - } - Self::HashJoin { - probe_child: left, - build_child: right, - .. - } - | Self::ForLoopJoin { left, right, .. } - | Self::AntiJoin { left, right, .. } - | Self::ForLoopLeftJoin { left, right, .. } => { - left.lookup_used_variables(callback); - right.lookup_used_variables(callback); - } - Self::HashLeftJoin { - left, - right, - expression, - .. - } => { - left.lookup_used_variables(callback); - right.lookup_used_variables(callback); - expression.lookup_used_variables(callback); - } - Self::Extend { - child, - variable, - expression, - } => { - callback(variable.encoded); - expression.lookup_used_variables(callback); - child.lookup_used_variables(callback); - } - Self::Sort { child, .. } - | Self::HashDeduplicate { child } - | Self::Reduced { child } - | Self::Skip { child, .. } - | Self::Limit { child, .. } => child.lookup_used_variables(callback), - Self::Service { - child, - service_name, - .. - } => { - service_name.lookup_variables(callback); - child.lookup_used_variables(callback); - } - Self::Project { mapping, child } => { - let child_bound = child.used_variables(); - for (child_i, output_i) in mapping.iter() { - if child_bound.contains(&child_i.encoded) { - callback(output_i.encoded); - } - } - } - Self::Aggregate { - key_variables, - aggregates, - .. - } => { - for var in key_variables.iter() { - callback(var.encoded); - } - for (_, var) in aggregates.iter() { - callback(var.encoded); - } - } - } - } -} - -#[derive(Debug, Clone)] -pub struct PlanTerm { - pub encoded: EncodedTerm, - pub plain: T, -} - -impl fmt::Display for PlanTerm { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.plain) - } -} - -#[derive(Debug, Clone)] -pub enum PatternValue { - Constant(PlanTerm), - Variable(PlanVariable), - TriplePattern(Box), -} - -impl PatternValue { - pub fn lookup_variables(&self, callback: &mut impl FnMut(usize)) { - if let Self::Variable(v) = self { - callback(v.encoded) - } else if let Self::TriplePattern(p) = self { - p.subject.lookup_variables(callback); - p.predicate.lookup_variables(callback); - p.object.lookup_variables(callback); - } - } -} - -impl fmt::Display for PatternValue { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Constant(c) => write!(f, "{c}"), - Self::Variable(v) => write!(f, "{v}"), - Self::TriplePattern(p) => write!(f, "{p}"), - } - } -} - -#[derive(Debug, Clone)] -pub enum PatternValueConstant { - NamedNode(NamedNode), - Literal(Literal), - Triple(Box), - DefaultGraph, -} - -impl fmt::Display for PatternValueConstant { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::NamedNode(n) => write!(f, "{n}"), - Self::Literal(l) => write!(f, "{l}"), - Self::Triple(t) => write!(f, "<< {t} >>"), - Self::DefaultGraph => f.write_str("DEFAULT"), - } - } -} - -#[derive(Debug, Clone)] -pub struct TriplePatternValue { - pub subject: PatternValue, - pub predicate: PatternValue, - pub object: PatternValue, -} - -impl fmt::Display for TriplePatternValue { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{} {} {}", self.subject, self.predicate, self.object) - } -} - -#[derive(Debug, Clone)] -pub struct PlanVariable

{ - pub encoded: usize, - pub plain: P, -} - -impl fmt::Display for PlanVariable

{ - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.plain) - } -} - -#[derive(Debug, Clone)] -pub enum PlanExpression { - NamedNode(PlanTerm), - Literal(PlanTerm), - Variable(PlanVariable), - Exists(Rc), - Or(Vec), - And(Vec), - Equal(Box, Box), - Greater(Box, Box), - GreaterOrEqual(Box, Box), - Less(Box, Box), - LessOrEqual(Box, Box), - Add(Box, Box), - Subtract(Box, Box), - Multiply(Box, Box), - Divide(Box, Box), - UnaryPlus(Box), - UnaryMinus(Box), - Not(Box), - Str(Box), - Lang(Box), - LangMatches(Box, Box), - Datatype(Box), - Bound(PlanVariable), - Iri(Box), - BNode(Option>), - Rand, - Abs(Box), - Ceil(Box), - Floor(Box), - Round(Box), - Concat(Vec), - SubStr(Box, Box, Option>), - StrLen(Box), - StaticReplace(Box, Regex, Box), - DynamicReplace(Box, Box, Box, Option>), - UCase(Box), - LCase(Box), - EncodeForUri(Box), - Contains(Box, Box), - StrStarts(Box, Box), - StrEnds(Box, Box), - StrBefore(Box, Box), - StrAfter(Box, Box), - Year(Box), - Month(Box), - Day(Box), - Hours(Box), - Minutes(Box), - Seconds(Box), - Timezone(Box), - Tz(Box), - Now, - Uuid, - StrUuid, - Md5(Box), - Sha1(Box), - Sha256(Box), - Sha384(Box), - Sha512(Box), - Coalesce(Vec), - If(Box, Box, Box), - StrLang(Box, Box), - StrDt(Box, Box), - SameTerm(Box, Box), - IsIri(Box), - IsBlank(Box), - IsLiteral(Box), - IsNumeric(Box), - StaticRegex(Box, Regex), - DynamicRegex(Box, Box, Option>), - Triple(Box, Box, Box), - Subject(Box), - Predicate(Box), - Object(Box), - IsTriple(Box), - Adjust(Box, Box), - BooleanCast(Box), - DoubleCast(Box), - FloatCast(Box), - DecimalCast(Box), - IntegerCast(Box), - DateCast(Box), - TimeCast(Box), - DateTimeCast(Box), - DurationCast(Box), - YearMonthDurationCast(Box), - DayTimeDurationCast(Box), - StringCast(Box), - CustomFunction(NamedNode, Vec), -} - -impl PlanExpression { - pub fn lookup_used_variables(&self, callback: &mut impl FnMut(usize)) { - match self { - Self::Variable(v) | Self::Bound(v) => { - callback(v.encoded); - } - Self::NamedNode(_) - | Self::Literal(_) - | Self::Rand - | Self::Now - | Self::Uuid - | Self::StrUuid - | Self::BNode(None) => (), - Self::UnaryPlus(e) - | Self::UnaryMinus(e) - | Self::Not(e) - | Self::BNode(Some(e)) - | Self::Str(e) - | Self::Lang(e) - | Self::Datatype(e) - | Self::Iri(e) - | Self::Abs(e) - | Self::Ceil(e) - | Self::Floor(e) - | Self::Round(e) - | Self::UCase(e) - | Self::LCase(e) - | Self::StrLen(e) - | Self::EncodeForUri(e) - | Self::StaticRegex(e, _) - | Self::Year(e) - | Self::Month(e) - | Self::Day(e) - | Self::Hours(e) - | Self::Minutes(e) - | Self::Seconds(e) - | Self::Timezone(e) - | Self::Tz(e) - | Self::Md5(e) - | Self::Sha1(e) - | Self::Sha256(e) - | Self::Sha384(e) - | Self::Sha512(e) - | Self::IsIri(e) - | Self::IsBlank(e) - | Self::IsLiteral(e) - | Self::IsNumeric(e) - | Self::IsTriple(e) - | Self::Subject(e) - | Self::Predicate(e) - | Self::Object(e) - | Self::BooleanCast(e) - | Self::DoubleCast(e) - | Self::FloatCast(e) - | Self::DecimalCast(e) - | Self::IntegerCast(e) - | Self::DateCast(e) - | Self::TimeCast(e) - | Self::DateTimeCast(e) - | Self::DurationCast(e) - | Self::YearMonthDurationCast(e) - | Self::DayTimeDurationCast(e) - | Self::StringCast(e) => e.lookup_used_variables(callback), - Self::Equal(a, b) - | Self::Greater(a, b) - | Self::GreaterOrEqual(a, b) - | Self::Less(a, b) - | Self::LessOrEqual(a, b) - | Self::Add(a, b) - | Self::Subtract(a, b) - | Self::Multiply(a, b) - | Self::Divide(a, b) - | Self::LangMatches(a, b) - | Self::Contains(a, b) - | Self::StaticReplace(a, _, b) - | Self::StrStarts(a, b) - | Self::StrEnds(a, b) - | Self::StrBefore(a, b) - | Self::StrAfter(a, b) - | Self::StrLang(a, b) - | Self::StrDt(a, b) - | Self::SameTerm(a, b) - | Self::SubStr(a, b, None) - | Self::DynamicRegex(a, b, None) - | Self::Adjust(a, b) => { - a.lookup_used_variables(callback); - b.lookup_used_variables(callback); - } - Self::If(a, b, c) - | Self::SubStr(a, b, Some(c)) - | Self::DynamicRegex(a, b, Some(c)) - | Self::DynamicReplace(a, b, c, None) - | Self::Triple(a, b, c) => { - a.lookup_used_variables(callback); - b.lookup_used_variables(callback); - c.lookup_used_variables(callback); - } - Self::DynamicReplace(a, b, c, Some(d)) => { - a.lookup_used_variables(callback); - b.lookup_used_variables(callback); - c.lookup_used_variables(callback); - d.lookup_used_variables(callback); - } - Self::Or(es) - | Self::And(es) - | Self::Concat(es) - | Self::Coalesce(es) - | Self::CustomFunction(_, es) => { - for e in es { - e.lookup_used_variables(callback); - } - } - Self::Exists(e) => { - e.lookup_used_variables(callback); - } - } - } -} - -impl fmt::Display for PlanExpression { - #[allow(clippy::many_single_char_names)] - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Variable(v) => { - write!(f, "{v}") - } - Self::Bound(v) => { - write!(f, "Bound({v})") - } - Self::NamedNode(n) => write!(f, "{n}"), - Self::Literal(l) => write!(f, "{l}"), - Self::Rand => write!(f, "Rand()"), - Self::Now => write!(f, "Now()"), - Self::Uuid => write!(f, "Uuid()"), - Self::StrUuid => write!(f, "StrUuid()"), - Self::UnaryPlus(e) => write!(f, "UnaryPlus({e})"), - Self::UnaryMinus(e) => write!(f, "UnaryMinus({e})"), - Self::Not(e) => write!(f, "Not({e})"), - Self::BNode(e) => { - if let Some(e) = e { - write!(f, "BNode({e})") - } else { - write!(f, "BNode()") - } - } - Self::Str(e) => write!(f, "Str({e})"), - Self::Lang(e) => write!(f, "Lang({e})"), - Self::Datatype(e) => write!(f, "Datatype({e})"), - Self::Iri(e) => write!(f, "Iri({e})"), - Self::Abs(e) => write!(f, "Abs({e})"), - Self::Ceil(e) => write!(f, "Ceil({e})"), - Self::Floor(e) => write!(f, "Floor({e})"), - Self::Round(e) => write!(f, "Round({e})"), - Self::UCase(e) => write!(f, "UCase({e})"), - Self::LCase(e) => write!(f, "LCase({e})"), - Self::StrLen(e) => write!(f, "StrLen({e})"), - Self::EncodeForUri(e) => write!(f, "EncodeForUri({e})"), - Self::StaticRegex(e, r) => write!(f, "StaticRegex({e}, {r})"), - Self::Year(e) => write!(f, "Year({e})"), - Self::Month(e) => write!(f, "Month({e})"), - Self::Day(e) => write!(f, "Day({e})"), - Self::Hours(e) => write!(f, "Hours({e})"), - Self::Minutes(e) => write!(f, "Minutes({e})"), - Self::Seconds(e) => write!(f, "Seconds({e})"), - Self::Timezone(e) => write!(f, "Timezone({e})"), - Self::Tz(e) => write!(f, "Tz({e})"), - Self::Md5(e) => write!(f, "Md5({e})"), - Self::Sha1(e) => write!(f, "Sha1({e})"), - Self::Sha256(e) => write!(f, "Sha256({e})"), - Self::Sha384(e) => write!(f, "Sha384({e})"), - Self::Sha512(e) => write!(f, "Sha512({e})"), - Self::IsIri(e) => write!(f, "IsIri({e})"), - Self::IsBlank(e) => write!(f, "IsBlank({e})"), - Self::IsLiteral(e) => write!(f, "IsLiteral({e})"), - Self::IsNumeric(e) => write!(f, "IsNumeric({e})"), - Self::IsTriple(e) => write!(f, "IsTriple({e})"), - Self::Subject(e) => write!(f, "Subject({e})"), - Self::Predicate(e) => write!(f, "Predicate({e})"), - Self::Object(e) => write!(f, "Object({e})"), - Self::BooleanCast(e) => write!(f, "BooleanCast({e})"), - Self::DoubleCast(e) => write!(f, "DoubleCast({e})"), - Self::FloatCast(e) => write!(f, "FloatCast({e})"), - Self::DecimalCast(e) => write!(f, "DecimalCast({e})"), - Self::IntegerCast(e) => write!(f, "IntegerCast({e})"), - Self::DateCast(e) => write!(f, "DateCast({e})"), - Self::TimeCast(e) => write!(f, "TimeCast({e})"), - Self::DateTimeCast(e) => write!(f, "DateTimeCast({e})"), - Self::DurationCast(e) => write!(f, "DurationCast({e})"), - Self::YearMonthDurationCast(e) => write!(f, "YearMonthDurationCast({e})"), - Self::DayTimeDurationCast(e) => write!(f, "DayTimeDurationCast({e})"), - Self::StringCast(e) => write!(f, "StringCast({e})"), - Self::Or(es) => { - write!(f, "Or(")?; - for (i, e) in es.iter().enumerate() { - if i > 0 { - write!(f, ", ")?; - } - write!(f, "{e}")?; - } - write!(f, ")") - } - Self::And(es) => { - write!(f, "And(")?; - for (i, e) in es.iter().enumerate() { - if i > 0 { - write!(f, ", ")?; - } - write!(f, "{e}")?; - } - write!(f, ")") - } - Self::Equal(a, b) => write!(f, "Equal({a}, {b})"), - Self::Greater(a, b) => write!(f, "Greater({a}, {b})"), - Self::GreaterOrEqual(a, b) => write!(f, "GreaterOrEqual({a}, {b})"), - Self::Less(a, b) => write!(f, "Less({a}, {b})"), - Self::LessOrEqual(a, b) => write!(f, "LessOrEqual({a}, {b})"), - Self::Add(a, b) => write!(f, "Add({a}, {b})"), - Self::Subtract(a, b) => write!(f, "Subtract({a}, {b})"), - Self::Multiply(a, b) => write!(f, "Multiply({a}, {b})"), - Self::Divide(a, b) => write!(f, "Divide({a}, {b})"), - Self::LangMatches(a, b) => write!(f, "LangMatches({a}, {b})"), - Self::Contains(a, b) => write!(f, "Contains({a}, {b})"), - Self::StaticReplace(a, b, c) => write!(f, "StaticReplace({a}, {b}, {c})"), - Self::StrStarts(a, b) => write!(f, "StrStarts({a}, {b})"), - Self::StrEnds(a, b) => write!(f, "StrEnds({a}, {b})"), - Self::StrBefore(a, b) => write!(f, "StrBefore({a}, {b})"), - Self::StrAfter(a, b) => write!(f, "StrAfter({a}, {b})"), - Self::StrLang(a, b) => write!(f, "StrLang({a}, {b})"), - Self::StrDt(a, b) => write!(f, "StrDt({a}, {b})"), - Self::SameTerm(a, b) => write!(f, "SameTerm({a}, {b})"), - Self::SubStr(a, b, None) => write!(f, "SubStr({a}, {b})"), - Self::DynamicRegex(a, b, None) => write!(f, "DynamicRegex({a}, {b})"), - Self::Adjust(a, b) => write!(f, "Adjust({a}, {b})"), - Self::If(a, b, c) => write!(f, "If({a}, {b}, {c})"), - Self::SubStr(a, b, Some(c)) => write!(f, "SubStr({a}, {b}, {c})"), - Self::DynamicRegex(a, b, Some(c)) => write!(f, "DynamicRegex({a}, {b}, {c})"), - Self::DynamicReplace(a, b, c, None) => write!(f, "DynamicReplace({a}, {b}, {c})"), - Self::Triple(a, b, c) => write!(f, "Triple({a}, {b}, {c})"), - Self::DynamicReplace(a, b, c, Some(d)) => { - write!(f, "DynamicReplace({a}, {b}, {c}, {d})") - } - Self::Concat(es) => { - write!(f, "Concat(")?; - for (i, e) in es.iter().enumerate() { - if i > 0 { - write!(f, ", ")?; - } - write!(f, "{e}")?; - } - write!(f, ")") - } - Self::Coalesce(es) => { - write!(f, "Coalesce(")?; - for (i, e) in es.iter().enumerate() { - if i > 0 { - write!(f, ", ")?; - } - write!(f, "{e}")?; - } - write!(f, ")") - } - Self::CustomFunction(name, es) => { - write!(f, "{name}(")?; - for (i, e) in es.iter().enumerate() { - if i > 0 { - write!(f, ", ")?; - } - write!(f, "{e}")?; - } - write!(f, ")") - } - Self::Exists(_) => write!(f, "Exists()"), //TODO - } - } -} - -#[derive(Debug, Clone)] -pub struct PlanAggregation { - pub function: PlanAggregationFunction, - pub parameter: Option, - pub distinct: bool, -} - -impl fmt::Display for PlanAggregation { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match &self.function { - PlanAggregationFunction::Count => { - write!(f, "Count") - } - PlanAggregationFunction::Sum => { - write!(f, "Sum") - } - PlanAggregationFunction::Min => { - write!(f, "Min") - } - PlanAggregationFunction::Max => { - write!(f, "Max") - } - PlanAggregationFunction::Avg => { - write!(f, "Avg") - } - PlanAggregationFunction::GroupConcat { .. } => { - write!(f, "GroupConcat") - } - PlanAggregationFunction::Sample => write!(f, "Sample"), - }?; - if self.distinct { - write!(f, "Distinct")?; - } - write!(f, "(")?; - if let Some(expr) = &self.parameter { - write!(f, "{expr}")?; - } - if let PlanAggregationFunction::GroupConcat { separator } = &self.function { - write!(f, "; separator={separator}")?; - } - write!(f, ")") - } -} - -#[derive(Debug, Clone)] -pub enum PlanAggregationFunction { - Count, - Sum, - Min, - Max, - Avg, - Sample, - GroupConcat { separator: Rc }, -} - -#[derive(Debug, Clone)] -pub enum PlanPropertyPath { - Path(PlanTerm), - Reverse(Rc), - Sequence(Rc, Rc), - Alternative(Rc, Rc), - ZeroOrMore(Rc), - OneOrMore(Rc), - ZeroOrOne(Rc), - NegatedPropertySet(Rc<[PlanTerm]>), -} - -impl fmt::Display for PlanPropertyPath { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Path(p) => write!(f, "{p}"), - Self::Reverse(p) => write!(f, "Reverse({p})"), - Self::Sequence(a, b) => write!(f, "Sequence{a}, {b}"), - Self::Alternative(a, b) => write!(f, "Alternative{a}, {b}"), - Self::ZeroOrMore(p) => write!(f, "ZeroOrMore({p})"), - Self::OneOrMore(p) => write!(f, "OneOrMore({p})"), - Self::ZeroOrOne(p) => write!(f, "ZeroOrOne({p})"), - Self::NegatedPropertySet(ps) => { - write!(f, "NegatedPropertySet(")?; - for (i, p) in ps.iter().enumerate() { - if i > 0 { - write!(f, ", ")?; - } - write!(f, "{p}")?; - } - write!(f, ")") - } - } - } -} - -#[derive(Debug, Clone)] -pub enum Comparator { - Asc(PlanExpression), - Desc(PlanExpression), -} - -impl fmt::Display for Comparator { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Asc(c) => write!(f, "Asc({c})"), - Self::Desc(c) => write!(f, "Desc({c})"), - } - } -} - -#[derive(Debug, Clone)] -pub struct TripleTemplate { - pub subject: TripleTemplateValue, - pub predicate: TripleTemplateValue, - pub object: TripleTemplateValue, -} - -impl fmt::Display for TripleTemplate { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{} {} {}", self.subject, self.predicate, self.object) - } -} - -#[derive(Debug, Clone)] -pub enum TripleTemplateValue { - Constant(PlanTerm), - BlankNode(PlanVariable), - Variable(PlanVariable), - Triple(Box), -} - -impl fmt::Display for TripleTemplateValue { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Constant(c) => write!(f, "{c}"), - Self::BlankNode(bn) => write!(f, "{bn}"), - Self::Variable(v) => write!(f, "{v}"), - Self::Triple(t) => write!(f, "<< {t} >>"), - } - } -} - -#[derive(Eq, PartialEq, Debug, Clone, Hash)] -pub struct EncodedTuple { - inner: Vec>, -} - -impl EncodedTuple { - pub fn with_capacity(capacity: usize) -> Self { - Self { - inner: Vec::with_capacity(capacity), - } - } - - pub fn capacity(&self) -> usize { - self.inner.capacity() - } - - pub fn contains(&self, index: usize) -> bool { - self.inner.get(index).map_or(false, Option::is_some) - } - - pub fn get(&self, index: usize) -> Option<&EncodedTerm> { - self.inner.get(index).unwrap_or(&None).as_ref() - } - - pub fn iter(&self) -> impl Iterator> + '_ { - self.inner.iter().cloned() - } - - pub fn set(&mut self, index: usize, value: EncodedTerm) { - if self.inner.len() <= index { - self.inner.resize(index + 1, None); - } - self.inner[index] = Some(value); - } - - pub fn combine_with(&self, other: &Self) -> Option { - if self.inner.len() < other.inner.len() { - let mut result = other.inner.clone(); - for (key, self_value) in self.inner.iter().enumerate() { - if let Some(self_value) = self_value { - match &other.inner[key] { - Some(other_value) => { - if self_value != other_value { - return None; - } - } - None => result[key] = Some(self_value.clone()), - } - } - } - Some(Self { inner: result }) - } else { - let mut result = self.inner.clone(); - for (key, other_value) in other.inner.iter().enumerate() { - if let Some(other_value) = other_value { - match &self.inner[key] { - Some(self_value) => { - if self_value != other_value { - return None; - } - } - None => result[key] = Some(other_value.clone()), - } - } - } - Some(Self { inner: result }) - } - } -} - -impl IntoIterator for EncodedTuple { - type Item = Option; - type IntoIter = std::vec::IntoIter>; - - fn into_iter(self) -> Self::IntoIter { - self.inner.into_iter() - } -} diff --git a/lib/src/sparql/plan_builder.rs b/lib/src/sparql/plan_builder.rs deleted file mode 100644 index 12faeb6c..00000000 --- a/lib/src/sparql/plan_builder.rs +++ /dev/null @@ -1,1109 +0,0 @@ -use crate::model::Term as OxTerm; -use crate::sparql::dataset::DatasetView; -use crate::sparql::error::EvaluationError; -use crate::sparql::eval::compile_pattern; -use crate::sparql::plan::*; -use crate::storage::numeric_encoder::{EncodedTerm, EncodedTriple}; -use oxrdf::vocab::xsd; -use oxrdf::{BlankNode, Term, TermRef, Triple}; -use regex::Regex; -use spargebra::term::{GroundSubject, GroundTriple, TermPattern, TriplePattern}; -use sparopt::algebra::*; -use sparopt::Optimizer; -use std::collections::HashMap; -use std::rc::Rc; - -pub struct PlanBuilder<'a> { - dataset: &'a DatasetView, - custom_functions: &'a HashMap Option>>, -} - -impl<'a> PlanBuilder<'a> { - pub fn build( - dataset: &'a DatasetView, - pattern: &spargebra::algebra::GraphPattern, - is_cardinality_meaningful: bool, - custom_functions: &'a HashMap Option>>, - without_optimizations: bool, - ) -> Result<(PlanNode, Vec), EvaluationError> { - let mut pattern = GraphPattern::from(pattern); - if !without_optimizations { - pattern = Optimizer::optimize_graph_pattern(pattern); - } - let mut variables = Vec::default(); - let plan = PlanBuilder { - dataset, - custom_functions, - } - .build_for_graph_pattern(&pattern, &mut variables)?; - let plan = if !without_optimizations && !is_cardinality_meaningful { - // let's reduce downstream task. - // TODO: avoid if already REDUCED or DISTINCT - PlanNode::Reduced { - child: Rc::new(plan), - } - } else { - plan - }; - Ok((plan, variables)) - } - - pub fn build_graph_template( - dataset: &'a DatasetView, - template: &[TriplePattern], - mut variables: Vec, - custom_functions: &'a HashMap Option>>, - ) -> Vec { - PlanBuilder { - dataset, - custom_functions, - } - .build_for_graph_template(template, &mut variables) - } - - fn build_for_graph_pattern( - &self, - pattern: &GraphPattern, - variables: &mut Vec, - ) -> Result { - Ok(match pattern { - GraphPattern::QuadPattern { - subject, - predicate, - object, - graph_name, - } => PlanNode::QuadPattern { - subject: self.pattern_value_from_ground_term_pattern(subject, variables), - predicate: self.pattern_value_from_named_node_or_variable(predicate, variables), - object: self.pattern_value_from_ground_term_pattern(object, variables), - graph_name: graph_name.as_ref().map_or( - PatternValue::Constant(PlanTerm { - encoded: EncodedTerm::DefaultGraph, - plain: PatternValueConstant::DefaultGraph, - }), - |g| self.pattern_value_from_named_node_or_variable(g, variables), - ), - }, - GraphPattern::Path { - subject, - path, - object, - graph_name, - } => PlanNode::PathPattern { - subject: self.pattern_value_from_ground_term_pattern(subject, variables), - path: Rc::new(self.build_for_path(path)), - object: self.pattern_value_from_ground_term_pattern(object, variables), - graph_name: graph_name.as_ref().map_or( - PatternValue::Constant(PlanTerm { - encoded: EncodedTerm::DefaultGraph, - plain: PatternValueConstant::DefaultGraph, - }), - |g| self.pattern_value_from_named_node_or_variable(g, variables), - ), - }, - GraphPattern::Join { - left, - right, - algorithm, - } => match algorithm { - JoinAlgorithm::HashBuildLeftProbeRight { keys } => PlanNode::HashJoin { - build_child: Rc::new(self.build_for_graph_pattern(left, variables)?), - probe_child: Rc::new(self.build_for_graph_pattern(right, variables)?), - keys: keys - .iter() - .map(|v| build_plan_variable(variables, v)) - .collect(), - }, - }, - GraphPattern::LeftJoin { - left, - right, - expression, - algorithm, - } => match algorithm { - LeftJoinAlgorithm::HashBuildRightProbeLeft { keys } => PlanNode::HashLeftJoin { - left: Rc::new(self.build_for_graph_pattern(left, variables)?), - right: Rc::new(self.build_for_graph_pattern(right, variables)?), - expression: Box::new(self.build_for_expression(expression, variables)?), - keys: keys - .iter() - .map(|v| build_plan_variable(variables, v)) - .collect(), - }, - }, - GraphPattern::Lateral { left, right } => { - if let GraphPattern::LeftJoin { - left: nested_left, - right: nested_right, - expression, - .. - } = right.as_ref() - { - if nested_left.is_empty_singleton() { - // We are in a ForLoopLeftJoin - let right = - GraphPattern::filter(nested_right.as_ref().clone(), expression.clone()); - PlanNode::ForLoopLeftJoin { - left: Rc::new(self.build_for_graph_pattern(left, variables)?), - right: Rc::new(self.build_for_graph_pattern(&right, variables)?), - } - } else { - PlanNode::ForLoopJoin { - left: Rc::new(self.build_for_graph_pattern(left, variables)?), - right: Rc::new(self.build_for_graph_pattern(right, variables)?), - } - } - } else { - PlanNode::ForLoopJoin { - left: Rc::new(self.build_for_graph_pattern(left, variables)?), - right: Rc::new(self.build_for_graph_pattern(right, variables)?), - } - } - } - GraphPattern::Filter { expression, inner } => PlanNode::Filter { - child: Rc::new(self.build_for_graph_pattern(inner, variables)?), - expression: Box::new(self.build_for_expression(expression, variables)?), - }, - GraphPattern::Union { inner } => PlanNode::Union { - children: inner - .iter() - .map(|p| Ok(Rc::new(self.build_for_graph_pattern(p, variables)?))) - .collect::>()?, - }, - GraphPattern::Extend { - inner, - variable, - expression, - } => PlanNode::Extend { - child: Rc::new(self.build_for_graph_pattern(inner, variables)?), - variable: build_plan_variable(variables, variable), - expression: Box::new(self.build_for_expression(expression, variables)?), - }, - GraphPattern::Minus { - left, - right, - algorithm, - } => match algorithm { - MinusAlgorithm::HashBuildRightProbeLeft { keys } => PlanNode::AntiJoin { - left: Rc::new(self.build_for_graph_pattern(left, variables)?), - right: Rc::new(self.build_for_graph_pattern(right, variables)?), - keys: keys - .iter() - .map(|v| build_plan_variable(variables, v)) - .collect(), - }, - }, - GraphPattern::Service { - name, - inner, - silent, - } => { - // Child building should be at the begging in order for `variables` to be filled - let child = self.build_for_graph_pattern(inner, variables)?; - let service_name = self.pattern_value_from_named_node_or_variable(name, variables); - PlanNode::Service { - service_name, - variables: Rc::from(variables.as_slice()), - child: Rc::new(child), - graph_pattern: Rc::new(inner.as_ref().into()), - silent: *silent, - } - } - GraphPattern::Group { - inner, - variables: by, - aggregates, - } => PlanNode::Aggregate { - child: Rc::new(self.build_for_graph_pattern(inner, variables)?), - key_variables: by - .iter() - .map(|k| build_plan_variable(variables, k)) - .collect(), - aggregates: aggregates - .iter() - .map(|(v, a)| { - Ok(( - self.build_for_aggregate(a, variables)?, - build_plan_variable(variables, v), - )) - }) - .collect::>()?, - }, - GraphPattern::Values { - variables: table_variables, - bindings, - } => { - let bindings_variables = table_variables - .iter() - .map(|v| build_plan_variable(variables, v)) - .collect::>(); - let encoded_tuples = bindings - .iter() - .map(|row| { - let mut result = EncodedTuple::with_capacity(variables.len()); - for (key, value) in row.iter().enumerate() { - if let Some(term) = value { - result.set( - bindings_variables[key].encoded, - match term { - GroundTerm::NamedNode(node) => self.build_term(node), - GroundTerm::Literal(literal) => self.build_term(literal), - GroundTerm::Triple(triple) => self.build_triple(triple), - }, - ); - } - } - result - }) - .collect(); - PlanNode::StaticBindings { - encoded_tuples, - variables: bindings_variables, - plain_bindings: bindings.clone(), - } - } - GraphPattern::OrderBy { inner, expression } => { - let condition: Result, EvaluationError> = expression - .iter() - .map(|comp| match comp { - OrderExpression::Asc(e) => { - Ok(Comparator::Asc(self.build_for_expression(e, variables)?)) - } - OrderExpression::Desc(e) => { - Ok(Comparator::Desc(self.build_for_expression(e, variables)?)) - } - }) - .collect(); - PlanNode::Sort { - child: Rc::new(self.build_for_graph_pattern(inner, variables)?), - by: condition?, - } - } - GraphPattern::Project { - inner, - variables: projection, - } => { - let mut inner_variables = projection.clone(); - PlanNode::Project { - child: Rc::new(self.build_for_graph_pattern(inner, &mut inner_variables)?), - mapping: projection - .iter() - .enumerate() - .map(|(new_variable, variable)| { - ( - PlanVariable { - encoded: new_variable, - plain: variable.clone(), - }, - build_plan_variable(variables, variable), - ) - }) - .collect(), - } - } - GraphPattern::Distinct { inner } => PlanNode::HashDeduplicate { - child: Rc::new(self.build_for_graph_pattern(inner, variables)?), - }, - GraphPattern::Reduced { inner } => PlanNode::Reduced { - child: Rc::new(self.build_for_graph_pattern(inner, variables)?), - }, - GraphPattern::Slice { - inner, - start, - length, - } => { - let mut plan = self.build_for_graph_pattern(inner, variables)?; - if *start > 0 { - plan = PlanNode::Skip { - child: Rc::new(plan), - count: *start, - }; - } - if let Some(length) = length { - plan = PlanNode::Limit { - child: Rc::new(plan), - count: *length, - }; - } - plan - } - }) - } - - fn build_for_path(&self, path: &PropertyPathExpression) -> PlanPropertyPath { - match path { - PropertyPathExpression::NamedNode(p) => PlanPropertyPath::Path(PlanTerm { - encoded: self.build_term(p), - plain: p.clone(), - }), - PropertyPathExpression::Reverse(p) => { - PlanPropertyPath::Reverse(Rc::new(self.build_for_path(p))) - } - PropertyPathExpression::Alternative(a, b) => PlanPropertyPath::Alternative( - Rc::new(self.build_for_path(a)), - Rc::new(self.build_for_path(b)), - ), - PropertyPathExpression::Sequence(a, b) => PlanPropertyPath::Sequence( - Rc::new(self.build_for_path(a)), - Rc::new(self.build_for_path(b)), - ), - PropertyPathExpression::ZeroOrMore(p) => { - PlanPropertyPath::ZeroOrMore(Rc::new(self.build_for_path(p))) - } - PropertyPathExpression::OneOrMore(p) => { - PlanPropertyPath::OneOrMore(Rc::new(self.build_for_path(p))) - } - PropertyPathExpression::ZeroOrOne(p) => { - PlanPropertyPath::ZeroOrOne(Rc::new(self.build_for_path(p))) - } - PropertyPathExpression::NegatedPropertySet(p) => PlanPropertyPath::NegatedPropertySet( - p.iter() - .map(|p| PlanTerm { - encoded: self.build_term(p), - plain: p.clone(), - }) - .collect(), - ), - } - } - - fn build_for_expression( - &self, - expression: &Expression, - variables: &mut Vec, - ) -> Result { - Ok(match expression { - Expression::NamedNode(node) => PlanExpression::NamedNode(PlanTerm { - encoded: self.build_term(node), - plain: node.clone(), - }), - Expression::Literal(l) => PlanExpression::Literal(PlanTerm { - encoded: self.build_term(l), - plain: l.clone(), - }), - Expression::Variable(v) => PlanExpression::Variable(build_plan_variable(variables, v)), - Expression::Or(inner) => PlanExpression::Or( - inner - .iter() - .map(|e| self.build_for_expression(e, variables)) - .collect::>()?, - ), - Expression::And(inner) => PlanExpression::And( - inner - .iter() - .map(|e| self.build_for_expression(e, variables)) - .collect::>()?, - ), - Expression::Equal(a, b) => PlanExpression::Equal( - Box::new(self.build_for_expression(a, variables)?), - Box::new(self.build_for_expression(b, variables)?), - ), - Expression::SameTerm(a, b) => PlanExpression::SameTerm( - Box::new(self.build_for_expression(a, variables)?), - Box::new(self.build_for_expression(b, variables)?), - ), - Expression::Greater(a, b) => PlanExpression::Greater( - Box::new(self.build_for_expression(a, variables)?), - Box::new(self.build_for_expression(b, variables)?), - ), - Expression::GreaterOrEqual(a, b) => PlanExpression::GreaterOrEqual( - Box::new(self.build_for_expression(a, variables)?), - Box::new(self.build_for_expression(b, variables)?), - ), - Expression::Less(a, b) => PlanExpression::Less( - Box::new(self.build_for_expression(a, variables)?), - Box::new(self.build_for_expression(b, variables)?), - ), - Expression::LessOrEqual(a, b) => PlanExpression::LessOrEqual( - Box::new(self.build_for_expression(a, variables)?), - Box::new(self.build_for_expression(b, variables)?), - ), - Expression::Add(a, b) => PlanExpression::Add( - Box::new(self.build_for_expression(a, variables)?), - Box::new(self.build_for_expression(b, variables)?), - ), - Expression::Subtract(a, b) => PlanExpression::Subtract( - Box::new(self.build_for_expression(a, variables)?), - Box::new(self.build_for_expression(b, variables)?), - ), - Expression::Multiply(a, b) => PlanExpression::Multiply( - Box::new(self.build_for_expression(a, variables)?), - Box::new(self.build_for_expression(b, variables)?), - ), - Expression::Divide(a, b) => PlanExpression::Divide( - Box::new(self.build_for_expression(a, variables)?), - Box::new(self.build_for_expression(b, variables)?), - ), - Expression::UnaryPlus(e) => { - PlanExpression::UnaryPlus(Box::new(self.build_for_expression(e, variables)?)) - } - Expression::UnaryMinus(e) => { - PlanExpression::UnaryMinus(Box::new(self.build_for_expression(e, variables)?)) - } - Expression::Not(e) => { - PlanExpression::Not(Box::new(self.build_for_expression(e, variables)?)) - } - Expression::FunctionCall(function, parameters) => match function { - Function::Str => PlanExpression::Str(Box::new( - self.build_for_expression(¶meters[0], variables)?, - )), - Function::Lang => PlanExpression::Lang(Box::new( - self.build_for_expression(¶meters[0], variables)?, - )), - Function::LangMatches => PlanExpression::LangMatches( - Box::new(self.build_for_expression(¶meters[0], variables)?), - Box::new(self.build_for_expression(¶meters[1], variables)?), - ), - Function::Datatype => PlanExpression::Datatype(Box::new( - self.build_for_expression(¶meters[0], variables)?, - )), - Function::Iri => PlanExpression::Iri(Box::new( - self.build_for_expression(¶meters[0], variables)?, - )), - Function::BNode => PlanExpression::BNode(match parameters.get(0) { - Some(e) => Some(Box::new(self.build_for_expression(e, variables)?)), - None => None, - }), - Function::Rand => PlanExpression::Rand, - Function::Abs => PlanExpression::Abs(Box::new( - self.build_for_expression(¶meters[0], variables)?, - )), - Function::Ceil => PlanExpression::Ceil(Box::new( - self.build_for_expression(¶meters[0], variables)?, - )), - Function::Floor => PlanExpression::Floor(Box::new( - self.build_for_expression(¶meters[0], variables)?, - )), - Function::Round => PlanExpression::Round(Box::new( - self.build_for_expression(¶meters[0], variables)?, - )), - Function::Concat => { - PlanExpression::Concat(self.expression_list(parameters, variables)?) - } - Function::SubStr => PlanExpression::SubStr( - Box::new(self.build_for_expression(¶meters[0], variables)?), - Box::new(self.build_for_expression(¶meters[1], variables)?), - match parameters.get(2) { - Some(flags) => Some(Box::new(self.build_for_expression(flags, variables)?)), - None => None, - }, - ), - Function::StrLen => PlanExpression::StrLen(Box::new( - self.build_for_expression(¶meters[0], variables)?, - )), - Function::Replace => { - if let Some(static_regex) = - compile_static_pattern_if_exists(¶meters[1], parameters.get(3)) - { - PlanExpression::StaticReplace( - Box::new(self.build_for_expression(¶meters[0], variables)?), - static_regex, - Box::new(self.build_for_expression(¶meters[2], variables)?), - ) - } else { - PlanExpression::DynamicReplace( - Box::new(self.build_for_expression(¶meters[0], variables)?), - Box::new(self.build_for_expression(¶meters[1], variables)?), - Box::new(self.build_for_expression(¶meters[2], variables)?), - match parameters.get(3) { - Some(flags) => { - Some(Box::new(self.build_for_expression(flags, variables)?)) - } - None => None, - }, - ) - } - } - Function::UCase => PlanExpression::UCase(Box::new( - self.build_for_expression(¶meters[0], variables)?, - )), - Function::LCase => PlanExpression::LCase(Box::new( - self.build_for_expression(¶meters[0], variables)?, - )), - Function::EncodeForUri => PlanExpression::EncodeForUri(Box::new( - self.build_for_expression(¶meters[0], variables)?, - )), - Function::Contains => PlanExpression::Contains( - Box::new(self.build_for_expression(¶meters[0], variables)?), - Box::new(self.build_for_expression(¶meters[1], variables)?), - ), - Function::StrStarts => PlanExpression::StrStarts( - Box::new(self.build_for_expression(¶meters[0], variables)?), - Box::new(self.build_for_expression(¶meters[1], variables)?), - ), - Function::StrEnds => PlanExpression::StrEnds( - Box::new(self.build_for_expression(¶meters[0], variables)?), - Box::new(self.build_for_expression(¶meters[1], variables)?), - ), - Function::StrBefore => PlanExpression::StrBefore( - Box::new(self.build_for_expression(¶meters[0], variables)?), - Box::new(self.build_for_expression(¶meters[1], variables)?), - ), - Function::StrAfter => PlanExpression::StrAfter( - Box::new(self.build_for_expression(¶meters[0], variables)?), - Box::new(self.build_for_expression(¶meters[1], variables)?), - ), - Function::Year => PlanExpression::Year(Box::new( - self.build_for_expression(¶meters[0], variables)?, - )), - Function::Month => PlanExpression::Month(Box::new( - self.build_for_expression(¶meters[0], variables)?, - )), - Function::Day => PlanExpression::Day(Box::new( - self.build_for_expression(¶meters[0], variables)?, - )), - Function::Hours => PlanExpression::Hours(Box::new( - self.build_for_expression(¶meters[0], variables)?, - )), - Function::Minutes => PlanExpression::Minutes(Box::new( - self.build_for_expression(¶meters[0], variables)?, - )), - Function::Seconds => PlanExpression::Seconds(Box::new( - self.build_for_expression(¶meters[0], variables)?, - )), - Function::Timezone => PlanExpression::Timezone(Box::new( - self.build_for_expression(¶meters[0], variables)?, - )), - Function::Tz => PlanExpression::Tz(Box::new( - self.build_for_expression(¶meters[0], variables)?, - )), - Function::Now => PlanExpression::Now, - Function::Uuid => PlanExpression::Uuid, - Function::StrUuid => PlanExpression::StrUuid, - Function::Md5 => PlanExpression::Md5(Box::new( - self.build_for_expression(¶meters[0], variables)?, - )), - Function::Sha1 => PlanExpression::Sha1(Box::new( - self.build_for_expression(¶meters[0], variables)?, - )), - Function::Sha256 => PlanExpression::Sha256(Box::new( - self.build_for_expression(¶meters[0], variables)?, - )), - Function::Sha384 => PlanExpression::Sha384(Box::new( - self.build_for_expression(¶meters[0], variables)?, - )), - Function::Sha512 => PlanExpression::Sha512(Box::new( - self.build_for_expression(¶meters[0], variables)?, - )), - Function::StrLang => PlanExpression::StrLang( - Box::new(self.build_for_expression(¶meters[0], variables)?), - Box::new(self.build_for_expression(¶meters[1], variables)?), - ), - Function::StrDt => PlanExpression::StrDt( - Box::new(self.build_for_expression(¶meters[0], variables)?), - Box::new(self.build_for_expression(¶meters[1], variables)?), - ), - Function::IsIri => PlanExpression::IsIri(Box::new( - self.build_for_expression(¶meters[0], variables)?, - )), - Function::IsBlank => PlanExpression::IsBlank(Box::new( - self.build_for_expression(¶meters[0], variables)?, - )), - Function::IsLiteral => PlanExpression::IsLiteral(Box::new( - self.build_for_expression(¶meters[0], variables)?, - )), - Function::IsNumeric => PlanExpression::IsNumeric(Box::new( - self.build_for_expression(¶meters[0], variables)?, - )), - Function::Regex => { - if let Some(static_regex) = - compile_static_pattern_if_exists(¶meters[1], parameters.get(2)) - { - PlanExpression::StaticRegex( - Box::new(self.build_for_expression(¶meters[0], variables)?), - static_regex, - ) - } else { - PlanExpression::DynamicRegex( - Box::new(self.build_for_expression(¶meters[0], variables)?), - Box::new(self.build_for_expression(¶meters[1], variables)?), - match parameters.get(2) { - Some(flags) => { - Some(Box::new(self.build_for_expression(flags, variables)?)) - } - None => None, - }, - ) - } - } - Function::Triple => PlanExpression::Triple( - Box::new(self.build_for_expression(¶meters[0], variables)?), - Box::new(self.build_for_expression(¶meters[1], variables)?), - Box::new(self.build_for_expression(¶meters[2], variables)?), - ), - Function::Subject => PlanExpression::Subject(Box::new( - self.build_for_expression(¶meters[0], variables)?, - )), - Function::Predicate => PlanExpression::Predicate(Box::new( - self.build_for_expression(¶meters[0], variables)?, - )), - Function::Object => PlanExpression::Object(Box::new( - self.build_for_expression(¶meters[0], variables)?, - )), - Function::IsTriple => PlanExpression::IsTriple(Box::new( - self.build_for_expression(¶meters[0], variables)?, - )), - Function::Adjust => PlanExpression::Adjust( - Box::new(self.build_for_expression(¶meters[0], variables)?), - Box::new(self.build_for_expression(¶meters[1], variables)?), - ), - Function::Custom(name) => { - if self.custom_functions.contains_key(name) { - PlanExpression::CustomFunction( - name.clone(), - parameters - .iter() - .map(|p| self.build_for_expression(p, variables)) - .collect::, EvaluationError>>()?, - ) - } else if name.as_ref() == xsd::BOOLEAN { - self.build_cast( - parameters, - PlanExpression::BooleanCast, - variables, - "boolean", - )? - } else if name.as_ref() == xsd::DOUBLE { - self.build_cast( - parameters, - PlanExpression::DoubleCast, - variables, - "double", - )? - } else if name.as_ref() == xsd::FLOAT { - self.build_cast(parameters, PlanExpression::FloatCast, variables, "float")? - } else if name.as_ref() == xsd::DECIMAL { - self.build_cast( - parameters, - PlanExpression::DecimalCast, - variables, - "decimal", - )? - } else if name.as_ref() == xsd::INTEGER { - self.build_cast( - parameters, - PlanExpression::IntegerCast, - variables, - "integer", - )? - } else if name.as_ref() == xsd::DATE { - self.build_cast(parameters, PlanExpression::DateCast, variables, "date")? - } else if name.as_ref() == xsd::TIME { - self.build_cast(parameters, PlanExpression::TimeCast, variables, "time")? - } else if name.as_ref() == xsd::DATE_TIME { - self.build_cast( - parameters, - PlanExpression::DateTimeCast, - variables, - "dateTime", - )? - } else if name.as_ref() == xsd::DURATION { - self.build_cast( - parameters, - PlanExpression::DurationCast, - variables, - "duration", - )? - } else if name.as_ref() == xsd::YEAR_MONTH_DURATION { - self.build_cast( - parameters, - PlanExpression::YearMonthDurationCast, - variables, - "yearMonthDuration", - )? - } else if name.as_ref() == xsd::DAY_TIME_DURATION { - self.build_cast( - parameters, - PlanExpression::DayTimeDurationCast, - variables, - "dayTimeDuration", - )? - } else if name.as_ref() == xsd::STRING { - self.build_cast( - parameters, - PlanExpression::StringCast, - variables, - "string", - )? - } else { - return Err(EvaluationError::msg(format!( - "Not supported custom function {name}" - ))); - } - } - }, - Expression::Bound(v) => PlanExpression::Bound(build_plan_variable(variables, v)), - Expression::If(a, b, c) => PlanExpression::If( - Box::new(self.build_for_expression(a, variables)?), - Box::new(self.build_for_expression(b, variables)?), - Box::new(self.build_for_expression(c, variables)?), - ), - Expression::Exists(n) => { - let mut variables = variables.clone(); // Do not expose the exists variables outside - PlanExpression::Exists(Rc::new(self.build_for_graph_pattern(n, &mut variables)?)) - } - Expression::Coalesce(parameters) => { - PlanExpression::Coalesce(self.expression_list(parameters, variables)?) - } - }) - } - - fn build_cast( - &self, - parameters: &[Expression], - constructor: impl Fn(Box) -> PlanExpression, - variables: &mut Vec, - name: &'static str, - ) -> Result { - if parameters.len() == 1 { - Ok(constructor(Box::new( - self.build_for_expression(¶meters[0], variables)?, - ))) - } else { - Err(EvaluationError::msg(format!( - "The xsd:{name} casting takes only one parameter" - ))) - } - } - - fn expression_list( - &self, - l: &[Expression], - variables: &mut Vec, - ) -> Result, EvaluationError> { - l.iter() - .map(|e| self.build_for_expression(e, variables)) - .collect() - } - - fn pattern_value_from_ground_term_pattern( - &self, - term_pattern: &GroundTermPattern, - variables: &mut Vec, - ) -> PatternValue { - match term_pattern { - GroundTermPattern::Variable(variable) => { - PatternValue::Variable(build_plan_variable(variables, variable)) - } - GroundTermPattern::NamedNode(node) => PatternValue::Constant(PlanTerm { - encoded: self.build_term(node), - plain: PatternValueConstant::NamedNode(node.clone()), - }), - GroundTermPattern::Literal(literal) => PatternValue::Constant(PlanTerm { - encoded: self.build_term(literal), - plain: PatternValueConstant::Literal(literal.clone()), - }), - GroundTermPattern::Triple(triple) => { - match ( - self.pattern_value_from_ground_term_pattern(&triple.subject, variables), - self.pattern_value_from_named_node_or_variable(&triple.predicate, variables), - self.pattern_value_from_ground_term_pattern(&triple.object, variables), - ) { - ( - PatternValue::Constant(PlanTerm { - encoded: encoded_subject, - plain: plain_subject, - }), - PatternValue::Constant(PlanTerm { - encoded: encoded_predicate, - plain: plain_predicate, - }), - PatternValue::Constant(PlanTerm { - encoded: encoded_object, - plain: plain_object, - }), - ) => PatternValue::Constant(PlanTerm { - encoded: EncodedTriple { - subject: encoded_subject, - predicate: encoded_predicate, - object: encoded_object, - } - .into(), - plain: PatternValueConstant::Triple(Box::new(Triple { - subject: match plain_subject { - PatternValueConstant::NamedNode(s) => s.into(), - PatternValueConstant::Triple(s) => s.into(), - PatternValueConstant::Literal(_) - | PatternValueConstant::DefaultGraph => unreachable!(), - }, - predicate: match plain_predicate { - PatternValueConstant::NamedNode(s) => s, - PatternValueConstant::Literal(_) - | PatternValueConstant::Triple(_) - | PatternValueConstant::DefaultGraph => unreachable!(), - }, - object: match plain_object { - PatternValueConstant::NamedNode(s) => s.into(), - PatternValueConstant::Literal(s) => s.into(), - PatternValueConstant::Triple(s) => s.into(), - PatternValueConstant::DefaultGraph => unreachable!(), - }, - })), - }), - (subject, predicate, object) => { - PatternValue::TriplePattern(Box::new(TriplePatternValue { - subject, - predicate, - object, - })) - } - } - } - } - } - - fn pattern_value_from_named_node_or_variable( - &self, - named_node_or_variable: &NamedNodePattern, - variables: &mut Vec, - ) -> PatternValue { - match named_node_or_variable { - NamedNodePattern::NamedNode(named_node) => PatternValue::Constant(PlanTerm { - encoded: self.build_term(named_node), - plain: PatternValueConstant::NamedNode(named_node.clone()), - }), - NamedNodePattern::Variable(variable) => { - PatternValue::Variable(build_plan_variable(variables, variable)) - } - } - } - - fn build_for_aggregate( - &self, - aggregate: &AggregateExpression, - variables: &mut Vec, - ) -> Result { - match aggregate { - AggregateExpression::Count { expr, distinct } => Ok(PlanAggregation { - function: PlanAggregationFunction::Count, - parameter: match expr { - Some(expr) => Some(self.build_for_expression(expr, variables)?), - None => None, - }, - distinct: *distinct, - }), - AggregateExpression::Sum { expr, distinct } => Ok(PlanAggregation { - function: PlanAggregationFunction::Sum, - parameter: Some(self.build_for_expression(expr, variables)?), - distinct: *distinct, - }), - AggregateExpression::Min { expr, distinct } => Ok(PlanAggregation { - function: PlanAggregationFunction::Min, - parameter: Some(self.build_for_expression(expr, variables)?), - distinct: *distinct, - }), - AggregateExpression::Max { expr, distinct } => Ok(PlanAggregation { - function: PlanAggregationFunction::Max, - parameter: Some(self.build_for_expression(expr, variables)?), - distinct: *distinct, - }), - AggregateExpression::Avg { expr, distinct } => Ok(PlanAggregation { - function: PlanAggregationFunction::Avg, - parameter: Some(self.build_for_expression(expr, variables)?), - distinct: *distinct, - }), - AggregateExpression::Sample { expr, distinct } => Ok(PlanAggregation { - function: PlanAggregationFunction::Sample, - parameter: Some(self.build_for_expression(expr, variables)?), - distinct: *distinct, - }), - AggregateExpression::GroupConcat { - expr, - distinct, - separator, - } => Ok(PlanAggregation { - function: PlanAggregationFunction::GroupConcat { - separator: Rc::from(separator.as_deref().unwrap_or(" ")), - }, - parameter: Some(self.build_for_expression(expr, variables)?), - distinct: *distinct, - }), - AggregateExpression::Custom { .. } => Err(EvaluationError::msg( - "Custom aggregation functions are not supported yet", - )), - } - } - - fn build_for_graph_template( - &self, - template: &[TriplePattern], - variables: &mut Vec, - ) -> Vec { - let mut bnodes = Vec::default(); - template - .iter() - .map(|triple| TripleTemplate { - subject: self.template_value_from_term_or_variable( - &triple.subject, - variables, - &mut bnodes, - ), - predicate: self - .template_value_from_named_node_or_variable(&triple.predicate, variables), - object: self.template_value_from_term_or_variable( - &triple.object, - variables, - &mut bnodes, - ), - }) - .collect() - } - - fn template_value_from_term_or_variable( - &self, - term_or_variable: &TermPattern, - variables: &mut Vec, - bnodes: &mut Vec, - ) -> TripleTemplateValue { - match term_or_variable { - TermPattern::Variable(variable) => { - TripleTemplateValue::Variable(build_plan_variable(variables, variable)) - } - TermPattern::NamedNode(node) => TripleTemplateValue::Constant(PlanTerm { - encoded: self.build_term(node), - plain: node.clone().into(), - }), - TermPattern::BlankNode(bnode) => TripleTemplateValue::BlankNode(PlanVariable { - encoded: bnode_key(bnodes, bnode), - plain: bnode.clone(), - }), - TermPattern::Literal(literal) => TripleTemplateValue::Constant(PlanTerm { - encoded: self.build_term(literal), - plain: literal.clone().into(), - }), - TermPattern::Triple(triple) => match ( - self.template_value_from_term_or_variable(&triple.subject, variables, bnodes), - self.template_value_from_named_node_or_variable(&triple.predicate, variables), - self.template_value_from_term_or_variable(&triple.object, variables, bnodes), - ) { - ( - TripleTemplateValue::Constant(subject), - TripleTemplateValue::Constant(predicate), - TripleTemplateValue::Constant(object), - ) => TripleTemplateValue::Constant(PlanTerm { - encoded: EncodedTriple { - subject: subject.encoded, - predicate: predicate.encoded, - object: object.encoded, - } - .into(), - plain: Triple { - subject: match subject.plain { - Term::NamedNode(node) => node.into(), - Term::BlankNode(node) => node.into(), - Term::Literal(_) => unreachable!(), - Term::Triple(node) => node.into(), - }, - predicate: match predicate.plain { - Term::NamedNode(node) => node, - Term::BlankNode(_) | Term::Literal(_) | Term::Triple(_) => { - unreachable!() - } - }, - object: object.plain, - } - .into(), - }), - (subject, predicate, object) => { - TripleTemplateValue::Triple(Box::new(TripleTemplate { - subject, - predicate, - object, - })) - } - }, - } - } - - fn template_value_from_named_node_or_variable( - &self, - named_node_or_variable: &NamedNodePattern, - variables: &mut Vec, - ) -> TripleTemplateValue { - match named_node_or_variable { - NamedNodePattern::Variable(variable) => { - TripleTemplateValue::Variable(build_plan_variable(variables, variable)) - } - NamedNodePattern::NamedNode(term) => TripleTemplateValue::Constant(PlanTerm { - encoded: self.build_term(term), - plain: term.clone().into(), - }), - } - } - - fn build_term<'b>(&self, term: impl Into>) -> EncodedTerm { - self.dataset.encode_term(term) - } - - fn build_triple(&self, triple: &GroundTriple) -> EncodedTerm { - EncodedTriple::new( - match &triple.subject { - GroundSubject::NamedNode(node) => self.build_term(node), - GroundSubject::Triple(triple) => self.build_triple(triple), - }, - self.build_term(&triple.predicate), - match &triple.object { - GroundTerm::NamedNode(node) => self.build_term(node), - GroundTerm::Literal(literal) => self.build_term(literal), - GroundTerm::Triple(triple) => self.build_triple(triple), - }, - ) - .into() - } -} - -fn build_plan_variable(variables: &mut Vec, variable: &Variable) -> PlanVariable { - let encoded = if let Some(key) = slice_key(variables, variable) { - key - } else { - variables.push(variable.clone()); - variables.len() - 1 - }; - PlanVariable { - plain: variable.clone(), - encoded, - } -} - -fn bnode_key(blank_nodes: &mut Vec, blank_node: &BlankNode) -> usize { - if let Some(key) = slice_key(blank_nodes, blank_node) { - key - } else { - blank_nodes.push(blank_node.clone()); - blank_nodes.len() - 1 - } -} - -fn slice_key(slice: &[T], element: &T) -> Option { - for (i, item) in slice.iter().enumerate() { - if item == element { - return Some(i); - } - } - None -} - -fn compile_static_pattern_if_exists( - pattern: &Expression, - options: Option<&Expression>, -) -> Option { - let static_pattern = if let Expression::Literal(pattern) = pattern { - (pattern.datatype() == xsd::STRING).then(|| pattern.value()) - } else { - None - }; - let static_options = if let Some(options) = options { - if let Expression::Literal(options) = options { - (options.datatype() == xsd::STRING).then(|| Some(options.value())) - } else { - None - } - } else { - Some(None) - }; - if let (Some(static_pattern), Some(static_options)) = (static_pattern, static_options) { - compile_pattern(static_pattern, static_options) - } else { - None - } -} diff --git a/lib/src/sparql/update.rs b/lib/src/sparql/update.rs index 3091172f..fb68c561 100644 --- a/lib/src/sparql/update.rs +++ b/lib/src/sparql/update.rs @@ -3,10 +3,8 @@ use crate::io::{GraphFormat, GraphParser}; use crate::model::{GraphName as OxGraphName, GraphNameRef, Quad as OxQuad}; use crate::sparql::algebra::QueryDataset; use crate::sparql::dataset::DatasetView; -use crate::sparql::eval::SimpleEvaluator; +use crate::sparql::eval::{EncodedTuple, SimpleEvaluator}; use crate::sparql::http::Client; -use crate::sparql::plan::EncodedTuple; -use crate::sparql::plan_builder::PlanBuilder; use crate::sparql::{EvaluationError, Update, UpdateOptions}; use crate::storage::numeric_encoder::{Decoder, EncodedTerm}; use crate::storage::StorageWriter; @@ -18,6 +16,7 @@ use spargebra::term::{ Quad, QuadPattern, Subject, Term, TermPattern, Triple, TriplePattern, Variable, }; use spargebra::GraphUpdateOperation; +use sparopt::Optimizer; use std::collections::HashMap; use std::io::BufReader; use std::rc::Rc; @@ -125,13 +124,12 @@ impl<'a, 'b: 'a> SimpleUpdateEvaluator<'a, 'b> { algebra: &GraphPattern, ) -> Result<(), EvaluationError> { let dataset = Rc::new(DatasetView::new(self.transaction.reader(), using)); - let (plan, variables) = PlanBuilder::build( - &dataset, - algebra, - false, - &self.options.query_options.custom_functions, - !self.options.query_options.without_optimizations, - )?; + let mut pattern = sparopt::algebra::GraphPattern::from(algebra); + if !self.options.query_options.without_optimizations { + pattern = Optimizer::optimize_graph_pattern(sparopt::algebra::GraphPattern::Reduced { + inner: Box::new(pattern), + }); + } let evaluator = SimpleEvaluator::new( Rc::clone(&dataset), self.base_iri.clone(), @@ -139,8 +137,9 @@ impl<'a, 'b: 'a> SimpleUpdateEvaluator<'a, 'b> { Rc::new(self.options.query_options.custom_functions.clone()), false, ); + let mut variables = Vec::new(); let mut bnodes = HashMap::new(); - let (eval, _) = evaluator.plan_evaluator(&plan); + let (eval, _) = evaluator.graph_pattern_evaluator(&pattern, &mut variables); let tuples = eval(EncodedTuple::with_capacity(variables.len())).collect::, _>>()?; //TODO: would be much better to stream for tuple in tuples { From 00f179058ebbdb27f8989c1a139be8d48bebbfd7 Mon Sep 17 00:00:00 2001 From: Tpt Date: Tue, 11 Jul 2023 21:03:03 +0200 Subject: [PATCH 033/217] Upgrades minimal Python version to 3.8 3.7 is EOL --- .github/workflows/artifacts.yml | 4 ++-- .github/workflows/tests.yml | 6 +++--- python/Cargo.toml | 2 +- python/generate_stubs.py | 5 +---- python/pyproject.toml | 3 +-- python/requirements.dev.txt | 2 +- 6 files changed, 9 insertions(+), 13 deletions(-) diff --git a/.github/workflows/artifacts.yml b/.github/workflows/artifacts.yml index e544b33e..a72c065d 100644 --- a/.github/workflows/artifacts.yml +++ b/.github/workflows/artifacts.yml @@ -177,7 +177,7 @@ jobs: - uses: Swatinem/rust-cache@v2 - uses: actions/setup-python@v4 with: - python-version: "3.10" + python-version: "3.11" cache: pip cache-dependency-path: '**/requirements.dev.txt' - run: pip install -r python/requirements.dev.txt @@ -211,7 +211,7 @@ jobs: - uses: Swatinem/rust-cache@v2 - uses: actions/setup-python@v4 with: - python-version: "3.10" + python-version: "3.11" cache: pip cache-dependency-path: '**/requirements.dev.txt' - run: Remove-Item -LiteralPath "C:\msys64\" -Force -Recurse diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e752ed0b..69788753 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -254,7 +254,7 @@ jobs: - uses: Swatinem/rust-cache@v2 - uses: actions/setup-python@v4 with: - python-version: "3.10" + python-version: "3.11" cache: pip cache-dependency-path: '**/requirements.dev.txt' - run: pip install -r python/requirements.dev.txt @@ -288,7 +288,7 @@ jobs: - uses: Swatinem/rust-cache@v2 - uses: actions/setup-python@v4 with: - python-version: "3.7" + python-version: "3.8" cache: pip cache-dependency-path: '**/requirements.dev.txt' - run: pip install -r python/requirements.dev.txt @@ -308,7 +308,7 @@ jobs: - uses: Swatinem/rust-cache@v2 - uses: actions/setup-python@v4 with: - python-version: "3.10" + python-version: "3.11" cache: pip cache-dependency-path: '**/requirements.dev.txt' - run: pip install "maturin~=1.0" diff --git a/python/Cargo.toml b/python/Cargo.toml index 6f3b59d4..242a4c0e 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -16,7 +16,7 @@ name = "pyoxigraph" doctest = false [features] -abi3 = ["pyo3/abi3-py37"] +abi3 = ["pyo3/abi3-py38"] [dependencies] oxigraph = { version = "0.4.0-alpha.1-dev", path="../lib", features = ["http_client"] } diff --git a/python/generate_stubs.py b/python/generate_stubs.py index 8a36f1b4..f343fd53 100644 --- a/python/generate_stubs.py +++ b/python/generate_stubs.py @@ -473,7 +473,7 @@ def build_doc_comment(doc: str) -> Optional[ast.Expr]: def format_with_black(code: str) -> str: result = subprocess.run( - ["python", "-m", "black", "-t", "py37", "--pyi", "-"], + ["python", "-m", "black", "-t", "py38", "--pyi", "-"], input=code.encode(), capture_output=True, ) @@ -498,9 +498,6 @@ if __name__ == "__main__": ) args = parser.parse_args() stub_content = ast.unparse(module_stubs(importlib.import_module(args.module_name))) - stub_content = stub_content.replace( - ", /", "" - ) # TODO: remove when targeting Python 3.8+ if args.black: stub_content = format_with_black(stub_content) args.out.write(stub_content) diff --git a/python/pyproject.toml b/python/pyproject.toml index e17c6a01..dbdc8cd1 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -11,7 +11,6 @@ classifiers = [ "License :: OSI Approved :: Apache Software License", "License :: OSI Approved :: MIT License", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", @@ -20,7 +19,7 @@ classifiers = [ "Topic :: Database :: Database Engines/Servers", "Topic :: Software Development :: Libraries :: Python Modules", ] -requires-python = ">=3.7" +requires-python = ">=3.8" [project.urls] Changelog = "https://github.com/oxigraph/oxigraph/blob/main/CHANGELOG.md" diff --git a/python/requirements.dev.txt b/python/requirements.dev.txt index 8e235e25..fb239ec8 100644 --- a/python/requirements.dev.txt +++ b/python/requirements.dev.txt @@ -3,4 +3,4 @@ furo maturin~=1.0 mypy~=1.0 ruff~=0.0.255 -sphinx~=5.3 +sphinx~=7.0 From 8a398db20e8dbc917ce563901d0f9139a813f8d0 Mon Sep 17 00:00:00 2001 From: Tpt Date: Wed, 12 Jul 2023 22:09:14 +0200 Subject: [PATCH 034/217] SPARQL: Do not unescape unicode escape everywhere but only in IRIs and strings Follows most systems behavior Issue #376 --- lib/spargebra/src/parser.rs | 298 ++++++++++-------------------------- 1 file changed, 78 insertions(+), 220 deletions(-) diff --git a/lib/spargebra/src/parser.rs b/lib/spargebra/src/parser.rs index daeddd55..0dccff3b 100644 --- a/lib/spargebra/src/parser.rs +++ b/lib/spargebra/src/parser.rs @@ -8,11 +8,9 @@ use oxrdf::vocab::{rdf, xsd}; use peg::parser; use peg::str::LineCol; use rand::random; -use std::borrow::Cow; use std::collections::{HashMap, HashSet}; use std::error::Error; use std::mem::take; -use std::str::Chars; use std::str::FromStr; use std::{char, fmt}; @@ -32,7 +30,7 @@ pub fn parse_query(query: &str, base_iri: Option<&str>) -> Result) -> Result Result, IriParseError> { + fn parse_iri(&self, iri: String) -> Result, IriParseError> { if let Some(base_iri) = &self.base_iri { - base_iri.resolve(iri) + base_iri.resolve(&iri) } else { - Iri::parse(iri.to_owned()) + Iri::parse(iri) } } @@ -746,211 +741,69 @@ impl ParserState { } } -pub fn unescape_unicode_codepoints(input: &str) -> Cow<'_, str> { - if needs_unescape_unicode_codepoints(input) { - UnescapeUnicodeCharIterator::new(input).collect() - } else { - input.into() - } -} - -fn needs_unescape_unicode_codepoints(input: &str) -> bool { - let bytes = input.as_bytes(); - for i in 1..bytes.len() { - if (bytes[i] == b'u' || bytes[i] == b'U') && bytes[i - 1] == b'\\' { - return true; - } +fn unescape_iriref(mut input: &str) -> Result { + let mut output = String::with_capacity(input.len()); + while let Some((before, after)) = input.split_once('\\') { + output.push_str(before); + let mut after = after.chars(); + let (escape, after) = match after.next() { + Some('u') => read_hex_char::<4>(after.as_str())?, + Some('U') => read_hex_char::<8>(after.as_str())?, + Some(_) => { + return Err( + "IRIs are only allowed to contain escape sequences \\uXXXX and \\UXXXXXXXX", + ) + } + None => return Err("IRIs are not allowed to end with a '\'"), + }; + output.push(escape); + input = after; } - false + output.push_str(input); + Ok(output) } -struct UnescapeUnicodeCharIterator<'a> { - iter: Chars<'a>, - buffer: String, -} - -impl<'a> UnescapeUnicodeCharIterator<'a> { - fn new(string: &'a str) -> Self { - Self { - iter: string.chars(), - buffer: String::with_capacity(9), - } +fn unescape_string(mut input: &str) -> Result { + let mut output = String::with_capacity(input.len()); + while let Some((before, after)) = input.split_once('\\') { + output.push_str(before); + let mut after = after.chars(); + let (escape, after) = match after.next() { + Some('t') => ('\u{0009}', after.as_str()), + Some('b') => ('\u{0008}', after.as_str()), + Some('n') => ('\u{000A}', after.as_str()), + Some('r') => ('\u{000D}', after.as_str()), + Some('f') => ('\u{000C}', after.as_str()), + Some('"') => ('\u{0022}', after.as_str()), + Some('\'') => ('\u{0027}', after.as_str()), + Some('\\') => ('\u{005C}', after.as_str()), + Some('u') => read_hex_char::<4>(after.as_str())?, + Some('U') => read_hex_char::<8>(after.as_str())?, + Some(_) => return Err("The character that can be escaped in strings are tbnrf\"'\\"), + None => return Err("strings are not allowed to end with a '\'"), + }; + output.push(escape); + input = after; } + output.push_str(input); + Ok(output) } -impl<'a> Iterator for UnescapeUnicodeCharIterator<'a> { - type Item = char; - - fn next(&mut self) -> Option { - if !self.buffer.is_empty() { - return Some(self.buffer.remove(0)); - } - match self.iter.next()? { - '\\' => match self.iter.next() { - Some('u') => { - self.buffer.push('u'); - for _ in 0..4 { - if let Some(c) = self.iter.next() { - self.buffer.push(c); - } else { - return Some('\\'); - } - } - if let Some(c) = u32::from_str_radix(&self.buffer[1..], 16) - .ok() - .and_then(char::from_u32) - { - self.buffer.clear(); - Some(c) - } else { - Some('\\') - } - } - Some('U') => { - self.buffer.push('U'); - for _ in 0..8 { - if let Some(c) = self.iter.next() { - self.buffer.push(c); - } else { - return Some('\\'); - } - } - if let Some(c) = u32::from_str_radix(&self.buffer[1..], 16) - .ok() - .and_then(char::from_u32) - { - self.buffer.clear(); - Some(c) - } else { - Some('\\') - } - } - Some(c) => { - self.buffer.push(c); - Some('\\') - } - None => Some('\\'), - }, - c => Some(c), +fn read_hex_char(input: &str) -> Result<(char, &str), &'static str> { + if let Some(escape) = input.get(..SIZE) { + if let Some(char) = u32::from_str_radix(escape, 16) + .ok() + .and_then(char::from_u32) + { + Ok((char, &input[SIZE..])) + } else { + Err("\\u escape sequence should be followed by hexadecimal digits") } - } -} - -pub fn unescape_characters<'a>( - input: &'a str, - characters: &'static [u8], - replacement: &'static StaticCharSliceMap, -) -> Cow<'a, str> { - if needs_unescape_characters(input, characters) { - UnescapeCharsIterator::new(input, replacement).collect() } else { - input.into() + Err("\\u escape sequence should be followed by hexadecimal digits") } } -fn needs_unescape_characters(input: &str, characters: &[u8]) -> bool { - let bytes = input.as_bytes(); - for i in 1..bytes.len() { - if bytes[i - 1] == b'\\' && characters.contains(&bytes[i]) { - return true; - } - } - false -} - -struct UnescapeCharsIterator<'a> { - iter: Chars<'a>, - buffer: Option, - replacement: &'static StaticCharSliceMap, -} - -impl<'a> UnescapeCharsIterator<'a> { - fn new(string: &'a str, replacement: &'static StaticCharSliceMap) -> Self { - Self { - iter: string.chars(), - buffer: None, - replacement, - } - } -} - -impl<'a> Iterator for UnescapeCharsIterator<'a> { - type Item = char; - - fn next(&mut self) -> Option { - if let Some(ch) = self.buffer { - self.buffer = None; - return Some(ch); - } - match self.iter.next()? { - '\\' => match self.iter.next() { - Some(ch) => { - if let Some(replace) = self.replacement.get(ch) { - Some(replace) - } else { - self.buffer = Some(ch); - Some('\\') - } - } - None => Some('\\'), - }, - c => Some(c), - } - } -} - -pub struct StaticCharSliceMap { - keys: &'static [char], - values: &'static [char], -} - -impl StaticCharSliceMap { - pub const fn new(keys: &'static [char], values: &'static [char]) -> Self { - Self { keys, values } - } - - pub fn get(&self, key: char) -> Option { - for i in 0..self.keys.len() { - if self.keys[i] == key { - return Some(self.values[i]); - } - } - None - } -} - -const UNESCAPE_CHARACTERS: [u8; 8] = [b't', b'b', b'n', b'r', b'f', b'"', b'\'', b'\\']; -const UNESCAPE_REPLACEMENT: StaticCharSliceMap = StaticCharSliceMap::new( - &['t', 'b', 'n', 'r', 'f', '"', '\'', '\\'], - &[ - '\u{0009}', '\u{0008}', '\u{000A}', '\u{000D}', '\u{000C}', '\u{0022}', '\u{0027}', - '\u{005C}', - ], -); - -fn unescape_echars(input: &str) -> Cow<'_, str> { - unescape_characters(input, &UNESCAPE_CHARACTERS, &UNESCAPE_REPLACEMENT) -} - -const UNESCAPE_PN_CHARACTERS: [u8; 20] = [ - b'_', b'~', b'.', b'-', b'!', b'$', b'&', b'\'', b'(', b')', b'*', b'+', b',', b';', b'=', - b'/', b'?', b'#', b'@', b'%', -]; -const UNESCAPE_PN_REPLACEMENT: StaticCharSliceMap = StaticCharSliceMap::new( - &[ - '_', '~', '.', '-', '!', '$', '&', '\'', '(', ')', '*', '+', ',', ';', '=', '/', '?', '#', - '@', '%', - ], - &[ - '_', '~', '.', '-', '!', '$', '&', '\'', '(', ')', '*', '+', ',', ';', '=', '/', '?', '#', - '@', '%', - ], -); - -pub fn unescape_pn_local(input: &str) -> Cow<'_, str> { - unescape_characters(input, &UNESCAPE_PN_CHARACTERS, &UNESCAPE_PN_REPLACEMENT) -} - fn variable() -> Variable { Variable::new_unchecked(format!("{:x}", random::())) } @@ -2143,7 +1996,7 @@ parser! { } / ANON() { BlankNode::default() } rule IRIREF() -> Iri = "<" i:$((!['>'] [_])*) ">" {? - state.parse_iri(i).map_err(|_| "IRI parsing failed") + state.parse_iri(unescape_iriref(i)?).map_err(|_| "IRI parsing failed") } rule PNAME_NS() -> &'input str = ns:$(PN_PREFIX()?) ":" { @@ -2152,8 +2005,11 @@ parser! { rule PNAME_LN() -> Iri = ns:PNAME_NS() local:$(PN_LOCAL()) {? if let Some(base) = state.namespaces.get(ns) { - let mut iri = base.clone(); - iri.push_str(&unescape_pn_local(local)); + let mut iri = String::with_capacity(base.len() + local.len()); + iri.push_str(base); + for chunk in local.split('\\') { // We remove \ + iri.push_str(chunk); + } Iri::parse(iri).map_err(|_| "IRI parsing failed") } else { Err("Prefix not found") @@ -2192,29 +2048,31 @@ parser! { rule EXPONENT() = ['e' | 'E'] ['+' | '-']? ['0'..='9']+ - rule STRING_LITERAL1() -> String = "'" l:$((STRING_LITERAL1_simple_char() / ECHAR())*) "'" { - unescape_echars(l).to_string() + rule STRING_LITERAL1() -> String = "'" l:$((STRING_LITERAL1_simple_char() / ECHAR() / UCHAR())*) "'" {? + unescape_string(l) } rule STRING_LITERAL1_simple_char() = !['\u{27}' | '\u{5C}' | '\u{A}' | '\u{D}'] [_] - rule STRING_LITERAL2() -> String = "\"" l:$((STRING_LITERAL2_simple_char() / ECHAR())*) "\"" { - unescape_echars(l).to_string() + rule STRING_LITERAL2() -> String = "\"" l:$((STRING_LITERAL2_simple_char() / ECHAR() / UCHAR())*) "\"" {? + unescape_string(l) } rule STRING_LITERAL2_simple_char() = !['\u{22}' | '\u{5C}' | '\u{A}' | '\u{D}'] [_] - rule STRING_LITERAL_LONG1() -> String = "'''" l:$(STRING_LITERAL_LONG1_inner()*) "'''" { - unescape_echars(l).to_string() + rule STRING_LITERAL_LONG1() -> String = "'''" l:$(STRING_LITERAL_LONG1_inner()*) "'''" {? + unescape_string(l) } - rule STRING_LITERAL_LONG1_inner() = ("''" / "'")? (STRING_LITERAL_LONG1_simple_char() / ECHAR()) + rule STRING_LITERAL_LONG1_inner() = ("''" / "'")? (STRING_LITERAL_LONG1_simple_char() / ECHAR() / UCHAR()) rule STRING_LITERAL_LONG1_simple_char() = !['\'' | '\\'] [_] - rule STRING_LITERAL_LONG2() -> String = "\"\"\"" l:$(STRING_LITERAL_LONG2_inner()*) "\"\"\"" { - unescape_echars(l).to_string() + rule STRING_LITERAL_LONG2() -> String = "\"\"\"" l:$(STRING_LITERAL_LONG2_inner()*) "\"\"\"" {? + unescape_string(l) } - rule STRING_LITERAL_LONG2_inner() = ("\"\"" / "\"")? (STRING_LITERAL_LONG2_simple_char() / ECHAR()) + rule STRING_LITERAL_LONG2_inner() = ("\"\"" / "\"")? (STRING_LITERAL_LONG2_simple_char() / ECHAR() / UCHAR()) rule STRING_LITERAL_LONG2_simple_char() = !['"' | '\\'] [_] + rule UCHAR() = "\\u" HEX() HEX() HEX() HEX() / "\\U" HEX() HEX() HEX() HEX() HEX() HEX() HEX() HEX() + rule ECHAR() = "\\" ['t' | 'b' | 'n' | 'r' | 'f' | '"' |'\'' | '\\'] rule NIL() = "(" WS()* ")" @@ -2223,7 +2081,7 @@ parser! { rule ANON() = "[" WS()* "]" - rule PN_CHARS_BASE() = ['A' ..= 'Z' | 'a' ..= 'z' | '\u{00C0}' ..='\u{00D6}' | '\u{00D8}'..='\u{00F6}' | '\u{00F8}'..='\u{02FF}' | '\u{0370}'..='\u{037D}' | '\u{037F}'..='\u{1FFF}' | '\u{200C}'..='\u{200D}' | '\u{2070}'..='\u{218F}' | '\u{2C00}'..='\u{2FEF}' | '\u{3001}'..='\u{D7FF}' | '\u{F900}'..='\u{FDCF}' | '\u{FDF0}'..='\u{FFFD}'] + rule PN_CHARS_BASE() = ['A' ..= 'Z' | 'a' ..= 'z' | '\u{00C0}'..='\u{00D6}' | '\u{00D8}'..='\u{00F6}' | '\u{00F8}'..='\u{02FF}' | '\u{0370}'..='\u{037D}' | '\u{037F}'..='\u{1FFF}' | '\u{200C}'..='\u{200D}' | '\u{2070}'..='\u{218F}' | '\u{2C00}'..='\u{2FEF}' | '\u{3001}'..='\u{D7FF}' | '\u{F900}'..='\u{FDCF}' | '\u{FDF0}'..='\u{FFFD}'] rule PN_CHARS_U() = ['_'] / PN_CHARS_BASE() From 1e1ed65d3b080624576916922df2d096ddccdd20 Mon Sep 17 00:00:00 2001 From: Tpt Date: Sun, 25 Jun 2023 13:27:50 +0200 Subject: [PATCH 035/217] Stop assuming JS platform when compiling to wasm32-unknown-unknown - Adds the "js" feature to enable JS support - Adds the "custom-now" feature to oxsdatatypes to inject a custom "now" implementation It is already possible for random with the getrandom "custom" feature Issue #471 --- .github/workflows/tests.yml | 15 ++++++- js/Cargo.toml | 2 +- lib/Cargo.toml | 5 ++- lib/oxsdatatypes/Cargo.toml | 6 ++- lib/oxsdatatypes/README.md | 16 ++++++++ lib/oxsdatatypes/src/date_time.rs | 35 ++++++++++++++-- lib/oxsdatatypes/src/duration.rs | 5 ++- lib/src/sparql/algebra.rs | 6 +-- lib/src/sparql/eval.rs | 68 ++++++++++++------------------- lib/src/sparql/mod.rs | 33 +++++++++++---- 10 files changed, 128 insertions(+), 63 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 69788753..c5010d9c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -75,6 +75,17 @@ jobs: - run: cargo clippy --lib --tests --target wasm32-wasi working-directory: ./lib + clippy_wasm_unknown: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + submodules: true + - run: rustup update && rustup target add wasm32-unknown-unknown && rustup component add clippy + - uses: Swatinem/rust-cache@v2 + - run: cargo clippy --lib --tests --target wasm32-unknown-unknown --features getrandom/custom --features oxsdatatypes/custom-now + working-directory: ./lib + clippy_msv: runs-on: ubuntu-latest steps: @@ -158,7 +169,7 @@ jobs: submodules: true - run: rustup update - uses: Swatinem/rust-cache@v2 - - run: cargo test --all-features + - run: cargo test env: RUST_BACKTRACE: 1 @@ -185,7 +196,7 @@ jobs: - run: rustup update - uses: Swatinem/rust-cache@v2 - run: Remove-Item -LiteralPath "C:\msys64\" -Force -Recurse - - run: cargo test --all-features + - run: cargo test env: RUST_BACKTRACE: 1 diff --git a/js/Cargo.toml b/js/Cargo.toml index ee103f79..86390266 100644 --- a/js/Cargo.toml +++ b/js/Cargo.toml @@ -14,7 +14,7 @@ crate-type = ["cdylib"] name = "oxigraph" [dependencies] -oxigraph = { version = "0.4.0-alpha.1-dev", path="../lib" } +oxigraph = { version = "0.4.0-alpha.1-dev", path="../lib", features = ["js"] } wasm-bindgen = "0.2" js-sys = "0.3" console_error_panic_hook = "0.1" diff --git a/lib/Cargo.toml b/lib/Cargo.toml index 1ea58475..d9be7aed 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -16,6 +16,7 @@ rust-version = "1.65" [features] default = [] +js = ["getrandom/js", "oxsdatatypes/js", "js-sys"] http_client = ["oxhttp", "oxhttp/rustls"] rocksdb_debug = [] @@ -46,8 +47,8 @@ oxrocksdb-sys = { version = "0.4.0-alpha.1-dev", path="../oxrocksdb-sys" } oxhttp = { version = "0.1", optional = true } [target.'cfg(all(target_family = "wasm", target_os = "unknown"))'.dependencies] -getrandom = { version = "0.2", features = ["js"] } -js-sys = "0.3" +getrandom = "0.2" +js-sys = { version = "0.3", optional = true } [target.'cfg(not(target_family = "wasm"))'.dev-dependencies] criterion = "0.5" diff --git a/lib/oxsdatatypes/Cargo.toml b/lib/oxsdatatypes/Cargo.toml index 1535a325..f3a1e3c3 100644 --- a/lib/oxsdatatypes/Cargo.toml +++ b/lib/oxsdatatypes/Cargo.toml @@ -13,8 +13,12 @@ An implementation of some XSD datatypes for SPARQL implementations edition = "2021" rust-version = "1.65" +[features] +js = ["js-sys"] +custom-now = [] + [target.'cfg(all(target_family = "wasm", target_os = "unknown"))'.dependencies] -js-sys = "0.3" +js-sys = { version = "0.3", optional = true } [package.metadata.docs.rs] all-features = true diff --git a/lib/oxsdatatypes/README.md b/lib/oxsdatatypes/README.md index 1f0b419a..a164f282 100644 --- a/lib/oxsdatatypes/README.md +++ b/lib/oxsdatatypes/README.md @@ -32,6 +32,22 @@ Each datatype provides: * `from_be_bytes` and `to_be_bytes` methods for serialization. +### `DateTime::now` behavior + +The `DateTime::now()` function needs special OS support. +Currently: +- If the `custom-now` feature is enabled, a function computing `now` must be set: + ```rust + use oxsdatatypes::{DateTimeError, Duration}; + + #[no_mangle] + fn custom_ox_now() -> Result { + unimplemented!("now implementation") + } + ``` +- For `wasm32-unknown-unknown` if the `js` feature is enabled the `Date.now()` ECMAScript API is used. +- For all other targets `SystemTime::now()` is used. + ## License This project is licensed under either of diff --git a/lib/oxsdatatypes/src/date_time.rs b/lib/oxsdatatypes/src/date_time.rs index b5cc8d9e..0bfe29d1 100644 --- a/lib/oxsdatatypes/src/date_time.rs +++ b/lib/oxsdatatypes/src/date_time.rs @@ -191,7 +191,7 @@ impl DateTime { pub fn checked_sub_day_time_duration(self, rhs: impl Into) -> Option { let rhs = rhs.into(); Some(Self { - timestamp: self.timestamp.checked_sub_seconds(rhs.all_seconds())?, + timestamp: self.timestamp.checked_sub_seconds(rhs.as_seconds())?, }) } @@ -1757,7 +1757,22 @@ impl Timestamp { } } -#[cfg(all(target_family = "wasm", target_os = "unknown"))] +#[cfg(feature = "custom-now")] +#[allow(unsafe_code)] +pub fn since_unix_epoch() -> Result { + extern "Rust" { + fn custom_ox_now() -> Result; + } + + unsafe { custom_ox_now() } +} + +#[cfg(all( + feature = "js", + not(feature = "custom-now"), + target_family = "wasm", + target_os = "unknown" +))] fn since_unix_epoch() -> Result { Ok(Duration::new( 0, @@ -1766,7 +1781,10 @@ fn since_unix_epoch() -> Result { )) } -#[cfg(not(all(target_family = "wasm", target_os = "unknown")))] +#[cfg(not(any( + feature = "custom-now", + all(feature = "js", target_family = "wasm", target_os = "unknown") +)))] fn since_unix_epoch() -> Result { use std::time::SystemTime; @@ -2605,6 +2623,17 @@ mod tests { Ok(()) } + #[cfg(feature = "custom-now")] + #[test] + fn custom_now() { + #[no_mangle] + fn custom_ox_now() -> Result { + Ok(Duration::default()) + } + assert!(DateTime::now().is_ok()); + } + + #[cfg(not(feature = "custom-now"))] #[test] fn now() -> Result<(), XsdParseError> { let now = DateTime::now().unwrap(); diff --git a/lib/oxsdatatypes/src/duration.rs b/lib/oxsdatatypes/src/duration.rs index 4815eb16..27ce5d97 100644 --- a/lib/oxsdatatypes/src/duration.rs +++ b/lib/oxsdatatypes/src/duration.rs @@ -85,7 +85,7 @@ impl Duration { #[inline] #[must_use] pub(super) const fn all_seconds(self) -> Decimal { - self.day_time.all_seconds() + self.day_time.as_seconds() } #[inline] @@ -443,8 +443,9 @@ impl DayTimeDuration { self.seconds.checked_rem(60).unwrap() } + /// The duration in seconds. #[inline] - pub(super) const fn all_seconds(self) -> Decimal { + pub const fn as_seconds(self) -> Decimal { self.seconds } diff --git a/lib/src/sparql/algebra.rs b/lib/src/sparql/algebra.rs index 41fad114..f468e69e 100644 --- a/lib/src/sparql/algebra.rs +++ b/lib/src/sparql/algebra.rs @@ -6,10 +6,10 @@ use crate::model::*; use crate::sparql::eval::Timer; +use oxsdatatypes::DayTimeDuration; use spargebra::GraphUpdateOperation; use std::fmt; use std::str::FromStr; -use std::time::Duration; /// A parsed [SPARQL query](https://www.w3.org/TR/sparql11-query/). /// @@ -32,7 +32,7 @@ use std::time::Duration; pub struct Query { pub(super) inner: spargebra::Query, pub(super) dataset: QueryDataset, - pub(super) parsing_duration: Option, + pub(super) parsing_duration: Option, } impl Query { @@ -43,7 +43,7 @@ impl Query { Ok(Self { dataset: query.dataset, inner: query.inner, - parsing_duration: Some(start.elapsed()), + parsing_duration: start.elapsed(), }) } diff --git a/lib/src/sparql/eval.rs b/lib/src/sparql/eval.rs index 82f0e06f..90a52efa 100644 --- a/lib/src/sparql/eval.rs +++ b/lib/src/sparql/eval.rs @@ -35,9 +35,6 @@ use std::hash::{Hash, Hasher}; use std::iter::Iterator; use std::iter::{empty, once}; use std::rc::Rc; -use std::time::Duration as StdDuration; -#[cfg(not(all(target_family = "wasm", target_os = "unknown")))] -use std::time::Instant; use std::{fmt, io, str}; const REGEX_SIZE_LIMIT: usize = 1_000_000; @@ -258,16 +255,19 @@ impl SimpleEvaluator { label: eval_node_label(pattern), children: stat_children, exec_count: Cell::new(0), - exec_duration: Cell::new(StdDuration::from_secs(0)), + exec_duration: Cell::new(self.run_stats.then(DayTimeDuration::default)), }); if self.run_stats { let stats = Rc::clone(&stats); evaluator = Rc::new(move |tuple| { let start = Timer::now(); let inner = evaluator(tuple); - stats - .exec_duration - .set(stats.exec_duration.get() + start.elapsed()); + stats.exec_duration.set( + stats + .exec_duration + .get() + .and_then(|stat| stat.checked_add(start.elapsed()?)), + ); Box::new(StatsIterator { inner, stats: Rc::clone(&stats), @@ -5648,9 +5648,12 @@ impl Iterator for StatsIterator { fn next(&mut self) -> Option> { let start = Timer::now(); let result = self.inner.next(); - self.stats - .exec_duration - .set(self.stats.exec_duration.get() + start.elapsed()); + self.stats.exec_duration.set( + self.stats + .exec_duration + .get() + .and_then(|stat| stat.checked_add(start.elapsed()?)), + ); if matches!(result, Some(Ok(_))) { self.stats.exec_count.set(self.stats.exec_count.get() + 1); } @@ -5662,7 +5665,7 @@ pub struct EvalNodeWithStats { pub label: String, pub children: Vec>, pub exec_count: Cell, - pub exec_duration: Cell, + pub exec_duration: Cell>, } impl EvalNodeWithStats { @@ -5677,10 +5680,10 @@ impl EvalNodeWithStats { if with_stats { writer.write_event(JsonEvent::ObjectKey("number of results"))?; writer.write_event(JsonEvent::Number(&self.exec_count.get().to_string()))?; - writer.write_event(JsonEvent::ObjectKey("duration in seconds"))?; - writer.write_event(JsonEvent::Number( - &self.exec_duration.get().as_secs_f32().to_string(), - ))?; + if let Some(duration) = self.exec_duration.get() { + writer.write_event(JsonEvent::ObjectKey("duration in seconds"))?; + writer.write_event(JsonEvent::Number(&duration.as_seconds().to_string()))?; + } } writer.write_event(JsonEvent::ObjectKey("children"))?; writer.write_event(JsonEvent::StartArray)?; @@ -5696,9 +5699,12 @@ impl fmt::Debug for EvalNodeWithStats { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let mut obj = f.debug_struct("Node"); obj.field("name", &self.label); - if self.exec_duration.get() > StdDuration::default() { + if let Some(exec_duration) = self.exec_duration.get() { obj.field("number of results", &self.exec_count.get()); - obj.field("duration in seconds", &self.exec_duration.get()); + obj.field( + "duration in seconds", + &f32::from(Float::from(exec_duration.as_seconds())), + ); } if !self.children.is_empty() { obj.field("children", &self.children); @@ -5844,39 +5850,19 @@ fn format_list(values: impl IntoIterator) -> String { .join(", ") } -#[cfg(all(target_family = "wasm", target_os = "unknown"))] -pub struct Timer { - timestamp_ms: f64, -} - -#[cfg(all(target_family = "wasm", target_os = "unknown"))] -impl Timer { - pub fn now() -> Self { - Self { - timestamp_ms: js_sys::Date::now(), - } - } - - pub fn elapsed(&self) -> StdDuration { - StdDuration::from_secs_f64((js_sys::Date::now() - self.timestamp_ms) / 1000.) - } -} - -#[cfg(not(all(target_family = "wasm", target_os = "unknown")))] pub struct Timer { - instant: Instant, + start: Option, } -#[cfg(not(all(target_family = "wasm", target_os = "unknown")))] impl Timer { pub fn now() -> Self { Self { - instant: Instant::now(), + start: DateTime::now().ok(), } } - pub fn elapsed(&self) -> StdDuration { - self.instant.elapsed() + pub fn elapsed(&self) -> Option { + DateTime::now().ok()?.checked_sub(self.start?) } } diff --git a/lib/src/sparql/mod.rs b/lib/src/sparql/mod.rs index d5e7233e..95249203 100644 --- a/lib/src/sparql/mod.rs +++ b/lib/src/sparql/mod.rs @@ -23,6 +23,7 @@ pub(crate) use crate::sparql::update::evaluate_update; use crate::storage::StorageReader; use json_event_parser::{JsonEvent, JsonWriter}; pub use oxrdf::{Variable, VariableNameParseError}; +use oxsdatatypes::{DayTimeDuration, Float}; pub use sparesults::QueryResultsFormat; pub use spargebra::ParseError; use sparopt::algebra::GraphPattern; @@ -272,8 +273,8 @@ impl From for UpdateOptions { pub struct QueryExplanation { inner: Rc, with_stats: bool, - parsing_duration: Option, - planning_duration: Duration, + parsing_duration: Option, + planning_duration: Option, } impl QueryExplanation { @@ -284,13 +285,15 @@ impl QueryExplanation { if let Some(parsing_duration) = self.parsing_duration { writer.write_event(JsonEvent::ObjectKey("parsing duration in seconds"))?; writer.write_event(JsonEvent::Number( - &parsing_duration.as_secs_f32().to_string(), + &parsing_duration.as_seconds().to_string(), + ))?; + } + if let Some(planning_duration) = self.planning_duration { + writer.write_event(JsonEvent::ObjectKey("planning duration in seconds"))?; + writer.write_event(JsonEvent::Number( + &planning_duration.as_seconds().to_string(), ))?; } - writer.write_event(JsonEvent::ObjectKey("planning duration in seconds"))?; - writer.write_event(JsonEvent::Number( - &self.planning_duration.as_secs_f32().to_string(), - ))?; writer.write_event(JsonEvent::ObjectKey("plan"))?; self.inner.json_node(&mut writer, self.with_stats)?; writer.write_event(JsonEvent::EndObject) @@ -299,6 +302,20 @@ impl QueryExplanation { impl fmt::Debug for QueryExplanation { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{:?}", self.inner) + let mut obj = f.debug_struct("QueryExplanation"); + if let Some(parsing_duration) = self.parsing_duration { + obj.field( + "parsing duration in seconds", + &f32::from(Float::from(parsing_duration.as_seconds())), + ); + } + if let Some(planning_duration) = self.planning_duration { + obj.field( + "planning duration in seconds", + &f32::from(Float::from(planning_duration.as_seconds())), + ); + } + obj.field("tree", &self.inner); + obj.finish() } } From b22e74379ae235846400ac427c5fd449327b524f Mon Sep 17 00:00:00 2001 From: Tpt Date: Sun, 16 Jul 2023 22:25:31 +0200 Subject: [PATCH 036/217] Run RDF canon tests to check isomorphism Automated report generation --- .gitmodules | 3 +++ testsuite/rdf-canon | 1 + testsuite/src/parser_evaluator.rs | 12 ++++++++++++ testsuite/tests/canonicalization.rs | 7 +++++++ 4 files changed, 23 insertions(+) create mode 160000 testsuite/rdf-canon create mode 100644 testsuite/tests/canonicalization.rs diff --git a/.gitmodules b/.gitmodules index 0c6310f6..fba23723 100644 --- a/.gitmodules +++ b/.gitmodules @@ -17,3 +17,6 @@ path = testsuite/N3 url = https://github.com/w3c/N3.git branch = master +[submodule "testsuite/rdf-canon"] + path = testsuite/rdf-canon + url = https://github.com/w3c/rdf-canon.git diff --git a/testsuite/rdf-canon b/testsuite/rdf-canon new file mode 160000 index 00000000..9b3efebe --- /dev/null +++ b/testsuite/rdf-canon @@ -0,0 +1 @@ +Subproject commit 9b3efebebe0e5338debfc4d3ddd02f6adf0a852c diff --git a/testsuite/src/parser_evaluator.rs b/testsuite/src/parser_evaluator.rs index 8e1a818c..975c2e9b 100644 --- a/testsuite/src/parser_evaluator.rs +++ b/testsuite/src/parser_evaluator.rs @@ -67,6 +67,18 @@ pub fn register_parser_tests(evaluator: &mut TestEvaluator) { evaluator.register("http://www.w3.org/ns/rdftest#TestTrigNegativeEval", |t| { evaluate_negative_dataset_syntax_test(t, DatasetFormat::TriG) }); + evaluator.register( + "https://w3c.github.io/rdf-canon/tests/vocab#RDFC10EvalTest", + |t| evaluate_positive_dataset_syntax_test(t, DatasetFormat::NQuads), //TODO: not a proper implementation! + ); + evaluator.register( + "https://w3c.github.io/rdf-canon/tests/vocab#RDFC10NegativeEvalTest", + |_| Ok(()), //TODO: not a proper implementation + ); + evaluator.register( + "https://w3c.github.io/rdf-canon/tests/vocab#RDFC10MapTest", + |_| Ok(()), //TODO: not a proper implementation + ); evaluator.register( "https://github.com/oxigraph/oxigraph/tests#TestNTripleRecovery", |t| evaluate_graph_eval_test(t, GraphFormat::NTriples, true), diff --git a/testsuite/tests/canonicalization.rs b/testsuite/tests/canonicalization.rs new file mode 100644 index 00000000..115e5c5a --- /dev/null +++ b/testsuite/tests/canonicalization.rs @@ -0,0 +1,7 @@ +use anyhow::Result; +use oxigraph_testsuite::check_testsuite; + +#[test] +fn rdf_canon_w3c_testsuite() -> Result<()> { + check_testsuite("http://w3c.github.io/rdf-canon/tests/manifest.ttl", &[]) +} From 077c1fc1a85bd27b783d78daa06c1b6b215b40b3 Mon Sep 17 00:00:00 2001 From: Tpt Date: Thu, 20 Jul 2023 18:47:45 +0200 Subject: [PATCH 037/217] Improves XSD errors and code organization --- lib/oxsdatatypes/README.md | 4 +- lib/oxsdatatypes/src/date_time.rs | 989 +++++++++++++++++++++++------- lib/oxsdatatypes/src/decimal.rs | 299 ++++++--- lib/oxsdatatypes/src/duration.rs | 484 +++++++++++++-- lib/oxsdatatypes/src/integer.rs | 64 +- lib/oxsdatatypes/src/lib.rs | 13 +- lib/oxsdatatypes/src/parser.rs | 626 ------------------- lib/src/sparql/eval.rs | 24 +- 8 files changed, 1492 insertions(+), 1011 deletions(-) delete mode 100644 lib/oxsdatatypes/src/parser.rs diff --git a/lib/oxsdatatypes/README.md b/lib/oxsdatatypes/README.md index a164f282..f198d5ae 100644 --- a/lib/oxsdatatypes/README.md +++ b/lib/oxsdatatypes/README.md @@ -38,10 +38,10 @@ The `DateTime::now()` function needs special OS support. Currently: - If the `custom-now` feature is enabled, a function computing `now` must be set: ```rust - use oxsdatatypes::{DateTimeError, Duration}; + use oxsdatatypes::Duration; #[no_mangle] - fn custom_ox_now() -> Result { + fn custom_ox_now() -> Duration { unimplemented!("now implementation") } ``` diff --git a/lib/oxsdatatypes/src/date_time.rs b/lib/oxsdatatypes/src/date_time.rs index 0bfe29d1..671d00a8 100644 --- a/lib/oxsdatatypes/src/date_time.rs +++ b/lib/oxsdatatypes/src/date_time.rs @@ -1,14 +1,11 @@ -use super::{DayTimeDuration, Decimal, Duration, XsdParseError, YearMonthDuration}; -use crate::parser::{ - parse_date, parse_date_time, parse_g_day, parse_g_month, parse_g_month_day, parse_g_year, - parse_g_year_month, parse_time, -}; +#![allow(clippy::expect_used)] + +use crate::{DayTimeDuration, Decimal, Duration, YearMonthDuration}; use std::cmp::{min, Ordering}; use std::error::Error; use std::fmt; use std::hash::{Hash, Hasher}; use std::str::FromStr; -use std::time::SystemTimeError; /// [XML Schema `dateTime` datatype](https://www.w3.org/TR/xmlschema11-2/#dateTime) /// @@ -29,7 +26,7 @@ impl DateTime { minute: u8, second: Decimal, timezone_offset: Option, - ) -> Result { + ) -> Result { Ok(Self { timestamp: Timestamp::new(&DateTimeSevenPropertyModel { year: Some(year), @@ -45,10 +42,10 @@ impl DateTime { /// [fn:current-dateTime](https://www.w3.org/TR/xpath-functions-31/#func-current-dateTime) #[inline] - pub fn now() -> Result { - Ok(Self { - timestamp: Timestamp::now()?, - }) + pub fn now() -> Self { + Self { + timestamp: Timestamp::now(), + } } #[inline] @@ -134,6 +131,8 @@ impl DateTime { } /// [op:subtract-dateTimes](https://www.w3.org/TR/xpath-functions-31/#func-subtract-dateTimes) + /// + /// Returns `None` in case of overflow ([`FODT0001`](https://www.w3.org/TR/xpath-functions-31/#ERRFODT0001)). #[inline] #[must_use] pub fn checked_sub(self, rhs: impl Into) -> Option { @@ -141,6 +140,8 @@ impl DateTime { } /// [op:add-yearMonthDuration-to-dateTime](https://www.w3.org/TR/xpath-functions-31/#func-add-yearMonthDuration-to-dateTime) + /// + /// Returns `None` in case of overflow ([`FODT0001`](https://www.w3.org/TR/xpath-functions-31/#ERRFODT0001)). #[inline] #[must_use] pub fn checked_add_year_month_duration( @@ -151,6 +152,8 @@ impl DateTime { } /// [op:add-dayTimeDuration-to-dateTime](https://www.w3.org/TR/xpath-functions-31/#func-add-dayTimeDuration-to-dateTime) + /// + /// Returns `None` in case of overflow ([`FODT0001`](https://www.w3.org/TR/xpath-functions-31/#ERRFODT0001)). #[inline] #[must_use] pub fn checked_add_day_time_duration(self, rhs: impl Into) -> Option { @@ -161,6 +164,8 @@ impl DateTime { } /// [op:add-yearMonthDuration-to-dateTime](https://www.w3.org/TR/xpath-functions-31/#func-add-yearMonthDuration-to-dateTime) and [op:add-dayTimeDuration-to-dateTime](https://www.w3.org/TR/xpath-functions-31/#func-add-dayTimeDuration-to-dateTime) + /// + /// Returns `None` in case of overflow ([`FODT0001`](https://www.w3.org/TR/xpath-functions-31/#ERRFODT0001)). #[inline] #[must_use] pub fn checked_add_duration(self, rhs: impl Into) -> Option { @@ -176,6 +181,8 @@ impl DateTime { } /// [op:subtract-yearMonthDuration-from-dateTime](https://www.w3.org/TR/xpath-functions-31/#func-subtract-yearMonthDuration-from-dateTime) + /// + /// Returns `None` in case of overflow ([`FODT0001`](https://www.w3.org/TR/xpath-functions-31/#ERRFODT0001)). #[inline] #[must_use] pub fn checked_sub_year_month_duration( @@ -186,6 +193,8 @@ impl DateTime { } /// [op:subtract-dayTimeDuration-from-dateTime](https://www.w3.org/TR/xpath-functions-31/#func-subtract-dayTimeDuration-from-dateTime) + /// + /// Returns `None` in case of overflow ([`FODT0001`](https://www.w3.org/TR/xpath-functions-31/#ERRFODT0001)). #[inline] #[must_use] pub fn checked_sub_day_time_duration(self, rhs: impl Into) -> Option { @@ -196,6 +205,8 @@ impl DateTime { } /// [op:subtract-yearMonthDuration-from-dateTime](https://www.w3.org/TR/xpath-functions-31/#func-subtract-yearMonthDuration-from-dateTime) and [op:subtract-dayTimeDuration-from-dateTime](https://www.w3.org/TR/xpath-functions-31/#func-subtract-dayTimeDuration-from-dateTime) + /// + /// Returns `None` in case of overflow ([`FODT0001`](https://www.w3.org/TR/xpath-functions-31/#ERRFODT0001)). #[inline] #[must_use] pub fn checked_sub_duration(self, rhs: impl Into) -> Option { @@ -214,6 +225,8 @@ impl DateTime { } /// [fn:adjust-dateTime-to-timezone](https://www.w3.org/TR/xpath-functions-31/#func-adjust-dateTime-to-timezone) + /// + /// Returns `None` in case of overflow ([`FODT0001`](https://www.w3.org/TR/xpath-functions-31/#ERRFODT0001)). #[inline] #[must_use] pub fn adjust(self, timezone_offset: Option) -> Option { @@ -228,14 +241,22 @@ impl DateTime { pub fn is_identical_with(self, other: Self) -> bool { self.timestamp.is_identical_with(other.timestamp) } + + pub const MIN: Self = Self { + timestamp: Timestamp::MIN, + }; + + pub const MAX: Self = Self { + timestamp: Timestamp::MAX, + }; } /// Conversion according to [XPath cast rules](https://www.w3.org/TR/xpath-functions-31/#casting-to-datetimes). impl TryFrom for DateTime { - type Error = DateTimeError; + type Error = DateTimeOverflowError; #[inline] - fn try_from(date: Date) -> Result { + fn try_from(date: Date) -> Result { Self::new( date.year(), date.month(), @@ -249,10 +270,10 @@ impl TryFrom for DateTime { } impl FromStr for DateTime { - type Err = XsdParseError; + type Err = ParseDateTimeError; - fn from_str(input: &str) -> Result { - parse_date_time(input) + fn from_str(input: &str) -> Result { + ensure_complete(input, date_time_lexical_rep) } } @@ -263,6 +284,7 @@ impl fmt::Display for DateTime { if year < 0 { write!(f, "-")?; } + let second = self.second(); write!( f, "{:04}-{:02}-{:02}T{:02}:{:02}:{}{}", @@ -271,12 +293,12 @@ impl fmt::Display for DateTime { self.day(), self.hour(), self.minute(), - if self.second().abs() >= 10.into() { - "" - } else { + if Decimal::from(-10) < second && second < Decimal::from(10) { "0" + } else { + "" }, - self.second() + second )?; if let Some(timezone_offset) = self.timezone_offset() { write!(f, "{timezone_offset}")?; @@ -296,12 +318,12 @@ pub struct Time { impl Time { #[inline] - pub(super) fn new( + fn new( mut hour: u8, minute: u8, second: Decimal, timezone_offset: Option, - ) -> Result { + ) -> Result { if hour == 24 && minute == 0 && second == Decimal::default() { hour = 0; } @@ -328,8 +350,10 @@ impl Time { /// [fn:current-time](https://www.w3.org/TR/xpath-functions-31/#func-current-time) #[inline] - pub fn now() -> Result { - DateTime::now()?.try_into() + pub fn now() -> Self { + Self { + timestamp: Timestamp::now(), + } } /// [fn:hour-from-time](https://www.w3.org/TR/xpath-functions-31/#func-hours-from-time) @@ -373,6 +397,8 @@ impl Time { } /// [op:subtract-times](https://www.w3.org/TR/xpath-functions-31/#func-subtract-times) + /// + /// Returns `None` in case of overflow ([`FODT0001`](https://www.w3.org/TR/xpath-functions-31/#ERRFODT0001)). #[inline] #[must_use] pub fn checked_sub(self, rhs: impl Into) -> Option { @@ -380,6 +406,8 @@ impl Time { } /// [op:add-dayTimeDuration-to-time](https://www.w3.org/TR/xpath-functions-31/#func-add-dayTimeDuration-to-time) + /// + /// Returns `None` in case of overflow ([`FODT0001`](https://www.w3.org/TR/xpath-functions-31/#ERRFODT0001)). #[inline] #[must_use] pub fn checked_add_day_time_duration(self, rhs: impl Into) -> Option { @@ -387,6 +415,8 @@ impl Time { } /// [op:add-dayTimeDuration-to-time](https://www.w3.org/TR/xpath-functions-31/#func-add-dayTimeDuration-to-time) + /// + /// Returns `None` in case of overflow ([`FODT0001`](https://www.w3.org/TR/xpath-functions-31/#ERRFODT0001)). #[inline] #[must_use] pub fn checked_add_duration(self, rhs: impl Into) -> Option { @@ -406,6 +436,8 @@ impl Time { } /// [op:subtract-dayTimeDuration-from-time](https://www.w3.org/TR/xpath-functions-31/#func-subtract-dayTimeDuration-from-time) + /// + /// Returns `None` in case of overflow ([`FODT0001`](https://www.w3.org/TR/xpath-functions-31/#ERRFODT0001)). #[inline] #[must_use] pub fn checked_sub_day_time_duration(self, rhs: impl Into) -> Option { @@ -413,6 +445,8 @@ impl Time { } /// [op:subtract-dayTimeDuration-from-time](https://www.w3.org/TR/xpath-functions-31/#func-subtract-dayTimeDuration-from-time) + /// + /// Returns `None` in case of overflow ([`FODT0001`](https://www.w3.org/TR/xpath-functions-31/#ERRFODT0001)). #[inline] #[must_use] pub fn checked_sub_duration(self, rhs: impl Into) -> Option { @@ -456,45 +490,61 @@ impl Time { pub fn is_identical_with(self, other: Self) -> bool { self.timestamp.is_identical_with(other.timestamp) } + + #[cfg(test)] + const MIN: Self = Self { + timestamp: Timestamp { + value: Decimal::new_from_i128_unchecked(62_230_154_400), + timezone_offset: Some(TimezoneOffset::MAX), + }, + }; + + #[cfg(test)] + const MAX: Self = Self { + timestamp: Timestamp { + value: Decimal::new_from_i128_unchecked(62_230_255_200), + timezone_offset: Some(TimezoneOffset::MIN), + }, + }; } /// Conversion according to [XPath cast rules](https://www.w3.org/TR/xpath-functions-31/#casting-to-datetimes). -impl TryFrom for Time { - type Error = DateTimeError; - +impl From for Time { #[inline] - fn try_from(date_time: DateTime) -> Result { + fn from(date_time: DateTime) -> Self { Self::new( date_time.hour(), date_time.minute(), date_time.second(), date_time.timezone_offset(), ) + .expect("Casting from xsd:dateTime to xsd:date can't fail") } } impl FromStr for Time { - type Err = XsdParseError; + type Err = ParseDateTimeError; - fn from_str(input: &str) -> Result { - parse_time(input) + fn from_str(input: &str) -> Result { + ensure_complete(input, time_lexical_rep) } } impl fmt::Display for Time { #[inline] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let second = self.second(); write!( f, "{:02}:{:02}:{}{}", self.hour(), self.minute(), - if self.second().abs() >= 10.into() { - "" - } else { + if Decimal::from(-10) < second && second < Decimal::from(10) { "0" + } else { + "" }, - self.second() + second )?; if let Some(timezone_offset) = self.timezone_offset() { write!(f, "{timezone_offset}")?; @@ -514,12 +564,12 @@ pub struct Date { impl Date { #[inline] - pub(super) fn new( + fn new( year: i64, month: u8, day: u8, timezone_offset: Option, - ) -> Result { + ) -> Result { Ok(Self { timestamp: Timestamp::new(&DateTimeSevenPropertyModel { year: Some(year), @@ -543,8 +593,10 @@ impl Date { /// [fn:current-date](https://www.w3.org/TR/xpath-functions-31/#func-current-date) #[inline] - pub fn now() -> Result { - DateTime::now()?.try_into() + pub fn now() -> Self { + DateTime::now() + .try_into() + .expect("The current time seems way in the future, it's strange") } /// [fn:year-from-date](https://www.w3.org/TR/xpath-functions-31/#func-year-from-date) @@ -588,6 +640,8 @@ impl Date { } /// [op:subtract-dates](https://www.w3.org/TR/xpath-functions-31/#func-subtract-dates) + /// + /// Returns `None` in case of overflow ([`FODT0001`](https://www.w3.org/TR/xpath-functions-31/#ERRFODT0001)). #[inline] #[must_use] pub fn checked_sub(self, rhs: impl Into) -> Option { @@ -595,6 +649,8 @@ impl Date { } /// [op:add-yearMonthDuration-to-date](https://www.w3.org/TR/xpath-functions-31/#func-add-yearMonthDuration-to-date) + /// + /// Returns `None` in case of overflow ([`FODT0001`](https://www.w3.org/TR/xpath-functions-31/#ERRFODT0001)). #[inline] #[must_use] pub fn checked_add_year_month_duration( @@ -605,6 +661,8 @@ impl Date { } /// [op:add-dayTimeDuration-to-dateTime](https://www.w3.org/TR/xpath-functions-31/#func-add-dayTimeDuration-to-date) + /// + /// Returns `None` in case of overflow ([`FODT0001`](https://www.w3.org/TR/xpath-functions-31/#ERRFODT0001)). #[inline] #[must_use] pub fn checked_add_day_time_duration(self, rhs: impl Into) -> Option { @@ -612,6 +670,8 @@ impl Date { } /// [op:add-yearMonthDuration-to-date](https://www.w3.org/TR/xpath-functions-31/#func-add-yearMonthDuration-to-date) and [op:add-dayTimeDuration-to-dateTime](https://www.w3.org/TR/xpath-functions-31/#func-add-dayTimeDuration-to-date) + /// + /// Returns `None` in case of overflow ([`FODT0001`](https://www.w3.org/TR/xpath-functions-31/#ERRFODT0001)). #[inline] #[must_use] pub fn checked_add_duration(self, rhs: impl Into) -> Option { @@ -623,6 +683,8 @@ impl Date { } /// [op:subtract-yearMonthDuration-from-date](https://www.w3.org/TR/xpath-functions-31/#func-subtract-yearMonthDuration-from-date) + /// + /// Returns `None` in case of overflow ([`FODT0001`](https://www.w3.org/TR/xpath-functions-31/#ERRFODT0001)). #[inline] #[must_use] pub fn checked_sub_year_month_duration( @@ -633,6 +695,8 @@ impl Date { } /// [op:subtract-dayTimeDuration-from-date](https://www.w3.org/TR/xpath-functions-31/#func-subtract-dayTimeDuration-from-date) + /// + /// Returns `None` in case of overflow ([`FODT0001`](https://www.w3.org/TR/xpath-functions-31/#ERRFODT0001)). #[inline] #[must_use] pub fn checked_sub_day_time_duration(self, rhs: impl Into) -> Option { @@ -675,14 +739,27 @@ impl Date { pub fn is_identical_with(self, other: Self) -> bool { self.timestamp.is_identical_with(other.timestamp) } + + pub const MIN: Self = Self { + timestamp: Timestamp { + value: Decimal::new_from_i128_unchecked(-170_141_183_460_469_216_800), + timezone_offset: Some(TimezoneOffset::MIN), + }, + }; + pub const MAX: Self = Self { + timestamp: Timestamp { + value: Decimal::new_from_i128_unchecked(170_141_183_460_469_216_800), + timezone_offset: Some(TimezoneOffset::MAX), + }, + }; } /// Conversion according to [XPath cast rules](https://www.w3.org/TR/xpath-functions-31/#casting-to-datetimes). impl TryFrom for Date { - type Error = DateTimeError; + type Error = DateTimeOverflowError; #[inline] - fn try_from(date_time: DateTime) -> Result { + fn try_from(date_time: DateTime) -> Result { Self::new( date_time.year(), date_time.month(), @@ -693,10 +770,10 @@ impl TryFrom for Date { } impl FromStr for Date { - type Err = XsdParseError; + type Err = ParseDateTimeError; - fn from_str(input: &str) -> Result { - parse_date(input) + fn from_str(input: &str) -> Result { + ensure_complete(input, date_lexical_rep) } } @@ -726,11 +803,11 @@ pub struct GYearMonth { impl GYearMonth { #[inline] - pub(super) fn new( + fn new( year: i64, month: u8, timezone_offset: Option, - ) -> Result { + ) -> Result { Ok(Self { timestamp: Timestamp::new(&DateTimeSevenPropertyModel { year: Some(year), @@ -796,14 +873,27 @@ impl GYearMonth { pub fn is_identical_with(self, other: Self) -> bool { self.timestamp.is_identical_with(other.timestamp) } + + pub const MIN: Self = Self { + timestamp: Timestamp { + value: Decimal::new_from_i128_unchecked(-170_141_183_460_466_970_400), + timezone_offset: Some(TimezoneOffset::MIN), + }, + }; + pub const MAX: Self = Self { + timestamp: Timestamp { + value: Decimal::new_from_i128_unchecked(170_141_183_460_469_216_800), + timezone_offset: Some(TimezoneOffset::MAX), + }, + }; } /// Conversion according to [XPath cast rules](https://www.w3.org/TR/xpath-functions-31/#casting-to-datetimes). impl TryFrom for GYearMonth { - type Error = DateTimeError; + type Error = DateTimeOverflowError; #[inline] - fn try_from(date_time: DateTime) -> Result { + fn try_from(date_time: DateTime) -> Result { Self::new( date_time.year(), date_time.month(), @@ -813,20 +903,19 @@ impl TryFrom for GYearMonth { } /// Conversion according to [XPath cast rules](https://www.w3.org/TR/xpath-functions-31/#casting-to-datetimes). -impl TryFrom for GYearMonth { - type Error = DateTimeError; - +impl From for GYearMonth { #[inline] - fn try_from(date: Date) -> Result { + fn from(date: Date) -> Self { Self::new(date.year(), date.month(), date.timezone_offset()) + .expect("Casting from xsd:date to xsd:gYearMonth can't fail") } } impl FromStr for GYearMonth { - type Err = XsdParseError; + type Err = ParseDateTimeError; - fn from_str(input: &str) -> Result { - parse_g_year_month(input) + fn from_str(input: &str) -> Result { + ensure_complete(input, g_year_month_lexical_rep) } } @@ -856,10 +945,10 @@ pub struct GYear { impl GYear { #[inline] - pub(super) fn new( + fn new( year: i64, timezone_offset: Option, - ) -> Result { + ) -> Result { Ok(Self { timestamp: Timestamp::new(&DateTimeSevenPropertyModel { year: Some(year), @@ -919,42 +1008,55 @@ impl GYear { pub fn is_identical_with(self, other: Self) -> bool { self.timestamp.is_identical_with(other.timestamp) } + + pub const MIN: Self = Self { + timestamp: Timestamp { + value: Decimal::new_from_i128_unchecked(-170_141_183_460_461_700_000), + timezone_offset: Some(TimezoneOffset::MIN), + }, + }; + pub const MAX: Self = Self { + timestamp: Timestamp { + value: Decimal::new_from_i128_unchecked(170_141_183_460_461_440_800), + timezone_offset: Some(TimezoneOffset::MAX), + }, + }; } /// Conversion according to [XPath cast rules](https://www.w3.org/TR/xpath-functions-31/#casting-to-datetimes). impl TryFrom for GYear { - type Error = DateTimeError; + type Error = DateTimeOverflowError; #[inline] - fn try_from(date_time: DateTime) -> Result { + fn try_from(date_time: DateTime) -> Result { Self::new(date_time.year(), date_time.timezone_offset()) } } /// Conversion according to [XPath cast rules](https://www.w3.org/TR/xpath-functions-31/#casting-to-datetimes). impl TryFrom for GYear { - type Error = DateTimeError; + type Error = DateTimeOverflowError; #[inline] - fn try_from(date: Date) -> Result { + fn try_from(date: Date) -> Result { Self::new(date.year(), date.timezone_offset()) } } impl TryFrom for GYear { - type Error = DateTimeError; + type Error = DateTimeOverflowError; #[inline] - fn try_from(year_month: GYearMonth) -> Result { + fn try_from(year_month: GYearMonth) -> Result { Self::new(year_month.year(), year_month.timezone_offset()) } } impl FromStr for GYear { - type Err = XsdParseError; + type Err = ParseDateTimeError; - fn from_str(input: &str) -> Result { - parse_g_year(input) + fn from_str(input: &str) -> Result { + ensure_complete(input, g_year_lexical_rep) } } @@ -984,11 +1086,11 @@ pub struct GMonthDay { impl GMonthDay { #[inline] - pub(super) fn new( + fn new( month: u8, day: u8, timezone_offset: Option, - ) -> Result { + ) -> Result { Ok(Self { timestamp: Timestamp::new(&DateTimeSevenPropertyModel { year: None, @@ -1057,34 +1159,32 @@ impl GMonthDay { } /// Conversion according to [XPath cast rules](https://www.w3.org/TR/xpath-functions-31/#casting-to-datetimes). -impl TryFrom for GMonthDay { - type Error = DateTimeError; - +impl From for GMonthDay { #[inline] - fn try_from(date_time: DateTime) -> Result { + fn from(date_time: DateTime) -> Self { Self::new( date_time.month(), date_time.day(), date_time.timezone_offset(), ) + .expect("Casting from xsd:dateTime to xsd:gMonthDay can't fail") } } /// Conversion according to [XPath cast rules](https://www.w3.org/TR/xpath-functions-31/#casting-to-datetimes). -impl TryFrom for GMonthDay { - type Error = DateTimeError; - +impl From for GMonthDay { #[inline] - fn try_from(date: Date) -> Result { + fn from(date: Date) -> Self { Self::new(date.month(), date.day(), date.timezone_offset()) + .expect("Casting from xsd:date to xsd:gMonthDay can't fail") } } impl FromStr for GMonthDay { - type Err = XsdParseError; + type Err = ParseDateTimeError; - fn from_str(input: &str) -> Result { - parse_g_month_day(input) + fn from_str(input: &str) -> Result { + ensure_complete(input, g_month_day_lexical_rep) } } @@ -1110,10 +1210,10 @@ pub struct GMonth { impl GMonth { #[inline] - pub(super) fn new( + fn new( month: u8, timezone_offset: Option, - ) -> Result { + ) -> Result { Ok(Self { timestamp: Timestamp::new(&DateTimeSevenPropertyModel { year: None, @@ -1176,48 +1276,44 @@ impl GMonth { } /// Conversion according to [XPath cast rules](https://www.w3.org/TR/xpath-functions-31/#casting-to-datetimes). -impl TryFrom for GMonth { - type Error = DateTimeError; - +impl From for GMonth { #[inline] - fn try_from(date_time: DateTime) -> Result { + fn from(date_time: DateTime) -> Self { Self::new(date_time.month(), date_time.timezone_offset()) + .expect("Casting from xsd:dateTime to xsd:gMonth can't fail") } } /// Conversion according to [XPath cast rules](https://www.w3.org/TR/xpath-functions-31/#casting-to-datetimes). -impl TryFrom for GMonth { - type Error = DateTimeError; - +impl From for GMonth { #[inline] - fn try_from(date: Date) -> Result { + fn from(date: Date) -> Self { Self::new(date.month(), date.timezone_offset()) + .expect("Casting from xsd:date to xsd:gMonth can't fail") } } -impl TryFrom for GMonth { - type Error = DateTimeError; - +impl From for GMonth { #[inline] - fn try_from(year_month: GYearMonth) -> Result { + fn from(year_month: GYearMonth) -> Self { Self::new(year_month.month(), year_month.timezone_offset()) + .expect("Casting from xsd:gYearMonth to xsd:gMonth can't fail") } } -impl TryFrom for GMonth { - type Error = DateTimeError; - +impl From for GMonth { #[inline] - fn try_from(month_day: GMonthDay) -> Result { + fn from(month_day: GMonthDay) -> Self { Self::new(month_day.month(), month_day.timezone_offset()) + .expect("Casting from xsd:gMonthDay to xsd:gMonth can't fail") } } impl FromStr for GMonth { - type Err = XsdParseError; + type Err = ParseDateTimeError; - fn from_str(input: &str) -> Result { - parse_g_month(input) + fn from_str(input: &str) -> Result { + ensure_complete(input, g_month_lexical_rep) } } @@ -1243,10 +1339,10 @@ pub struct GDay { impl GDay { #[inline] - pub(super) fn new( + fn new( day: u8, timezone_offset: Option, - ) -> Result { + ) -> Result { Ok(Self { timestamp: Timestamp::new(&DateTimeSevenPropertyModel { year: None, @@ -1309,39 +1405,36 @@ impl GDay { } /// Conversion according to [XPath cast rules](https://www.w3.org/TR/xpath-functions-31/#casting-to-datetimes). -impl TryFrom for GDay { - type Error = DateTimeError; - +impl From for GDay { #[inline] - fn try_from(date_time: DateTime) -> Result { + fn from(date_time: DateTime) -> Self { Self::new(date_time.day(), date_time.timezone_offset()) + .expect("Casting from xsd:dateTime to xsd:gDay can't fail") } } /// Conversion according to [XPath cast rules](https://www.w3.org/TR/xpath-functions-31/#casting-to-datetimes). -impl TryFrom for GDay { - type Error = DateTimeError; - +impl From for GDay { #[inline] - fn try_from(date: Date) -> Result { + fn from(date: Date) -> Self { Self::new(date.day(), date.timezone_offset()) + .expect("Casting from xsd:date to xsd:gDay can't fail") } } -impl TryFrom for GDay { - type Error = DateTimeError; - +impl From for GDay { #[inline] - fn try_from(month_day: GMonthDay) -> Result { + fn from(month_day: GMonthDay) -> Self { Self::new(month_day.day(), month_day.timezone_offset()) + .expect("Casting from xsd:gMonthDay to xsd:gDay can't fail") } } impl FromStr for GDay { - type Err = XsdParseError; + type Err = ParseDateTimeError; - fn from_str(input: &str) -> Result { - parse_g_day(input) + fn from_str(input: &str) -> Result { + ensure_complete(input, g_day_lexical_rep) } } @@ -1367,14 +1460,16 @@ pub struct TimezoneOffset { impl TimezoneOffset { /// From offset in minute with respect to UTC #[inline] - pub fn new(offset_in_minutes: i16) -> Result { + pub fn new(offset_in_minutes: i16) -> Result { let value = Self { offset: offset_in_minutes, }; if Self::MIN <= value && value <= Self::MAX { Ok(value) } else { - Err(DATE_TIME_OVERFLOW) + Err(InvalidTimezoneError { + offset_in_minutes: offset_in_minutes.into(), + }) } } @@ -1398,31 +1493,34 @@ impl TimezoneOffset { } impl TryFrom for TimezoneOffset { - type Error = DateTimeError; + type Error = InvalidTimezoneError; #[inline] - fn try_from(value: DayTimeDuration) -> Result { + fn try_from(value: DayTimeDuration) -> Result { + let offset_in_minutes = value.minutes() + value.hours() * 60; let result = Self::new( - (value.minutes() + value.hours() * 60) + offset_in_minutes .try_into() - .map_err(|_| DATE_TIME_OVERFLOW)?, + .map_err(|_| InvalidTimezoneError { offset_in_minutes })?, )?; if DayTimeDuration::from(result) == value { Ok(result) } else { // The value is not an integral number of minutes or overflow problems - Err(DATE_TIME_OVERFLOW) + Err(InvalidTimezoneError { offset_in_minutes }) } } } impl TryFrom for TimezoneOffset { - type Error = DateTimeError; + type Error = InvalidTimezoneError; #[inline] - fn try_from(value: Duration) -> Result { + fn try_from(value: Duration) -> Result { DayTimeDuration::try_from(value) - .map_err(|_| DATE_TIME_OVERFLOW)? + .map_err(|_| InvalidTimezoneError { + offset_in_minutes: 0, + })? .try_into() } } @@ -1522,28 +1620,18 @@ impl Hash for Timestamp { impl Timestamp { #[inline] - fn new(props: &DateTimeSevenPropertyModel) -> Result { - // Validation - if let (Some(day), Some(month)) = (props.day, props.month) { - // Constraint: Day-of-month Values - if day > days_in_month(props.year, month) { - return Err(DateTimeError { - kind: DateTimeErrorKind::InvalidDayOfMonth { day, month }, - }); - } - } - + fn new(props: &DateTimeSevenPropertyModel) -> Result { Ok(Self { timezone_offset: props.timezone_offset, - value: time_on_timeline(props).ok_or(DATE_TIME_OVERFLOW)?, + value: time_on_timeline(props).ok_or(DateTimeOverflowError)?, }) } #[inline] - fn now() -> Result { + fn now() -> Self { Self::new( &date_time_plus_duration( - since_unix_epoch()?, + since_unix_epoch(), &DateTimeSevenPropertyModel { year: Some(1970), month: Some(1), @@ -1554,8 +1642,9 @@ impl Timestamp { timezone_offset: Some(TimezoneOffset::UTC), }, ) - .ok_or(DATE_TIME_OVERFLOW)?, + .expect("The current time seems way in the future, it's strange"), ) + .expect("The current time seems way in the future, it's strange") } #[inline] @@ -1669,7 +1758,11 @@ impl Timestamp { #[inline] #[must_use] fn second(&self) -> Decimal { - self.value.checked_rem_euclid(60).unwrap().abs() + self.value + .checked_rem_euclid(60) + .unwrap() + .checked_abs() + .unwrap() } #[inline] @@ -1755,13 +1848,23 @@ impl Timestamp { pub fn is_identical_with(self, other: Self) -> bool { self.value == other.value && self.timezone_offset == other.timezone_offset } + + pub const MIN: Self = Self { + value: Decimal::MIN, + timezone_offset: Some(TimezoneOffset::MIN), + }; + + pub const MAX: Self = Self { + value: Decimal::MAX, + timezone_offset: Some(TimezoneOffset::MAX), + }; } #[cfg(feature = "custom-now")] #[allow(unsafe_code)] -pub fn since_unix_epoch() -> Result { +pub fn since_unix_epoch() -> Duration { extern "Rust" { - fn custom_ox_now() -> Result; + fn custom_ox_now() -> Duration; } unsafe { custom_ox_now() } @@ -1773,25 +1876,26 @@ pub fn since_unix_epoch() -> Result { target_family = "wasm", target_os = "unknown" ))] -fn since_unix_epoch() -> Result { - Ok(Duration::new( +fn since_unix_epoch() -> Duration { + Duration::new( 0, Decimal::try_from(crate::Double::from(js_sys::Date::now() / 1000.)) - .map_err(|_| DATE_TIME_OVERFLOW)?, - )) + .expect("The current time seems way in the future, it's strange"), + ) } #[cfg(not(any( feature = "custom-now", all(feature = "js", target_family = "wasm", target_os = "unknown") )))] -fn since_unix_epoch() -> Result { +fn since_unix_epoch() -> Duration { use std::time::SystemTime; SystemTime::now() - .duration_since(SystemTime::UNIX_EPOCH)? + .duration_since(SystemTime::UNIX_EPOCH) + .expect("System time before UNIX epoch") .try_into() - .map_err(|_| DATE_TIME_OVERFLOW) + .expect("The current time seems way in the future, it's strange") } /// The [normalizeMonth](https://www.w3.org/TR/xmlschema11-2/#f-dt-normMo) function @@ -1933,61 +2037,424 @@ fn time_on_timeline(props: &DateTimeSevenPropertyModel) -> Option { .checked_add(se) } -/// An error when doing [`DateTime`] operations. +/// A parsing error #[derive(Debug, Clone)] -pub struct DateTimeError { - kind: DateTimeErrorKind, +pub struct ParseDateTimeError { + kind: ParseDateTimeErrorKind, } #[derive(Debug, Clone)] -enum DateTimeErrorKind { +enum ParseDateTimeErrorKind { InvalidDayOfMonth { day: u8, month: u8 }, - Overflow, - SystemTime(SystemTimeError), + Overflow(DateTimeOverflowError), + InvalidTimezone(InvalidTimezoneError), + Message(&'static str), } -impl fmt::Display for DateTimeError { - #[inline] +impl fmt::Display for ParseDateTimeError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match &self.kind { - DateTimeErrorKind::InvalidDayOfMonth { day, month } => { + ParseDateTimeErrorKind::InvalidDayOfMonth { day, month } => { write!(f, "{day} is not a valid day of {month}") } - DateTimeErrorKind::Overflow => write!(f, "Overflow during date time normalization"), - DateTimeErrorKind::SystemTime(error) => error.fmt(f), + ParseDateTimeErrorKind::Overflow(error) => error.fmt(f), + ParseDateTimeErrorKind::InvalidTimezone(error) => error.fmt(f), + ParseDateTimeErrorKind::Message(msg) => write!(f, "{msg}"), } } } -impl Error for DateTimeError { - #[inline] - fn source(&self) -> Option<&(dyn Error + 'static)> { - match &self.kind { - DateTimeErrorKind::SystemTime(error) => Some(error), - _ => None, +impl ParseDateTimeError { + const fn msg(message: &'static str) -> Self { + Self { + kind: ParseDateTimeErrorKind::Message(message), } } } -impl From for DateTimeError { - #[inline] - fn from(error: SystemTimeError) -> Self { - Self { - kind: DateTimeErrorKind::SystemTime(error), +impl Error for ParseDateTimeError {} + +// [16] dateTimeLexicalRep ::= yearFrag '-' monthFrag '-' dayFrag 'T' ((hourFrag ':' minuteFrag ':' secondFrag) | endOfDayFrag) timezoneFrag? +fn date_time_lexical_rep(input: &str) -> Result<(DateTime, &str), ParseDateTimeError> { + let (year, input) = year_frag(input)?; + let input = expect_char(input, '-', "The year and month must be separated by '-'")?; + let (month, input) = month_frag(input)?; + let input = expect_char(input, '-', "The month and day must be separated by '-'")?; + let (day, input) = day_frag(input)?; + let input = expect_char(input, 'T', "The date and time must be separated by 'T'")?; + let (hour, input) = hour_frag(input)?; + let input = expect_char(input, ':', "The hours and minutes must be separated by ':'")?; + let (minute, input) = minute_frag(input)?; + let input = expect_char( + input, + ':', + "The minutes and seconds must be separated by ':'", + )?; + let (second, input) = second_frag(input)?; + // We validate 24:00:00 + if hour == 24 && minute != 0 && second != Decimal::from(0) { + return Err(ParseDateTimeError::msg( + "Times are not allowed to be after 24:00:00", + )); + } + let (timezone_offset, input) = optional_end(input, timezone_frag)?; + validate_day_of_month(Some(year), month, day)?; + Ok(( + DateTime::new(year, month, day, hour, minute, second, timezone_offset)?, + input, + )) +} + +// [17] timeLexicalRep ::= ((hourFrag ':' minuteFrag ':' secondFrag) | endOfDayFrag) timezoneFrag? +fn time_lexical_rep(input: &str) -> Result<(Time, &str), ParseDateTimeError> { + let (hour, input) = hour_frag(input)?; + let input = expect_char(input, ':', "The hours and minutes must be separated by ':'")?; + let (minute, input) = minute_frag(input)?; + let input = expect_char( + input, + ':', + "The minutes and seconds must be separated by ':'", + )?; + let (second, input) = second_frag(input)?; + // We validate 24:00:00 + if hour == 24 && minute != 0 && second != Decimal::from(0) { + return Err(ParseDateTimeError::msg( + "Times are not allowed to be after 24:00:00", + )); + } + let (timezone_offset, input) = optional_end(input, timezone_frag)?; + Ok((Time::new(hour, minute, second, timezone_offset)?, input)) +} + +// [18] dateLexicalRep ::= yearFrag '-' monthFrag '-' dayFrag timezoneFrag? Constraint: Day-of-month Representations +fn date_lexical_rep(input: &str) -> Result<(Date, &str), ParseDateTimeError> { + let (year, input) = year_frag(input)?; + let input = expect_char(input, '-', "The year and month must be separated by '-'")?; + let (month, input) = month_frag(input)?; + let input = expect_char(input, '-', "The month and day must be separated by '-'")?; + let (day, input) = day_frag(input)?; + let (timezone_offset, input) = optional_end(input, timezone_frag)?; + validate_day_of_month(Some(year), month, day)?; + Ok((Date::new(year, month, day, timezone_offset)?, input)) +} + +// [19] gYearMonthLexicalRep ::= yearFrag '-' monthFrag timezoneFrag? +fn g_year_month_lexical_rep(input: &str) -> Result<(GYearMonth, &str), ParseDateTimeError> { + let (year, input) = year_frag(input)?; + let input = expect_char(input, '-', "The year and month must be separated by '-'")?; + let (month, input) = month_frag(input)?; + let (timezone_offset, input) = optional_end(input, timezone_frag)?; + Ok((GYearMonth::new(year, month, timezone_offset)?, input)) +} + +// [20] gYearLexicalRep ::= yearFrag timezoneFrag? +fn g_year_lexical_rep(input: &str) -> Result<(GYear, &str), ParseDateTimeError> { + let (year, input) = year_frag(input)?; + let (timezone_offset, input) = optional_end(input, timezone_frag)?; + Ok((GYear::new(year, timezone_offset)?, input)) +} + +// [21] gMonthDayLexicalRep ::= '--' monthFrag '-' dayFrag timezoneFrag? Constraint: Day-of-month Representations +fn g_month_day_lexical_rep(input: &str) -> Result<(GMonthDay, &str), ParseDateTimeError> { + let input = expect_char(input, '-', "gMonthDay values must start with '--'")?; + let input = expect_char(input, '-', "gMonthDay values must start with '--'")?; + let (month, input) = month_frag(input)?; + let input = expect_char(input, '-', "The month and day must be separated by '-'")?; + let (day, input) = day_frag(input)?; + let (timezone_offset, input) = optional_end(input, timezone_frag)?; + validate_day_of_month(None, month, day)?; + Ok((GMonthDay::new(month, day, timezone_offset)?, input)) +} + +// [22] gDayLexicalRep ::= '---' dayFrag timezoneFrag? +fn g_day_lexical_rep(input: &str) -> Result<(GDay, &str), ParseDateTimeError> { + let input = expect_char(input, '-', "gDay values must start with '---'")?; + let input = expect_char(input, '-', "gDay values must start with '---'")?; + let input = expect_char(input, '-', "gDay values must start with '---'")?; + let (day, input) = day_frag(input)?; + let (timezone_offset, input) = optional_end(input, timezone_frag)?; + Ok((GDay::new(day, timezone_offset)?, input)) +} + +// [23] gMonthLexicalRep ::= '--' monthFrag timezoneFrag? +fn g_month_lexical_rep(input: &str) -> Result<(GMonth, &str), ParseDateTimeError> { + let input = expect_char(input, '-', "gMonth values must start with '--'")?; + let input = expect_char(input, '-', "gMonth values must start with '--'")?; + let (month, input) = month_frag(input)?; + let (timezone_offset, input) = optional_end(input, timezone_frag)?; + Ok((GMonth::new(month, timezone_offset)?, input)) +} + +// [56] yearFrag ::= '-'? (([1-9] digit digit digit+)) | ('0' digit digit digit)) +fn year_frag(input: &str) -> Result<(i64, &str), ParseDateTimeError> { + let (sign, input) = if let Some(left) = input.strip_prefix('-') { + (-1, left) + } else { + (1, input) + }; + let (number_str, input) = integer_prefix(input); + if number_str.len() < 4 { + return Err(ParseDateTimeError::msg( + "The year should be encoded on 4 digits", + )); + } + if number_str.len() > 4 && number_str.starts_with('0') { + return Err(ParseDateTimeError::msg( + "The years value must not start with 0 if it can be encoded in at least 4 digits", + )); + } + let number = i64::from_str(number_str).expect("valid integer"); + Ok((sign * number, input)) +} + +// [57] monthFrag ::= ('0' [1-9]) | ('1' [0-2]) +fn month_frag(input: &str) -> Result<(u8, &str), ParseDateTimeError> { + let (number_str, input) = integer_prefix(input); + if number_str.len() != 2 { + return Err(ParseDateTimeError::msg( + "Month must be encoded with two digits", + )); + } + let number = u8::from_str(number_str).expect("valid integer"); + if !(1..=12).contains(&number) { + return Err(ParseDateTimeError::msg("Month must be between 01 and 12")); + } + Ok((number, input)) +} + +// [58] dayFrag ::= ('0' [1-9]) | ([12] digit) | ('3' [01]) +fn day_frag(input: &str) -> Result<(u8, &str), ParseDateTimeError> { + let (number_str, input) = integer_prefix(input); + if number_str.len() != 2 { + return Err(ParseDateTimeError::msg( + "Day must be encoded with two digits", + )); + } + let number = u8::from_str(number_str).expect("valid integer"); + if !(1..=31).contains(&number) { + return Err(ParseDateTimeError::msg("Day must be between 01 and 31")); + } + Ok((number, input)) +} + +// [59] hourFrag ::= ([01] digit) | ('2' [0-3]) +// We also allow 24 for ease of parsing +fn hour_frag(input: &str) -> Result<(u8, &str), ParseDateTimeError> { + let (number_str, input) = integer_prefix(input); + if number_str.len() != 2 { + return Err(ParseDateTimeError::msg( + "Hours must be encoded with two digits", + )); + } + let number = u8::from_str(number_str).expect("valid integer"); + if !(0..=24).contains(&number) { + return Err(ParseDateTimeError::msg("Hours must be between 00 and 24")); + } + Ok((number, input)) +} + +// [60] minuteFrag ::= [0-5] digit +fn minute_frag(input: &str) -> Result<(u8, &str), ParseDateTimeError> { + let (number_str, input) = integer_prefix(input); + if number_str.len() != 2 { + return Err(ParseDateTimeError::msg( + "Minutes must be encoded with two digits", + )); + } + let number = u8::from_str(number_str).expect("valid integer"); + if !(0..=59).contains(&number) { + return Err(ParseDateTimeError::msg("Minutes must be between 00 and 59")); + } + Ok((number, input)) +} + +// [61] secondFrag ::= ([0-5] digit) ('.' digit+)? +fn second_frag(input: &str) -> Result<(Decimal, &str), ParseDateTimeError> { + let (number_str, input) = decimal_prefix(input); + let (before_dot_str, _) = number_str.split_once('.').unwrap_or((number_str, "")); + if before_dot_str.len() != 2 { + return Err(ParseDateTimeError::msg( + "Seconds must be encoded with two digits", + )); + } + let number = Decimal::from_str(number_str) + .map_err(|_| ParseDateTimeError::msg("The second precision is too large"))?; + if number < Decimal::from(0) || number >= Decimal::from(60) { + return Err(ParseDateTimeError::msg("Seconds must be between 00 and 60")); + } + if number_str.ends_with('.') { + return Err(ParseDateTimeError::msg( + "Seconds are not allowed to end with a dot", + )); + } + Ok((number, input)) +} + +// [63] timezoneFrag ::= 'Z' | ('+' | '-') (('0' digit | '1' [0-3]) ':' minuteFrag | '14:00') +fn timezone_frag(input: &str) -> Result<(TimezoneOffset, &str), ParseDateTimeError> { + if let Some(left) = input.strip_prefix('Z') { + return Ok((TimezoneOffset::UTC, left)); + } + let (sign, input) = if let Some(left) = input.strip_prefix('-') { + (-1, left) + } else if let Some(left) = input.strip_prefix('+') { + (1, left) + } else { + (1, input) + }; + + let (hour_str, input) = integer_prefix(input); + if hour_str.len() != 2 { + return Err(ParseDateTimeError::msg( + "The timezone hours must be encoded with two digits", + )); + } + let hours = i16::from_str(hour_str).expect("valid integer"); + + let input = expect_char( + input, + ':', + "The timezone hours and minutes must be separated by ':'", + )?; + let (minutes, input) = minute_frag(input)?; + + if hours > 13 && !(hours == 14 && minutes == 0) { + return Err(ParseDateTimeError::msg( + "The timezone hours must be between 00 and 13", + )); + } + + Ok(( + TimezoneOffset::new(sign * (hours * 60 + i16::from(minutes))).map_err(|e| { + ParseDateTimeError { + kind: ParseDateTimeErrorKind::InvalidTimezone(e), + } + })?, + input, + )) +} + +fn ensure_complete( + input: &str, + parse: impl FnOnce(&str) -> Result<(T, &str), ParseDateTimeError>, +) -> Result { + let (result, left) = parse(input)?; + if !left.is_empty() { + return Err(ParseDateTimeError::msg("Unrecognized value suffix")); + } + Ok(result) +} + +fn expect_char<'a>( + input: &'a str, + constant: char, + error_message: &'static str, +) -> Result<&'a str, ParseDateTimeError> { + if let Some(left) = input.strip_prefix(constant) { + Ok(left) + } else { + Err(ParseDateTimeError::msg(error_message)) + } +} + +fn integer_prefix(input: &str) -> (&str, &str) { + let mut end = input.len(); + for (i, c) in input.char_indices() { + if !c.is_ascii_digit() { + end = i; + break; + } + } + input.split_at(end) +} + +fn decimal_prefix(input: &str) -> (&str, &str) { + let mut end = input.len(); + let mut dot_seen = false; + for (i, c) in input.char_indices() { + if c.is_ascii_digit() { + // Ok + } else if c == '.' && !dot_seen { + dot_seen = true; + } else { + end = i; + break; + } + } + input.split_at(end) +} + +fn optional_end( + input: &str, + parse: impl FnOnce(&str) -> Result<(T, &str), ParseDateTimeError>, +) -> Result<(Option, &str), ParseDateTimeError> { + Ok(if input.is_empty() { + (None, input) + } else { + let (result, input) = parse(input)?; + (Some(result), input) + }) +} + +fn validate_day_of_month(year: Option, month: u8, day: u8) -> Result<(), ParseDateTimeError> { + // Constraint: Day-of-month Values + if day > days_in_month(year, month) { + return Err(ParseDateTimeError { + kind: ParseDateTimeErrorKind::InvalidDayOfMonth { day, month }, + }); + } + Ok(()) +} + +/// An overflow during [`DateTime`]-related operations. +/// +/// Matches XPath [`FODT0001` error](https://www.w3.org/TR/xpath-functions-31/#ERRFODT0001). +#[derive(Debug, Clone, Copy)] +pub struct DateTimeOverflowError; + +impl fmt::Display for DateTimeOverflowError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "overflow during xsd:dateTime computation") + } +} + +impl Error for DateTimeOverflowError {} + +impl From for ParseDateTimeError { + fn from(error: DateTimeOverflowError) -> Self { + ParseDateTimeError { + kind: ParseDateTimeErrorKind::Overflow(error), } } } -const DATE_TIME_OVERFLOW: DateTimeError = DateTimeError { - kind: DateTimeErrorKind::Overflow, -}; +/// The value provided as timezone is not valid. +/// +/// Matches XPath [`FODT0003` error](https://www.w3.org/TR/xpath-functions-31/#ERRFODT0003). +#[derive(Debug, Clone, Copy)] +pub struct InvalidTimezoneError { + offset_in_minutes: i64, +} + +impl fmt::Display for InvalidTimezoneError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "invalid timezone offset {}:{}", + self.offset_in_minutes / 60, + self.offset_in_minutes.abs() % 60 + ) + } +} + +impl Error for InvalidTimezoneError {} #[cfg(test)] mod tests { use super::*; #[test] - fn from_str() -> Result<(), XsdParseError> { + fn from_str() -> Result<(), ParseDateTimeError> { assert_eq!(Time::from_str("00:00:00Z")?.to_string(), "00:00:00Z"); assert_eq!(Time::from_str("00:00:00+00:00")?.to_string(), "00:00:00Z"); assert_eq!(Time::from_str("00:00:00-00:00")?.to_string(), "00:00:00Z"); @@ -2148,11 +2615,57 @@ mod tests { assert!(GYear::from_str("02020").is_err()); assert!(GYear::from_str("+2020").is_err()); assert!(GYear::from_str("33").is_err()); + + assert_eq!(Time::from_str("00:00:00+14:00")?, Time::MIN); + assert_eq!(Time::from_str("24:00:00-14:00")?, Time::MAX); + Ok(()) + } + + #[test] + fn to_be_bytes() -> Result<(), ParseDateTimeError> { + assert_eq!( + DateTime::from_be_bytes(DateTime::MIN.to_be_bytes()), + DateTime::MIN + ); + assert_eq!( + DateTime::from_be_bytes(DateTime::MAX.to_be_bytes()), + DateTime::MAX + ); + assert_eq!( + DateTime::from_be_bytes(DateTime::from_str("2022-01-03T01:02:03")?.to_be_bytes()), + DateTime::from_str("2022-01-03T01:02:03")? + ); + assert_eq!(Date::from_be_bytes(Date::MIN.to_be_bytes()), Date::MIN); + assert_eq!(Date::from_be_bytes(Date::MAX.to_be_bytes()), Date::MAX); + assert_eq!( + Date::from_be_bytes(Date::from_str("2022-01-03")?.to_be_bytes()), + Date::from_str("2022-01-03")? + ); + assert_eq!(Time::from_be_bytes(Time::MIN.to_be_bytes()), Time::MIN); + assert_eq!(Time::from_be_bytes(Time::MAX.to_be_bytes()), Time::MAX); + assert_eq!( + Time::from_be_bytes(Time::from_str("01:02:03")?.to_be_bytes()), + Time::from_str("01:02:03")? + ); + assert_eq!( + Time::from_be_bytes(Time::from_str("01:02:03")?.to_be_bytes()), + Time::from_str("01:02:03")? + ); + assert_eq!( + GYearMonth::from_be_bytes(GYearMonth::MIN.to_be_bytes()), + GYearMonth::MIN + ); + assert_eq!( + GYearMonth::from_be_bytes(GYearMonth::MAX.to_be_bytes()), + GYearMonth::MAX + ); + assert_eq!(GYear::from_be_bytes(GYear::MIN.to_be_bytes()), GYear::MIN); + assert_eq!(GYear::from_be_bytes(GYear::MAX.to_be_bytes()), GYear::MAX); Ok(()) } #[test] - fn equals() -> Result<(), XsdParseError> { + fn equals() -> Result<(), ParseDateTimeError> { assert_eq!( DateTime::from_str("2002-04-02T12:00:00-01:00")?, DateTime::from_str("2002-04-02T17:00:00+04:00")? @@ -2253,7 +2766,7 @@ mod tests { #[test] #[allow(clippy::neg_cmp_op_on_partial_ord)] - fn cmp() -> Result<(), XsdParseError> { + fn cmp() -> Result<(), ParseDateTimeError> { assert!(Date::from_str("2004-12-25Z")? < Date::from_str("2004-12-25-05:00")?); assert!(!(Date::from_str("2004-12-25-12:00")? < Date::from_str("2004-12-26+12:00")?)); @@ -2280,7 +2793,7 @@ mod tests { } #[test] - fn year() -> Result<(), XsdParseError> { + fn year() -> Result<(), ParseDateTimeError> { assert_eq!( DateTime::from_str("1999-05-31T13:20:00-05:00")?.year(), 1999 @@ -2303,7 +2816,7 @@ mod tests { } #[test] - fn month() -> Result<(), XsdParseError> { + fn month() -> Result<(), ParseDateTimeError> { assert_eq!(DateTime::from_str("1999-05-31T13:20:00-05:00")?.month(), 5); assert_eq!(DateTime::from_str("1999-12-31T19:20:00-05:00")?.month(), 12); @@ -2317,7 +2830,7 @@ mod tests { } #[test] - fn day() -> Result<(), XsdParseError> { + fn day() -> Result<(), ParseDateTimeError> { assert_eq!(DateTime::from_str("1999-05-31T13:20:00-05:00")?.day(), 31); assert_eq!(DateTime::from_str("1999-12-31T20:00:00-05:00")?.day(), 31); @@ -2330,7 +2843,7 @@ mod tests { } #[test] - fn hour() -> Result<(), XsdParseError> { + fn hour() -> Result<(), ParseDateTimeError> { assert_eq!(DateTime::from_str("1999-05-31T08:20:00-05:00")?.hour(), 8); assert_eq!(DateTime::from_str("1999-12-31T21:20:00-05:00")?.hour(), 21); assert_eq!(DateTime::from_str("1999-12-31T12:00:00")?.hour(), 12); @@ -2344,7 +2857,7 @@ mod tests { } #[test] - fn minute() -> Result<(), XsdParseError> { + fn minute() -> Result<(), ParseDateTimeError> { assert_eq!( DateTime::from_str("1999-05-31T13:20:00-05:00")?.minute(), 20 @@ -2359,7 +2872,7 @@ mod tests { } #[test] - fn second() -> Result<(), XsdParseError> { + fn second() -> Result<(), Box> { assert_eq!( DateTime::from_str("1999-05-31T13:20:00-05:00")?.second(), Decimal::from(0) @@ -2373,7 +2886,7 @@ mod tests { } #[test] - fn timezone() -> Result<(), XsdParseError> { + fn timezone() -> Result<(), Box> { assert_eq!( DateTime::from_str("1999-05-31T13:20:00-05:00")?.timezone(), Some(DayTimeDuration::from_str("-PT5H")?) @@ -2402,7 +2915,7 @@ mod tests { } #[test] - fn sub() -> Result<(), XsdParseError> { + fn sub() -> Result<(), Box> { assert_eq!( DateTime::from_str("2000-10-30T06:12:00-05:00")? .checked_sub(DateTime::from_str("1999-11-28T09:00:00Z")?), @@ -2442,7 +2955,7 @@ mod tests { } #[test] - fn add_duration() -> Result<(), XsdParseError> { + fn add_duration() -> Result<(), Box> { assert_eq!( DateTime::from_str("2000-01-12T12:13:14Z")? .checked_add_duration(Duration::from_str("P1Y3M5DT7H10M3.3S")?), @@ -2506,7 +3019,7 @@ mod tests { } #[test] - fn sub_duration() -> Result<(), XsdParseError> { + fn sub_duration() -> Result<(), Box> { assert_eq!( DateTime::from_str("2000-10-30T11:12:00")? .checked_sub_duration(Duration::from_str("P1Y2M")?), @@ -2548,17 +3061,15 @@ mod tests { } #[test] - fn adjust() -> Result<(), XsdParseError> { + fn adjust() -> Result<(), Box> { assert_eq!( - DateTime::from_str("2002-03-07T10:00:00-07:00")?.adjust(Some( - DayTimeDuration::from_str("PT10H")?.try_into().unwrap() - )), + DateTime::from_str("2002-03-07T10:00:00-07:00")? + .adjust(Some(DayTimeDuration::from_str("PT10H")?.try_into()?)), Some(DateTime::from_str("2002-03-08T03:00:00+10:00")?) ); assert_eq!( - DateTime::from_str("2002-03-07T00:00:00+01:00")?.adjust(Some( - DayTimeDuration::from_str("-PT8H")?.try_into().unwrap() - )), + DateTime::from_str("2002-03-07T00:00:00+01:00")? + .adjust(Some(DayTimeDuration::from_str("-PT8H")?.try_into()?)), Some(DateTime::from_str("2002-03-06T15:00:00-08:00")?) ); assert_eq!( @@ -2571,18 +3082,13 @@ mod tests { ); assert_eq!( - Date::from_str("2002-03-07")?.adjust(Some( - DayTimeDuration::from_str("-PT10H")?.try_into().unwrap() - )), + Date::from_str("2002-03-07")? + .adjust(Some(DayTimeDuration::from_str("-PT10H")?.try_into()?)), Some(Date::from_str("2002-03-07-10:00")?) ); assert_eq!( - Date::from_str("2002-03-07-07:00")?.adjust(Some( - DayTimeDuration::from_str("-PT10H") - .unwrap() - .try_into() - .unwrap() - )), + Date::from_str("2002-03-07-07:00")? + .adjust(Some(DayTimeDuration::from_str("-PT10H")?.try_into()?)), Some(Date::from_str("2002-03-06-10:00")?) ); assert_eq!( @@ -2595,15 +3101,13 @@ mod tests { ); assert_eq!( - Time::from_str("10:00:00")?.adjust(Some( - DayTimeDuration::from_str("-PT10H")?.try_into().unwrap() - )), + Time::from_str("10:00:00")? + .adjust(Some(DayTimeDuration::from_str("-PT10H")?.try_into()?)), Some(Time::from_str("10:00:00-10:00")?) ); assert_eq!( - Time::from_str("10:00:00-07:00")?.adjust(Some( - DayTimeDuration::from_str("-PT10H")?.try_into().unwrap() - )), + Time::from_str("10:00:00-07:00")? + .adjust(Some(DayTimeDuration::from_str("-PT10H")?.try_into()?)), Some(Time::from_str("07:00:00-10:00")?) ); assert_eq!( @@ -2615,35 +3119,78 @@ mod tests { Some(Time::from_str("10:00:00")?) ); assert_eq!( - Time::from_str("10:00:00-07:00")?.adjust(Some( - DayTimeDuration::from_str("PT10H")?.try_into().unwrap() - )), + Time::from_str("10:00:00-07:00")? + .adjust(Some(DayTimeDuration::from_str("PT10H")?.try_into()?)), Some(Time::from_str("03:00:00+10:00")?) ); Ok(()) } + #[test] + fn time_from_datetime() -> Result<(), ParseDateTimeError> { + assert_eq!( + Time::from(DateTime::MIN), + Time::from_str("19:51:08.312696284115894272-14:00")? + ); + assert_eq!( + Time::from(DateTime::MAX), + Time::from_str("04:08:51.687303715884105727+14:00")? + ); + Ok(()) + } + + #[test] + fn date_from_datetime() -> Result<(), Box> { + assert_eq!( + Date::try_from( + DateTime::MIN + .checked_add_day_time_duration(DayTimeDuration::from_str("P1D")?) + .unwrap() + )?, + Date::MIN + ); + assert_eq!(Date::try_from(DateTime::MAX)?, Date::MAX); + Ok(()) + } + + #[test] + fn g_year_month_from_date() -> Result<(), ParseDateTimeError> { + assert_eq!(GYearMonth::from(Date::MIN), GYearMonth::MIN); + assert_eq!(GYearMonth::from(Date::MAX), GYearMonth::MAX); + Ok(()) + } + + #[test] + fn g_year_from_g_year_month() -> Result<(), ParseDateTimeError> { + assert_eq!(GYear::try_from(GYearMonth::MIN)?, GYear::MIN); + assert_eq!( + GYear::try_from(GYearMonth::from_str("5391559471918-12+14:00")?)?, + GYear::MAX + ); + Ok(()) + } + #[cfg(feature = "custom-now")] #[test] fn custom_now() { #[no_mangle] - fn custom_ox_now() -> Result { - Ok(Duration::default()) + fn custom_ox_now() -> Duration { + Duration::default() } - assert!(DateTime::now().is_ok()); + DateTime::now(); } #[cfg(not(feature = "custom-now"))] #[test] - fn now() -> Result<(), XsdParseError> { - let now = DateTime::now().unwrap(); + fn now() -> Result<(), ParseDateTimeError> { + let now = DateTime::now(); assert!(DateTime::from_str("2022-01-01T00:00:00Z")? < now); assert!(now < DateTime::from_str("2100-01-01T00:00:00Z")?); Ok(()) } #[test] - fn minimally_conformant() -> Result<(), XsdParseError> { + fn minimally_conformant() -> Result<(), ParseDateTimeError> { // All minimally conforming processors must support nonnegative year values less than 10000 // (i.e., those expressible with four digits) in all datatypes which // use the seven-property model defined in The Seven-property Model (§D.2.1) diff --git a/lib/oxsdatatypes/src/decimal.rs b/lib/oxsdatatypes/src/decimal.rs index f3880dcb..6a309108 100644 --- a/lib/oxsdatatypes/src/decimal.rs +++ b/lib/oxsdatatypes/src/decimal.rs @@ -1,4 +1,4 @@ -use crate::{Boolean, Double, Float, Integer}; +use crate::{Boolean, Double, Float, Integer, TooLargeForIntegerError}; use std::error::Error; use std::fmt; use std::fmt::Write; @@ -21,15 +21,20 @@ pub struct Decimal { impl Decimal { /// Constructs the decimal i / 10^n #[inline] - pub fn new(i: i128, n: u32) -> Result { - let shift = DECIMAL_PART_DIGITS - .checked_sub(n) - .ok_or(DecimalOverflowError)?; - Ok(Self { - value: i - .checked_mul(10_i128.pow(shift)) - .ok_or(DecimalOverflowError)?, - }) + pub const fn new(i: i128, n: u32) -> Result { + let Some(shift) = DECIMAL_PART_DIGITS.checked_sub(n) else { + return Err(TooLargeForDecimalError); + }; + let Some(value) = i.checked_mul(10_i128.pow(shift)) else { + return Err(TooLargeForDecimalError); + }; + Ok(Self { value }) + } + + pub(crate) const fn new_from_i128_unchecked(value: i128) -> Self { + Self { + value: value * DECIMAL_PART_POW, + } } #[inline] @@ -47,6 +52,8 @@ impl Decimal { } /// [op:numeric-add](https://www.w3.org/TR/xpath-functions-31/#func-numeric-add) + /// + /// Returns `None` in case of overflow ([FOAR0002](https://www.w3.org/TR/xpath-functions-31/#ERRFOAR0002)). #[inline] #[must_use] pub fn checked_add(self, rhs: impl Into) -> Option { @@ -56,6 +63,8 @@ impl Decimal { } /// [op:numeric-subtract](https://www.w3.org/TR/xpath-functions-31/#func-numeric-subtract) + /// + /// Returns `None` in case of overflow ([FOAR0002](https://www.w3.org/TR/xpath-functions-31/#ERRFOAR0002)). #[inline] #[must_use] pub fn checked_sub(self, rhs: impl Into) -> Option { @@ -65,6 +74,8 @@ impl Decimal { } /// [op:numeric-multiply](https://www.w3.org/TR/xpath-functions-31/#func-numeric-multiply) + /// + /// Returns `None` in case of overflow ([FOAR0002](https://www.w3.org/TR/xpath-functions-31/#ERRFOAR0002)). #[inline] #[must_use] pub fn checked_mul(self, rhs: impl Into) -> Option { @@ -98,6 +109,8 @@ impl Decimal { } /// [op:numeric-divide](https://www.w3.org/TR/xpath-functions-31/#func-numeric-divide) + /// + /// Returns `None` in case of division by 0 ([FOAR0001](https://www.w3.org/TR/xpath-functions-31/#ERRFOAR0001)) or overflow ([FOAR0002](https://www.w3.org/TR/xpath-functions-31/#ERRFOAR0002)). #[inline] #[must_use] pub fn checked_div(self, rhs: impl Into) -> Option { @@ -132,6 +145,8 @@ impl Decimal { } /// [op:numeric-mod](https://www.w3.org/TR/xpath-functions-31/#func-numeric-mod) + /// + /// Returns `None` in case of division by 0 ([FOAR0001](https://www.w3.org/TR/xpath-functions-31/#ERRFOAR0001)) or overflow ([FOAR0002](https://www.w3.org/TR/xpath-functions-31/#ERRFOAR0002)). #[inline] #[must_use] pub fn checked_rem(self, rhs: impl Into) -> Option { @@ -140,6 +155,9 @@ impl Decimal { }) } + /// Euclidean remainder + /// + /// Returns `None` in case of division by 0 ([FOAR0001](https://www.w3.org/TR/xpath-functions-31/#ERRFOAR0001)) or overflow ([FOAR0002](https://www.w3.org/TR/xpath-functions-31/#ERRFOAR0002)). #[inline] #[must_use] pub fn checked_rem_euclid(self, rhs: impl Into) -> Option { @@ -149,6 +167,8 @@ impl Decimal { } /// [op:numeric-unary-minus](https://www.w3.org/TR/xpath-functions-31/#func-numeric-unary-minus) + /// + /// Returns `None` in case of overflow ([FOAR0002](https://www.w3.org/TR/xpath-functions-31/#ERRFOAR0002)). #[inline] #[must_use] pub fn checked_neg(self) -> Option { @@ -158,52 +178,63 @@ impl Decimal { } /// [fn:abs](https://www.w3.org/TR/xpath-functions-31/#func-abs) + /// + /// Returns `None` in case of overflow ([FOAR0002](https://www.w3.org/TR/xpath-functions-31/#ERRFOAR0002)). #[inline] #[must_use] - pub const fn abs(self) -> Self { - Self { - value: self.value.abs(), - } + pub fn checked_abs(self) -> Option { + Some(Self { + value: self.value.checked_abs()?, + }) } /// [fn:round](https://www.w3.org/TR/xpath-functions-31/#func-round) + /// + /// Returns `None` in case of overflow ([FOAR0002](https://www.w3.org/TR/xpath-functions-31/#ERRFOAR0002)). #[inline] #[must_use] - pub fn round(self) -> Self { + pub fn checked_round(self) -> Option { let value = self.value / DECIMAL_PART_POW_MINUS_ONE; - Self { + Some(Self { value: if value >= 0 { - (value / 10 + i128::from(value % 10 >= 5)) * DECIMAL_PART_POW + value / 10 + i128::from(value % 10 >= 5) } else { - (value / 10 - i128::from(-value % 10 > 5)) * DECIMAL_PART_POW - }, - } + value / 10 - i128::from(-value % 10 > 5) + } + .checked_mul(DECIMAL_PART_POW)?, + }) } /// [fn:ceiling](https://www.w3.org/TR/xpath-functions-31/#func-ceiling) + /// + /// Returns `None` in case of overflow ([FOAR0002](https://www.w3.org/TR/xpath-functions-31/#ERRFOAR0002)). #[inline] #[must_use] - pub fn ceil(self) -> Self { - Self { - value: if self.value >= 0 && self.value % DECIMAL_PART_POW != 0 { - (self.value / DECIMAL_PART_POW + 1) * DECIMAL_PART_POW + pub fn checked_ceil(self) -> Option { + Some(Self { + value: if self.value > 0 && self.value % DECIMAL_PART_POW != 0 { + self.value / DECIMAL_PART_POW + 1 } else { - (self.value / DECIMAL_PART_POW) * DECIMAL_PART_POW - }, - } + self.value / DECIMAL_PART_POW + } + .checked_mul(DECIMAL_PART_POW)?, + }) } /// [fn:floor](https://www.w3.org/TR/xpath-functions-31/#func-floor) + /// + /// Returns `None` in case of overflow ([FOAR0002](https://www.w3.org/TR/xpath-functions-31/#ERRFOAR0002)). #[inline] #[must_use] - pub fn floor(self) -> Self { - Self { + pub fn checked_floor(self) -> Option { + Some(Self { value: if self.value >= 0 || self.value % DECIMAL_PART_POW == 0 { - (self.value / DECIMAL_PART_POW) * DECIMAL_PART_POW + self.value / DECIMAL_PART_POW } else { - (self.value / DECIMAL_PART_POW - 1) * DECIMAL_PART_POW - }, - } + self.value / DECIMAL_PART_POW - 1 + } + .checked_mul(DECIMAL_PART_POW)?, + }) } #[inline] @@ -328,28 +359,28 @@ impl From for Decimal { } impl TryFrom for Decimal { - type Error = DecimalOverflowError; + type Error = TooLargeForDecimalError; #[inline] - fn try_from(value: i128) -> Result { + fn try_from(value: i128) -> Result { Ok(Self { value: value .checked_mul(DECIMAL_PART_POW) - .ok_or(DecimalOverflowError)?, + .ok_or(TooLargeForDecimalError)?, }) } } impl TryFrom for Decimal { - type Error = DecimalOverflowError; + type Error = TooLargeForDecimalError; #[inline] - fn try_from(value: u128) -> Result { + fn try_from(value: u128) -> Result { Ok(Self { value: i128::try_from(value) - .map_err(|_| DecimalOverflowError)? + .map_err(|_| TooLargeForDecimalError)? .checked_mul(DECIMAL_PART_POW) - .ok_or(DecimalOverflowError)?, + .ok_or(TooLargeForDecimalError)?, }) } } @@ -362,27 +393,27 @@ impl From for Decimal { } impl TryFrom for Decimal { - type Error = DecimalOverflowError; + type Error = TooLargeForDecimalError; #[inline] - fn try_from(value: Float) -> Result { + fn try_from(value: Float) -> Result { Double::from(value).try_into() } } impl TryFrom for Decimal { - type Error = DecimalOverflowError; + type Error = TooLargeForDecimalError; #[inline] #[allow(clippy::cast_precision_loss, clippy::cast_possible_truncation)] - fn try_from(value: Double) -> Result { + fn try_from(value: Double) -> Result { let shifted = f64::from(value) * (DECIMAL_PART_POW as f64); - if shifted.is_finite() && (i128::MIN as f64) <= shifted && shifted <= (i128::MAX as f64) { + if (i128::MIN as f64) <= shifted && shifted <= (i128::MAX as f64) { Ok(Self { value: shifted as i128, }) } else { - Err(DecimalOverflowError) + Err(TooLargeForDecimalError) } } } @@ -415,17 +446,17 @@ impl From for Double { } impl TryFrom for Integer { - type Error = DecimalOverflowError; + type Error = TooLargeForIntegerError; #[inline] - fn try_from(value: Decimal) -> Result { + fn try_from(value: Decimal) -> Result { Ok(i64::try_from( value .value .checked_div(DECIMAL_PART_POW) - .ok_or(DecimalOverflowError)?, + .ok_or(TooLargeForIntegerError)?, ) - .map_err(|_| DecimalOverflowError)? + .map_err(|_| TooLargeForIntegerError)? .into()) } } @@ -620,25 +651,27 @@ impl fmt::Display for ParseDecimalError { impl Error for ParseDecimalError {} -impl From for ParseDecimalError { - fn from(_: DecimalOverflowError) -> Self { +impl From for ParseDecimalError { + fn from(_: TooLargeForDecimalError) -> Self { Self { kind: DecimalParseErrorKind::Overflow, } } } -/// An overflow in [`Decimal`] computations. +/// The input is too large to fit into a [`Decimal`]. +/// +/// Matches XPath [`FOCA0001` error](https://www.w3.org/TR/xpath-functions-31/#ERRFOCA0001). #[derive(Debug, Clone, Copy)] -pub struct DecimalOverflowError; +pub struct TooLargeForDecimalError; -impl fmt::Display for DecimalOverflowError { +impl fmt::Display for TooLargeForDecimalError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "Value overflow") + write!(f, "Value too large for xsd:decimal internal representation") } } -impl Error for DecimalOverflowError {} +impl Error for TooLargeForDecimalError {} #[cfg(test)] mod tests { @@ -797,45 +830,153 @@ mod tests { Some(Decimal::from_str("0.9")?) ); assert_eq!(Decimal::from(1).checked_rem(0), None); + assert_eq!( + Decimal::MAX.checked_rem(1), + Some(Decimal::from_str("0.687303715884105727")?) + ); + assert_eq!( + Decimal::MIN.checked_rem(1), + Some(Decimal::from_str("-0.687303715884105728")?) + ); + assert_eq!( + Decimal::MAX.checked_rem(Decimal::STEP), + Some(Decimal::default()) + ); + assert_eq!( + Decimal::MIN.checked_rem(Decimal::STEP), + Some(Decimal::default()) + ); + assert_eq!( + Decimal::MAX.checked_rem(Decimal::MAX), + Some(Decimal::default()) + ); + assert_eq!( + Decimal::MIN.checked_rem(Decimal::MIN), + Some(Decimal::default()) + ); Ok(()) } #[test] fn round() -> Result<(), ParseDecimalError> { - assert_eq!(Decimal::from(10).round(), Decimal::from(10)); - assert_eq!(Decimal::from(-10).round(), Decimal::from(-10)); - assert_eq!(Decimal::from_str("2.5")?.round(), Decimal::from(3)); - assert_eq!(Decimal::from_str("2.4999")?.round(), Decimal::from(2)); - assert_eq!(Decimal::from_str("-2.5")?.round(), Decimal::from(-2)); - assert_eq!(Decimal::from(i64::MIN).round(), Decimal::from(i64::MIN)); - assert_eq!(Decimal::from(i64::MAX).round(), Decimal::from(i64::MAX)); + assert_eq!(Decimal::from(10).checked_round(), Some(Decimal::from(10))); + assert_eq!(Decimal::from(-10).checked_round(), Some(Decimal::from(-10))); + assert_eq!( + Decimal::from(i64::MIN).checked_round(), + Some(Decimal::from(i64::MIN)) + ); + assert_eq!( + Decimal::from(i64::MAX).checked_round(), + Some(Decimal::from(i64::MAX)) + ); + assert_eq!( + Decimal::from_str("2.5")?.checked_round(), + Some(Decimal::from(3)) + ); + assert_eq!( + Decimal::from_str("2.4999")?.checked_round(), + Some(Decimal::from(2)) + ); + assert_eq!( + Decimal::from_str("-2.5")?.checked_round(), + Some(Decimal::from(-2)) + ); + assert_eq!(Decimal::MAX.checked_round(), None); + assert_eq!( + (Decimal::MAX.checked_sub(Decimal::from_str("0.5")?)) + .unwrap() + .checked_round(), + Some(Decimal::from_str("170141183460469231731")?) + ); + assert_eq!(Decimal::MIN.checked_round(), None); + assert_eq!( + (Decimal::MIN.checked_add(Decimal::from_str("0.5")?)) + .unwrap() + .checked_round(), + Some(Decimal::from_str("-170141183460469231731")?) + ); Ok(()) } #[test] fn ceil() -> Result<(), ParseDecimalError> { - assert_eq!(Decimal::from(10).ceil(), Decimal::from(10)); - assert_eq!(Decimal::from(-10).ceil(), Decimal::from(-10)); - assert_eq!(Decimal::from_str("10.5")?.ceil(), Decimal::from(11)); - assert_eq!(Decimal::from_str("-10.5")?.ceil(), Decimal::from(-10)); - assert_eq!(Decimal::from(i64::MIN).ceil(), Decimal::from(i64::MIN)); - assert_eq!(Decimal::from(i64::MAX).ceil(), Decimal::from(i64::MAX)); + assert_eq!(Decimal::from(10).checked_ceil(), Some(Decimal::from(10))); + assert_eq!(Decimal::from(-10).checked_ceil(), Some(Decimal::from(-10))); + assert_eq!( + Decimal::from_str("10.5")?.checked_ceil(), + Some(Decimal::from(11)) + ); + assert_eq!( + Decimal::from_str("-10.5")?.checked_ceil(), + Some(Decimal::from(-10)) + ); + assert_eq!( + Decimal::from(i64::MIN).checked_ceil(), + Some(Decimal::from(i64::MIN)) + ); + assert_eq!( + Decimal::from(i64::MAX).checked_ceil(), + Some(Decimal::from(i64::MAX)) + ); + assert_eq!(Decimal::MAX.checked_ceil(), None); + assert_eq!( + Decimal::MAX + .checked_sub(Decimal::from(1)) + .unwrap() + .checked_ceil(), + Some(Decimal::from_str("170141183460469231731")?) + ); + assert_eq!( + Decimal::MIN.checked_ceil(), + Some(Decimal::from_str("-170141183460469231731")?) + ); Ok(()) } #[test] fn floor() -> Result<(), ParseDecimalError> { - assert_eq!(Decimal::from(10).ceil(), Decimal::from(10)); - assert_eq!(Decimal::from(-10).ceil(), Decimal::from(-10)); - assert_eq!(Decimal::from_str("10.5")?.floor(), Decimal::from(10)); - assert_eq!(Decimal::from_str("-10.5")?.floor(), Decimal::from(-11)); - assert_eq!(Decimal::from(i64::MIN).floor(), Decimal::from(i64::MIN)); - assert_eq!(Decimal::from(i64::MAX).floor(), Decimal::from(i64::MAX)); + assert_eq!(Decimal::from(10).checked_floor(), Some(Decimal::from(10))); + assert_eq!(Decimal::from(-10).checked_floor(), Some(Decimal::from(-10))); + assert_eq!( + Decimal::from_str("10.5")?.checked_floor(), + Some(Decimal::from(10)) + ); + assert_eq!( + Decimal::from_str("-10.5")?.checked_floor(), + Some(Decimal::from(-11)) + ); + assert_eq!( + Decimal::from(i64::MIN).checked_floor(), + Some(Decimal::from(i64::MIN)) + ); + assert_eq!( + Decimal::from(i64::MAX).checked_floor(), + Some(Decimal::from(i64::MAX)) + ); + assert_eq!( + Decimal::MAX.checked_floor(), + Some(Decimal::from_str("170141183460469231731")?) + ); + assert_eq!(Decimal::MIN.checked_floor(), None); + assert_eq!( + (Decimal::MIN.checked_add(Decimal::from_str("1")?)) + .unwrap() + .checked_floor(), + Some(Decimal::from_str("-170141183460469231731")?) + ); Ok(()) } #[test] fn to_be_bytes() -> Result<(), ParseDecimalError> { + assert_eq!( + Decimal::from_be_bytes(Decimal::MIN.to_be_bytes()), + Decimal::MIN + ); + assert_eq!( + Decimal::from_be_bytes(Decimal::MAX.to_be_bytes()), + Decimal::MAX + ); assert_eq!( Decimal::from_be_bytes(Decimal::from(i64::MIN).to_be_bytes()), Decimal::from(i64::MIN) @@ -889,7 +1030,8 @@ mod tests { .unwrap() .checked_sub(Decimal::from(1_672_507_293_696_i64)) .unwrap() - .abs() + .checked_abs() + .unwrap() < Decimal::from(1) ); Ok(()) @@ -914,7 +1056,8 @@ mod tests { .unwrap() .checked_sub(Decimal::from(1_672_507_302_466_i64)) .unwrap() - .abs() + .checked_abs() + .unwrap() < Decimal::from(1) ); assert!(Decimal::try_from(Double::from(f64::NAN)).is_err()); diff --git a/lib/oxsdatatypes/src/duration.rs b/lib/oxsdatatypes/src/duration.rs index 27ce5d97..87be5a22 100644 --- a/lib/oxsdatatypes/src/duration.rs +++ b/lib/oxsdatatypes/src/duration.rs @@ -1,7 +1,6 @@ -use super::decimal::DecimalOverflowError; -use super::parser::*; -use super::*; +use crate::{DateTime, Decimal}; use std::cmp::Ordering; +use std::error::Error; use std::fmt; use std::str::FromStr; use std::time::Duration as StdDuration; @@ -78,13 +77,13 @@ impl Duration { #[inline] #[must_use] - pub(super) const fn all_months(self) -> i64 { + pub(crate) const fn all_months(self) -> i64 { self.year_month.all_months() } #[inline] #[must_use] - pub(super) const fn all_seconds(self) -> Decimal { + pub(crate) const fn all_seconds(self) -> Decimal { self.day_time.as_seconds() } @@ -98,6 +97,8 @@ impl Duration { } /// [op:add-yearMonthDurations](https://www.w3.org/TR/xpath-functions-31/#func-add-yearMonthDurations) and [op:add-dayTimeDurations](https://www.w3.org/TR/xpath-functions-31/#func-add-dayTimeDurations) + /// + /// Returns `None` in case of overflow ([`FODT0002`](https://www.w3.org/TR/xpath-functions-31/#ERRFODT0002)). #[inline] #[must_use] pub fn checked_add(self, rhs: impl Into) -> Option { @@ -109,6 +110,8 @@ impl Duration { } /// [op:subtract-yearMonthDurations](https://www.w3.org/TR/xpath-functions-31/#func-subtract-yearMonthDurations) and [op:subtract-dayTimeDurations](https://www.w3.org/TR/xpath-functions-31/#func-subtract-dayTimeDurations) + /// + /// Returns `None` in case of overflow ([`FODT0002`](https://www.w3.org/TR/xpath-functions-31/#ERRFODT0002)). #[inline] #[must_use] pub fn checked_sub(self, rhs: impl Into) -> Option { @@ -119,6 +122,9 @@ impl Duration { }) } + /// Unary negation. + /// + /// Returns `None` in case of overflow ([`FODT0002`](https://www.w3.org/TR/xpath-functions-31/#ERRFODT0002)). #[inline] #[must_use] pub fn checked_neg(self) -> Option { @@ -134,22 +140,39 @@ impl Duration { pub fn is_identical_with(self, other: Self) -> bool { self == other } + + pub const MIN: Self = Self { + year_month: YearMonthDuration::MIN, + day_time: DayTimeDuration::MIN, + }; + + pub const MAX: Self = Self { + year_month: YearMonthDuration::MAX, + day_time: DayTimeDuration::MAX, + }; } impl TryFrom for Duration { - type Error = DecimalOverflowError; + type Error = DurationOverflowError; #[inline] - fn try_from(value: StdDuration) -> Result { + fn try_from(value: StdDuration) -> Result { Ok(DayTimeDuration::try_from(value)?.into()) } } impl FromStr for Duration { - type Err = XsdParseError; + type Err = ParseDurationError; - fn from_str(input: &str) -> Result { - parse_duration(input) + fn from_str(input: &str) -> Result { + let parts = ensure_complete(input, duration_parts)?; + if parts.year_month.is_none() && parts.day_time.is_none() { + return Err(ParseDurationError::msg("Empty duration")); + } + Ok(Self::new( + parts.year_month.unwrap_or(0), + parts.day_time.unwrap_or_default(), + )) } } @@ -208,7 +231,7 @@ impl fmt::Display for Duration { write!(f, "{}M", m.abs())?; } if s != 0.into() { - write!(f, "{}S", s.abs())?; + write!(f, "{}S", s.checked_abs().ok_or(fmt::Error)?)?; } } } @@ -282,7 +305,7 @@ impl YearMonthDuration { } #[inline] - pub(super) const fn all_months(self) -> i64 { + pub(crate) const fn all_months(self) -> i64 { self.months } @@ -292,6 +315,8 @@ impl YearMonthDuration { } /// [op:add-yearMonthDurations](https://www.w3.org/TR/xpath-functions-31/#func-add-yearMonthDurations) + /// + /// Returns `None` in case of overflow ([`FODT0002`](https://www.w3.org/TR/xpath-functions-31/#ERRFODT0002)). #[inline] pub fn checked_add(self, rhs: impl Into) -> Option { let rhs = rhs.into(); @@ -301,6 +326,8 @@ impl YearMonthDuration { } /// [op:subtract-yearMonthDurations](https://www.w3.org/TR/xpath-functions-31/#func-subtract-yearMonthDurations) + /// + /// Returns `None` in case of overflow ([`FODT0002`](https://www.w3.org/TR/xpath-functions-31/#ERRFODT0002)). #[inline] pub fn checked_sub(self, rhs: impl Into) -> Option { let rhs = rhs.into(); @@ -309,6 +336,9 @@ impl YearMonthDuration { }) } + /// Unary negation. + /// + /// Returns `None` in case of overflow ([`FODT0002`](https://www.w3.org/TR/xpath-functions-31/#ERRFODT0002)). #[inline] pub fn checked_neg(self) -> Option { Some(Self { @@ -321,6 +351,10 @@ impl YearMonthDuration { pub fn is_identical_with(self, other: Self) -> bool { self == other } + + pub const MIN: Self = Self { months: i64::MIN }; + + pub const MAX: Self = Self { months: i64::MAX }; } impl From for Duration { @@ -334,23 +368,31 @@ impl From for Duration { } impl TryFrom for YearMonthDuration { - type Error = DecimalOverflowError; + type Error = DurationOverflowError; #[inline] - fn try_from(value: Duration) -> Result { + fn try_from(value: Duration) -> Result { if value.day_time == DayTimeDuration::default() { Ok(value.year_month) } else { - Err(DecimalOverflowError {}) + Err(DurationOverflowError) } } } impl FromStr for YearMonthDuration { - type Err = XsdParseError; - - fn from_str(input: &str) -> Result { - parse_year_month_duration(input) + type Err = ParseDurationError; + + fn from_str(input: &str) -> Result { + let parts = ensure_complete(input, duration_parts)?; + if parts.day_time.is_some() { + return Err(ParseDurationError::msg( + "There must not be any day or time component in a yearMonthDuration", + )); + } + Ok(Self::new(parts.year_month.ok_or( + ParseDurationError::msg("No year and month values found"), + )?)) } } @@ -455,6 +497,8 @@ impl DayTimeDuration { } /// [op:add-dayTimeDurations](https://www.w3.org/TR/xpath-functions-31/#func-add-dayTimeDurations) + /// + /// Returns `None` in case of overflow ([`FODT0002`](https://www.w3.org/TR/xpath-functions-31/#ERRFODT0002)). #[inline] pub fn checked_add(self, rhs: impl Into) -> Option { let rhs = rhs.into(); @@ -464,6 +508,8 @@ impl DayTimeDuration { } /// [op:subtract-dayTimeDurations](https://www.w3.org/TR/xpath-functions-31/#func-subtract-dayTimeDurations) + /// + /// Returns `None` in case of overflow ([`FODT0002`](https://www.w3.org/TR/xpath-functions-31/#ERRFODT0002)). #[inline] pub fn checked_sub(self, rhs: impl Into) -> Option { let rhs = rhs.into(); @@ -472,6 +518,9 @@ impl DayTimeDuration { }) } + /// Unary negation. + /// + /// Returns `None` in case of overflow ([`FODT0002`](https://www.w3.org/TR/xpath-functions-31/#ERRFODT0002)). #[inline] pub fn checked_neg(self) -> Option { Some(Self { @@ -484,6 +533,14 @@ impl DayTimeDuration { pub fn is_identical_with(self, other: Self) -> bool { self == other } + + pub const MIN: Self = Self { + seconds: Decimal::MIN, + }; + + pub const MAX: Self = Self { + seconds: Decimal::MAX, + }; } impl From for Duration { @@ -497,65 +554,75 @@ impl From for Duration { } impl TryFrom for DayTimeDuration { - type Error = DecimalOverflowError; + type Error = DurationOverflowError; #[inline] - fn try_from(value: Duration) -> Result { + fn try_from(value: Duration) -> Result { if value.year_month == YearMonthDuration::default() { Ok(value.day_time) } else { - Err(DecimalOverflowError {}) + Err(DurationOverflowError) } } } impl TryFrom for DayTimeDuration { - type Error = DecimalOverflowError; + type Error = DurationOverflowError; #[inline] - fn try_from(value: StdDuration) -> Result { + fn try_from(value: StdDuration) -> Result { Ok(Self { seconds: Decimal::new( - i128::try_from(value.as_nanos()).map_err(|_| DecimalOverflowError)?, + i128::try_from(value.as_nanos()).map_err(|_| DurationOverflowError)?, 9, - )?, + ) + .map_err(|_| DurationOverflowError)?, }) } } impl TryFrom for StdDuration { - type Error = DecimalOverflowError; + type Error = DurationOverflowError; #[inline] - fn try_from(value: DayTimeDuration) -> Result { + fn try_from(value: DayTimeDuration) -> Result { if value.seconds.is_negative() { - return Err(DecimalOverflowError); + return Err(DurationOverflowError); } - let secs = value.seconds.floor(); + let secs = value.seconds.checked_floor().ok_or(DurationOverflowError)?; let nanos = value .seconds .checked_sub(secs) - .ok_or(DecimalOverflowError)? + .ok_or(DurationOverflowError)? .checked_mul(1_000_000_000) - .ok_or(DecimalOverflowError)? - .floor(); + .ok_or(DurationOverflowError)? + .checked_floor() + .ok_or(DurationOverflowError)?; Ok(StdDuration::new( secs.as_i128() .try_into() - .map_err(|_| DecimalOverflowError)?, + .map_err(|_| DurationOverflowError)?, nanos .as_i128() .try_into() - .map_err(|_| DecimalOverflowError)?, + .map_err(|_| DurationOverflowError)?, )) } } impl FromStr for DayTimeDuration { - type Err = XsdParseError; - - fn from_str(input: &str) -> Result { - parse_day_time_duration(input) + type Err = ParseDurationError; + + fn from_str(input: &str) -> Result { + let parts = ensure_complete(input, duration_parts)?; + if parts.year_month.is_some() { + return Err(ParseDurationError::msg( + "There must not be any year or month component in a dayTimeDuration", + )); + } + Ok(Self::new(parts.day_time.ok_or(ParseDurationError::msg( + "No day or time values found", + ))?)) } } @@ -622,12 +689,273 @@ impl PartialOrd for YearMonthDuration { } } +// [6] duYearFrag ::= unsignedNoDecimalPtNumeral 'Y' +// [7] duMonthFrag ::= unsignedNoDecimalPtNumeral 'M' +// [8] duDayFrag ::= unsignedNoDecimalPtNumeral 'D' +// [9] duHourFrag ::= unsignedNoDecimalPtNumeral 'H' +// [10] duMinuteFrag ::= unsignedNoDecimalPtNumeral 'M' +// [11] duSecondFrag ::= (unsignedNoDecimalPtNumeral | unsignedDecimalPtNumeral) 'S' +// [12] duYearMonthFrag ::= (duYearFrag duMonthFrag?) | duMonthFrag +// [13] duTimeFrag ::= 'T' ((duHourFrag duMinuteFrag? duSecondFrag?) | (duMinuteFrag duSecondFrag?) | duSecondFrag) +// [14] duDayTimeFrag ::= (duDayFrag duTimeFrag?) | duTimeFrag +// [15] durationLexicalRep ::= '-'? 'P' ((duYearMonthFrag duDayTimeFrag?) | duDayTimeFrag) +struct DurationParts { + year_month: Option, + day_time: Option, +} + +fn duration_parts(input: &str) -> Result<(DurationParts, &str), ParseDurationError> { + // States + const START: u32 = 0; + const AFTER_YEAR: u32 = 1; + const AFTER_MONTH: u32 = 2; + const AFTER_DAY: u32 = 3; + const AFTER_T: u32 = 4; + const AFTER_HOUR: u32 = 5; + const AFTER_MINUTE: u32 = 6; + const AFTER_SECOND: u32 = 7; + + let (is_negative, input) = if let Some(left) = input.strip_prefix('-') { + (true, left) + } else { + (false, input) + }; + let mut input = expect_char(input, 'P', "Durations must start with 'P'")?; + let mut state = START; + let mut year_month: Option = None; + let mut day_time: Option = None; + while !input.is_empty() { + if let Some(left) = input.strip_prefix('T') { + if state >= AFTER_T { + return Err(ParseDurationError::msg("Duplicated time separator 'T'")); + } + state = AFTER_T; + input = left; + } else { + let (number_str, left) = decimal_prefix(input); + match left.chars().next() { + Some('Y') if state < AFTER_YEAR => { + year_month = Some( + year_month + .unwrap_or_default() + .checked_add( + apply_i64_neg( + i64::from_str(number_str).map_err(|_| OVERFLOW_ERROR)?, + is_negative, + )? + .checked_mul(12) + .ok_or(OVERFLOW_ERROR)?, + ) + .ok_or(OVERFLOW_ERROR)?, + ); + state = AFTER_YEAR; + } + Some('M') if state < AFTER_MONTH => { + year_month = Some( + year_month + .unwrap_or_default() + .checked_add(apply_i64_neg( + i64::from_str(number_str).map_err(|_| OVERFLOW_ERROR)?, + is_negative, + )?) + .ok_or(OVERFLOW_ERROR)?, + ); + state = AFTER_MONTH; + } + Some('D') if state < AFTER_DAY => { + if number_str.contains('.') { + return Err(ParseDurationError::msg( + "Decimal numbers are not allowed for days", + )); + } + day_time = Some( + day_time + .unwrap_or_default() + .checked_add( + apply_decimal_neg( + Decimal::from_str(number_str).map_err(|_| OVERFLOW_ERROR)?, + is_negative, + )? + .checked_mul(86400) + .ok_or(OVERFLOW_ERROR)?, + ) + .ok_or(OVERFLOW_ERROR)?, + ); + state = AFTER_DAY; + } + Some('H') if state == AFTER_T => { + if number_str.contains('.') { + return Err(ParseDurationError::msg( + "Decimal numbers are not allowed for hours", + )); + } + day_time = Some( + day_time + .unwrap_or_default() + .checked_add( + apply_decimal_neg( + Decimal::from_str(number_str).map_err(|_| OVERFLOW_ERROR)?, + is_negative, + )? + .checked_mul(3600) + .ok_or(OVERFLOW_ERROR)?, + ) + .ok_or(OVERFLOW_ERROR)?, + ); + state = AFTER_HOUR; + } + Some('M') if (AFTER_T..AFTER_MINUTE).contains(&state) => { + if number_str.contains('.') { + return Err(ParseDurationError::msg( + "Decimal numbers are not allowed for minutes", + )); + } + day_time = Some( + day_time + .unwrap_or_default() + .checked_add( + apply_decimal_neg( + Decimal::from_str(number_str).map_err(|_| OVERFLOW_ERROR)?, + is_negative, + )? + .checked_mul(60) + .ok_or(OVERFLOW_ERROR)?, + ) + .ok_or(OVERFLOW_ERROR)?, + ); + state = AFTER_MINUTE; + } + Some('S') if (AFTER_T..AFTER_SECOND).contains(&state) => { + day_time = Some( + day_time + .unwrap_or_default() + .checked_add(apply_decimal_neg( + Decimal::from_str(number_str).map_err(|_| OVERFLOW_ERROR)?, + is_negative, + )?) + .ok_or(OVERFLOW_ERROR)?, + ); + state = AFTER_SECOND; + } + Some(_) => return Err(ParseDurationError::msg("Unexpected type character")), + None => { + return Err(ParseDurationError::msg( + "Numbers in durations must be followed by a type character", + )) + } + } + input = &left[1..]; + } + } + + Ok(( + DurationParts { + year_month, + day_time, + }, + input, + )) +} + +fn apply_i64_neg(value: i64, is_negative: bool) -> Result { + if is_negative { + value.checked_neg().ok_or(OVERFLOW_ERROR) + } else { + Ok(value) + } +} + +fn apply_decimal_neg(value: Decimal, is_negative: bool) -> Result { + if is_negative { + value.checked_neg().ok_or(OVERFLOW_ERROR) + } else { + Ok(value) + } +} + +fn ensure_complete( + input: &str, + parse: impl FnOnce(&str) -> Result<(T, &str), ParseDurationError>, +) -> Result { + let (result, left) = parse(input)?; + if !left.is_empty() { + return Err(ParseDurationError::msg("Unrecognized value suffix")); + } + Ok(result) +} + +fn expect_char<'a>( + input: &'a str, + constant: char, + error_message: &'static str, +) -> Result<&'a str, ParseDurationError> { + if let Some(left) = input.strip_prefix(constant) { + Ok(left) + } else { + Err(ParseDurationError::msg(error_message)) + } +} + +fn decimal_prefix(input: &str) -> (&str, &str) { + let mut end = input.len(); + let mut dot_seen = false; + for (i, c) in input.char_indices() { + if c.is_ascii_digit() { + // Ok + } else if c == '.' && !dot_seen { + dot_seen = true; + } else { + end = i; + break; + } + } + input.split_at(end) +} + +/// A parsing error +#[derive(Debug, Clone)] +pub struct ParseDurationError { + msg: &'static str, +} + +const OVERFLOW_ERROR: ParseDurationError = ParseDurationError { + msg: "Overflow error", +}; + +impl fmt::Display for ParseDurationError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.msg) + } +} + +impl ParseDurationError { + const fn msg(msg: &'static str) -> Self { + Self { msg } + } +} + +impl Error for ParseDurationError {} + +/// An overflow during [`Duration`]-related operations. +/// +/// Matches XPath [`FODT0002` error](https://www.w3.org/TR/xpath-functions-31/#ERRFODT0002). +#[derive(Debug, Clone, Copy)] +pub struct DurationOverflowError; + +impl fmt::Display for DurationOverflowError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "overflow during xsd:duration computation") + } +} + +impl Error for DurationOverflowError {} + #[cfg(test)] mod tests { use super::*; #[test] - fn from_str() -> Result<(), XsdParseError> { + fn from_str() -> Result<(), ParseDurationError> { let min = Duration::new(i64::MIN, Decimal::MIN); let max = Duration::new(i64::MAX, Decimal::MAX); @@ -667,25 +995,52 @@ mod tests { } #[test] - fn from_std() { + fn from_std() -> Result<(), DurationOverflowError> { assert_eq!( - Duration::try_from(StdDuration::new(10, 10)) - .unwrap() - .to_string(), + Duration::try_from(StdDuration::new(10, 10))?.to_string(), "PT10.00000001S" ); + Ok(()) } #[test] - fn to_std() -> Result<(), XsdParseError> { - let duration = StdDuration::try_from(DayTimeDuration::from_str("PT10.00000001S")?).unwrap(); + fn to_std() -> Result<(), Box> { + let duration = StdDuration::try_from(DayTimeDuration::from_str("PT10.00000001S")?)?; assert_eq!(duration.as_secs(), 10); assert_eq!(duration.subsec_nanos(), 10); Ok(()) } #[test] - fn equals() -> Result<(), XsdParseError> { + fn to_be_bytes() { + assert_eq!( + Duration::from_be_bytes(Duration::MIN.to_be_bytes()), + Duration::MIN + ); + assert_eq!( + Duration::from_be_bytes(Duration::MAX.to_be_bytes()), + Duration::MAX + ); + assert_eq!( + YearMonthDuration::from_be_bytes(YearMonthDuration::MIN.to_be_bytes()), + YearMonthDuration::MIN + ); + assert_eq!( + YearMonthDuration::from_be_bytes(YearMonthDuration::MAX.to_be_bytes()), + YearMonthDuration::MAX + ); + assert_eq!( + DayTimeDuration::from_be_bytes(DayTimeDuration::MIN.to_be_bytes()), + DayTimeDuration::MIN + ); + assert_eq!( + DayTimeDuration::from_be_bytes(DayTimeDuration::MAX.to_be_bytes()), + DayTimeDuration::MAX + ); + } + + #[test] + fn equals() -> Result<(), ParseDurationError> { assert_eq!( YearMonthDuration::from_str("P1Y")?, YearMonthDuration::from_str("P12M")? @@ -730,7 +1085,24 @@ mod tests { } #[test] - fn years() -> Result<(), XsdParseError> { + #[allow(clippy::neg_cmp_op_on_partial_ord)] + fn cmp() -> Result<(), ParseDurationError> { + assert!(Duration::from_str("P1Y1D")? < Duration::from_str("P13MT25H")?); + assert!(YearMonthDuration::from_str("P1Y")? < YearMonthDuration::from_str("P13M")?); + assert!(Duration::from_str("P1Y")? < YearMonthDuration::from_str("P13M")?); + assert!(YearMonthDuration::from_str("P1Y")? < Duration::from_str("P13M")?); + assert!(DayTimeDuration::from_str("P1D")? < DayTimeDuration::from_str("PT25H")?); + assert!(DayTimeDuration::from_str("PT1H")? < DayTimeDuration::from_str("PT61M")?); + assert!(DayTimeDuration::from_str("PT1M")? < DayTimeDuration::from_str("PT61S")?); + assert!(Duration::from_str("PT1H")? < DayTimeDuration::from_str("PT61M")?); + assert!(DayTimeDuration::from_str("PT1H")? < Duration::from_str("PT61M")?); + assert!(YearMonthDuration::from_str("P1M")? < DayTimeDuration::from_str("P40D")?); + assert!(DayTimeDuration::from_str("P25D")? < YearMonthDuration::from_str("P1M")?); + Ok(()) + } + + #[test] + fn years() -> Result<(), ParseDurationError> { assert_eq!(Duration::from_str("P20Y15M")?.years(), 21); assert_eq!(Duration::from_str("-P15M")?.years(), -1); assert_eq!(Duration::from_str("-P2DT15H")?.years(), 0); @@ -738,7 +1110,7 @@ mod tests { } #[test] - fn months() -> Result<(), XsdParseError> { + fn months() -> Result<(), ParseDurationError> { assert_eq!(Duration::from_str("P20Y15M")?.months(), 3); assert_eq!(Duration::from_str("-P20Y18M")?.months(), -6); assert_eq!(Duration::from_str("-P2DT15H0M0S")?.months(), 0); @@ -746,7 +1118,7 @@ mod tests { } #[test] - fn days() -> Result<(), XsdParseError> { + fn days() -> Result<(), ParseDurationError> { assert_eq!(Duration::from_str("P3DT10H")?.days(), 3); assert_eq!(Duration::from_str("P3DT55H")?.days(), 5); assert_eq!(Duration::from_str("P3Y5M")?.days(), 0); @@ -754,7 +1126,7 @@ mod tests { } #[test] - fn hours() -> Result<(), XsdParseError> { + fn hours() -> Result<(), ParseDurationError> { assert_eq!(Duration::from_str("P3DT10H")?.hours(), 10); assert_eq!(Duration::from_str("P3DT12H32M12S")?.hours(), 12); assert_eq!(Duration::from_str("PT123H")?.hours(), 3); @@ -763,14 +1135,14 @@ mod tests { } #[test] - fn minutes() -> Result<(), XsdParseError> { + fn minutes() -> Result<(), ParseDurationError> { assert_eq!(Duration::from_str("P3DT10H")?.minutes(), 0); assert_eq!(Duration::from_str("-P5DT12H30M")?.minutes(), -30); Ok(()) } #[test] - fn seconds() -> Result<(), XsdParseError> { + fn seconds() -> Result<(), Box> { assert_eq!( Duration::from_str("P3DT10H12.5S")?.seconds(), Decimal::from_str("12.5")? @@ -783,7 +1155,7 @@ mod tests { } #[test] - fn add() -> Result<(), XsdParseError> { + fn add() -> Result<(), ParseDurationError> { assert_eq!( Duration::from_str("P2Y11M")?.checked_add(Duration::from_str("P3Y3M")?), Some(Duration::from_str("P6Y2M")?) @@ -796,7 +1168,7 @@ mod tests { } #[test] - fn sub() -> Result<(), XsdParseError> { + fn sub() -> Result<(), ParseDurationError> { assert_eq!( Duration::from_str("P2Y11M")?.checked_sub(Duration::from_str("P3Y3M")?), Some(Duration::from_str("-P4M")?) @@ -809,7 +1181,7 @@ mod tests { } #[test] - fn minimally_conformant() -> Result<(), XsdParseError> { + fn minimally_conformant() -> Result<(), ParseDurationError> { // All minimally conforming processors must support fractional-second duration values // to milliseconds (i.e. those expressible with three fraction digits). assert_eq!(Duration::from_str("PT0.001S")?.to_string(), "PT0.001S"); diff --git a/lib/oxsdatatypes/src/integer.rs b/lib/oxsdatatypes/src/integer.rs index f376a57d..352e521a 100644 --- a/lib/oxsdatatypes/src/integer.rs +++ b/lib/oxsdatatypes/src/integer.rs @@ -1,4 +1,5 @@ -use crate::{Boolean, Decimal, DecimalOverflowError, Double, Float}; +use crate::{Boolean, Decimal, Double, Float}; +use std::error::Error; use std::fmt; use std::num::ParseIntError; use std::str::FromStr; @@ -28,6 +29,8 @@ impl Integer { } /// [op:numeric-add](https://www.w3.org/TR/xpath-functions-31/#func-numeric-add) + /// + /// Returns `None` in case of overflow ([FOAR0002](https://www.w3.org/TR/xpath-functions-31/#ERRFOAR0002)). #[inline] #[must_use] pub fn checked_add(self, rhs: impl Into) -> Option { @@ -37,6 +40,8 @@ impl Integer { } /// [op:numeric-subtract](https://www.w3.org/TR/xpath-functions-31/#func-numeric-subtract) + /// + /// Returns `None` in case of overflow ([FOAR0002](https://www.w3.org/TR/xpath-functions-31/#ERRFOAR0002)). #[inline] #[must_use] pub fn checked_sub(self, rhs: impl Into) -> Option { @@ -46,6 +51,8 @@ impl Integer { } /// [op:numeric-multiply](https://www.w3.org/TR/xpath-functions-31/#func-numeric-multiply) + /// + /// Returns `None` in case of overflow ([FOAR0002](https://www.w3.org/TR/xpath-functions-31/#ERRFOAR0002)). #[inline] #[must_use] pub fn checked_mul(self, rhs: impl Into) -> Option { @@ -55,6 +62,8 @@ impl Integer { } /// [op:numeric-integer-divide](https://www.w3.org/TR/xpath-functions-31/#func-numeric-integer-divide) + /// + /// Returns `None` in case of division by 0 ([FOAR0001](https://www.w3.org/TR/xpath-functions-31/#ERRFOAR0001)) or overflow ([FOAR0002](https://www.w3.org/TR/xpath-functions-31/#ERRFOAR0002)). #[inline] #[must_use] pub fn checked_div(self, rhs: impl Into) -> Option { @@ -64,6 +73,8 @@ impl Integer { } /// [op:numeric-mod](https://www.w3.org/TR/xpath-functions-31/#func-numeric-mod) + /// + /// Returns `None` in case of division by 0 ([FOAR0001](https://www.w3.org/TR/xpath-functions-31/#ERRFOAR0001)) or overflow ([FOAR0002](https://www.w3.org/TR/xpath-functions-31/#ERRFOAR0002)). #[inline] #[must_use] pub fn checked_rem(self, rhs: impl Into) -> Option { @@ -72,6 +83,9 @@ impl Integer { }) } + /// Euclidean remainder + /// + /// Returns `None` in case of division by 0 ([FOAR0001](https://www.w3.org/TR/xpath-functions-31/#ERRFOAR0001)) or overflow ([FOAR0002](https://www.w3.org/TR/xpath-functions-31/#ERRFOAR0002)). #[inline] #[must_use] pub fn checked_rem_euclid(self, rhs: impl Into) -> Option { @@ -81,6 +95,8 @@ impl Integer { } /// [op:numeric-unary-minus](https://www.w3.org/TR/xpath-functions-31/#func-numeric-unary-minus) + /// + /// Returns `None` in case of overflow ([FOAR0002](https://www.w3.org/TR/xpath-functions-31/#ERRFOAR0002)). #[inline] #[must_use] pub fn checked_neg(self) -> Option { @@ -90,12 +106,14 @@ impl Integer { } /// [fn:abs](https://www.w3.org/TR/xpath-functions-31/#func-abs) + /// + /// Returns `None` in case of overflow ([FOAR0002](https://www.w3.org/TR/xpath-functions-31/#ERRFOAR0002)). #[inline] #[must_use] - pub const fn abs(self) -> Self { - Self { - value: self.value.abs(), - } + pub fn checked_abs(self) -> Option { + Some(Self { + value: self.value.checked_abs()?, + }) } #[inline] @@ -223,23 +241,41 @@ impl fmt::Display for Integer { } impl TryFrom for Integer { - type Error = DecimalOverflowError; + type Error = TooLargeForIntegerError; #[inline] - fn try_from(value: Float) -> Result { - Decimal::try_from(value)?.try_into() + fn try_from(value: Float) -> Result { + Decimal::try_from(value) + .map_err(|_| TooLargeForIntegerError)? + .try_into() } } impl TryFrom for Integer { - type Error = DecimalOverflowError; + type Error = TooLargeForIntegerError; #[inline] - fn try_from(value: Double) -> Result { - Decimal::try_from(value)?.try_into() + fn try_from(value: Double) -> Result { + Decimal::try_from(value) + .map_err(|_| TooLargeForIntegerError)? + .try_into() } } +/// The input is too large to fit into an [`Integer`]. +/// +/// Matches XPath [`FOCA0003` error](https://www.w3.org/TR/xpath-functions-31/#ERRFOCA0003). +#[derive(Debug, Clone, Copy)] +pub struct TooLargeForIntegerError; + +impl fmt::Display for TooLargeForIntegerError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Value too large for xsd:integer internal representation") + } +} + +impl Error for TooLargeForIntegerError {} + #[cfg(test)] mod tests { use super::*; @@ -278,7 +314,8 @@ mod tests { .unwrap() .checked_sub(Integer::from_str("1672507300000")?) .unwrap() - .abs() + .checked_abs() + .unwrap() < Integer::from(1_000_000) ); Ok(()) @@ -303,7 +340,8 @@ mod tests { .unwrap() .checked_sub(Integer::from_str("1672507300000").unwrap()) .unwrap() - .abs() + .checked_abs() + .unwrap() < Integer::from(10) ); assert!(Integer::try_from(Double::from(f64::NAN)).is_err()); diff --git a/lib/oxsdatatypes/src/lib.rs b/lib/oxsdatatypes/src/lib.rs index 6e1cd28f..a31caf61 100644 --- a/lib/oxsdatatypes/src/lib.rs +++ b/lib/oxsdatatypes/src/lib.rs @@ -11,15 +11,16 @@ mod double; mod duration; mod float; mod integer; -mod parser; pub use self::boolean::Boolean; pub use self::date_time::{ - Date, DateTime, DateTimeError, GDay, GMonth, GMonthDay, GYear, GYearMonth, Time, TimezoneOffset, + Date, DateTime, DateTimeOverflowError, GDay, GMonth, GMonthDay, GYear, GYearMonth, + InvalidTimezoneError, ParseDateTimeError, Time, TimezoneOffset, }; -pub use self::decimal::{Decimal, DecimalOverflowError, ParseDecimalError}; +pub use self::decimal::{Decimal, ParseDecimalError, TooLargeForDecimalError}; pub use self::double::Double; -pub use self::duration::{DayTimeDuration, Duration, YearMonthDuration}; +pub use self::duration::{ + DayTimeDuration, Duration, DurationOverflowError, ParseDurationError, YearMonthDuration, +}; pub use self::float::Float; -pub use self::integer::Integer; -pub use self::parser::XsdParseError; +pub use self::integer::{Integer, TooLargeForIntegerError}; diff --git a/lib/oxsdatatypes/src/parser.rs b/lib/oxsdatatypes/src/parser.rs deleted file mode 100644 index 942c71e4..00000000 --- a/lib/oxsdatatypes/src/parser.rs +++ /dev/null @@ -1,626 +0,0 @@ -use super::date_time::{DateTimeError, GDay, GMonth, GMonthDay, GYear, GYearMonth, TimezoneOffset}; -use super::decimal::ParseDecimalError; -use super::duration::{DayTimeDuration, YearMonthDuration}; -use super::*; -use std::error::Error; -use std::fmt; -use std::num::ParseIntError; -use std::str::FromStr; - -/// A parsing error -#[derive(Debug, Clone)] -pub struct XsdParseError { - kind: XsdParseErrorKind, -} - -#[derive(Debug, Clone)] -enum XsdParseErrorKind { - ParseInt(ParseIntError), - ParseDecimal(ParseDecimalError), - DateTime(DateTimeError), - Message(&'static str), -} - -const OVERFLOW_ERROR: XsdParseError = XsdParseError { - kind: XsdParseErrorKind::Message("Overflow error"), -}; - -impl fmt::Display for XsdParseError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match &self.kind { - XsdParseErrorKind::ParseInt(error) => { - write!(f, "Error while parsing integer: {error}") - } - XsdParseErrorKind::ParseDecimal(error) => { - write!(f, "Error while parsing decimal: {error}") - } - XsdParseErrorKind::DateTime(error) => error.fmt(f), - XsdParseErrorKind::Message(msg) => write!(f, "{msg}"), - } - } -} - -impl XsdParseError { - const fn msg(message: &'static str) -> Self { - Self { - kind: XsdParseErrorKind::Message(message), - } - } -} - -impl Error for XsdParseError { - fn source(&self) -> Option<&(dyn Error + 'static)> { - match &self.kind { - XsdParseErrorKind::ParseInt(error) => Some(error), - XsdParseErrorKind::ParseDecimal(error) => Some(error), - XsdParseErrorKind::DateTime(error) => Some(error), - XsdParseErrorKind::Message(_) => None, - } - } -} - -impl From for XsdParseError { - fn from(error: ParseIntError) -> Self { - Self { - kind: XsdParseErrorKind::ParseInt(error), - } - } -} - -impl From for XsdParseError { - fn from(error: ParseDecimalError) -> Self { - Self { - kind: XsdParseErrorKind::ParseDecimal(error), - } - } -} - -impl From for XsdParseError { - fn from(error: DateTimeError) -> Self { - Self { - kind: XsdParseErrorKind::DateTime(error), - } - } -} - -// [6] duYearFrag ::= unsignedNoDecimalPtNumeral 'Y' -// [7] duMonthFrag ::= unsignedNoDecimalPtNumeral 'M' -// [8] duDayFrag ::= unsignedNoDecimalPtNumeral 'D' -// [9] duHourFrag ::= unsignedNoDecimalPtNumeral 'H' -// [10] duMinuteFrag ::= unsignedNoDecimalPtNumeral 'M' -// [11] duSecondFrag ::= (unsignedNoDecimalPtNumeral | unsignedDecimalPtNumeral) 'S' -// [12] duYearMonthFrag ::= (duYearFrag duMonthFrag?) | duMonthFrag -// [13] duTimeFrag ::= 'T' ((duHourFrag duMinuteFrag? duSecondFrag?) | (duMinuteFrag duSecondFrag?) | duSecondFrag) -// [14] duDayTimeFrag ::= (duDayFrag duTimeFrag?) | duTimeFrag -// [15] durationLexicalRep ::= '-'? 'P' ((duYearMonthFrag duDayTimeFrag?) | duDayTimeFrag) -struct DurationParts { - year_month: Option, - day_time: Option, -} - -fn duration_parts(input: &str) -> Result<(DurationParts, &str), XsdParseError> { - // States - const START: u32 = 0; - const AFTER_YEAR: u32 = 1; - const AFTER_MONTH: u32 = 2; - const AFTER_DAY: u32 = 3; - const AFTER_T: u32 = 4; - const AFTER_HOUR: u32 = 5; - const AFTER_MINUTE: u32 = 6; - const AFTER_SECOND: u32 = 7; - - let (is_negative, input) = if let Some(left) = input.strip_prefix('-') { - (true, left) - } else { - (false, input) - }; - let mut input = expect_char(input, 'P', "Durations must start with 'P'")?; - let mut state = START; - let mut year_month: Option = None; - let mut day_time: Option = None; - while !input.is_empty() { - if let Some(left) = input.strip_prefix('T') { - if state >= AFTER_T { - return Err(XsdParseError::msg("Duplicated time separator 'T'")); - } - state = AFTER_T; - input = left; - } else { - let (number_str, left) = decimal_prefix(input); - match left.chars().next() { - Some('Y') if state < AFTER_YEAR => { - year_month = Some( - year_month - .unwrap_or_default() - .checked_add( - apply_i64_neg(i64::from_str(number_str)?, is_negative)? - .checked_mul(12) - .ok_or(OVERFLOW_ERROR)?, - ) - .ok_or(OVERFLOW_ERROR)?, - ); - state = AFTER_YEAR; - } - Some('M') if state < AFTER_MONTH => { - year_month = Some( - year_month - .unwrap_or_default() - .checked_add(apply_i64_neg(i64::from_str(number_str)?, is_negative)?) - .ok_or(OVERFLOW_ERROR)?, - ); - state = AFTER_MONTH; - } - Some('D') if state < AFTER_DAY => { - if number_str.contains('.') { - return Err(XsdParseError::msg( - "Decimal numbers are not allowed for days", - )); - } - day_time = Some( - day_time - .unwrap_or_default() - .checked_add( - apply_decimal_neg(Decimal::from_str(number_str)?, is_negative)? - .checked_mul(86400) - .ok_or(OVERFLOW_ERROR)?, - ) - .ok_or(OVERFLOW_ERROR)?, - ); - state = AFTER_DAY; - } - Some('H') if state == AFTER_T => { - if number_str.contains('.') { - return Err(XsdParseError::msg( - "Decimal numbers are not allowed for hours", - )); - } - day_time = Some( - day_time - .unwrap_or_default() - .checked_add( - apply_decimal_neg(Decimal::from_str(number_str)?, is_negative)? - .checked_mul(3600) - .ok_or(OVERFLOW_ERROR)?, - ) - .ok_or(OVERFLOW_ERROR)?, - ); - state = AFTER_HOUR; - } - Some('M') if (AFTER_T..AFTER_MINUTE).contains(&state) => { - if number_str.contains('.') { - return Err(XsdParseError::msg( - "Decimal numbers are not allowed for minutes", - )); - } - day_time = Some( - day_time - .unwrap_or_default() - .checked_add( - apply_decimal_neg(Decimal::from_str(number_str)?, is_negative)? - .checked_mul(60) - .ok_or(OVERFLOW_ERROR)?, - ) - .ok_or(OVERFLOW_ERROR)?, - ); - state = AFTER_MINUTE; - } - Some('S') if (AFTER_T..AFTER_SECOND).contains(&state) => { - day_time = Some( - day_time - .unwrap_or_default() - .checked_add(apply_decimal_neg( - Decimal::from_str(number_str)?, - is_negative, - )?) - .ok_or(OVERFLOW_ERROR)?, - ); - state = AFTER_SECOND; - } - Some(_) => return Err(XsdParseError::msg("Unexpected type character")), - None => { - return Err(XsdParseError::msg( - "Numbers in durations must be followed by a type character", - )) - } - } - input = &left[1..]; - } - } - - Ok(( - DurationParts { - year_month, - day_time, - }, - input, - )) -} - -fn apply_i64_neg(value: i64, is_negative: bool) -> Result { - if is_negative { - value.checked_neg().ok_or(OVERFLOW_ERROR) - } else { - Ok(value) - } -} - -fn apply_decimal_neg(value: Decimal, is_negative: bool) -> Result { - if is_negative { - value.checked_neg().ok_or(OVERFLOW_ERROR) - } else { - Ok(value) - } -} - -pub fn parse_duration(input: &str) -> Result { - let parts = ensure_complete(input, duration_parts)?; - if parts.year_month.is_none() && parts.day_time.is_none() { - return Err(XsdParseError::msg("Empty duration")); - } - Ok(Duration::new( - parts.year_month.unwrap_or(0), - parts.day_time.unwrap_or_default(), - )) -} - -pub fn parse_year_month_duration(input: &str) -> Result { - let parts = ensure_complete(input, duration_parts)?; - if parts.day_time.is_some() { - return Err(XsdParseError::msg( - "There must not be any day or time component in a yearMonthDuration", - )); - } - Ok(YearMonthDuration::new(parts.year_month.ok_or( - XsdParseError::msg("No year and month values found"), - )?)) -} - -pub fn parse_day_time_duration(input: &str) -> Result { - let parts = ensure_complete(input, duration_parts)?; - if parts.year_month.is_some() { - return Err(XsdParseError::msg( - "There must not be any year or month component in a dayTimeDuration", - )); - } - Ok(DayTimeDuration::new(parts.day_time.ok_or( - XsdParseError::msg("No day or time values found"), - )?)) -} - -// [16] dateTimeLexicalRep ::= yearFrag '-' monthFrag '-' dayFrag 'T' ((hourFrag ':' minuteFrag ':' secondFrag) | endOfDayFrag) timezoneFrag? -fn date_time_lexical_rep(input: &str) -> Result<(DateTime, &str), XsdParseError> { - let (year, input) = year_frag(input)?; - let input = expect_char(input, '-', "The year and month must be separated by '-'")?; - let (month, input) = month_frag(input)?; - let input = expect_char(input, '-', "The month and day must be separated by '-'")?; - let (day, input) = day_frag(input)?; - let input = expect_char(input, 'T', "The date and time must be separated by 'T'")?; - let (hour, input) = hour_frag(input)?; - let input = expect_char(input, ':', "The hours and minutes must be separated by ':'")?; - let (minute, input) = minute_frag(input)?; - let input = expect_char( - input, - ':', - "The minutes and seconds must be separated by ':'", - )?; - let (second, input) = second_frag(input)?; - // We validate 24:00:00 - if hour == 24 && minute != 0 && second != Decimal::from(0) { - return Err(XsdParseError::msg( - "Times are not allowed to be after 24:00:00", - )); - } - let (timezone_offset, input) = optional_end(input, timezone_frag)?; - Ok(( - DateTime::new(year, month, day, hour, minute, second, timezone_offset)?, - input, - )) -} - -pub fn parse_date_time(input: &str) -> Result { - ensure_complete(input, date_time_lexical_rep) -} - -// [17] timeLexicalRep ::= ((hourFrag ':' minuteFrag ':' secondFrag) | endOfDayFrag) timezoneFrag? -fn time_lexical_rep(input: &str) -> Result<(Time, &str), XsdParseError> { - let (hour, input) = hour_frag(input)?; - let input = expect_char(input, ':', "The hours and minutes must be separated by ':'")?; - let (minute, input) = minute_frag(input)?; - let input = expect_char( - input, - ':', - "The minutes and seconds must be separated by ':'", - )?; - let (second, input) = second_frag(input)?; - // We validate 24:00:00 - if hour == 24 && minute != 0 && second != Decimal::from(0) { - return Err(XsdParseError::msg( - "Times are not allowed to be after 24:00:00", - )); - } - let (timezone_offset, input) = optional_end(input, timezone_frag)?; - Ok((Time::new(hour, minute, second, timezone_offset)?, input)) -} - -pub fn parse_time(input: &str) -> Result { - ensure_complete(input, time_lexical_rep) -} - -// [18] dateLexicalRep ::= yearFrag '-' monthFrag '-' dayFrag timezoneFrag? Constraint: Day-of-month Representations -fn date_lexical_rep(input: &str) -> Result<(Date, &str), XsdParseError> { - let (year, input) = year_frag(input)?; - let input = expect_char(input, '-', "The year and month must be separated by '-'")?; - let (month, input) = month_frag(input)?; - let input = expect_char(input, '-', "The month and day must be separated by '-'")?; - let (day, input) = day_frag(input)?; - let (timezone_offset, input) = optional_end(input, timezone_frag)?; - Ok((Date::new(year, month, day, timezone_offset)?, input)) -} - -pub fn parse_date(input: &str) -> Result { - ensure_complete(input, date_lexical_rep) -} - -// [19] gYearMonthLexicalRep ::= yearFrag '-' monthFrag timezoneFrag? -fn g_year_month_lexical_rep(input: &str) -> Result<(GYearMonth, &str), XsdParseError> { - let (year, input) = year_frag(input)?; - let input = expect_char(input, '-', "The year and month must be separated by '-'")?; - let (month, input) = month_frag(input)?; - let (timezone_offset, input) = optional_end(input, timezone_frag)?; - Ok((GYearMonth::new(year, month, timezone_offset)?, input)) -} - -pub fn parse_g_year_month(input: &str) -> Result { - ensure_complete(input, g_year_month_lexical_rep) -} - -// [20] gYearLexicalRep ::= yearFrag timezoneFrag? -fn g_year_lexical_rep(input: &str) -> Result<(GYear, &str), XsdParseError> { - let (year, input) = year_frag(input)?; - let (timezone_offset, input) = optional_end(input, timezone_frag)?; - Ok((GYear::new(year, timezone_offset)?, input)) -} - -pub fn parse_g_year(input: &str) -> Result { - ensure_complete(input, g_year_lexical_rep) -} - -// [21] gMonthDayLexicalRep ::= '--' monthFrag '-' dayFrag timezoneFrag? Constraint: Day-of-month Representations -fn g_month_day_lexical_rep(input: &str) -> Result<(GMonthDay, &str), XsdParseError> { - let input = expect_char(input, '-', "gMonthDay values must start with '--'")?; - let input = expect_char(input, '-', "gMonthDay values must start with '--'")?; - let (month, input) = month_frag(input)?; - let input = expect_char(input, '-', "The month and day must be separated by '-'")?; - let (day, input) = day_frag(input)?; - let (timezone_offset, input) = optional_end(input, timezone_frag)?; - Ok((GMonthDay::new(month, day, timezone_offset)?, input)) -} - -pub fn parse_g_month_day(input: &str) -> Result { - ensure_complete(input, g_month_day_lexical_rep) -} - -// [22] gDayLexicalRep ::= '---' dayFrag timezoneFrag? -fn g_day_lexical_rep(input: &str) -> Result<(GDay, &str), XsdParseError> { - let input = expect_char(input, '-', "gDay values must start with '---'")?; - let input = expect_char(input, '-', "gDay values must start with '---'")?; - let input = expect_char(input, '-', "gDay values must start with '---'")?; - let (day, input) = day_frag(input)?; - let (timezone_offset, input) = optional_end(input, timezone_frag)?; - Ok((GDay::new(day, timezone_offset)?, input)) -} - -pub fn parse_g_day(input: &str) -> Result { - ensure_complete(input, g_day_lexical_rep) -} - -// [23] gMonthLexicalRep ::= '--' monthFrag timezoneFrag? -fn g_month_lexical_rep(input: &str) -> Result<(GMonth, &str), XsdParseError> { - let input = expect_char(input, '-', "gMonth values must start with '--'")?; - let input = expect_char(input, '-', "gMonth values must start with '--'")?; - let (month, input) = month_frag(input)?; - let (timezone_offset, input) = optional_end(input, timezone_frag)?; - Ok((GMonth::new(month, timezone_offset)?, input)) -} - -pub fn parse_g_month(input: &str) -> Result { - ensure_complete(input, g_month_lexical_rep) -} - -// [56] yearFrag ::= '-'? (([1-9] digit digit digit+)) | ('0' digit digit digit)) -fn year_frag(input: &str) -> Result<(i64, &str), XsdParseError> { - let (sign, input) = if let Some(left) = input.strip_prefix('-') { - (-1, left) - } else { - (1, input) - }; - let (number_str, input) = integer_prefix(input); - if number_str.len() < 4 { - return Err(XsdParseError::msg("The year should be encoded on 4 digits")); - } - if number_str.len() > 4 && number_str.starts_with('0') { - return Err(XsdParseError::msg( - "The years value must not start with 0 if it can be encoded in at least 4 digits", - )); - } - let number = i64::from_str(number_str)?; - Ok((sign * number, input)) -} - -// [57] monthFrag ::= ('0' [1-9]) | ('1' [0-2]) -fn month_frag(input: &str) -> Result<(u8, &str), XsdParseError> { - let (number_str, input) = integer_prefix(input); - if number_str.len() != 2 { - return Err(XsdParseError::msg("Month must be encoded with two digits")); - } - let number = u8::from_str(number_str)?; - if !(1..=12).contains(&number) { - return Err(XsdParseError::msg("Month must be between 01 and 12")); - } - Ok((number, input)) -} - -// [58] dayFrag ::= ('0' [1-9]) | ([12] digit) | ('3' [01]) -fn day_frag(input: &str) -> Result<(u8, &str), XsdParseError> { - let (number_str, input) = integer_prefix(input); - if number_str.len() != 2 { - return Err(XsdParseError::msg("Day must be encoded with two digits")); - } - let number = u8::from_str(number_str)?; - if !(1..=31).contains(&number) { - return Err(XsdParseError::msg("Day must be between 01 and 31")); - } - Ok((number, input)) -} - -// [59] hourFrag ::= ([01] digit) | ('2' [0-3]) -// We also allow 24 for ease of parsing -fn hour_frag(input: &str) -> Result<(u8, &str), XsdParseError> { - let (number_str, input) = integer_prefix(input); - if number_str.len() != 2 { - return Err(XsdParseError::msg("Hours must be encoded with two digits")); - } - let number = u8::from_str(number_str)?; - if !(0..=24).contains(&number) { - return Err(XsdParseError::msg("Hours must be between 00 and 24")); - } - Ok((number, input)) -} - -// [60] minuteFrag ::= [0-5] digit -fn minute_frag(input: &str) -> Result<(u8, &str), XsdParseError> { - let (number_str, input) = integer_prefix(input); - if number_str.len() != 2 { - return Err(XsdParseError::msg( - "Minutes must be encoded with two digits", - )); - } - let number = u8::from_str(number_str)?; - if !(0..=59).contains(&number) { - return Err(XsdParseError::msg("Minutes must be between 00 and 59")); - } - Ok((number, input)) -} - -// [61] secondFrag ::= ([0-5] digit) ('.' digit+)? -fn second_frag(input: &str) -> Result<(Decimal, &str), XsdParseError> { - let (number_str, input) = decimal_prefix(input); - let (before_dot_str, _) = number_str.split_once('.').unwrap_or((number_str, "")); - if before_dot_str.len() != 2 { - return Err(XsdParseError::msg( - "Seconds must be encoded with two digits", - )); - } - let number = Decimal::from_str(number_str)?; - if number < Decimal::from(0) || number >= Decimal::from(60) { - return Err(XsdParseError::msg("Seconds must be between 00 and 60")); - } - if number_str.ends_with('.') { - return Err(XsdParseError::msg( - "Seconds are not allowed to end with a dot", - )); - } - Ok((number, input)) -} - -// [63] timezoneFrag ::= 'Z' | ('+' | '-') (('0' digit | '1' [0-3]) ':' minuteFrag | '14:00') -fn timezone_frag(input: &str) -> Result<(TimezoneOffset, &str), XsdParseError> { - if let Some(left) = input.strip_prefix('Z') { - return Ok((TimezoneOffset::UTC, left)); - } - let (sign, input) = if let Some(left) = input.strip_prefix('-') { - (-1, left) - } else if let Some(left) = input.strip_prefix('+') { - (1, left) - } else { - (1, input) - }; - - let (hour_str, input) = integer_prefix(input); - if hour_str.len() != 2 { - return Err(XsdParseError::msg( - "The timezone hours must be encoded with two digits", - )); - } - let hours = i16::from_str(hour_str)?; - - let input = expect_char( - input, - ':', - "The timezone hours and minutes must be separated by ':'", - )?; - let (minutes, input) = minute_frag(input)?; - - if hours > 13 && !(hours == 14 && minutes == 0) { - return Err(XsdParseError::msg( - "The timezone hours must be between 00 and 13", - )); - } - - Ok(( - TimezoneOffset::new(sign * (hours * 60 + i16::from(minutes)))?, - input, - )) -} - -fn ensure_complete( - input: &str, - parse: impl FnOnce(&str) -> Result<(T, &str), XsdParseError>, -) -> Result { - let (result, left) = parse(input)?; - if !left.is_empty() { - return Err(XsdParseError::msg("Unrecognized value suffix")); - } - Ok(result) -} - -fn expect_char<'a>( - input: &'a str, - constant: char, - error_message: &'static str, -) -> Result<&'a str, XsdParseError> { - if let Some(left) = input.strip_prefix(constant) { - Ok(left) - } else { - Err(XsdParseError::msg(error_message)) - } -} - -fn integer_prefix(input: &str) -> (&str, &str) { - let mut end = input.len(); - for (i, c) in input.char_indices() { - if !c.is_ascii_digit() { - end = i; - break; - } - } - input.split_at(end) -} - -fn decimal_prefix(input: &str) -> (&str, &str) { - let mut end = input.len(); - let mut dot_seen = false; - for (i, c) in input.char_indices() { - if c.is_ascii_digit() { - // Ok - } else if c == '.' && !dot_seen { - dot_seen = true; - } else { - end = i; - break; - } - } - input.split_at(end) -} - -fn optional_end( - input: &str, - parse: impl FnOnce(&str) -> Result<(T, &str), XsdParseError>, -) -> Result<(Option, &str), XsdParseError> { - Ok(if input.is_empty() { - (None, input) - } else { - let (result, input) = parse(input)?; - (Some(result), input) - }) -} diff --git a/lib/src/sparql/eval.rs b/lib/src/sparql/eval.rs index 90a52efa..ad50cd2f 100644 --- a/lib/src/sparql/eval.rs +++ b/lib/src/sparql/eval.rs @@ -142,7 +142,7 @@ impl SimpleEvaluator { Self { dataset, base_iri, - now: DateTime::now().unwrap(), + now: DateTime::now(), service_handler, custom_functions, run_stats, @@ -1605,8 +1605,8 @@ impl SimpleEvaluator { stat_children, ); Rc::new(move |tuple| match e(tuple)? { - EncodedTerm::IntegerLiteral(value) => Some(value.abs().into()), - EncodedTerm::DecimalLiteral(value) => Some(value.abs().into()), + EncodedTerm::IntegerLiteral(value) => Some(value.checked_abs()?.into()), + EncodedTerm::DecimalLiteral(value) => Some(value.checked_abs()?.into()), EncodedTerm::FloatLiteral(value) => Some(value.abs().into()), EncodedTerm::DoubleLiteral(value) => Some(value.abs().into()), _ => None, @@ -1620,7 +1620,9 @@ impl SimpleEvaluator { ); Rc::new(move |tuple| match e(tuple)? { EncodedTerm::IntegerLiteral(value) => Some(value.into()), - EncodedTerm::DecimalLiteral(value) => Some(value.ceil().into()), + EncodedTerm::DecimalLiteral(value) => { + Some(value.checked_ceil()?.into()) + } EncodedTerm::FloatLiteral(value) => Some(value.ceil().into()), EncodedTerm::DoubleLiteral(value) => Some(value.ceil().into()), _ => None, @@ -1634,7 +1636,9 @@ impl SimpleEvaluator { ); Rc::new(move |tuple| match e(tuple)? { EncodedTerm::IntegerLiteral(value) => Some(value.into()), - EncodedTerm::DecimalLiteral(value) => Some(value.floor().into()), + EncodedTerm::DecimalLiteral(value) => { + Some(value.checked_floor()?.into()) + } EncodedTerm::FloatLiteral(value) => Some(value.floor().into()), EncodedTerm::DoubleLiteral(value) => Some(value.floor().into()), _ => None, @@ -1648,7 +1652,9 @@ impl SimpleEvaluator { ); Rc::new(move |tuple| match e(tuple)? { EncodedTerm::IntegerLiteral(value) => Some(value.into()), - EncodedTerm::DecimalLiteral(value) => Some(value.round().into()), + EncodedTerm::DecimalLiteral(value) => { + Some(value.checked_round()?.into()) + } EncodedTerm::FloatLiteral(value) => Some(value.round().into()), EncodedTerm::DoubleLiteral(value) => Some(value.round().into()), _ => None, @@ -5851,18 +5857,18 @@ fn format_list(values: impl IntoIterator) -> String { } pub struct Timer { - start: Option, + start: DateTime, } impl Timer { pub fn now() -> Self { Self { - start: DateTime::now().ok(), + start: DateTime::now(), } } pub fn elapsed(&self) -> Option { - DateTime::now().ok()?.checked_sub(self.start?) + DateTime::now().checked_sub(self.start) } } From 922023b1da2dd47091c896d4098a2f049adf9b86 Mon Sep 17 00:00:00 2001 From: Tpt Date: Fri, 21 Jul 2023 18:52:42 +0200 Subject: [PATCH 038/217] Parsers do not need BufRead anymore --- lib/benches/store.rs | 8 ++++---- lib/src/io/read.rs | 22 +++++++++++----------- lib/src/sparql/update.rs | 3 +-- lib/src/store.rs | 14 +++++++------- python/src/io.rs | 30 +++++------------------------- server/src/main.rs | 25 ++++++++----------------- testsuite/src/files.rs | 8 ++++---- testsuite/src/sparql_evaluator.rs | 7 +++++-- 8 files changed, 45 insertions(+), 72 deletions(-) diff --git a/lib/benches/store.rs b/lib/benches/store.rs index 0f35a623..17779b19 100644 --- a/lib/benches/store.rs +++ b/lib/benches/store.rs @@ -92,7 +92,7 @@ fn store_query_and_update(c: &mut Criterion) { .read_to_end(&mut data) .unwrap(); - let operations = read_data("mix-exploreAndUpdate-1000.tsv.zst") + let operations = BufReader::new(read_data("mix-exploreAndUpdate-1000.tsv.zst")) .lines() .map(|l| { let l = l.unwrap(); @@ -167,7 +167,7 @@ fn sparql_parsing(c: &mut Criterion) { .read_to_end(&mut data) .unwrap(); - let operations = read_data("mix-exploreAndUpdate-1000.tsv.zst") + let operations = BufReader::new(read_data("mix-exploreAndUpdate-1000.tsv.zst")) .lines() .map(|l| { let l = l.unwrap(); @@ -213,7 +213,7 @@ criterion_group!(store, sparql_parsing, store_query_and_update, store_load); criterion_main!(store); -fn read_data(file: &str) -> impl BufRead { +fn read_data(file: &str) -> impl Read { if !Path::new(file).exists() { let mut client = oxhttp::Client::new(); client.set_redirection_limit(5); @@ -228,7 +228,7 @@ fn read_data(file: &str) -> impl BufRead { ); std::io::copy(&mut response.into_body(), &mut File::create(file).unwrap()).unwrap(); } - BufReader::new(zstd::Decoder::new(File::open(file).unwrap()).unwrap()) + zstd::Decoder::new(File::open(file).unwrap()).unwrap() } #[derive(Clone)] diff --git a/lib/src/io/read.rs b/lib/src/io/read.rs index 9489a168..3b3cadc4 100644 --- a/lib/src/io/read.rs +++ b/lib/src/io/read.rs @@ -10,7 +10,7 @@ use oxttl::ntriples::{FromReadNTriplesReader, NTriplesParser}; use oxttl::trig::{FromReadTriGReader, TriGParser}; use oxttl::turtle::{FromReadTurtleReader, TurtleParser}; use std::collections::HashMap; -use std::io::BufRead; +use std::io::Read; /// Parsers for RDF graph serialization formats. /// @@ -83,8 +83,8 @@ impl GraphParser { }) } - /// Executes the parsing itself on a [`BufRead`] implementation and returns an iterator of triples. - pub fn read_triples(&self, reader: R) -> TripleReader { + /// Executes the parsing itself on a [`Read`] implementation and returns an iterator of triples. + pub fn read_triples(&self, reader: R) -> TripleReader { TripleReader { mapper: BlankNodeMapper::default(), parser: match &self.inner { @@ -114,19 +114,19 @@ impl GraphParser { /// # std::io::Result::Ok(()) /// ``` #[must_use] -pub struct TripleReader { +pub struct TripleReader { mapper: BlankNodeMapper, parser: TripleReaderKind, } #[allow(clippy::large_enum_variant)] -enum TripleReaderKind { +enum TripleReaderKind { NTriples(FromReadNTriplesReader), Turtle(FromReadTurtleReader), RdfXml(FromReadRdfXmlReader), } -impl Iterator for TripleReader { +impl Iterator for TripleReader { type Item = Result; fn next(&mut self) -> Option> { @@ -214,8 +214,8 @@ impl DatasetParser { }) } - /// Executes the parsing itself on a [`BufRead`] implementation and returns an iterator of quads. - pub fn read_quads(&self, reader: R) -> QuadReader { + /// Executes the parsing itself on a [`Read`] implementation and returns an iterator of quads. + pub fn read_quads(&self, reader: R) -> QuadReader { QuadReader { mapper: BlankNodeMapper::default(), parser: match &self.inner { @@ -242,17 +242,17 @@ impl DatasetParser { /// # std::io::Result::Ok(()) /// ``` #[must_use] -pub struct QuadReader { +pub struct QuadReader { mapper: BlankNodeMapper, parser: QuadReaderKind, } -enum QuadReaderKind { +enum QuadReaderKind { NQuads(FromReadNQuadsReader), TriG(FromReadTriGReader), } -impl Iterator for QuadReader { +impl Iterator for QuadReader { type Item = Result; fn next(&mut self) -> Option> { diff --git a/lib/src/sparql/update.rs b/lib/src/sparql/update.rs index fb68c561..a62f99a8 100644 --- a/lib/src/sparql/update.rs +++ b/lib/src/sparql/update.rs @@ -18,7 +18,6 @@ use spargebra::term::{ use spargebra::GraphUpdateOperation; use sparopt::Optimizer; use std::collections::HashMap; -use std::io::BufReader; use std::rc::Rc; pub fn evaluate_update<'a, 'b: 'a>( @@ -182,7 +181,7 @@ impl<'a, 'b: 'a> SimpleUpdateEvaluator<'a, 'b> { .with_base_iri(base_iri.as_str()) .map_err(|e| ParseError::invalid_base_iri(base_iri, e))?; } - for t in parser.read_triples(BufReader::new(body)) { + for t in parser.read_triples(body) { self.transaction .insert(t?.as_ref().in_graph(to_graph_name))?; } diff --git a/lib/src/store.rs b/lib/src/store.rs index c174bcb1..eed5dce2 100644 --- a/lib/src/store.rs +++ b/lib/src/store.rs @@ -40,7 +40,7 @@ use crate::storage::{ }; pub use crate::storage::{CorruptionError, LoaderError, SerializerError, StorageError}; use std::error::Error; -use std::io::{BufRead, Write}; +use std::io::{Read, Write}; #[cfg(not(target_family = "wasm"))] use std::path::Path; use std::{fmt, str}; @@ -469,7 +469,7 @@ impl Store { /// ``` pub fn load_graph<'a>( &self, - reader: impl BufRead, + reader: impl Read, format: GraphFormat, to_graph_name: impl Into>, base_iri: Option<&str>, @@ -513,7 +513,7 @@ impl Store { /// ``` pub fn load_dataset( &self, - reader: impl BufRead, + reader: impl Read, format: DatasetFormat, base_iri: Option<&str>, ) -> Result<(), LoaderError> { @@ -1077,7 +1077,7 @@ impl<'a> Transaction<'a> { /// ``` pub fn load_graph<'b>( &mut self, - reader: impl BufRead, + reader: impl Read, format: GraphFormat, to_graph_name: impl Into>, base_iri: Option<&str>, @@ -1119,7 +1119,7 @@ impl<'a> Transaction<'a> { /// ``` pub fn load_dataset( &mut self, - reader: impl BufRead, + reader: impl Read, format: DatasetFormat, base_iri: Option<&str>, ) -> Result<(), LoaderError> { @@ -1462,7 +1462,7 @@ impl BulkLoader { /// ``` pub fn load_dataset( &self, - reader: impl BufRead, + reader: impl Read, format: DatasetFormat, base_iri: Option<&str>, ) -> Result<(), LoaderError> { @@ -1517,7 +1517,7 @@ impl BulkLoader { /// ``` pub fn load_graph<'a>( &self, - reader: impl BufRead, + reader: impl Read, format: GraphFormat, to_graph_name: impl Into>, base_iri: Option<&str>, diff --git a/python/src/io.rs b/python/src/io.rs index 382a8c79..d29bfb9d 100644 --- a/python/src/io.rs +++ b/python/src/io.rs @@ -12,7 +12,7 @@ use pyo3::{intern, wrap_pyfunction}; use std::cmp::max; use std::error::Error; use std::fs::File; -use std::io::{self, BufRead, BufReader, BufWriter, Cursor, Read, Write}; +use std::io::{self, BufWriter, Cursor, Read, Write}; pub fn add_to_module(module: &PyModule) -> PyResult<()> { module.add_wrapped(wrap_pyfunction!(parse))?; @@ -193,15 +193,13 @@ impl PyQuadReader { pub enum PyReadable { Bytes(Cursor>), - Io(BufReader), - File(BufReader), + Io(PyIo), + File(File), } impl PyReadable { pub fn from_file(file: &str, py: Python<'_>) -> io::Result { - Ok(Self::File(BufReader::new( - py.allow_threads(|| File::open(file))?, - ))) + Ok(Self::File(py.allow_threads(|| File::open(file))?)) } pub fn from_data(data: PyObject, py: Python<'_>) -> Self { @@ -210,7 +208,7 @@ impl PyReadable { } else if let Ok(string) = data.extract::(py) { Self::Bytes(Cursor::new(string.into_bytes())) } else { - Self::Io(BufReader::new(PyIo(data))) + Self::Io(PyIo(data)) } } } @@ -225,24 +223,6 @@ impl Read for PyReadable { } } -impl BufRead for PyReadable { - fn fill_buf(&mut self) -> io::Result<&[u8]> { - match self { - Self::Bytes(bytes) => bytes.fill_buf(), - Self::Io(io) => io.fill_buf(), - Self::File(file) => file.fill_buf(), - } - } - - fn consume(&mut self, amt: usize) { - match self { - Self::Bytes(bytes) => bytes.consume(amt), - Self::Io(io) => io.consume(amt), - Self::File(file) => file.consume(amt), - } - } -} - pub enum PyWritable { Io(BufWriter), File(BufWriter), diff --git a/server/src/main.rs b/server/src/main.rs index 7474cae4..5db006b9 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -21,7 +21,7 @@ use std::cmp::{max, min}; use std::env; use std::ffi::OsStr; use std::fs::File; -use std::io::{self, stdin, stdout, BufRead, BufReader, BufWriter, Read, Write}; +use std::io::{self, stdin, stdout, BufWriter, Read, Write}; #[cfg(target_os = "linux")] use std::os::unix::net::UnixDatagram; use std::path::{Path, PathBuf}; @@ -377,7 +377,7 @@ pub fn main() -> anyhow::Result<()> { if file.extension().map_or(false, |e| e == OsStr::new("gz")) { bulk_load( &loader, - BufReader::new(MultiGzDecoder::new(fp)), + MultiGzDecoder::new(fp), format.unwrap_or_else(|| { GraphOrDatasetFormat::from_path( &file.with_extension(""), @@ -390,7 +390,7 @@ pub fn main() -> anyhow::Result<()> { } else { bulk_load( &loader, - BufReader::new(fp), + fp, format.unwrap_or_else(|| { GraphOrDatasetFormat::from_path(&file).unwrap() }), @@ -645,7 +645,7 @@ pub fn main() -> anyhow::Result<()> { fn bulk_load( loader: &BulkLoader, - reader: impl BufRead, + reader: impl Read, format: GraphOrDatasetFormat, base_iri: Option<&str>, to_graph_name: Option>, @@ -1531,18 +1531,13 @@ fn web_load_graph( }; if url_query_parameter(request, "no_transaction").is_some() { web_bulk_loader(store, request).load_graph( - BufReader::new(request.body_mut()), + request.body_mut(), format, to_graph_name, base_iri, ) } else { - store.load_graph( - BufReader::new(request.body_mut()), - format, - to_graph_name, - base_iri, - ) + store.load_graph(request.body_mut(), format, to_graph_name, base_iri) } .map_err(loader_to_http_error) } @@ -1553,13 +1548,9 @@ fn web_load_dataset( format: DatasetFormat, ) -> Result<(), HttpError> { if url_query_parameter(request, "no_transaction").is_some() { - web_bulk_loader(store, request).load_dataset( - BufReader::new(request.body_mut()), - format, - None, - ) + web_bulk_loader(store, request).load_dataset(request.body_mut(), format, None) } else { - store.load_dataset(BufReader::new(request.body_mut()), format, None) + store.load_dataset(request.body_mut(), format, None) } .map_err(loader_to_http_error) } diff --git a/testsuite/src/files.rs b/testsuite/src/files.rs index f7101e9d..2d9914c7 100644 --- a/testsuite/src/files.rs +++ b/testsuite/src/files.rs @@ -1,13 +1,13 @@ -use anyhow::{anyhow, bail, Result}; +use anyhow::{anyhow, bail, Context, Result}; use oxigraph::io::{DatasetFormat, DatasetParser, GraphFormat, GraphParser}; use oxigraph::model::{Dataset, Graph}; use oxttl::n3::N3Quad; use oxttl::N3Parser; use std::fs::File; -use std::io::{BufRead, BufReader, Read}; +use std::io::Read; use std::path::PathBuf; -pub fn read_file(url: &str) -> Result { +pub fn read_file(url: &str) -> Result { let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); path.push(if url.starts_with("http://w3c.github.io/") { url.replace("http://w3c.github.io/", "") @@ -25,7 +25,7 @@ pub fn read_file(url: &str) -> Result { } else { bail!("Not supported url for file: {url}") }); - Ok(BufReader::new(File::open(&path)?)) + File::open(&path).with_context(|| format!("Failed to read {}", path.display())) } pub fn read_file_to_string(url: &str) -> Result { diff --git a/testsuite/src/sparql_evaluator.rs b/testsuite/src/sparql_evaluator.rs index da4f0347..ab2b6643 100644 --- a/testsuite/src/sparql_evaluator.rs +++ b/testsuite/src/sparql_evaluator.rs @@ -11,7 +11,7 @@ use oxigraph::store::Store; use sparopt::Optimizer; use std::collections::HashMap; use std::fmt::Write; -use std::io::{self, Cursor}; +use std::io::{self, BufReader, Cursor}; use std::str::FromStr; use std::sync::Arc; @@ -296,7 +296,10 @@ fn load_sparql_query_result(url: &str) -> Result { .rsplit_once('.') .and_then(|(_, extension)| QueryResultsFormat::from_extension(extension)) { - StaticQueryResults::from_query_results(QueryResults::read(read_file(url)?, format)?, false) + StaticQueryResults::from_query_results( + QueryResults::read(BufReader::new(read_file(url)?), format)?, + false, + ) } else { StaticQueryResults::from_graph(&load_graph(url, guess_graph_format(url)?, false)?) } From 3adf33d2f485cf1c0f72c84469e22e52e2d7407c Mon Sep 17 00:00:00 2001 From: Tpt Date: Thu, 3 Aug 2023 18:18:15 +0200 Subject: [PATCH 039/217] Renames "parse_from_read" to "parse_read" --- fuzz/fuzz_targets/nquads.rs | 2 +- fuzz/fuzz_targets/rdf_xml.rs | 4 ++-- fuzz/fuzz_targets/trig.rs | 2 +- lib/oxrdfxml/README.md | 2 +- lib/oxrdfxml/src/parser.rs | 18 +++++++++--------- lib/oxttl/README.md | 2 +- lib/oxttl/src/n3.rs | 22 +++++++++++----------- lib/oxttl/src/nquads.rs | 22 +++++++++++----------- lib/oxttl/src/ntriples.rs | 22 +++++++++++----------- lib/oxttl/src/toolkit/parser.rs | 4 ++-- lib/oxttl/src/trig.rs | 22 +++++++++++----------- lib/oxttl/src/turtle.rs | 22 +++++++++++----------- lib/src/io/read.rs | 12 +++++------- testsuite/src/files.rs | 2 +- 14 files changed, 78 insertions(+), 80 deletions(-) diff --git a/fuzz/fuzz_targets/nquads.rs b/fuzz/fuzz_targets/nquads.rs index 80cf9160..b8b1ac6e 100644 --- a/fuzz/fuzz_targets/nquads.rs +++ b/fuzz/fuzz_targets/nquads.rs @@ -33,7 +33,7 @@ fuzz_target!(|data: &[u8]| { // We parse the serialization let new_quads = NQuadsParser::new() .with_quoted_triples() - .parse_from_read(new_serialization.as_slice()) + .parse_read(new_serialization.as_slice()) .collect::, _>>() .map_err(|e| { format!( diff --git a/fuzz/fuzz_targets/rdf_xml.rs b/fuzz/fuzz_targets/rdf_xml.rs index ae0cb6b1..e1efc7e9 100644 --- a/fuzz/fuzz_targets/rdf_xml.rs +++ b/fuzz/fuzz_targets/rdf_xml.rs @@ -6,7 +6,7 @@ use oxrdfxml::{RdfXmlParser, RdfXmlSerializer}; fuzz_target!(|data: &[u8]| { // We parse let mut triples = Vec::new(); - for triple in RdfXmlParser::new().parse_from_read(data) { + for triple in RdfXmlParser::new().parse_read(data) { if let Ok(triple) = triple { triples.push(triple); } @@ -21,7 +21,7 @@ fuzz_target!(|data: &[u8]| { // We parse the serialization let new_triples = RdfXmlParser::new() - .parse_from_read(new_serialization.as_slice()) + .parse_read(new_serialization.as_slice()) .collect::, _>>() .map_err(|e| { format!( diff --git a/fuzz/fuzz_targets/trig.rs b/fuzz/fuzz_targets/trig.rs index 9c2385df..e6ed06c7 100644 --- a/fuzz/fuzz_targets/trig.rs +++ b/fuzz/fuzz_targets/trig.rs @@ -37,7 +37,7 @@ fuzz_target!(|data: &[u8]| { // We parse the serialization let new_quads = TriGParser::new() .with_quoted_triples() - .parse_from_read(new_serialization.as_slice()) + .parse_read(new_serialization.as_slice()) .collect::, _>>() .map_err(|e| { format!( diff --git a/lib/oxrdfxml/README.md b/lib/oxrdfxml/README.md index 66f9f563..d2c0390d 100644 --- a/lib/oxrdfxml/README.md +++ b/lib/oxrdfxml/README.md @@ -25,7 +25,7 @@ let file = b" let schema_person = NamedNodeRef::new("http://schema.org/Person").unwrap(); let mut count = 0; -for triple in RdfXmlParser::new().parse_from_read(file.as_ref()) { +for triple in RdfXmlParser::new().parse_read(file.as_ref()) { let triple = triple.unwrap(); if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { count += 1; diff --git a/lib/oxrdfxml/src/parser.rs b/lib/oxrdfxml/src/parser.rs index 999becad..ddb11fd3 100644 --- a/lib/oxrdfxml/src/parser.rs +++ b/lib/oxrdfxml/src/parser.rs @@ -40,7 +40,7 @@ use tokio::io::{AsyncRead, BufReader as AsyncBufReader}; /// /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; /// let mut count = 0; -/// for triple in RdfXmlParser::new().parse_from_read(file.as_ref()) { +/// for triple in RdfXmlParser::new().parse_read(file.as_ref()) { /// let triple = triple?; /// if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { /// count += 1; @@ -85,7 +85,7 @@ impl RdfXmlParser { /// /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; /// let mut count = 0; - /// for triple in RdfXmlParser::new().parse_from_read(file.as_ref()) { + /// for triple in RdfXmlParser::new().parse_read(file.as_ref()) { /// let triple = triple?; /// if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { /// count += 1; @@ -94,7 +94,7 @@ impl RdfXmlParser { /// assert_eq!(2, count); /// # Result::<_,Box>::Ok(()) /// ``` - pub fn parse_from_read(&self, read: R) -> FromReadRdfXmlReader { + pub fn parse_read(&self, read: R) -> FromReadRdfXmlReader { FromReadRdfXmlReader { results: Vec::new(), reader: self.parse(BufReader::new(read)), @@ -122,7 +122,7 @@ impl RdfXmlParser { /// /// let schema_person = NamedNodeRef::new_unchecked("http://schema.org/Person"); /// let mut count = 0; - /// let mut parser = RdfXmlParser::new().parse_from_tokio_async_read(file.as_ref()); + /// let mut parser = RdfXmlParser::new().parse_tokio_async_read(file.as_ref()); /// while let Some(triple) = parser.next().await { /// let triple = triple?; /// if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { @@ -134,7 +134,7 @@ impl RdfXmlParser { /// } /// ``` #[cfg(feature = "async-tokio")] - pub fn parse_from_tokio_async_read( + pub fn parse_tokio_async_read( &self, read: R, ) -> FromTokioAsyncReadRdfXmlReader { @@ -161,7 +161,7 @@ impl RdfXmlParser { } } -/// Parses a RDF/XML file from a [`Read`] implementation. Can be built using [`RdfXmlParser::parse_from_read`]. +/// Parses a RDF/XML file from a [`Read`] implementation. Can be built using [`RdfXmlParser::parse_read`]. /// /// Count the number of people: /// ``` @@ -179,7 +179,7 @@ impl RdfXmlParser { /// /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; /// let mut count = 0; -/// for triple in RdfXmlParser::new().parse_from_read(file.as_ref()) { +/// for triple in RdfXmlParser::new().parse_read(file.as_ref()) { /// let triple = triple?; /// if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { /// count += 1; @@ -227,7 +227,7 @@ impl FromReadRdfXmlReader { } } -/// Parses a RDF/XML file from a [`AsyncRead`] implementation. Can be built using [`RdfXmlParser::parse_from_tokio_async_read`]. +/// Parses a RDF/XML file from a [`AsyncRead`] implementation. Can be built using [`RdfXmlParser::parse_tokio_async_read`]. /// /// Count the number of people: /// ``` @@ -247,7 +247,7 @@ impl FromReadRdfXmlReader { /// /// let schema_person = NamedNodeRef::new_unchecked("http://schema.org/Person"); /// let mut count = 0; -/// let mut parser = RdfXmlParser::new().parse_from_tokio_async_read(file.as_ref()); +/// let mut parser = RdfXmlParser::new().parse_tokio_async_read(file.as_ref()); /// while let Some(triple) = parser.next().await { /// let triple = triple?; /// if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { diff --git a/lib/oxttl/README.md b/lib/oxttl/README.md index e99dd2fc..ace0955d 100644 --- a/lib/oxttl/README.md +++ b/lib/oxttl/README.md @@ -27,7 +27,7 @@ let file = b"@base . let schema_person = NamedNodeRef::new("http://schema.org/Person").unwrap(); let mut count = 0; -for triple in TurtleParser::new().parse_from_read(file.as_ref()) { +for triple in TurtleParser::new().parse_read(file.as_ref()) { let triple = triple.unwrap(); if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { count += 1; diff --git a/lib/oxttl/src/n3.rs b/lib/oxttl/src/n3.rs index 106263a2..7167df09 100644 --- a/lib/oxttl/src/n3.rs +++ b/lib/oxttl/src/n3.rs @@ -194,7 +194,7 @@ impl From for N3Quad { /// let rdf_type = N3Term::NamedNode(rdf::TYPE.into_owned()); /// let schema_person = N3Term::NamedNode(NamedNode::new("http://schema.org/Person")?); /// let mut count = 0; -/// for triple in N3Parser::new().parse_from_read(file.as_ref()) { +/// for triple in N3Parser::new().parse_read(file.as_ref()) { /// let triple = triple?; /// if triple.predicate == rdf_type && triple.object == schema_person { /// count += 1; @@ -250,7 +250,7 @@ impl N3Parser { /// let rdf_type = N3Term::NamedNode(NamedNode::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?); /// let schema_person = N3Term::NamedNode(NamedNode::new("http://schema.org/Person")?); /// let mut count = 0; - /// for triple in N3Parser::new().parse_from_read(file.as_ref()) { + /// for triple in N3Parser::new().parse_read(file.as_ref()) { /// let triple = triple?; /// if triple.predicate == rdf_type && triple.object == schema_person { /// count += 1; @@ -259,9 +259,9 @@ impl N3Parser { /// assert_eq!(2, count); /// # Result::<_,Box>::Ok(()) /// ``` - pub fn parse_from_read(&self, read: R) -> FromReadN3Reader { + pub fn parse_read(&self, read: R) -> FromReadN3Reader { FromReadN3Reader { - inner: self.parse().parser.parse_from_read(read), + inner: self.parse().parser.parse_read(read), } } @@ -285,7 +285,7 @@ impl N3Parser { /// let rdf_type = N3Term::NamedNode(rdf::TYPE.into_owned()); /// let schema_person = N3Term::NamedNode(NamedNode::new_unchecked("http://schema.org/Person")); /// let mut count = 0; - /// let mut parser = N3Parser::new().parse_from_tokio_async_read(file.as_ref()); + /// let mut parser = N3Parser::new().parse_tokio_async_read(file.as_ref()); /// while let Some(triple) = parser.next().await { /// let triple = triple?; /// if triple.predicate == rdf_type && triple.object == schema_person { @@ -297,12 +297,12 @@ impl N3Parser { /// } /// ``` #[cfg(feature = "async-tokio")] - pub fn parse_from_tokio_async_read( + pub fn parse_tokio_async_read( &self, read: R, ) -> FromTokioAsyncReadN3Reader { FromTokioAsyncReadN3Reader { - inner: self.parse().parser.parse_from_tokio_async_read(read), + inner: self.parse().parser.parse_tokio_async_read(read), } } @@ -350,7 +350,7 @@ impl N3Parser { } } -/// Parses a N3 file from a [`Read`] implementation. Can be built using [`N3Parser::parse_from_read`]. +/// Parses a N3 file from a [`Read`] implementation. Can be built using [`N3Parser::parse_read`]. /// /// Count the number of people: /// ``` @@ -367,7 +367,7 @@ impl N3Parser { /// let rdf_type = N3Term::NamedNode(rdf::TYPE.into_owned()); /// let schema_person = N3Term::NamedNode(NamedNode::new("http://schema.org/Person")?); /// let mut count = 0; -/// for triple in N3Parser::new().parse_from_read(file.as_ref()) { +/// for triple in N3Parser::new().parse_read(file.as_ref()) { /// let triple = triple?; /// if triple.predicate == rdf_type && triple.object == schema_person { /// count += 1; @@ -388,7 +388,7 @@ impl Iterator for FromReadN3Reader { } } -/// Parses a N3 file from a [`AsyncRead`] implementation. Can be built using [`N3Parser::parse_from_tokio_async_read`]. +/// Parses a N3 file from a [`AsyncRead`] implementation. Can be built using [`N3Parser::parse_tokio_async_read`]. /// /// Count the number of people: /// ``` @@ -408,7 +408,7 @@ impl Iterator for FromReadN3Reader { /// let rdf_type = N3Term::NamedNode(rdf::TYPE.into_owned()); /// let schema_person = N3Term::NamedNode(NamedNode::new_unchecked("http://schema.org/Person")); /// let mut count = 0; -/// let mut parser = N3Parser::new().parse_from_tokio_async_read(file.as_ref()); +/// let mut parser = N3Parser::new().parse_tokio_async_read(file.as_ref()); /// while let Some(triple) = parser.next().await { /// let triple = triple?; /// if triple.predicate == rdf_type && triple.object == schema_person { diff --git a/lib/oxttl/src/nquads.rs b/lib/oxttl/src/nquads.rs index 5cd27dae..140f25a7 100644 --- a/lib/oxttl/src/nquads.rs +++ b/lib/oxttl/src/nquads.rs @@ -25,7 +25,7 @@ use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; /// /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; /// let mut count = 0; -/// for quad in NQuadsParser::new().parse_from_read(file.as_ref()) { +/// for quad in NQuadsParser::new().parse_read(file.as_ref()) { /// let quad = quad?; /// if quad.predicate == rdf::TYPE && quad.object == schema_person.into() { /// count += 1; @@ -70,7 +70,7 @@ impl NQuadsParser { /// /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; /// let mut count = 0; - /// for quad in NQuadsParser::new().parse_from_read(file.as_ref()) { + /// for quad in NQuadsParser::new().parse_read(file.as_ref()) { /// let quad = quad?; /// if quad.predicate == rdf::TYPE && quad.object == schema_person.into() { /// count += 1; @@ -79,9 +79,9 @@ impl NQuadsParser { /// assert_eq!(2, count); /// # Result::<_,Box>::Ok(()) /// ``` - pub fn parse_from_read(&self, read: R) -> FromReadNQuadsReader { + pub fn parse_read(&self, read: R) -> FromReadNQuadsReader { FromReadNQuadsReader { - inner: self.parse().parser.parse_from_read(read), + inner: self.parse().parser.parse_read(read), } } @@ -101,7 +101,7 @@ impl NQuadsParser { /// /// let schema_person = NamedNodeRef::new_unchecked("http://schema.org/Person"); /// let mut count = 0; - /// let mut parser = NQuadsParser::new().parse_from_tokio_async_read(file.as_ref()); + /// let mut parser = NQuadsParser::new().parse_tokio_async_read(file.as_ref()); /// while let Some(triple) = parser.next().await { /// let triple = triple?; /// if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { @@ -113,12 +113,12 @@ impl NQuadsParser { /// } /// ``` #[cfg(feature = "async-tokio")] - pub fn parse_from_tokio_async_read( + pub fn parse_tokio_async_read( &self, read: R, ) -> FromTokioAsyncReadNQuadsReader { FromTokioAsyncReadNQuadsReader { - inner: self.parse().parser.parse_from_tokio_async_read(read), + inner: self.parse().parser.parse_tokio_async_read(read), } } @@ -170,7 +170,7 @@ impl NQuadsParser { } } -/// Parses a N-Quads file from a [`Read`] implementation. Can be built using [`NQuadsParser::parse_from_read`]. +/// Parses a N-Quads file from a [`Read`] implementation. Can be built using [`NQuadsParser::parse_read`]. /// /// Count the number of people: /// ``` @@ -184,7 +184,7 @@ impl NQuadsParser { /// /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; /// let mut count = 0; -/// for quad in NQuadsParser::new().parse_from_read(file.as_ref()) { +/// for quad in NQuadsParser::new().parse_read(file.as_ref()) { /// let quad = quad?; /// if quad.predicate == rdf::TYPE && quad.object == schema_person.into() { /// count += 1; @@ -205,7 +205,7 @@ impl Iterator for FromReadNQuadsReader { } } -/// Parses a N-Quads file from a [`AsyncRead`] implementation. Can be built using [`NQuadsParser::parse_from_tokio_async_read`]. +/// Parses a N-Quads file from a [`AsyncRead`] implementation. Can be built using [`NQuadsParser::parse_tokio_async_read`]. /// /// Count the number of people: /// ``` @@ -221,7 +221,7 @@ impl Iterator for FromReadNQuadsReader { /// /// let schema_person = NamedNodeRef::new_unchecked("http://schema.org/Person"); /// let mut count = 0; -/// let mut parser = NQuadsParser::new().parse_from_tokio_async_read(file.as_ref()); +/// let mut parser = NQuadsParser::new().parse_tokio_async_read(file.as_ref()); /// while let Some(triple) = parser.next().await { /// let triple = triple?; /// if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { diff --git a/lib/oxttl/src/ntriples.rs b/lib/oxttl/src/ntriples.rs index 2278db81..3014c5d8 100644 --- a/lib/oxttl/src/ntriples.rs +++ b/lib/oxttl/src/ntriples.rs @@ -26,7 +26,7 @@ use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; /// /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; /// let mut count = 0; -/// for triple in NTriplesParser::new().parse_from_read(file.as_ref()) { +/// for triple in NTriplesParser::new().parse_read(file.as_ref()) { /// let triple = triple?; /// if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { /// count += 1; @@ -71,7 +71,7 @@ impl NTriplesParser { /// /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; /// let mut count = 0; - /// for triple in NTriplesParser::new().parse_from_read(file.as_ref()) { + /// for triple in NTriplesParser::new().parse_read(file.as_ref()) { /// let triple = triple?; /// if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { /// count += 1; @@ -80,9 +80,9 @@ impl NTriplesParser { /// assert_eq!(2, count); /// # Result::<_,Box>::Ok(()) /// ``` - pub fn parse_from_read(&self, read: R) -> FromReadNTriplesReader { + pub fn parse_read(&self, read: R) -> FromReadNTriplesReader { FromReadNTriplesReader { - inner: self.parse().parser.parse_from_read(read), + inner: self.parse().parser.parse_read(read), } } @@ -102,7 +102,7 @@ impl NTriplesParser { /// /// let schema_person = NamedNodeRef::new_unchecked("http://schema.org/Person"); /// let mut count = 0; - /// let mut parser = NTriplesParser::new().parse_from_tokio_async_read(file.as_ref()); + /// let mut parser = NTriplesParser::new().parse_tokio_async_read(file.as_ref()); /// while let Some(triple) = parser.next().await { /// let triple = triple?; /// if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { @@ -114,12 +114,12 @@ impl NTriplesParser { /// } /// ``` #[cfg(feature = "async-tokio")] - pub fn parse_from_tokio_async_read( + pub fn parse_tokio_async_read( &self, read: R, ) -> FromTokioAsyncReadNTriplesReader { FromTokioAsyncReadNTriplesReader { - inner: self.parse().parser.parse_from_tokio_async_read(read), + inner: self.parse().parser.parse_tokio_async_read(read), } } @@ -171,7 +171,7 @@ impl NTriplesParser { } } -/// Parses a N-Triples file from a [`Read`] implementation. Can be built using [`NTriplesParser::parse_from_read`]. +/// Parses a N-Triples file from a [`Read`] implementation. Can be built using [`NTriplesParser::parse_read`]. /// /// Count the number of people: /// ``` @@ -185,7 +185,7 @@ impl NTriplesParser { /// /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; /// let mut count = 0; -/// for triple in NTriplesParser::new().parse_from_read(file.as_ref()) { +/// for triple in NTriplesParser::new().parse_read(file.as_ref()) { /// let triple = triple?; /// if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { /// count += 1; @@ -206,7 +206,7 @@ impl Iterator for FromReadNTriplesReader { } } -/// Parses a N-Triples file from a [`AsyncRead`] implementation. Can be built using [`NTriplesParser::parse_from_tokio_async_read`]. +/// Parses a N-Triples file from a [`AsyncRead`] implementation. Can be built using [`NTriplesParser::parse_tokio_async_read`]. /// /// Count the number of people: /// ``` @@ -222,7 +222,7 @@ impl Iterator for FromReadNTriplesReader { /// /// let schema_person = NamedNodeRef::new_unchecked("http://schema.org/Person"); /// let mut count = 0; -/// let mut parser = NTriplesParser::new().parse_from_tokio_async_read(file.as_ref()); +/// let mut parser = NTriplesParser::new().parse_tokio_async_read(file.as_ref()); /// while let Some(triple) = parser.next().await { /// let triple = triple?; /// if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { diff --git a/lib/oxttl/src/toolkit/parser.rs b/lib/oxttl/src/toolkit/parser.rs index c5808199..38419477 100644 --- a/lib/oxttl/src/toolkit/parser.rs +++ b/lib/oxttl/src/toolkit/parser.rs @@ -113,12 +113,12 @@ impl Parser { } } - pub fn parse_from_read(self, read: R) -> FromReadIterator { + pub fn parse_read(self, read: R) -> FromReadIterator { FromReadIterator { read, parser: self } } #[cfg(feature = "async-tokio")] - pub fn parse_from_tokio_async_read( + pub fn parse_tokio_async_read( self, read: R, ) -> FromTokioAsyncReadIterator { diff --git a/lib/oxttl/src/trig.rs b/lib/oxttl/src/trig.rs index 5ad21402..146e56b4 100644 --- a/lib/oxttl/src/trig.rs +++ b/lib/oxttl/src/trig.rs @@ -30,7 +30,7 @@ use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; /// /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; /// let mut count = 0; -/// for quad in TriGParser::new().parse_from_read(file.as_ref()) { +/// for quad in TriGParser::new().parse_read(file.as_ref()) { /// let quad = quad?; /// if quad.predicate == rdf::TYPE && quad.object == schema_person.into() { /// count += 1; @@ -96,7 +96,7 @@ impl TriGParser { /// /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; /// let mut count = 0; - /// for quad in TriGParser::new().parse_from_read(file.as_ref()) { + /// for quad in TriGParser::new().parse_read(file.as_ref()) { /// let quad = quad?; /// if quad.predicate == rdf::TYPE && quad.object == schema_person.into() { /// count += 1; @@ -105,9 +105,9 @@ impl TriGParser { /// assert_eq!(2, count); /// # Result::<_,Box>::Ok(()) /// ``` - pub fn parse_from_read(&self, read: R) -> FromReadTriGReader { + pub fn parse_read(&self, read: R) -> FromReadTriGReader { FromReadTriGReader { - inner: self.parse().parser.parse_from_read(read), + inner: self.parse().parser.parse_read(read), } } @@ -129,7 +129,7 @@ impl TriGParser { /// /// let schema_person = NamedNodeRef::new_unchecked("http://schema.org/Person"); /// let mut count = 0; - /// let mut parser = TriGParser::new().parse_from_tokio_async_read(file.as_ref()); + /// let mut parser = TriGParser::new().parse_tokio_async_read(file.as_ref()); /// while let Some(triple) = parser.next().await { /// let triple = triple?; /// if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { @@ -141,12 +141,12 @@ impl TriGParser { /// } /// ``` #[cfg(feature = "async-tokio")] - pub fn parse_from_tokio_async_read( + pub fn parse_tokio_async_read( &self, read: R, ) -> FromTokioAsyncReadTriGReader { FromTokioAsyncReadTriGReader { - inner: self.parse().parser.parse_from_tokio_async_read(read), + inner: self.parse().parser.parse_tokio_async_read(read), } } @@ -199,7 +199,7 @@ impl TriGParser { } } -/// Parses a TriG file from a [`Read`] implementation. Can be built using [`TriGParser::parse_from_read`]. +/// Parses a TriG file from a [`Read`] implementation. Can be built using [`TriGParser::parse_read`]. /// /// Count the number of people: /// ``` @@ -215,7 +215,7 @@ impl TriGParser { /// /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; /// let mut count = 0; -/// for quad in TriGParser::new().parse_from_read(file.as_ref()) { +/// for quad in TriGParser::new().parse_read(file.as_ref()) { /// let quad = quad?; /// if quad.predicate == rdf::TYPE && quad.object == schema_person.into() { /// count += 1; @@ -236,7 +236,7 @@ impl Iterator for FromReadTriGReader { } } -/// Parses a TriG file from a [`AsyncRead`] implementation. Can be built using [`TriGParser::parse_from_tokio_async_read`]. +/// Parses a TriG file from a [`AsyncRead`] implementation. Can be built using [`TriGParser::parse_tokio_async_read`]. /// /// Count the number of people: /// ``` @@ -254,7 +254,7 @@ impl Iterator for FromReadTriGReader { /// /// let schema_person = NamedNodeRef::new_unchecked("http://schema.org/Person"); /// let mut count = 0; -/// let mut parser = TriGParser::new().parse_from_tokio_async_read(file.as_ref()); +/// let mut parser = TriGParser::new().parse_tokio_async_read(file.as_ref()); /// while let Some(triple) = parser.next().await { /// let triple = triple?; /// if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { diff --git a/lib/oxttl/src/turtle.rs b/lib/oxttl/src/turtle.rs index 0c875b8a..b5a1b0cd 100644 --- a/lib/oxttl/src/turtle.rs +++ b/lib/oxttl/src/turtle.rs @@ -32,7 +32,7 @@ use tokio::io::{AsyncRead, AsyncWrite}; /// /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; /// let mut count = 0; -/// for triple in TurtleParser::new().parse_from_read(file.as_ref()) { +/// for triple in TurtleParser::new().parse_read(file.as_ref()) { /// let triple = triple?; /// if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { /// count += 1; @@ -98,7 +98,7 @@ impl TurtleParser { /// /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; /// let mut count = 0; - /// for triple in TurtleParser::new().parse_from_read(file.as_ref()) { + /// for triple in TurtleParser::new().parse_read(file.as_ref()) { /// let triple = triple?; /// if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { /// count += 1; @@ -107,9 +107,9 @@ impl TurtleParser { /// assert_eq!(2, count); /// # Result::<_,Box>::Ok(()) /// ``` - pub fn parse_from_read(&self, read: R) -> FromReadTurtleReader { + pub fn parse_read(&self, read: R) -> FromReadTurtleReader { FromReadTurtleReader { - inner: self.parse().parser.parse_from_read(read), + inner: self.parse().parser.parse_read(read), } } @@ -131,7 +131,7 @@ impl TurtleParser { /// /// let schema_person = NamedNodeRef::new_unchecked("http://schema.org/Person"); /// let mut count = 0; - /// let mut parser = TurtleParser::new().parse_from_tokio_async_read(file.as_ref()); + /// let mut parser = TurtleParser::new().parse_tokio_async_read(file.as_ref()); /// while let Some(triple) = parser.next().await { /// let triple = triple?; /// if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { @@ -143,12 +143,12 @@ impl TurtleParser { /// } /// ``` #[cfg(feature = "async-tokio")] - pub fn parse_from_tokio_async_read( + pub fn parse_tokio_async_read( &self, read: R, ) -> FromTokioAsyncReadTurtleReader { FromTokioAsyncReadTurtleReader { - inner: self.parse().parser.parse_from_tokio_async_read(read), + inner: self.parse().parser.parse_tokio_async_read(read), } } @@ -201,7 +201,7 @@ impl TurtleParser { } } -/// Parses a Turtle file from a [`Read`] implementation. Can be built using [`TurtleParser::parse_from_read`]. +/// Parses a Turtle file from a [`Read`] implementation. Can be built using [`TurtleParser::parse_read`]. /// /// Count the number of people: /// ``` @@ -217,7 +217,7 @@ impl TurtleParser { /// /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; /// let mut count = 0; -/// for triple in TurtleParser::new().parse_from_read(file.as_ref()) { +/// for triple in TurtleParser::new().parse_read(file.as_ref()) { /// let triple = triple?; /// if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { /// count += 1; @@ -238,7 +238,7 @@ impl Iterator for FromReadTurtleReader { } } -/// Parses a Turtle file from a [`AsyncRead`] implementation. Can be built using [`TurtleParser::parse_from_tokio_async_read`]. +/// Parses a Turtle file from a [`AsyncRead`] implementation. Can be built using [`TurtleParser::parse_tokio_async_read`]. /// /// Count the number of people: /// ``` @@ -256,7 +256,7 @@ impl Iterator for FromReadTurtleReader { /// /// let schema_person = NamedNodeRef::new_unchecked("http://schema.org/Person"); /// let mut count = 0; -/// let mut parser = TurtleParser::new().parse_from_tokio_async_read(file.as_ref()); +/// let mut parser = TurtleParser::new().parse_tokio_async_read(file.as_ref()); /// while let Some(triple) = parser.next().await { /// let triple = triple?; /// if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { diff --git a/lib/src/io/read.rs b/lib/src/io/read.rs index 3b3cadc4..eddaabed 100644 --- a/lib/src/io/read.rs +++ b/lib/src/io/read.rs @@ -88,11 +88,9 @@ impl GraphParser { TripleReader { mapper: BlankNodeMapper::default(), parser: match &self.inner { - GraphParserKind::NTriples(p) => { - TripleReaderKind::NTriples(p.parse_from_read(reader)) - } - GraphParserKind::Turtle(p) => TripleReaderKind::Turtle(p.parse_from_read(reader)), - GraphParserKind::RdfXml(p) => TripleReaderKind::RdfXml(p.parse_from_read(reader)), + GraphParserKind::NTriples(p) => TripleReaderKind::NTriples(p.parse_read(reader)), + GraphParserKind::Turtle(p) => TripleReaderKind::Turtle(p.parse_read(reader)), + GraphParserKind::RdfXml(p) => TripleReaderKind::RdfXml(p.parse_read(reader)), }, } } @@ -219,8 +217,8 @@ impl DatasetParser { QuadReader { mapper: BlankNodeMapper::default(), parser: match &self.inner { - DatasetParserKind::NQuads(p) => QuadReaderKind::NQuads(p.parse_from_read(reader)), - DatasetParserKind::TriG(p) => QuadReaderKind::TriG(p.parse_from_read(reader)), + DatasetParserKind::NQuads(p) => QuadReaderKind::NQuads(p.parse_read(reader)), + DatasetParserKind::TriG(p) => QuadReaderKind::TriG(p.parse_read(reader)), }, } } diff --git a/testsuite/src/files.rs b/testsuite/src/files.rs index 2d9914c7..9a9dc839 100644 --- a/testsuite/src/files.rs +++ b/testsuite/src/files.rs @@ -107,7 +107,7 @@ pub fn load_n3(url: &str, ignore_errors: bool) -> Result> { for q in N3Parser::new() .with_base_iri(url)? .with_prefix("", format!("{url}#"))? - .parse_from_read(read_file(url)?) + .parse_read(read_file(url)?) { match q { Ok(q) => quads.push(q), From 4f7445104ae144fb6e599fd97bb4fe4feefb754c Mon Sep 17 00:00:00 2001 From: Tpt Date: Sun, 30 Jul 2023 17:17:52 +0200 Subject: [PATCH 040/217] Testsuite: Upgrades to new RDF syntax test structure --- testsuite/benches/parser.rs | 4 +- testsuite/rdf-star | 2 +- testsuite/rdf-tests | 2 +- testsuite/src/files.rs | 6 +-- testsuite/tests/canonicalization.rs | 2 +- testsuite/tests/parser.rs | 71 +++++++++++++++++------------ 6 files changed, 49 insertions(+), 38 deletions(-) diff --git a/testsuite/benches/parser.rs b/testsuite/benches/parser.rs index ed7205eb..c14fb921 100644 --- a/testsuite/benches/parser.rs +++ b/testsuite/benches/parser.rs @@ -23,14 +23,14 @@ fn test_data_from_testsuite(manifest_uri: String, include_tests_types: &[&str]) fn ntriples_test_data() -> Result> { test_data_from_testsuite( - "http://w3c.github.io/rdf-tests/ntriples/manifest.ttl".to_owned(), + "https://w3c.github.io/rdf-tests/rdf/rdf11/rdf-n-triples/manifest.ttl".to_owned(), &["http://www.w3.org/ns/rdftest#TestNTriplesPositiveSyntax"], ) } fn turtle_test_data() -> Result> { test_data_from_testsuite( - "http://w3c.github.io/rdf-tests/turtle/manifest.ttl".to_owned(), + "https://w3c.github.io/rdf-tests/rdf/rdf11/rdf-turtle/manifest.ttl".to_owned(), &[ "http://www.w3.org/ns/rdftest#TestTurtlePositiveSyntax", "http://www.w3.org/ns/rdftest#TestTurtleEval", diff --git a/testsuite/rdf-star b/testsuite/rdf-star index 8c7039dd..59eb0293 160000 --- a/testsuite/rdf-star +++ b/testsuite/rdf-star @@ -1 +1 @@ -Subproject commit 8c7039dd152915a48f266c3d9a10c75adeb24181 +Subproject commit 59eb0293b2b620023ee38252228c725769ad206f diff --git a/testsuite/rdf-tests b/testsuite/rdf-tests index fb410326..aaa24e47 160000 --- a/testsuite/rdf-tests +++ b/testsuite/rdf-tests @@ -1 +1 @@ -Subproject commit fb41032699d22b9991bba2e427b805b3b82648b6 +Subproject commit aaa24e4729a89bdee004bc3042814159eb689a19 diff --git a/testsuite/src/files.rs b/testsuite/src/files.rs index 9a9dc839..7e4f3b17 100644 --- a/testsuite/src/files.rs +++ b/testsuite/src/files.rs @@ -9,12 +9,8 @@ use std::path::PathBuf; pub fn read_file(url: &str) -> Result { let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - path.push(if url.starts_with("http://w3c.github.io/") { - url.replace("http://w3c.github.io/", "") - } else if url.starts_with("https://w3c.github.io/") { + path.push(if url.starts_with("https://w3c.github.io/") { url.replace("https://w3c.github.io/", "") - } else if url.starts_with("http://www.w3.org/2013/RDFXMLTests/") { - url.replace("http://www.w3.org/2013/RDFXMLTests/", "rdf-tests/rdf-xml/") } else if url.starts_with("https://github.com/oxigraph/oxigraph/tests/") { url.replace( "https://github.com/oxigraph/oxigraph/tests/", diff --git a/testsuite/tests/canonicalization.rs b/testsuite/tests/canonicalization.rs index 115e5c5a..c6e5f0e2 100644 --- a/testsuite/tests/canonicalization.rs +++ b/testsuite/tests/canonicalization.rs @@ -3,5 +3,5 @@ use oxigraph_testsuite::check_testsuite; #[test] fn rdf_canon_w3c_testsuite() -> Result<()> { - check_testsuite("http://w3c.github.io/rdf-canon/tests/manifest.ttl", &[]) + check_testsuite("https://w3c.github.io/rdf-canon/tests/manifest.ttl", &[]) } diff --git a/testsuite/tests/parser.rs b/testsuite/tests/parser.rs index e9458256..2846b8f0 100644 --- a/testsuite/tests/parser.rs +++ b/testsuite/tests/parser.rs @@ -2,92 +2,107 @@ use anyhow::Result; use oxigraph_testsuite::check_testsuite; #[test] -fn ntriples_w3c_testsuite() -> Result<()> { - check_testsuite("http://w3c.github.io/rdf-tests/ntriples/manifest.ttl", &[]) +fn rdf11_n_triples_w3c_testsuite() -> Result<()> { + check_testsuite( + "https://w3c.github.io/rdf-tests/rdf/rdf11/rdf-n-triples/manifest.ttl", + &[], + ) } #[test] -fn nquads_w3c_testsuite() -> Result<()> { - check_testsuite("http://w3c.github.io/rdf-tests/nquads/manifest.ttl", &[]) +fn rdf12_n_triples_syntax_w3c_testsuite() -> Result<()> { + check_testsuite( + "https://w3c.github.io/rdf-tests/rdf/rdf12/rdf-n-triples/syntax/manifest.ttl", + &[], + ) } -#[cfg(not(windows))] // Tests don't like git auto "\r\n" on Windows #[test] -fn turtle_w3c_testsuite() -> Result<()> { - check_testsuite("http://w3c.github.io/rdf-tests/turtle/manifest.ttl", &[]) +fn rdf11_n_quads_w3c_testsuite() -> Result<()> { + check_testsuite( + "https://w3c.github.io/rdf-tests/rdf/rdf11/rdf-n-quads/manifest.ttl", + &[], + ) } #[cfg(not(windows))] // Tests don't like git auto "\r\n" on Windows #[test] -fn trig_w3c_testsuite() -> Result<()> { - check_testsuite("http://w3c.github.io/rdf-tests/trig/manifest.ttl", &[]) +fn rdf11_turtle_w3c_testsuite() -> Result<()> { + check_testsuite( + "https://w3c.github.io/rdf-tests/rdf/rdf11/rdf-turtle/manifest.ttl", + &[], + ) } #[test] -fn n3_parser_testsuite() -> Result<()> { +fn rdf12_turtle_syntax_w3c_testsuite() -> Result<()> { check_testsuite( - "https://w3c.github.io/N3/tests/N3Tests/manifest-parser.ttl", + "https://w3c.github.io/rdf-tests/rdf/rdf12/rdf-turtle/syntax/manifest.ttl", &[], ) } + #[test] -fn n3_extended_testsuite() -> Result<()> { +fn rdf12_turtle_eval_w3c_testsuite() -> Result<()> { check_testsuite( - "https://w3c.github.io/N3/tests/N3Tests/manifest-extended.ttl", + "https://w3c.github.io/rdf-tests/rdf/rdf12/rdf-turtle/eval/manifest.ttl", &[], ) } #[cfg(not(windows))] // Tests don't like git auto "\r\n" on Windows #[test] -fn n3_turtle_testsuite() -> Result<()> { +fn rdf11_trig_w3c_testsuite() -> Result<()> { check_testsuite( - "https://w3c.github.io/N3/tests/TurtleTests/manifest.ttl", + "https://w3c.github.io/rdf-tests/rdf/rdf11/rdf-trig/manifest.ttl", &[], ) } #[test] -fn rdf_xml_w3c_testsuite() -> Result<()> { - check_testsuite("http://www.w3.org/2013/RDFXMLTests/manifest.ttl", &[]) +fn rdf12_trig_syntax_w3c_testsuite() -> Result<()> { + check_testsuite( + "https://w3c.github.io/rdf-tests/rdf/rdf12/rdf-trig/syntax/manifest.ttl", + &[], + ) } #[test] -fn ntriples_star_w3c_testsuite() -> Result<()> { +fn rdf12_trig_eval_w3c_testsuite() -> Result<()> { check_testsuite( - "https://w3c.github.io/rdf-star/tests/nt/syntax/manifest.ttl", + "https://w3c.github.io/rdf-tests/rdf/rdf12/rdf-trig/eval/manifest.ttl", &[], ) } #[test] -fn turtle_star_syntax_w3c_testsuite() -> Result<()> { +fn rdf11_xml_w3c_testsuite() -> Result<()> { check_testsuite( - "https://w3c.github.io/rdf-star/tests/turtle/syntax/manifest.ttl", + "https://w3c.github.io/rdf-tests/rdf/rdf11/rdf-xml/manifest.ttl", &[], ) } #[test] -fn turtle_star_eval_w3c_testsuite() -> Result<()> { +fn n3_parser_testsuite() -> Result<()> { check_testsuite( - "https://w3c.github.io/rdf-star/tests/turtle/eval/manifest.ttl", + "https://w3c.github.io/N3/tests/N3Tests/manifest-parser.ttl", &[], ) } - #[test] -fn trig_star_syntax_w3c_testsuite() -> Result<()> { +fn n3_extended_testsuite() -> Result<()> { check_testsuite( - "https://w3c.github.io/rdf-star/tests/trig/syntax/manifest.ttl", + "https://w3c.github.io/N3/tests/N3Tests/manifest-extended.ttl", &[], ) } +#[cfg(not(windows))] // Tests don't like git auto "\r\n" on Windows #[test] -fn trig_star_eval_w3c_testsuite() -> Result<()> { +fn n3_turtle_testsuite() -> Result<()> { check_testsuite( - "https://w3c.github.io/rdf-star/tests/trig/eval/manifest.ttl", + "https://w3c.github.io/N3/tests/TurtleTests/manifest.ttl", &[], ) } From afaabf6110c10121613e89daf10ab7bdb2c4242c Mon Sep 17 00:00:00 2001 From: Tpt Date: Fri, 4 Aug 2023 18:22:49 +0200 Subject: [PATCH 041/217] Simplifies OxTTL lexer buffer management --- lib/oxttl/src/toolkit/lexer.rs | 83 ++++++++++++++++------------------ 1 file changed, 38 insertions(+), 45 deletions(-) diff --git a/lib/oxttl/src/toolkit/lexer.rs b/lib/oxttl/src/toolkit/lexer.rs index 664e35fe..5b980f94 100644 --- a/lib/oxttl/src/toolkit/lexer.rs +++ b/lib/oxttl/src/toolkit/lexer.rs @@ -1,4 +1,5 @@ use memchr::memchr2; +use std::cmp::min; use std::error::Error; use std::fmt; use std::io::{self, Read}; @@ -56,7 +57,6 @@ pub struct Lexer { parser: R, data: Vec, start: usize, - end: usize, is_ending: bool, position: usize, min_buffer_size: usize, @@ -77,7 +77,6 @@ impl Lexer { parser, data: Vec::new(), start: 0, - end: 0, is_ending: false, position: 0, min_buffer_size, @@ -88,10 +87,8 @@ impl Lexer { } pub fn extend_from_slice(&mut self, other: &[u8]) { - self.shrink_if_useful(); - self.data.truncate(self.end); + self.shrink_data(); self.data.extend_from_slice(other); - self.end = self.data.len(); } #[inline] @@ -100,26 +97,25 @@ impl Lexer { } pub fn extend_from_read(&mut self, read: &mut impl Read) -> io::Result<()> { - self.shrink_if_useful(); - let min_end = self.end + self.min_buffer_size; - if min_end > self.max_buffer_size { + self.shrink_data(); + if self.data.len() == self.max_buffer_size { return Err(io::Error::new( io::ErrorKind::OutOfMemory, format!( - "The buffer maximal size is {} < {min_end}", + "Reached the buffer maximal size of {}", self.max_buffer_size ), )); } - if self.data.len() < min_end { - self.data.resize(min_end, 0); - } + let min_end = min(self.data.len() + self.min_buffer_size, self.max_buffer_size); + let new_start = self.data.len(); + self.data.resize(min_end, 0); if self.data.len() < self.data.capacity() { // We keep extending to have as much space as available without reallocation self.data.resize(self.data.capacity(), 0); } - let read = read.read(&mut self.data[self.end..])?; - self.end += read; + let read = read.read(&mut self.data[new_start..])?; + self.data.truncate(new_start + read); self.is_ending = read == 0; Ok(()) } @@ -129,26 +125,25 @@ impl Lexer { &mut self, read: &mut (impl AsyncRead + Unpin), ) -> io::Result<()> { - self.shrink_if_useful(); - let min_end = self.end + self.min_buffer_size; - if min_end > self.max_buffer_size { + self.shrink_data(); + if self.data.len() == self.max_buffer_size { return Err(io::Error::new( io::ErrorKind::OutOfMemory, format!( - "The buffer maximal size is {} < {min_end}", + "Reached the buffer maximal size of {}", self.max_buffer_size ), )); } - if self.data.len() < min_end { - self.data.resize(min_end, 0); - } + let min_end = min(self.data.len() + self.min_buffer_size, self.max_buffer_size); + let new_start = self.data.len(); + self.data.resize(min_end, 0); if self.data.len() < self.data.capacity() { // We keep extending to have as much space as available without reallocation self.data.resize(self.data.capacity(), 0); } - let read = read.read(&mut self.data[self.end..]).await?; - self.end += read; + let read = read.read(&mut self.data[new_start..]).await?; + self.data.truncate(new_start + read); self.is_ending = read == 0; Ok(()) } @@ -158,22 +153,21 @@ impl Lexer { options: &R::Options, ) -> Option>, LexerError>> { self.skip_whitespaces_and_comments()?; - let (consumed, result) = if let Some(r) = self.parser.recognize_next_token( - &self.data[self.start..self.end], - self.is_ending, - options, - ) { + let (consumed, result) = if let Some(r) = + self.parser + .recognize_next_token(&self.data[self.start..], self.is_ending, options) + { r } else { return if self.is_ending { - if self.start == self.end { + if self.start == self.data.len() { None // We have finished } else { let error = LexerError { - position: self.position..self.position + (self.end - self.start), + position: self.position..self.position + (self.data.len() - self.start), message: "Unexpected end of file".into(), }; - self.end = self.start; // We consume everything + self.start = self.data.len(); // We consume everything Some(Err(error)) } } else { @@ -185,9 +179,9 @@ impl Lexer { "The lexer must consume at least one byte each time" ); debug_assert!( - self.start + consumed <= self.end, + self.start + consumed <= self.data.len(), "The lexer tried to consumed {consumed} bytes but only {} bytes are readable", - self.end - self.start + self.data.len() - self.start ); let old_position = self.position; self.start += consumed; @@ -205,14 +199,14 @@ impl Lexer { } pub fn is_end(&self) -> bool { - self.is_ending && self.end == self.start + self.is_ending && self.data.len() == self.start } fn skip_whitespaces_and_comments(&mut self) -> Option<()> { loop { self.skip_whitespaces(); - let buf = &self.data[self.start..self.end]; + let buf = &self.data[self.start..]; if let Some(line_comment_start) = self.line_comment_start { if buf.starts_with(line_comment_start) { // Comment @@ -222,7 +216,7 @@ impl Lexer { continue; } if self.is_ending { - self.end = self.start; // EOF + self.start = self.data.len(); // EOF return Some(()); } return None; // We need more data @@ -234,7 +228,7 @@ impl Lexer { fn skip_whitespaces(&mut self) { if self.is_line_jump_whitespace { - for (i, c) in self.data[self.start..self.end].iter().enumerate() { + for (i, c) in self.data[self.start..].iter().enumerate() { if !matches!(c, b' ' | b'\t' | b'\r' | b'\n') { self.start += i; self.position += i; @@ -243,7 +237,7 @@ impl Lexer { //TODO: SIMD } } else { - for (i, c) in self.data[self.start..self.end].iter().enumerate() { + for (i, c) in self.data[self.start..].iter().enumerate() { if !matches!(c, b' ' | b'\t') { self.start += i; self.position += i; @@ -253,15 +247,14 @@ impl Lexer { } } // We only have whitespaces - self.position += self.end - self.start; - self.end = self.start; + self.position += self.data.len() - self.start; + self.start = self.data.len(); } - fn shrink_if_useful(&mut self) { - if self.start * 2 > self.data.len() { - // We have read more than half of the buffer, let's move the data to the beginning - self.data.copy_within(self.start..self.end, 0); - self.end -= self.start; + fn shrink_data(&mut self) { + if self.start > 0 { + self.data.copy_within(self.start.., 0); + self.data.truncate(self.data.len() - self.start); self.start = 0; } } From 4cb377bda4d945e96e9cc59fb94b4837b184b213 Mon Sep 17 00:00:00 2001 From: Tpt Date: Wed, 9 Aug 2023 22:35:40 +0200 Subject: [PATCH 042/217] Adds documentation field to Cargo.toml --- lib/Cargo.toml | 1 + lib/oxrdf/Cargo.toml | 1 + lib/oxrdfxml/Cargo.toml | 1 + lib/oxsdatatypes/Cargo.toml | 1 + lib/oxttl/Cargo.toml | 1 + lib/sparesults/Cargo.toml | 1 + lib/spargebra/Cargo.toml | 1 + lib/sparopt/Cargo.toml | 1 + lib/sparql-smith/Cargo.toml | 1 + oxrocksdb-sys/Cargo.toml | 1 + 10 files changed, 10 insertions(+) diff --git a/lib/Cargo.toml b/lib/Cargo.toml index d9be7aed..aef5f012 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -8,6 +8,7 @@ keywords = ["RDF", "SPARQL", "graph-database", "database"] categories = ["database-implementations"] repository = "https://github.com/oxigraph/oxigraph/tree/main/lib" homepage = "https://oxigraph.org/" +documentation = "https://docs.rs/oxigraph" description = """ a SPARQL database and RDF toolkit """ diff --git a/lib/oxrdf/Cargo.toml b/lib/oxrdf/Cargo.toml index 344d46fe..48845b04 100644 --- a/lib/oxrdf/Cargo.toml +++ b/lib/oxrdf/Cargo.toml @@ -10,6 +10,7 @@ homepage = "https://oxigraph.org/" description = """ A library providing basic data structures related to RDF """ +documentation = "https://docs.rs/oxrdf" edition = "2021" rust-version = "1.65" diff --git a/lib/oxrdfxml/Cargo.toml b/lib/oxrdfxml/Cargo.toml index 895159b5..c411bc9f 100644 --- a/lib/oxrdfxml/Cargo.toml +++ b/lib/oxrdfxml/Cargo.toml @@ -10,6 +10,7 @@ homepage = "https://oxigraph.org/" description = """ Parser for the RDF/XML language """ +documentation = "https://docs.rs/oxrdfxml" edition = "2021" rust-version = "1.65" diff --git a/lib/oxsdatatypes/Cargo.toml b/lib/oxsdatatypes/Cargo.toml index f3a1e3c3..f4f42ab8 100644 --- a/lib/oxsdatatypes/Cargo.toml +++ b/lib/oxsdatatypes/Cargo.toml @@ -10,6 +10,7 @@ homepage = "https://oxigraph.org/" description = """ An implementation of some XSD datatypes for SPARQL implementations """ +documentation = "https://docs.rs/oxsdatatypes" edition = "2021" rust-version = "1.65" diff --git a/lib/oxttl/Cargo.toml b/lib/oxttl/Cargo.toml index 06871c96..f8039069 100644 --- a/lib/oxttl/Cargo.toml +++ b/lib/oxttl/Cargo.toml @@ -10,6 +10,7 @@ homepage = "https://oxigraph.org/" description = """ Parser for languages related to RDF Turtle (N-Triples, N-Quads, Turtle, TriG and N3) """ +documentation = "https://docs.rs/oxttl" edition = "2021" rust-version = "1.65" diff --git a/lib/sparesults/Cargo.toml b/lib/sparesults/Cargo.toml index c345b2a0..ef023da7 100644 --- a/lib/sparesults/Cargo.toml +++ b/lib/sparesults/Cargo.toml @@ -10,6 +10,7 @@ homepage = "https://oxigraph.org/" description = """ SPARQL query results formats parsers and serializers """ +documentation = "https://docs.rs/sparesults" edition = "2021" rust-version = "1.65" diff --git a/lib/spargebra/Cargo.toml b/lib/spargebra/Cargo.toml index d13d8037..323502ac 100644 --- a/lib/spargebra/Cargo.toml +++ b/lib/spargebra/Cargo.toml @@ -7,6 +7,7 @@ readme = "README.md" keywords = ["SPARQL"] repository = "https://github.com/oxigraph/oxigraph/tree/main/lib/spargebra" homepage = "https://oxigraph.org/" +documentation = "https://docs.rs/spargebra" description = """ A SPARQL parser """ diff --git a/lib/sparopt/Cargo.toml b/lib/sparopt/Cargo.toml index 06934952..3406930d 100644 --- a/lib/sparopt/Cargo.toml +++ b/lib/sparopt/Cargo.toml @@ -7,6 +7,7 @@ readme = "README.md" keywords = ["SPARQL"] repository = "https://github.com/oxigraph/oxigraph/tree/main/lib/sparopt" homepage = "https://oxigraph.org/" +documentation = "https://docs.rs/sparopt" description = """ A SPARQL optimizer """ diff --git a/lib/sparql-smith/Cargo.toml b/lib/sparql-smith/Cargo.toml index a36096bb..c96e416a 100644 --- a/lib/sparql-smith/Cargo.toml +++ b/lib/sparql-smith/Cargo.toml @@ -7,6 +7,7 @@ readme = "README.md" keywords = ["SPARQL"] repository = "https://github.com/oxigraph/oxigraph/tree/main/lib/sparql-smith" homepage = "https://oxigraph.org/" +documentation = "https://docs.rs/sparql-smith" description = """ A SPARQL test cases generator """ diff --git a/oxrocksdb-sys/Cargo.toml b/oxrocksdb-sys/Cargo.toml index 5100e198..d25d237e 100644 --- a/oxrocksdb-sys/Cargo.toml +++ b/oxrocksdb-sys/Cargo.toml @@ -8,6 +8,7 @@ readme = "README.md" description = """ Rust bindings for RocksDB for Oxigraph usage. """ +documentation = "https://docs.rs/oxrocksdb-sys" edition = "2021" rust-version = "1.65" build = "build.rs" From 12a738279f9359938a4a9775bef7086da4411ddc Mon Sep 17 00:00:00 2001 From: Tpt Date: Tue, 8 Aug 2023 21:15:11 +0200 Subject: [PATCH 043/217] Python: allows giving pathlib.Path for input --- python/src/io.rs | 17 +++++++++-------- python/src/store.rs | 23 ++++++++++++----------- python/tests/test_store.py | 8 ++++---- 3 files changed, 25 insertions(+), 23 deletions(-) diff --git a/python/src/io.rs b/python/src/io.rs index d29bfb9d..28245b39 100644 --- a/python/src/io.rs +++ b/python/src/io.rs @@ -13,6 +13,7 @@ use std::cmp::max; use std::error::Error; use std::fs::File; use std::io::{self, BufWriter, Cursor, Read, Write}; +use std::path::{Path, PathBuf}; pub fn add_to_module(module: &PyModule) -> PyResult<()> { module.add_wrapped(wrap_pyfunction!(parse))?; @@ -34,7 +35,7 @@ pub fn add_to_module(module: &PyModule) -> PyResult<()> { /// and ``application/xml`` for `RDF/XML `_. /// /// :param input: The binary I/O object or file path to read from. For example, it could be a file path as a string or a file reader opened in binary mode with ``open('my_file.ttl', 'rb')``. -/// :type input: io(bytes) or io(str) or str +/// :type input: io(bytes) or io(str) or str or pathlib.Path /// :param mime_type: the MIME type of the RDF serialization. /// :type mime_type: str /// :param base_iri: the base IRI used to resolve the relative IRIs in the file or :py:const:`None` if relative IRI resolution should not be done. @@ -55,8 +56,8 @@ pub fn parse( base_iri: Option<&str>, py: Python<'_>, ) -> PyResult { - let input = if let Ok(path) = input.extract::<&str>(py) { - PyReadable::from_file(path, py).map_err(map_io_err)? + let input = if let Ok(path) = input.extract::(py) { + PyReadable::from_file(&path, py).map_err(map_io_err)? } else { PyReadable::from_data(input, py) }; @@ -106,7 +107,7 @@ pub fn parse( /// :param input: the RDF triples and quads to serialize. /// :type input: iterable(Triple) or iterable(Quad) /// :param output: The binary I/O object or file path to write to. For example, it could be a file path as a string or a file writer opened in binary mode with ``open('my_file.ttl', 'wb')``. -/// :type output: io(bytes) or str +/// :type output: io(bytes) or str or pathlib.Path /// :param mime_type: the MIME type of the RDF serialization. /// :type mime_type: str /// :rtype: None @@ -119,8 +120,8 @@ pub fn parse( /// b' "1" .\n' #[pyfunction] pub fn serialize(input: &PyAny, output: PyObject, mime_type: &str, py: Python<'_>) -> PyResult<()> { - let output = if let Ok(path) = output.extract::<&str>(py) { - PyWritable::from_file(path, py).map_err(map_io_err)? + let output = if let Ok(path) = output.extract::(py) { + PyWritable::from_file(&path, py).map_err(map_io_err)? } else { PyWritable::from_data(output) }; @@ -198,7 +199,7 @@ pub enum PyReadable { } impl PyReadable { - pub fn from_file(file: &str, py: Python<'_>) -> io::Result { + pub fn from_file(file: &Path, py: Python<'_>) -> io::Result { Ok(Self::File(py.allow_threads(|| File::open(file))?)) } @@ -229,7 +230,7 @@ pub enum PyWritable { } impl PyWritable { - pub fn from_file(file: &str, py: Python<'_>) -> io::Result { + pub fn from_file(file: &Path, py: Python<'_>) -> io::Result { Ok(Self::File(BufWriter::new( py.allow_threads(|| File::create(file))?, ))) diff --git a/python/src/store.rs b/python/src/store.rs index 5e3c1df9..957c1b5c 100644 --- a/python/src/store.rs +++ b/python/src/store.rs @@ -9,6 +9,7 @@ use oxigraph::sparql::Update; use oxigraph::store::{self, LoaderError, SerializerError, StorageError, Store}; use pyo3::exceptions::{PyIOError, PyRuntimeError, PyValueError}; use pyo3::prelude::*; +use std::path::PathBuf; /// RDF store. /// @@ -26,7 +27,7 @@ use pyo3::prelude::*; /// :param path: the path of the directory in which the store should read and write its data. If the directory does not exist, it is created. /// If no directory is provided a temporary one is created and removed when the Python garbage collector removes the store. /// In this case, the store data are kept in memory and never written on disk. -/// :type path: str or None, optional +/// :type path: str or pathlib.Path or None, optional /// :raises IOError: if the target directory contains invalid data or could not be accessed. /// /// The :py:func:`str` function provides a serialization of the store in NQuads: @@ -45,7 +46,7 @@ pub struct PyStore { impl PyStore { #[new] #[pyo3(signature = (path = None))] - fn new(path: Option<&str>, py: Python<'_>) -> PyResult { + fn new(path: Option, py: Python<'_>) -> PyResult { py.allow_threads(|| { Ok(Self { inner: if let Some(path) = path { @@ -357,7 +358,7 @@ impl PyStore { /// and ``application/xml`` for `RDF/XML `_. /// /// :param input: The binary I/O object or file path to read from. For example, it could be a file path as a string or a file reader opened in binary mode with ``open('my_file.ttl', 'rb')``. - /// :type input: io(bytes) or io(str) or str + /// :type input: io(bytes) or io(str) or str or pathlib.Path /// :param mime_type: the MIME type of the RDF serialization. /// :type mime_type: str /// :param base_iri: the base IRI used to resolve the relative IRIs in the file or :py:const:`None` if relative IRI resolution should not be done. @@ -387,8 +388,8 @@ impl PyStore { } else { None }; - let input = if let Ok(path) = input.extract::<&str>(py) { - PyReadable::from_file(path, py).map_err(map_io_err)? + let input = if let Ok(path) = input.extract::(py) { + PyReadable::from_file(&path, py).map_err(map_io_err)? } else { PyReadable::from_data(input, py) }; @@ -439,7 +440,7 @@ impl PyStore { /// and ``application/xml`` for `RDF/XML `_. /// /// :param input: The binary I/O object or file path to read from. For example, it could be a file path as a string or a file reader opened in binary mode with ``open('my_file.ttl', 'rb')``. - /// :type input: io(bytes) or io(str) or str + /// :type input: io(bytes) or io(str) or str or pathlib.Path /// :param mime_type: the MIME type of the RDF serialization. /// :type mime_type: str /// :param base_iri: the base IRI used to resolve the relative IRIs in the file or :py:const:`None` if relative IRI resolution should not be done. @@ -469,8 +470,8 @@ impl PyStore { } else { None }; - let input = if let Ok(path) = input.extract::<&str>(py) { - PyReadable::from_file(path, py).map_err(map_io_err)? + let input = if let Ok(path) = input.extract::(py) { + PyReadable::from_file(&path, py).map_err(map_io_err)? } else { PyReadable::from_data(input, py) }; @@ -518,7 +519,7 @@ impl PyStore { /// and ``application/xml`` for `RDF/XML `_. /// /// :param output: The binary I/O object or file path to write to. For example, it could be a file path as a string or a file writer opened in binary mode with ``open('my_file.ttl', 'wb')``. - /// :type output: io(bytes) or str + /// :type output: io(bytes) or str or pathlib.Path /// :param mime_type: the MIME type of the RDF serialization. /// :type mime_type: str /// :param from_graph: if a triple based format is requested, the store graph from which dump the triples. By default, the default graph is used. @@ -541,8 +542,8 @@ impl PyStore { from_graph: Option<&PyAny>, py: Python<'_>, ) -> PyResult<()> { - let output = if let Ok(path) = output.extract::<&str>(py) { - PyWritable::from_file(path, py).map_err(map_io_err)? + let output = if let Ok(path) = output.extract::(py) { + PyWritable::from_file(&path, py).map_err(map_io_err)? } else { PyWritable::from_data(output) }; diff --git a/python/tests/test_store.py b/python/tests/test_store.py index b29447c8..287bd27e 100644 --- a/python/tests/test_store.py +++ b/python/tests/test_store.py @@ -266,11 +266,11 @@ class TestStore(unittest.TestCase): def test_load_file(self) -> None: with NamedTemporaryFile(delete=False) as fp: - file_name = fp.name + file_name = Path(fp.name) fp.write(b" .") store = Store() store.load(file_name, mime_type="application/n-quads") - Path(file_name).unlink() + file_name.unlink() self.assertEqual(set(store), {Quad(foo, bar, baz, graph)}) def test_load_with_io_error(self) -> None: @@ -311,12 +311,12 @@ class TestStore(unittest.TestCase): def test_dump_file(self) -> None: with NamedTemporaryFile(delete=False) as fp: - file_name = fp.name + file_name = Path(fp.name) store = Store() store.add(Quad(foo, bar, baz, graph)) store.dump(file_name, "application/n-quads") self.assertEqual( - Path(file_name).read_text(), + file_name.read_text(), " .\n", ) From b06d6506cb52ead568f70b47e44866a5d15d7160 Mon Sep 17 00:00:00 2001 From: Tpt Date: Sat, 12 Aug 2023 16:52:15 +0200 Subject: [PATCH 044/217] Fixes "let else" formatting --- lib/oxrdfxml/src/parser.rs | 6 ++++-- lib/oxsdatatypes/src/decimal.rs | 4 ++-- lib/src/storage/backend/rocksdb.rs | 8 ++++---- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/lib/oxrdfxml/src/parser.rs b/lib/oxrdfxml/src/parser.rs index ddb11fd3..9f64da00 100644 --- a/lib/oxrdfxml/src/parser.rs +++ b/lib/oxrdfxml/src/parser.rs @@ -734,11 +734,13 @@ impl RdfXmlReader { } RdfXmlNextProduction::PropertyElt { subject } => { let iri = if *tag_name == *RDF_LI { - let Some(RdfXmlState::NodeElt { li_counter, .. }) = self.state.last_mut() else { + let Some(RdfXmlState::NodeElt { li_counter, .. }) = self.state.last_mut() + else { return Err(SyntaxError::msg(format!( "Invalid property element tag name: {}", &tag_name - )).into()); + )) + .into()); }; *li_counter += 1; NamedNode::new_unchecked(format!( diff --git a/lib/oxsdatatypes/src/decimal.rs b/lib/oxsdatatypes/src/decimal.rs index 6a309108..c93ef56a 100644 --- a/lib/oxsdatatypes/src/decimal.rs +++ b/lib/oxsdatatypes/src/decimal.rs @@ -26,8 +26,8 @@ impl Decimal { return Err(TooLargeForDecimalError); }; let Some(value) = i.checked_mul(10_i128.pow(shift)) else { - return Err(TooLargeForDecimalError); - }; + return Err(TooLargeForDecimalError); + }; Ok(Self { value }) } diff --git a/lib/src/storage/backend/rocksdb.rs b/lib/src/storage/backend/rocksdb.rs index 0e9fe986..68e88744 100644 --- a/lib/src/storage/backend/rocksdb.rs +++ b/lib/src/storage/backend/rocksdb.rs @@ -942,10 +942,10 @@ impl Reader { } InnerReader::Transaction(inner) => { let Some(inner) = inner.upgrade() else { - return Err(StorageError::Other( - "The transaction is already ended".into(), - )); - }; + return Err(StorageError::Other( + "The transaction is already ended".into(), + )); + }; ffi_result!(rocksdb_transaction_get_pinned_cf_with_status( *inner, self.options, From 73af297b4c5b12841d78eb21078bad1d075eca7a Mon Sep 17 00:00:00 2001 From: Tpt Date: Wed, 9 Aug 2023 22:46:24 +0200 Subject: [PATCH 045/217] Adds From to NamedNode --- lib/oxrdf/src/named_node.rs | 18 ++++++++++++++++++ lib/oxrdfxml/src/parser.rs | 20 +++++++++----------- lib/oxttl/src/line_formats.rs | 10 +++++----- lib/oxttl/src/n3.rs | 6 +++--- lib/oxttl/src/terse.rs | 16 ++++++++-------- lib/spargebra/src/parser.rs | 2 +- server/src/main.rs | 12 +++++------- 7 files changed, 49 insertions(+), 35 deletions(-) diff --git a/lib/oxrdf/src/named_node.rs b/lib/oxrdf/src/named_node.rs index 23a156d6..9b545bcc 100644 --- a/lib/oxrdf/src/named_node.rs +++ b/lib/oxrdf/src/named_node.rs @@ -216,3 +216,21 @@ impl PartialOrd> for NamedNode { self.as_ref().partial_cmp(other) } } + +impl From> for NamedNode { + #[inline] + fn from(iri: Iri) -> Self { + Self { + iri: iri.into_inner(), + } + } +} + +impl<'a> From> for NamedNodeRef<'a> { + #[inline] + fn from(iri: Iri<&'a str>) -> Self { + Self { + iri: iri.into_inner(), + } + } +} diff --git a/lib/oxrdfxml/src/parser.rs b/lib/oxrdfxml/src/parser.rs index 9f64da00..279d0301 100644 --- a/lib/oxrdfxml/src/parser.rs +++ b/lib/oxrdfxml/src/parser.rs @@ -1157,17 +1157,15 @@ impl RdfXmlReader { fn resolve(base_iri: &Option>, relative_iri: String) -> Result { if let Some(base_iri) = base_iri { - Ok(NamedNode::new_unchecked( - base_iri - .resolve(&relative_iri) - .map_err(|error| SyntaxError { - inner: SyntaxErrorKind::InvalidIri { - iri: relative_iri, - error, - }, - })? - .into_inner(), - )) + Ok(base_iri + .resolve(&relative_iri) + .map_err(|error| SyntaxError { + inner: SyntaxErrorKind::InvalidIri { + iri: relative_iri, + error, + }, + })? + .into()) } else { NamedNode::new(relative_iri.clone()).map_err(|error| SyntaxError { inner: SyntaxErrorKind::InvalidIri { diff --git a/lib/oxttl/src/line_formats.rs b/lib/oxttl/src/line_formats.rs index 6a253658..5990c889 100644 --- a/lib/oxttl/src/line_formats.rs +++ b/lib/oxttl/src/line_formats.rs @@ -59,7 +59,7 @@ impl RuleRecognizer for NQuadsRecognizer { NQuadsState::ExpectSubject => match token { N3Token::IriRef(s) => { self.subjects - .push(NamedNode::new_unchecked(s.into_inner()).into()); + .push(NamedNode::from(s).into()); self.stack.push(NQuadsState::ExpectPredicate); self } @@ -82,7 +82,7 @@ impl RuleRecognizer for NQuadsRecognizer { NQuadsState::ExpectPredicate => match token { N3Token::IriRef(p) => { self.predicates - .push(NamedNode::new_unchecked(p.into_inner())); + .push(p.into()); self.stack.push(NQuadsState::ExpectedObject); self } @@ -94,7 +94,7 @@ impl RuleRecognizer for NQuadsRecognizer { NQuadsState::ExpectedObject => match token { N3Token::IriRef(o) => { self.objects - .push(NamedNode::new_unchecked(o.into_inner()).into()); + .push(NamedNode::from(o).into()); self.stack .push(NQuadsState::ExpectPossibleGraphOrEndOfQuotedTriple); self @@ -151,7 +151,7 @@ impl RuleRecognizer for NQuadsRecognizer { self.objects.push( Literal::new_typed_literal( value, - NamedNode::new_unchecked(d.into_inner()), + d ) .into(), ); @@ -167,7 +167,7 @@ impl RuleRecognizer for NQuadsRecognizer { N3Token::IriRef(g) if self.with_graph_name => { self.emit_quad( results, - NamedNode::new_unchecked(g.into_inner()).into(), + NamedNode::from(g).into(), ); self.stack.push(NQuadsState::ExpectDot); self diff --git a/lib/oxttl/src/n3.rs b/lib/oxttl/src/n3.rs index 7167df09..d1847991 100644 --- a/lib/oxttl/src/n3.rs +++ b/lib/oxttl/src/n3.rs @@ -778,7 +778,7 @@ impl RuleRecognizer for N3Recognizer { N3State::PathItem => { match token { N3Token::IriRef(iri) => { - self.terms.push(NamedNode::new_unchecked(iri.into_inner()).into()); + self.terms.push(NamedNode::from(iri).into()); self } N3Token::PrefixedName { prefix, local, might_be_invalid_iri } => match resolve_local_name(prefix, &local, might_be_invalid_iri, &self.prefixes) { @@ -860,7 +860,7 @@ impl RuleRecognizer for N3Recognizer { } N3State::IriPropertyList => match token { N3Token::IriRef(id) => { - self.terms.push(NamedNode::new_unchecked(id.into_inner()).into()); + self.terms.push(NamedNode::from(id).into()); self.stack.push(N3State::PropertyListEnd); self.stack.push(N3State::PredicateObjectList); self @@ -940,7 +940,7 @@ impl RuleRecognizer for N3Recognizer { N3State::LiteralExpectDatatype { value } => { match token { N3Token::IriRef(datatype) => { - self.terms.push(Literal::new_typed_literal(value, NamedNode::new_unchecked(datatype.into_inner())).into()); + self.terms.push(Literal::new_typed_literal(value, datatype).into()); self }, N3Token::PrefixedName { prefix, local, might_be_invalid_iri } => match resolve_local_name(prefix, &local, might_be_invalid_iri, &self.prefixes) { diff --git a/lib/oxttl/src/terse.rs b/lib/oxttl/src/terse.rs index 151ee062..cedf089e 100644 --- a/lib/oxttl/src/terse.rs +++ b/lib/oxttl/src/terse.rs @@ -127,7 +127,7 @@ impl RuleRecognizer for TriGRecognizer { TriGState::TriplesOrGraph => match token { N3Token::IriRef(iri) => { self.stack.push(TriGState::WrappedGraphOrPredicateObjectList { - term: NamedNode::new_unchecked(iri.into_inner()).into() + term: NamedNode::from(iri).into() }); self } @@ -291,7 +291,7 @@ impl RuleRecognizer for TriGRecognizer { self } N3Token::IriRef(iri) => { - self.cur_subject.push(NamedNode::new_unchecked(iri.into_inner()).into()); + self.cur_subject.push(NamedNode::from(iri).into()); self.stack.push(TriGState::PredicateObjectList); self } @@ -337,7 +337,7 @@ impl RuleRecognizer for TriGRecognizer { // [7g] labelOrSubject ::= iri | BlankNode TriGState::GraphName => match token { N3Token::IriRef(iri) => { - self.cur_graph = NamedNode::new_unchecked(iri.into_inner()).into(); + self.cur_graph = NamedNode::from(iri).into(); self } N3Token::PrefixedName { prefix, local, might_be_invalid_iri } => match resolve_local_name(prefix, &local, might_be_invalid_iri, &self.prefixes) { @@ -451,7 +451,7 @@ impl RuleRecognizer for TriGRecognizer { self } N3Token::IriRef(iri) => { - self.cur_predicate.push(NamedNode::new_unchecked(iri.into_inner())); + self.cur_predicate.push(NamedNode::from(iri)); self } N3Token::PrefixedName { prefix, local, might_be_invalid_iri } => match resolve_local_name(prefix, &local, might_be_invalid_iri, &self.prefixes) { @@ -479,7 +479,7 @@ impl RuleRecognizer for TriGRecognizer { // [137s] BlankNode ::= BLANK_NODE_LABEL | ANON TriGState::Object => match token { N3Token::IriRef(iri) => { - self.cur_object.push(NamedNode::new_unchecked(iri.into_inner()).into()); + self.cur_object.push(NamedNode::from(iri).into()); self.emit_quad(results); self } @@ -632,7 +632,7 @@ impl RuleRecognizer for TriGRecognizer { TriGState::LiteralExpectDatatype { value, emit } => { match token { N3Token::IriRef(datatype) => { - self.cur_object.push(Literal::new_typed_literal(value, NamedNode::new_unchecked(datatype.into_inner())).into()); + self.cur_object.push(Literal::new_typed_literal(value, datatype).into()); if emit { self.emit_quad(results); } @@ -694,7 +694,7 @@ impl RuleRecognizer for TriGRecognizer { self } N3Token::IriRef(iri) => { - self.cur_subject.push(NamedNode::new_unchecked(iri.into_inner()).into()); + self.cur_subject.push(NamedNode::from(iri).into()); self } N3Token::PrefixedName { prefix, local, might_be_invalid_iri } => match resolve_local_name(prefix, &local, might_be_invalid_iri, &self.prefixes) { @@ -726,7 +726,7 @@ impl RuleRecognizer for TriGRecognizer { self } N3Token::IriRef(iri) => { - self.cur_object.push(NamedNode::new_unchecked(iri.into_inner()).into()); + self.cur_object.push(NamedNode::from(iri).into()); self } N3Token::PrefixedName { prefix, local, might_be_invalid_iri } => match resolve_local_name(prefix, &local, might_be_invalid_iri, &self.prefixes) { diff --git a/lib/spargebra/src/parser.rs b/lib/spargebra/src/parser.rs index 0dccff3b..32522c5c 100644 --- a/lib/spargebra/src/parser.rs +++ b/lib/spargebra/src/parser.rs @@ -1975,7 +1975,7 @@ parser! { rule String() -> String = STRING_LITERAL_LONG1() / STRING_LITERAL_LONG2() / STRING_LITERAL1() / STRING_LITERAL2() rule iri() -> NamedNode = i:(IRIREF() / PrefixedName()) { - NamedNode::new_unchecked(i.into_inner()) + NamedNode::from(i) } rule PrefixedName() -> Iri = PNAME_LN() / diff --git a/server/src/main.rs b/server/src/main.rs index 5db006b9..aed17921 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -1102,13 +1102,11 @@ fn base_url(request: &Request) -> String { } fn resolve_with_base(request: &Request, url: &str) -> Result { - Ok(NamedNode::new_unchecked( - Iri::parse(base_url(request)) - .map_err(bad_request)? - .resolve(url) - .map_err(bad_request)? - .into_inner(), - )) + Ok(Iri::parse(base_url(request)) + .map_err(bad_request)? + .resolve(url) + .map_err(bad_request)? + .into()) } fn url_query(request: &Request) -> &[u8] { From 7cd383af79b812f51e0d05959b6ca624fc1570f1 Mon Sep 17 00:00:00 2001 From: Tpt Date: Sun, 6 Aug 2023 21:48:39 +0200 Subject: [PATCH 046/217] Introduces OxRDF I/O stand-alone crate --- .github/workflows/tests.yml | 6 +- Cargo.lock | 13 +- Cargo.toml | 1 + lib/Cargo.toml | 3 +- lib/oxrdfio/Cargo.toml | 33 ++ lib/oxrdfio/README.md | 61 ++++ lib/oxrdfio/src/error.rs | 148 +++++++++ lib/oxrdfio/src/format.rs | 203 ++++++++++++ lib/oxrdfio/src/lib.rs | 19 ++ lib/oxrdfio/src/parser.rs | 577 ++++++++++++++++++++++++++++++++++ lib/oxrdfio/src/serializer.rs | 322 +++++++++++++++++++ lib/src/io/error.rs | 45 +-- lib/src/io/format.rs | 21 ++ lib/src/io/mod.rs | 4 + lib/src/io/read.rs | 191 ++--------- lib/src/io/write.rs | 84 ++--- 16 files changed, 1465 insertions(+), 266 deletions(-) create mode 100644 lib/oxrdfio/Cargo.toml create mode 100644 lib/oxrdfio/README.md create mode 100644 lib/oxrdfio/src/error.rs create mode 100644 lib/oxrdfio/src/format.rs create mode 100644 lib/oxrdfio/src/lib.rs create mode 100644 lib/oxrdfio/src/parser.rs create mode 100644 lib/oxrdfio/src/serializer.rs diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c5010d9c..a58e710c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -36,6 +36,8 @@ jobs: working-directory: ./lib/oxrdfxml - run: cargo clippy working-directory: ./lib/oxttl + - run: cargo clippy + working-directory: ./lib/oxrdfio - run: cargo clippy working-directory: ./lib/sparesults - run: cargo clippy @@ -102,6 +104,8 @@ jobs: working-directory: ./lib/oxrdfxml - run: cargo clippy -- -D warnings -D clippy::all working-directory: ./lib/oxttl + - run: cargo clippy -- -D warnings -D clippy::all + working-directory: ./lib/oxrdfio - run: cargo clippy -- -D warnings -D clippy::all working-directory: ./lib/sparesults - run: cargo clippy -- -D warnings -D clippy::all @@ -159,7 +163,7 @@ jobs: - run: rustup update - uses: Swatinem/rust-cache@v2 - run: cargo install cargo-semver-checks || true - - run: cargo semver-checks check-release --exclude oxrocksdb-sys --exclude oxigraph_js --exclude pyoxigraph --exclude oxigraph_testsuite --exclude oxigraph_server --exclude oxrdfxml --exclude oxttl --exclude sparopt + - run: cargo semver-checks check-release --exclude oxrocksdb-sys --exclude oxigraph_js --exclude pyoxigraph --exclude oxigraph_testsuite --exclude oxigraph_server --exclude oxrdfxml --exclude oxttl --exclude oxrdfio --exclude sparopt test_linux: runs-on: ubuntu-latest diff --git a/Cargo.lock b/Cargo.lock index ab40b360..1e3337d5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -952,10 +952,9 @@ dependencies = [ "oxilangtag", "oxiri", "oxrdf", - "oxrdfxml", + "oxrdfio", "oxrocksdb-sys", "oxsdatatypes", - "oxttl", "rand", "regex", "sha-1", @@ -1036,6 +1035,16 @@ dependencies = [ "rand", ] +[[package]] +name = "oxrdfio" +version = "0.1.0-alpha.1-dev" +dependencies = [ + "oxrdf", + "oxrdfxml", + "oxttl", + "tokio", +] + [[package]] name = "oxrdfxml" version = "0.1.0-alpha.1-dev" diff --git a/Cargo.toml b/Cargo.toml index 0ad8536d..8b6c3e48 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,7 @@ members = [ "js", "lib", "lib/oxrdf", + "lib/oxrdfio", "lib/oxrdfxml", "lib/oxsdatatypes", "lib/oxttl", diff --git a/lib/Cargo.toml b/lib/Cargo.toml index aef5f012..0cf839de 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -35,9 +35,8 @@ siphasher = "0.3" lazy_static = "1" json-event-parser = "0.1" oxrdf = { version = "0.2.0-alpha.1-dev", path = "oxrdf", features = ["rdf-star", "oxsdatatypes"] } -oxrdfxml = { version = "0.1.0-alpha.1-dev", path = "oxrdfxml" } oxsdatatypes = { version = "0.2.0-alpha.1-dev", path="oxsdatatypes" } -oxttl = { version = "0.1.0-alpha.1-dev" , path = "oxttl", features = ["rdf-star"] } +oxrdfio = { version = "0.1.0-alpha.1-dev" , path = "oxrdfio", features = ["rdf-star"] } spargebra = { version = "0.3.0-alpha.1-dev", path = "spargebra", features = ["rdf-star", "sep-0002", "sep-0006"] } sparopt = { version = "0.1.0-alpha.1-dev", path="sparopt", features = ["rdf-star", "sep-0002", "sep-0006"] } sparesults = { version = "0.2.0-alpha.1-dev", path = "sparesults", features = ["rdf-star"] } diff --git a/lib/oxrdfio/Cargo.toml b/lib/oxrdfio/Cargo.toml new file mode 100644 index 00000000..70f266b0 --- /dev/null +++ b/lib/oxrdfio/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "oxrdfio" +version = "0.1.0-alpha.1-dev" +authors = ["Tpt "] +license = "MIT OR Apache-2.0" +readme = "README.md" +keywords = ["RDF"] +repository = "https://github.com/oxigraph/oxigraph/tree/master/lib/oxrdfxml" +homepage = "https://oxigraph.org/" +documentation = "https://docs.rs/oxrdfio" +description = """ +Parser for various RDF serializations +""" +edition = "2021" +rust-version = "1.65" + +[features] +default = [] +async-tokio = ["dep:tokio", "oxrdfxml/async-tokio", "oxttl/async-tokio"] +rdf-star = ["oxrdf/rdf-star", "oxttl/rdf-star"] + +[dependencies] +oxrdf = { version = "0.2.0-alpha.1-dev", path = "../oxrdf" } +oxrdfxml = { version = "0.1.0-alpha.1-dev", path = "../oxrdfxml" } +oxttl = { version = "0.1.0-alpha.1-dev" , path = "../oxttl" } +tokio = { version = "1", optional = true, features = ["io-util"] } + +[dev-dependencies] +tokio = { version = "1", features = ["rt", "macros"] } + +[package.metadata.docs.rs] +all-features = true +rustdoc-args = ["--cfg", "docsrs"] diff --git a/lib/oxrdfio/README.md b/lib/oxrdfio/README.md new file mode 100644 index 00000000..1712b8ae --- /dev/null +++ b/lib/oxrdfio/README.md @@ -0,0 +1,61 @@ +OxRDF I/O +========= + +[![Latest Version](https://img.shields.io/crates/v/oxrdfio.svg)](https://crates.io/crates/oxrdfio) +[![Released API docs](https://docs.rs/oxrdfio/badge.svg)](https://docs.rs/oxrdfio) +[![Crates.io downloads](https://img.shields.io/crates/d/oxrdfio)](https://crates.io/crates/oxrdfio) +[![actions status](https://github.com/oxigraph/oxigraph/workflows/build/badge.svg)](https://github.com/oxigraph/oxigraph/actions) +[![Gitter](https://badges.gitter.im/oxigraph/community.svg)](https://gitter.im/oxigraph/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) + +OxRDF I/O is a set of parsers and serializers for RDF. + +It supports: +* [N3](https://w3c.github.io/N3/spec/) using [`oxttl`](https://crates.io/crates/oxttl) +* [N-Quads](https://www.w3.org/TR/n-quads/) using [`oxttl`](https://crates.io/crates/oxttl) +* [N-Triples](https://www.w3.org/TR/n-triples/) using [`oxttl`](https://crates.io/crates/oxttl) +* [RDF/XML](https://www.w3.org/TR/rdf-syntax-grammar/) using [`oxrdfxml`](https://crates.io/crates/oxrdfxml) +* [TriG](https://www.w3.org/TR/trig/) using [`oxttl`](https://crates.io/crates/oxttl) +* [Turtle](https://www.w3.org/TR/turtle/) using [`oxttl`](https://crates.io/crates/oxttl) + +Support for [SPARQL-star](https://w3c.github.io/rdf-star/cg-spec/2021-12-17.html) is also available behind the `rdf-star`feature for [Turtle-star](https://w3c.github.io/rdf-star/cg-spec/2021-12-17.html#turtle-star), [TriG-star](https://w3c.github.io/rdf-star/cg-spec/2021-12-17.html#trig-star), [N-Triples-star](https://w3c.github.io/rdf-star/cg-spec/2021-12-17.html#n-triples-star) and [N-Quads-star](https://w3c.github.io/rdf-star/cg-spec/2021-12-17.html#n-quads-star). + +It is designed as a low level parser compatible with both synchronous and asynchronous I/O (behind the `async-tokio` feature). + +Usage example counting the number of people in a Turtle file: +```rust +use oxrdf::{NamedNodeRef, vocab::rdf}; +use oxrdfio::{RdfFormat, RdfParser}; + +let file = b"@base . +@prefix schema: . + a schema:Person ; + schema:name \"Foo\" . + a schema:Person ; + schema:name \"Bar\" ."; + +let schema_person = NamedNodeRef::new("http://schema.org/Person").unwrap(); +let mut count = 0; +for quad in RdfParser::from_format(RdfFormat::Turtle).parse_read(file.as_ref()) { + let quad = quad.unwrap(); + if quad.predicate == rdf::TYPE && quad.object == schema_person.into() { + count += 1; + } +} +assert_eq!(2, count); +``` + +## License + +This project is licensed under either of + +* Apache License, Version 2.0, ([LICENSE-APACHE](../LICENSE-APACHE) or + ``) +* MIT license ([LICENSE-MIT](../LICENSE-MIT) or + ``) + +at your option. + + +### Contribution + +Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in Oxigraph by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. diff --git a/lib/oxrdfio/src/error.rs b/lib/oxrdfio/src/error.rs new file mode 100644 index 00000000..ac8173a7 --- /dev/null +++ b/lib/oxrdfio/src/error.rs @@ -0,0 +1,148 @@ +use std::error::Error; +use std::{fmt, io}; + +/// Error returned during RDF format parsing. +#[derive(Debug)] +pub enum ParseError { + /// I/O error during parsing (file not found...). + Io(io::Error), + /// An error in the file syntax. + Syntax(SyntaxError), +} + +impl ParseError { + pub(crate) fn msg(msg: &'static str) -> Self { + Self::Syntax(SyntaxError { + inner: SyntaxErrorKind::Msg { msg }, + }) + } +} + +impl fmt::Display for ParseError { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Io(e) => e.fmt(f), + Self::Syntax(e) => e.fmt(f), + } + } +} + +impl Error for ParseError { + #[inline] + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Io(e) => Some(e), + Self::Syntax(e) => Some(e), + } + } +} + +impl From for SyntaxError { + #[inline] + fn from(error: oxttl::SyntaxError) -> Self { + SyntaxError { + inner: SyntaxErrorKind::Turtle(error), + } + } +} + +impl From for ParseError { + #[inline] + fn from(error: oxttl::ParseError) -> Self { + match error { + oxttl::ParseError::Syntax(e) => Self::Syntax(e.into()), + oxttl::ParseError::Io(e) => Self::Io(e), + } + } +} + +impl From for SyntaxError { + #[inline] + fn from(error: oxrdfxml::SyntaxError) -> Self { + SyntaxError { + inner: SyntaxErrorKind::RdfXml(error), + } + } +} + +impl From for ParseError { + #[inline] + fn from(error: oxrdfxml::ParseError) -> Self { + match error { + oxrdfxml::ParseError::Syntax(e) => Self::Syntax(e.into()), + oxrdfxml::ParseError::Io(e) => Self::Io(e), + } + } +} + +impl From for ParseError { + #[inline] + fn from(error: io::Error) -> Self { + Self::Io(error) + } +} + +impl From for ParseError { + #[inline] + fn from(error: SyntaxError) -> Self { + Self::Syntax(error) + } +} + +impl From for io::Error { + #[inline] + fn from(error: ParseError) -> Self { + match error { + ParseError::Io(error) => error, + ParseError::Syntax(error) => error.into(), + } + } +} + +/// An error in the syntax of the parsed file. +#[derive(Debug)] +pub struct SyntaxError { + inner: SyntaxErrorKind, +} + +#[derive(Debug)] +enum SyntaxErrorKind { + Turtle(oxttl::SyntaxError), + RdfXml(oxrdfxml::SyntaxError), + + Msg { msg: &'static str }, +} + +impl fmt::Display for SyntaxError { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match &self.inner { + SyntaxErrorKind::Turtle(e) => e.fmt(f), + SyntaxErrorKind::RdfXml(e) => e.fmt(f), + SyntaxErrorKind::Msg { msg } => write!(f, "{msg}"), + } + } +} + +impl Error for SyntaxError { + #[inline] + fn source(&self) -> Option<&(dyn Error + 'static)> { + match &self.inner { + SyntaxErrorKind::Turtle(e) => Some(e), + SyntaxErrorKind::RdfXml(e) => Some(e), + SyntaxErrorKind::Msg { .. } => None, + } + } +} + +impl From for io::Error { + #[inline] + fn from(error: SyntaxError) -> Self { + match error.inner { + SyntaxErrorKind::Turtle(error) => error.into(), + SyntaxErrorKind::RdfXml(error) => error.into(), + SyntaxErrorKind::Msg { msg } => io::Error::new(io::ErrorKind::InvalidData, msg), + } + } +} diff --git a/lib/oxrdfio/src/format.rs b/lib/oxrdfio/src/format.rs new file mode 100644 index 00000000..8c4ce230 --- /dev/null +++ b/lib/oxrdfio/src/format.rs @@ -0,0 +1,203 @@ +use std::fmt; + +/// RDF serialization formats. +/// +/// This enumeration is non exhaustive. New formats like JSON-LD might be added in the future. +#[derive(Eq, PartialEq, Debug, Clone, Copy, Hash)] +#[non_exhaustive] +pub enum RdfFormat { + /// [N3](https://w3c.github.io/N3/spec/) + N3, + /// [N-Quads](https://www.w3.org/TR/n-quads/) + NQuads, + /// [N-Triples](https://www.w3.org/TR/n-triples/) + NTriples, + /// [RDF/XML](https://www.w3.org/TR/rdf-syntax-grammar/) + RdfXml, + /// [TriG](https://www.w3.org/TR/trig/) + TriG, + /// [Turtle](https://www.w3.org/TR/turtle/) + Turtle, +} + +impl RdfFormat { + /// The format canonical IRI according to the [Unique URIs for file formats registry](https://www.w3.org/ns/formats/). + /// + /// ``` + /// use oxrdfio::RdfFormat; + /// + /// assert_eq!(RdfFormat::NTriples.iri(), "http://www.w3.org/ns/formats/N-Triples") + /// ``` + #[inline] + pub const fn iri(self) -> &'static str { + match self { + Self::N3 => "http://www.w3.org/ns/formats/N3", + Self::NQuads => "http://www.w3.org/ns/formats/N-Quads", + Self::NTriples => "http://www.w3.org/ns/formats/N-Triples", + Self::RdfXml => "http://www.w3.org/ns/formats/RDF_XML", + Self::TriG => "http://www.w3.org/ns/formats/TriG", + Self::Turtle => "http://www.w3.org/ns/formats/Turtle", + } + } + + /// The format [IANA media type](https://tools.ietf.org/html/rfc2046). + /// + /// ``` + /// use oxrdfio::RdfFormat; + /// + /// assert_eq!(RdfFormat::NTriples.media_type(), "application/n-triples") + /// ``` + #[inline] + pub const fn media_type(self) -> &'static str { + match self { + Self::N3 => "text/n3", + Self::NQuads => "application/n-quads", + Self::NTriples => "application/n-triples", + Self::RdfXml => "application/rdf+xml", + Self::TriG => "application/trig", + Self::Turtle => "text/turtle", + } + } + + /// The format [IANA-registered](https://tools.ietf.org/html/rfc2046) file extension. + /// + /// ``` + /// use oxrdfio::RdfFormat; + /// + /// assert_eq!(RdfFormat::NTriples.file_extension(), "nt") + /// ``` + #[inline] + pub const fn file_extension(self) -> &'static str { + match self { + Self::N3 => "n3", + Self::NQuads => "nq", + Self::NTriples => "nt", + Self::RdfXml => "rdf", + Self::TriG => "trig", + Self::Turtle => "ttl", + } + } + + /// The format name. + /// + /// ``` + /// use oxrdfio::RdfFormat; + /// + /// assert_eq!(RdfFormat::NTriples.name(), "N-Triples") + /// ``` + #[inline] + pub const fn name(self) -> &'static str { + match self { + Self::N3 => "N3", + Self::NQuads => "N-Quads", + Self::NTriples => "N-Triples", + Self::RdfXml => "RDF/XML", + Self::TriG => "TriG", + Self::Turtle => "Turtle", + } + } + + /// Checks if the formats supports [RDF datasets](https://www.w3.org/TR/rdf11-concepts/#dfn-rdf-dataset) and not only [RDF graphs](https://www.w3.org/TR/rdf11-concepts/#dfn-rdf-graph). + /// + /// ``` + /// use oxrdfio::RdfFormat; + /// + /// assert_eq!(RdfFormat::NTriples.supports_datasets(), false); + /// assert_eq!(RdfFormat::NQuads.supports_datasets(), true); + /// ``` + #[inline] + pub const fn supports_datasets(self) -> bool { + matches!(self, Self::NQuads | Self::TriG) + } + + /// Checks if the formats supports [RDF-star quoted triples](https://w3c.github.io/rdf-star/cg-spec/2021-12-17.html#dfn-quoted). + /// + /// ``` + /// use oxrdfio::RdfFormat; + /// + /// assert_eq!(RdfFormat::NTriples.supports_rdf_star(), true); + /// assert_eq!(RdfFormat::RdfXml.supports_rdf_star(), false); + /// ``` + #[inline] + #[cfg(feature = "rdf-star")] + pub const fn supports_rdf_star(self) -> bool { + matches!( + self, + Self::NTriples | Self::NQuads | Self::Turtle | Self::TriG + ) + } + + /// Looks for a known format from a media type. + /// + /// It supports some media type aliases. + /// For example, "application/xml" is going to return `RdfFormat::RdfXml` even if it is not its canonical media type. + /// + /// Example: + /// ``` + /// use oxrdfio::RdfFormat; + /// + /// assert_eq!(RdfFormat::from_media_type("text/turtle; charset=utf-8"), Some(RdfFormat::Turtle)) + /// ``` + #[inline] + pub fn from_media_type(media_type: &str) -> Option { + const MEDIA_TYPES: [(&str, RdfFormat); 14] = [ + ("application/n-quads", RdfFormat::NQuads), + ("application/n-triples", RdfFormat::NTriples), + ("application/rdf+xml", RdfFormat::RdfXml), + ("application/trig", RdfFormat::TriG), + ("application/turtle", RdfFormat::Turtle), + ("application/xml", RdfFormat::RdfXml), + ("application/x-trig", RdfFormat::TriG), + ("application/x-turtle", RdfFormat::Turtle), + ("text/n3", RdfFormat::N3), + ("text/nquads", RdfFormat::NQuads), + ("text/plain", RdfFormat::NTriples), + ("text/turtle", RdfFormat::Turtle), + ("text/xml", RdfFormat::RdfXml), + ("text/x-nquads", RdfFormat::NQuads), + ]; + let media_type = media_type.split(';').next()?.trim(); + for (candidate_media_type, candidate_id) in MEDIA_TYPES { + if candidate_media_type.eq_ignore_ascii_case(media_type) { + return Some(candidate_id); + } + } + None + } + + /// Looks for a known format from an extension. + /// + /// It supports some aliases. + /// + /// Example: + /// ``` + /// use oxrdfio::RdfFormat; + /// + /// assert_eq!(RdfFormat::from_extension("nt"), Some(RdfFormat::NTriples)) + /// ``` + #[inline] + pub fn from_extension(extension: &str) -> Option { + const MEDIA_TYPES: [(&str, RdfFormat); 8] = [ + ("n3", RdfFormat::N3), + ("nq", RdfFormat::NQuads), + ("nt", RdfFormat::NTriples), + ("rdf", RdfFormat::RdfXml), + ("trig", RdfFormat::TriG), + ("ttl", RdfFormat::Turtle), + ("txt", RdfFormat::NTriples), + ("xml", RdfFormat::RdfXml), + ]; + for (candidate_extension, candidate_id) in MEDIA_TYPES { + if candidate_extension.eq_ignore_ascii_case(extension) { + return Some(candidate_id); + } + } + None + } +} + +impl fmt::Display for RdfFormat { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.name()) + } +} diff --git a/lib/oxrdfio/src/lib.rs b/lib/oxrdfio/src/lib.rs new file mode 100644 index 00000000..d31ec656 --- /dev/null +++ b/lib/oxrdfio/src/lib.rs @@ -0,0 +1,19 @@ +#![doc = include_str!("../README.md")] +#![doc(test(attr(deny(warnings))))] +#![cfg_attr(docsrs, feature(doc_auto_cfg))] +#![doc(html_favicon_url = "https://raw.githubusercontent.com/oxigraph/oxigraph/main/logo.svg")] +#![doc(html_logo_url = "https://raw.githubusercontent.com/oxigraph/oxigraph/main/logo.svg")] + +mod error; +mod format; +mod parser; +mod serializer; + +pub use error::{ParseError, SyntaxError}; +pub use format::RdfFormat; +#[cfg(feature = "async-tokio")] +pub use parser::FromTokioAsyncReadQuadReader; +pub use parser::{FromReadQuadReader, RdfParser}; +#[cfg(feature = "async-tokio")] +pub use serializer::ToTokioAsyncWriteQuadWriter; +pub use serializer::{RdfSerializer, ToWriteQuadWriter}; diff --git a/lib/oxrdfio/src/parser.rs b/lib/oxrdfio/src/parser.rs new file mode 100644 index 00000000..b0e4a419 --- /dev/null +++ b/lib/oxrdfio/src/parser.rs @@ -0,0 +1,577 @@ +//! Utilities to read RDF graphs and datasets. + +pub use crate::error::{ParseError, SyntaxError}; +use crate::format::RdfFormat; +use oxrdf::{BlankNode, GraphName, IriParseError, Quad, Subject, Term, Triple}; +#[cfg(feature = "async-tokio")] +use oxrdfxml::FromTokioAsyncReadRdfXmlReader; +use oxrdfxml::{FromReadRdfXmlReader, RdfXmlParser}; +#[cfg(feature = "async-tokio")] +use oxttl::n3::FromTokioAsyncReadN3Reader; +use oxttl::n3::{FromReadN3Reader, N3Parser, N3Quad, N3Term}; +#[cfg(feature = "async-tokio")] +use oxttl::nquads::FromTokioAsyncReadNQuadsReader; +use oxttl::nquads::{FromReadNQuadsReader, NQuadsParser}; +#[cfg(feature = "async-tokio")] +use oxttl::ntriples::FromTokioAsyncReadNTriplesReader; +use oxttl::ntriples::{FromReadNTriplesReader, NTriplesParser}; +#[cfg(feature = "async-tokio")] +use oxttl::trig::FromTokioAsyncReadTriGReader; +use oxttl::trig::{FromReadTriGReader, TriGParser}; +#[cfg(feature = "async-tokio")] +use oxttl::turtle::FromTokioAsyncReadTurtleReader; +use oxttl::turtle::{FromReadTurtleReader, TurtleParser}; +use std::collections::HashMap; +use std::io::Read; +#[cfg(feature = "async-tokio")] +use tokio::io::AsyncRead; + +/// Parsers for RDF serialization formats. +/// +/// It currently supports the following formats: +/// * [N3](https://w3c.github.io/N3/spec/) ([`RdfFormat::N3`]) +/// * [N-Quads](https://www.w3.org/TR/n-quads/) ([`RdfFormat::NQuads`]) +/// * [N-Triples](https://www.w3.org/TR/n-triples/) ([`RdfFormat::NTriples`]) +/// * [RDF/XML](https://www.w3.org/TR/rdf-syntax-grammar/) ([`RdfFormat::RdfXml`]) +/// * [TriG](https://www.w3.org/TR/trig/) ([`RdfFormat::TriG`]) +/// * [Turtle](https://www.w3.org/TR/turtle/) ([`RdfFormat::Turtle`]) +/// +/// Note the useful options: +/// - [`with_base_iri`](RdfParser::with_base_iri) to resolve the relative IRIs. +/// - [`rename_blank_nodes`](RdfParser::rename_blank_nodes) to rename the blank nodes to auto-generated numbers to avoid conflicts when merging RDF graphs together. +/// - [`without_named_graphs`](RdfParser::without_named_graphs) to parse a single graph. +/// +/// ``` +/// use oxrdfio::{RdfFormat, RdfParser}; +/// +/// let file = " ."; +/// +/// let parser = RdfParser::from_format(RdfFormat::NTriples); +/// let quads = parser.parse_read(file.as_bytes()).collect::,_>>()?; +/// +/// assert_eq!(quads.len(), 1); +/// assert_eq!(quads[0].subject.to_string(), ""); +/// # std::io::Result::Ok(()) +/// ``` +pub struct RdfParser { + inner: RdfParserKind, + default_graph: GraphName, + without_named_graphs: bool, + rename_blank_nodes: bool, +} + +enum RdfParserKind { + N3(N3Parser), + NQuads(NQuadsParser), + NTriples(NTriplesParser), + RdfXml(RdfXmlParser), + TriG(TriGParser), + Turtle(TurtleParser), +} + +impl RdfParser { + /// Builds a parser for the given format. + #[inline] + #[must_use] + pub fn from_format(format: RdfFormat) -> Self { + Self { + inner: match format { + RdfFormat::N3 => RdfParserKind::N3(N3Parser::new()), + RdfFormat::NQuads => RdfParserKind::NQuads({ + #[cfg(feature = "rdf-star")] + { + NQuadsParser::new().with_quoted_triples() + } + #[cfg(not(feature = "rdf-star"))] + { + NQuadsParser::new() + } + }), + RdfFormat::NTriples => RdfParserKind::NTriples({ + #[cfg(feature = "rdf-star")] + { + NTriplesParser::new().with_quoted_triples() + } + #[cfg(not(feature = "rdf-star"))] + { + NTriplesParser::new() + } + }), + RdfFormat::RdfXml => RdfParserKind::RdfXml(RdfXmlParser::new()), + RdfFormat::TriG => RdfParserKind::TriG({ + #[cfg(feature = "rdf-star")] + { + TriGParser::new().with_quoted_triples() + } + #[cfg(not(feature = "rdf-star"))] + { + TriGParser::new() + } + }), + RdfFormat::Turtle => RdfParserKind::Turtle({ + #[cfg(feature = "rdf-star")] + { + TurtleParser::new().with_quoted_triples() + } + #[cfg(not(feature = "rdf-star"))] + { + TurtleParser::new() + } + }), + }, + default_graph: GraphName::DefaultGraph, + without_named_graphs: false, + rename_blank_nodes: false, + } + } + + /// Provides an IRI that could be used to resolve the file relative IRIs. + /// + /// ``` + /// use oxrdfio::{RdfFormat, RdfParser}; + /// + /// let file = "

."; + /// + /// let parser = RdfParser::from_format(RdfFormat::Turtle).with_base_iri("http://example.com")?; + /// let quads = parser.parse_read(file.as_bytes()).collect::,_>>()?; + /// + /// assert_eq!(quads.len(), 1); + /// assert_eq!(quads[0].subject.to_string(), ""); + /// # Result::<_,Box>::Ok(()) + /// ``` + #[inline] + pub fn with_base_iri(self, base_iri: impl Into) -> Result { + Ok(Self { + inner: match self.inner { + RdfParserKind::N3(p) => RdfParserKind::N3(p), + RdfParserKind::NTriples(p) => RdfParserKind::NTriples(p), + RdfParserKind::NQuads(p) => RdfParserKind::NQuads(p), + RdfParserKind::RdfXml(p) => RdfParserKind::RdfXml(p.with_base_iri(base_iri)?), + RdfParserKind::TriG(p) => RdfParserKind::TriG(p.with_base_iri(base_iri)?), + RdfParserKind::Turtle(p) => RdfParserKind::Turtle(p.with_base_iri(base_iri)?), + }, + default_graph: self.default_graph, + without_named_graphs: self.without_named_graphs, + rename_blank_nodes: self.rename_blank_nodes, + }) + } + + /// Provides the name graph name that should replace the default graph in the returned quads. + /// + /// ``` + /// use oxrdf::NamedNode; + /// use oxrdfio::{RdfFormat, RdfParser}; + /// + /// let file = " ."; + /// + /// let parser = RdfParser::from_format(RdfFormat::Turtle).with_default_graph(NamedNode::new("http://example.com/g")?); + /// let quads = parser.parse_read(file.as_bytes()).collect::,_>>()?; + /// + /// assert_eq!(quads.len(), 1); + /// assert_eq!(quads[0].graph_name.to_string(), ""); + /// # Result::<_,Box>::Ok(()) + /// ``` + #[inline] + #[must_use] + pub fn with_default_graph(self, default_graph: impl Into) -> Self { + Self { + inner: self.inner, + default_graph: default_graph.into(), + without_named_graphs: self.without_named_graphs, + rename_blank_nodes: self.rename_blank_nodes, + } + } + + /// Sets that the parser must fail if parsing a named graph. + /// + /// This function restricts the parser to only parse a single [RDF graph](https://www.w3.org/TR/rdf11-concepts/#dfn-rdf-graph) and not an [RDF dataset](https://www.w3.org/TR/rdf11-concepts/#dfn-rdf-dataset). + /// + /// ``` + /// use oxrdfio::{RdfFormat, RdfParser}; + /// + /// let file = " ."; + /// + /// let parser = RdfParser::from_format(RdfFormat::NQuads).without_named_graphs(); + /// assert!(parser.parse_read(file.as_bytes()).next().unwrap().is_err()); + /// ``` + #[inline] + #[must_use] + pub fn without_named_graphs(self) -> Self { + Self { + inner: self.inner, + default_graph: self.default_graph, + without_named_graphs: true, + rename_blank_nodes: self.rename_blank_nodes, + } + } + + /// Renames the blank nodes ids from the ones set in the serialization to random ids. + /// + /// This allows to avoid id conflicts when merging graphs together. + /// + /// ``` + /// use oxrdfio::{RdfFormat, RdfParser}; + /// + /// let file = "_:a ."; + /// + /// let parser = RdfParser::from_format(RdfFormat::NQuads).rename_blank_nodes(); + /// let result1 = parser.parse_read(file.as_bytes()).collect::,_>>()?; + /// let result2 = parser.parse_read(file.as_bytes()).collect::,_>>()?; + /// assert_ne!(result1, result2); + /// # Result::<_,Box>::Ok(()) + /// ``` + #[inline] + #[must_use] + pub fn rename_blank_nodes(self) -> Self { + Self { + inner: self.inner, + default_graph: self.default_graph, + without_named_graphs: self.without_named_graphs, + rename_blank_nodes: true, + } + } + + /// Parses from a [`Read`] implementation and returns an iterator of quads. + /// + /// Reads are buffered. + /// + /// ``` + /// use oxrdfio::{RdfFormat, RdfParser}; + /// + /// let file = " ."; + /// + /// let parser = RdfParser::from_format(RdfFormat::NTriples); + /// let quads = parser.parse_read(file.as_bytes()).collect::,_>>()?; + /// + /// assert_eq!(quads.len(), 1); + /// assert_eq!(quads[0].subject.to_string(), ""); + /// # std::io::Result::Ok(()) + /// ``` + pub fn parse_read(&self, reader: R) -> FromReadQuadReader { + FromReadQuadReader { + parser: match &self.inner { + RdfParserKind::N3(p) => FromReadQuadReaderKind::N3(p.parse_read(reader)), + RdfParserKind::NQuads(p) => FromReadQuadReaderKind::NQuads(p.parse_read(reader)), + RdfParserKind::NTriples(p) => { + FromReadQuadReaderKind::NTriples(p.parse_read(reader)) + } + RdfParserKind::RdfXml(p) => FromReadQuadReaderKind::RdfXml(p.parse_read(reader)), + RdfParserKind::TriG(p) => FromReadQuadReaderKind::TriG(p.parse_read(reader)), + RdfParserKind::Turtle(p) => FromReadQuadReaderKind::Turtle(p.parse_read(reader)), + }, + mapper: QuadMapper { + default_graph: self.default_graph.clone(), + without_named_graphs: self.without_named_graphs, + blank_node_map: self.rename_blank_nodes.then(HashMap::new), + }, + } + } + + /// Parses from a Tokio [`AsyncRead`] implementation and returns an async iterator of quads. + /// + /// Reads are buffered. + /// + /// ``` + /// use oxrdfio::{RdfFormat, RdfParser, ParseError}; + /// + /// #[tokio::main(flavor = "current_thread")] + /// async fn main() -> Result<(), ParseError> { + /// let file = " ."; + /// + /// let parser = RdfParser::from_format(RdfFormat::NTriples); + /// let mut reader = parser.parse_tokio_async_read(file.as_bytes()); + /// if let Some(quad) = reader.next().await { + /// assert_eq!(quad?.subject.to_string(), ""); + /// } + /// Ok(()) + /// } + /// ``` + #[cfg(feature = "async-tokio")] + pub fn parse_tokio_async_read( + &self, + reader: R, + ) -> FromTokioAsyncReadQuadReader { + FromTokioAsyncReadQuadReader { + parser: match &self.inner { + RdfParserKind::N3(p) => { + FromTokioAsyncReadQuadReaderKind::N3(p.parse_tokio_async_read(reader)) + } + RdfParserKind::NQuads(p) => { + FromTokioAsyncReadQuadReaderKind::NQuads(p.parse_tokio_async_read(reader)) + } + RdfParserKind::NTriples(p) => { + FromTokioAsyncReadQuadReaderKind::NTriples(p.parse_tokio_async_read(reader)) + } + RdfParserKind::RdfXml(p) => { + FromTokioAsyncReadQuadReaderKind::RdfXml(p.parse_tokio_async_read(reader)) + } + RdfParserKind::TriG(p) => { + FromTokioAsyncReadQuadReaderKind::TriG(p.parse_tokio_async_read(reader)) + } + RdfParserKind::Turtle(p) => { + FromTokioAsyncReadQuadReaderKind::Turtle(p.parse_tokio_async_read(reader)) + } + }, + mapper: QuadMapper { + default_graph: self.default_graph.clone(), + without_named_graphs: self.without_named_graphs, + blank_node_map: self.rename_blank_nodes.then(HashMap::new), + }, + } + } +} + +/// Parses a RDF file from a [`Read`] implementation. Can be built using [`RdfParser::parse_read`]. +/// +/// Reads are buffered. +/// +/// ``` +/// use oxrdfio::{RdfFormat, RdfParser}; +/// +/// let file = " ."; +/// +/// let parser = RdfParser::from_format(RdfFormat::NTriples); +/// let quads = parser.parse_read(file.as_bytes()).collect::,_>>()?; +/// +/// assert_eq!(quads.len(), 1); +/// assert_eq!(quads[0].subject.to_string(), ""); +/// # std::io::Result::Ok(()) +/// ``` +#[must_use] +pub struct FromReadQuadReader { + parser: FromReadQuadReaderKind, + mapper: QuadMapper, +} + +enum FromReadQuadReaderKind { + N3(FromReadN3Reader), + NQuads(FromReadNQuadsReader), + NTriples(FromReadNTriplesReader), + RdfXml(FromReadRdfXmlReader), + TriG(FromReadTriGReader), + Turtle(FromReadTurtleReader), +} + +impl Iterator for FromReadQuadReader { + type Item = Result; + + fn next(&mut self) -> Option> { + Some(match &mut self.parser { + FromReadQuadReaderKind::N3(parser) => match parser.next()? { + Ok(quad) => self.mapper.map_n3_quad(quad), + Err(e) => Err(e.into()), + }, + FromReadQuadReaderKind::NQuads(parser) => match parser.next()? { + Ok(quad) => self.mapper.map_quad(quad), + Err(e) => Err(e.into()), + }, + FromReadQuadReaderKind::NTriples(parser) => match parser.next()? { + Ok(triple) => Ok(self.mapper.map_triple_to_quad(triple)), + Err(e) => Err(e.into()), + }, + FromReadQuadReaderKind::RdfXml(parser) => match parser.next()? { + Ok(triple) => Ok(self.mapper.map_triple_to_quad(triple)), + Err(e) => Err(e.into()), + }, + FromReadQuadReaderKind::TriG(parser) => match parser.next()? { + Ok(quad) => self.mapper.map_quad(quad), + Err(e) => Err(e.into()), + }, + FromReadQuadReaderKind::Turtle(parser) => match parser.next()? { + Ok(triple) => Ok(self.mapper.map_triple_to_quad(triple)), + Err(e) => Err(e.into()), + }, + }) + } +} + +/// Parses a RDF file from a Tokio [`AsyncRead`] implementation. Can be built using [`RdfParser::parse_tokio_async_read`]. +/// +/// Reads are buffered. +/// +/// ``` +/// use oxrdfio::{RdfFormat, RdfParser, ParseError}; +/// +/// #[tokio::main(flavor = "current_thread")] +/// async fn main() -> Result<(), ParseError> { +/// let file = " ."; +/// +/// let parser = RdfParser::from_format(RdfFormat::NTriples); +/// let mut reader = parser.parse_tokio_async_read(file.as_bytes()); +/// if let Some(quad) = reader.next().await { +/// assert_eq!(quad?.subject.to_string(), ""); +/// } +/// Ok(()) +/// } +/// ``` +#[must_use] +#[cfg(feature = "async-tokio")] +pub struct FromTokioAsyncReadQuadReader { + parser: FromTokioAsyncReadQuadReaderKind, + mapper: QuadMapper, +} + +#[cfg(feature = "async-tokio")] +enum FromTokioAsyncReadQuadReaderKind { + N3(FromTokioAsyncReadN3Reader), + NQuads(FromTokioAsyncReadNQuadsReader), + NTriples(FromTokioAsyncReadNTriplesReader), + RdfXml(FromTokioAsyncReadRdfXmlReader), + TriG(FromTokioAsyncReadTriGReader), + Turtle(FromTokioAsyncReadTurtleReader), +} + +#[cfg(feature = "async-tokio")] +impl FromTokioAsyncReadQuadReader { + pub async fn next(&mut self) -> Option> { + Some(match &mut self.parser { + FromTokioAsyncReadQuadReaderKind::N3(parser) => match parser.next().await? { + Ok(quad) => self.mapper.map_n3_quad(quad), + Err(e) => Err(e.into()), + }, + FromTokioAsyncReadQuadReaderKind::NQuads(parser) => match parser.next().await? { + Ok(quad) => self.mapper.map_quad(quad), + Err(e) => Err(e.into()), + }, + FromTokioAsyncReadQuadReaderKind::NTriples(parser) => match parser.next().await? { + Ok(triple) => Ok(self.mapper.map_triple_to_quad(triple)), + Err(e) => Err(e.into()), + }, + FromTokioAsyncReadQuadReaderKind::RdfXml(parser) => match parser.next().await? { + Ok(triple) => Ok(self.mapper.map_triple_to_quad(triple)), + Err(e) => Err(e.into()), + }, + FromTokioAsyncReadQuadReaderKind::TriG(parser) => match parser.next().await? { + Ok(quad) => self.mapper.map_quad(quad), + Err(e) => Err(e.into()), + }, + FromTokioAsyncReadQuadReaderKind::Turtle(parser) => match parser.next().await? { + Ok(triple) => Ok(self.mapper.map_triple_to_quad(triple)), + Err(e) => Err(e.into()), + }, + }) + } +} + +struct QuadMapper { + default_graph: GraphName, + without_named_graphs: bool, + blank_node_map: Option>, +} + +impl QuadMapper { + fn map_blank_node(&mut self, node: BlankNode) -> BlankNode { + if let Some(blank_node_map) = &mut self.blank_node_map { + blank_node_map + .entry(node) + .or_insert_with(BlankNode::default) + .clone() + } else { + node + } + } + + fn map_subject(&mut self, node: Subject) -> Subject { + match node { + Subject::NamedNode(node) => node.into(), + Subject::BlankNode(node) => self.map_blank_node(node).into(), + #[cfg(feature = "rdf-star")] + Subject::Triple(triple) => self.map_triple(*triple).into(), + } + } + + fn map_term(&mut self, node: Term) -> Term { + match node { + Term::NamedNode(node) => node.into(), + Term::BlankNode(node) => self.map_blank_node(node).into(), + Term::Literal(literal) => literal.into(), + #[cfg(feature = "rdf-star")] + Term::Triple(triple) => self.map_triple(*triple).into(), + } + } + + fn map_triple(&mut self, triple: Triple) -> Triple { + Triple { + subject: self.map_subject(triple.subject), + predicate: triple.predicate, + object: self.map_term(triple.object), + } + } + + fn map_graph_name(&mut self, graph_name: GraphName) -> Result { + match graph_name { + GraphName::NamedNode(node) => { + if self.without_named_graphs { + Err(ParseError::msg("Named graphs are not allowed")) + } else { + Ok(node.into()) + } + } + GraphName::BlankNode(node) => { + if self.without_named_graphs { + Err(ParseError::msg("Named graphs are not allowed")) + } else { + Ok(self.map_blank_node(node).into()) + } + } + GraphName::DefaultGraph => Ok(self.default_graph.clone()), + } + } + + fn map_quad(&mut self, quad: Quad) -> Result { + Ok(Quad { + subject: self.map_subject(quad.subject), + predicate: quad.predicate, + object: self.map_term(quad.object), + graph_name: self.map_graph_name(quad.graph_name)?, + }) + } + + fn map_triple_to_quad(&mut self, triple: Triple) -> Quad { + self.map_triple(triple).in_graph(self.default_graph.clone()) + } + + fn map_n3_quad(&mut self, quad: N3Quad) -> Result { + Ok(Quad { + subject: match quad.subject { + N3Term::NamedNode(s) => Ok(s.into()), + N3Term::BlankNode(s) => Ok(self.map_blank_node(s).into()), + N3Term::Literal(_) => Err(ParseError::msg( + "literals are not allowed in regular RDF subjects", + )), + #[cfg(feature = "rdf-star")] + N3Term::Triple(s) => Ok(self.map_triple(*s).into()), + N3Term::Variable(_) => Err(ParseError::msg( + "variables are not allowed in regular RDF subjects", + )), + }?, + predicate: match quad.predicate { + N3Term::NamedNode(p) => Ok(p), + N3Term::BlankNode(_) => Err(ParseError::msg( + "blank nodes are not allowed in regular RDF predicates", + )), + N3Term::Literal(_) => Err(ParseError::msg( + "literals are not allowed in regular RDF predicates", + )), + #[cfg(feature = "rdf-star")] + N3Term::Triple(_) => Err(ParseError::msg( + "quoted triples are not allowed in regular RDF predicates", + )), + N3Term::Variable(_) => Err(ParseError::msg( + "variables are not allowed in regular RDF predicates", + )), + }?, + object: match quad.object { + N3Term::NamedNode(o) => Ok(o.into()), + N3Term::BlankNode(o) => Ok(self.map_blank_node(o).into()), + N3Term::Literal(o) => Ok(o.into()), + #[cfg(feature = "rdf-star")] + N3Term::Triple(o) => Ok(self.map_triple(*o).into()), + N3Term::Variable(_) => Err(ParseError::msg( + "variables are not allowed in regular RDF objects", + )), + }?, + graph_name: self.map_graph_name(quad.graph_name)?, + }) + } +} diff --git a/lib/oxrdfio/src/serializer.rs b/lib/oxrdfio/src/serializer.rs new file mode 100644 index 00000000..35a9f29e --- /dev/null +++ b/lib/oxrdfio/src/serializer.rs @@ -0,0 +1,322 @@ +//! Utilities to write RDF graphs and datasets. + +use crate::format::RdfFormat; +use oxrdf::{GraphNameRef, QuadRef, TripleRef}; +#[cfg(feature = "async-tokio")] +use oxrdfxml::ToTokioAsyncWriteRdfXmlWriter; +use oxrdfxml::{RdfXmlSerializer, ToWriteRdfXmlWriter}; +#[cfg(feature = "async-tokio")] +use oxttl::nquads::ToTokioAsyncWriteNQuadsWriter; +use oxttl::nquads::{NQuadsSerializer, ToWriteNQuadsWriter}; +#[cfg(feature = "async-tokio")] +use oxttl::ntriples::ToTokioAsyncWriteNTriplesWriter; +use oxttl::ntriples::{NTriplesSerializer, ToWriteNTriplesWriter}; +#[cfg(feature = "async-tokio")] +use oxttl::trig::ToTokioAsyncWriteTriGWriter; +use oxttl::trig::{ToWriteTriGWriter, TriGSerializer}; +#[cfg(feature = "async-tokio")] +use oxttl::turtle::ToTokioAsyncWriteTurtleWriter; +use oxttl::turtle::{ToWriteTurtleWriter, TurtleSerializer}; +use std::io::{self, Write}; +#[cfg(feature = "async-tokio")] +use tokio::io::{AsyncWrite, AsyncWriteExt}; + +/// A serializer for RDF serialization formats. +/// +/// It currently supports the following formats: +/// * [N3](https://w3c.github.io/N3/spec/) ([`RdfFormat::N3`]) +/// * [N-Quads](https://www.w3.org/TR/n-quads/) ([`RdfFormat::NQuads`]) +/// * [N-Triples](https://www.w3.org/TR/n-triples/) ([`RdfFormat::NTriples`]) +/// * [RDF/XML](https://www.w3.org/TR/rdf-syntax-grammar/) ([`RdfFormat::RdfXml`]) +/// * [TriG](https://www.w3.org/TR/trig/) ([`RdfFormat::TriG`]) +/// * [Turtle](https://www.w3.org/TR/turtle/) ([`RdfFormat::Turtle`]) +/// +/// ``` +/// use oxrdfio::{RdfFormat, RdfSerializer}; +/// use oxrdf::{Quad, NamedNode}; +/// +/// let mut buffer = Vec::new(); +/// let mut writer = RdfSerializer::from_format(RdfFormat::NQuads).serialize_to_write(&mut buffer); +/// writer.write_quad(&Quad { +/// subject: NamedNode::new("http://example.com/s")?.into(), +/// predicate: NamedNode::new("http://example.com/p")?, +/// object: NamedNode::new("http://example.com/o")?.into(), +/// graph_name: NamedNode::new("http://example.com/g")?.into() +/// })?; +/// writer.finish()?; +/// +/// assert_eq!(buffer.as_slice(), " .\n".as_bytes()); +/// # Result::<_,Box>::Ok(()) +/// ``` +pub struct RdfSerializer { + format: RdfFormat, +} + +impl RdfSerializer { + /// Builds a serializer for the given format + #[inline] + pub fn from_format(format: RdfFormat) -> Self { + Self { format } + } + + /// Writes to a [`Write`] implementation. + /// + /// Warning: Do not forget to run the [`finish`](ToWriteQuadWriter::finish()) method to properly write the last bytes of the file. + /// + /// Warning: This writer does unbuffered writes. You might want to use [`BufWriter`](io::BufWriter) to avoid that. + /// + /// ``` + /// use oxrdfio::{RdfFormat, RdfSerializer}; + /// use oxrdf::{Quad, NamedNode}; + /// + /// let mut buffer = Vec::new(); + /// let mut writer = RdfSerializer::from_format(RdfFormat::NQuads).serialize_to_write(&mut buffer); + /// writer.write_quad(&Quad { + /// subject: NamedNode::new("http://example.com/s")?.into(), + /// predicate: NamedNode::new("http://example.com/p")?, + /// object: NamedNode::new("http://example.com/o")?.into(), + /// graph_name: NamedNode::new("http://example.com/g")?.into() + /// })?; + /// writer.finish()?; + /// + /// assert_eq!(buffer.as_slice(), " .\n".as_bytes()); + /// # Result::<_,Box>::Ok(()) + /// ``` + pub fn serialize_to_write(&self, writer: W) -> ToWriteQuadWriter { + ToWriteQuadWriter { + formatter: match self.format { + RdfFormat::NQuads => ToWriteQuadWriterKind::NQuads( + NQuadsSerializer::new().serialize_to_write(writer), + ), + RdfFormat::NTriples => ToWriteQuadWriterKind::NTriples( + NTriplesSerializer::new().serialize_to_write(writer), + ), + RdfFormat::RdfXml => ToWriteQuadWriterKind::RdfXml( + RdfXmlSerializer::new().serialize_to_write(writer), + ), + RdfFormat::TriG => { + ToWriteQuadWriterKind::TriG(TriGSerializer::new().serialize_to_write(writer)) + } + RdfFormat::Turtle | RdfFormat::N3 => ToWriteQuadWriterKind::Turtle( + TurtleSerializer::new().serialize_to_write(writer), + ), + }, + } + } + + /// Writes to a Tokio [`AsyncWrite`] implementation. + /// + /// Warning: Do not forget to run the [`finish`](ToTokioAsyncWriteQuadWriter::finish()) method to properly write the last bytes of the file. + /// + /// Warning: This writer does unbuffered writes. You might want to use [`BufWriter`](tokio::io::BufWriter) to avoid that. + /// + /// ``` + /// use oxrdfio::{RdfFormat, RdfSerializer}; + /// use oxrdf::{Quad, NamedNode}; + /// use std::io; + /// + /// #[tokio::main(flavor = "current_thread")] + /// async fn main() -> io::Result<()> { + /// let mut buffer = Vec::new(); + /// let mut writer = RdfSerializer::from_format(RdfFormat::NQuads).serialize_to_tokio_async_write(&mut buffer); + /// writer.write_quad(&Quad { + /// subject: NamedNode::new_unchecked("http://example.com/s").into(), + /// predicate: NamedNode::new_unchecked("http://example.com/p"), + /// object: NamedNode::new_unchecked("http://example.com/o").into(), + /// graph_name: NamedNode::new_unchecked("http://example.com/g").into() + /// }).await?; + /// writer.finish().await?; + /// + /// assert_eq!(buffer.as_slice(), " .\n".as_bytes()); + /// Ok(()) + /// } + /// ``` + #[cfg(feature = "async-tokio")] + pub fn serialize_to_tokio_async_write( + &self, + writer: W, + ) -> ToTokioAsyncWriteQuadWriter { + ToTokioAsyncWriteQuadWriter { + formatter: match self.format { + RdfFormat::NQuads => ToTokioAsyncWriteQuadWriterKind::NQuads( + NQuadsSerializer::new().serialize_to_tokio_async_write(writer), + ), + RdfFormat::NTriples => ToTokioAsyncWriteQuadWriterKind::NTriples( + NTriplesSerializer::new().serialize_to_tokio_async_write(writer), + ), + RdfFormat::RdfXml => ToTokioAsyncWriteQuadWriterKind::RdfXml( + RdfXmlSerializer::new().serialize_to_tokio_async_write(writer), + ), + RdfFormat::TriG => ToTokioAsyncWriteQuadWriterKind::TriG( + TriGSerializer::new().serialize_to_tokio_async_write(writer), + ), + RdfFormat::Turtle | RdfFormat::N3 => ToTokioAsyncWriteQuadWriterKind::Turtle( + TurtleSerializer::new().serialize_to_tokio_async_write(writer), + ), + }, + } + } +} + +/// Writes quads or triples to a [`Write`] implementation. +/// +/// Can be built using [`RdfSerializer::serialize_to_write`]. +/// +/// Warning: Do not forget to run the [`finish`](ToWriteQuadWriter::finish()) method to properly write the last bytes of the file. +/// +/// Warning: This writer does unbuffered writes. You might want to use [`BufWriter`](io::BufWriter) to avoid that. +/// +/// ``` +/// use oxrdfio::{RdfFormat, RdfSerializer}; +/// use oxrdf::{Quad, NamedNode}; +/// +/// let mut buffer = Vec::new(); +/// let mut writer = RdfSerializer::from_format(RdfFormat::NQuads).serialize_to_write(&mut buffer); +/// writer.write_quad(&Quad { +/// subject: NamedNode::new("http://example.com/s")?.into(), +/// predicate: NamedNode::new("http://example.com/p")?, +/// object: NamedNode::new("http://example.com/o")?.into(), +/// graph_name: NamedNode::new("http://example.com/g")?.into(), +/// })?; +/// writer.finish()?; +/// +/// assert_eq!(buffer.as_slice(), " .\n".as_bytes()); +/// # Result::<_,Box>::Ok(()) +/// ``` +#[must_use] +pub struct ToWriteQuadWriter { + formatter: ToWriteQuadWriterKind, +} + +enum ToWriteQuadWriterKind { + NQuads(ToWriteNQuadsWriter), + NTriples(ToWriteNTriplesWriter), + RdfXml(ToWriteRdfXmlWriter), + TriG(ToWriteTriGWriter), + Turtle(ToWriteTurtleWriter), +} + +impl ToWriteQuadWriter { + /// Writes a [`QuadRef`] + pub fn write_quad<'a>(&mut self, quad: impl Into>) -> io::Result<()> { + match &mut self.formatter { + ToWriteQuadWriterKind::NQuads(writer) => writer.write_quad(quad), + ToWriteQuadWriterKind::NTriples(writer) => writer.write_triple(to_triple(quad)?), + ToWriteQuadWriterKind::RdfXml(writer) => writer.write_triple(to_triple(quad)?), + ToWriteQuadWriterKind::TriG(writer) => writer.write_quad(quad), + ToWriteQuadWriterKind::Turtle(writer) => writer.write_triple(to_triple(quad)?), + } + } + + /// Writes a [`TripleRef`] + pub fn write_triple<'a>(&mut self, triple: impl Into>) -> io::Result<()> { + self.write_quad(triple.into().in_graph(GraphNameRef::DefaultGraph)) + } + + /// Writes the last bytes of the file + pub fn finish(self) -> io::Result<()> { + match self.formatter { + ToWriteQuadWriterKind::NQuads(writer) => writer.finish(), + ToWriteQuadWriterKind::NTriples(writer) => writer.finish(), + ToWriteQuadWriterKind::RdfXml(writer) => writer.finish()?, + ToWriteQuadWriterKind::TriG(writer) => writer.finish()?, + ToWriteQuadWriterKind::Turtle(writer) => writer.finish()?, + } + .flush() + } +} + +/// Writes quads or triples to a [`Write`] implementation. +/// +/// Can be built using [`RdfSerializer::serialize_to_write`]. +/// +/// Warning: Do not forget to run the [`finish`](ToWriteQuadWriter::finish()) method to properly write the last bytes of the file. +/// +/// Warning: This writer does unbuffered writes. You might want to use [`BufWriter`](io::BufWriter) to avoid that. +/// +/// ``` +/// use oxrdfio::{RdfFormat, RdfSerializer}; +/// use oxrdf::{Quad, NamedNode}; +/// use std::io; +/// +/// #[tokio::main(flavor = "current_thread")] +/// async fn main() -> io::Result<()> { +/// let mut buffer = Vec::new(); +/// let mut writer = RdfSerializer::from_format(RdfFormat::NQuads).serialize_to_tokio_async_write(&mut buffer); +/// writer.write_quad(&Quad { +/// subject: NamedNode::new_unchecked("http://example.com/s").into(), +/// predicate: NamedNode::new_unchecked("http://example.com/p"), +/// object: NamedNode::new_unchecked("http://example.com/o").into(), +/// graph_name: NamedNode::new_unchecked("http://example.com/g").into() +/// }).await?; +/// writer.finish().await?; +/// +/// assert_eq!(buffer.as_slice(), " .\n".as_bytes()); +/// Ok(()) +/// } +/// ``` +#[must_use] +#[cfg(feature = "async-tokio")] +pub struct ToTokioAsyncWriteQuadWriter { + formatter: ToTokioAsyncWriteQuadWriterKind, +} + +#[cfg(feature = "async-tokio")] +enum ToTokioAsyncWriteQuadWriterKind { + NQuads(ToTokioAsyncWriteNQuadsWriter), + NTriples(ToTokioAsyncWriteNTriplesWriter), + RdfXml(ToTokioAsyncWriteRdfXmlWriter), + TriG(ToTokioAsyncWriteTriGWriter), + Turtle(ToTokioAsyncWriteTurtleWriter), +} + +#[cfg(feature = "async-tokio")] +impl ToTokioAsyncWriteQuadWriter { + /// Writes a [`QuadRef`] + pub async fn write_quad<'a>(&mut self, quad: impl Into>) -> io::Result<()> { + match &mut self.formatter { + ToTokioAsyncWriteQuadWriterKind::NQuads(writer) => writer.write_quad(quad).await, + ToTokioAsyncWriteQuadWriterKind::NTriples(writer) => { + writer.write_triple(to_triple(quad)?).await + } + ToTokioAsyncWriteQuadWriterKind::RdfXml(writer) => { + writer.write_triple(to_triple(quad)?).await + } + ToTokioAsyncWriteQuadWriterKind::TriG(writer) => writer.write_quad(quad).await, + ToTokioAsyncWriteQuadWriterKind::Turtle(writer) => { + writer.write_triple(to_triple(quad)?).await + } + } + } + + /// Writes a [`TripleRef`] + pub async fn write_triple<'a>(&mut self, triple: impl Into>) -> io::Result<()> { + self.write_quad(triple.into().in_graph(GraphNameRef::DefaultGraph)) + .await + } + + /// Writes the last bytes of the file + pub async fn finish(self) -> io::Result<()> { + match self.formatter { + ToTokioAsyncWriteQuadWriterKind::NQuads(writer) => writer.finish(), + ToTokioAsyncWriteQuadWriterKind::NTriples(writer) => writer.finish(), + ToTokioAsyncWriteQuadWriterKind::RdfXml(writer) => writer.finish().await?, + ToTokioAsyncWriteQuadWriterKind::TriG(writer) => writer.finish().await?, + ToTokioAsyncWriteQuadWriterKind::Turtle(writer) => writer.finish().await?, + } + .flush() + .await + } +} + +fn to_triple<'a>(quad: impl Into>) -> io::Result> { + let quad = quad.into(); + if quad.graph_name.is_default_graph() { + Ok(quad.into()) + } else { + Err(io::Error::new( + io::ErrorKind::InvalidInput, + "Only quads in the default graph can be serialized to a RDF graph format", + )) + } +} diff --git a/lib/src/io/error.rs b/lib/src/io/error.rs index bea22d8e..7cbdc8ac 100644 --- a/lib/src/io/error.rs +++ b/lib/src/io/error.rs @@ -43,40 +43,21 @@ impl Error for ParseError { } } -impl From for SyntaxError { +impl From for SyntaxError { #[inline] - fn from(error: oxttl::SyntaxError) -> Self { + fn from(error: oxrdfio::SyntaxError) -> Self { SyntaxError { - inner: SyntaxErrorKind::Turtle(error), + inner: SyntaxErrorKind::IO(error), } } } -impl From for ParseError { +impl From for ParseError { #[inline] - fn from(error: oxttl::ParseError) -> Self { + fn from(error: oxrdfio::ParseError) -> Self { match error { - oxttl::ParseError::Syntax(e) => Self::Syntax(e.into()), - oxttl::ParseError::Io(e) => Self::Io(e), - } - } -} - -impl From for SyntaxError { - #[inline] - fn from(error: oxrdfxml::SyntaxError) -> Self { - SyntaxError { - inner: SyntaxErrorKind::RdfXml(error), - } - } -} - -impl From for ParseError { - #[inline] - fn from(error: oxrdfxml::ParseError) -> Self { - match error { - oxrdfxml::ParseError::Syntax(e) => Self::Syntax(e.into()), - oxrdfxml::ParseError::Io(e) => Self::Io(e), + oxrdfio::ParseError::Syntax(e) => Self::Syntax(e.into()), + oxrdfio::ParseError::Io(e) => Self::Io(e), } } } @@ -113,8 +94,7 @@ pub struct SyntaxError { #[derive(Debug)] enum SyntaxErrorKind { - Turtle(oxttl::SyntaxError), - RdfXml(oxrdfxml::SyntaxError), + IO(oxrdfio::SyntaxError), InvalidBaseIri { iri: String, error: IriParseError }, } @@ -122,8 +102,7 @@ impl fmt::Display for SyntaxError { #[inline] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match &self.inner { - SyntaxErrorKind::Turtle(e) => e.fmt(f), - SyntaxErrorKind::RdfXml(e) => e.fmt(f), + SyntaxErrorKind::IO(e) => e.fmt(f), SyntaxErrorKind::InvalidBaseIri { iri, error } => { write!(f, "Invalid base IRI '{iri}': {error}") } @@ -135,8 +114,7 @@ impl Error for SyntaxError { #[inline] fn source(&self) -> Option<&(dyn Error + 'static)> { match &self.inner { - SyntaxErrorKind::Turtle(e) => Some(e), - SyntaxErrorKind::RdfXml(e) => Some(e), + SyntaxErrorKind::IO(e) => Some(e), SyntaxErrorKind::InvalidBaseIri { .. } => None, } } @@ -146,8 +124,7 @@ impl From for io::Error { #[inline] fn from(error: SyntaxError) -> Self { match error.inner { - SyntaxErrorKind::Turtle(error) => error.into(), - SyntaxErrorKind::RdfXml(error) => error.into(), + SyntaxErrorKind::IO(error) => error.into(), SyntaxErrorKind::InvalidBaseIri { iri, error } => Self::new( io::ErrorKind::InvalidInput, format!("Invalid IRI '{iri}': {error}"), diff --git a/lib/src/io/format.rs b/lib/src/io/format.rs index 7c95ddcb..01e112ac 100644 --- a/lib/src/io/format.rs +++ b/lib/src/io/format.rs @@ -1,3 +1,5 @@ +use oxrdfio::RdfFormat; + /// [RDF graph](https://www.w3.org/TR/rdf11-concepts/#dfn-rdf-graph) serialization formats. /// /// This enumeration is non exhaustive. New formats like JSON-LD will be added in the future. @@ -102,6 +104,16 @@ impl GraphFormat { } } +impl From for RdfFormat { + fn from(format: GraphFormat) -> Self { + match format { + GraphFormat::NTriples => Self::NTriples, + GraphFormat::Turtle => Self::Turtle, + GraphFormat::RdfXml => Self::RdfXml, + } + } +} + /// [RDF dataset](https://www.w3.org/TR/rdf11-concepts/#dfn-rdf-dataset) serialization formats. /// /// This enumeration is non exhaustive. New formats like JSON-LD will be added in the future. @@ -198,6 +210,15 @@ impl DatasetFormat { } } +impl From for RdfFormat { + fn from(format: DatasetFormat) -> Self { + match format { + DatasetFormat::NQuads => Self::NQuads, + DatasetFormat::TriG => Self::TriG, + } + } +} + impl TryFrom for GraphFormat { type Error = (); diff --git a/lib/src/io/mod.rs b/lib/src/io/mod.rs index 2e5269de..a8185918 100644 --- a/lib/src/io/mod.rs +++ b/lib/src/io/mod.rs @@ -8,3 +8,7 @@ pub mod write; pub use self::format::{DatasetFormat, GraphFormat}; pub use self::read::{DatasetParser, GraphParser}; pub use self::write::{DatasetSerializer, GraphSerializer}; +pub use oxrdfio::{ + FromReadQuadReader, ParseError, RdfFormat, RdfParser, RdfSerializer, SyntaxError, + ToWriteQuadWriter, +}; diff --git a/lib/src/io/read.rs b/lib/src/io/read.rs index eddaabed..fe414aa1 100644 --- a/lib/src/io/read.rs +++ b/lib/src/io/read.rs @@ -4,12 +4,7 @@ pub use crate::io::error::{ParseError, SyntaxError}; use crate::io::{DatasetFormat, GraphFormat}; use crate::model::*; use oxiri::IriParseError; -use oxrdfxml::{FromReadRdfXmlReader, RdfXmlParser}; -use oxttl::nquads::{FromReadNQuadsReader, NQuadsParser}; -use oxttl::ntriples::{FromReadNTriplesReader, NTriplesParser}; -use oxttl::trig::{FromReadTriGReader, TriGParser}; -use oxttl::turtle::{FromReadTurtleReader, TurtleParser}; -use std::collections::HashMap; +use oxrdfio::{FromReadQuadReader, RdfParser}; use std::io::Read; /// Parsers for RDF graph serialization formats. @@ -27,18 +22,12 @@ use std::io::Read; /// let parser = GraphParser::from_format(GraphFormat::NTriples); /// let triples = parser.read_triples(file.as_bytes()).collect::,_>>()?; /// -///assert_eq!(triples.len(), 1); -///assert_eq!(triples[0].subject.to_string(), ""); +/// assert_eq!(triples.len(), 1); +/// assert_eq!(triples[0].subject.to_string(), ""); /// # std::io::Result::Ok(()) /// ``` pub struct GraphParser { - inner: GraphParserKind, -} - -enum GraphParserKind { - NTriples(NTriplesParser), - Turtle(TurtleParser), - RdfXml(RdfXmlParser), + inner: RdfParser, } impl GraphParser { @@ -46,15 +35,9 @@ impl GraphParser { #[inline] pub fn from_format(format: GraphFormat) -> Self { Self { - inner: match format { - GraphFormat::NTriples => { - GraphParserKind::NTriples(NTriplesParser::new().with_quoted_triples()) - } - GraphFormat::Turtle => { - GraphParserKind::Turtle(TurtleParser::new().with_quoted_triples()) - } - GraphFormat::RdfXml => GraphParserKind::RdfXml(RdfXmlParser::new()), - }, + inner: RdfParser::from_format(format.into()) + .without_named_graphs() + .rename_blank_nodes(), } } @@ -68,30 +51,21 @@ impl GraphParser { /// let parser = GraphParser::from_format(GraphFormat::Turtle).with_base_iri("http://example.com")?; /// let triples = parser.read_triples(file.as_bytes()).collect::,_>>()?; /// - ///assert_eq!(triples.len(), 1); - ///assert_eq!(triples[0].subject.to_string(), ""); + /// assert_eq!(triples.len(), 1); + /// assert_eq!(triples[0].subject.to_string(), ""); /// # Result::<_,Box>::Ok(()) /// ``` #[inline] pub fn with_base_iri(self, base_iri: impl Into) -> Result { Ok(Self { - inner: match self.inner { - GraphParserKind::NTriples(p) => GraphParserKind::NTriples(p), - GraphParserKind::Turtle(p) => GraphParserKind::Turtle(p.with_base_iri(base_iri)?), - GraphParserKind::RdfXml(p) => GraphParserKind::RdfXml(p.with_base_iri(base_iri)?), - }, + inner: self.inner.with_base_iri(base_iri)?, }) } /// Executes the parsing itself on a [`Read`] implementation and returns an iterator of triples. pub fn read_triples(&self, reader: R) -> TripleReader { TripleReader { - mapper: BlankNodeMapper::default(), - parser: match &self.inner { - GraphParserKind::NTriples(p) => TripleReaderKind::NTriples(p.parse_read(reader)), - GraphParserKind::Turtle(p) => TripleReaderKind::Turtle(p.parse_read(reader)), - GraphParserKind::RdfXml(p) => TripleReaderKind::RdfXml(p.parse_read(reader)), - }, + parser: self.inner.parse_read(reader), } } } @@ -107,41 +81,20 @@ impl GraphParser { /// let parser = GraphParser::from_format(GraphFormat::NTriples); /// let triples = parser.read_triples(file.as_bytes()).collect::,_>>()?; /// -///assert_eq!(triples.len(), 1); -///assert_eq!(triples[0].subject.to_string(), ""); +/// assert_eq!(triples.len(), 1); +/// assert_eq!(triples[0].subject.to_string(), ""); /// # std::io::Result::Ok(()) /// ``` #[must_use] pub struct TripleReader { - mapper: BlankNodeMapper, - parser: TripleReaderKind, -} - -#[allow(clippy::large_enum_variant)] -enum TripleReaderKind { - NTriples(FromReadNTriplesReader), - Turtle(FromReadTurtleReader), - RdfXml(FromReadRdfXmlReader), + parser: FromReadQuadReader, } impl Iterator for TripleReader { type Item = Result; fn next(&mut self) -> Option> { - Some(match &mut self.parser { - TripleReaderKind::NTriples(parser) => match parser.next()? { - Ok(triple) => Ok(self.mapper.triple(triple)), - Err(e) => Err(e.into()), - }, - TripleReaderKind::Turtle(parser) => match parser.next()? { - Ok(triple) => Ok(self.mapper.triple(triple)), - Err(e) => Err(e.into()), - }, - TripleReaderKind::RdfXml(parser) => match parser.next()? { - Ok(triple) => Ok(self.mapper.triple(triple)), - Err(e) => Err(e.into()), - }, - }) + Some(self.parser.next()?.map(Into::into).map_err(Into::into)) } } @@ -159,17 +112,12 @@ impl Iterator for TripleReader { /// let parser = DatasetParser::from_format(DatasetFormat::NQuads); /// let quads = parser.read_quads(file.as_bytes()).collect::,_>>()?; /// -///assert_eq!(quads.len(), 1); -///assert_eq!(quads[0].subject.to_string(), ""); +/// assert_eq!(quads.len(), 1); +/// assert_eq!(quads[0].subject.to_string(), ""); /// # std::io::Result::Ok(()) /// ``` pub struct DatasetParser { - inner: DatasetParserKind, -} - -enum DatasetParserKind { - NQuads(NQuadsParser), - TriG(TriGParser), + inner: RdfParser, } impl DatasetParser { @@ -177,14 +125,7 @@ impl DatasetParser { #[inline] pub fn from_format(format: DatasetFormat) -> Self { Self { - inner: match format { - DatasetFormat::NQuads => { - DatasetParserKind::NQuads(NQuadsParser::new().with_quoted_triples()) - } - DatasetFormat::TriG => { - DatasetParserKind::TriG(TriGParser::new().with_quoted_triples()) - } - }, + inner: RdfParser::from_format(format.into()).rename_blank_nodes(), } } @@ -198,28 +139,21 @@ impl DatasetParser { /// let parser = DatasetParser::from_format(DatasetFormat::TriG).with_base_iri("http://example.com")?; /// let triples = parser.read_quads(file.as_bytes()).collect::,_>>()?; /// - ///assert_eq!(triples.len(), 1); - ///assert_eq!(triples[0].subject.to_string(), ""); + /// assert_eq!(triples.len(), 1); + /// assert_eq!(triples[0].subject.to_string(), ""); /// # Result::<_,Box>::Ok(()) /// ``` #[inline] pub fn with_base_iri(self, base_iri: impl Into) -> Result { Ok(Self { - inner: match self.inner { - DatasetParserKind::NQuads(p) => DatasetParserKind::NQuads(p), - DatasetParserKind::TriG(p) => DatasetParserKind::TriG(p.with_base_iri(base_iri)?), - }, + inner: self.inner.with_base_iri(base_iri)?, }) } /// Executes the parsing itself on a [`Read`] implementation and returns an iterator of quads. pub fn read_quads(&self, reader: R) -> QuadReader { QuadReader { - mapper: BlankNodeMapper::default(), - parser: match &self.inner { - DatasetParserKind::NQuads(p) => QuadReaderKind::NQuads(p.parse_read(reader)), - DatasetParserKind::TriG(p) => QuadReaderKind::TriG(p.parse_read(reader)), - }, + parser: self.inner.parse_read(reader), } } } @@ -235,90 +169,19 @@ impl DatasetParser { /// let parser = DatasetParser::from_format(DatasetFormat::NQuads); /// let quads = parser.read_quads(file.as_bytes()).collect::,_>>()?; /// -///assert_eq!(quads.len(), 1); -///assert_eq!(quads[0].subject.to_string(), ""); +/// assert_eq!(quads.len(), 1); +/// assert_eq!(quads[0].subject.to_string(), ""); /// # std::io::Result::Ok(()) /// ``` #[must_use] pub struct QuadReader { - mapper: BlankNodeMapper, - parser: QuadReaderKind, -} - -enum QuadReaderKind { - NQuads(FromReadNQuadsReader), - TriG(FromReadTriGReader), + parser: FromReadQuadReader, } impl Iterator for QuadReader { type Item = Result; fn next(&mut self) -> Option> { - Some(match &mut self.parser { - QuadReaderKind::NQuads(parser) => match parser.next()? { - Ok(quad) => Ok(self.mapper.quad(quad)), - Err(e) => Err(e.into()), - }, - QuadReaderKind::TriG(parser) => match parser.next()? { - Ok(quad) => Ok(self.mapper.quad(quad)), - Err(e) => Err(e.into()), - }, - }) - } -} - -#[derive(Default)] -struct BlankNodeMapper { - bnode_map: HashMap, -} - -impl BlankNodeMapper { - fn blank_node(&mut self, node: BlankNode) -> BlankNode { - self.bnode_map - .entry(node) - .or_insert_with(BlankNode::default) - .clone() - } - - fn subject(&mut self, node: Subject) -> Subject { - match node { - Subject::NamedNode(node) => node.into(), - Subject::BlankNode(node) => self.blank_node(node).into(), - Subject::Triple(triple) => self.triple(*triple).into(), - } - } - - fn term(&mut self, node: Term) -> Term { - match node { - Term::NamedNode(node) => node.into(), - Term::BlankNode(node) => self.blank_node(node).into(), - Term::Literal(literal) => literal.into(), - Term::Triple(triple) => self.triple(*triple).into(), - } - } - - fn triple(&mut self, triple: Triple) -> Triple { - Triple { - subject: self.subject(triple.subject), - predicate: triple.predicate, - object: self.term(triple.object), - } - } - - fn graph_name(&mut self, graph_name: GraphName) -> GraphName { - match graph_name { - GraphName::NamedNode(node) => node.into(), - GraphName::BlankNode(node) => self.blank_node(node).into(), - GraphName::DefaultGraph => GraphName::DefaultGraph, - } - } - - fn quad(&mut self, quad: Quad) -> Quad { - Quad { - subject: self.subject(quad.subject), - predicate: quad.predicate, - object: self.term(quad.object), - graph_name: self.graph_name(quad.graph_name), - } + Some(self.parser.next()?.map_err(Into::into)) } } diff --git a/lib/src/io/write.rs b/lib/src/io/write.rs index 1661343f..b9373afc 100644 --- a/lib/src/io/write.rs +++ b/lib/src/io/write.rs @@ -2,11 +2,7 @@ use crate::io::{DatasetFormat, GraphFormat}; use crate::model::*; -use oxrdfxml::{RdfXmlSerializer, ToWriteRdfXmlWriter}; -use oxttl::nquads::{NQuadsSerializer, ToWriteNQuadsWriter}; -use oxttl::ntriples::{NTriplesSerializer, ToWriteNTriplesWriter}; -use oxttl::trig::{ToWriteTriGWriter, TriGSerializer}; -use oxttl::turtle::{ToWriteTurtleWriter, TurtleSerializer}; +use oxrdfio::{RdfSerializer, ToWriteQuadWriter}; use std::io::{self, Write}; /// A serializer for RDF graph serialization formats. @@ -29,34 +25,26 @@ use std::io::{self, Write}; /// })?; /// writer.finish()?; /// -///assert_eq!(buffer.as_slice(), " .\n".as_bytes()); +/// assert_eq!(buffer.as_slice(), " .\n".as_bytes()); /// # Result::<_,Box>::Ok(()) /// ``` pub struct GraphSerializer { - format: GraphFormat, + inner: RdfSerializer, } impl GraphSerializer { /// Builds a serializer for the given format #[inline] pub fn from_format(format: GraphFormat) -> Self { - Self { format } + Self { + inner: RdfSerializer::from_format(format.into()), + } } /// Returns a [`TripleWriter`] allowing writing triples into the given [`Write`] implementation pub fn triple_writer(&self, writer: W) -> TripleWriter { TripleWriter { - formatter: match self.format { - GraphFormat::NTriples => { - TripleWriterKind::NTriples(NTriplesSerializer::new().serialize_to_write(writer)) - } - GraphFormat::Turtle => { - TripleWriterKind::Turtle(TurtleSerializer::new().serialize_to_write(writer)) - } - GraphFormat::RdfXml => { - TripleWriterKind::RdfXml(RdfXmlSerializer::new().serialize_to_write(writer)) - } - }, + writer: self.inner.serialize_to_write(writer), } } } @@ -79,37 +67,23 @@ impl GraphSerializer { /// })?; /// writer.finish()?; /// -///assert_eq!(buffer.as_slice(), " .\n".as_bytes()); +/// assert_eq!(buffer.as_slice(), " .\n".as_bytes()); /// # Result::<_,Box>::Ok(()) /// ``` #[must_use] pub struct TripleWriter { - formatter: TripleWriterKind, -} - -enum TripleWriterKind { - NTriples(ToWriteNTriplesWriter), - Turtle(ToWriteTurtleWriter), - RdfXml(ToWriteRdfXmlWriter), + writer: ToWriteQuadWriter, } impl TripleWriter { /// Writes a triple pub fn write<'a>(&mut self, triple: impl Into>) -> io::Result<()> { - match &mut self.formatter { - TripleWriterKind::NTriples(writer) => writer.write_triple(triple), - TripleWriterKind::Turtle(writer) => writer.write_triple(triple), - TripleWriterKind::RdfXml(writer) => writer.write_triple(triple), - } + self.writer.write_triple(triple) } /// Writes the last bytes of the file pub fn finish(self) -> io::Result<()> { - match self.formatter { - TripleWriterKind::NTriples(writer) => writer.finish().flush(), - TripleWriterKind::Turtle(writer) => writer.finish()?.flush(), - TripleWriterKind::RdfXml(formatter) => formatter.finish()?.flush(), - } + self.writer.finish() } } @@ -133,31 +107,26 @@ impl TripleWriter { /// })?; /// writer.finish()?; /// -///assert_eq!(buffer.as_slice(), " .\n".as_bytes()); +/// assert_eq!(buffer.as_slice(), " .\n".as_bytes()); /// # Result::<_,Box>::Ok(()) /// ``` pub struct DatasetSerializer { - format: DatasetFormat, + inner: RdfSerializer, } impl DatasetSerializer { /// Builds a serializer for the given format #[inline] pub fn from_format(format: DatasetFormat) -> Self { - Self { format } + Self { + inner: RdfSerializer::from_format(format.into()), + } } /// Returns a [`QuadWriter`] allowing writing triples into the given [`Write`] implementation pub fn quad_writer(&self, writer: W) -> QuadWriter { QuadWriter { - formatter: match self.format { - DatasetFormat::NQuads => { - QuadWriterKind::NQuads(NQuadsSerializer::new().serialize_to_write(writer)) - } - DatasetFormat::TriG => { - QuadWriterKind::TriG(TriGSerializer::new().serialize_to_write(writer)) - } - }, + writer: self.inner.serialize_to_write(writer), } } } @@ -181,33 +150,22 @@ impl DatasetSerializer { /// })?; /// writer.finish()?; /// -///assert_eq!(buffer.as_slice(), " .\n".as_bytes()); +/// assert_eq!(buffer.as_slice(), " .\n".as_bytes()); /// # Result::<_,Box>::Ok(()) /// ``` #[must_use] pub struct QuadWriter { - formatter: QuadWriterKind, -} - -enum QuadWriterKind { - NQuads(ToWriteNQuadsWriter), - TriG(ToWriteTriGWriter), + writer: ToWriteQuadWriter, } impl QuadWriter { /// Writes a quad pub fn write<'a>(&mut self, quad: impl Into>) -> io::Result<()> { - match &mut self.formatter { - QuadWriterKind::NQuads(writer) => writer.write_quad(quad), - QuadWriterKind::TriG(writer) => writer.write_quad(quad), - } + self.writer.write_quad(quad) } /// Writes the last bytes of the file pub fn finish(self) -> io::Result<()> { - match self.formatter { - QuadWriterKind::NQuads(writer) => writer.finish().flush(), - QuadWriterKind::TriG(writer) => writer.finish()?.flush(), - } + self.writer.finish() } } From 217abaf7ee80b2a960f6e33fee87c4aa31476039 Mon Sep 17 00:00:00 2001 From: Tpt Date: Sat, 12 Aug 2023 16:30:48 +0200 Subject: [PATCH 047/217] Adopt new I/O API for serialization --- js/src/store.rs | 30 +++----- lib/src/io/mod.rs | 1 + lib/src/io/write.rs | 4 + lib/src/sparql/error.rs | 11 ++- lib/src/sparql/model.rs | 13 ++-- lib/src/sparql/update.rs | 21 ++--- lib/src/storage/error.rs | 12 ++- lib/src/store.rs | 24 +++--- lib/tests/store.rs | 10 +-- python/src/io.rs | 45 +++++------ python/src/sparql.rs | 4 +- python/src/store.rs | 36 ++++----- python/tests/test_store.py | 4 +- server/src/main.rs | 154 ++++++++++++++++++++----------------- 14 files changed, 189 insertions(+), 180 deletions(-) diff --git a/js/src/store.rs b/js/src/store.rs index 9fce396d..ef8673bc 100644 --- a/js/src/store.rs +++ b/js/src/store.rs @@ -4,7 +4,7 @@ use crate::format_err; use crate::model::*; use crate::utils::to_err; use js_sys::{Array, Map}; -use oxigraph::io::{DatasetFormat, GraphFormat}; +use oxigraph::io::{DatasetFormat, GraphFormat, RdfFormat}; use oxigraph::model::*; use oxigraph::sparql::QueryResults; use oxigraph::store::Store; @@ -191,34 +191,22 @@ impl JsStore { } pub fn dump(&self, mime_type: &str, from_graph_name: &JsValue) -> Result { + let Some(format) = RdfFormat::from_media_type(mime_type) else { + return Err(format_err!("Not supported MIME type: {mime_type}")); + }; let from_graph_name = if let Some(graph_name) = FROM_JS.with(|c| c.to_optional_term(from_graph_name))? { - Some(graph_name.try_into()?) + Some(GraphName::try_from(graph_name)?) } else { None }; - let mut buffer = Vec::new(); - if let Some(graph_format) = GraphFormat::from_media_type(mime_type) { - self.store - .dump_graph( - &mut buffer, - graph_format, - &from_graph_name.unwrap_or(GraphName::DefaultGraph), - ) - .map_err(to_err)?; - } else if let Some(dataset_format) = DatasetFormat::from_media_type(mime_type) { - if from_graph_name.is_some() { - return Err(format_err!( - "The target graph name parameter is not available for dataset formats" - )); - } - self.store - .dump_dataset(&mut buffer, dataset_format) - .map_err(to_err)?; + if let Some(from_graph_name) = &from_graph_name { + self.store.dump_graph(&mut buffer, format, from_graph_name) } else { - return Err(format_err!("Not supported MIME type: {mime_type}")); + self.store.dump_dataset(&mut buffer, format) } + .map_err(to_err)?; String::from_utf8(buffer).map_err(to_err) } } diff --git a/lib/src/io/mod.rs b/lib/src/io/mod.rs index a8185918..9d91c881 100644 --- a/lib/src/io/mod.rs +++ b/lib/src/io/mod.rs @@ -7,6 +7,7 @@ pub mod write; pub use self::format::{DatasetFormat, GraphFormat}; pub use self::read::{DatasetParser, GraphParser}; +#[allow(deprecated)] pub use self::write::{DatasetSerializer, GraphSerializer}; pub use oxrdfio::{ FromReadQuadReader, ParseError, RdfFormat, RdfParser, RdfSerializer, SyntaxError, diff --git a/lib/src/io/write.rs b/lib/src/io/write.rs index b9373afc..7955f3a3 100644 --- a/lib/src/io/write.rs +++ b/lib/src/io/write.rs @@ -1,3 +1,5 @@ +#![allow(deprecated)] + //! Utilities to write RDF graphs and datasets. use crate::io::{DatasetFormat, GraphFormat}; @@ -28,6 +30,7 @@ use std::io::{self, Write}; /// assert_eq!(buffer.as_slice(), " .\n".as_bytes()); /// # Result::<_,Box>::Ok(()) /// ``` +#[deprecated(note = "Use RdfSerializer instead")] pub struct GraphSerializer { inner: RdfSerializer, } @@ -110,6 +113,7 @@ impl TripleWriter { /// assert_eq!(buffer.as_slice(), " .\n".as_bytes()); /// # Result::<_,Box>::Ok(()) /// ``` +#[deprecated(note = "Use RdfSerializer instead")] pub struct DatasetSerializer { inner: RdfSerializer, } diff --git a/lib/src/sparql/error.rs b/lib/src/sparql/error.rs index 101b0165..62e5d821 100644 --- a/lib/src/sparql/error.rs +++ b/lib/src/sparql/error.rs @@ -1,4 +1,4 @@ -use crate::io::read::ParseError; +use crate::io::{ParseError, SyntaxError}; use crate::storage::StorageError; use std::convert::Infallible; use std::error; @@ -14,10 +14,10 @@ pub enum EvaluationError { /// An error from the storage. Storage(StorageError), /// An error while parsing an external RDF file. - GraphParsing(ParseError), + GraphParsing(SyntaxError), /// An error while parsing an external result file (likely from a federated query). ResultsParsing(sparesults::ParseError), - /// An error returned during store IOs or during results write. + /// An error returned during store or results I/Os. Io(io::Error), /// An error returned during the query evaluation itself (not supported custom function...). Query(QueryError), @@ -132,7 +132,10 @@ impl From for EvaluationError { impl From for EvaluationError { #[inline] fn from(error: ParseError) -> Self { - Self::GraphParsing(error) + match error { + ParseError::Syntax(error) => Self::GraphParsing(error), + ParseError::Io(error) => Self::Io(error), + } } } diff --git a/lib/src/sparql/model.rs b/lib/src/sparql/model.rs index 1aec94a1..aa7c83fe 100644 --- a/lib/src/sparql/model.rs +++ b/lib/src/sparql/model.rs @@ -1,5 +1,4 @@ -use crate::io::GraphFormat; -use crate::io::GraphSerializer; +use crate::io::{RdfFormat, RdfSerializer}; use crate::model::*; use crate::sparql::error::EvaluationError; use oxrdf::{Variable, VariableRef}; @@ -96,7 +95,7 @@ impl QueryResults { /// /// ``` /// use oxigraph::store::Store; - /// use oxigraph::io::GraphFormat; + /// use oxigraph::io::{RdfFormat, GraphFormat}; /// use oxigraph::model::*; /// /// let graph = " .\n"; @@ -105,19 +104,19 @@ impl QueryResults { /// store.load_graph(graph.as_bytes(), GraphFormat::NTriples, GraphNameRef::DefaultGraph, None)?; /// /// let mut results = Vec::new(); - /// store.query("CONSTRUCT WHERE { ?s ?p ?o }")?.write_graph(&mut results, GraphFormat::NTriples)?; + /// store.query("CONSTRUCT WHERE { ?s ?p ?o }")?.write_graph(&mut results, RdfFormat::NTriples)?; /// assert_eq!(results, graph.as_bytes()); /// # Result::<_,Box>::Ok(()) /// ``` pub fn write_graph( self, write: impl Write, - format: GraphFormat, + format: impl Into, ) -> Result<(), EvaluationError> { if let Self::Graph(triples) = self { - let mut writer = GraphSerializer::from_format(format).triple_writer(write); + let mut writer = RdfSerializer::from_format(format.into()).serialize_to_write(write); for triple in triples { - writer.write(&triple?)?; + writer.write_triple(&triple?)?; } writer.finish()?; Ok(()) diff --git a/lib/src/sparql/update.rs b/lib/src/sparql/update.rs index a62f99a8..ef4f9c54 100644 --- a/lib/src/sparql/update.rs +++ b/lib/src/sparql/update.rs @@ -1,5 +1,4 @@ -use crate::io::read::ParseError; -use crate::io::{GraphFormat, GraphParser}; +use crate::io::{RdfFormat, RdfParser}; use crate::model::{GraphName as OxGraphName, GraphNameRef, Quad as OxQuad}; use crate::sparql::algebra::QueryDataset; use crate::sparql::dataset::DatasetView; @@ -166,7 +165,7 @@ impl<'a, 'b: 'a> SimpleUpdateEvaluator<'a, 'b> { from.as_str(), "application/n-triples, text/turtle, application/rdf+xml", )?; - let format = GraphFormat::from_media_type(&content_type).ok_or_else(|| { + let format = RdfFormat::from_media_type(&content_type).ok_or_else(|| { EvaluationError::msg(format!( "Unsupported Content-Type returned by {from}: {content_type}" )) @@ -175,15 +174,17 @@ impl<'a, 'b: 'a> SimpleUpdateEvaluator<'a, 'b> { GraphName::NamedNode(graph_name) => graph_name.into(), GraphName::DefaultGraph => GraphNameRef::DefaultGraph, }; - let mut parser = GraphParser::from_format(format); + let mut parser = RdfParser::from_format(format) + .rename_blank_nodes() + .without_named_graphs() + .with_default_graph(to_graph_name); if let Some(base_iri) = &self.base_iri { - parser = parser - .with_base_iri(base_iri.as_str()) - .map_err(|e| ParseError::invalid_base_iri(base_iri, e))?; + parser = parser.with_base_iri(base_iri.as_str()).map_err(|e| { + EvaluationError::msg(format!("The LOAD IRI '{base_iri}' is invalid: {e}")) + })?; } - for t in parser.read_triples(body) { - self.transaction - .insert(t?.as_ref().in_graph(to_graph_name))?; + for q in parser.parse_read(body) { + self.transaction.insert(q?.as_ref())?; } Ok(()) } diff --git a/lib/src/storage/error.rs b/lib/src/storage/error.rs index f0d5b841..754513eb 100644 --- a/lib/src/storage/error.rs +++ b/lib/src/storage/error.rs @@ -1,4 +1,4 @@ -use crate::io::read::ParseError; +use crate::io::{read::ParseError, RdfFormat}; use std::error::Error; use std::fmt; use std::io; @@ -179,6 +179,8 @@ pub enum SerializerError { Io(io::Error), /// An error raised during the lookup in the store. Storage(StorageError), + /// A format compatible with [RDF dataset](https://www.w3.org/TR/rdf11-concepts/#dfn-rdf-dataset) is required. + DatasetFormatExpected(RdfFormat), } impl fmt::Display for SerializerError { @@ -187,6 +189,10 @@ impl fmt::Display for SerializerError { match self { Self::Io(e) => e.fmt(f), Self::Storage(e) => e.fmt(f), + Self::DatasetFormatExpected(format) => write!( + f, + "A RDF format supporting datasets was expected, {format} found" + ), } } } @@ -197,6 +203,7 @@ impl Error for SerializerError { match self { Self::Io(e) => Some(e), Self::Storage(e) => Some(e), + Self::DatasetFormatExpected(_) => None, } } } @@ -221,6 +228,9 @@ impl From for io::Error { match error { SerializerError::Storage(error) => error.into(), SerializerError::Io(error) => error, + SerializerError::DatasetFormatExpected(_) => { + io::Error::new(io::ErrorKind::InvalidInput, error.to_string()) + } } } } diff --git a/lib/src/store.rs b/lib/src/store.rs index eed5dce2..3923bf80 100644 --- a/lib/src/store.rs +++ b/lib/src/store.rs @@ -24,9 +24,7 @@ //! # Result::<_, Box>::Ok(()) //! ``` use crate::io::read::ParseError; -use crate::io::{ - DatasetFormat, DatasetParser, DatasetSerializer, GraphFormat, GraphParser, GraphSerializer, -}; +use crate::io::{DatasetFormat, DatasetParser, GraphFormat, GraphParser, RdfFormat, RdfSerializer}; use crate::model::*; use crate::sparql::{ evaluate_query, evaluate_update, EvaluationError, Query, QueryExplanation, QueryOptions, @@ -612,13 +610,13 @@ impl Store { /// ``` pub fn dump_graph<'a>( &self, - writer: impl Write, - format: GraphFormat, + write: impl Write, + format: impl Into, from_graph_name: impl Into>, ) -> Result<(), SerializerError> { - let mut writer = GraphSerializer::from_format(format).triple_writer(writer); + let mut writer = RdfSerializer::from_format(format.into()).serialize_to_write(write); for quad in self.quads_for_pattern(None, None, None, Some(from_graph_name.into())) { - writer.write(quad?.as_ref())?; + writer.write_triple(quad?.as_ref())?; } writer.finish()?; Ok(()) @@ -642,12 +640,16 @@ impl Store { /// ``` pub fn dump_dataset( &self, - writer: impl Write, - format: DatasetFormat, + write: impl Write, + format: impl Into, ) -> Result<(), SerializerError> { - let mut writer = DatasetSerializer::from_format(format).quad_writer(writer); + let format = format.into(); + if !format.supports_datasets() { + return Err(SerializerError::DatasetFormatExpected(format)); + } + let mut writer = RdfSerializer::from_format(format).serialize_to_write(write); for quad in self.iter() { - writer.write(&quad?)?; + writer.write_quad(&quad?)?; } writer.finish()?; Ok(()) diff --git a/lib/tests/store.rs b/lib/tests/store.rs index 7328ef4a..750d74c8 100644 --- a/lib/tests/store.rs +++ b/lib/tests/store.rs @@ -1,4 +1,4 @@ -use oxigraph::io::{DatasetFormat, GraphFormat}; +use oxigraph::io::{DatasetFormat, GraphFormat, RdfFormat}; use oxigraph::model::vocab::{rdf, xsd}; use oxigraph::model::*; use oxigraph::store::Store; @@ -211,11 +211,7 @@ fn test_dump_graph() -> Result<(), Box> { } let mut buffer = Vec::new(); - store.dump_graph( - &mut buffer, - GraphFormat::NTriples, - GraphNameRef::DefaultGraph, - )?; + store.dump_graph(&mut buffer, RdfFormat::NTriples, GraphNameRef::DefaultGraph)?; assert_eq!( buffer.into_iter().filter(|c| *c == b'\n').count(), NUMBER_OF_TRIPLES @@ -231,7 +227,7 @@ fn test_dump_dataset() -> Result<(), Box> { } let mut buffer = Vec::new(); - store.dump_dataset(&mut buffer, DatasetFormat::NQuads)?; + store.dump_dataset(&mut buffer, RdfFormat::NQuads)?; assert_eq!( buffer.into_iter().filter(|c| *c == b'\n').count(), NUMBER_OF_TRIPLES diff --git a/python/src/io.rs b/python/src/io.rs index 28245b39..9b6d4075 100644 --- a/python/src/io.rs +++ b/python/src/io.rs @@ -3,8 +3,9 @@ use crate::model::{PyQuad, PyTriple}; use oxigraph::io::read::{ParseError, QuadReader, TripleReader}; use oxigraph::io::{ - DatasetFormat, DatasetParser, DatasetSerializer, GraphFormat, GraphParser, GraphSerializer, + DatasetFormat, DatasetParser, GraphFormat, GraphParser, RdfFormat, RdfSerializer, }; +use oxigraph::model::QuadRef; use pyo3::exceptions::{PyIOError, PySyntaxError, PyValueError}; use pyo3::prelude::*; use pyo3::types::PyBytes; @@ -120,34 +121,34 @@ pub fn parse( /// b' "1" .\n' #[pyfunction] pub fn serialize(input: &PyAny, output: PyObject, mime_type: &str, py: Python<'_>) -> PyResult<()> { + let Some(format) = RdfFormat::from_media_type(mime_type) else { + return Err(PyValueError::new_err(format!( + "Not supported MIME type: {mime_type}" + ))); + }; let output = if let Ok(path) = output.extract::(py) { PyWritable::from_file(&path, py).map_err(map_io_err)? } else { PyWritable::from_data(output) }; - if let Some(graph_format) = GraphFormat::from_media_type(mime_type) { - let mut writer = GraphSerializer::from_format(graph_format).triple_writer(output); - for i in input.iter()? { - writer - .write(&*i?.extract::>()?) - .map_err(map_io_err)?; - } - writer.finish().map_err(map_io_err)?; - Ok(()) - } else if let Some(dataset_format) = DatasetFormat::from_media_type(mime_type) { - let mut writer = DatasetSerializer::from_format(dataset_format).quad_writer(output); - for i in input.iter()? { - writer - .write(&*i?.extract::>()?) - .map_err(map_io_err)?; + let mut writer = RdfSerializer::from_format(format).serialize_to_write(output); + for i in input.iter()? { + let i = i?; + if let Ok(triple) = i.extract::>() { + writer.write_triple(&*triple) + } else { + let quad = i.extract::>()?; + let quad = QuadRef::from(&*quad); + if !quad.graph_name.is_default_graph() && !format.supports_datasets() { + return Err(PyValueError::new_err( + "The {format} format does not support named graphs", + )); + } + writer.write_quad(quad) } - writer.finish().map_err(map_io_err)?; - Ok(()) - } else { - Err(PyValueError::new_err(format!( - "Not supported MIME type: {mime_type}" - ))) + .map_err(map_io_err)?; } + writer.finish().map_err(map_io_err) } #[pyclass(name = "TripleReader", module = "pyoxigraph")] diff --git a/python/src/sparql.rs b/python/src/sparql.rs index 01298fa6..408f6f0c 100644 --- a/python/src/sparql.rs +++ b/python/src/sparql.rs @@ -1,4 +1,4 @@ -use crate::io::{allow_threads_unsafe, map_io_err, map_parse_error}; +use crate::io::{allow_threads_unsafe, map_io_err}; use crate::map_storage_error; use crate::model::*; use oxigraph::model::Term; @@ -234,7 +234,7 @@ pub fn map_evaluation_error(error: EvaluationError) -> PyErr { EvaluationError::Parsing(error) => PySyntaxError::new_err(error.to_string()), EvaluationError::Storage(error) => map_storage_error(error), EvaluationError::Io(error) => map_io_err(error), - EvaluationError::GraphParsing(error) => map_parse_error(error), + EvaluationError::GraphParsing(error) => PySyntaxError::new_err(error.to_string()), EvaluationError::Query(error) => PyValueError::new_err(error.to_string()), _ => PyRuntimeError::new_err(error.to_string()), } diff --git a/python/src/store.rs b/python/src/store.rs index 957c1b5c..692b485f 100644 --- a/python/src/store.rs +++ b/python/src/store.rs @@ -3,7 +3,7 @@ use crate::io::{allow_threads_unsafe, map_io_err, map_parse_error, PyReadable, PyWritable}; use crate::model::*; use crate::sparql::*; -use oxigraph::io::{DatasetFormat, GraphFormat}; +use oxigraph::io::{DatasetFormat, GraphFormat, RdfFormat}; use oxigraph::model::{GraphName, GraphNameRef}; use oxigraph::sparql::Update; use oxigraph::store::{self, LoaderError, SerializerError, StorageError, Store}; @@ -522,10 +522,10 @@ impl PyStore { /// :type output: io(bytes) or str or pathlib.Path /// :param mime_type: the MIME type of the RDF serialization. /// :type mime_type: str - /// :param from_graph: if a triple based format is requested, the store graph from which dump the triples. By default, the default graph is used. + /// :param from_graph: the store graph from which dump the triples. Required if the serialization format does not support named graphs. If it does supports named graphs the full dataset is written. /// :type from_graph: NamedNode or BlankNode or DefaultGraph or None, optional /// :rtype: None - /// :raises ValueError: if the MIME type is not supported or the `from_graph` parameter is given with a quad syntax. + /// :raises ValueError: if the MIME type is not supported or the `from_graph` parameter is not given with a syntax not supporting named graphs. /// :raises IOError: if an I/O error happens during a quad lookup /// /// >>> store = Store() @@ -547,34 +547,23 @@ impl PyStore { } else { PyWritable::from_data(output) }; + let Some(format) = RdfFormat::from_media_type(mime_type) else { + return Err(PyValueError::new_err(format!( + "Not supported MIME type: {mime_type}" + ))); + }; let from_graph_name = if let Some(graph_name) = from_graph { Some(GraphName::from(&PyGraphNameRef::try_from(graph_name)?)) } else { None }; py.allow_threads(|| { - if let Some(graph_format) = GraphFormat::from_media_type(mime_type) { - self.inner - .dump_graph( - output, - graph_format, - &from_graph_name.unwrap_or(GraphName::DefaultGraph), - ) - .map_err(map_serializer_error) - } else if let Some(dataset_format) = DatasetFormat::from_media_type(mime_type) { - if from_graph_name.is_some() { - return Err(PyValueError::new_err( - "The target graph name parameter is not available for dataset formats", - )); - } - self.inner - .dump_dataset(output, dataset_format) - .map_err(map_serializer_error) + if let Some(from_graph_name) = &from_graph_name { + self.inner.dump_graph(output, format, from_graph_name) } else { - Err(PyValueError::new_err(format!( - "Not supported MIME type: {mime_type}" - ))) + self.inner.dump_dataset(output, format) } + .map_err(map_serializer_error) }) } @@ -878,6 +867,7 @@ pub fn map_serializer_error(error: SerializerError) -> PyErr { match error { SerializerError::Storage(error) => map_storage_error(error), SerializerError::Io(error) => PyIOError::new_err(error.to_string()), + SerializerError::DatasetFormatExpected(_) => PyValueError::new_err(error.to_string()), } } diff --git a/python/tests/test_store.py b/python/tests/test_store.py index 287bd27e..ee02e830 100644 --- a/python/tests/test_store.py +++ b/python/tests/test_store.py @@ -321,8 +321,10 @@ class TestStore(unittest.TestCase): ) def test_dump_with_io_error(self) -> None: + store = Store() + store.add(Quad(foo, bar, bar)) with self.assertRaises(OSError) as _, TemporaryFile("rb") as fp: - Store().dump(fp, mime_type="application/rdf+xml") + store.dump(fp, mime_type="application/trig") def test_write_in_read(self) -> None: store = Store() diff --git a/server/src/main.rs b/server/src/main.rs index aed17921..eb9b3116 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -1,10 +1,10 @@ #![allow(clippy::print_stderr, clippy::cast_precision_loss, clippy::use_debug)] -use anyhow::{anyhow, bail, Context, Error}; +use anyhow::{anyhow, bail, ensure, Context, Error}; use clap::{Parser, Subcommand}; use flate2::read::MultiGzDecoder; use oxhttp::model::{Body, HeaderName, HeaderValue, Method, Request, Response, Status}; use oxhttp::Server; -use oxigraph::io::{DatasetFormat, DatasetSerializer, GraphFormat, GraphSerializer}; +use oxigraph::io::{DatasetFormat, GraphFormat, RdfFormat, RdfSerializer}; use oxigraph::model::{ GraphName, GraphNameRef, IriParseError, NamedNode, NamedNodeRef, NamedOrBlankNode, }; @@ -424,9 +424,9 @@ pub fn main() -> anyhow::Result<()> { .ok_or_else(|| anyhow!("The --location argument is required"))?, )?; let format = if let Some(format) = format { - GraphOrDatasetFormat::from_str(&format)? + rdf_format_from_name(&format)? } else if let Some(file) = &file { - GraphOrDatasetFormat::from_path(file)? + rdf_format_from_path(file)? } else { bail!("The --format option must be set when writing to stdout") }; @@ -554,34 +554,25 @@ pub fn main() -> anyhow::Result<()> { } } QueryResults::Graph(triples) => { - let format = if let Some(name) = results_format { - if let Some(format) = GraphFormat::from_extension(&name) { - format - } else if let Some(format) = GraphFormat::from_media_type(&name) { - format - } else { - bail!("The file format '{name}' is unknown") - } + let format = if let Some(name) = &results_format { + rdf_format_from_name(name) } else if let Some(results_file) = &results_file { - format_from_path(results_file, |ext| { - GraphFormat::from_extension(ext) - .ok_or_else(|| anyhow!("The file extension '{ext}' is unknown")) - })? + rdf_format_from_path(results_file) } else { bail!("The --results-format option must be set when writing to stdout") - }; + }?; + let serializer = RdfSerializer::from_format(format); if let Some(results_file) = results_file { - let mut writer = GraphSerializer::from_format(format) - .triple_writer(BufWriter::new(File::create(results_file)?)); + let mut writer = serializer + .serialize_to_write(BufWriter::new(File::create(results_file)?)); for triple in triples { - writer.write(triple?.as_ref())?; + writer.write_triple(triple?.as_ref())?; } writer.finish()?; } else { - let mut writer = - GraphSerializer::from_format(format).triple_writer(stdout().lock()); + let mut writer = serializer.serialize_to_write(stdout().lock()); for triple in triples { - writer.write(triple?.as_ref())?; + writer.write_triple(triple?.as_ref())?; } writer.finish()?; } @@ -670,22 +661,15 @@ fn bulk_load( fn dump( store: &Store, writer: impl Write, - format: GraphOrDatasetFormat, + format: RdfFormat, to_graph_name: Option, ) -> anyhow::Result<()> { - match format { - GraphOrDatasetFormat::Graph(format) => store.dump_graph( - writer, - format, - &to_graph_name.ok_or_else(|| anyhow!("The --graph option is required when writing a graph format like NTriples, Turtle or RDF/XML"))?, - )?, - GraphOrDatasetFormat::Dataset(format) => { - if to_graph_name.is_some() { - bail!("The --graph option is not allowed when writing a dataset format like NQuads or TriG"); - } - store.dump_dataset(writer, format)? - } - } + ensure!(format.supports_datasets() || to_graph_name.is_some(), "The --graph option is required when writing a format not supporting datasets like NTriples, Turtle or RDF/XML"); + if let Some(to_graph_name) = &to_graph_name { + store.dump_graph(writer, format, to_graph_name) + } else { + store.dump_dataset(writer, format) + }?; Ok(()) } @@ -761,6 +745,23 @@ impl FromStr for GraphOrDatasetFormat { } } +fn rdf_format_from_path(path: &Path) -> anyhow::Result { + format_from_path(path, |ext| { + RdfFormat::from_extension(ext) + .ok_or_else(|| anyhow!("The file extension '{ext}' is unknown")) + }) +} + +fn rdf_format_from_name(name: &str) -> anyhow::Result { + if let Some(t) = RdfFormat::from_extension(name) { + return Ok(t); + } + if let Some(t) = RdfFormat::from_media_type(name) { + return Ok(t); + } + bail!("The file format '{name}' is unknown") +} + fn serve(store: Store, bind: String, read_only: bool, cors: bool) -> anyhow::Result<()> { let mut server = if cors { Server::new(cors_middleware(move |request| { @@ -917,8 +918,9 @@ fn handle_request( (path, "GET") if path.starts_with("/store") => { if let Some(target) = store_target(request)? { assert_that_graph_exists(&store, &target)?; - let format = graph_content_negotiation(request)?; - let triples = store.quads_for_pattern( + let format = rdf_content_negotiation(request)?; + + let quads = store.quads_for_pattern( None, None, None, @@ -927,14 +929,14 @@ fn handle_request( ReadForWrite::build_response( move |w| { Ok(( - GraphSerializer::from_format(format).triple_writer(w), - triples, + RdfSerializer::from_format(format).serialize_to_write(w), + quads, )) }, - |(mut writer, mut triples)| { - Ok(if let Some(t) = triples.next() { - writer.write(&t?.into())?; - Some((writer, triples)) + |(mut writer, mut quads)| { + Ok(if let Some(q) = quads.next() { + writer.write_triple(&q?.into())?; + Some((writer, quads)) } else { writer.finish()?; None @@ -943,17 +945,22 @@ fn handle_request( format.media_type(), ) } else { - let format = dataset_content_negotiation(request)?; + let format = rdf_content_negotiation(request)?; + if !format.supports_datasets() { + return Err(bad_request(format!( + "It is not possible to serialize the full RDF dataset using {format} that does not support named graphs" + ))); + } ReadForWrite::build_response( move |w| { Ok(( - DatasetSerializer::from_format(format).quad_writer(w), + RdfSerializer::from_format(format).serialize_to_write(w), store.iter(), )) }, |(mut writer, mut quads)| { Ok(if let Some(q) = quads.next() { - writer.write(&q?)?; + writer.write_quad(&q?)?; Some((writer, quads)) } else { writer.finish()?; @@ -1227,17 +1234,17 @@ fn evaluate_sparql_query( .with_body(body)) } QueryResults::Graph(triples) => { - let format = graph_content_negotiation(request)?; + let format = rdf_content_negotiation(request)?; ReadForWrite::build_response( move |w| { Ok(( - GraphSerializer::from_format(format).triple_writer(w), + RdfSerializer::from_format(format).serialize_to_write(w), triples, )) }, |(mut writer, mut triples)| { Ok(if let Some(t) = triples.next() { - writer.write(&t?)?; + writer.write_triple(&t?)?; Some((writer, triples)) } else { writer.finish()?; @@ -1403,26 +1410,26 @@ impl From for GraphName { } } -fn graph_content_negotiation(request: &Request) -> Result { - content_negotiation( - request, - &[ - GraphFormat::NTriples.media_type(), - GraphFormat::Turtle.media_type(), - GraphFormat::RdfXml.media_type(), - ], - GraphFormat::from_media_type, - ) -} - -fn dataset_content_negotiation(request: &Request) -> Result { +fn rdf_content_negotiation(request: &Request) -> Result { content_negotiation( request, &[ - DatasetFormat::NQuads.media_type(), - DatasetFormat::TriG.media_type(), + "application/n-quads", + "application/n-triples", + "application/rdf+xml", + "application/trig", + "application/turtle", + "application/xml", + "application/x-trig", + "application/x-turtle", + "text/n3", + "text/nquads", + "text/plain", + "text/turtle", + "text/xml", + "text/x-nquads", ], - DatasetFormat::from_media_type, + RdfFormat::from_media_type, ) } @@ -1430,10 +1437,15 @@ fn query_results_content_negotiation(request: &Request) -> Result Date: Sun, 13 Aug 2023 12:24:32 +0200 Subject: [PATCH 048/217] Adopt new I/O API for parsing --- fuzz/fuzz_targets/sparql_eval.rs | 8 +- js/src/store.rs | 52 ++++------ js/test/store.mjs | 2 +- lib/benches/store.rs | 18 +--- lib/src/io/error.rs | 134 ------------------------ lib/src/io/format.rs | 4 + lib/src/io/mod.rs | 3 +- lib/src/io/read.rs | 7 +- lib/src/lib.rs | 1 + lib/src/sparql/model.rs | 4 +- lib/src/storage/error.rs | 10 +- lib/src/store.rs | 162 +++++++++++++++++------------- lib/tests/store.rs | 20 ++-- python/src/io.rs | 86 ++++++---------- python/src/store.rs | 76 ++++++-------- python/tests/test_io.py | 35 ++++++- server/src/main.rs | 156 +++++++++------------------- testsuite/src/files.rs | 30 +++--- testsuite/src/manifest.rs | 4 +- testsuite/src/parser_evaluator.rs | 97 +++++------------- testsuite/src/report.rs | 16 +-- testsuite/src/sparql_evaluator.rs | 25 ++--- 22 files changed, 333 insertions(+), 617 deletions(-) delete mode 100644 lib/src/io/error.rs diff --git a/fuzz/fuzz_targets/sparql_eval.rs b/fuzz/fuzz_targets/sparql_eval.rs index 969bafe1..5b52f4bd 100644 --- a/fuzz/fuzz_targets/sparql_eval.rs +++ b/fuzz/fuzz_targets/sparql_eval.rs @@ -2,7 +2,7 @@ use lazy_static::lazy_static; use libfuzzer_sys::fuzz_target; -use oxigraph::io::DatasetFormat; +use oxigraph::io::RdfFormat; use oxigraph::sparql::{Query, QueryOptions, QueryResults, QuerySolutionIter}; use oxigraph::store::Store; @@ -10,11 +10,7 @@ lazy_static! { static ref STORE: Store = { let store = Store::new().unwrap(); store - .load_dataset( - sparql_smith::DATA_TRIG.as_bytes(), - DatasetFormat::TriG, - None, - ) + .load_dataset(sparql_smith::DATA_TRIG.as_bytes(), RdfFormat::TriG, None) .unwrap(); store }; diff --git a/js/src/store.rs b/js/src/store.rs index ef8673bc..2ec83cd6 100644 --- a/js/src/store.rs +++ b/js/src/store.rs @@ -4,7 +4,7 @@ use crate::format_err; use crate::model::*; use crate::utils::to_err; use js_sys::{Array, Map}; -use oxigraph::io::{DatasetFormat, GraphFormat, RdfFormat}; +use oxigraph::io::RdfFormat; use oxigraph::model::*; use oxigraph::sparql::QueryResults; use oxigraph::store::Store; @@ -148,6 +148,9 @@ impl JsStore { base_iri: &JsValue, to_graph_name: &JsValue, ) -> Result<(), JsValue> { + let Some(format) = RdfFormat::from_media_type(mime_type) else { + return Err(format_err!("Not supported MIME type: {mime_type}")); + }; let base_iri = if base_iri.is_null() || base_iri.is_undefined() { None } else if base_iri.is_string() { @@ -160,49 +163,28 @@ impl JsStore { )); }; - let to_graph_name = - if let Some(graph_name) = FROM_JS.with(|c| c.to_optional_term(to_graph_name))? { - Some(graph_name.try_into()?) - } else { - None - }; - - if let Some(graph_format) = GraphFormat::from_media_type(mime_type) { - self.store - .load_graph( - data.as_bytes(), - graph_format, - &to_graph_name.unwrap_or(GraphName::DefaultGraph), - base_iri.as_deref(), - ) - .map_err(to_err) - } else if let Some(dataset_format) = DatasetFormat::from_media_type(mime_type) { - if to_graph_name.is_some() { - return Err(format_err!( - "The target graph name parameter is not available for dataset formats" - )); - } - self.store - .load_dataset(data.as_bytes(), dataset_format, base_iri.as_deref()) - .map_err(to_err) + if let Some(to_graph_name) = FROM_JS.with(|c| c.to_optional_term(to_graph_name))? { + self.store.load_graph( + data.as_bytes(), + format, + GraphName::try_from(to_graph_name)?, + base_iri.as_deref(), + ) } else { - Err(format_err!("Not supported MIME type: {mime_type}")) + self.store + .load_dataset(data.as_bytes(), format, base_iri.as_deref()) } + .map_err(to_err) } pub fn dump(&self, mime_type: &str, from_graph_name: &JsValue) -> Result { let Some(format) = RdfFormat::from_media_type(mime_type) else { return Err(format_err!("Not supported MIME type: {mime_type}")); }; - let from_graph_name = - if let Some(graph_name) = FROM_JS.with(|c| c.to_optional_term(from_graph_name))? { - Some(GraphName::try_from(graph_name)?) - } else { - None - }; let mut buffer = Vec::new(); - if let Some(from_graph_name) = &from_graph_name { - self.store.dump_graph(&mut buffer, format, from_graph_name) + if let Some(from_graph_name) = FROM_JS.with(|c| c.to_optional_term(from_graph_name))? { + self.store + .dump_graph(&mut buffer, format, &GraphName::try_from(from_graph_name)?) } else { self.store.dump_dataset(&mut buffer, format) } diff --git a/js/test/store.mjs b/js/test/store.mjs index 2317c022..ed50cb40 100644 --- a/js/test/store.mjs +++ b/js/test/store.mjs @@ -186,7 +186,7 @@ describe("Store", function () { it("dump default graph content", function () { const store = new Store([dataModel.quad(ex, ex, ex, ex)]); - assert.strictEqual("", store.dump("application/n-triples")); + assert.strictEqual("", store.dump("application/n-triples", dataModel.defaultGraph())); }); }); }); diff --git a/lib/benches/store.rs b/lib/benches/store.rs index 17779b19..eaaf71e7 100644 --- a/lib/benches/store.rs +++ b/lib/benches/store.rs @@ -1,7 +1,7 @@ use criterion::{criterion_group, criterion_main, Criterion, Throughput}; use oxhttp::model::{Method, Request, Status}; -use oxigraph::io::GraphFormat; -use oxigraph::model::GraphNameRef; +use oxigraph::io::RdfFormat; +use oxigraph::model::{GraphName, GraphNameRef}; use oxigraph::sparql::{Query, QueryResults, Update}; use oxigraph::store::Store; use rand::random; @@ -63,12 +63,7 @@ fn store_load(c: &mut Criterion) { fn do_load(store: &Store, data: &[u8]) { store - .load_graph( - data, - GraphFormat::NTriples, - GraphNameRef::DefaultGraph, - None, - ) + .load_graph(data, RdfFormat::NTriples, GraphName::DefaultGraph, None) .unwrap(); store.optimize().unwrap(); } @@ -76,12 +71,7 @@ fn do_load(store: &Store, data: &[u8]) { fn do_bulk_load(store: &Store, data: &[u8]) { store .bulk_loader() - .load_graph( - data, - GraphFormat::NTriples, - GraphNameRef::DefaultGraph, - None, - ) + .load_graph(data, RdfFormat::NTriples, GraphNameRef::DefaultGraph, None) .unwrap(); store.optimize().unwrap(); } diff --git a/lib/src/io/error.rs b/lib/src/io/error.rs deleted file mode 100644 index 7cbdc8ac..00000000 --- a/lib/src/io/error.rs +++ /dev/null @@ -1,134 +0,0 @@ -use oxiri::IriParseError; -use std::error::Error; -use std::{fmt, io}; - -/// Error returned during RDF format parsing. -#[derive(Debug)] -pub enum ParseError { - /// I/O error during parsing (file not found...). - Io(io::Error), - /// An error in the file syntax. - Syntax(SyntaxError), -} - -impl ParseError { - #[inline] - pub(crate) fn invalid_base_iri(iri: &str, error: IriParseError) -> Self { - Self::Syntax(SyntaxError { - inner: SyntaxErrorKind::InvalidBaseIri { - iri: iri.to_owned(), - error, - }, - }) - } -} - -impl fmt::Display for ParseError { - #[inline] - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Io(e) => e.fmt(f), - Self::Syntax(e) => e.fmt(f), - } - } -} - -impl Error for ParseError { - #[inline] - fn source(&self) -> Option<&(dyn Error + 'static)> { - match self { - Self::Io(e) => Some(e), - Self::Syntax(e) => Some(e), - } - } -} - -impl From for SyntaxError { - #[inline] - fn from(error: oxrdfio::SyntaxError) -> Self { - SyntaxError { - inner: SyntaxErrorKind::IO(error), - } - } -} - -impl From for ParseError { - #[inline] - fn from(error: oxrdfio::ParseError) -> Self { - match error { - oxrdfio::ParseError::Syntax(e) => Self::Syntax(e.into()), - oxrdfio::ParseError::Io(e) => Self::Io(e), - } - } -} - -impl From for ParseError { - #[inline] - fn from(error: io::Error) -> Self { - Self::Io(error) - } -} - -impl From for ParseError { - #[inline] - fn from(error: SyntaxError) -> Self { - Self::Syntax(error) - } -} - -impl From for io::Error { - #[inline] - fn from(error: ParseError) -> Self { - match error { - ParseError::Io(error) => error, - ParseError::Syntax(error) => error.into(), - } - } -} - -/// An error in the syntax of the parsed file. -#[derive(Debug)] -pub struct SyntaxError { - inner: SyntaxErrorKind, -} - -#[derive(Debug)] -enum SyntaxErrorKind { - IO(oxrdfio::SyntaxError), - InvalidBaseIri { iri: String, error: IriParseError }, -} - -impl fmt::Display for SyntaxError { - #[inline] - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match &self.inner { - SyntaxErrorKind::IO(e) => e.fmt(f), - SyntaxErrorKind::InvalidBaseIri { iri, error } => { - write!(f, "Invalid base IRI '{iri}': {error}") - } - } - } -} - -impl Error for SyntaxError { - #[inline] - fn source(&self) -> Option<&(dyn Error + 'static)> { - match &self.inner { - SyntaxErrorKind::IO(e) => Some(e), - SyntaxErrorKind::InvalidBaseIri { .. } => None, - } - } -} - -impl From for io::Error { - #[inline] - fn from(error: SyntaxError) -> Self { - match error.inner { - SyntaxErrorKind::IO(error) => error.into(), - SyntaxErrorKind::InvalidBaseIri { iri, error } => Self::new( - io::ErrorKind::InvalidInput, - format!("Invalid IRI '{iri}': {error}"), - ), - } - } -} diff --git a/lib/src/io/format.rs b/lib/src/io/format.rs index 01e112ac..b100d392 100644 --- a/lib/src/io/format.rs +++ b/lib/src/io/format.rs @@ -1,3 +1,5 @@ +#![allow(deprecated)] + use oxrdfio::RdfFormat; /// [RDF graph](https://www.w3.org/TR/rdf11-concepts/#dfn-rdf-graph) serialization formats. @@ -5,6 +7,7 @@ use oxrdfio::RdfFormat; /// This enumeration is non exhaustive. New formats like JSON-LD will be added in the future. #[derive(Eq, PartialEq, Debug, Clone, Copy, Hash)] #[non_exhaustive] +#[deprecated(note = "Use RdfFormat instead")] pub enum GraphFormat { /// [N-Triples](https://www.w3.org/TR/n-triples/) NTriples, @@ -119,6 +122,7 @@ impl From for RdfFormat { /// This enumeration is non exhaustive. New formats like JSON-LD will be added in the future. #[derive(Eq, PartialEq, Debug, Clone, Copy, Hash)] #[non_exhaustive] +#[deprecated(note = "Use RdfFormat instead")] pub enum DatasetFormat { /// [N-Quads](https://www.w3.org/TR/n-quads/) NQuads, diff --git a/lib/src/io/mod.rs b/lib/src/io/mod.rs index 9d91c881..f183157d 100644 --- a/lib/src/io/mod.rs +++ b/lib/src/io/mod.rs @@ -1,11 +1,12 @@ //! Utilities to read and write RDF graphs and datasets. -mod error; mod format; pub mod read; pub mod write; +#[allow(deprecated)] pub use self::format::{DatasetFormat, GraphFormat}; +#[allow(deprecated)] pub use self::read::{DatasetParser, GraphParser}; #[allow(deprecated)] pub use self::write::{DatasetSerializer, GraphSerializer}; diff --git a/lib/src/io/read.rs b/lib/src/io/read.rs index fe414aa1..3e2f0e2a 100644 --- a/lib/src/io/read.rs +++ b/lib/src/io/read.rs @@ -1,10 +1,11 @@ +#![allow(deprecated)] + //! Utilities to read RDF graphs and datasets. -pub use crate::io::error::{ParseError, SyntaxError}; use crate::io::{DatasetFormat, GraphFormat}; use crate::model::*; use oxiri::IriParseError; -use oxrdfio::{FromReadQuadReader, RdfParser}; +use oxrdfio::{FromReadQuadReader, ParseError, RdfParser}; use std::io::Read; /// Parsers for RDF graph serialization formats. @@ -26,6 +27,7 @@ use std::io::Read; /// assert_eq!(triples[0].subject.to_string(), ""); /// # std::io::Result::Ok(()) /// ``` +#[deprecated(note = "Use RdfParser instead")] pub struct GraphParser { inner: RdfParser, } @@ -116,6 +118,7 @@ impl Iterator for TripleReader { /// assert_eq!(quads[0].subject.to_string(), ""); /// # std::io::Result::Ok(()) /// ``` +#[deprecated(note = "Use RdfParser instead")] pub struct DatasetParser { inner: RdfParser, } diff --git a/lib/src/lib.rs b/lib/src/lib.rs index 1ccbe197..e5a680d8 100644 --- a/lib/src/lib.rs +++ b/lib/src/lib.rs @@ -1,5 +1,6 @@ #![doc = include_str!("../README.md")] #![doc(test(attr(deny(warnings))))] +#![doc(test(attr(allow(deprecated))))] #![cfg_attr(docsrs, feature(doc_auto_cfg))] #![doc(html_favicon_url = "https://raw.githubusercontent.com/oxigraph/oxigraph/main/logo.svg")] #![doc(html_logo_url = "https://raw.githubusercontent.com/oxigraph/oxigraph/main/logo.svg")] diff --git a/lib/src/sparql/model.rs b/lib/src/sparql/model.rs index aa7c83fe..1cd64158 100644 --- a/lib/src/sparql/model.rs +++ b/lib/src/sparql/model.rs @@ -95,13 +95,13 @@ impl QueryResults { /// /// ``` /// use oxigraph::store::Store; - /// use oxigraph::io::{RdfFormat, GraphFormat}; + /// use oxigraph::io::RdfFormat; /// use oxigraph::model::*; /// /// let graph = " .\n"; /// /// let store = Store::new()?; - /// store.load_graph(graph.as_bytes(), GraphFormat::NTriples, GraphNameRef::DefaultGraph, None)?; + /// store.load_graph(graph.as_bytes(), RdfFormat::NTriples, GraphName::DefaultGraph, None)?; /// /// let mut results = Vec::new(); /// store.query("CONSTRUCT WHERE { ?s ?p ?o }")?.write_graph(&mut results, RdfFormat::NTriples)?; diff --git a/lib/src/storage/error.rs b/lib/src/storage/error.rs index 754513eb..690b6bfd 100644 --- a/lib/src/storage/error.rs +++ b/lib/src/storage/error.rs @@ -1,4 +1,5 @@ -use crate::io::{read::ParseError, RdfFormat}; +use crate::io::{ParseError, RdfFormat}; +use oxiri::IriParseError; use std::error::Error; use std::fmt; use std::io; @@ -126,6 +127,8 @@ pub enum LoaderError { Parsing(ParseError), /// An error raised during the insertion in the store. Storage(StorageError), + /// The base IRI is invalid. + InvalidBaseIri { iri: String, error: IriParseError }, } impl fmt::Display for LoaderError { @@ -134,6 +137,7 @@ impl fmt::Display for LoaderError { match self { Self::Parsing(e) => e.fmt(f), Self::Storage(e) => e.fmt(f), + Self::InvalidBaseIri { iri, error } => write!(f, "Invalid base IRI '{iri}': {error}"), } } } @@ -144,6 +148,7 @@ impl Error for LoaderError { match self { Self::Parsing(e) => Some(e), Self::Storage(e) => Some(e), + Self::InvalidBaseIri { error, .. } => Some(error), } } } @@ -168,6 +173,9 @@ impl From for io::Error { match error { LoaderError::Storage(error) => error.into(), LoaderError::Parsing(error) => error.into(), + LoaderError::InvalidBaseIri { .. } => { + io::Error::new(io::ErrorKind::InvalidInput, error.to_string()) + } } } } diff --git a/lib/src/store.rs b/lib/src/store.rs index 3923bf80..0667f019 100644 --- a/lib/src/store.rs +++ b/lib/src/store.rs @@ -23,8 +23,9 @@ //! }; //! # Result::<_, Box>::Ok(()) //! ``` -use crate::io::read::ParseError; -use crate::io::{DatasetFormat, DatasetParser, GraphFormat, GraphParser, RdfFormat, RdfSerializer}; +#[cfg(not(target_family = "wasm"))] +use crate::io::ParseError; +use crate::io::{RdfFormat, RdfParser, RdfSerializer}; use crate::model::*; use crate::sparql::{ evaluate_query, evaluate_update, EvaluationError, Query, QueryExplanation, QueryOptions, @@ -451,38 +452,43 @@ impl Store { /// Usage example: /// ``` /// use oxigraph::store::Store; - /// use oxigraph::io::GraphFormat; + /// use oxigraph::io::RdfFormat; /// use oxigraph::model::*; /// /// let store = Store::new()?; /// /// // insertion /// let file = b" ."; - /// store.load_graph(file.as_ref(), GraphFormat::NTriples, GraphNameRef::DefaultGraph, None)?; + /// store.load_graph(file.as_ref(), RdfFormat::NTriples, GraphName::DefaultGraph, None)?; /// /// // we inspect the store contents /// let ex = NamedNodeRef::new("http://example.com")?; /// assert!(store.contains(QuadRef::new(ex, ex, ex, GraphNameRef::DefaultGraph))?); /// # Result::<_, Box>::Ok(()) /// ``` - pub fn load_graph<'a>( + pub fn load_graph( &self, - reader: impl Read, - format: GraphFormat, - to_graph_name: impl Into>, + read: impl Read, + format: impl Into, + to_graph_name: impl Into, base_iri: Option<&str>, ) -> Result<(), LoaderError> { - let mut parser = GraphParser::from_format(format); + let mut parser = RdfParser::from_format(format.into()) + .without_named_graphs() + .with_default_graph(to_graph_name) + .rename_blank_nodes(); if let Some(base_iri) = base_iri { parser = parser .with_base_iri(base_iri) - .map_err(|e| ParseError::invalid_base_iri(base_iri, e))?; + .map_err(|e| LoaderError::InvalidBaseIri { + iri: base_iri.into(), + error: e, + })?; } - let quads = parser.read_triples(reader).collect::, _>>()?; - let to_graph_name = to_graph_name.into(); + let quads = parser.parse_read(read).collect::, _>>()?; self.storage.transaction(move |mut t| { for quad in &quads { - t.insert(quad.as_ref().in_graph(to_graph_name))?; + t.insert(quad.as_ref())?; } Ok(()) }) @@ -495,14 +501,14 @@ impl Store { /// Usage example: /// ``` /// use oxigraph::store::Store; - /// use oxigraph::io::DatasetFormat; + /// use oxigraph::io::RdfFormat; /// use oxigraph::model::*; /// /// let store = Store::new()?; /// /// // insertion /// let file = b" ."; - /// store.load_dataset(file.as_ref(), DatasetFormat::NQuads, None)?; + /// store.load_dataset(file.as_ref(), RdfFormat::NQuads, None)?; /// /// // we inspect the store contents /// let ex = NamedNodeRef::new("http://example.com")?; @@ -511,20 +517,23 @@ impl Store { /// ``` pub fn load_dataset( &self, - reader: impl Read, - format: DatasetFormat, + read: impl Read, + format: impl Into, base_iri: Option<&str>, ) -> Result<(), LoaderError> { - let mut parser = DatasetParser::from_format(format); + let mut parser = RdfParser::from_format(format.into()).rename_blank_nodes(); if let Some(base_iri) = base_iri { parser = parser .with_base_iri(base_iri) - .map_err(|e| ParseError::invalid_base_iri(base_iri, e))?; + .map_err(|e| LoaderError::InvalidBaseIri { + iri: base_iri.into(), + error: e, + })?; } - let quads = parser.read_quads(reader).collect::, _>>()?; + let quads = parser.parse_read(read).collect::, _>>()?; self.storage.transaction(move |mut t| { for quad in &quads { - t.insert(quad.into())?; + t.insert(quad.as_ref())?; } Ok(()) }) @@ -595,16 +604,16 @@ impl Store { /// Usage example: /// ``` /// use oxigraph::store::Store; - /// use oxigraph::io::GraphFormat; - /// use oxigraph::model::GraphNameRef; + /// use oxigraph::io::RdfFormat; + /// use oxigraph::model::*; /// /// let file = " .\n".as_bytes(); /// /// let store = Store::new()?; - /// store.load_graph(file, GraphFormat::NTriples, GraphNameRef::DefaultGraph, None)?; + /// store.load_graph(file, RdfFormat::NTriples, GraphName::DefaultGraph, None)?; /// /// let mut buffer = Vec::new(); - /// store.dump_graph(&mut buffer, GraphFormat::NTriples, GraphNameRef::DefaultGraph)?; + /// store.dump_graph(&mut buffer, RdfFormat::NTriples, GraphNameRef::DefaultGraph)?; /// assert_eq!(file, buffer.as_slice()); /// # std::io::Result::Ok(()) /// ``` @@ -626,15 +635,15 @@ impl Store { /// /// ``` /// use oxigraph::store::Store; - /// use oxigraph::io::DatasetFormat; + /// use oxigraph::io::RdfFormat; /// /// let file = " .\n".as_bytes(); /// /// let store = Store::new()?; - /// store.load_dataset(file, DatasetFormat::NQuads, None)?; + /// store.load_dataset(file, RdfFormat::NQuads, None)?; /// /// let mut buffer = Vec::new(); - /// store.dump_dataset(&mut buffer, DatasetFormat::NQuads)?; + /// store.dump_dataset(&mut buffer, RdfFormat::NQuads)?; /// assert_eq!(file, buffer.as_slice()); /// # std::io::Result::Ok(()) /// ``` @@ -841,14 +850,14 @@ impl Store { /// Usage example: /// ``` /// use oxigraph::store::Store; - /// use oxigraph::io::DatasetFormat; + /// use oxigraph::io::RdfFormat; /// use oxigraph::model::*; /// /// let store = Store::new()?; /// /// // quads file insertion /// let file = b" ."; - /// store.bulk_loader().load_dataset(file.as_ref(), DatasetFormat::NQuads, None)?; + /// store.bulk_loader().load_dataset(file.as_ref(), RdfFormat::NQuads, None)?; /// /// // we inspect the store contents /// let ex = NamedNodeRef::new("http://example.com")?; @@ -1061,7 +1070,7 @@ impl<'a> Transaction<'a> { /// Usage example: /// ``` /// use oxigraph::store::Store; - /// use oxigraph::io::GraphFormat; + /// use oxigraph::io::RdfFormat; /// use oxigraph::model::*; /// /// let store = Store::new()?; @@ -1069,7 +1078,7 @@ impl<'a> Transaction<'a> { /// // insertion /// let file = b" ."; /// store.transaction(|mut transaction| { - /// transaction.load_graph(file.as_ref(), GraphFormat::NTriples, GraphNameRef::DefaultGraph, None) + /// transaction.load_graph(file.as_ref(), RdfFormat::NTriples, GraphName::DefaultGraph, None) /// })?; /// /// // we inspect the store contents @@ -1077,23 +1086,27 @@ impl<'a> Transaction<'a> { /// assert!(store.contains(QuadRef::new(ex, ex, ex, GraphNameRef::DefaultGraph))?); /// # Result::<_,oxigraph::store::LoaderError>::Ok(()) /// ``` - pub fn load_graph<'b>( + pub fn load_graph( &mut self, - reader: impl Read, - format: GraphFormat, - to_graph_name: impl Into>, + read: impl Read, + format: impl Into, + to_graph_name: impl Into, base_iri: Option<&str>, ) -> Result<(), LoaderError> { - let mut parser = GraphParser::from_format(format); + let mut parser = RdfParser::from_format(format.into()) + .without_named_graphs() + .with_default_graph(to_graph_name) + .rename_blank_nodes(); if let Some(base_iri) = base_iri { parser = parser .with_base_iri(base_iri) - .map_err(|e| ParseError::invalid_base_iri(base_iri, e))?; + .map_err(|e| LoaderError::InvalidBaseIri { + iri: base_iri.into(), + error: e, + })?; } - let to_graph_name = to_graph_name.into(); - for triple in parser.read_triples(reader) { - self.writer - .insert(triple?.as_ref().in_graph(to_graph_name))?; + for quad in parser.parse_read(read) { + self.writer.insert(quad?.as_ref())?; } Ok(()) } @@ -1103,7 +1116,7 @@ impl<'a> Transaction<'a> { /// Usage example: /// ``` /// use oxigraph::store::Store; - /// use oxigraph::io::DatasetFormat; + /// use oxigraph::io::RdfFormat; /// use oxigraph::model::*; /// /// let store = Store::new()?; @@ -1111,7 +1124,7 @@ impl<'a> Transaction<'a> { /// // insertion /// let file = b" ."; /// store.transaction(|mut transaction| { - /// transaction.load_dataset(file.as_ref(), DatasetFormat::NQuads, None) + /// transaction.load_dataset(file.as_ref(), RdfFormat::NQuads, None) /// })?; /// /// // we inspect the store contents @@ -1121,17 +1134,20 @@ impl<'a> Transaction<'a> { /// ``` pub fn load_dataset( &mut self, - reader: impl Read, - format: DatasetFormat, + read: impl Read, + format: impl Into, base_iri: Option<&str>, ) -> Result<(), LoaderError> { - let mut parser = DatasetParser::from_format(format); + let mut parser = RdfParser::from_format(format.into()).rename_blank_nodes(); if let Some(base_iri) = base_iri { parser = parser .with_base_iri(base_iri) - .map_err(|e| ParseError::invalid_base_iri(base_iri, e))?; + .map_err(|e| LoaderError::InvalidBaseIri { + iri: base_iri.into(), + error: e, + })?; } - for quad in parser.read_quads(reader) { + for quad in parser.parse_read(read) { self.writer.insert(quad?.as_ref())?; } Ok(()) @@ -1365,14 +1381,14 @@ impl Iterator for GraphNameIter { /// Usage example with loading a dataset: /// ``` /// use oxigraph::store::Store; -/// use oxigraph::io::DatasetFormat; +/// use oxigraph::io::RdfFormat; /// use oxigraph::model::*; /// /// let store = Store::new()?; /// /// // quads file insertion /// let file = b" ."; -/// store.bulk_loader().load_dataset(file.as_ref(), DatasetFormat::NQuads, None)?; +/// store.bulk_loader().load_dataset(file.as_ref(), RdfFormat::NQuads, None)?; /// /// // we inspect the store contents /// let ex = NamedNodeRef::new("http://example.com")?; @@ -1448,14 +1464,14 @@ impl BulkLoader { /// Usage example: /// ``` /// use oxigraph::store::Store; - /// use oxigraph::io::DatasetFormat; + /// use oxigraph::io::RdfFormat; /// use oxigraph::model::*; /// /// let store = Store::new()?; /// /// // insertion /// let file = b" ."; - /// store.bulk_loader().load_dataset(file.as_ref(), DatasetFormat::NQuads, None)?; + /// store.bulk_loader().load_dataset(file.as_ref(), RdfFormat::NQuads, None)?; /// /// // we inspect the store contents /// let ex = NamedNodeRef::new("http://example.com")?; @@ -1464,17 +1480,20 @@ impl BulkLoader { /// ``` pub fn load_dataset( &self, - reader: impl Read, - format: DatasetFormat, + read: impl Read, + format: impl Into, base_iri: Option<&str>, ) -> Result<(), LoaderError> { - let mut parser = DatasetParser::from_format(format); + let mut parser = RdfParser::from_format(format.into()).rename_blank_nodes(); if let Some(base_iri) = base_iri { parser = parser .with_base_iri(base_iri) - .map_err(|e| ParseError::invalid_base_iri(base_iri, e))?; + .map_err(|e| LoaderError::InvalidBaseIri { + iri: base_iri.into(), + error: e, + })?; } - self.load_ok_quads(parser.read_quads(reader).filter_map(|r| match r { + self.load_ok_quads(parser.parse_read(read).filter_map(|r| match r { Ok(q) => Some(Ok(q)), Err(e) => { if let Some(callback) = &self.on_parse_error { @@ -1503,36 +1522,41 @@ impl BulkLoader { /// Usage example: /// ``` /// use oxigraph::store::Store; - /// use oxigraph::io::GraphFormat; + /// use oxigraph::io::RdfFormat; /// use oxigraph::model::*; /// /// let store = Store::new()?; /// /// // insertion /// let file = b" ."; - /// store.bulk_loader().load_graph(file.as_ref(), GraphFormat::NTriples, GraphNameRef::DefaultGraph, None)?; + /// store.bulk_loader().load_graph(file.as_ref(), RdfFormat::NTriples, GraphName::DefaultGraph, None)?; /// /// // we inspect the store contents /// let ex = NamedNodeRef::new("http://example.com")?; /// assert!(store.contains(QuadRef::new(ex, ex, ex, GraphNameRef::DefaultGraph))?); /// # Result::<_, Box>::Ok(()) /// ``` - pub fn load_graph<'a>( + pub fn load_graph( &self, - reader: impl Read, - format: GraphFormat, - to_graph_name: impl Into>, + read: impl Read, + format: impl Into, + to_graph_name: impl Into, base_iri: Option<&str>, ) -> Result<(), LoaderError> { - let mut parser = GraphParser::from_format(format); + let mut parser = RdfParser::from_format(format.into()) + .without_named_graphs() + .with_default_graph(to_graph_name) + .rename_blank_nodes(); if let Some(base_iri) = base_iri { parser = parser .with_base_iri(base_iri) - .map_err(|e| ParseError::invalid_base_iri(base_iri, e))?; + .map_err(|e| LoaderError::InvalidBaseIri { + iri: base_iri.into(), + error: e, + })?; } - let to_graph_name = to_graph_name.into(); - self.load_ok_quads(parser.read_triples(reader).filter_map(|r| match r { - Ok(q) => Some(Ok(q.in_graph(to_graph_name.into_owned()))), + self.load_ok_quads(parser.parse_read(read).filter_map(|r| match r { + Ok(q) => Some(Ok(q)), Err(e) => { if let Some(callback) = &self.on_parse_error { if let Err(e) = callback(e) { diff --git a/lib/tests/store.rs b/lib/tests/store.rs index 750d74c8..1b28a03d 100644 --- a/lib/tests/store.rs +++ b/lib/tests/store.rs @@ -1,4 +1,4 @@ -use oxigraph::io::{DatasetFormat, GraphFormat, RdfFormat}; +use oxigraph::io::RdfFormat; use oxigraph::model::vocab::{rdf, xsd}; use oxigraph::model::*; use oxigraph::store::Store; @@ -109,7 +109,7 @@ fn test_load_graph() -> Result<(), Box> { let store = Store::new()?; store.load_graph( DATA.as_bytes(), - GraphFormat::Turtle, + RdfFormat::Turtle, GraphNameRef::DefaultGraph, None, )?; @@ -126,8 +126,8 @@ fn test_bulk_load_graph() -> Result<(), Box> { let store = Store::new()?; store.bulk_loader().load_graph( DATA.as_bytes(), - GraphFormat::Turtle, - GraphNameRef::DefaultGraph, + RdfFormat::Turtle, + GraphName::DefaultGraph, None, )?; for q in quads(GraphNameRef::DefaultGraph) { @@ -143,8 +143,8 @@ fn test_bulk_load_graph_lenient() -> Result<(), Box> { let store = Store::new()?; store.bulk_loader().on_parse_error(|_| Ok(())).load_graph( b" .\n .".as_slice(), - GraphFormat::NTriples, - GraphNameRef::DefaultGraph, + RdfFormat::NTriples, + GraphName::DefaultGraph, None, )?; assert_eq!(store.len()?, 1); @@ -161,7 +161,7 @@ fn test_bulk_load_graph_lenient() -> Result<(), Box> { #[test] fn test_load_dataset() -> Result<(), Box> { let store = Store::new()?; - store.load_dataset(GRAPH_DATA.as_bytes(), DatasetFormat::TriG, None)?; + store.load_dataset(GRAPH_DATA.as_bytes(), RdfFormat::TriG, None)?; for q in quads(NamedNodeRef::new_unchecked( "http://www.wikidata.org/wiki/Special:EntityData/Q90", )) { @@ -177,7 +177,7 @@ fn test_bulk_load_dataset() -> Result<(), Box> { let store = Store::new()?; store .bulk_loader() - .load_dataset(GRAPH_DATA.as_bytes(), DatasetFormat::TriG, None)?; + .load_dataset(GRAPH_DATA.as_bytes(), RdfFormat::TriG, None)?; let graph_name = NamedNodeRef::new_unchecked("http://www.wikidata.org/wiki/Special:EntityData/Q90"); for q in quads(graph_name) { @@ -194,8 +194,8 @@ fn test_load_graph_generates_new_blank_nodes() -> Result<(), Box> { for _ in 0..2 { store.load_graph( "_:a .".as_bytes(), - GraphFormat::NTriples, - GraphNameRef::DefaultGraph, + RdfFormat::NTriples, + GraphName::DefaultGraph, None, )?; } diff --git a/python/src/io.rs b/python/src/io.rs index 9b6d4075..f6661474 100644 --- a/python/src/io.rs +++ b/python/src/io.rs @@ -1,10 +1,7 @@ #![allow(clippy::needless_option_as_deref)] use crate::model::{PyQuad, PyTriple}; -use oxigraph::io::read::{ParseError, QuadReader, TripleReader}; -use oxigraph::io::{ - DatasetFormat, DatasetParser, GraphFormat, GraphParser, RdfFormat, RdfSerializer, -}; +use oxigraph::io::{FromReadQuadReader, ParseError, RdfFormat, RdfParser, RdfSerializer}; use oxigraph::model::QuadRef; use pyo3::exceptions::{PyIOError, PySyntaxError, PyValueError}; use pyo3::prelude::*; @@ -41,54 +38,54 @@ pub fn add_to_module(module: &PyModule) -> PyResult<()> { /// :type mime_type: str /// :param base_iri: the base IRI used to resolve the relative IRIs in the file or :py:const:`None` if relative IRI resolution should not be done. /// :type base_iri: str or None, optional +/// :param without_named_graphs: Sets that the parser must fail if parsing a named graph. +/// :type without_named_graphs: bool, optional +/// :param rename_blank_nodes: Renames the blank nodes ids from the ones set in the serialization to random ids. This allows to avoid id conflicts when merging graphs together. +/// :type rename_blank_nodes: bool, optional /// :return: an iterator of RDF triples or quads depending on the format. -/// :rtype: iterator(Triple) or iterator(Quad) +/// :rtype: iterator(Quad) /// :raises ValueError: if the MIME type is not supported. /// :raises SyntaxError: if the provided data is invalid. /// /// >>> input = io.BytesIO(b'

"1" .') /// >>> list(parse(input, "text/turtle", base_iri="http://example.com/")) -/// [ predicate= object=>>] +/// [ predicate= object=> graph_name=>] #[pyfunction] -#[pyo3(signature = (input, mime_type, *, base_iri = None))] +#[pyo3(signature = (input, mime_type, *, base_iri = None, without_named_graphs = false, rename_blank_nodes = false))] pub fn parse( input: PyObject, mime_type: &str, base_iri: Option<&str>, + without_named_graphs: bool, + rename_blank_nodes: bool, py: Python<'_>, ) -> PyResult { + let Some(format) = RdfFormat::from_media_type(mime_type) else { + return Err(PyValueError::new_err(format!( + "Not supported MIME type: {mime_type}" + ))); + }; let input = if let Ok(path) = input.extract::(py) { PyReadable::from_file(&path, py).map_err(map_io_err)? } else { PyReadable::from_data(input, py) }; - if let Some(graph_format) = GraphFormat::from_media_type(mime_type) { - let mut parser = GraphParser::from_format(graph_format); - if let Some(base_iri) = base_iri { - parser = parser - .with_base_iri(base_iri) - .map_err(|e| PyValueError::new_err(e.to_string()))?; - } - Ok(PyTripleReader { - inner: parser.read_triples(input), - } - .into_py(py)) - } else if let Some(dataset_format) = DatasetFormat::from_media_type(mime_type) { - let mut parser = DatasetParser::from_format(dataset_format); - if let Some(base_iri) = base_iri { - parser = parser - .with_base_iri(base_iri) - .map_err(|e| PyValueError::new_err(e.to_string()))?; - } - Ok(PyQuadReader { - inner: parser.read_quads(input), - } - .into_py(py)) - } else { - Err(PyValueError::new_err(format!( - "Not supported MIME type: {mime_type}" - ))) + let mut parser = RdfParser::from_format(format); + if let Some(base_iri) = base_iri { + parser = parser + .with_base_iri(base_iri) + .map_err(|e| PyValueError::new_err(e.to_string()))?; } + if without_named_graphs { + parser = parser.without_named_graphs(); + } + if rename_blank_nodes { + parser = parser.rename_blank_nodes(); + } + Ok(PyQuadReader { + inner: parser.parse_read(input), + } + .into_py(py)) } /// Serializes an RDF graph or dataset. @@ -151,30 +148,9 @@ pub fn serialize(input: &PyAny, output: PyObject, mime_type: &str, py: Python<'_ writer.finish().map_err(map_io_err) } -#[pyclass(name = "TripleReader", module = "pyoxigraph")] -pub struct PyTripleReader { - inner: TripleReader, -} - -#[pymethods] -impl PyTripleReader { - fn __iter__(slf: PyRef<'_, Self>) -> PyRef { - slf - } - - fn __next__(&mut self, py: Python<'_>) -> PyResult> { - py.allow_threads(|| { - self.inner - .next() - .map(|q| Ok(q.map_err(map_parse_error)?.into())) - .transpose() - }) - } -} - #[pyclass(name = "QuadReader", module = "pyoxigraph")] pub struct PyQuadReader { - inner: QuadReader, + inner: FromReadQuadReader, } #[pymethods] diff --git a/python/src/store.rs b/python/src/store.rs index 692b485f..cfb10a3a 100644 --- a/python/src/store.rs +++ b/python/src/store.rs @@ -3,7 +3,7 @@ use crate::io::{allow_threads_unsafe, map_io_err, map_parse_error, PyReadable, PyWritable}; use crate::model::*; use crate::sparql::*; -use oxigraph::io::{DatasetFormat, GraphFormat, RdfFormat}; +use oxigraph::io::RdfFormat; use oxigraph::model::{GraphName, GraphNameRef}; use oxigraph::sparql::Update; use oxigraph::store::{self, LoaderError, SerializerError, StorageError, Store}; @@ -366,7 +366,7 @@ impl PyStore { /// :param to_graph: if it is a file composed of triples, the graph in which the triples should be stored. By default, the default graph is used. /// :type to_graph: NamedNode or BlankNode or DefaultGraph or None, optional /// :rtype: None - /// :raises ValueError: if the MIME type is not supported or the `to_graph` parameter is given with a quad file. + /// :raises ValueError: if the MIME type is not supported. /// :raises SyntaxError: if the provided data is invalid. /// :raises IOError: if an I/O error happens during a quad insertion. /// @@ -383,6 +383,11 @@ impl PyStore { to_graph: Option<&PyAny>, py: Python<'_>, ) -> PyResult<()> { + let Some(format) = RdfFormat::from_media_type(mime_type) else { + return Err(PyValueError::new_err(format!( + "Not supported MIME type: {mime_type}" + ))); + }; let to_graph_name = if let Some(graph_name) = to_graph { Some(GraphName::from(&PyGraphNameRef::try_from(graph_name)?)) } else { @@ -394,29 +399,13 @@ impl PyStore { PyReadable::from_data(input, py) }; py.allow_threads(|| { - if let Some(graph_format) = GraphFormat::from_media_type(mime_type) { - self.inner - .load_graph( - input, - graph_format, - to_graph_name.as_ref().unwrap_or(&GraphName::DefaultGraph), - base_iri, - ) - .map_err(map_loader_error) - } else if let Some(dataset_format) = DatasetFormat::from_media_type(mime_type) { - if to_graph_name.is_some() { - return Err(PyValueError::new_err( - "The target graph name parameter is not available for dataset formats", - )); - } + if let Some(to_graph_name) = to_graph_name { self.inner - .load_dataset(input, dataset_format, base_iri) - .map_err(map_loader_error) + .load_graph(input, format, to_graph_name, base_iri) } else { - Err(PyValueError::new_err(format!( - "Not supported MIME type: {mime_type}" - ))) + self.inner.load_dataset(input, format, base_iri) } + .map_err(map_loader_error) }) } @@ -448,7 +437,7 @@ impl PyStore { /// :param to_graph: if it is a file composed of triples, the graph in which the triples should be stored. By default, the default graph is used. /// :type to_graph: NamedNode or BlankNode or DefaultGraph or None, optional /// :rtype: None - /// :raises ValueError: if the MIME type is not supported or the `to_graph` parameter is given with a quad file. + /// :raises ValueError: if the MIME type is not supported. /// :raises SyntaxError: if the provided data is invalid. /// :raises IOError: if an I/O error happens during a quad insertion. /// @@ -465,6 +454,11 @@ impl PyStore { to_graph: Option<&PyAny>, py: Python<'_>, ) -> PyResult<()> { + let Some(format) = RdfFormat::from_media_type(mime_type) else { + return Err(PyValueError::new_err(format!( + "Not supported MIME type: {mime_type}" + ))); + }; let to_graph_name = if let Some(graph_name) = to_graph { Some(GraphName::from(&PyGraphNameRef::try_from(graph_name)?)) } else { @@ -476,31 +470,16 @@ impl PyStore { PyReadable::from_data(input, py) }; py.allow_threads(|| { - if let Some(graph_format) = GraphFormat::from_media_type(mime_type) { + if let Some(to_graph_name) = to_graph_name { self.inner .bulk_loader() - .load_graph( - input, - graph_format, - &to_graph_name.unwrap_or(GraphName::DefaultGraph), - base_iri, - ) - .map_err(map_loader_error) - } else if let Some(dataset_format) = DatasetFormat::from_media_type(mime_type) { - if to_graph_name.is_some() { - return Err(PyValueError::new_err( - "The target graph name parameter is not available for dataset formats", - )); - } + .load_graph(input, format, to_graph_name, base_iri) + } else { self.inner .bulk_loader() - .load_dataset(input, dataset_format, base_iri) - .map_err(map_loader_error) - } else { - Err(PyValueError::new_err(format!( - "Not supported MIME type: {mime_type}" - ))) + .load_dataset(input, format, base_iri) } + .map_err(map_loader_error) }) } @@ -542,11 +521,6 @@ impl PyStore { from_graph: Option<&PyAny>, py: Python<'_>, ) -> PyResult<()> { - let output = if let Ok(path) = output.extract::(py) { - PyWritable::from_file(&path, py).map_err(map_io_err)? - } else { - PyWritable::from_data(output) - }; let Some(format) = RdfFormat::from_media_type(mime_type) else { return Err(PyValueError::new_err(format!( "Not supported MIME type: {mime_type}" @@ -557,6 +531,11 @@ impl PyStore { } else { None }; + let output = if let Ok(path) = output.extract::(py) { + PyWritable::from_file(&path, py).map_err(map_io_err)? + } else { + PyWritable::from_data(output) + }; py.allow_threads(|| { if let Some(from_graph_name) = &from_graph_name { self.inner.dump_graph(output, format, from_graph_name) @@ -860,6 +839,7 @@ pub fn map_loader_error(error: LoaderError) -> PyErr { match error { LoaderError::Storage(error) => map_storage_error(error), LoaderError::Parsing(error) => map_parse_error(error), + LoaderError::InvalidBaseIri { .. } => PyValueError::new_err(error.to_string()), } } diff --git a/python/tests/test_io.py b/python/tests/test_io.py index 2d291bc8..d3f535c4 100644 --- a/python/tests/test_io.py +++ b/python/tests/test_io.py @@ -2,9 +2,9 @@ import unittest from io import BytesIO, StringIO, UnsupportedOperation from tempfile import NamedTemporaryFile, TemporaryFile -from pyoxigraph import Literal, NamedNode, Quad, Triple, parse, serialize +from pyoxigraph import Literal, NamedNode, Quad, parse, serialize -EXAMPLE_TRIPLE = Triple( +EXAMPLE_TRIPLE = Quad( NamedNode("http://example.com/foo"), NamedNode("http://example.com/p"), Literal("éù"), @@ -83,11 +83,40 @@ class TestParse(unittest.TestCase): [EXAMPLE_QUAD], ) + def test_parse_without_named_graphs(self) -> None: + with self.assertRaises(SyntaxError) as _: + list( + parse( + StringIO(' {

"1" }'), + "application/trig", + base_iri="http://example.com/", + without_named_graphs=True, + ) + ) + + def test_parse_rename_blank_nodes(self) -> None: + self.assertNotEqual( + list( + parse( + StringIO('_:s "o" .'), + "application/n-triples", + rename_blank_nodes=True, + ) + ), + list( + parse( + StringIO('_:s "o" .'), + "application/n-triples", + rename_blank_nodes=True, + ) + ), + ) + class TestSerialize(unittest.TestCase): def test_serialize_to_bytes_io(self) -> None: output = BytesIO() - serialize([EXAMPLE_TRIPLE], output, "text/turtle") + serialize([EXAMPLE_TRIPLE.triple], output, "text/turtle") self.assertEqual( output.getvalue().decode(), ' "éù" .\n', diff --git a/server/src/main.rs b/server/src/main.rs index eb9b3116..5749d7f6 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -1,10 +1,10 @@ #![allow(clippy::print_stderr, clippy::cast_precision_loss, clippy::use_debug)] -use anyhow::{anyhow, bail, ensure, Context, Error}; +use anyhow::{anyhow, bail, ensure, Context}; use clap::{Parser, Subcommand}; use flate2::read::MultiGzDecoder; use oxhttp::model::{Body, HeaderName, HeaderValue, Method, Request, Response, Status}; use oxhttp::Server; -use oxigraph::io::{DatasetFormat, GraphFormat, RdfFormat, RdfSerializer}; +use oxigraph::io::{RdfFormat, RdfSerializer}; use oxigraph::model::{ GraphName, GraphNameRef, IriParseError, NamedNode, NamedNodeRef, NamedOrBlankNode, }; @@ -295,13 +295,13 @@ pub fn main() -> anyhow::Result<()> { Store::new() }?; let format = if let Some(format) = format { - Some(GraphOrDatasetFormat::from_str(&format)?) + Some(rdf_format_from_name(&format)?) } else { None }; let graph = if let Some(iri) = &graph { Some( - NamedNodeRef::new(iri) + NamedNode::new(iri) .with_context(|| format!("The target graph name {iri} is invalid"))?, ) } else { @@ -342,6 +342,7 @@ pub fn main() -> anyhow::Result<()> { .scope(|s| { for file in file { let store = store.clone(); + let graph = graph.clone(); s.spawn(move |_| { let f = file.clone(); let start = Instant::now(); @@ -379,10 +380,8 @@ pub fn main() -> anyhow::Result<()> { &loader, MultiGzDecoder::new(fp), format.unwrap_or_else(|| { - GraphOrDatasetFormat::from_path( - &file.with_extension(""), - ) - .unwrap() + rdf_format_from_path(&file.with_extension("")) + .unwrap() }), None, graph, @@ -392,7 +391,7 @@ pub fn main() -> anyhow::Result<()> { &loader, fp, format.unwrap_or_else(|| { - GraphOrDatasetFormat::from_path(&file).unwrap() + rdf_format_from_path(&file).unwrap() }), None, graph, @@ -432,7 +431,7 @@ pub fn main() -> anyhow::Result<()> { }; let graph = if let Some(graph) = &graph { Some(if graph.eq_ignore_ascii_case("default") { - GraphName::DefaultGraph + GraphNameRef::DefaultGraph } else { NamedNodeRef::new(graph) .with_context(|| format!("The target graph name {graph} is invalid"))? @@ -637,24 +636,15 @@ pub fn main() -> anyhow::Result<()> { fn bulk_load( loader: &BulkLoader, reader: impl Read, - format: GraphOrDatasetFormat, + format: RdfFormat, base_iri: Option<&str>, - to_graph_name: Option>, + to_graph_name: Option, ) -> anyhow::Result<()> { - match format { - GraphOrDatasetFormat::Graph(format) => loader.load_graph( - reader, - format, - to_graph_name.map_or(GraphNameRef::DefaultGraph, GraphNameRef::from), - base_iri, - )?, - GraphOrDatasetFormat::Dataset(format) => { - if to_graph_name.is_some() { - bail!("The --graph option is not allowed when loading a dataset format like NQuads or TriG"); - } - loader.load_dataset(reader, format, base_iri)? - } - } + if let Some(to_graph_name) = to_graph_name { + loader.load_graph(reader, format, to_graph_name, base_iri) + } else { + loader.load_dataset(reader, format, base_iri) + }?; Ok(()) } @@ -662,57 +652,17 @@ fn dump( store: &Store, writer: impl Write, format: RdfFormat, - to_graph_name: Option, + from_graph_name: Option>, ) -> anyhow::Result<()> { - ensure!(format.supports_datasets() || to_graph_name.is_some(), "The --graph option is required when writing a format not supporting datasets like NTriples, Turtle or RDF/XML"); - if let Some(to_graph_name) = &to_graph_name { - store.dump_graph(writer, format, to_graph_name) + ensure!(format.supports_datasets() || from_graph_name.is_some(), "The --graph option is required when writing a format not supporting datasets like NTriples, Turtle or RDF/XML"); + if let Some(from_graph_name) = from_graph_name { + store.dump_graph(writer, format, from_graph_name) } else { store.dump_dataset(writer, format) }?; Ok(()) } -#[derive(Copy, Clone)] -enum GraphOrDatasetFormat { - Graph(GraphFormat), - Dataset(DatasetFormat), -} - -impl GraphOrDatasetFormat { - fn from_path(path: &Path) -> anyhow::Result { - format_from_path(path, Self::from_extension) - } - - fn from_extension(name: &str) -> anyhow::Result { - Ok(match (GraphFormat::from_extension(name), DatasetFormat::from_extension(name)) { - (Some(g), Some(d)) => bail!("The file extension '{name}' can be resolved to both '{}' and '{}', not sure what to pick", g.file_extension(), d.file_extension()), - (Some(g), None) => Self::Graph(g), - (None, Some(d)) => Self::Dataset(d), - (None, None) => - bail!("The file extension '{name}' is unknown") - }) - } - - fn from_media_type(name: &str) -> anyhow::Result { - Ok( - match ( - GraphFormat::from_media_type(name), - DatasetFormat::from_media_type(name), - ) { - (Some(g), Some(d)) => bail!( - "The media type '{name}' can be resolved to both '{}' and '{}', not sure what to pick", - g.file_extension(), - d.file_extension() - ), - (Some(g), None) => Self::Graph(g), - (None, Some(d)) => Self::Dataset(d), - (None, None) => bail!("The media type '{name}' is unknown"), - }, - ) - } -} - fn format_from_path( path: &Path, from_extension: impl FnOnce(&str) -> anyhow::Result, @@ -731,20 +681,6 @@ fn format_from_path( } } -impl FromStr for GraphOrDatasetFormat { - type Err = Error; - - fn from_str(name: &str) -> anyhow::Result { - if let Ok(t) = Self::from_extension(name) { - return Ok(t); - } - if let Ok(t) = Self::from_media_type(name) { - return Ok(t); - } - bail!("The file format '{name}' is unknown") - } -} - fn rdf_format_from_path(path: &Path) -> anyhow::Result { format_from_path(path, |ext| { RdfFormat::from_extension(ext) @@ -978,7 +914,7 @@ fn handle_request( let content_type = content_type(request).ok_or_else(|| bad_request("No Content-Type given"))?; if let Some(target) = store_target(request)? { - let format = GraphFormat::from_media_type(&content_type) + let format = RdfFormat::from_media_type(&content_type) .ok_or_else(|| unsupported_media_type(&content_type))?; let new = !match &target { NamedGraphName::NamedNode(target) => { @@ -1002,7 +938,7 @@ fn handle_request( true } }; - web_load_graph(&store, request, format, GraphName::from(target).as_ref())?; + web_load_graph(&store, request, format, &GraphName::from(target))?; Ok(Response::builder(if new { Status::CREATED } else { @@ -1010,7 +946,7 @@ fn handle_request( }) .build()) } else { - let format = DatasetFormat::from_media_type(&content_type) + let format = RdfFormat::from_media_type(&content_type) .ok_or_else(|| unsupported_media_type(&content_type))?; store.clear().map_err(internal_server_error)?; web_load_dataset(&store, request, format)?; @@ -1054,10 +990,10 @@ fn handle_request( let content_type = content_type(request).ok_or_else(|| bad_request("No Content-Type given"))?; if let Some(target) = store_target(request)? { - let format = GraphFormat::from_media_type(&content_type) + let format = RdfFormat::from_media_type(&content_type) .ok_or_else(|| unsupported_media_type(&content_type))?; let new = assert_that_graph_exists(&store, &target).is_ok(); - web_load_graph(&store, request, format, GraphName::from(target).as_ref())?; + web_load_graph(&store, request, format, &GraphName::from(target))?; Ok(Response::builder(if new { Status::CREATED } else { @@ -1065,22 +1001,19 @@ fn handle_request( }) .build()) } else { - match GraphOrDatasetFormat::from_media_type(&content_type) - .map_err(|_| unsupported_media_type(&content_type))? - { - GraphOrDatasetFormat::Graph(format) => { - let graph = - resolve_with_base(request, &format!("/store/{:x}", random::()))?; - web_load_graph(&store, request, format, graph.as_ref().into())?; - Ok(Response::builder(Status::CREATED) - .with_header(HeaderName::LOCATION, graph.into_string()) - .unwrap() - .build()) - } - GraphOrDatasetFormat::Dataset(format) => { - web_load_dataset(&store, request, format)?; - Ok(Response::builder(Status::NO_CONTENT).build()) - } + let format = RdfFormat::from_media_type(&content_type) + .ok_or_else(|| unsupported_media_type(&content_type))?; + if format.supports_datasets() { + web_load_dataset(&store, request, format)?; + Ok(Response::builder(Status::NO_CONTENT).build()) + } else { + let graph = + resolve_with_base(request, &format!("/store/{:x}", random::()))?; + web_load_graph(&store, request, format, &graph.clone().into())?; + Ok(Response::builder(Status::CREATED) + .with_header(HeaderName::LOCATION, graph.as_str()) + .unwrap() + .build()) } } } @@ -1531,10 +1464,10 @@ fn content_type(request: &Request) -> Option { fn web_load_graph( store: &Store, request: &mut Request, - format: GraphFormat, - to_graph_name: GraphNameRef<'_>, + format: RdfFormat, + to_graph_name: &GraphName, ) -> Result<(), HttpError> { - let base_iri = if let GraphNameRef::NamedNode(graph_name) = to_graph_name { + let base_iri = if let GraphName::NamedNode(graph_name) = to_graph_name { Some(graph_name.as_str()) } else { None @@ -1543,11 +1476,11 @@ fn web_load_graph( web_bulk_loader(store, request).load_graph( request.body_mut(), format, - to_graph_name, + to_graph_name.clone(), base_iri, ) } else { - store.load_graph(request.body_mut(), format, to_graph_name, base_iri) + store.load_graph(request.body_mut(), format, to_graph_name.clone(), base_iri) } .map_err(loader_to_http_error) } @@ -1555,7 +1488,7 @@ fn web_load_graph( fn web_load_dataset( store: &Store, request: &mut Request, - format: DatasetFormat, + format: RdfFormat, ) -> Result<(), HttpError> { if url_query_parameter(request, "no_transaction").is_some() { web_bulk_loader(store, request).load_dataset(request.body_mut(), format, None) @@ -1616,6 +1549,7 @@ fn loader_to_http_error(e: LoaderError) -> HttpError { match e { LoaderError::Parsing(e) => bad_request(e), LoaderError::Storage(e) => internal_server_error(e), + LoaderError::InvalidBaseIri { .. } => bad_request(e), } } diff --git a/testsuite/src/files.rs b/testsuite/src/files.rs index 7e4f3b17..fc7f5a20 100644 --- a/testsuite/src/files.rs +++ b/testsuite/src/files.rs @@ -1,5 +1,5 @@ use anyhow::{anyhow, bail, Context, Result}; -use oxigraph::io::{DatasetFormat, DatasetParser, GraphFormat, GraphParser}; +use oxigraph::io::{RdfFormat, RdfParser}; use oxigraph::model::{Dataset, Graph}; use oxttl::n3::N3Quad; use oxttl::N3Parser; @@ -33,14 +33,14 @@ pub fn read_file_to_string(url: &str) -> Result { pub fn load_to_graph( url: &str, graph: &mut Graph, - format: GraphFormat, + format: RdfFormat, ignore_errors: bool, ) -> Result<()> { - let parser = GraphParser::from_format(format).with_base_iri(url)?; - for t in parser.read_triples(read_file(url)?) { + let parser = RdfParser::from_format(format).with_base_iri(url)?; + for t in parser.parse_read(read_file(url)?) { match t { Ok(t) => { - graph.insert(&t); + graph.insert(&t.into()); } Err(e) => { if !ignore_errors { @@ -52,26 +52,20 @@ pub fn load_to_graph( Ok(()) } -pub fn load_graph(url: &str, format: GraphFormat, ignore_errors: bool) -> Result { +pub fn load_graph(url: &str, format: RdfFormat, ignore_errors: bool) -> Result { let mut graph = Graph::new(); load_to_graph(url, &mut graph, format, ignore_errors)?; Ok(graph) } -pub fn guess_graph_format(url: &str) -> Result { - url.rsplit_once('.') - .and_then(|(_, extension)| GraphFormat::from_extension(extension)) - .ok_or_else(|| anyhow!("Serialization type not found for {url}")) -} - pub fn load_to_dataset( url: &str, dataset: &mut Dataset, - format: DatasetFormat, + format: RdfFormat, ignore_errors: bool, ) -> Result<()> { - let parser = DatasetParser::from_format(format).with_base_iri(url)?; - for q in parser.read_quads(read_file(url)?) { + let parser = RdfParser::from_format(format).with_base_iri(url)?; + for q in parser.parse_read(read_file(url)?) { match q { Ok(q) => { dataset.insert(&q); @@ -86,15 +80,15 @@ pub fn load_to_dataset( Ok(()) } -pub fn load_dataset(url: &str, format: DatasetFormat, ignore_errors: bool) -> Result { +pub fn load_dataset(url: &str, format: RdfFormat, ignore_errors: bool) -> Result { let mut dataset = Dataset::new(); load_to_dataset(url, &mut dataset, format, ignore_errors)?; Ok(dataset) } -pub fn guess_dataset_format(url: &str) -> Result { +pub fn guess_rdf_format(url: &str) -> Result { url.rsplit_once('.') - .and_then(|(_, extension)| DatasetFormat::from_extension(extension)) + .and_then(|(_, extension)| RdfFormat::from_extension(extension)) .ok_or_else(|| anyhow!("Serialization type not found for {url}")) } diff --git a/testsuite/src/manifest.rs b/testsuite/src/manifest.rs index f854b95e..fd450fe2 100644 --- a/testsuite/src/manifest.rs +++ b/testsuite/src/manifest.rs @@ -1,4 +1,4 @@ -use crate::files::{guess_graph_format, load_to_graph}; +use crate::files::{guess_rdf_format, load_to_graph}; use crate::vocab::*; use anyhow::{bail, Result}; use oxigraph::model::vocab::*; @@ -275,7 +275,7 @@ impl TestManifest { return Ok(None); }; self.graph.clear(); - load_to_graph(&url, &mut self.graph, guess_graph_format(&url)?, false)?; + load_to_graph(&url, &mut self.graph, guess_rdf_format(&url)?, false)?; let manifests = self .graph diff --git a/testsuite/src/parser_evaluator.rs b/testsuite/src/parser_evaluator.rs index 975c2e9b..dbf660da 100644 --- a/testsuite/src/parser_evaluator.rs +++ b/testsuite/src/parser_evaluator.rs @@ -1,27 +1,27 @@ use crate::evaluator::TestEvaluator; -use crate::files::{guess_dataset_format, guess_graph_format, load_dataset, load_graph, load_n3}; +use crate::files::{guess_rdf_format, load_dataset, load_n3}; use crate::manifest::Test; -use crate::report::{dataset_diff, graph_diff}; +use crate::report::dataset_diff; use anyhow::{anyhow, bail, Result}; -use oxigraph::io::{DatasetFormat, GraphFormat}; +use oxigraph::io::RdfFormat; use oxigraph::model::{BlankNode, Dataset, Quad}; use oxttl::n3::{N3Quad, N3Term}; pub fn register_parser_tests(evaluator: &mut TestEvaluator) { evaluator.register( "http://www.w3.org/ns/rdftest#TestNTriplesPositiveSyntax", - |t| evaluate_positive_graph_syntax_test(t, GraphFormat::NTriples), + |t| evaluate_positive_syntax_test(t, RdfFormat::NTriples), ); evaluator.register( "http://www.w3.org/ns/rdftest#TestNQuadsPositiveSyntax", - |t| evaluate_positive_dataset_syntax_test(t, DatasetFormat::NQuads), + |t| evaluate_positive_syntax_test(t, RdfFormat::NQuads), ); evaluator.register( "http://www.w3.org/ns/rdftest#TestTurtlePositiveSyntax", - |t| evaluate_positive_graph_syntax_test(t, GraphFormat::Turtle), + |t| evaluate_positive_syntax_test(t, RdfFormat::Turtle), ); evaluator.register("http://www.w3.org/ns/rdftest#TestTrigPositiveSyntax", |t| { - evaluate_positive_dataset_syntax_test(t, DatasetFormat::TriG) + evaluate_positive_syntax_test(t, RdfFormat::TriG) }); evaluator.register( "https://w3c.github.io/N3/tests/test.n3#TestN3PositiveSyntax", @@ -29,47 +29,47 @@ pub fn register_parser_tests(evaluator: &mut TestEvaluator) { ); evaluator.register( "http://www.w3.org/ns/rdftest#TestNTriplesNegativeSyntax", - |t| evaluate_negative_graph_syntax_test(t, GraphFormat::NTriples), + |t| evaluate_negative_syntax_test(t, RdfFormat::NTriples), ); evaluator.register( "http://www.w3.org/ns/rdftest#TestNQuadsNegativeSyntax", - |t| evaluate_negative_dataset_syntax_test(t, DatasetFormat::NQuads), + |t| evaluate_negative_syntax_test(t, RdfFormat::NQuads), ); evaluator.register( "http://www.w3.org/ns/rdftest#TestTurtleNegativeSyntax", - |t| evaluate_negative_graph_syntax_test(t, GraphFormat::Turtle), + |t| evaluate_negative_syntax_test(t, RdfFormat::Turtle), ); evaluator.register("http://www.w3.org/ns/rdftest#TestTrigNegativeSyntax", |t| { - evaluate_negative_dataset_syntax_test(t, DatasetFormat::TriG) + evaluate_negative_syntax_test(t, RdfFormat::TriG) }); evaluator.register("http://www.w3.org/ns/rdftest#TestXMLNegativeSyntax", |t| { - evaluate_negative_graph_syntax_test(t, GraphFormat::RdfXml) + evaluate_negative_syntax_test(t, RdfFormat::RdfXml) }); evaluator.register( "https://w3c.github.io/N3/tests/test.n3#TestN3NegativeSyntax", evaluate_negative_n3_syntax_test, ); evaluator.register("http://www.w3.org/ns/rdftest#TestTurtleEval", |t| { - evaluate_graph_eval_test(t, GraphFormat::Turtle, false) + evaluate_eval_test(t, RdfFormat::Turtle, false) }); evaluator.register("http://www.w3.org/ns/rdftest#TestTrigEval", |t| { - evaluate_dataset_eval_test(t, DatasetFormat::TriG, false) + evaluate_eval_test(t, RdfFormat::TriG, false) }); evaluator.register("http://www.w3.org/ns/rdftest#TestXMLEval", |t| { - evaluate_graph_eval_test(t, GraphFormat::RdfXml, false) + evaluate_eval_test(t, RdfFormat::RdfXml, false) }); evaluator.register("https://w3c.github.io/N3/tests/test.n3#TestN3Eval", |t| { evaluate_n3_eval_test(t, false) }); evaluator.register("http://www.w3.org/ns/rdftest#TestTurtleNegativeEval", |t| { - evaluate_negative_graph_syntax_test(t, GraphFormat::Turtle) + evaluate_negative_syntax_test(t, RdfFormat::Turtle) }); evaluator.register("http://www.w3.org/ns/rdftest#TestTrigNegativeEval", |t| { - evaluate_negative_dataset_syntax_test(t, DatasetFormat::TriG) + evaluate_negative_syntax_test(t, RdfFormat::TriG) }); evaluator.register( "https://w3c.github.io/rdf-canon/tests/vocab#RDFC10EvalTest", - |t| evaluate_positive_dataset_syntax_test(t, DatasetFormat::NQuads), //TODO: not a proper implementation! + |t| evaluate_positive_syntax_test(t, RdfFormat::NQuads), //TODO: not a proper implementation! ); evaluator.register( "https://w3c.github.io/rdf-canon/tests/vocab#RDFC10NegativeEvalTest", @@ -81,11 +81,11 @@ pub fn register_parser_tests(evaluator: &mut TestEvaluator) { ); evaluator.register( "https://github.com/oxigraph/oxigraph/tests#TestNTripleRecovery", - |t| evaluate_graph_eval_test(t, GraphFormat::NTriples, true), + |t| evaluate_eval_test(t, RdfFormat::NTriples, true), ); evaluator.register( "https://github.com/oxigraph/oxigraph/tests#TestTurtleRecovery", - |t| evaluate_graph_eval_test(t, GraphFormat::Turtle, true), + |t| evaluate_eval_test(t, RdfFormat::Turtle, true), ); evaluator.register( "https://github.com/oxigraph/oxigraph/tests#TestN3Recovery", @@ -93,16 +93,7 @@ pub fn register_parser_tests(evaluator: &mut TestEvaluator) { ); } -fn evaluate_positive_graph_syntax_test(test: &Test, format: GraphFormat) -> Result<()> { - let action = test - .action - .as_deref() - .ok_or_else(|| anyhow!("No action found for test {test}"))?; - load_graph(action, format, false).map_err(|e| anyhow!("Parse error: {e}"))?; - Ok(()) -} - -fn evaluate_positive_dataset_syntax_test(test: &Test, format: DatasetFormat) -> Result<()> { +fn evaluate_positive_syntax_test(test: &Test, format: RdfFormat) -> Result<()> { let action = test .action .as_deref() @@ -120,18 +111,7 @@ fn evaluate_positive_n3_syntax_test(test: &Test) -> Result<()> { Ok(()) } -fn evaluate_negative_graph_syntax_test(test: &Test, format: GraphFormat) -> Result<()> { - let action = test - .action - .as_deref() - .ok_or_else(|| anyhow!("No action found for test {test}"))?; - match load_graph(action, format, false) { - Ok(_) => bail!("File parsed without errors even if it should not"), - Err(_) => Ok(()), - } -} - -fn evaluate_negative_dataset_syntax_test(test: &Test, format: DatasetFormat) -> Result<()> { +fn evaluate_negative_syntax_test(test: &Test, format: RdfFormat) -> Result<()> { let action = test .action .as_deref() @@ -153,36 +133,7 @@ fn evaluate_negative_n3_syntax_test(test: &Test) -> Result<()> { } } -fn evaluate_graph_eval_test(test: &Test, format: GraphFormat, ignore_errors: bool) -> Result<()> { - let action = test - .action - .as_deref() - .ok_or_else(|| anyhow!("No action found for test {test}"))?; - let mut actual_graph = load_graph(action, format, ignore_errors) - .map_err(|e| anyhow!("Parse error on file {action}: {e}"))?; - actual_graph.canonicalize(); - let results = test - .result - .as_ref() - .ok_or_else(|| anyhow!("No tests result found"))?; - let mut expected_graph = load_graph(results, guess_graph_format(results)?, false) - .map_err(|e| anyhow!("Parse error on file {results}: {e}"))?; - expected_graph.canonicalize(); - if expected_graph == actual_graph { - Ok(()) - } else { - bail!( - "The two files are not isomorphic. Diff:\n{}", - graph_diff(&expected_graph, &actual_graph) - ) - } -} - -fn evaluate_dataset_eval_test( - test: &Test, - format: DatasetFormat, - ignore_errors: bool, -) -> Result<()> { +fn evaluate_eval_test(test: &Test, format: RdfFormat, ignore_errors: bool) -> Result<()> { let action = test .action .as_deref() @@ -194,7 +145,7 @@ fn evaluate_dataset_eval_test( .result .as_ref() .ok_or_else(|| anyhow!("No tests result found"))?; - let mut expected_dataset = load_dataset(results, guess_dataset_format(results)?, false) + let mut expected_dataset = load_dataset(results, guess_rdf_format(results)?, false) .map_err(|e| anyhow!("Parse error on file {results}: {e}"))?; expected_dataset.canonicalize(); if expected_dataset == actual_dataset { diff --git a/testsuite/src/report.rs b/testsuite/src/report.rs index ba24ee2c..be27110a 100644 --- a/testsuite/src/report.rs +++ b/testsuite/src/report.rs @@ -1,5 +1,5 @@ use anyhow::Result; -use oxigraph::model::{Dataset, Graph, NamedNode}; +use oxigraph::model::{Dataset, NamedNode}; use std::fmt::Write; use text_diff::{diff, Difference}; use time::format_description::well_known::Rfc3339; @@ -26,20 +26,6 @@ fn normalize_dataset_text(store: &Dataset) -> String { quads.join("\n") } -pub(super) fn graph_diff(expected: &Graph, actual: &Graph) -> String { - format_diff( - &normalize_graph_text(expected), - &normalize_graph_text(actual), - "triples", - ) -} - -fn normalize_graph_text(store: &Graph) -> String { - let mut triples: Vec<_> = store.iter().map(|q| q.to_string()).collect(); - triples.sort(); - triples.join("\n") -} - pub(super) fn format_diff(expected: &str, actual: &str, kind: &str) -> String { let (_, changeset) = diff(expected, actual, "\n"); let mut ret = String::new(); diff --git a/testsuite/src/sparql_evaluator.rs b/testsuite/src/sparql_evaluator.rs index ab2b6643..f440b6ad 100644 --- a/testsuite/src/sparql_evaluator.rs +++ b/testsuite/src/sparql_evaluator.rs @@ -158,7 +158,7 @@ fn evaluate_evaluation_test(test: &Test) -> Result<()> { load_dataset_to_store(data, &store)?; } for (name, value) in &test.graph_data { - load_graph_to_store(value, &store, name)?; + load_graph_to_store(value, &store, name.clone())?; } let query_file = test .query @@ -251,7 +251,7 @@ fn evaluate_update_evaluation_test(test: &Test) -> Result<()> { load_dataset_to_store(data, &store)?; } for (name, value) in &test.graph_data { - load_graph_to_store(value, &store, name)?; + load_graph_to_store(value, &store, name.clone())?; } let result_store = Store::new()?; @@ -259,7 +259,7 @@ fn evaluate_update_evaluation_test(test: &Test) -> Result<()> { load_dataset_to_store(data, &result_store)?; } for (name, value) in &test.result_graph_data { - load_graph_to_store(value, &result_store, name)?; + load_graph_to_store(value, &result_store, name.clone())?; } let update_file = test @@ -301,7 +301,7 @@ fn load_sparql_query_result(url: &str) -> Result { false, ) } else { - StaticQueryResults::from_graph(&load_graph(url, guess_graph_format(url)?, false)?) + StaticQueryResults::from_graph(&load_graph(url, guess_rdf_format(url)?, false)?) } } @@ -698,14 +698,14 @@ fn solutions_to_string(solutions: Vec>, ordered: bool) -> lines.join("\n") } -fn load_graph_to_store<'a>( +fn load_graph_to_store( url: &str, store: &Store, - to_graph_name: impl Into>, + to_graph_name: impl Into, ) -> Result<()> { store.load_graph( read_file(url)?, - guess_graph_format(url)?, + guess_rdf_format(url)?, to_graph_name, Some(url), )?; @@ -713,16 +713,7 @@ fn load_graph_to_store<'a>( } fn load_dataset_to_store(url: &str, store: &Store) -> Result<()> { - if let Ok(format) = guess_dataset_format(url) { - store.load_dataset(read_file(url)?, format, Some(url)) - } else { - store.load_graph( - read_file(url)?, - guess_graph_format(url)?, - GraphNameRef::DefaultGraph, - Some(url), - ) - }?; + store.load_dataset(read_file(url)?, guess_rdf_format(url)?, Some(url))?; Ok(()) } From 4a798ed3eac99ec230a159e12a15289c665350ab Mon Sep 17 00:00:00 2001 From: Tpt Date: Thu, 17 Aug 2023 18:47:16 +0200 Subject: [PATCH 049/217] Python: use OSError instead of IOError to map io::Error --- python/src/io.rs | 13 +++++++----- python/src/store.rs | 50 ++++++++++++++++++++++----------------------- 2 files changed, 33 insertions(+), 30 deletions(-) diff --git a/python/src/io.rs b/python/src/io.rs index f6661474..97bee867 100644 --- a/python/src/io.rs +++ b/python/src/io.rs @@ -3,7 +3,7 @@ use crate::model::{PyQuad, PyTriple}; use oxigraph::io::{FromReadQuadReader, ParseError, RdfFormat, RdfParser, RdfSerializer}; use oxigraph::model::QuadRef; -use pyo3::exceptions::{PyIOError, PySyntaxError, PyValueError}; +use pyo3::exceptions::{PySyntaxError, PyValueError}; use pyo3::prelude::*; use pyo3::types::PyBytes; use pyo3::{intern, wrap_pyfunction}; @@ -272,14 +272,17 @@ impl Write for PyIo { fn flush(&mut self) -> io::Result<()> { Python::with_gil(|py| { - self.0.as_ref(py).call_method0(intern!(py, "flush"))?; + self.0 + .as_ref(py) + .call_method0(intern!(py, "flush")) + .map_err(to_io_err)?; Ok(()) }) } } -fn to_io_err(error: impl Into) -> io::Error { - io::Error::new(io::ErrorKind::Other, error.into()) +fn to_io_err(error: PyErr) -> io::Error { + io::Error::new(io::ErrorKind::Other, error) } pub fn map_io_err(error: io::Error) -> PyErr { @@ -289,7 +292,7 @@ pub fn map_io_err(error: io::Error) -> PyErr { { *error.into_inner().unwrap().downcast().unwrap() } else { - PyIOError::new_err(error.to_string()) + error.into() } } diff --git a/python/src/store.rs b/python/src/store.rs index cfb10a3a..d689c997 100644 --- a/python/src/store.rs +++ b/python/src/store.rs @@ -7,7 +7,7 @@ use oxigraph::io::RdfFormat; use oxigraph::model::{GraphName, GraphNameRef}; use oxigraph::sparql::Update; use oxigraph::store::{self, LoaderError, SerializerError, StorageError, Store}; -use pyo3::exceptions::{PyIOError, PyRuntimeError, PyValueError}; +use pyo3::exceptions::{PyRuntimeError, PyValueError}; use pyo3::prelude::*; use std::path::PathBuf; @@ -28,7 +28,7 @@ use std::path::PathBuf; /// If no directory is provided a temporary one is created and removed when the Python garbage collector removes the store. /// In this case, the store data are kept in memory and never written on disk. /// :type path: str or pathlib.Path or None, optional -/// :raises IOError: if the target directory contains invalid data or could not be accessed. +/// :raises OSError: if the target directory contains invalid data or could not be accessed. /// /// The :py:func:`str` function provides a serialization of the store in NQuads: /// @@ -68,7 +68,7 @@ impl PyStore { /// :type path: str /// :return: the opened store. /// :rtype: Store - /// :raises IOError: if the target directory contains invalid data or could not be accessed. + /// :raises OSError: if the target directory contains invalid data or could not be accessed. #[staticmethod] fn read_only(path: &str, py: Python<'_>) -> PyResult { py.allow_threads(|| { @@ -92,7 +92,7 @@ impl PyStore { /// :type secondary_path: str or None, optional /// :return: the opened store. /// :rtype: Store - /// :raises IOError: if the target directories contain invalid data or could not be accessed. + /// :raises OSError: if the target directories contain invalid data or could not be accessed. #[staticmethod] #[pyo3(signature = (primary_path, secondary_path = None))] fn secondary( @@ -117,7 +117,7 @@ impl PyStore { /// :param quad: the quad to add. /// :type quad: Quad /// :rtype: None - /// :raises IOError: if an I/O error happens during the quad insertion. + /// :raises OSError: if an error happens during the quad insertion. /// /// >>> store = Store() /// >>> store.add(Quad(NamedNode('http://example.com'), NamedNode('http://example.com/p'), Literal('1'), NamedNode('http://example.com/g'))) @@ -138,7 +138,7 @@ impl PyStore { /// :param quads: the quads to add. /// :type quads: iterable(Quad) /// :rtype: None - /// :raises IOError: if an I/O error happens during the quad insertion. + /// :raises OSError: if an error happens during the quad insertion. /// /// >>> store = Store() /// >>> store.extend([Quad(NamedNode('http://example.com'), NamedNode('http://example.com/p'), Literal('1'), NamedNode('http://example.com/g'))]) @@ -163,7 +163,7 @@ impl PyStore { /// :param quads: the quads to add. /// :type quads: iterable(Quad) /// :rtype: None - /// :raises IOError: if an I/O error happens during the quad insertion. + /// :raises OSError: if an error happens during the quad insertion. /// /// >>> store = Store() /// >>> store.bulk_extend([Quad(NamedNode('http://example.com'), NamedNode('http://example.com/p'), Literal('1'), NamedNode('http://example.com/g'))]) @@ -183,7 +183,7 @@ impl PyStore { /// :param quad: the quad to remove. /// :type quad: Quad /// :rtype: None - /// :raises IOError: if an I/O error happens during the quad removal. + /// :raises OSError: if an error happens during the quad removal. /// /// >>> store = Store() /// >>> quad = Quad(NamedNode('http://example.com'), NamedNode('http://example.com/p'), Literal('1'), NamedNode('http://example.com/g')) @@ -210,7 +210,7 @@ impl PyStore { /// :type graph_name: NamedNode or BlankNode or DefaultGraph or None, optional /// :return: an iterator of the quads matching the pattern. /// :rtype: iterator(Quad) - /// :raises IOError: if an I/O error happens during the quads lookup. + /// :raises OSError: if an error happens during the quads lookup. /// /// >>> store = Store() /// >>> store.add(Quad(NamedNode('http://example.com'), NamedNode('http://example.com/p'), Literal('1'), NamedNode('http://example.com/g'))) @@ -251,7 +251,7 @@ impl PyStore { /// :return: a :py:class:`bool` for ``ASK`` queries, an iterator of :py:class:`Triple` for ``CONSTRUCT`` and ``DESCRIBE`` queries and an iterator of :py:class:`QuerySolution` for ``SELECT`` queries. /// :rtype: QuerySolutions or QueryTriples or bool /// :raises SyntaxError: if the provided query is invalid. - /// :raises IOError: if an I/O error happens while reading the store. + /// :raises OSError: if an error happens while reading the store. /// /// ``SELECT`` query: /// @@ -305,7 +305,7 @@ impl PyStore { /// :type base_iri: str or None, optional /// :rtype: None /// :raises SyntaxError: if the provided update is invalid. - /// :raises IOError: if an I/O error happens while reading the store. + /// :raises OSError: if an error happens while reading the store. /// /// ``INSERT DATA`` update: /// @@ -368,7 +368,7 @@ impl PyStore { /// :rtype: None /// :raises ValueError: if the MIME type is not supported. /// :raises SyntaxError: if the provided data is invalid. - /// :raises IOError: if an I/O error happens during a quad insertion. + /// :raises OSError: if an error happens during a quad insertion. /// /// >>> store = Store() /// >>> store.load(io.BytesIO(b'

"1" .'), "text/turtle", base_iri="http://example.com/", to_graph=NamedNode("http://example.com/g")) @@ -439,7 +439,7 @@ impl PyStore { /// :rtype: None /// :raises ValueError: if the MIME type is not supported. /// :raises SyntaxError: if the provided data is invalid. - /// :raises IOError: if an I/O error happens during a quad insertion. + /// :raises OSError: if an error happens during a quad insertion. /// /// >>> store = Store() /// >>> store.bulk_load(io.BytesIO(b'

"1" .'), "text/turtle", base_iri="http://example.com/", to_graph=NamedNode("http://example.com/g")) @@ -505,7 +505,7 @@ impl PyStore { /// :type from_graph: NamedNode or BlankNode or DefaultGraph or None, optional /// :rtype: None /// :raises ValueError: if the MIME type is not supported or the `from_graph` parameter is not given with a syntax not supporting named graphs. - /// :raises IOError: if an I/O error happens during a quad lookup + /// :raises OSError: if an error happens during a quad lookup /// /// >>> store = Store() /// >>> store.add(Quad(NamedNode('http://example.com'), NamedNode('http://example.com/p'), Literal('1'), NamedNode('http://example.com/g'))) @@ -550,7 +550,7 @@ impl PyStore { /// /// :return: an iterator of the store graph names. /// :rtype: iterator(NamedNode or BlankNode) - /// :raises IOError: if an I/O error happens during the named graphs lookup. + /// :raises OSError: if an error happens during the named graphs lookup. /// /// >>> store = Store() /// >>> store.add(Quad(NamedNode('http://example.com'), NamedNode('http://example.com/p'), Literal('1'), NamedNode('http://example.com/g'))) @@ -567,7 +567,7 @@ impl PyStore { /// :param graph_name: the name of the named graph. /// :type graph_name: NamedNode or BlankNode or DefaultGraph /// :rtype: bool - /// :raises IOError: if an I/O error happens during the named graph lookup. + /// :raises OSError: if an error happens during the named graph lookup. /// /// >>> store = Store() /// >>> store.add_graph(NamedNode('http://example.com/g')) @@ -588,7 +588,7 @@ impl PyStore { /// :param graph_name: the name of the name graph to add. /// :type graph_name: NamedNode or BlankNode or DefaultGraph /// :rtype: None - /// :raises IOError: if an I/O error happens during the named graph insertion. + /// :raises OSError: if an error happens during the named graph insertion. /// /// >>> store = Store() /// >>> store.add_graph(NamedNode('http://example.com/g')) @@ -615,7 +615,7 @@ impl PyStore { /// :param graph_name: the name of the name graph to clear. /// :type graph_name: NamedNode or BlankNode or DefaultGraph /// :rtype: None - /// :raises IOError: if an I/O error happens during the operation. + /// :raises OSError: if an error happens during the operation. /// /// >>> store = Store() /// >>> store.add(Quad(NamedNode('http://example.com'), NamedNode('http://example.com/p'), Literal('1'), NamedNode('http://example.com/g'))) @@ -640,7 +640,7 @@ impl PyStore { /// :param graph_name: the name of the name graph to remove. /// :type graph_name: NamedNode or BlankNode or DefaultGraph /// :rtype: None - /// :raises IOError: if an I/O error happens during the named graph removal. + /// :raises OSError: if an error happens during the named graph removal. /// /// >>> store = Store() /// >>> store.add(Quad(NamedNode('http://example.com'), NamedNode('http://example.com/p'), Literal('1'), NamedNode('http://example.com/g'))) @@ -666,7 +666,7 @@ impl PyStore { /// Clears the store by removing all its contents. /// /// :rtype: None - /// :raises IOError: if an I/O error happens during the operation. + /// :raises OSError: if an error happens during the operation. /// /// >>> store = Store() /// >>> store.add(Quad(NamedNode('http://example.com'), NamedNode('http://example.com/p'), Literal('1'), NamedNode('http://example.com/g'))) @@ -684,7 +684,7 @@ impl PyStore { /// Flushes are automatically done using background threads but might lag a little bit. /// /// :rtype: None - /// :raises IOError: if an I/O error happens during the flush. + /// :raises OSError: if an error happens during the flush. fn flush(&self, py: Python<'_>) -> PyResult<()> { py.allow_threads(|| self.inner.flush().map_err(map_storage_error)) } @@ -694,7 +694,7 @@ impl PyStore { /// Useful to call after a batch upload or another similar operation. /// /// :rtype: None - /// :raises IOError: if an I/O error happens during the optimization. + /// :raises OSError: if an error happens during the optimization. fn optimize(&self, py: Python<'_>) -> PyResult<()> { py.allow_threads(|| self.inner.optimize().map_err(map_storage_error)) } @@ -719,7 +719,7 @@ impl PyStore { /// :param target_directory: the directory name to save the database to. /// :type target_directory: str /// :rtype: None - /// :raises IOError: if an I/O error happens during the backup. + /// :raises OSError: if an error happens during the backup. fn backup(&self, target_directory: &str, py: Python<'_>) -> PyResult<()> { py.allow_threads(|| { self.inner @@ -830,7 +830,7 @@ pub fn extract_quads_pattern<'a>( pub fn map_storage_error(error: StorageError) -> PyErr { match error { - StorageError::Io(error) => PyIOError::new_err(error.to_string()), + StorageError::Io(error) => error.into(), _ => PyRuntimeError::new_err(error.to_string()), } } @@ -846,7 +846,7 @@ pub fn map_loader_error(error: LoaderError) -> PyErr { pub fn map_serializer_error(error: SerializerError) -> PyErr { match error { SerializerError::Storage(error) => map_storage_error(error), - SerializerError::Io(error) => PyIOError::new_err(error.to_string()), + SerializerError::Io(error) => error.into(), SerializerError::DatasetFormatExpected(_) => PyValueError::new_err(error.to_string()), } } From 7c227830e95d085d22ab60209ddf21181695536e Mon Sep 17 00:00:00 2001 From: Tpt Date: Fri, 18 Aug 2023 10:50:16 +0200 Subject: [PATCH 050/217] Upgrades RocksDB --- Cargo.lock | 518 +++++++++++++++++--------------------- lib/oxrdfxml/Cargo.toml | 2 +- lib/sparesults/Cargo.toml | 2 +- oxrocksdb-sys/Cargo.toml | 2 +- oxrocksdb-sys/rocksdb | 2 +- python/src/lib.rs | 3 +- testsuite/Cargo.toml | 6 +- 7 files changed, 245 insertions(+), 290 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1e3337d5..c27daea8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3,25 +3,25 @@ version = 3 [[package]] -name = "adler" -version = "1.0.2" +name = "addr2line" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +checksum = "f4fa78e18c64fce05e902adecd7a5eed15a5e0a3439f7b0e169f0252214865e3" +dependencies = [ + "gimli", +] [[package]] -name = "aho-corasick" -version = "0.7.20" +name = "adler" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" -dependencies = [ - "memchr", -] +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "aho-corasick" -version = "1.0.2" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41" +checksum = "6748e8def348ed4d14996fa801f4122cd763fff530258cdc03f64b25f89d3a5a" dependencies = [ "memchr", ] @@ -49,15 +49,15 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41ed9a86bf92ae6580e0a31281f65a1b1d867c0cc68d5346e2ae128dddfa6a7d" +checksum = "3a30da5c5f2d5e72842e00bcb57657162cdabef0931f40e2deb9b4140440cecd" [[package]] name = "anstyle-parse" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e765fd216e48e067936442276d1d57399e37bce53c264d6fefbe298080cb57ee" +checksum = "938874ff5980b03a87c5524b3ae5b59cf99b1d6bc836848df7bc5ada9643c333" dependencies = [ "utf8parse", ] @@ -68,24 +68,24 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" dependencies = [ - "windows-sys 0.48.0", + "windows-sys", ] [[package]] name = "anstyle-wincon" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "180abfa45703aebe0093f79badacc01b8fd4ea2e35118747e5811127f926e188" +checksum = "c677ab05e09154296dd37acecd46420c17b9713e8366facafa8fc0885167cf4c" dependencies = [ "anstyle", - "windows-sys 0.48.0", + "windows-sys", ] [[package]] name = "anyhow" -version = "1.0.71" +version = "1.0.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8" +checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" [[package]] name = "arbitrary" @@ -98,9 +98,9 @@ dependencies = [ [[package]] name = "assert_cmd" -version = "2.0.11" +version = "2.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86d6b683edf8d1119fe420a94f8a7e389239666aa72e65495d91c00462510151" +checksum = "88903cb14723e4d4003335bb7f8a14f27691649105346a0f0957466c096adfe6" dependencies = [ "anstyle", "bstr", @@ -132,6 +132,21 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "backtrace" +version = "0.3.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4319208da049c43661739c5fade2ba182f09d1dc2299b32298d3a31692b17e12" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + [[package]] name = "base64" version = "0.21.2" @@ -140,11 +155,11 @@ checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d" [[package]] name = "bindgen" -version = "0.65.1" +version = "0.66.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfdf7b466f9a4903edc73f95d6d2bcd5baf8ae620638762244d3f60143643cc5" +checksum = "f2b84e06fc203107bfbad243f4aba2af864eb7db3b1cf46ea0a023b0b433d2a7" dependencies = [ - "bitflags", + "bitflags 2.4.0", "cexpr", "clang-sys", "lazy_static", @@ -157,7 +172,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.18", + "syn 2.0.29", "which", ] @@ -167,6 +182,12 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitflags" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" + [[package]] name = "block-buffer" version = "0.10.4" @@ -178,12 +199,11 @@ dependencies = [ [[package]] name = "bstr" -version = "1.5.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a246e68bb43f6cd9db24bea052a53e40405417c5fb372e3d1a8a7f770a564ef5" +checksum = "6798148dccfbff0fae41c7574d2fa8f1ef3492fba0face179de5d8d447d67b05" dependencies = [ "memchr", - "once_cell", "regex-automata", "serde", ] @@ -208,11 +228,12 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.0.79" +version = "1.0.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" +checksum = "305fe645edc1442a0fa8b6726ba61d422798d37a52e12eaecf4b022ebbb88f01" dependencies = [ "jobserver", + "libc", ] [[package]] @@ -270,9 +291,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.3.3" +version = "4.3.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca8f255e4b8027970e78db75e78831229c9815fdbfa67eb1a1b777a62e24b4a0" +checksum = "b417ae4361bca3f5de378294fc7472d3c4ed86a5ef9f49e93ae722f432aae8d2" dependencies = [ "clap_builder", "clap_derive", @@ -281,27 +302,26 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.3.3" +version = "4.3.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acd4f3c17c83b0ba34ffbc4f8bbd74f079413f747f84a6f89292f138057e36ab" +checksum = "9c90dc0f0e42c64bff177ca9d7be6fcc9ddb0f26a6e062174a61c84dd6c644d4" dependencies = [ "anstream", "anstyle", - "bitflags", "clap_lex", "strsim", ] [[package]] name = "clap_derive" -version = "4.3.2" +version = "4.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8cd2b2a819ad6eec39e8f1d6b53001af1e5469f8c177579cdaeb313115b825f" +checksum = "54a9bb5758fc5dfe728d1019941681eccaf0cf8a4189b692a0ee2f2ecf90a050" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.29", ] [[package]] @@ -344,9 +364,9 @@ checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" [[package]] name = "cpufeatures" -version = "0.2.7" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e4c1eaa2012c47becbbad2ab175484c2a84d1185b566fb2cc5b8707343dfe58" +checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1" dependencies = [ "libc", ] @@ -457,7 +477,7 @@ checksum = "53e0efad4403bfc52dc201159c4b842a246a14b98c64b55dfd0f2d89729dfeb8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.29", ] [[package]] @@ -484,19 +504,19 @@ checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" [[package]] name = "either" -version = "1.8.1" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" +checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" [[package]] name = "errno" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" +checksum = "6b30f669a7961ef1631673d2766cc92f52d64f7ef354d4fe0ddfd30ed52f0f4f" dependencies = [ "errno-dragonfly", "libc", - "windows-sys 0.48.0", + "windows-sys", ] [[package]] @@ -511,9 +531,9 @@ dependencies = [ [[package]] name = "escargot" -version = "0.5.7" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5584ba17d7ab26a8a7284f13e5bd196294dd2f2d79773cff29b9e9edef601a6" +checksum = "768064bd3a0e2bedcba91dc87ace90beea91acc41b6a01a3ca8e9aa8827461bf" dependencies = [ "log", "once_cell", @@ -523,18 +543,15 @@ dependencies = [ [[package]] name = "fastrand" -version = "1.9.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" -dependencies = [ - "instant", -] +checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" [[package]] name = "flate2" -version = "1.0.26" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b9429470923de8e8cbd4d2dc513535400b4b3fef0319fb5c4e1f520a7bef743" +checksum = "c6c98ee8095e9d1dcbf2fcc6d95acccb90d1c81db1e44725c6a984b1dbdfb010" dependencies = [ "crc32fast", "miniz_oxide", @@ -596,6 +613,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "gimli" +version = "0.27.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e" + [[package]] name = "glob" version = "0.3.1" @@ -604,11 +627,11 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] name = "globset" -version = "0.4.10" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "029d74589adefde59de1a0c4f4732695c32805624aec7b68d91503d4dba79afc" +checksum = "759c97c1e17c55525b57192c06a267cda0ac5210b222d6b82189a2338fa1c13d" dependencies = [ - "aho-corasick 0.7.20", + "aho-corasick", "bstr", "fnv", "log", @@ -621,7 +644,7 @@ version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93e3af942408868f6934a7b85134a3230832b9977cf66125df2f9edcfce4ddcc" dependencies = [ - "bitflags", + "bitflags 1.3.2", "ignore", "walkdir", ] @@ -640,18 +663,9 @@ checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] name = "hermit-abi" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" -dependencies = [ - "libc", -] - -[[package]] -name = "hermit-abi" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" +checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" [[package]] name = "hex" @@ -698,36 +712,15 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfa799dd5ed20a7e349f3b4639aa80d74549c81716d9ec4f994c9b5815598306" -[[package]] -name = "instant" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "io-lifetimes" -version = "1.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" -dependencies = [ - "hermit-abi 0.3.1", - "libc", - "windows-sys 0.48.0", -] - [[package]] name = "is-terminal" -version = "0.4.7" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adcf93614601c8129ddf72e2d5633df827ba6551541c6d8c59520a371475be1f" +checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" dependencies = [ - "hermit-abi 0.3.1", - "io-lifetimes", + "hermit-abi", "rustix", - "windows-sys 0.48.0", + "windows-sys", ] [[package]] @@ -741,9 +734,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.6" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" +checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" [[package]] name = "jobserver" @@ -793,9 +786,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.146" +version = "0.2.147" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f92be4933c13fd498862a9e02a3055f8a8d9c039ce33db97306fd5a6caa7f29b" +checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" [[package]] name = "libloading" @@ -809,9 +802,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.3.8" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" +checksum = "57bcfdad1b858c2db7c38303a6d2ad4dfaf5eb53dfeb0910128b2c26d6158503" [[package]] name = "lock_api" @@ -825,9 +818,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.19" +version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4" +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" [[package]] name = "md-5" @@ -886,23 +879,32 @@ checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" [[package]] name = "num-traits" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2" dependencies = [ "autocfg", ] [[package]] name = "num_cpus" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi 0.2.6", + "hermit-abi", "libc", ] +[[package]] +name = "object" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bda667d9f2b5051b8833f59f3bf748b28ef54f850f4fcb389a252aa383866d1" +dependencies = [ + "memchr", +] + [[package]] name = "once_cell" version = "1.18.0" @@ -1147,9 +1149,9 @@ checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" [[package]] name = "pin-project-lite" -version = "0.2.9" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" +checksum = "12cc1b0bf1727a77a54b6654e7b5f1af8604923edc8b81885f8ec92f9e3f0a05" [[package]] name = "pkg-config" @@ -1159,9 +1161,9 @@ checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" [[package]] name = "plotters" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2538b639e642295546c50fcd545198c9d64ee2a38620a628724a3b266d5fbf97" +checksum = "d2c224ba00d7cadd4d5c660deaf2098e5e80e07846537c51f9cfa4be50c1fd45" dependencies = [ "num-traits", "plotters-backend", @@ -1172,15 +1174,15 @@ dependencies = [ [[package]] name = "plotters-backend" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "193228616381fecdc1224c62e96946dfbc73ff4384fba576e052ff8c1bea8142" +checksum = "9e76628b4d3a7581389a35d5b6e2139607ad7c75b17aed325f210aa91f4a9609" [[package]] name = "plotters-svg" -version = "0.3.3" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9a81d2759aae1dae668f783c308bc5c8ebd191ff4184aaa1b37f65a6ae5a56f" +checksum = "38f6d39893cca0701371e3c27294f09797214b86f1fb951b89ade8ec04e2abab" dependencies = [ "plotters-backend", ] @@ -1224,28 +1226,28 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.2.6" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b69d39aab54d069e7f2fe8cb970493e7834601ca2d8c65fd7bbd183578080d1" +checksum = "6c64d9ba0963cdcea2e1b2230fbae2bab30eb25a174be395c41e764bfb65dd62" dependencies = [ "proc-macro2", - "syn 2.0.18", + "syn 2.0.29", ] [[package]] name = "proc-macro2" -version = "1.0.60" +version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dec2b086b7a862cf4de201096214fa870344cf922b2b30c167badb3af3195406" +checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" dependencies = [ "unicode-ident", ] [[package]] name = "pyo3" -version = "0.19.0" +version = "0.19.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cffef52f74ec3b1a1baf295d9b8fcc3070327aefc39a6d00656b13c1d0b8885c" +checksum = "e681a6cfdc4adcc93b4d3cf993749a4552018ee0a9b65fc0ccfad74352c72a38" dependencies = [ "cfg-if", "indoc", @@ -1260,9 +1262,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.19.0" +version = "0.19.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "713eccf888fb05f1a96eb78c0dbc51907fee42b3377272dc902eb38985f418d5" +checksum = "076c73d0bc438f7a4ef6fdd0c3bb4732149136abd952b110ac93e4edb13a6ba5" dependencies = [ "once_cell", "target-lexicon", @@ -1270,9 +1272,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.19.0" +version = "0.19.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b2ecbdcfb01cbbf56e179ce969a048fd7305a66d4cdf3303e0da09d69afe4c3" +checksum = "e53cee42e77ebe256066ba8aa77eff722b3bb91f3419177cf4cd0f304d3284d9" dependencies = [ "libc", "pyo3-build-config", @@ -1280,9 +1282,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.19.0" +version = "0.19.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b78fdc0899f2ea781c463679b20cb08af9247febc8d052de941951024cd8aea0" +checksum = "dfeb4c99597e136528c6dd7d5e3de5434d1ceaf487436a3f03b2d56b6fc9efd1" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -1292,9 +1294,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.19.0" +version = "0.19.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60da7b84f1227c3e2fe7593505de274dcf4c8928b4e0a1c23d551a14e4e80a0f" +checksum = "947dc12175c254889edc0c02e399476c2f652b4b9ebd123aa655c224de259536" dependencies = [ "proc-macro2", "quote", @@ -1311,9 +1313,9 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.29.0" +version = "0.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81b9228215d82c7b61490fec1de287136b5de6f5700f6e58ea9ad61a7964ca51" +checksum = "eff6510e86862b57b210fd8cbe8ed3f0d7d600b9c2863cd4549a2e033c66e956" dependencies = [ "memchr", "tokio", @@ -1321,9 +1323,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.28" +version = "1.0.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488" +checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" dependencies = [ "proc-macro2", ] @@ -1386,31 +1388,37 @@ version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" dependencies = [ - "bitflags", + "bitflags 1.3.2", ] [[package]] name = "regex" -version = "1.8.4" +version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0ab3ca65655bb1e41f2a8c8cd662eb4fb035e67c3f78da1d61dffe89d07300f" +checksum = "81bc1d4caf89fac26a70747fe603c130093b53c773888797a6329091246d651a" dependencies = [ - "aho-corasick 1.0.2", + "aho-corasick", "memchr", + "regex-automata", "regex-syntax", ] [[package]] name = "regex-automata" -version = "0.1.10" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +checksum = "fed1ceff11a1dddaee50c9dc8e4938bd106e9d89ae372f192311e7da498e3b69" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] [[package]] name = "regex-syntax" -version = "0.7.2" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78" +checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2" [[package]] name = "ring" @@ -1444,6 +1452,12 @@ dependencies = [ "rio_api", ] +[[package]] +name = "rustc-demangle" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" + [[package]] name = "rustc-hash" version = "1.1.0" @@ -1452,16 +1466,15 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "rustix" -version = "0.37.20" +version = "0.38.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b96e891d04aa506a6d1f318d2771bcb1c7dfda84e126660ace067c9b474bb2c0" +checksum = "19ed4fa021d81c8392ce04db050a3da9a60299050b7ae1cf482d862b54a7218f" dependencies = [ - "bitflags", + "bitflags 2.4.0", "errno", - "io-lifetimes", "libc", "linux-raw-sys", - "windows-sys 0.48.0", + "windows-sys", ] [[package]] @@ -1478,9 +1491,9 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0167bac7a9f490495f3c33013e7722b53cb087ecbe082fb0c6387c96f634ea50" +checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" dependencies = [ "openssl-probe", "rustls-pemfile", @@ -1490,18 +1503,18 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d194b56d58803a43635bdc398cd17e383d6f71f9182b9a192c127ca42494a59b" +checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2" dependencies = [ "base64", ] [[package]] name = "ryu" -version = "1.0.13" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" +checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" [[package]] name = "same-file" @@ -1514,18 +1527,18 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.21" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "713cfb06c7059f3588fb8044c0fad1d09e3c01d225e25b9220dbfdcf16dbb1b3" +checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" dependencies = [ - "windows-sys 0.42.0", + "windows-sys", ] [[package]] name = "scopeguard" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "sct" @@ -1539,11 +1552,11 @@ dependencies = [ [[package]] name = "security-framework" -version = "2.9.1" +version = "2.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fc758eb7bffce5b308734e9b0c1468893cae9ff70ebf13e7090be8dcbcc83a8" +checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" dependencies = [ - "bitflags", + "bitflags 1.3.2", "core-foundation", "core-foundation-sys", "libc", @@ -1552,9 +1565,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.9.0" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f51d0c0d83bec45f16480d0ce0058397a69e48fcdc52d1dc8855fb68acbd31a7" +checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" dependencies = [ "core-foundation-sys", "libc", @@ -1562,29 +1575,29 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.164" +version = "1.0.183" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e8c8cf938e98f769bc164923b06dce91cea1751522f46f8466461af04c9027d" +checksum = "32ac8da02677876d532745a130fc9d8e6edfa81a269b107c5b00829b91d8eb3c" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.164" +version = "1.0.183" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9735b638ccc51c28bf6914d90a2e9725b377144fc612c49a611fddd1b631d68" +checksum = "aafe972d60b0b9bee71a91b92fee2d4fb3c9d7e8f6b179aa99f27203d99a4816" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.29", ] [[package]] name = "serde_json" -version = "1.0.96" +version = "1.0.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1" +checksum = "693151e1ac27563d6dbcec9dee9fbd5da8539b20fa14ad3752b2e6d363ace360" dependencies = [ "itoa", "ryu", @@ -1604,9 +1617,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.6" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" +checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8" dependencies = [ "cfg-if", "cpufeatures", @@ -1627,9 +1640,9 @@ checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de" [[package]] name = "smallvec" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" +checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" [[package]] name = "sparesults" @@ -1692,9 +1705,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.18" +version = "2.0.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32d41677bcbe24c20c52e7c70b0d8db04134c5d1066bf98662e2871ad200ea3e" +checksum = "c324c494eba9d92503e6f1ef2e6df781e78f6a7705a0202d9801b198807d518a" dependencies = [ "proc-macro2", "quote", @@ -1703,22 +1716,21 @@ dependencies = [ [[package]] name = "target-lexicon" -version = "0.12.7" +version = "0.12.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd1ba337640d60c3e96bc6f0638a939b9c9a7f2c316a1598c279828b3d1dc8c5" +checksum = "9d0e916b1148c8e263850e1ebcbd046f333e0683c724876bb0da63ea4373dc8a" [[package]] name = "tempfile" -version = "3.6.0" +version = "3.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31c0432476357e58790aaa47a8efb0c5138f137343f3b5f23bd36a27e3b0a6d6" +checksum = "dc02fddf48964c42031a0b3fe0428320ecf3a73c401040fc0096f97794310651" dependencies = [ - "autocfg", "cfg-if", "fastrand", "redox_syscall", "rustix", - "windows-sys 0.48.0", + "windows-sys", ] [[package]] @@ -1759,9 +1771,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.22" +version = "0.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea9e1b3cf1243ae005d9e74085d4d542f3125458f3a81af210d901dcd7411efd" +checksum = "59e399c068f43a5d116fedaf73b203fa4f9c519f17e2b34f63221d3792f81446" dependencies = [ "itoa", "serde", @@ -1777,9 +1789,9 @@ checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" [[package]] name = "time-macros" -version = "0.2.9" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "372950940a5f07bf38dbe211d7283c9e6d7327df53794992d293e534c733d09b" +checksum = "96ba15a897f3c86766b757e5ac7221554c6750054d74d5b28844fce5fb36a6c4" dependencies = [ "time-core", ] @@ -1811,15 +1823,14 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.28.2" +version = "1.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94d7b1cfd2aa4011f2de74c2c4c63665e27a71006b0a192dcd2710272e73dfa2" +checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" dependencies = [ - "autocfg", + "backtrace", "bytes", "pin-project-lite", "tokio-macros", - "windows-sys 0.48.0", ] [[package]] @@ -1830,7 +1841,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.29", ] [[package]] @@ -1847,9 +1858,9 @@ checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" [[package]] name = "unicode-ident" -version = "1.0.9" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b15811caf2415fb889178633e7724bad2509101cde276048e013b9def5e51fa0" +checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" [[package]] name = "unicode-normalization" @@ -1947,7 +1958,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.29", "wasm-bindgen-shared", ] @@ -1969,7 +1980,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.29", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -2054,21 +2065,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -[[package]] -name = "windows-sys" -version = "0.42.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" -dependencies = [ - "windows_aarch64_gnullvm 0.42.2", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm 0.42.2", - "windows_x86_64_msvc 0.42.2", -] - [[package]] name = "windows-sys" version = "0.48.0" @@ -2080,117 +2076,75 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.48.0" +version = "0.48.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" +checksum = "d92ecb8ae0317859f509f17b19adc74b0763b0fa3b085dea8ed01085c8dac222" dependencies = [ - "windows_aarch64_gnullvm 0.48.0", - "windows_aarch64_msvc 0.48.0", - "windows_i686_gnu 0.48.0", - "windows_i686_msvc 0.48.0", - "windows_x86_64_gnu 0.48.0", - "windows_x86_64_gnullvm 0.48.0", - "windows_x86_64_msvc 0.48.0", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", ] [[package]] name = "windows_aarch64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.48.0" +version = "0.48.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" +checksum = "d14b0ee96970be7108701212f097ce67ca772fd84cb0ffbc86d26a94e77ba929" [[package]] name = "windows_aarch64_msvc" -version = "0.42.2" +version = "0.48.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" +checksum = "1332277d49f440c8fc6014941e320ee47ededfcce10cb272728470f56cc092c9" [[package]] name = "windows_i686_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" - -[[package]] -name = "windows_i686_gnu" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" - -[[package]] -name = "windows_i686_msvc" -version = "0.42.2" +version = "0.48.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" +checksum = "d992130ac399d56f02c20564e9975ac5ba08cb25cb832849bbc0d736a101abe5" [[package]] name = "windows_i686_msvc" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.42.2" +version = "0.48.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" +checksum = "962e96d0fa4b4773c63977977ea6564f463fb10e34a6e07360428b53ae7a3f71" [[package]] name = "windows_x86_64_gnu" -version = "0.48.0" +version = "0.48.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" +checksum = "30652a53018a48a9735fbc2986ff0446c37bc8bed0d3f98a0ed4d04cdb80027e" [[package]] name = "windows_x86_64_gnullvm" -version = "0.42.2" +version = "0.48.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" +checksum = "b5bb3f0331abfe1a95af56067f1e64b3791b55b5373b03869560b6025de809bf" [[package]] name = "windows_x86_64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.48.0" +version = "0.48.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" +checksum = "bd1df36d9fd0bbe4849461de9b969f765170f4e0f90497d580a235d515722b10" [[package]] name = "zstd" -version = "0.12.3+zstd.1.5.2" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76eea132fb024e0e13fd9c2f5d5d595d8a967aa72382ac2f9d39fcc95afd0806" +checksum = "1a27595e173641171fc74a1232b7b1c7a7cb6e18222c11e9dfb9888fa424c53c" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "6.0.5+zstd.1.5.4" +version = "6.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d56d9e60b4b1758206c238a10165fbcae3ca37b01744e394c463463f6529d23b" +checksum = "ee98ffd0b48ee95e6c5168188e44a54550b1564d9d530ee21d5f0eaed1069581" dependencies = [ "libc", "zstd-sys", diff --git a/lib/oxrdfxml/Cargo.toml b/lib/oxrdfxml/Cargo.toml index c411bc9f..371eb19f 100644 --- a/lib/oxrdfxml/Cargo.toml +++ b/lib/oxrdfxml/Cargo.toml @@ -22,7 +22,7 @@ async-tokio = ["dep:tokio", "quick-xml/async-tokio"] oxrdf = { version = "0.2.0-alpha.1-dev", path = "../oxrdf" } oxilangtag = "0.1" oxiri = "0.2" -quick-xml = "0.29" +quick-xml = "0.30" tokio = { version = "1", optional = true, features = ["io-util"] } [dev-dependencies] diff --git a/lib/sparesults/Cargo.toml b/lib/sparesults/Cargo.toml index ef023da7..54e3032a 100644 --- a/lib/sparesults/Cargo.toml +++ b/lib/sparesults/Cargo.toml @@ -21,7 +21,7 @@ rdf-star = ["oxrdf/rdf-star"] [dependencies] json-event-parser = "0.1" oxrdf = { version = "0.2.0-alpha.1-dev", path="../oxrdf" } -quick-xml = "0.29" +quick-xml = "0.30" [package.metadata.docs.rs] all-features = true diff --git a/oxrocksdb-sys/Cargo.toml b/oxrocksdb-sys/Cargo.toml index d25d237e..8cbdba15 100644 --- a/oxrocksdb-sys/Cargo.toml +++ b/oxrocksdb-sys/Cargo.toml @@ -18,5 +18,5 @@ links = "rocksdb" libc = "0.2" [build-dependencies] -bindgen = "0.65" +bindgen = "0.66" cc = { version = "1", features = ["parallel"] } diff --git a/oxrocksdb-sys/rocksdb b/oxrocksdb-sys/rocksdb index 443333d8..5f2d6f0c 160000 --- a/oxrocksdb-sys/rocksdb +++ b/oxrocksdb-sys/rocksdb @@ -1 +1 @@ -Subproject commit 443333d8c059c87db408ec2d11685db00031b30a +Subproject commit 5f2d6f0cba9858130be48ae129dd9c9dcafe0f97 diff --git a/python/src/lib.rs b/python/src/lib.rs index 736cbfc9..bdc38681 100644 --- a/python/src/lib.rs +++ b/python/src/lib.rs @@ -1,7 +1,8 @@ #![allow( clippy::used_underscore_binding, clippy::unused_self, - clippy::trivially_copy_pass_by_ref + clippy::trivially_copy_pass_by_ref, + unused_qualifications )] mod io; diff --git a/testsuite/Cargo.toml b/testsuite/Cargo.toml index 23bfb246..af2e859d 100644 --- a/testsuite/Cargo.toml +++ b/testsuite/Cargo.toml @@ -14,17 +14,17 @@ publish = false [dependencies] anyhow = "1" clap = { version = "4", features = ["derive"] } -time = { version = "0.3", features = ["formatting"] } oxigraph = { path = "../lib" } oxttl = { path= "../lib/oxttl" } sparopt = { path = "../lib/sparopt" } spargebra = { path = "../lib/spargebra" } text-diff = "0.4" -rio_api = "0.8" -rio_turtle = "0.8" +time = { version = "=0.3.23", features = ["formatting"] } [dev-dependencies] criterion = "0.5" +rio_api = "0.8" +rio_turtle = "0.8" [[bench]] name = "parser" From c6e55c706af3b10ae93940df42f2d070300960c4 Mon Sep 17 00:00:00 2001 From: Tpt Date: Sat, 19 Aug 2023 12:27:01 +0200 Subject: [PATCH 051/217] RDF serialization: removes implicit flush Allows to the user to control flushing --- js/src/store.rs | 16 +++++----- lib/oxrdfio/src/serializer.rs | 21 ++++++------- lib/src/io/write.rs | 4 +-- lib/src/store.rs | 21 ++++++------- lib/tests/store.rs | 3 +- python/src/io.rs | 28 +++++++++++++----- python/src/store.rs | 8 ++++- server/src/main.rs | 56 ++++++++++++++++++++--------------- 8 files changed, 90 insertions(+), 67 deletions(-) diff --git a/js/src/store.rs b/js/src/store.rs index 2ec83cd6..bffac273 100644 --- a/js/src/store.rs +++ b/js/src/store.rs @@ -181,14 +181,14 @@ impl JsStore { let Some(format) = RdfFormat::from_media_type(mime_type) else { return Err(format_err!("Not supported MIME type: {mime_type}")); }; - let mut buffer = Vec::new(); - if let Some(from_graph_name) = FROM_JS.with(|c| c.to_optional_term(from_graph_name))? { - self.store - .dump_graph(&mut buffer, format, &GraphName::try_from(from_graph_name)?) - } else { - self.store.dump_dataset(&mut buffer, format) - } - .map_err(to_err)?; + let buffer = + if let Some(from_graph_name) = FROM_JS.with(|c| c.to_optional_term(from_graph_name))? { + self.store + .dump_graph(Vec::new(), format, &GraphName::try_from(from_graph_name)?) + } else { + self.store.dump_dataset(Vec::new(), format) + } + .map_err(to_err)?; String::from_utf8(buffer).map_err(to_err) } } diff --git a/lib/oxrdfio/src/serializer.rs b/lib/oxrdfio/src/serializer.rs index 35a9f29e..6ec4e666 100644 --- a/lib/oxrdfio/src/serializer.rs +++ b/lib/oxrdfio/src/serializer.rs @@ -19,7 +19,7 @@ use oxttl::turtle::ToTokioAsyncWriteTurtleWriter; use oxttl::turtle::{ToWriteTurtleWriter, TurtleSerializer}; use std::io::{self, Write}; #[cfg(feature = "async-tokio")] -use tokio::io::{AsyncWrite, AsyncWriteExt}; +use tokio::io::AsyncWrite; /// A serializer for RDF serialization formats. /// @@ -214,15 +214,16 @@ impl ToWriteQuadWriter { } /// Writes the last bytes of the file - pub fn finish(self) -> io::Result<()> { - match self.formatter { + /// + /// Note that this function does not flush the writer. You need to do that if you are using a [`BufWriter`](io::BufWriter). + pub fn finish(self) -> io::Result { + Ok(match self.formatter { ToWriteQuadWriterKind::NQuads(writer) => writer.finish(), ToWriteQuadWriterKind::NTriples(writer) => writer.finish(), ToWriteQuadWriterKind::RdfXml(writer) => writer.finish()?, ToWriteQuadWriterKind::TriG(writer) => writer.finish()?, ToWriteQuadWriterKind::Turtle(writer) => writer.finish()?, - } - .flush() + }) } } @@ -296,16 +297,16 @@ impl ToTokioAsyncWriteQuadWriter { } /// Writes the last bytes of the file - pub async fn finish(self) -> io::Result<()> { - match self.formatter { + /// + /// Note that this function does not flush the writer. You need to do that if you are using a [`BufWriter`](io::BufWriter). + pub async fn finish(self) -> io::Result { + Ok(match self.formatter { ToTokioAsyncWriteQuadWriterKind::NQuads(writer) => writer.finish(), ToTokioAsyncWriteQuadWriterKind::NTriples(writer) => writer.finish(), ToTokioAsyncWriteQuadWriterKind::RdfXml(writer) => writer.finish().await?, ToTokioAsyncWriteQuadWriterKind::TriG(writer) => writer.finish().await?, ToTokioAsyncWriteQuadWriterKind::Turtle(writer) => writer.finish().await?, - } - .flush() - .await + }) } } diff --git a/lib/src/io/write.rs b/lib/src/io/write.rs index 7955f3a3..7c308c86 100644 --- a/lib/src/io/write.rs +++ b/lib/src/io/write.rs @@ -86,7 +86,7 @@ impl TripleWriter { /// Writes the last bytes of the file pub fn finish(self) -> io::Result<()> { - self.writer.finish() + self.writer.finish()?.flush() } } @@ -170,6 +170,6 @@ impl QuadWriter { /// Writes the last bytes of the file pub fn finish(self) -> io::Result<()> { - self.writer.finish() + self.writer.finish()?.flush() } } diff --git a/lib/src/store.rs b/lib/src/store.rs index 0667f019..f8ea0af5 100644 --- a/lib/src/store.rs +++ b/lib/src/store.rs @@ -617,18 +617,17 @@ impl Store { /// assert_eq!(file, buffer.as_slice()); /// # std::io::Result::Ok(()) /// ``` - pub fn dump_graph<'a>( + pub fn dump_graph<'a, W: Write>( &self, - write: impl Write, + write: W, format: impl Into, from_graph_name: impl Into>, - ) -> Result<(), SerializerError> { + ) -> Result { let mut writer = RdfSerializer::from_format(format.into()).serialize_to_write(write); for quad in self.quads_for_pattern(None, None, None, Some(from_graph_name.into())) { writer.write_triple(quad?.as_ref())?; } - writer.finish()?; - Ok(()) + Ok(writer.finish()?) } /// Dumps the store into a file. @@ -642,16 +641,15 @@ impl Store { /// let store = Store::new()?; /// store.load_dataset(file, RdfFormat::NQuads, None)?; /// - /// let mut buffer = Vec::new(); - /// store.dump_dataset(&mut buffer, RdfFormat::NQuads)?; + /// let buffer = store.dump_dataset(Vec::new(), RdfFormat::NQuads)?; /// assert_eq!(file, buffer.as_slice()); /// # std::io::Result::Ok(()) /// ``` - pub fn dump_dataset( + pub fn dump_dataset( &self, - write: impl Write, + write: W, format: impl Into, - ) -> Result<(), SerializerError> { + ) -> Result { let format = format.into(); if !format.supports_datasets() { return Err(SerializerError::DatasetFormatExpected(format)); @@ -660,8 +658,7 @@ impl Store { for quad in self.iter() { writer.write_quad(&quad?)?; } - writer.finish()?; - Ok(()) + Ok(writer.finish()?) } /// Returns all the store named graphs. diff --git a/lib/tests/store.rs b/lib/tests/store.rs index 1b28a03d..3deeeaac 100644 --- a/lib/tests/store.rs +++ b/lib/tests/store.rs @@ -226,8 +226,7 @@ fn test_dump_dataset() -> Result<(), Box> { store.insert(q)?; } - let mut buffer = Vec::new(); - store.dump_dataset(&mut buffer, RdfFormat::NQuads)?; + let buffer = store.dump_dataset(Vec::new(), RdfFormat::NQuads)?; assert_eq!( buffer.into_iter().filter(|c| *c == b'\n').count(), NUMBER_OF_TRIPLES diff --git a/python/src/io.rs b/python/src/io.rs index 97bee867..9585dcdc 100644 --- a/python/src/io.rs +++ b/python/src/io.rs @@ -128,7 +128,7 @@ pub fn serialize(input: &PyAny, output: PyObject, mime_type: &str, py: Python<'_ } else { PyWritable::from_data(output) }; - let mut writer = RdfSerializer::from_format(format).serialize_to_write(output); + let mut writer = RdfSerializer::from_format(format).serialize_to_write(BufWriter::new(output)); for i in input.iter()? { let i = i?; if let Ok(triple) = i.extract::>() { @@ -145,7 +145,13 @@ pub fn serialize(input: &PyAny, output: PyObject, mime_type: &str, py: Python<'_ } .map_err(map_io_err)?; } - writer.finish().map_err(map_io_err) + writer + .finish() + .map_err(map_io_err)? + .into_inner() + .map_err(|e| map_io_err(e.into_error()))? + .close() + .map_err(map_io_err) } #[pyclass(name = "QuadReader", module = "pyoxigraph")] @@ -202,19 +208,25 @@ impl Read for PyReadable { } pub enum PyWritable { - Io(BufWriter), - File(BufWriter), + Io(PyIo), + File(File), } impl PyWritable { pub fn from_file(file: &Path, py: Python<'_>) -> io::Result { - Ok(Self::File(BufWriter::new( - py.allow_threads(|| File::create(file))?, - ))) + Ok(Self::File(py.allow_threads(|| File::create(file))?)) } pub fn from_data(data: PyObject) -> Self { - Self::Io(BufWriter::new(PyIo(data))) + Self::Io(PyIo(data)) + } + + pub fn close(mut self) -> io::Result<()> { + self.flush()?; + if let Self::File(file) = self { + file.sync_all()?; + } + Ok(()) } } diff --git a/python/src/store.rs b/python/src/store.rs index d689c997..da20686a 100644 --- a/python/src/store.rs +++ b/python/src/store.rs @@ -9,6 +9,7 @@ use oxigraph::sparql::Update; use oxigraph::store::{self, LoaderError, SerializerError, StorageError, Store}; use pyo3::exceptions::{PyRuntimeError, PyValueError}; use pyo3::prelude::*; +use std::io::BufWriter; use std::path::PathBuf; /// RDF store. @@ -537,12 +538,17 @@ impl PyStore { PyWritable::from_data(output) }; py.allow_threads(|| { + let output = BufWriter::new(output); if let Some(from_graph_name) = &from_graph_name { self.inner.dump_graph(output, format, from_graph_name) } else { self.inner.dump_dataset(output, format) } - .map_err(map_serializer_error) + .map_err(map_serializer_error)? + .into_inner() + .map_err(|e| map_io_err(e.into_error()))? + .close() + .map_err(map_io_err) }) } diff --git a/server/src/main.rs b/server/src/main.rs index 5749d7f6..67a5d527 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -441,17 +441,16 @@ pub fn main() -> anyhow::Result<()> { None }; if let Some(file) = file { - dump( + close_file_writer(dump( &store, - BufWriter::new(File::create(&file).map_err(|e| { - anyhow!("Error while opening file {}: {e}", file.display()) - })?), + BufWriter::new(File::create(file)?), format, graph, - ) + )?)?; } else { - dump(&store, stdout().lock(), format, graph) + dump(&store, stdout().lock(), format, graph)?.flush()?; } + Ok(()) } Command::Query { query, @@ -509,7 +508,7 @@ pub fn main() -> anyhow::Result<()> { for solution in solutions { writer.write(&solution?)?; } - writer.finish()?; + close_file_writer(writer.finish()?)?; } else { let mut writer = QueryResultsSerializer::from_format(format) .solutions_writer( @@ -519,8 +518,7 @@ pub fn main() -> anyhow::Result<()> { for solution in solutions { writer.write(&solution?)?; } - #[allow(clippy::let_underscore_must_use)] - let _ = writer.finish()?; + writer.finish()?.flush()?; } } QueryResults::Boolean(result) => { @@ -542,14 +540,16 @@ pub fn main() -> anyhow::Result<()> { bail!("The --results-format option must be set when writing to stdout") }; if let Some(results_file) = results_file { - QueryResultsSerializer::from_format(format).write_boolean_result( - BufWriter::new(File::create(results_file)?), - result, + close_file_writer( + QueryResultsSerializer::from_format(format).write_boolean_result( + BufWriter::new(File::create(results_file)?), + result, + )?, )?; } else { - #[allow(clippy::let_underscore_must_use)] - let _ = QueryResultsSerializer::from_format(format) - .write_boolean_result(stdout().lock(), result)?; + QueryResultsSerializer::from_format(format) + .write_boolean_result(stdout().lock(), result)? + .flush()?; } } QueryResults::Graph(triples) => { @@ -567,13 +567,13 @@ pub fn main() -> anyhow::Result<()> { for triple in triples { writer.write_triple(triple?.as_ref())?; } - writer.finish()?; + close_file_writer(writer.finish()?)?; } else { let mut writer = serializer.serialize_to_write(stdout().lock()); for triple in triples { writer.write_triple(triple?.as_ref())?; } - writer.finish()?; + writer.finish()?.flush()?; } } } @@ -585,13 +585,14 @@ pub fn main() -> anyhow::Result<()> { .extension() .and_then(OsStr::to_str) { Some("json") => { - explanation.write_in_json(file)?; + explanation.write_in_json(&mut file)?; }, Some("txt") => { write!(file, "{:?}", explanation)?; }, _ => bail!("The given explanation file {} must have an extension that is .json or .txt", explain_file.display()) } + close_file_writer(file)?; } else if explain || stats { eprintln!("{:#?}", explanation); } @@ -648,19 +649,18 @@ fn bulk_load( Ok(()) } -fn dump( +fn dump( store: &Store, - writer: impl Write, + writer: W, format: RdfFormat, from_graph_name: Option>, -) -> anyhow::Result<()> { +) -> anyhow::Result { ensure!(format.supports_datasets() || from_graph_name.is_some(), "The --graph option is required when writing a format not supporting datasets like NTriples, Turtle or RDF/XML"); - if let Some(from_graph_name) = from_graph_name { + Ok(if let Some(from_graph_name) = from_graph_name { store.dump_graph(writer, format, from_graph_name) } else { store.dump_dataset(writer, format) - }?; - Ok(()) + }?) } fn format_from_path( @@ -1631,6 +1631,14 @@ impl Write for ReadForWriteWriter { } } +fn close_file_writer(writer: BufWriter) -> io::Result<()> { + let mut file = writer + .into_inner() + .map_err(io::IntoInnerError::into_error)?; + file.flush()?; + file.sync_all() +} + #[cfg(target_os = "linux")] fn systemd_notify_ready() -> io::Result<()> { if let Some(path) = env::var_os("NOTIFY_SOCKET") { From 5fee36e587cfb26c9865db8f06a2c19690babb05 Mon Sep 17 00:00:00 2001 From: Tpt Date: Sat, 19 Aug 2023 11:09:56 +0200 Subject: [PATCH 052/217] Improves naming of artifacts built by the CI --- .github/workflows/artifacts.yml | 25 ++++++++++++++++--------- .github/workflows/manylinux_build.sh | 8 ++++---- .github/workflows/musllinux_build.sh | 8 ++++---- 3 files changed, 24 insertions(+), 17 deletions(-) diff --git a/.github/workflows/artifacts.yml b/.github/workflows/artifacts.yml index a72c065d..bf91b537 100644 --- a/.github/workflows/artifacts.yml +++ b/.github/workflows/artifacts.yml @@ -37,7 +37,7 @@ jobs: path: target/release/oxigraph_server - uses: actions/upload-artifact@v3 with: - name: oxigraph_server_aarch64-linux_gnu + name: oxigraph_server_aarch64_linux_gnu path: target/aarch64-unknown-linux-gnu/release/oxigraph_server - run: mv target/release/oxigraph_server oxigraph_server_${{ github.event.release.tag_name }}_x86_64_linux_gnu if: github.event_name == 'release' @@ -126,7 +126,7 @@ jobs: if: github.event_name == 'release' || matrix.architecture == 'x86_64' - uses: actions/upload-artifact@v3 with: - name: pyoxigraph_wheel_x86_64_linux + name: pyoxigraph_wheel_x86_64_linux_gnu path: target/wheels/*.whl - uses: pypa/gh-action-pypi-publish@release/v1 with: @@ -181,15 +181,19 @@ jobs: cache: pip cache-dependency-path: '**/requirements.dev.txt' - run: pip install -r python/requirements.dev.txt - - run: maturin build --release -m python/Cargo.toml --features abi3 + - run: maturin build --release --features abi3 + working-directory: ./python - run: pip install --no-index --find-links=target/wheels/ pyoxigraph - run: rm -r target/wheels - run: python generate_stubs.py pyoxigraph pyoxigraph.pyi --black working-directory: ./python - - run: maturin build --release --target universal2-apple-darwin -m python/Cargo.toml --features abi3 - - run: maturin build --release -m python/Cargo.toml --features abi3 + - run: maturin build --release --target universal2-apple-darwin --features abi3 + working-directory: ./python + - run: maturin build --release --features abi3 + working-directory: ./python if: github.event_name == 'release' - - run: maturin build --release --target aarch64-apple-darwin -m python/Cargo.toml --features abi3 + - run: maturin build --release --target aarch64-apple-darwin --features abi3 + working-directory: ./python if: github.event_name == 'release' - uses: actions/upload-artifact@v3 with: @@ -216,13 +220,16 @@ jobs: cache-dependency-path: '**/requirements.dev.txt' - run: Remove-Item -LiteralPath "C:\msys64\" -Force -Recurse - run: pip install -r python/requirements.dev.txt - - run: maturin build --release -m python/Cargo.toml --features abi3 + - run: maturin build --release --features abi3 + working-directory: ./python - run: pip install --no-index --find-links=target/wheels/ pyoxigraph - run: rm -r target/wheels - run: python generate_stubs.py pyoxigraph pyoxigraph.pyi --black working-directory: ./python - - run: maturin build --release -m python/Cargo.toml --features abi3 - - run: maturin sdist -m python/Cargo.toml + - run: maturin build --release --features abi3 + working-directory: ./python + - run: maturin sdist + working-directory: ./python - uses: actions/upload-artifact@v3 with: name: pyoxigraph_wheel_x86_64_windows diff --git a/.github/workflows/manylinux_build.sh b/.github/workflows/manylinux_build.sh index 1ed3f3e2..ae72000d 100644 --- a/.github/workflows/manylinux_build.sh +++ b/.github/workflows/manylinux_build.sh @@ -8,14 +8,14 @@ chmod +x rustup-init source "$HOME/.cargo/env" export PATH="${PATH}:/opt/python/cp37-cp37m/bin:/opt/python/cp38-cp38/bin:/opt/python/cp39-cp39/bin:/opt/python/cp310-cp310/bin:/opt/python/cp311-cp311/bin" cd python -python3.10 -m venv venv +python3.11 -m venv venv source venv/bin/activate pip install -r requirements.dev.txt -maturin develop --release -m Cargo.toml +maturin develop --release python generate_stubs.py pyoxigraph pyoxigraph.pyi --black -maturin build --release -m Cargo.toml --features abi3 --compatibility manylinux2014 +maturin build --release --features abi3 --compatibility manylinux2014 if [ %for_each_version% ]; then for VERSION in 7 8 9 10 11; do - maturin build --release -m Cargo.toml --interpreter "python3.$VERSION" --compatibility manylinux2014 + maturin build --release --interpreter "python3.$VERSION" --compatibility manylinux2014 done fi diff --git a/.github/workflows/musllinux_build.sh b/.github/workflows/musllinux_build.sh index 7dd2fb4c..8abc5b0d 100644 --- a/.github/workflows/musllinux_build.sh +++ b/.github/workflows/musllinux_build.sh @@ -6,14 +6,14 @@ chmod +x rustup-init source "$HOME/.cargo/env" export PATH="${PATH}:/opt/python/cp37-cp37m/bin:/opt/python/cp38-cp38/bin:/opt/python/cp39-cp39/bin:/opt/python/cp310-cp310/bin:/opt/python/cp311-cp311/bin" cd python -python3.10 -m venv venv +python3.11 -m venv venv source venv/bin/activate pip install -r requirements.dev.txt -maturin develop --release -m Cargo.toml +maturin develop --release python generate_stubs.py pyoxigraph pyoxigraph.pyi --black -maturin build --release -m Cargo.toml --features abi3 --compatibility musllinux_1_1 +maturin build --release --features abi3 --compatibility musllinux_1_1 if [ %for_each_version% ]; then for VERSION in 7 8 9 10 11; do - maturin build --release -m Cargo.toml --interpreter "python3.$VERSION" --compatibility musllinux_1_1 + maturin build --release --interpreter "python3.$VERSION" --compatibility musllinux_1_1 done fi From 807cf0d436167b78979548a7adfcb26dd923a291 Mon Sep 17 00:00:00 2001 From: Tpt Date: Sun, 16 Jul 2023 21:34:22 +0200 Subject: [PATCH 053/217] Isomorphism: make sure to also take quoted triples into account and fixes interning stability issue --- lib/oxrdf/src/dataset.rs | 323 +++++++++++++++++++++++++------------ lib/oxrdf/src/interning.rs | 43 +++-- 2 files changed, 251 insertions(+), 115 deletions(-) diff --git a/lib/oxrdf/src/dataset.rs b/lib/oxrdf/src/dataset.rs index 95ecef75..dbf82c86 100644 --- a/lib/oxrdf/src/dataset.rs +++ b/lib/oxrdf/src/dataset.rs @@ -294,6 +294,18 @@ impl Dataset { .map(|(o, s, p, g)| (s, p, o, g)) } + pub fn quads_for_graph_name<'a, 'b>( + &'a self, + graph_name: impl Into>, + ) -> impl Iterator> + 'a { + let graph_name = self + .encoded_graph_name(graph_name) + .unwrap_or_else(InternedGraphName::impossible); + + self.interned_quads_for_graph_name(&graph_name) + .map(move |q| self.decode_spog(q)) + } + fn interned_quads_for_graph_name( &self, graph_name: &InternedGraphName, @@ -526,9 +538,12 @@ impl Dataset { /// Warning 3: This implementation worst-case complexity is in *O(b!)* with *b* the number of blank nodes in the input dataset. pub fn canonicalize(&mut self) { let bnodes = self.blank_nodes(); - let (hash, partition) = - self.hash_bnodes(bnodes.into_iter().map(|bnode| (bnode, 0)).collect()); - let new_quads = self.distinguish(&hash, &partition); + let quads_per_blank_node = self.quads_per_blank_nodes(); + let (hash, partition) = self.hash_bnodes( + bnodes.into_iter().map(|bnode| (bnode, 0)).collect(), + &quads_per_blank_node, + ); + let new_quads = self.distinguish(&hash, &partition, &quads_per_blank_node); self.clear(); for quad in new_quads { self.insert_encoded(quad); @@ -573,107 +588,168 @@ impl Dataset { } } + fn quads_per_blank_nodes(&self) -> QuadsPerBlankNode { + let mut map: HashMap<_, Vec<_>> = HashMap::new(); + for quad in &self.spog { + if let InternedSubject::BlankNode(bnode) = &quad.0 { + map.entry(*bnode).or_default().push(quad.clone()); + } + #[cfg(feature = "rdf-star")] + if let InternedSubject::Triple(t) = &quad.0 { + Self::add_quad_with_quoted_triple_to_quad_per_blank_nodes_map(quad, t, &mut map); + } + if let InternedTerm::BlankNode(bnode) = &quad.2 { + map.entry(*bnode).or_default().push(quad.clone()); + } + #[cfg(feature = "rdf-star")] + if let InternedTerm::Triple(t) = &quad.2 { + Self::add_quad_with_quoted_triple_to_quad_per_blank_nodes_map(quad, t, &mut map); + } + if let InternedGraphName::BlankNode(bnode) = &quad.3 { + map.entry(*bnode).or_default().push(quad.clone()); + } + } + map + } + + #[cfg(feature = "rdf-star")] + fn add_quad_with_quoted_triple_to_quad_per_blank_nodes_map( + quad: &( + InternedSubject, + InternedNamedNode, + InternedTerm, + InternedGraphName, + ), + triple: &InternedTriple, + map: &mut QuadsPerBlankNode, + ) { + if let InternedSubject::BlankNode(bnode) = &triple.subject { + map.entry(*bnode).or_default().push(quad.clone()); + } + if let InternedSubject::Triple(t) = &triple.subject { + Self::add_quad_with_quoted_triple_to_quad_per_blank_nodes_map(quad, t, map); + } + if let InternedTerm::BlankNode(bnode) = &triple.object { + map.entry(*bnode).or_default().push(quad.clone()); + } + if let InternedTerm::Triple(t) = &triple.object { + Self::add_quad_with_quoted_triple_to_quad_per_blank_nodes_map(quad, t, map); + } + } + fn hash_bnodes( &self, mut hashes: HashMap, + quads_per_blank_node: &QuadsPerBlankNode, ) -> ( HashMap, Vec<(u64, Vec)>, ) { let mut to_hash = Vec::new(); - let mut partition: HashMap> = HashMap::new(); - let mut partition_len = 0; - loop { - //TODO: improve termination - let mut new_hashes = HashMap::new(); - for (bnode, old_hash) in &hashes { - for (_, p, o, g) in - self.interned_quads_for_subject(&InternedSubject::BlankNode(*bnode)) - { - to_hash.push(( - self.hash_named_node(*p), - self.hash_term(o, &hashes), - self.hash_graph_name(g, &hashes), - 0, - )); - } - for (s, p, _, g) in self.interned_quads_for_object(&InternedTerm::BlankNode(*bnode)) - { - to_hash.push(( - self.hash_subject(s, &hashes), - self.hash_named_node(*p), - self.hash_graph_name(g, &hashes), - 1, - )); - } - for (s, p, o, _) in - self.interned_quads_for_graph_name(&InternedGraphName::BlankNode(*bnode)) - { + let mut to_do = hashes.keys().copied().collect::>(); + let mut partition = HashMap::<_, Vec<_>>::with_capacity(hashes.len()); + let mut partition_count = to_do.len(); + while !to_do.is_empty() { + partition.clear(); + let mut new_hashes = hashes.clone(); + let mut new_todo = Vec::with_capacity(to_do.len()); + for bnode in to_do { + for (s, p, o, g) in &quads_per_blank_node[&bnode] { to_hash.push(( - self.hash_subject(s, &hashes), + self.hash_subject(s, bnode, &hashes), self.hash_named_node(*p), - self.hash_term(o, &hashes), - 2, + self.hash_term(o, bnode, &hashes), + self.hash_graph_name(g, bnode, &hashes), )); } to_hash.sort_unstable(); - let hash = Self::hash_tuple((old_hash, &to_hash)); + let hash = Self::hash_tuple((&to_hash,)); to_hash.clear(); - new_hashes.insert(*bnode, hash); - partition.entry(hash).or_default().push(*bnode); - } - if partition.len() == partition_len { - let mut partition: Vec<_> = partition.into_iter().collect(); - partition.sort_by(|(h1, b1), (h2, b2)| (b1.len(), h1).cmp(&(b2.len(), h2))); - return (hashes, partition); + if hash != hashes[&bnode] { + new_hashes.insert(bnode, hash); + new_todo.push(bnode); + } + partition.entry(hash).or_default().push(bnode); } hashes = new_hashes; - partition_len = partition.len(); - partition.clear(); + to_do = new_todo; + if partition_count == partition.len() { + break; // no improvement + } + partition_count = partition.len(); } + let mut partition: Vec<_> = partition.into_iter().collect(); + partition.sort_unstable_by(|(h1, b1), (h2, b2)| (b1.len(), h1).cmp(&(b2.len(), h2))); + (hashes, partition) } fn hash_named_node(&self, node: InternedNamedNode) -> u64 { Self::hash_tuple(node.decode_from(&self.interner)) } + fn hash_blank_node( + node: InternedBlankNode, + current_blank_node: InternedBlankNode, + bnodes_hash: &HashMap, + ) -> u64 { + if node == current_blank_node { + u64::MAX + } else { + bnodes_hash[&node] + } + } + fn hash_subject( &self, node: &InternedSubject, + current_blank_node: InternedBlankNode, bnodes_hash: &HashMap, ) -> u64 { - #[cfg(feature = "rdf-star")] - if let InternedSubject::Triple(triple) = node { - return self.hash_triple(triple, bnodes_hash); - } - if let InternedSubject::BlankNode(bnode) = node { - bnodes_hash[bnode] - } else { - Self::hash_tuple(node.decode_from(&self.interner)) + match node { + InternedSubject::NamedNode(node) => Self::hash_tuple(node.decode_from(&self.interner)), + InternedSubject::BlankNode(bnode) => { + Self::hash_blank_node(*bnode, current_blank_node, bnodes_hash) + } + #[cfg(feature = "rdf-star")] + InternedSubject::Triple(triple) => { + self.hash_triple(triple, current_blank_node, bnodes_hash) + } } } - fn hash_term(&self, term: &InternedTerm, bnodes_hash: &HashMap) -> u64 { - #[cfg(feature = "rdf-star")] - if let InternedTerm::Triple(triple) = term { - return self.hash_triple(triple, bnodes_hash); - } - if let InternedTerm::BlankNode(bnode) = term { - bnodes_hash[bnode] - } else { - Self::hash_tuple(term.decode_from(&self.interner)) + fn hash_term( + &self, + term: &InternedTerm, + current_blank_node: InternedBlankNode, + bnodes_hash: &HashMap, + ) -> u64 { + match term { + InternedTerm::NamedNode(node) => Self::hash_tuple(node.decode_from(&self.interner)), + InternedTerm::BlankNode(bnode) => { + Self::hash_blank_node(*bnode, current_blank_node, bnodes_hash) + } + InternedTerm::Literal(literal) => Self::hash_tuple(literal.decode_from(&self.interner)), + #[cfg(feature = "rdf-star")] + InternedTerm::Triple(triple) => { + self.hash_triple(triple, current_blank_node, bnodes_hash) + } } } fn hash_graph_name( &self, graph_name: &InternedGraphName, + current_blank_node: InternedBlankNode, bnodes_hash: &HashMap, ) -> u64 { - if let InternedGraphName::BlankNode(bnode) = graph_name { - bnodes_hash[bnode] - } else { - Self::hash_tuple(graph_name.decode_from(&self.interner)) + match graph_name { + InternedGraphName::NamedNode(node) => { + Self::hash_tuple(node.decode_from(&self.interner)) + } + InternedGraphName::BlankNode(bnode) => { + Self::hash_blank_node(*bnode, current_blank_node, bnodes_hash) + } + InternedGraphName::DefaultGraph => 0, } } @@ -681,12 +757,13 @@ impl Dataset { fn hash_triple( &self, triple: &InternedTriple, + current_blank_node: InternedBlankNode, bnodes_hash: &HashMap, ) -> u64 { Self::hash_tuple(( - self.hash_subject(&triple.subject, bnodes_hash), + self.hash_subject(&triple.subject, current_blank_node, bnodes_hash), self.hash_named_node(triple.predicate), - self.hash_term(&triple.object, bnodes_hash), + self.hash_term(&triple.object, current_blank_node, bnodes_hash), )) } @@ -700,6 +777,7 @@ impl Dataset { &mut self, hash: &HashMap, partition: &[(u64, Vec)], + quads_per_blank_node: &QuadsPerBlankNode, ) -> Vec<( InternedSubject, InternedNamedNode, @@ -713,8 +791,9 @@ impl Dataset { .map(|b| { let mut hash_prime = hash.clone(); hash_prime.insert(*b, Self::hash_tuple((hash_prime[b], 22))); - let (hash_prime_prime, partition_prime) = self.hash_bnodes(hash_prime); - self.distinguish(&hash_prime_prime, &partition_prime) + let (hash_prime_prime, partition_prime) = + self.hash_bnodes(hash_prime, quads_per_blank_node); + self.distinguish(&hash_prime_prime, &partition_prime, quads_per_blank_node) }) .reduce(min) .unwrap_or_default() @@ -738,54 +817,43 @@ impl Dataset { .into_iter() .map(|(s, p, o, g)| { ( - if let InternedSubject::BlankNode(bnode) = s { - InternedSubject::BlankNode(self.map_bnode(bnode, hashes)) - } else { - #[cfg(feature = "rdf-star")] - { - if let InternedSubject::Triple(triple) = s { - InternedSubject::Triple(Box::new(InternedTriple::encoded_into( - self.label_triple(&triple, hashes).as_ref(), - &mut self.interner, - ))) - } else { - s - } + match s { + InternedSubject::NamedNode(_) => s, + InternedSubject::BlankNode(bnode) => { + InternedSubject::BlankNode(self.map_bnode(bnode, hashes)) } - #[cfg(not(feature = "rdf-star"))] - { - s + #[cfg(feature = "rdf-star")] + InternedSubject::Triple(triple) => { + InternedSubject::Triple(Box::new(InternedTriple::encoded_into( + self.label_triple(&triple, hashes).as_ref(), + &mut self.interner, + ))) } }, p, - if let InternedTerm::BlankNode(bnode) = o { - InternedTerm::BlankNode(self.map_bnode(bnode, hashes)) - } else { - #[cfg(feature = "rdf-star")] - { - if let InternedTerm::Triple(triple) = o { - InternedTerm::Triple(Box::new(InternedTriple::encoded_into( - self.label_triple(&triple, hashes).as_ref(), - &mut self.interner, - ))) - } else { - o - } + match o { + InternedTerm::NamedNode(_) | InternedTerm::Literal(_) => o, + InternedTerm::BlankNode(bnode) => { + InternedTerm::BlankNode(self.map_bnode(bnode, hashes)) } - #[cfg(not(feature = "rdf-star"))] - { - o + #[cfg(feature = "rdf-star")] + InternedTerm::Triple(triple) => { + InternedTerm::Triple(Box::new(InternedTriple::encoded_into( + self.label_triple(&triple, hashes).as_ref(), + &mut self.interner, + ))) } }, - if let InternedGraphName::BlankNode(bnode) = g { - InternedGraphName::BlankNode(self.map_bnode(bnode, hashes)) - } else { - g + match g { + InternedGraphName::NamedNode(_) | InternedGraphName::DefaultGraph => g, + InternedGraphName::BlankNode(bnode) => { + InternedGraphName::BlankNode(self.map_bnode(bnode, hashes)) + } }, ) }) .collect(); - quads.sort(); + quads.sort_unstable(); quads } @@ -1483,3 +1551,46 @@ impl<'a> Iterator for GraphViewIter<'a> { .map(|(_, s, p, o)| self.dataset.decode_spo((s, p, o))) } } + +type QuadsPerBlankNode = HashMap< + InternedBlankNode, + Vec<( + InternedSubject, + InternedNamedNode, + InternedTerm, + InternedGraphName, + )>, +>; + +#[test] +fn test_canon() { + let mut dataset = Dataset::new(); + dataset.insert(QuadRef::new( + BlankNode::default().as_ref(), + NamedNodeRef::new_unchecked("http://ex"), + BlankNode::default().as_ref(), + GraphNameRef::DefaultGraph, + )); + dataset.insert(QuadRef::new( + BlankNode::default().as_ref(), + NamedNodeRef::new_unchecked("http://ex"), + BlankNode::default().as_ref(), + GraphNameRef::DefaultGraph, + )); + dataset.canonicalize(); + let mut dataset2 = Dataset::new(); + dataset2.insert(QuadRef::new( + BlankNode::default().as_ref(), + NamedNodeRef::new_unchecked("http://ex"), + BlankNode::default().as_ref(), + GraphNameRef::DefaultGraph, + )); + dataset2.insert(QuadRef::new( + BlankNode::default().as_ref(), + NamedNodeRef::new_unchecked("http://ex"), + BlankNode::default().as_ref(), + GraphNameRef::DefaultGraph, + )); + dataset2.canonicalize(); + assert_eq!(dataset, dataset2); +} diff --git a/lib/oxrdf/src/interning.rs b/lib/oxrdf/src/interning.rs index 4b7b8705..3414d51a 100644 --- a/lib/oxrdf/src/interning.rs +++ b/lib/oxrdf/src/interning.rs @@ -8,6 +8,7 @@ use std::hash::{BuildHasher, Hasher}; pub struct Interner { hasher: RandomState, string_for_hash: HashMap, + string_for_blank_node_id: HashMap, #[cfg(feature = "rdf-star")] triples: HashMap, } @@ -119,29 +120,53 @@ impl InternedNamedNode { } #[derive(Eq, PartialEq, Ord, PartialOrd, Debug, Clone, Copy, Hash)] -pub struct InternedBlankNode { - id: Key, +pub enum InternedBlankNode { + Number { id: u128 }, + Other { id: Key }, } impl InternedBlankNode { pub fn encoded_into(blank_node: BlankNodeRef<'_>, interner: &mut Interner) -> Self { - Self { - id: interner.get_or_intern(blank_node.as_str()), + if let Some(id) = blank_node.unique_id() { + interner + .string_for_blank_node_id + .entry(id) + .or_insert_with(|| blank_node.as_str().into()); + Self::Number { id } + } else { + Self::Other { + id: interner.get_or_intern(blank_node.as_str()), + } } } pub fn encoded_from(blank_node: BlankNodeRef<'_>, interner: &Interner) -> Option { - Some(Self { - id: interner.get(blank_node.as_str())?, - }) + if let Some(id) = blank_node.unique_id() { + interner + .string_for_blank_node_id + .contains_key(&id) + .then_some(Self::Number { id }) + } else { + Some(Self::Other { + id: interner.get(blank_node.as_str())?, + }) + } } pub fn decode_from(self, interner: &Interner) -> BlankNodeRef { - BlankNodeRef::new_unchecked(interner.resolve(self.id)) + BlankNodeRef::new_unchecked(match self { + Self::Number { id } => &interner.string_for_blank_node_id[&id], + Self::Other { id } => interner.resolve(id), + }) } pub fn next(self) -> Self { - Self { id: self.id.next() } + match self { + Self::Number { id } => Self::Number { + id: id.saturating_add(1), + }, + Self::Other { id } => Self::Other { id: id.next() }, + } } } From 88e49f6c66b376e284d92df31ae343d46e6bb67b Mon Sep 17 00:00:00 2001 From: Tpt Date: Sat, 19 Aug 2023 14:09:00 +0200 Subject: [PATCH 054/217] Server: adds the "convert" command --- lib/oxrdfio/src/serializer.rs | 36 ++--- lib/src/io/write.rs | 8 +- server/src/main.rs | 264 ++++++++++++++++++++++++++++++++-- 3 files changed, 272 insertions(+), 36 deletions(-) diff --git a/lib/oxrdfio/src/serializer.rs b/lib/oxrdfio/src/serializer.rs index 6ec4e666..1a721b6c 100644 --- a/lib/oxrdfio/src/serializer.rs +++ b/lib/oxrdfio/src/serializer.rs @@ -82,24 +82,24 @@ impl RdfSerializer { /// assert_eq!(buffer.as_slice(), " .\n".as_bytes()); /// # Result::<_,Box>::Ok(()) /// ``` - pub fn serialize_to_write(&self, writer: W) -> ToWriteQuadWriter { + pub fn serialize_to_write(&self, write: W) -> ToWriteQuadWriter { ToWriteQuadWriter { formatter: match self.format { - RdfFormat::NQuads => ToWriteQuadWriterKind::NQuads( - NQuadsSerializer::new().serialize_to_write(writer), - ), + RdfFormat::NQuads => { + ToWriteQuadWriterKind::NQuads(NQuadsSerializer::new().serialize_to_write(write)) + } RdfFormat::NTriples => ToWriteQuadWriterKind::NTriples( - NTriplesSerializer::new().serialize_to_write(writer), - ), - RdfFormat::RdfXml => ToWriteQuadWriterKind::RdfXml( - RdfXmlSerializer::new().serialize_to_write(writer), + NTriplesSerializer::new().serialize_to_write(write), ), + RdfFormat::RdfXml => { + ToWriteQuadWriterKind::RdfXml(RdfXmlSerializer::new().serialize_to_write(write)) + } RdfFormat::TriG => { - ToWriteQuadWriterKind::TriG(TriGSerializer::new().serialize_to_write(writer)) + ToWriteQuadWriterKind::TriG(TriGSerializer::new().serialize_to_write(write)) + } + RdfFormat::Turtle | RdfFormat::N3 => { + ToWriteQuadWriterKind::Turtle(TurtleSerializer::new().serialize_to_write(write)) } - RdfFormat::Turtle | RdfFormat::N3 => ToWriteQuadWriterKind::Turtle( - TurtleSerializer::new().serialize_to_write(writer), - ), }, } } @@ -134,24 +134,24 @@ impl RdfSerializer { #[cfg(feature = "async-tokio")] pub fn serialize_to_tokio_async_write( &self, - writer: W, + write: W, ) -> ToTokioAsyncWriteQuadWriter { ToTokioAsyncWriteQuadWriter { formatter: match self.format { RdfFormat::NQuads => ToTokioAsyncWriteQuadWriterKind::NQuads( - NQuadsSerializer::new().serialize_to_tokio_async_write(writer), + NQuadsSerializer::new().serialize_to_tokio_async_write(write), ), RdfFormat::NTriples => ToTokioAsyncWriteQuadWriterKind::NTriples( - NTriplesSerializer::new().serialize_to_tokio_async_write(writer), + NTriplesSerializer::new().serialize_to_tokio_async_write(write), ), RdfFormat::RdfXml => ToTokioAsyncWriteQuadWriterKind::RdfXml( - RdfXmlSerializer::new().serialize_to_tokio_async_write(writer), + RdfXmlSerializer::new().serialize_to_tokio_async_write(write), ), RdfFormat::TriG => ToTokioAsyncWriteQuadWriterKind::TriG( - TriGSerializer::new().serialize_to_tokio_async_write(writer), + TriGSerializer::new().serialize_to_tokio_async_write(write), ), RdfFormat::Turtle | RdfFormat::N3 => ToTokioAsyncWriteQuadWriterKind::Turtle( - TurtleSerializer::new().serialize_to_tokio_async_write(writer), + TurtleSerializer::new().serialize_to_tokio_async_write(write), ), }, } diff --git a/lib/src/io/write.rs b/lib/src/io/write.rs index 7c308c86..4c398791 100644 --- a/lib/src/io/write.rs +++ b/lib/src/io/write.rs @@ -45,9 +45,9 @@ impl GraphSerializer { } /// Returns a [`TripleWriter`] allowing writing triples into the given [`Write`] implementation - pub fn triple_writer(&self, writer: W) -> TripleWriter { + pub fn triple_writer(&self, write: W) -> TripleWriter { TripleWriter { - writer: self.inner.serialize_to_write(writer), + writer: self.inner.serialize_to_write(write), } } } @@ -128,9 +128,9 @@ impl DatasetSerializer { } /// Returns a [`QuadWriter`] allowing writing triples into the given [`Write`] implementation - pub fn quad_writer(&self, writer: W) -> QuadWriter { + pub fn quad_writer(&self, write: W) -> QuadWriter { QuadWriter { - writer: self.inner.serialize_to_write(writer), + writer: self.inner.serialize_to_write(write), } } } diff --git a/server/src/main.rs b/server/src/main.rs index 67a5d527..638f0e60 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -4,7 +4,7 @@ use clap::{Parser, Subcommand}; use flate2::read::MultiGzDecoder; use oxhttp::model::{Body, HeaderName, HeaderValue, Method, Request, Response, Status}; use oxhttp::Server; -use oxigraph::io::{RdfFormat, RdfSerializer}; +use oxigraph::io::{RdfFormat, RdfParser, RdfSerializer}; use oxigraph::model::{ GraphName, GraphNameRef, IriParseError, NamedNode, NamedNodeRef, NamedOrBlankNode, }; @@ -129,9 +129,10 @@ enum Command { /// By default the format is guessed from the loaded file extension. #[arg(long, required_unless_present = "file")] format: Option, + /// Base IRI of the file(s) to load. + #[arg(long)] + base: Option, /// Attempt to keep loading even if the data file is invalid. - /// - /// Only works with N-Triples and N-Quads for now. #[arg(long)] lenient: bool, /// Name of the graph to load the data to. @@ -174,7 +175,7 @@ enum Command { /// If no query or query file are given, stdin is used. #[arg(long, conflicts_with = "query")] query_file: Option, - /// Base URI of the query. + /// Base IRI of the query. #[arg(long)] query_base: Option, /// File in which the query results will be stored. @@ -219,7 +220,7 @@ enum Command { /// If no update or update file are given, stdin is used. #[arg(long, conflicts_with = "update")] update_file: Option, - /// Base URI of the update. + /// Base IRI of the update. #[arg(long)] update_base: Option, }, @@ -228,6 +229,52 @@ enum Command { /// Done by default in the background when serving requests. /// It is likely to not be useful in most of cases except if you provide a read-only SPARQL endpoint under heavy load. Optimize {}, + /// Converts a RDF serialization from one format to an other. + Convert { + /// File to convert from. + /// + /// If no file is given, stdin is read. + #[arg(short, long)] + from_file: Option, + /// The format of the file(s) to convert from. + /// + /// Can be an extension like "nt" or a MIME type like "application/n-triples". + /// + /// By default the format is guessed from the input file extension. + #[arg(long, required_unless_present = "from_file")] + from_format: Option, + /// Base IRI of the file to read. + #[arg(long)] + from_base: Option, + /// File to convert to. + /// + /// If no file is given, stdout is written. + #[arg(short, long)] + to_file: Option, + /// The format of the file(s) to convert from. + /// + /// Can be an extension like "nt" or a MIME type like "application/n-triples". + /// + /// By default the format is guessed from the target file extension. + #[arg(long, required_unless_present = "to_file")] + to_format: Option, + /// Attempt to keep converting even if the data file is invalid. + #[arg(long)] + lenient: bool, + /// Only load the given named graph from the input file. + /// + /// By default all graphs are loaded. + #[arg(long, conflicts_with = "from_default_graph")] + from_graph: Option, + /// Only load the default graph from the input file. + #[arg(long, conflicts_with = "from_graph")] + from_default_graph: bool, + /// Name of the graph to map the default graph to. + /// + /// By default the default graph is used. + #[arg(long)] + to_graph: Option, + }, } pub fn main() -> anyhow::Result<()> { @@ -286,6 +333,7 @@ pub fn main() -> anyhow::Result<()> { file, lenient, format, + base, graph, } => { let store = if let Some(location) = matches.location { @@ -331,7 +379,7 @@ pub fn main() -> anyhow::Result<()> { format.ok_or_else(|| { anyhow!("The --format option must be set when loading from stdin") })?, - None, + base.as_deref(), graph, ) } else { @@ -343,6 +391,7 @@ pub fn main() -> anyhow::Result<()> { for file in file { let store = store.clone(); let graph = graph.clone(); + let base = base.clone(); s.spawn(move |_| { let f = file.clone(); let start = Instant::now(); @@ -383,7 +432,7 @@ pub fn main() -> anyhow::Result<()> { rdf_format_from_path(&file.with_extension("")) .unwrap() }), - None, + base.as_deref(), graph, ) } else { @@ -393,7 +442,7 @@ pub fn main() -> anyhow::Result<()> { format.unwrap_or_else(|| { rdf_format_from_path(&file).unwrap() }), - None, + base.as_deref(), graph, ) } @@ -631,6 +680,101 @@ pub fn main() -> anyhow::Result<()> { store.optimize()?; Ok(()) } + Command::Convert { + from_file, + from_format, + from_base, + to_file, + to_format, + lenient, + from_graph, + from_default_graph, + to_graph, + } => { + let from_format = if let Some(format) = from_format { + rdf_format_from_name(&format)? + } else if let Some(file) = &from_file { + rdf_format_from_path(file)? + } else { + bail!("The --from-format option must be set when reading from stdin") + }; + let mut parser = RdfParser::from_format(from_format); + if let Some(base) = from_base { + parser = parser + .with_base_iri(&base) + .with_context(|| anyhow!("Invalid base IRI {base}"))?; + } + + let to_format = if let Some(format) = to_format { + rdf_format_from_name(&format)? + } else if let Some(file) = &from_file { + rdf_format_from_path(file)? + } else { + bail!("The --to-format option must be set when writing to stdout") + }; + let serializer = RdfSerializer::from_format(to_format); + + let from_graph = if let Some(from_graph) = from_graph { + Some( + NamedNode::new(&from_graph) + .with_context(|| format!("The source graph name {from_graph} is invalid"))? + .into(), + ) + } else if from_default_graph { + Some(GraphName::DefaultGraph) + } else { + None + }; + let to_graph = if let Some(to_graph) = to_graph { + NamedNode::new(&to_graph) + .with_context(|| format!("The target graph name {to_graph} is invalid"))? + .into() + } else { + GraphName::DefaultGraph + }; + + match (from_file, to_file) { + (Some(from_file), Some(to_file)) => close_file_writer(do_convert( + &parser, + File::open(from_file)?, + &serializer, + BufWriter::new(File::create(to_file)?), + lenient, + &from_graph, + &to_graph, + )?), + (Some(from_file), None) => do_convert( + &parser, + File::open(from_file)?, + &serializer, + stdout().lock(), + lenient, + &from_graph, + &to_graph, + )? + .flush(), + (None, Some(to_file)) => close_file_writer(do_convert( + &parser, + stdin().lock(), + &serializer, + BufWriter::new(File::create(to_file)?), + lenient, + &from_graph, + &to_graph, + )?), + (None, None) => do_convert( + &parser, + stdin().lock(), + &serializer, + stdout().lock(), + lenient, + &from_graph, + &to_graph, + )? + .flush(), + }?; + Ok(()) + } } } @@ -651,18 +795,55 @@ fn bulk_load( fn dump( store: &Store, - writer: W, + write: W, format: RdfFormat, from_graph_name: Option>, ) -> anyhow::Result { ensure!(format.supports_datasets() || from_graph_name.is_some(), "The --graph option is required when writing a format not supporting datasets like NTriples, Turtle or RDF/XML"); Ok(if let Some(from_graph_name) = from_graph_name { - store.dump_graph(writer, format, from_graph_name) + store.dump_graph(write, format, from_graph_name) } else { - store.dump_dataset(writer, format) + store.dump_dataset(write, format) }?) } +fn do_convert( + parser: &RdfParser, + read: R, + serializer: &RdfSerializer, + write: W, + lenient: bool, + from_graph: &Option, + default_graph: &GraphName, +) -> anyhow::Result { + let mut writer = serializer.serialize_to_write(write); + for quad_result in parser.parse_read(read) { + match quad_result { + Ok(mut quad) => { + if let Some(from_graph) = from_graph { + if quad.graph_name == *from_graph { + quad.graph_name = GraphName::DefaultGraph; + } else { + continue; + } + } + if quad.graph_name.is_default_graph() { + quad.graph_name = default_graph.clone(); + } + writer.write_quad(&quad)?; + } + Err(e) => { + if lenient { + eprintln!("Parsing error: {e}"); + } else { + return Err(e.into()); + } + } + } + } + Ok(writer.finish()?) +} + fn format_from_path( path: &Path, from_extension: impl FnOnce(&str) -> anyhow::Result, @@ -1710,15 +1891,16 @@ mod tests { #[test] fn cli_load_optimize_and_dump_graph() -> Result<()> { let store_dir = TempDir::new()?; - let input_file = NamedTempFile::new("input.nt")?; - input_file - .write_str(" .")?; + let input_file = NamedTempFile::new("input.ttl")?; + input_file.write_str(" .")?; cli_command()? .arg("--location") .arg(store_dir.path()) .arg("load") .arg("--file") .arg(input_file.path()) + .arg("--base") + .arg("http://example.com/") .assert() .success(); @@ -2041,6 +2223,60 @@ mod tests { ) } + #[test] + fn cli_convert_file() -> Result<()> { + let input_file = NamedTempFile::new("input.ttl")?; + input_file.write_str("

.")?; + let output_file = NamedTempFile::new("output.nt")?; + cli_command()? + .arg("convert") + .arg("--from-file") + .arg(input_file.path()) + .arg("--from-base") + .arg("http://example.com/") + .arg("--to-file") + .arg(output_file.path()) + .assert() + .success(); + output_file + .assert(" .\n"); + Ok(()) + } + + #[test] + fn cli_convert_from_default_graph_to_named_graph() -> Result<()> { + cli_command()? + .arg("convert") + .arg("--from-format") + .arg("trig") + .arg("--to-format") + .arg("nq") + .arg("--from-default-graph") + .arg("--to-graph") + .arg("http://example.com/t") + .write_stdin("@base .

. { . }") + .assert() + .stdout(" .\n") + .success(); + Ok(()) + } + + #[test] + fn cli_convert_from_named_graph() -> Result<()> { + cli_command()? + .arg("convert") + .arg("--from-format") + .arg("trig") + .arg("--to-format") + .arg("nq") + .arg("--from-graph") + .arg("http://example.com/g") + .write_stdin("@base .

. { . }") + .assert() + .stdout(" .\n"); + Ok(()) + } + #[test] fn get_ui() -> Result<()> { ServerTest::new()?.test_status( From f586cc048f823d8faba31b5eba26ac635050e7fe Mon Sep 17 00:00:00 2001 From: Tpt Date: Thu, 6 Jul 2023 17:56:05 +0200 Subject: [PATCH 055/217] Fuzzer: ensure that NQuad/TriG segmentation does not affect results --- fuzz/Cargo.toml | 2 +- fuzz/fuzz_targets/nquads.rs | 34 +++++++--- fuzz/fuzz_targets/sparql_query.rs | 2 +- fuzz/fuzz_targets/sparql_results_json.rs | 2 +- fuzz/fuzz_targets/sparql_results_tsv.rs | 2 +- fuzz/fuzz_targets/sparql_results_xml.rs | 2 +- fuzz/fuzz_targets/sparql_update.rs | 2 +- fuzz/fuzz_targets/trig.rs | 80 ++++++++++++++++++++---- 8 files changed, 101 insertions(+), 25 deletions(-) diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index 8bef5528..f1524903 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -1,7 +1,6 @@ [package] name = "oxigraph-fuzz" version = "0.0.0" -authors = ["Automatically generated"] publish = false edition = "2021" @@ -12,6 +11,7 @@ cargo-fuzz = true anyhow = "1" lazy_static = "1" libfuzzer-sys = "0.4" +oxrdf = { path = "../lib/oxrdf", features = ["rdf-star"] } oxttl = { path = "../lib/oxttl", features = ["rdf-star"] } oxrdfxml = { path = "../lib/oxrdfxml" } spargebra = { path = "../lib/spargebra", features = ["rdf-star", "sep-0006"] } diff --git a/fuzz/fuzz_targets/nquads.rs b/fuzz/fuzz_targets/nquads.rs index b8b1ac6e..a7de4913 100644 --- a/fuzz/fuzz_targets/nquads.rs +++ b/fuzz/fuzz_targets/nquads.rs @@ -1,27 +1,45 @@ #![no_main] use libfuzzer_sys::fuzz_target; -use oxttl::{NQuadsParser, NQuadsSerializer}; +use oxrdf::Quad; +use oxttl::{NQuadsParser, NQuadsSerializer, SyntaxError}; -fuzz_target!(|data: &[u8]| { - // We parse +fn parse<'a>(chunks: impl IntoIterator) -> (Vec, Vec) { let mut quads = Vec::new(); + let mut errors = Vec::new(); let mut parser = NQuadsParser::new().with_quoted_triples().parse(); - for chunk in data.split(|c| *c == 0xFF) { + for chunk in chunks { parser.extend_from_slice(chunk); while let Some(result) = parser.read_next() { - if let Ok(quad) = result { - quads.push(quad); + match result { + Ok(quad) => quads.push(quad), + Err(error) => errors.push(error), } } } parser.end(); while let Some(result) = parser.read_next() { - if let Ok(quad) = result { - quads.push(quad); + match result { + Ok(quad) => quads.push(quad), + Err(error) => errors.push(error), } } assert!(parser.is_end()); + (quads, errors) +} + +fuzz_target!(|data: &[u8]| { + // We parse with splitting + let (quads, errors) = parse(data.split(|c| *c == 0xFF)); + // We parse without splitting + let (quads_without_split, errors_without_split) = parse([data + .iter() + .copied() + .filter(|c| *c != 0xFF) + .collect::>() + .as_slice()]); + assert_eq!(quads, quads_without_split); + assert_eq!(errors.len(), errors_without_split.len()); // We serialize let mut writer = NQuadsSerializer::new().serialize_to_write(Vec::new()); diff --git a/fuzz/fuzz_targets/sparql_query.rs b/fuzz/fuzz_targets/sparql_query.rs index 889d3e79..136a2c0f 100644 --- a/fuzz/fuzz_targets/sparql_query.rs +++ b/fuzz/fuzz_targets/sparql_query.rs @@ -3,5 +3,5 @@ use libfuzzer_sys::fuzz_target; use spargebra::Query; fuzz_target!(|data: &str| { - Query::parse(data, None); + let _ = Query::parse(data, None); }); diff --git a/fuzz/fuzz_targets/sparql_results_json.rs b/fuzz/fuzz_targets/sparql_results_json.rs index cd917481..f9588f8b 100644 --- a/fuzz/fuzz_targets/sparql_results_json.rs +++ b/fuzz/fuzz_targets/sparql_results_json.rs @@ -3,4 +3,4 @@ use libfuzzer_sys::fuzz_target; use oxigraph_fuzz::result_format::fuzz_result_format; use sparesults::QueryResultsFormat; -fuzz_target!(|data: &[u8]| { fuzz_result_format(QueryResultsFormat::Json, data) }); +fuzz_target!(|data: &[u8]| fuzz_result_format(QueryResultsFormat::Json, data)); diff --git a/fuzz/fuzz_targets/sparql_results_tsv.rs b/fuzz/fuzz_targets/sparql_results_tsv.rs index 4cf3f4cf..1aa600b7 100644 --- a/fuzz/fuzz_targets/sparql_results_tsv.rs +++ b/fuzz/fuzz_targets/sparql_results_tsv.rs @@ -3,4 +3,4 @@ use libfuzzer_sys::fuzz_target; use oxigraph_fuzz::result_format::fuzz_result_format; use sparesults::QueryResultsFormat; -fuzz_target!(|data: &[u8]| { fuzz_result_format(QueryResultsFormat::Tsv, data) }); +fuzz_target!(|data: &[u8]| fuzz_result_format(QueryResultsFormat::Tsv, data)); diff --git a/fuzz/fuzz_targets/sparql_results_xml.rs b/fuzz/fuzz_targets/sparql_results_xml.rs index 6c4747ec..451f528a 100644 --- a/fuzz/fuzz_targets/sparql_results_xml.rs +++ b/fuzz/fuzz_targets/sparql_results_xml.rs @@ -3,4 +3,4 @@ use libfuzzer_sys::fuzz_target; use oxigraph_fuzz::result_format::fuzz_result_format; use sparesults::QueryResultsFormat; -fuzz_target!(|data: &[u8]| { fuzz_result_format(QueryResultsFormat::Xml, data) }); +fuzz_target!(|data: &[u8]| fuzz_result_format(QueryResultsFormat::Xml, data)); diff --git a/fuzz/fuzz_targets/sparql_update.rs b/fuzz/fuzz_targets/sparql_update.rs index 15c0a995..56ffdae3 100644 --- a/fuzz/fuzz_targets/sparql_update.rs +++ b/fuzz/fuzz_targets/sparql_update.rs @@ -4,5 +4,5 @@ use spargebra::Update; use std::str; fuzz_target!(|data: &str| { - Update::parse(data, None); + let _ = Update::parse(data, None); }); diff --git a/fuzz/fuzz_targets/trig.rs b/fuzz/fuzz_targets/trig.rs index e6ed06c7..a96ca86c 100644 --- a/fuzz/fuzz_targets/trig.rs +++ b/fuzz/fuzz_targets/trig.rs @@ -1,38 +1,96 @@ #![no_main] use libfuzzer_sys::fuzz_target; -use oxttl::{TriGParser, TriGSerializer}; +use oxrdf::{Dataset, GraphName, Quad, Subject, Term, Triple}; +use oxttl::{SyntaxError, TriGParser, TriGSerializer}; -fuzz_target!(|data: &[u8]| { - // We parse +fn parse<'a>(chunks: impl IntoIterator) -> (Vec, Vec) { let mut quads = Vec::new(); + let mut errors = Vec::new(); let mut parser = TriGParser::new() .with_quoted_triples() .with_base_iri("http://example.com/") .unwrap() .parse(); - for chunk in data.split(|c| *c == 0xFF) { + for chunk in chunks { parser.extend_from_slice(chunk); while let Some(result) = parser.read_next() { - if let Ok(quad) = result { - quads.push(quad); + match result { + Ok(quad) => quads.push(quad), + Err(error) => errors.push(error), } } } parser.end(); while let Some(result) = parser.read_next() { - if let Ok(quad) = result { - quads.push(quad); + match result { + Ok(quad) => quads.push(quad), + Err(error) => errors.push(error), } } assert!(parser.is_end()); + (quads, errors) +} - // We serialize +fn count_triple_blank_nodes(triple: &Triple) -> usize { + (match &triple.subject { + Subject::BlankNode(_) => 1, + Subject::Triple(t) => count_triple_blank_nodes(t), + _ => 0, + }) + (match &triple.object { + Term::BlankNode(_) => 1, + Term::Triple(t) => count_triple_blank_nodes(t), + _ => 0, + }) +} + +fn count_quad_blank_nodes(quad: &Quad) -> usize { + (match &quad.subject { + Subject::BlankNode(_) => 1, + Subject::Triple(t) => count_triple_blank_nodes(t), + _ => 0, + }) + (match &quad.object { + Term::BlankNode(_) => 1, + Term::Triple(t) => count_triple_blank_nodes(t), + _ => 0, + }) + usize::from(matches!(quad.graph_name, GraphName::BlankNode(_))) +} + +fn serialize_quads(quads: &[Quad]) -> Vec { let mut writer = TriGSerializer::new().serialize_to_write(Vec::new()); - for quad in &quads { + for quad in quads { writer.write_quad(quad).unwrap(); } - let new_serialization = writer.finish().unwrap(); + writer.finish().unwrap() +} + +fuzz_target!(|data: &[u8]| { + // We parse with splitting + let (quads, errors) = parse(data.split(|c| *c == 0xFF)); + // We parse without splitting + let (quads_without_split, errors_without_split) = parse([data + .iter() + .copied() + .filter(|c| *c != 0xFF) + .collect::>() + .as_slice()]); + if quads.iter().map(count_quad_blank_nodes).sum::() < 2 { + let mut dataset_with_split = quads.iter().collect::(); + let mut dataset_without_split = quads_without_split.iter().collect::(); + dataset_with_split.canonicalize(); + dataset_without_split.canonicalize(); + assert_eq!( + dataset_with_split, + dataset_without_split, + "With split:\n{}\nWithout split:\n{}", + String::from_utf8_lossy(&serialize_quads(&quads)), + String::from_utf8_lossy(&serialize_quads(&quads_without_split)) + ); + } + assert_eq!(errors.len(), errors_without_split.len()); + + // We serialize + let new_serialization = serialize_quads(&quads); // We parse the serialization let new_quads = TriGParser::new() From 788450932a71e1d01afdd8fe7adf34d3fde2cbd8 Mon Sep 17 00:00:00 2001 From: Tpt Date: Sat, 19 Aug 2023 15:25:43 +0200 Subject: [PATCH 056/217] Server: removes the "location" argument to relevant commands --- server/README.md | 6 +-- server/src/main.rs | 130 ++++++++++++++++++++++----------------------- 2 files changed, 67 insertions(+), 69 deletions(-) diff --git a/server/README.md b/server/README.md index 851d7521..31075667 100644 --- a/server/README.md +++ b/server/README.md @@ -44,7 +44,7 @@ It will create a fat binary in `target/release/oxigraph_server`. ## Usage -Run `oxigraph_server --location my_data_storage_directory serve` to start the server where `my_data_storage_directory` is the directory where you want Oxigraph data to be stored. It listens by default on `localhost:7878`. +Run `oxigraph_server serve --location my_data_storage_directory` to start the server where `my_data_storage_directory` is the directory where you want Oxigraph data to be stored. It listens by default on `localhost:7878`. The server provides an HTML UI, based on [YASGUI](https://yasgui.triply.cc), with a form to execute SPARQL requests. @@ -81,7 +81,7 @@ It provides the following REST actions: Use `oxigraph_server --help` to see the possible options when starting the server. It is also possible to load RDF data offline using bulk loading: -`oxigraph_server --location my_data_storage_directory load --file my_file.nq` +`oxigraph_server load --location my_data_storage_directory --file my_file.nq` ## Using a Docker image @@ -93,7 +93,7 @@ docker run --rm ghcr.io/oxigraph/oxigraph --help ### Run the Webserver Expose the server on port `7878` of the host machine, and save data on the local `./data` folder ```sh -docker run --rm -v $PWD/data:/data -p 7878:7878 ghcr.io/oxigraph/oxigraph --location /data serve --bind 0.0.0.0:7878 +docker run --rm -v $PWD/data:/data -p 7878:7878 ghcr.io/oxigraph/oxigraph serve --location /data --bind 0.0.0.0:7878 ``` You can then access it from your machine on port `7878`: diff --git a/server/src/main.rs b/server/src/main.rs index 638f0e60..1178d1dc 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -41,11 +41,6 @@ const LOGO: &str = include_str!("../logo.svg"); #[command(about, version)] /// Oxigraph SPARQL server. struct Args { - /// Directory in which the data should be persisted. - /// - /// If not present. An in-memory storage will be used. - #[arg(short, long, global = true)] - location: Option, //TODO: move into commands on next breaking release #[command(subcommand)] command: Command, } @@ -54,6 +49,11 @@ struct Args { enum Command { /// Start Oxigraph HTTP server in read-write mode. Serve { + /// Directory in which the data should be persisted. + /// + /// If not present. An in-memory storage will be used. + #[arg(short, long)] + location: Option, /// Host and port to listen to. #[arg(short, long, default_value = "localhost:7878")] bind: String, @@ -67,6 +67,9 @@ enum Command { /// Opening as read-only while having an other process writing the database is undefined behavior. /// Please use the serve-secondary command in this case. ServeReadOnly { + /// Directory in which Oxigraph data are persisted. + #[arg(short, long)] + location: PathBuf, /// Host and port to listen to. #[arg(short, long, default_value = "localhost:7878")] bind: String, @@ -83,8 +86,8 @@ enum Command { /// Dirty reads might happen. ServeSecondary { /// Directory where the primary Oxigraph instance is writing to. - #[arg(long, conflicts_with = "location")] - primary_location: Option, + #[arg(long)] + primary_location: PathBuf, /// Directory to which the current secondary instance might write to. /// /// By default, temporary storage is used. @@ -109,12 +112,18 @@ enum Command { /// /// If you want to move your data to another RDF storage system, you should use the dump operation instead. Backup { + /// Directory in which Oxigraph data are persisted. + #[arg(short, long)] + location: PathBuf, /// Directory in which the backup will be written. #[arg(short, long)] destination: PathBuf, }, /// Load file(s) into the store. Load { + /// Directory in which Oxigraph data are persisted. + #[arg(short, long)] + location: PathBuf, /// File(s) to load. /// /// If multiple files are provided they are loaded in parallel. @@ -145,6 +154,9 @@ enum Command { }, /// Dump the store content into a file. Dump { + /// Directory in which Oxigraph data are persisted. + #[arg(short, long)] + location: PathBuf, /// File to dump to. /// /// If no file is given, stdout is used. @@ -165,6 +177,9 @@ enum Command { }, /// Executes a SPARQL query against the store. Query { + /// Directory in which Oxigraph data are persisted. + #[arg(short, long)] + location: PathBuf, /// The SPARQL query to execute. /// /// If no query or query file are given, stdin is used. @@ -210,6 +225,9 @@ enum Command { }, /// Executes a SPARQL update against the store. Update { + /// Directory in which Oxigraph data are persisted. + #[arg(short, long)] + location: PathBuf, /// The SPARQL update to execute. /// /// If no query or query file are given, stdin is used. @@ -228,7 +246,11 @@ enum Command { /// /// Done by default in the background when serving requests. /// It is likely to not be useful in most of cases except if you provide a read-only SPARQL endpoint under heavy load. - Optimize {}, + Optimize { + /// Directory in which Oxigraph data are persisted. + #[arg(short, long)] + location: PathBuf, + }, /// Converts a RDF serialization from one format to an other. Convert { /// File to convert from. @@ -280,8 +302,12 @@ enum Command { pub fn main() -> anyhow::Result<()> { let matches = Args::parse(); match matches.command { - Command::Serve { bind, cors } => serve( - if let Some(location) = matches.location { + Command::Serve { + location, + bind, + cors, + } => serve( + if let Some(location) = location { Store::open(location) } else { Store::new() @@ -290,58 +316,43 @@ pub fn main() -> anyhow::Result<()> { false, cors, ), - Command::ServeReadOnly { bind, cors } => serve( - Store::open_read_only( - matches - .location - .ok_or_else(|| anyhow!("The --location argument is required"))?, - )?, + Command::ServeReadOnly { + location, bind, - true, cors, - ), + } => serve(Store::open_read_only(location)?, bind, true, cors), Command::ServeSecondary { primary_location, secondary_location, bind, cors, + } => serve( + if let Some(secondary_location) = secondary_location { + Store::open_persistent_secondary(primary_location, secondary_location) + } else { + Store::open_secondary(primary_location) + }?, + bind, + true, + cors, + ), + Command::Backup { + location, + destination, } => { - let primary_location = primary_location.or(matches.location).ok_or_else(|| { - anyhow!("Either the --location or the --primary-location argument is required") - })?; - serve( - if let Some(secondary_location) = secondary_location { - Store::open_persistent_secondary(primary_location, secondary_location) - } else { - Store::open_secondary(primary_location) - }?, - bind, - true, - cors, - ) - } - Command::Backup { destination } => { - let store = Store::open_read_only( - matches - .location - .ok_or_else(|| anyhow!("The --location argument is required"))?, - )?; + let store = Store::open_read_only(location)?; store.backup(destination)?; Ok(()) } Command::Load { + location, file, lenient, format, base, graph, } => { - let store = if let Some(location) = matches.location { - Store::open(location) - } else { - eprintln!("Warning: opening an in-memory store. It will not be possible to read the written data."); - Store::new() - }?; + let store = Store::open(location)?; let format = if let Some(format) = format { Some(rdf_format_from_name(&format)?) } else { @@ -462,15 +473,12 @@ pub fn main() -> anyhow::Result<()> { } } Command::Dump { + location, file, format, graph, } => { - let store = Store::open_read_only( - matches - .location - .ok_or_else(|| anyhow!("The --location argument is required"))?, - )?; + let store = Store::open_read_only(location)?; let format = if let Some(format) = format { rdf_format_from_name(&format)? } else if let Some(file) = &file { @@ -502,6 +510,7 @@ pub fn main() -> anyhow::Result<()> { Ok(()) } Command::Query { + location, query, query_file, query_base, @@ -521,11 +530,7 @@ pub fn main() -> anyhow::Result<()> { io::read_to_string(stdin().lock())? }; let query = Query::parse(&query, query_base.as_deref())?; - let store = Store::open_read_only( - matches - .location - .ok_or_else(|| anyhow!("The --location argument is required"))?, - )?; + let store = Store::open_read_only(location)?; let (results, explanation) = store.explain_query_opt(query, QueryOptions::default(), stats)?; let print_result = (|| { @@ -648,6 +653,7 @@ pub fn main() -> anyhow::Result<()> { print_result } Command::Update { + location, update, update_file, update_base, @@ -662,21 +668,13 @@ pub fn main() -> anyhow::Result<()> { io::read_to_string(stdin().lock())? }; let update = Update::parse(&update, update_base.as_deref())?; - let store = Store::open( - matches - .location - .ok_or_else(|| anyhow!("The --location argument is required"))?, - )?; + let store = Store::open(location)?; store.update(update)?; store.flush()?; Ok(()) } - Command::Optimize {} => { - let store = Store::open( - matches - .location - .ok_or_else(|| anyhow!("The --location argument is required"))?, - )?; + Command::Optimize { location } => { + let store = Store::open(location)?; store.optimize()?; Ok(()) } @@ -1894,9 +1892,9 @@ mod tests { let input_file = NamedTempFile::new("input.ttl")?; input_file.write_str(" .")?; cli_command()? + .arg("load") .arg("--location") .arg(store_dir.path()) - .arg("load") .arg("--file") .arg(input_file.path()) .arg("--base") From bbf184f7aea4d60ec01a33d680d8a3dbb6567170 Mon Sep 17 00:00:00 2001 From: Tpt Date: Thu, 6 Jul 2023 17:56:05 +0200 Subject: [PATCH 057/217] Isomorphism: makes sure that new hashes depends on the old ones Allows to make the "distinguish" step work --- fuzz/fuzz_targets/trig.rs | 11 +++++++- lib/oxrdf/src/dataset.rs | 56 +++++++++++++++++++++------------------ 2 files changed, 40 insertions(+), 27 deletions(-) diff --git a/fuzz/fuzz_targets/trig.rs b/fuzz/fuzz_targets/trig.rs index a96ca86c..6a930a97 100644 --- a/fuzz/fuzz_targets/trig.rs +++ b/fuzz/fuzz_targets/trig.rs @@ -74,7 +74,16 @@ fuzz_target!(|data: &[u8]| { .filter(|c| *c != 0xFF) .collect::>() .as_slice()]); - if quads.iter().map(count_quad_blank_nodes).sum::() < 2 { + let bnodes_count = quads.iter().map(count_quad_blank_nodes).sum::(); + if bnodes_count == 0 { + assert_eq!( + quads, + quads_without_split, + "With split:\n{}\nWithout split:\n{}", + String::from_utf8_lossy(&serialize_quads(&quads)), + String::from_utf8_lossy(&serialize_quads(&quads_without_split)) + ); + } else if bnodes_count <= 4 { let mut dataset_with_split = quads.iter().collect::(); let mut dataset_without_split = quads_without_split.iter().collect::(); dataset_with_split.canonicalize(); diff --git a/lib/oxrdf/src/dataset.rs b/lib/oxrdf/src/dataset.rs index dbf82c86..4b7736eb 100644 --- a/lib/oxrdf/src/dataset.rs +++ b/lib/oxrdf/src/dataset.rs @@ -646,37 +646,41 @@ impl Dataset { Vec<(u64, Vec)>, ) { let mut to_hash = Vec::new(); - let mut to_do = hashes.keys().copied().collect::>(); + let mut to_do = hashes + .keys() + .map(|bnode| (*bnode, true)) + .collect::>(); let mut partition = HashMap::<_, Vec<_>>::with_capacity(hashes.len()); - let mut partition_count = to_do.len(); - while !to_do.is_empty() { + let mut old_partition_count = usize::MAX; + while old_partition_count != partition.len() { + old_partition_count = partition.len(); partition.clear(); let mut new_hashes = hashes.clone(); - let mut new_todo = Vec::with_capacity(to_do.len()); - for bnode in to_do { - for (s, p, o, g) in &quads_per_blank_node[&bnode] { - to_hash.push(( - self.hash_subject(s, bnode, &hashes), - self.hash_named_node(*p), - self.hash_term(o, bnode, &hashes), - self.hash_graph_name(g, bnode, &hashes), - )); - } - to_hash.sort_unstable(); - let hash = Self::hash_tuple((&to_hash,)); - to_hash.clear(); - if hash != hashes[&bnode] { - new_hashes.insert(bnode, hash); - new_todo.push(bnode); - } - partition.entry(hash).or_default().push(bnode); + for bnode in hashes.keys() { + let hash = if to_do.contains_key(bnode) { + for (s, p, o, g) in &quads_per_blank_node[bnode] { + to_hash.push(( + self.hash_subject(s, *bnode, &hashes), + self.hash_named_node(*p), + self.hash_term(o, *bnode, &hashes), + self.hash_graph_name(g, *bnode, &hashes), + )); + } + to_hash.sort_unstable(); + let hash = Self::hash_tuple((&to_hash, hashes[bnode])); + to_hash.clear(); + if hash == hashes[bnode] { + to_do.insert(*bnode, false); + } else { + new_hashes.insert(*bnode, hash); + } + hash + } else { + hashes[bnode] + }; + partition.entry(hash).or_default().push(*bnode); } hashes = new_hashes; - to_do = new_todo; - if partition_count == partition.len() { - break; // no improvement - } - partition_count = partition.len(); } let mut partition: Vec<_> = partition.into_iter().collect(); partition.sort_unstable_by(|(h1, b1), (h2, b2)| (b1.len(), h1).cmp(&(b2.len(), h2))); From 010196c974f964b70f78a38061dc2fe7edecc904 Mon Sep 17 00:00:00 2001 From: Tpt Date: Sun, 20 Aug 2023 19:02:43 +0200 Subject: [PATCH 058/217] Makes parse_ and serialize_ method take owned Parser and Serializer It is very rare to reuse parser and this allows to avoid copies --- lib/oxrdfio/src/parser.rs | 17 ++++++++++------- lib/oxrdfio/src/serializer.rs | 4 ++-- lib/oxrdfxml/src/parser.rs | 4 ++-- lib/oxrdfxml/src/serializer.rs | 4 ++-- lib/oxttl/src/n3.rs | 8 ++++---- lib/oxttl/src/nquads.rs | 10 +++++----- lib/oxttl/src/ntriples.rs | 10 +++++----- lib/oxttl/src/trig.rs | 14 +++++++------- lib/oxttl/src/turtle.rs | 14 +++++++------- lib/src/io/read.rs | 4 ++-- lib/src/io/write.rs | 4 ++-- server/src/main.rs | 20 ++++++++++---------- 12 files changed, 58 insertions(+), 55 deletions(-) diff --git a/lib/oxrdfio/src/parser.rs b/lib/oxrdfio/src/parser.rs index b0e4a419..4c7090b6 100644 --- a/lib/oxrdfio/src/parser.rs +++ b/lib/oxrdfio/src/parser.rs @@ -214,9 +214,12 @@ impl RdfParser { /// /// let file = "_:a ."; /// - /// let parser = RdfParser::from_format(RdfFormat::NQuads).rename_blank_nodes(); - /// let result1 = parser.parse_read(file.as_bytes()).collect::,_>>()?; - /// let result2 = parser.parse_read(file.as_bytes()).collect::,_>>()?; + /// let result1 = RdfParser::from_format(RdfFormat::NQuads) + /// .rename_blank_nodes() + /// .parse_read(file.as_bytes()).collect::,_>>()?; + /// let result2 = RdfParser::from_format(RdfFormat::NQuads) + /// .rename_blank_nodes() + /// .parse_read(file.as_bytes()).collect::,_>>()?; /// assert_ne!(result1, result2); /// # Result::<_,Box>::Ok(()) /// ``` @@ -247,9 +250,9 @@ impl RdfParser { /// assert_eq!(quads[0].subject.to_string(), ""); /// # std::io::Result::Ok(()) /// ``` - pub fn parse_read(&self, reader: R) -> FromReadQuadReader { + pub fn parse_read(self, reader: R) -> FromReadQuadReader { FromReadQuadReader { - parser: match &self.inner { + parser: match self.inner { RdfParserKind::N3(p) => FromReadQuadReaderKind::N3(p.parse_read(reader)), RdfParserKind::NQuads(p) => FromReadQuadReaderKind::NQuads(p.parse_read(reader)), RdfParserKind::NTriples(p) => { @@ -288,11 +291,11 @@ impl RdfParser { /// ``` #[cfg(feature = "async-tokio")] pub fn parse_tokio_async_read( - &self, + self, reader: R, ) -> FromTokioAsyncReadQuadReader { FromTokioAsyncReadQuadReader { - parser: match &self.inner { + parser: match self.inner { RdfParserKind::N3(p) => { FromTokioAsyncReadQuadReaderKind::N3(p.parse_tokio_async_read(reader)) } diff --git a/lib/oxrdfio/src/serializer.rs b/lib/oxrdfio/src/serializer.rs index 1a721b6c..26229e5d 100644 --- a/lib/oxrdfio/src/serializer.rs +++ b/lib/oxrdfio/src/serializer.rs @@ -82,7 +82,7 @@ impl RdfSerializer { /// assert_eq!(buffer.as_slice(), " .\n".as_bytes()); /// # Result::<_,Box>::Ok(()) /// ``` - pub fn serialize_to_write(&self, write: W) -> ToWriteQuadWriter { + pub fn serialize_to_write(self, write: W) -> ToWriteQuadWriter { ToWriteQuadWriter { formatter: match self.format { RdfFormat::NQuads => { @@ -133,7 +133,7 @@ impl RdfSerializer { /// ``` #[cfg(feature = "async-tokio")] pub fn serialize_to_tokio_async_write( - &self, + self, write: W, ) -> ToTokioAsyncWriteQuadWriter { ToTokioAsyncWriteQuadWriter { diff --git a/lib/oxrdfxml/src/parser.rs b/lib/oxrdfxml/src/parser.rs index 279d0301..a17f9ca4 100644 --- a/lib/oxrdfxml/src/parser.rs +++ b/lib/oxrdfxml/src/parser.rs @@ -94,7 +94,7 @@ impl RdfXmlParser { /// assert_eq!(2, count); /// # Result::<_,Box>::Ok(()) /// ``` - pub fn parse_read(&self, read: R) -> FromReadRdfXmlReader { + pub fn parse_read(self, read: R) -> FromReadRdfXmlReader { FromReadRdfXmlReader { results: Vec::new(), reader: self.parse(BufReader::new(read)), @@ -135,7 +135,7 @@ impl RdfXmlParser { /// ``` #[cfg(feature = "async-tokio")] pub fn parse_tokio_async_read( - &self, + self, read: R, ) -> FromTokioAsyncReadRdfXmlReader { FromTokioAsyncReadRdfXmlReader { diff --git a/lib/oxrdfxml/src/serializer.rs b/lib/oxrdfxml/src/serializer.rs index 51802af5..e3d93065 100644 --- a/lib/oxrdfxml/src/serializer.rs +++ b/lib/oxrdfxml/src/serializer.rs @@ -57,7 +57,7 @@ impl RdfXmlSerializer { /// # Result::<_,Box>::Ok(()) /// ``` #[allow(clippy::unused_self)] - pub fn serialize_to_write(&self, write: W) -> ToWriteRdfXmlWriter { + pub fn serialize_to_write(self, write: W) -> ToWriteRdfXmlWriter { ToWriteRdfXmlWriter { writer: Writer::new_with_indent(write, b'\t', 1), inner: InnerRdfXmlWriter { @@ -93,7 +93,7 @@ impl RdfXmlSerializer { #[allow(clippy::unused_self)] #[cfg(feature = "async-tokio")] pub fn serialize_to_tokio_async_write( - &self, + self, write: W, ) -> ToTokioAsyncWriteRdfXmlWriter { ToTokioAsyncWriteRdfXmlWriter { diff --git a/lib/oxttl/src/n3.rs b/lib/oxttl/src/n3.rs index d1847991..0edb9d13 100644 --- a/lib/oxttl/src/n3.rs +++ b/lib/oxttl/src/n3.rs @@ -259,7 +259,7 @@ impl N3Parser { /// assert_eq!(2, count); /// # Result::<_,Box>::Ok(()) /// ``` - pub fn parse_read(&self, read: R) -> FromReadN3Reader { + pub fn parse_read(self, read: R) -> FromReadN3Reader { FromReadN3Reader { inner: self.parse().parser.parse_read(read), } @@ -298,7 +298,7 @@ impl N3Parser { /// ``` #[cfg(feature = "async-tokio")] pub fn parse_tokio_async_read( - &self, + self, read: R, ) -> FromTokioAsyncReadN3Reader { FromTokioAsyncReadN3Reader { @@ -343,9 +343,9 @@ impl N3Parser { /// assert_eq!(2, count); /// # Result::<_,Box>::Ok(()) /// ``` - pub fn parse(&self) -> LowLevelN3Reader { + pub fn parse(self) -> LowLevelN3Reader { LowLevelN3Reader { - parser: N3Recognizer::new_parser(self.base.clone(), self.prefixes.clone()), + parser: N3Recognizer::new_parser(self.base, self.prefixes), } } } diff --git a/lib/oxttl/src/nquads.rs b/lib/oxttl/src/nquads.rs index 140f25a7..71ac2d4c 100644 --- a/lib/oxttl/src/nquads.rs +++ b/lib/oxttl/src/nquads.rs @@ -79,7 +79,7 @@ impl NQuadsParser { /// assert_eq!(2, count); /// # Result::<_,Box>::Ok(()) /// ``` - pub fn parse_read(&self, read: R) -> FromReadNQuadsReader { + pub fn parse_read(self, read: R) -> FromReadNQuadsReader { FromReadNQuadsReader { inner: self.parse().parser.parse_read(read), } @@ -114,7 +114,7 @@ impl NQuadsParser { /// ``` #[cfg(feature = "async-tokio")] pub fn parse_tokio_async_read( - &self, + self, read: R, ) -> FromTokioAsyncReadNQuadsReader { FromTokioAsyncReadNQuadsReader { @@ -159,7 +159,7 @@ impl NQuadsParser { /// # Result::<_,Box>::Ok(()) /// ``` #[allow(clippy::unused_self)] - pub fn parse(&self) -> LowLevelNQuadsReader { + pub fn parse(self) -> LowLevelNQuadsReader { LowLevelNQuadsReader { parser: NQuadsRecognizer::new_parser( true, @@ -362,7 +362,7 @@ impl NQuadsSerializer { /// ); /// # Result::<_,Box>::Ok(()) /// ``` - pub fn serialize_to_write(&self, write: W) -> ToWriteNQuadsWriter { + pub fn serialize_to_write(self, write: W) -> ToWriteNQuadsWriter { ToWriteNQuadsWriter { write, writer: self.serialize(), @@ -394,7 +394,7 @@ impl NQuadsSerializer { /// ``` #[cfg(feature = "async-tokio")] pub fn serialize_to_tokio_async_write( - &self, + self, write: W, ) -> ToTokioAsyncWriteNQuadsWriter { ToTokioAsyncWriteNQuadsWriter { diff --git a/lib/oxttl/src/ntriples.rs b/lib/oxttl/src/ntriples.rs index 3014c5d8..07672f1a 100644 --- a/lib/oxttl/src/ntriples.rs +++ b/lib/oxttl/src/ntriples.rs @@ -80,7 +80,7 @@ impl NTriplesParser { /// assert_eq!(2, count); /// # Result::<_,Box>::Ok(()) /// ``` - pub fn parse_read(&self, read: R) -> FromReadNTriplesReader { + pub fn parse_read(self, read: R) -> FromReadNTriplesReader { FromReadNTriplesReader { inner: self.parse().parser.parse_read(read), } @@ -115,7 +115,7 @@ impl NTriplesParser { /// ``` #[cfg(feature = "async-tokio")] pub fn parse_tokio_async_read( - &self, + self, read: R, ) -> FromTokioAsyncReadNTriplesReader { FromTokioAsyncReadNTriplesReader { @@ -160,7 +160,7 @@ impl NTriplesParser { /// # Result::<_,Box>::Ok(()) /// ``` #[allow(clippy::unused_self)] - pub fn parse(&self) -> LowLevelNTriplesReader { + pub fn parse(self) -> LowLevelNTriplesReader { LowLevelNTriplesReader { parser: NQuadsRecognizer::new_parser( false, @@ -361,7 +361,7 @@ impl NTriplesSerializer { /// ); /// # Result::<_,Box>::Ok(()) /// ``` - pub fn serialize_to_write(&self, write: W) -> ToWriteNTriplesWriter { + pub fn serialize_to_write(self, write: W) -> ToWriteNTriplesWriter { ToWriteNTriplesWriter { write, writer: self.serialize(), @@ -392,7 +392,7 @@ impl NTriplesSerializer { /// ``` #[cfg(feature = "async-tokio")] pub fn serialize_to_tokio_async_write( - &self, + self, write: W, ) -> ToTokioAsyncWriteNTriplesWriter { ToTokioAsyncWriteNTriplesWriter { diff --git a/lib/oxttl/src/trig.rs b/lib/oxttl/src/trig.rs index 146e56b4..f2c24ca0 100644 --- a/lib/oxttl/src/trig.rs +++ b/lib/oxttl/src/trig.rs @@ -105,7 +105,7 @@ impl TriGParser { /// assert_eq!(2, count); /// # Result::<_,Box>::Ok(()) /// ``` - pub fn parse_read(&self, read: R) -> FromReadTriGReader { + pub fn parse_read(self, read: R) -> FromReadTriGReader { FromReadTriGReader { inner: self.parse().parser.parse_read(read), } @@ -142,7 +142,7 @@ impl TriGParser { /// ``` #[cfg(feature = "async-tokio")] pub fn parse_tokio_async_read( - &self, + self, read: R, ) -> FromTokioAsyncReadTriGReader { FromTokioAsyncReadTriGReader { @@ -186,14 +186,14 @@ impl TriGParser { /// assert_eq!(2, count); /// # Result::<_,Box>::Ok(()) /// ``` - pub fn parse(&self) -> LowLevelTriGReader { + pub fn parse(self) -> LowLevelTriGReader { LowLevelTriGReader { parser: TriGRecognizer::new_parser( true, #[cfg(feature = "rdf-star")] self.with_quoted_triples, - self.base.clone(), - self.prefixes.clone(), + self.base, + self.prefixes, ), } } @@ -395,7 +395,7 @@ impl TriGSerializer { /// ); /// # Result::<_,Box>::Ok(()) /// ``` - pub fn serialize_to_write(&self, write: W) -> ToWriteTriGWriter { + pub fn serialize_to_write(self, write: W) -> ToWriteTriGWriter { ToWriteTriGWriter { write, writer: self.serialize(), @@ -427,7 +427,7 @@ impl TriGSerializer { /// ``` #[cfg(feature = "async-tokio")] pub fn serialize_to_tokio_async_write( - &self, + self, write: W, ) -> ToTokioAsyncWriteTriGWriter { ToTokioAsyncWriteTriGWriter { diff --git a/lib/oxttl/src/turtle.rs b/lib/oxttl/src/turtle.rs index b5a1b0cd..70be2806 100644 --- a/lib/oxttl/src/turtle.rs +++ b/lib/oxttl/src/turtle.rs @@ -107,7 +107,7 @@ impl TurtleParser { /// assert_eq!(2, count); /// # Result::<_,Box>::Ok(()) /// ``` - pub fn parse_read(&self, read: R) -> FromReadTurtleReader { + pub fn parse_read(self, read: R) -> FromReadTurtleReader { FromReadTurtleReader { inner: self.parse().parser.parse_read(read), } @@ -144,7 +144,7 @@ impl TurtleParser { /// ``` #[cfg(feature = "async-tokio")] pub fn parse_tokio_async_read( - &self, + self, read: R, ) -> FromTokioAsyncReadTurtleReader { FromTokioAsyncReadTurtleReader { @@ -188,14 +188,14 @@ impl TurtleParser { /// assert_eq!(2, count); /// # Result::<_,Box>::Ok(()) /// ``` - pub fn parse(&self) -> LowLevelTurtleReader { + pub fn parse(self) -> LowLevelTurtleReader { LowLevelTurtleReader { parser: TriGRecognizer::new_parser( false, #[cfg(feature = "rdf-star")] self.with_quoted_triples, - self.base.clone(), - self.prefixes.clone(), + self.base, + self.prefixes, ), } } @@ -397,7 +397,7 @@ impl TurtleSerializer { /// ); /// # Result::<_,Box>::Ok(()) /// ``` - pub fn serialize_to_write(&self, write: W) -> ToWriteTurtleWriter { + pub fn serialize_to_write(self, write: W) -> ToWriteTurtleWriter { ToWriteTurtleWriter { inner: self.inner.serialize_to_write(write), } @@ -427,7 +427,7 @@ impl TurtleSerializer { /// ``` #[cfg(feature = "async-tokio")] pub fn serialize_to_tokio_async_write( - &self, + self, write: W, ) -> ToTokioAsyncWriteTurtleWriter { ToTokioAsyncWriteTurtleWriter { diff --git a/lib/src/io/read.rs b/lib/src/io/read.rs index 3e2f0e2a..6960112a 100644 --- a/lib/src/io/read.rs +++ b/lib/src/io/read.rs @@ -65,7 +65,7 @@ impl GraphParser { } /// Executes the parsing itself on a [`Read`] implementation and returns an iterator of triples. - pub fn read_triples(&self, reader: R) -> TripleReader { + pub fn read_triples(self, reader: R) -> TripleReader { TripleReader { parser: self.inner.parse_read(reader), } @@ -154,7 +154,7 @@ impl DatasetParser { } /// Executes the parsing itself on a [`Read`] implementation and returns an iterator of quads. - pub fn read_quads(&self, reader: R) -> QuadReader { + pub fn read_quads(self, reader: R) -> QuadReader { QuadReader { parser: self.inner.parse_read(reader), } diff --git a/lib/src/io/write.rs b/lib/src/io/write.rs index 4c398791..26a8c441 100644 --- a/lib/src/io/write.rs +++ b/lib/src/io/write.rs @@ -45,7 +45,7 @@ impl GraphSerializer { } /// Returns a [`TripleWriter`] allowing writing triples into the given [`Write`] implementation - pub fn triple_writer(&self, write: W) -> TripleWriter { + pub fn triple_writer(self, write: W) -> TripleWriter { TripleWriter { writer: self.inner.serialize_to_write(write), } @@ -128,7 +128,7 @@ impl DatasetSerializer { } /// Returns a [`QuadWriter`] allowing writing triples into the given [`Write`] implementation - pub fn quad_writer(&self, write: W) -> QuadWriter { + pub fn quad_writer(self, write: W) -> QuadWriter { QuadWriter { writer: self.inner.serialize_to_write(write), } diff --git a/server/src/main.rs b/server/src/main.rs index 1178d1dc..14cb725e 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -733,18 +733,18 @@ pub fn main() -> anyhow::Result<()> { match (from_file, to_file) { (Some(from_file), Some(to_file)) => close_file_writer(do_convert( - &parser, + parser, File::open(from_file)?, - &serializer, + serializer, BufWriter::new(File::create(to_file)?), lenient, &from_graph, &to_graph, )?), (Some(from_file), None) => do_convert( - &parser, + parser, File::open(from_file)?, - &serializer, + serializer, stdout().lock(), lenient, &from_graph, @@ -752,18 +752,18 @@ pub fn main() -> anyhow::Result<()> { )? .flush(), (None, Some(to_file)) => close_file_writer(do_convert( - &parser, + parser, stdin().lock(), - &serializer, + serializer, BufWriter::new(File::create(to_file)?), lenient, &from_graph, &to_graph, )?), (None, None) => do_convert( - &parser, + parser, stdin().lock(), - &serializer, + serializer, stdout().lock(), lenient, &from_graph, @@ -806,9 +806,9 @@ fn dump( } fn do_convert( - parser: &RdfParser, + parser: RdfParser, read: R, - serializer: &RdfSerializer, + serializer: RdfSerializer, write: W, lenient: bool, from_graph: &Option, From 3de3f9c4bc04aa80d95010c9f920477c7b599a4a Mon Sep 17 00:00:00 2001 From: Tpt Date: Sun, 20 Aug 2023 21:28:30 +0200 Subject: [PATCH 059/217] Server: Adds ValueHint annotations --- server/src/main.rs | 58 +++++++++++++++++++++++----------------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/server/src/main.rs b/server/src/main.rs index 14cb725e..4082ed52 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -1,6 +1,6 @@ #![allow(clippy::print_stderr, clippy::cast_precision_loss, clippy::use_debug)] use anyhow::{anyhow, bail, ensure, Context}; -use clap::{Parser, Subcommand}; +use clap::{Parser, Subcommand, ValueHint}; use flate2::read::MultiGzDecoder; use oxhttp::model::{Body, HeaderName, HeaderValue, Method, Request, Response, Status}; use oxhttp::Server; @@ -52,10 +52,10 @@ enum Command { /// Directory in which the data should be persisted. /// /// If not present. An in-memory storage will be used. - #[arg(short, long)] + #[arg(short, long, value_hint = ValueHint::DirPath)] location: Option, /// Host and port to listen to. - #[arg(short, long, default_value = "localhost:7878")] + #[arg(short, long, default_value = "localhost:7878", value_hint = ValueHint::Hostname)] bind: String, /// Allows cross-origin requests #[arg(long)] @@ -68,7 +68,7 @@ enum Command { /// Please use the serve-secondary command in this case. ServeReadOnly { /// Directory in which Oxigraph data are persisted. - #[arg(short, long)] + #[arg(short, long, value_hint = ValueHint::DirPath)] location: PathBuf, /// Host and port to listen to. #[arg(short, long, default_value = "localhost:7878")] @@ -86,12 +86,12 @@ enum Command { /// Dirty reads might happen. ServeSecondary { /// Directory where the primary Oxigraph instance is writing to. - #[arg(long)] + #[arg(long, value_hint = ValueHint::DirPath)] primary_location: PathBuf, /// Directory to which the current secondary instance might write to. /// /// By default, temporary storage is used. - #[arg(long)] + #[arg(long, value_hint = ValueHint::DirPath)] secondary_location: Option, /// Host and port to listen to. #[arg(short, long, default_value = "localhost:7878")] @@ -113,23 +113,23 @@ enum Command { /// If you want to move your data to another RDF storage system, you should use the dump operation instead. Backup { /// Directory in which Oxigraph data are persisted. - #[arg(short, long)] + #[arg(short, long, value_hint = ValueHint::DirPath)] location: PathBuf, /// Directory in which the backup will be written. - #[arg(short, long)] + #[arg(short, long, value_hint = ValueHint::DirPath)] destination: PathBuf, }, /// Load file(s) into the store. Load { /// Directory in which Oxigraph data are persisted. - #[arg(short, long)] + #[arg(short, long, value_hint = ValueHint::DirPath)] location: PathBuf, /// File(s) to load. /// /// If multiple files are provided they are loaded in parallel. /// /// If no file is given, stdin is read. - #[arg(short, long, num_args = 0..)] + #[arg(short, long, num_args = 0.., value_hint = ValueHint::FilePath)] file: Vec, /// The format of the file(s) to load. /// @@ -139,7 +139,7 @@ enum Command { #[arg(long, required_unless_present = "file")] format: Option, /// Base IRI of the file(s) to load. - #[arg(long)] + #[arg(long, value_hint = ValueHint::Url)] base: Option, /// Attempt to keep loading even if the data file is invalid. #[arg(long)] @@ -149,18 +149,18 @@ enum Command { /// By default the default graph is used. /// /// Only available when loading a graph file (N-Triples, Turtle...) and not a dataset file (N-Quads, TriG...). - #[arg(long)] + #[arg(long, value_hint = ValueHint::Url)] graph: Option, }, /// Dump the store content into a file. Dump { /// Directory in which Oxigraph data are persisted. - #[arg(short, long)] + #[arg(short, long, value_hint = ValueHint::DirPath)] location: PathBuf, /// File to dump to. /// /// If no file is given, stdout is used. - #[arg(short, long)] + #[arg(short, long, value_hint = ValueHint::FilePath)] file: Option, /// The format of the file(s) to dump. /// @@ -172,13 +172,13 @@ enum Command { /// Name of the graph to dump. /// /// By default all graphs are dumped if the output format supports datasets. - #[arg(long)] + #[arg(long, value_hint = ValueHint::Url)] graph: Option, }, /// Executes a SPARQL query against the store. Query { /// Directory in which Oxigraph data are persisted. - #[arg(short, long)] + #[arg(short, long, value_hint = ValueHint::DirPath)] location: PathBuf, /// The SPARQL query to execute. /// @@ -188,15 +188,15 @@ enum Command { /// File in which the query is stored. /// /// If no query or query file are given, stdin is used. - #[arg(long, conflicts_with = "query")] + #[arg(long, conflicts_with = "query", value_hint = ValueHint::FilePath)] query_file: Option, /// Base IRI of the query. - #[arg(long)] + #[arg(long, value_hint = ValueHint::Url)] query_base: Option, /// File in which the query results will be stored. /// /// If no file is given, stdout is used. - #[arg(short, long)] + #[arg(short, long, value_hint = ValueHint::FilePath)] results_file: Option, /// The format of the results. /// @@ -215,7 +215,7 @@ enum Command { /// If the file extension is .json the JSON format is used, if .txt a human readable format is used. /// /// Use the stats option to print also query evaluation statistics. - #[arg(long, conflicts_with = "explain")] + #[arg(long, conflicts_with = "explain", value_hint = ValueHint::FilePath)] explain_file: Option, /// Computes some evaluation statistics to print as part of the query explanations. /// @@ -226,7 +226,7 @@ enum Command { /// Executes a SPARQL update against the store. Update { /// Directory in which Oxigraph data are persisted. - #[arg(short, long)] + #[arg(short, long, value_hint = ValueHint::DirPath)] location: PathBuf, /// The SPARQL update to execute. /// @@ -236,10 +236,10 @@ enum Command { /// File in which the update is stored. /// /// If no update or update file are given, stdin is used. - #[arg(long, conflicts_with = "update")] + #[arg(long, conflicts_with = "update", value_hint = ValueHint::FilePath)] update_file: Option, /// Base IRI of the update. - #[arg(long)] + #[arg(long, value_hint = ValueHint::Url)] update_base: Option, }, /// Optimizes the database storage. @@ -248,7 +248,7 @@ enum Command { /// It is likely to not be useful in most of cases except if you provide a read-only SPARQL endpoint under heavy load. Optimize { /// Directory in which Oxigraph data are persisted. - #[arg(short, long)] + #[arg(short, long, value_hint = ValueHint::DirPath)] location: PathBuf, }, /// Converts a RDF serialization from one format to an other. @@ -256,7 +256,7 @@ enum Command { /// File to convert from. /// /// If no file is given, stdin is read. - #[arg(short, long)] + #[arg(short, long, value_hint = ValueHint::FilePath)] from_file: Option, /// The format of the file(s) to convert from. /// @@ -266,12 +266,12 @@ enum Command { #[arg(long, required_unless_present = "from_file")] from_format: Option, /// Base IRI of the file to read. - #[arg(long)] + #[arg(long, value_hint = ValueHint::Url)] from_base: Option, /// File to convert to. /// /// If no file is given, stdout is written. - #[arg(short, long)] + #[arg(short, long, value_hint = ValueHint::FilePath)] to_file: Option, /// The format of the file(s) to convert from. /// @@ -286,7 +286,7 @@ enum Command { /// Only load the given named graph from the input file. /// /// By default all graphs are loaded. - #[arg(long, conflicts_with = "from_default_graph")] + #[arg(long, conflicts_with = "from_default_graph", value_hint = ValueHint::Url)] from_graph: Option, /// Only load the default graph from the input file. #[arg(long, conflicts_with = "from_graph")] @@ -294,7 +294,7 @@ enum Command { /// Name of the graph to map the default graph to. /// /// By default the default graph is used. - #[arg(long)] + #[arg(long, value_hint = ValueHint::Url)] to_graph: Option, }, } From 872111ab8803f8a0ba6044ca996385b5b4543dc3 Mon Sep 17 00:00:00 2001 From: Tpt Date: Sun, 20 Aug 2023 18:55:44 +0200 Subject: [PATCH 060/217] Makes Graph and Dataset implement Clone --- lib/oxrdf/src/dataset.rs | 2 +- lib/oxrdf/src/graph.rs | 2 +- lib/oxrdf/src/interning.rs | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/oxrdf/src/dataset.rs b/lib/oxrdf/src/dataset.rs index 4b7736eb..e5688cb9 100644 --- a/lib/oxrdf/src/dataset.rs +++ b/lib/oxrdf/src/dataset.rs @@ -62,7 +62,7 @@ use std::hash::{Hash, Hasher}; /// assert_eq!(vec![TripleRef::new(ex, ex, ex)], results); /// # Result::<_,Box>::Ok(()) /// ``` -#[derive(Debug, Default)] +#[derive(Debug, Default, Clone)] pub struct Dataset { interner: Interner, gspo: BTreeSet<( diff --git a/lib/oxrdf/src/graph.rs b/lib/oxrdf/src/graph.rs index 4298684c..a40d1293 100644 --- a/lib/oxrdf/src/graph.rs +++ b/lib/oxrdf/src/graph.rs @@ -48,7 +48,7 @@ use std::fmt; /// assert_eq!(vec![triple], results); /// # Result::<_,Box>::Ok(()) /// ``` -#[derive(Debug, Default)] +#[derive(Debug, Default, Clone)] pub struct Graph { dataset: Dataset, } diff --git a/lib/oxrdf/src/interning.rs b/lib/oxrdf/src/interning.rs index 3414d51a..bc00c4a8 100644 --- a/lib/oxrdf/src/interning.rs +++ b/lib/oxrdf/src/interning.rs @@ -4,7 +4,7 @@ use crate::*; use std::collections::hash_map::{Entry, HashMap, RandomState}; use std::hash::{BuildHasher, Hasher}; -#[derive(Debug, Default)] +#[derive(Debug, Default, Clone)] pub struct Interner { hasher: RandomState, string_for_hash: HashMap, @@ -503,7 +503,7 @@ impl InternedTriple { } } -#[derive(Default)] +#[derive(Default, Clone)] struct IdentityHasherBuilder; impl BuildHasher for IdentityHasherBuilder { From 9e76323e2b700fb7fe83dec6f8cc933f38ed53dd Mon Sep 17 00:00:00 2001 From: Tpt Date: Mon, 21 Aug 2023 21:31:55 +0200 Subject: [PATCH 061/217] CI: Add cargo caching for linux wheel build --- .github/workflows/artifacts.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/artifacts.yml b/.github/workflows/artifacts.yml index bf91b537..ddcc158d 100644 --- a/.github/workflows/artifacts.yml +++ b/.github/workflows/artifacts.yml @@ -121,6 +121,7 @@ jobs: with: platforms: linux/${{ matrix.architecture }} if: github.event_name == 'release' && matrix.architecture != 'x86_64' + - uses: Swatinem/rust-cache@v2 - run: sed 's/%arch%/${{ matrix.architecture }}/g' .github/workflows/manylinux_build.sh | sed 's/%for_each_version%/${{ github.event_name == 'release' || '' }}/g' > .github/workflows/manylinux_build_script.sh - run: docker run -v "$(pwd)":/workdir --platform linux/${{ matrix.architecture }} quay.io/pypa/manylinux2014_${{ matrix.architecture }} /bin/bash /workdir/.github/workflows/manylinux_build_script.sh if: github.event_name == 'release' || matrix.architecture == 'x86_64' @@ -149,6 +150,7 @@ jobs: with: platforms: linux/${{ matrix.architecture }} if: github.event_name == 'release' && matrix.architecture != 'x86_64' + - uses: Swatinem/rust-cache@v2 - run: sed 's/%arch%/${{ matrix.architecture }}/g' .github/workflows/musllinux_build.sh | sed 's/%for_each_version%/${{ github.event_name == 'release' || '' }}/g' > .github/workflows/musllinux_build_script.sh - run: docker run -v "$(pwd)":/workdir --platform linux/${{ matrix.architecture }} quay.io/pypa/musllinux_1_1_${{ matrix.architecture }} /bin/bash /workdir/.github/workflows/musllinux_build_script.sh if: github.event_name == 'release' || matrix.architecture == 'x86_64' From d2306cea52c78c8bd442585cc45eb56486096ea4 Mon Sep 17 00:00:00 2001 From: Tpt Date: Mon, 21 Aug 2023 08:41:50 +0200 Subject: [PATCH 062/217] Improves type inference on zero-args functions --- lib/sparopt/src/type_inference.rs | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/lib/sparopt/src/type_inference.rs b/lib/sparopt/src/type_inference.rs index 03960adb..d52476bd 100644 --- a/lib/sparopt/src/type_inference.rs +++ b/lib/sparopt/src/type_inference.rs @@ -182,9 +182,17 @@ pub fn infer_expression_type(expression: &Expression, types: &VariableTypes) -> Expression::FunctionCall(Function::Predicate, _) => { VariableType::NAMED_NODE | VariableType::UNDEF } - Expression::FunctionCall(Function::BNode, _) => { - VariableType::BLANK_NODE | VariableType::UNDEF + Expression::FunctionCall(Function::BNode, args) => { + if args.is_empty() { + VariableType::BLANK_NODE + } else { + VariableType::BLANK_NODE | VariableType::UNDEF + } } + Expression::FunctionCall( + Function::Rand | Function::Now | Function::Uuid | Function::StrUuid, + _, + ) => VariableType::LITERAL, Expression::Or(_) | Expression::And(_) | Expression::Equal(_, _) @@ -203,7 +211,6 @@ pub fn infer_expression_type(expression: &Expression, types: &VariableTypes) -> Function::Str | Function::Lang | Function::LangMatches - | Function::Rand | Function::Abs | Function::Ceil | Function::Floor @@ -228,9 +235,6 @@ pub fn infer_expression_type(expression: &Expression, types: &VariableTypes) -> | Function::Seconds | Function::Timezone | Function::Tz - | Function::Now - | Function::Uuid - | Function::StrUuid | Function::Md5 | Function::Sha1 | Function::Sha256 From 570f21748d9c88168ecf37736b390ae9d4787668 Mon Sep 17 00:00:00 2001 From: Tpt Date: Mon, 21 Aug 2023 08:06:29 +0200 Subject: [PATCH 063/217] Cargo.toml: share some common fields in the workspace --- .github/workflows/tests.yml | 3 ++- Cargo.lock | 2 +- Cargo.toml | 8 ++++++++ js/Cargo.toml | 10 ++++++---- lib/Cargo.toml | 12 ++++++------ lib/oxrdf/Cargo.toml | 10 +++++----- lib/oxrdfio/Cargo.toml | 10 +++++----- lib/oxrdfxml/Cargo.toml | 10 +++++----- lib/oxsdatatypes/Cargo.toml | 10 +++++----- lib/oxttl/Cargo.toml | 10 +++++----- lib/sparesults/Cargo.toml | 10 +++++----- lib/spargebra/Cargo.toml | 10 +++++----- lib/sparopt/Cargo.toml | 10 +++++----- lib/sparql-smith/Cargo.toml | 9 +++++---- oxrocksdb-sys/Cargo.toml | 8 ++++---- python/Cargo.toml | 10 ++++++---- server/Cargo.toml | 10 +++++----- testsuite/Cargo.toml | 11 +++++------ 18 files changed, 88 insertions(+), 75 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a58e710c..5dec1690 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -156,6 +156,7 @@ jobs: semver_checks: runs-on: ubuntu-latest + if: false steps: - uses: actions/checkout@v3 with: @@ -163,7 +164,7 @@ jobs: - run: rustup update - uses: Swatinem/rust-cache@v2 - run: cargo install cargo-semver-checks || true - - run: cargo semver-checks check-release --exclude oxrocksdb-sys --exclude oxigraph_js --exclude pyoxigraph --exclude oxigraph_testsuite --exclude oxigraph_server --exclude oxrdfxml --exclude oxttl --exclude oxrdfio --exclude sparopt + - run: cargo semver-checks check-release --exclude oxrocksdb-sys --exclude oxigraph_js --exclude pyoxigraph --exclude oxigraph_testsuite --exclude oxigraph_server test_linux: runs-on: ubuntu-latest diff --git a/Cargo.lock b/Cargo.lock index c27daea8..aa576fdc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1000,7 +1000,7 @@ dependencies = [ [[package]] name = "oxigraph_testsuite" -version = "0.0.0" +version = "0.4.0-alpha.1-dev" dependencies = [ "anyhow", "clap", diff --git a/Cargo.toml b/Cargo.toml index 8b6c3e48..466be35b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,14 @@ members = [ ] resolver = "2" +[workspace.package] +version = "0.4.0-alpha.1-dev" +authors = ["Tpt "] +license = "MIT OR Apache-2.0" +homepage = "https://oxigraph.org/" +edition = "2021" +rust-version = "1.65" + [profile.release] lto = true codegen-units = 1 diff --git a/js/Cargo.toml b/js/Cargo.toml index 86390266..0b3f5738 100644 --- a/js/Cargo.toml +++ b/js/Cargo.toml @@ -1,13 +1,15 @@ [package] name = "oxigraph_js" -version = "0.4.0-alpha.1-dev" -authors = ["Tpt "] -license = "MIT OR Apache-2.0" +version.workspace = true +authors.workspace = true +license.workspace = true readme = "README.md" keywords = ["RDF", "N-Triples", "Turtle", "RDF/XML", "SPARQL"] repository = "https://github.com/oxigraph/oxigraph/tree/main/js" description = "JavaScript bindings of Oxigraph" -edition = "2021" +edition.workspace = true +rust-version.workspace = true +publish = false [lib] crate-type = ["cdylib"] diff --git a/lib/Cargo.toml b/lib/Cargo.toml index 0cf839de..d3eb9c04 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -1,19 +1,19 @@ [package] name = "oxigraph" -version = "0.4.0-alpha.1-dev" -authors = ["Tpt "] -license = "MIT OR Apache-2.0" +version.workspace = true +authors.workspace = true +license.workspace = true readme = "README.md" keywords = ["RDF", "SPARQL", "graph-database", "database"] categories = ["database-implementations"] repository = "https://github.com/oxigraph/oxigraph/tree/main/lib" -homepage = "https://oxigraph.org/" +homepage.workspace = true documentation = "https://docs.rs/oxigraph" description = """ a SPARQL database and RDF toolkit """ -edition = "2021" -rust-version = "1.65" +edition.workspace = true +rust-version.workspace = true [features] default = [] diff --git a/lib/oxrdf/Cargo.toml b/lib/oxrdf/Cargo.toml index 48845b04..a61523a3 100644 --- a/lib/oxrdf/Cargo.toml +++ b/lib/oxrdf/Cargo.toml @@ -1,18 +1,18 @@ [package] name = "oxrdf" version = "0.2.0-alpha.1-dev" -authors = ["Tpt "] -license = "MIT OR Apache-2.0" +authors.workspace = true +license.workspace = true readme = "README.md" keywords = ["RDF"] repository = "https://github.com/oxigraph/oxigraph/tree/main/lib/oxrdf" -homepage = "https://oxigraph.org/" +homepage.workspace = true description = """ A library providing basic data structures related to RDF """ documentation = "https://docs.rs/oxrdf" -edition = "2021" -rust-version = "1.65" +edition.workspace = true +rust-version.workspace = true [features] default = [] diff --git a/lib/oxrdfio/Cargo.toml b/lib/oxrdfio/Cargo.toml index 70f266b0..c04c16fa 100644 --- a/lib/oxrdfio/Cargo.toml +++ b/lib/oxrdfio/Cargo.toml @@ -1,18 +1,18 @@ [package] name = "oxrdfio" version = "0.1.0-alpha.1-dev" -authors = ["Tpt "] -license = "MIT OR Apache-2.0" +authors.workspace = true +license.workspace = true readme = "README.md" keywords = ["RDF"] repository = "https://github.com/oxigraph/oxigraph/tree/master/lib/oxrdfxml" -homepage = "https://oxigraph.org/" +homepage.workspace = true documentation = "https://docs.rs/oxrdfio" description = """ Parser for various RDF serializations """ -edition = "2021" -rust-version = "1.65" +edition.workspace = true +rust-version.workspace = true [features] default = [] diff --git a/lib/oxrdfxml/Cargo.toml b/lib/oxrdfxml/Cargo.toml index 371eb19f..f4c4d67d 100644 --- a/lib/oxrdfxml/Cargo.toml +++ b/lib/oxrdfxml/Cargo.toml @@ -1,18 +1,18 @@ [package] name = "oxrdfxml" version = "0.1.0-alpha.1-dev" -authors = ["Tpt "] -license = "MIT OR Apache-2.0" +authors.workspace = true +license.workspace = true readme = "README.md" keywords = ["RDF/XML", "RDF"] repository = "https://github.com/oxigraph/oxigraph/tree/master/lib/oxrdfxml" -homepage = "https://oxigraph.org/" +homepage.workspace = true description = """ Parser for the RDF/XML language """ documentation = "https://docs.rs/oxrdfxml" -edition = "2021" -rust-version = "1.65" +edition.workspace = true +rust-version.workspace = true [features] default = [] diff --git a/lib/oxsdatatypes/Cargo.toml b/lib/oxsdatatypes/Cargo.toml index f4f42ab8..64b974a9 100644 --- a/lib/oxsdatatypes/Cargo.toml +++ b/lib/oxsdatatypes/Cargo.toml @@ -1,18 +1,18 @@ [package] name = "oxsdatatypes" version = "0.2.0-alpha.1-dev" -authors = ["Tpt "] -license = "MIT OR Apache-2.0" +authors.workspace = true +license.workspace = true readme = "README.md" keywords = ["XSD"] repository = "https://github.com/oxigraph/oxigraph/tree/main/lib/oxsdatatypes" -homepage = "https://oxigraph.org/" +homepage.workspace = true description = """ An implementation of some XSD datatypes for SPARQL implementations """ documentation = "https://docs.rs/oxsdatatypes" -edition = "2021" -rust-version = "1.65" +edition.workspace = true +rust-version.workspace = true [features] js = ["js-sys"] diff --git a/lib/oxttl/Cargo.toml b/lib/oxttl/Cargo.toml index f8039069..f5dda6c6 100644 --- a/lib/oxttl/Cargo.toml +++ b/lib/oxttl/Cargo.toml @@ -1,18 +1,18 @@ [package] name = "oxttl" version = "0.1.0-alpha.1-dev" -authors = ["Tpt "] -license = "MIT OR Apache-2.0" +authors.workspace = true +license.workspace = true readme = "README.md" keywords = ["N-Triples", "N-Quads", "Turtle", "TriG", "N3", "RDF"] repository = "https://github.com/oxigraph/oxigraph/tree/master/lib/oxttl" -homepage = "https://oxigraph.org/" +homepage.workspace = true description = """ Parser for languages related to RDF Turtle (N-Triples, N-Quads, Turtle, TriG and N3) """ documentation = "https://docs.rs/oxttl" -edition = "2021" -rust-version = "1.65" +edition.workspace = true +rust-version.workspace = true [features] default = [] diff --git a/lib/sparesults/Cargo.toml b/lib/sparesults/Cargo.toml index 54e3032a..8cee3164 100644 --- a/lib/sparesults/Cargo.toml +++ b/lib/sparesults/Cargo.toml @@ -1,18 +1,18 @@ [package] name = "sparesults" version = "0.2.0-alpha.1-dev" -authors = ["Tpt "] -license = "MIT OR Apache-2.0" +authors.workspace = true +license.workspace = true readme = "README.md" keywords = ["SPARQL"] repository = "https://github.com/oxigraph/oxigraph/tree/main/lib/sparesults" -homepage = "https://oxigraph.org/" +homepage.workspace = true description = """ SPARQL query results formats parsers and serializers """ documentation = "https://docs.rs/sparesults" -edition = "2021" -rust-version = "1.65" +edition.workspace = true +rust-version.workspace = true [features] default = [] diff --git a/lib/spargebra/Cargo.toml b/lib/spargebra/Cargo.toml index 323502ac..cfc2e8cb 100644 --- a/lib/spargebra/Cargo.toml +++ b/lib/spargebra/Cargo.toml @@ -1,18 +1,18 @@ [package] name = "spargebra" version = "0.3.0-alpha.1-dev" -authors = ["Tpt "] -license = "MIT OR Apache-2.0" +authors.workspace = true +license.workspace = true readme = "README.md" keywords = ["SPARQL"] repository = "https://github.com/oxigraph/oxigraph/tree/main/lib/spargebra" -homepage = "https://oxigraph.org/" +homepage.workspace = true documentation = "https://docs.rs/spargebra" description = """ A SPARQL parser """ -edition = "2021" -rust-version = "1.65" +edition.workspace = true +rust-version.workspace = true [features] default = [] diff --git a/lib/sparopt/Cargo.toml b/lib/sparopt/Cargo.toml index 3406930d..586388bd 100644 --- a/lib/sparopt/Cargo.toml +++ b/lib/sparopt/Cargo.toml @@ -1,18 +1,18 @@ [package] name = "sparopt" version = "0.1.0-alpha.1-dev" -authors = ["Tpt "] -license = "MIT OR Apache-2.0" +authors.workspace = true +license.workspace = true readme = "README.md" keywords = ["SPARQL"] repository = "https://github.com/oxigraph/oxigraph/tree/main/lib/sparopt" -homepage = "https://oxigraph.org/" +homepage.workspace = true documentation = "https://docs.rs/sparopt" description = """ A SPARQL optimizer """ -edition = "2021" -rust-version = "1.60" +edition.workspace = true +rust-version.workspace = true [features] default = [] diff --git a/lib/sparql-smith/Cargo.toml b/lib/sparql-smith/Cargo.toml index c96e416a..5fe5e33c 100644 --- a/lib/sparql-smith/Cargo.toml +++ b/lib/sparql-smith/Cargo.toml @@ -1,17 +1,18 @@ [package] name = "sparql-smith" version = "0.1.0-alpha.5-dev" -authors = ["Tpt "] -license = "MIT OR Apache-2.0" +authors.workspace = true +license.workspace = true readme = "README.md" keywords = ["SPARQL"] repository = "https://github.com/oxigraph/oxigraph/tree/main/lib/sparql-smith" -homepage = "https://oxigraph.org/" +homepage.workspace = true documentation = "https://docs.rs/sparql-smith" description = """ A SPARQL test cases generator """ -edition = "2021" +edition.workspace = true +rust-version.workspace = true [features] default = [] diff --git a/oxrocksdb-sys/Cargo.toml b/oxrocksdb-sys/Cargo.toml index 8cbdba15..e91881e9 100644 --- a/oxrocksdb-sys/Cargo.toml +++ b/oxrocksdb-sys/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "oxrocksdb-sys" -version = "0.4.0-alpha.1-dev" -authors = ["Tpt "] +version.workspace = true +authors.workspace = true license = "GPL-2.0 OR Apache-2.0" repository = "https://github.com/oxigraph/oxigraph/tree/main/oxrocksdb-sys" readme = "README.md" @@ -9,8 +9,8 @@ description = """ Rust bindings for RocksDB for Oxigraph usage. """ documentation = "https://docs.rs/oxrocksdb-sys" -edition = "2021" -rust-version = "1.65" +edition.workspace = true +rust-version.workspace = true build = "build.rs" links = "rocksdb" diff --git a/python/Cargo.toml b/python/Cargo.toml index 242a4c0e..a8f610e1 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -1,14 +1,16 @@ [package] name = "pyoxigraph" -version = "0.4.0-alpha.1-dev" -authors = ["Tpt"] -license = "MIT OR Apache-2.0" +version.workspace = true +authors.workspace = true +license.workspace = true readme = "README.md" keywords = ["RDF", "SPARQL", "graph-database", "database"] repository = "https://github.com/oxigraph/oxigraph/tree/main/python" homepage = "https://pyoxigraph.readthedocs.io/" description = "Python bindings of Oxigraph, a SPARQL database and RDF toolkit" -edition = "2021" +edition.workspace = true +rust-version.workspace = true +publish = false [lib] crate-type = ["cdylib"] diff --git a/server/Cargo.toml b/server/Cargo.toml index 36b5fc00..9e61e54d 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -1,16 +1,16 @@ [package] name = "oxigraph_server" -version = "0.4.0-alpha.1-dev" -authors = ["Tpt "] -license = "MIT OR Apache-2.0" +version.workspace = true +authors.workspace = true +license.workspace = true readme = "README.md" repository = "https://github.com/oxigraph/oxigraph/tree/main/server" homepage = "https://oxigraph.org/server/" description = """ Oxigraph SPARQL HTTP server """ -edition = "2021" -rust-version = "1.65" +edition.workspace = true +rust-version.workspace = true [dependencies] anyhow = "1" diff --git a/testsuite/Cargo.toml b/testsuite/Cargo.toml index af2e859d..a32411f3 100644 --- a/testsuite/Cargo.toml +++ b/testsuite/Cargo.toml @@ -1,14 +1,13 @@ [package] name = "oxigraph_testsuite" -version = "0.0.0" -authors = ["Tpt "] -license = "MIT OR Apache-2.0" -readme = "../README.md" -repository = "https://github.com/oxigraph/oxigraph" +version.workspace = true +authors.workspace = true +license.workspace = true description = """ Implementation of W3C testsuites for Oxigraph """ -edition = "2021" +edition.workspace = true +rust-version.workspace = true publish = false [dependencies] From d44f9bee7a43179584a9591c6d4fd00e68ddcb0e Mon Sep 17 00:00:00 2001 From: Tpt Date: Sun, 20 Aug 2023 19:29:50 +0200 Subject: [PATCH 064/217] I/O adds extra #[must_use] annotations --- lib/oxrdfio/src/parser.rs | 5 +---- lib/oxrdfio/src/serializer.rs | 1 + lib/oxrdfxml/src/parser.rs | 3 +++ lib/oxrdfxml/src/serializer.rs | 3 +++ lib/oxttl/src/n3.rs | 3 +++ lib/oxttl/src/nquads.rs | 7 ++++++- lib/oxttl/src/ntriples.rs | 7 ++++++- lib/oxttl/src/trig.rs | 7 ++++++- lib/oxttl/src/turtle.rs | 7 ++++++- 9 files changed, 35 insertions(+), 8 deletions(-) diff --git a/lib/oxrdfio/src/parser.rs b/lib/oxrdfio/src/parser.rs index 4c7090b6..25f86598 100644 --- a/lib/oxrdfio/src/parser.rs +++ b/lib/oxrdfio/src/parser.rs @@ -53,6 +53,7 @@ use tokio::io::AsyncRead; /// assert_eq!(quads[0].subject.to_string(), ""); /// # std::io::Result::Ok(()) /// ``` +#[must_use] pub struct RdfParser { inner: RdfParserKind, default_graph: GraphName, @@ -72,7 +73,6 @@ enum RdfParserKind { impl RdfParser { /// Builds a parser for the given format. #[inline] - #[must_use] pub fn from_format(format: RdfFormat) -> Self { Self { inner: match format { @@ -172,7 +172,6 @@ impl RdfParser { /// # Result::<_,Box>::Ok(()) /// ``` #[inline] - #[must_use] pub fn with_default_graph(self, default_graph: impl Into) -> Self { Self { inner: self.inner, @@ -195,7 +194,6 @@ impl RdfParser { /// assert!(parser.parse_read(file.as_bytes()).next().unwrap().is_err()); /// ``` #[inline] - #[must_use] pub fn without_named_graphs(self) -> Self { Self { inner: self.inner, @@ -224,7 +222,6 @@ impl RdfParser { /// # Result::<_,Box>::Ok(()) /// ``` #[inline] - #[must_use] pub fn rename_blank_nodes(self) -> Self { Self { inner: self.inner, diff --git a/lib/oxrdfio/src/serializer.rs b/lib/oxrdfio/src/serializer.rs index 26229e5d..6289ed28 100644 --- a/lib/oxrdfio/src/serializer.rs +++ b/lib/oxrdfio/src/serializer.rs @@ -48,6 +48,7 @@ use tokio::io::AsyncWrite; /// assert_eq!(buffer.as_slice(), " .\n".as_bytes()); /// # Result::<_,Box>::Ok(()) /// ``` +#[must_use] pub struct RdfSerializer { format: RdfFormat, } diff --git a/lib/oxrdfxml/src/parser.rs b/lib/oxrdfxml/src/parser.rs index a17f9ca4..747b2419 100644 --- a/lib/oxrdfxml/src/parser.rs +++ b/lib/oxrdfxml/src/parser.rs @@ -50,6 +50,7 @@ use tokio::io::{AsyncRead, BufReader as AsyncBufReader}; /// # Result::<_,Box>::Ok(()) /// ``` #[derive(Default)] +#[must_use] pub struct RdfXmlParser { base: Option>, } @@ -188,6 +189,7 @@ impl RdfXmlParser { /// assert_eq!(2, count); /// # Result::<_,Box>::Ok(()) /// ``` +#[must_use] pub struct FromReadRdfXmlReader { results: Vec, reader: RdfXmlReader>, @@ -259,6 +261,7 @@ impl FromReadRdfXmlReader { /// } /// ``` #[cfg(feature = "async-tokio")] +#[must_use] pub struct FromTokioAsyncReadRdfXmlReader { results: Vec, reader: RdfXmlReader>, diff --git a/lib/oxrdfxml/src/serializer.rs b/lib/oxrdfxml/src/serializer.rs index e3d93065..d3e9949f 100644 --- a/lib/oxrdfxml/src/serializer.rs +++ b/lib/oxrdfxml/src/serializer.rs @@ -27,6 +27,7 @@ use tokio::io::AsyncWrite; /// # Result::<_,Box>::Ok(()) /// ``` #[derive(Default)] +#[must_use] pub struct RdfXmlSerializer; impl RdfXmlSerializer { @@ -123,6 +124,7 @@ impl RdfXmlSerializer { /// ); /// # Result::<_,Box>::Ok(()) /// ``` +#[must_use] pub struct ToWriteRdfXmlWriter { writer: Writer, inner: InnerRdfXmlWriter, @@ -176,6 +178,7 @@ impl ToWriteRdfXmlWriter { /// } /// ``` #[cfg(feature = "async-tokio")] +#[must_use] pub struct ToTokioAsyncWriteRdfXmlWriter { writer: Writer, inner: InnerRdfXmlWriter, diff --git a/lib/oxttl/src/n3.rs b/lib/oxttl/src/n3.rs index 0edb9d13..14c58413 100644 --- a/lib/oxttl/src/n3.rs +++ b/lib/oxttl/src/n3.rs @@ -204,6 +204,7 @@ impl From for N3Quad { /// # Result::<_,Box>::Ok(()) /// ``` #[derive(Default)] +#[must_use] pub struct N3Parser { base: Option>, prefixes: HashMap>, @@ -376,6 +377,7 @@ impl N3Parser { /// assert_eq!(2, count); /// # Result::<_,Box>::Ok(()) /// ``` +#[must_use] pub struct FromReadN3Reader { inner: FromReadIterator, } @@ -420,6 +422,7 @@ impl Iterator for FromReadN3Reader { /// } /// ``` #[cfg(feature = "async-tokio")] +#[must_use] pub struct FromTokioAsyncReadN3Reader { inner: FromTokioAsyncReadIterator, } diff --git a/lib/oxttl/src/nquads.rs b/lib/oxttl/src/nquads.rs index 71ac2d4c..caa7c642 100644 --- a/lib/oxttl/src/nquads.rs +++ b/lib/oxttl/src/nquads.rs @@ -35,6 +35,7 @@ use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; /// # Result::<_,Box>::Ok(()) /// ``` #[derive(Default)] +#[must_use] pub struct NQuadsParser { #[cfg(feature = "rdf-star")] with_quoted_triples: bool, @@ -50,7 +51,6 @@ impl NQuadsParser { /// Enables [N-Quads-star](https://w3c.github.io/rdf-star/cg-spec/2021-12-17.html#n-quads-star). #[cfg(feature = "rdf-star")] #[inline] - #[must_use] pub fn with_quoted_triples(mut self) -> Self { self.with_quoted_triples = true; self @@ -193,6 +193,7 @@ impl NQuadsParser { /// assert_eq!(2, count); /// # Result::<_,Box>::Ok(()) /// ``` +#[must_use] pub struct FromReadNQuadsReader { inner: FromReadIterator, } @@ -233,6 +234,7 @@ impl Iterator for FromReadNQuadsReader { /// } /// ``` #[cfg(feature = "async-tokio")] +#[must_use] pub struct FromTokioAsyncReadNQuadsReader { inner: FromTokioAsyncReadIterator, } @@ -334,6 +336,7 @@ impl LowLevelNQuadsReader { /// # Result::<_,Box>::Ok(()) /// ``` #[derive(Default)] +#[must_use] pub struct NQuadsSerializer; impl NQuadsSerializer { @@ -449,6 +452,7 @@ impl NQuadsSerializer { /// ); /// # Result::<_,Box>::Ok(()) /// ``` +#[must_use] pub struct ToWriteNQuadsWriter { write: W, writer: LowLevelNQuadsWriter, @@ -490,6 +494,7 @@ impl ToWriteNQuadsWriter { /// } /// ``` #[cfg(feature = "async-tokio")] +#[must_use] pub struct ToTokioAsyncWriteNQuadsWriter { write: W, writer: LowLevelNQuadsWriter, diff --git a/lib/oxttl/src/ntriples.rs b/lib/oxttl/src/ntriples.rs index 07672f1a..f8c32b12 100644 --- a/lib/oxttl/src/ntriples.rs +++ b/lib/oxttl/src/ntriples.rs @@ -36,6 +36,7 @@ use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; /// # Result::<_,Box>::Ok(()) /// ``` #[derive(Default)] +#[must_use] pub struct NTriplesParser { #[cfg(feature = "rdf-star")] with_quoted_triples: bool, @@ -51,7 +52,6 @@ impl NTriplesParser { /// Enables [N-Triples-star](https://w3c.github.io/rdf-star/cg-spec/2021-12-17.html#n-triples-star). #[cfg(feature = "rdf-star")] #[inline] - #[must_use] pub fn with_quoted_triples(mut self) -> Self { self.with_quoted_triples = true; self @@ -194,6 +194,7 @@ impl NTriplesParser { /// assert_eq!(2, count); /// # Result::<_,Box>::Ok(()) /// ``` +#[must_use] pub struct FromReadNTriplesReader { inner: FromReadIterator, } @@ -234,6 +235,7 @@ impl Iterator for FromReadNTriplesReader { /// } /// ``` #[cfg(feature = "async-tokio")] +#[must_use] pub struct FromTokioAsyncReadNTriplesReader { inner: FromTokioAsyncReadIterator, } @@ -334,6 +336,7 @@ impl LowLevelNTriplesReader { /// # Result::<_,Box>::Ok(()) /// ``` #[derive(Default)] +#[must_use] pub struct NTriplesSerializer; impl NTriplesSerializer { @@ -445,6 +448,7 @@ impl NTriplesSerializer { /// ); /// # Result::<_,Box>::Ok(()) /// ``` +#[must_use] pub struct ToWriteNTriplesWriter { write: W, writer: LowLevelNTriplesWriter, @@ -485,6 +489,7 @@ impl ToWriteNTriplesWriter { /// } /// ``` #[cfg(feature = "async-tokio")] +#[must_use] pub struct ToTokioAsyncWriteNTriplesWriter { write: W, writer: LowLevelNTriplesWriter, diff --git a/lib/oxttl/src/trig.rs b/lib/oxttl/src/trig.rs index f2c24ca0..6dfb2b43 100644 --- a/lib/oxttl/src/trig.rs +++ b/lib/oxttl/src/trig.rs @@ -40,6 +40,7 @@ use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; /// # Result::<_,Box>::Ok(()) /// ``` #[derive(Default)] +#[must_use] pub struct TriGParser { base: Option>, prefixes: HashMap>, @@ -74,7 +75,6 @@ impl TriGParser { /// Enables [TriG-star](https://w3c.github.io/rdf-star/cg-spec/2021-12-17.html#trig-star). #[cfg(feature = "rdf-star")] #[inline] - #[must_use] pub fn with_quoted_triples(mut self) -> Self { self.with_quoted_triples = true; self @@ -224,6 +224,7 @@ impl TriGParser { /// assert_eq!(2, count); /// # Result::<_,Box>::Ok(()) /// ``` +#[must_use] pub struct FromReadTriGReader { inner: FromReadIterator, } @@ -266,6 +267,7 @@ impl Iterator for FromReadTriGReader { /// } /// ``` #[cfg(feature = "async-tokio")] +#[must_use] pub struct FromTokioAsyncReadTriGReader { inner: FromTokioAsyncReadIterator, } @@ -367,6 +369,7 @@ impl LowLevelTriGReader { /// # Result::<_,Box>::Ok(()) /// ``` #[derive(Default)] +#[must_use] pub struct TriGSerializer; impl TriGSerializer { @@ -486,6 +489,7 @@ impl TriGSerializer { /// ); /// # Result::<_,Box>::Ok(()) /// ``` +#[must_use] pub struct ToWriteTriGWriter { write: W, writer: LowLevelTriGWriter, @@ -528,6 +532,7 @@ impl ToWriteTriGWriter { /// } /// ``` #[cfg(feature = "async-tokio")] +#[must_use] pub struct ToTokioAsyncWriteTriGWriter { write: W, writer: LowLevelTriGWriter, diff --git a/lib/oxttl/src/turtle.rs b/lib/oxttl/src/turtle.rs index 70be2806..133c9cca 100644 --- a/lib/oxttl/src/turtle.rs +++ b/lib/oxttl/src/turtle.rs @@ -42,6 +42,7 @@ use tokio::io::{AsyncRead, AsyncWrite}; /// # Result::<_,Box>::Ok(()) /// ``` #[derive(Default)] +#[must_use] pub struct TurtleParser { base: Option>, prefixes: HashMap>, @@ -76,7 +77,6 @@ impl TurtleParser { /// Enables [Turtle-star](https://w3c.github.io/rdf-star/cg-spec/2021-12-17.html#turtle-star). #[cfg(feature = "rdf-star")] #[inline] - #[must_use] pub fn with_quoted_triples(mut self) -> Self { self.with_quoted_triples = true; self @@ -226,6 +226,7 @@ impl TurtleParser { /// assert_eq!(2, count); /// # Result::<_,Box>::Ok(()) /// ``` +#[must_use] pub struct FromReadTurtleReader { inner: FromReadIterator, } @@ -268,6 +269,7 @@ impl Iterator for FromReadTurtleReader { /// } /// ``` #[cfg(feature = "async-tokio")] +#[must_use] pub struct FromTokioAsyncReadTurtleReader { inner: FromTokioAsyncReadIterator, } @@ -368,6 +370,7 @@ impl LowLevelTurtleReader { /// # Result::<_,Box>::Ok(()) /// ``` #[derive(Default)] +#[must_use] pub struct TurtleSerializer { inner: TriGSerializer, } @@ -480,6 +483,7 @@ impl TurtleSerializer { /// ); /// # Result::<_,Box>::Ok(()) /// ``` +#[must_use] pub struct ToWriteTurtleWriter { inner: ToWriteTriGWriter, } @@ -520,6 +524,7 @@ impl ToWriteTurtleWriter { /// } /// ``` #[cfg(feature = "async-tokio")] +#[must_use] pub struct ToTokioAsyncWriteTurtleWriter { inner: ToTokioAsyncWriteTriGWriter, } From c9ec5f7c0cd8cd6240aabab14194ac2049a01bbd Mon Sep 17 00:00:00 2001 From: Tpt Date: Mon, 21 Aug 2023 21:31:11 +0200 Subject: [PATCH 065/217] Python and & JS: Renames "mime_type" parameter to "format" - adds support of extensions - MIME type is a deprecated wording --- js/README.md | 30 ++++++------ js/src/store.rs | 22 +++++---- python/src/io.rs | 72 +++++++++++++++------------- python/src/store.rs | 98 +++++++++++++++++--------------------- python/tests/test_io.py | 2 +- python/tests/test_store.py | 20 ++++---- 6 files changed, 125 insertions(+), 119 deletions(-) diff --git a/js/README.md b/js/README.md index 576c9739..0ee92e9b 100644 --- a/js/README.md +++ b/js/README.md @@ -197,40 +197,42 @@ Example of update: store.update("DELETE WHERE { ?p ?o }") ``` -#### `Store.prototype.load(String data, String mimeType, NamedNode|String? baseIRI, NamedNode|BlankNode|DefaultGraph? toNamedGraph)` +#### `Store.prototype.load(String data, String format, NamedNode|String? baseIRI, NamedNode|BlankNode|DefaultGraph? toNamedGraph)` Loads serialized RDF triples or quad into the store. The method arguments are: 1. `data`: the serialized RDF triples or quads. -2. `mimeType`: the MIME type of the serialization. See below for the supported mime types. +2. `format`: the format of the serialization. See below for the supported formats. 3. `baseIRI`: the base IRI to use to resolve the relative IRIs in the serialization. 4. `toNamedGraph`: for triple serialization formats, the name of the named graph the triple should be loaded to. The available formats are: -* [Turtle](https://www.w3.org/TR/turtle/): `text/turtle` -* [TriG](https://www.w3.org/TR/trig/): `application/trig` -* [N-Triples](https://www.w3.org/TR/n-triples/): `application/n-triples` -* [N-Quads](https://www.w3.org/TR/n-quads/): `application/n-quads` -* [RDF/XML](https://www.w3.org/TR/rdf-syntax-grammar/): `application/rdf+xml` +* [Turtle](https://www.w3.org/TR/turtle/): `text/turtle` or `ttl` +* [TriG](https://www.w3.org/TR/trig/): `application/trig` or `trig` +* [N-Triples](https://www.w3.org/TR/n-triples/): `application/n-triples` or `nt` +* [N-Quads](https://www.w3.org/TR/n-quads/): `application/n-quads` or `nq` +* [N3](https://w3c.github.io/N3/spec/): `text/n3` or `n3` +* [RDF/XML](https://www.w3.org/TR/rdf-syntax-grammar/): `application/rdf+xml` or `rdf` Example of loading a Turtle file into the named graph `` with the base IRI `http://example.com`: ```js store.load(" <> .", "text/turtle", "http://example.com", oxigraph.namedNode("http://example.com/graph")); ``` -#### `Store.prototype.dump(String mimeType, NamedNode|BlankNode|DefaultGraph? fromNamedGraph)` +#### `Store.prototype.dump(String format, NamedNode|BlankNode|DefaultGraph? fromNamedGraph)` Returns serialized RDF triples or quad from the store. The method arguments are: -1. `mimeType`: the MIME type of the serialization. See below for the supported mime types. +1. `format`: the format type of the serialization. See below for the supported types. 2. `fromNamedGraph`: for triple serialization formats, the name of the named graph the triple should be loaded from. The available formats are: -* [Turtle](https://www.w3.org/TR/turtle/): `text/turtle` -* [TriG](https://www.w3.org/TR/trig/): `application/trig` -* [N-Triples](https://www.w3.org/TR/n-triples/): `application/n-triples` -* [N-Quads](https://www.w3.org/TR/n-quads/): `application/n-quads` -* [RDF/XML](https://www.w3.org/TR/rdf-syntax-grammar/): `application/rdf+xml` +* [Turtle](https://www.w3.org/TR/turtle/): `text/turtle` or `ttl` +* [TriG](https://www.w3.org/TR/trig/): `application/trig` or `trig` +* [N-Triples](https://www.w3.org/TR/n-triples/): `application/n-triples` or `nt` +* [N-Quads](https://www.w3.org/TR/n-quads/): `application/n-quads` or `nq` +* [N3](https://w3c.github.io/N3/spec/): `text/n3` or `n3` +* [RDF/XML](https://www.w3.org/TR/rdf-syntax-grammar/): `application/rdf+xml` or `rdf` Example of building a Turtle file from the named graph ``: ```js diff --git a/js/src/store.rs b/js/src/store.rs index bffac273..008f24f0 100644 --- a/js/src/store.rs +++ b/js/src/store.rs @@ -144,13 +144,11 @@ impl JsStore { pub fn load( &self, data: &str, - mime_type: &str, + format: &str, base_iri: &JsValue, to_graph_name: &JsValue, ) -> Result<(), JsValue> { - let Some(format) = RdfFormat::from_media_type(mime_type) else { - return Err(format_err!("Not supported MIME type: {mime_type}")); - }; + let format = rdf_format(format)?; let base_iri = if base_iri.is_null() || base_iri.is_undefined() { None } else if base_iri.is_string() { @@ -177,10 +175,8 @@ impl JsStore { .map_err(to_err) } - pub fn dump(&self, mime_type: &str, from_graph_name: &JsValue) -> Result { - let Some(format) = RdfFormat::from_media_type(mime_type) else { - return Err(format_err!("Not supported MIME type: {mime_type}")); - }; + pub fn dump(&self, format: &str, from_graph_name: &JsValue) -> Result { + let format = rdf_format(format)?; let buffer = if let Some(from_graph_name) = FROM_JS.with(|c| c.to_optional_term(from_graph_name))? { self.store @@ -192,3 +188,13 @@ impl JsStore { String::from_utf8(buffer).map_err(to_err) } } + +fn rdf_format(format: &str) -> Result { + if format.contains('/') { + RdfFormat::from_media_type(format) + .ok_or_else(|| format_err!("Not supported RDF format media type: {format}")) + } else { + RdfFormat::from_extension(format) + .ok_or_else(|| format_err!("Not supported RDF format extension: {format}")) + } +} diff --git a/python/src/io.rs b/python/src/io.rs index 9585dcdc..f365af50 100644 --- a/python/src/io.rs +++ b/python/src/io.rs @@ -22,20 +22,21 @@ pub fn add_to_module(module: &PyModule) -> PyResult<()> { /// /// It currently supports the following formats: /// -/// * `N-Triples `_ (``application/n-triples``) -/// * `N-Quads `_ (``application/n-quads``) -/// * `Turtle `_ (``text/turtle``) -/// * `TriG `_ (``application/trig``) -/// * `RDF/XML `_ (``application/rdf+xml``) +/// * `N-Triples `_ (``application/n-triples`` or ``nt``) +/// * `N-Quads `_ (``application/n-quads`` or ``nq``) +/// * `Turtle `_ (``text/turtle`` or ``ttl``) +/// * `TriG `_ (``application/trig`` or ``trig``) +/// * `N3 `_ (``text/n3`` or ``n3``) +/// * `RDF/XML `_ (``application/rdf+xml`` or ``rdf``) /// -/// It supports also some MIME type aliases. +/// It supports also some media type and extension aliases. /// For example, ``application/turtle`` could also be used for `Turtle `_ -/// and ``application/xml`` for `RDF/XML `_. +/// and ``application/xml`` or ``xml`` for `RDF/XML `_. /// /// :param input: The binary I/O object or file path to read from. For example, it could be a file path as a string or a file reader opened in binary mode with ``open('my_file.ttl', 'rb')``. /// :type input: io(bytes) or io(str) or str or pathlib.Path -/// :param mime_type: the MIME type of the RDF serialization. -/// :type mime_type: str +/// :param format: the format of the RDF serialization using a media type like ``text/turtle`` or an extension like `ttl`. +/// :type format: str /// :param base_iri: the base IRI used to resolve the relative IRIs in the file or :py:const:`None` if relative IRI resolution should not be done. /// :type base_iri: str or None, optional /// :param without_named_graphs: Sets that the parser must fail if parsing a named graph. @@ -44,27 +45,23 @@ pub fn add_to_module(module: &PyModule) -> PyResult<()> { /// :type rename_blank_nodes: bool, optional /// :return: an iterator of RDF triples or quads depending on the format. /// :rtype: iterator(Quad) -/// :raises ValueError: if the MIME type is not supported. +/// :raises ValueError: if the format is not supported. /// :raises SyntaxError: if the provided data is invalid. /// /// >>> input = io.BytesIO(b'

"1" .') /// >>> list(parse(input, "text/turtle", base_iri="http://example.com/")) /// [ predicate= object=> graph_name=>] #[pyfunction] -#[pyo3(signature = (input, mime_type, *, base_iri = None, without_named_graphs = false, rename_blank_nodes = false))] +#[pyo3(signature = (input, format, *, base_iri = None, without_named_graphs = false, rename_blank_nodes = false))] pub fn parse( input: PyObject, - mime_type: &str, + format: &str, base_iri: Option<&str>, without_named_graphs: bool, rename_blank_nodes: bool, py: Python<'_>, ) -> PyResult { - let Some(format) = RdfFormat::from_media_type(mime_type) else { - return Err(PyValueError::new_err(format!( - "Not supported MIME type: {mime_type}" - ))); - }; + let format = rdf_format(format)?; let input = if let Ok(path) = input.extract::(py) { PyReadable::from_file(&path, py).map_err(map_io_err)? } else { @@ -92,24 +89,25 @@ pub fn parse( /// /// It currently supports the following formats: /// -/// * `N-Triples `_ (``application/n-triples``) -/// * `N-Quads `_ (``application/n-quads``) -/// * `Turtle `_ (``text/turtle``) -/// * `TriG `_ (``application/trig``) -/// * `RDF/XML `_ (``application/rdf+xml``) +/// * `N-Triples `_ (``application/n-triples`` or ``nt``) +/// * `N-Quads `_ (``application/n-quads`` or ``nq``) +/// * `Turtle `_ (``text/turtle`` or ``ttl``) +/// * `TriG `_ (``application/trig`` or ``trig``) +/// * `N3 `_ (``text/n3`` or ``n3``) +/// * `RDF/XML `_ (``application/rdf+xml`` or ``rdf``) /// -/// It supports also some MIME type aliases. +/// It supports also some media type and extension aliases. /// For example, ``application/turtle`` could also be used for `Turtle `_ -/// and ``application/xml`` for `RDF/XML `_. +/// and ``application/xml`` or ``xml`` for `RDF/XML `_. /// /// :param input: the RDF triples and quads to serialize. /// :type input: iterable(Triple) or iterable(Quad) /// :param output: The binary I/O object or file path to write to. For example, it could be a file path as a string or a file writer opened in binary mode with ``open('my_file.ttl', 'wb')``. /// :type output: io(bytes) or str or pathlib.Path -/// :param mime_type: the MIME type of the RDF serialization. -/// :type mime_type: str +/// :param format: the format of the RDF serialization using a media type like ``text/turtle`` or an extension like `ttl`. +/// :type format: str /// :rtype: None -/// :raises ValueError: if the MIME type is not supported. +/// :raises ValueError: if the format is not supported. /// :raises TypeError: if a triple is given during a quad format serialization or reverse. /// /// >>> output = io.BytesIO() @@ -117,12 +115,8 @@ pub fn parse( /// >>> output.getvalue() /// b' "1" .\n' #[pyfunction] -pub fn serialize(input: &PyAny, output: PyObject, mime_type: &str, py: Python<'_>) -> PyResult<()> { - let Some(format) = RdfFormat::from_media_type(mime_type) else { - return Err(PyValueError::new_err(format!( - "Not supported MIME type: {mime_type}" - ))); - }; +pub fn serialize(input: &PyAny, output: PyObject, format: &str, py: Python<'_>) -> PyResult<()> { + let format = rdf_format(format)?; let output = if let Ok(path) = output.extract::(py) { PyWritable::from_file(&path, py).map_err(map_io_err)? } else { @@ -293,6 +287,18 @@ impl Write for PyIo { } } +pub fn rdf_format(format: &str) -> PyResult { + if format.contains('/') { + RdfFormat::from_media_type(format).ok_or_else(|| { + PyValueError::new_err(format!("Not supported RDF format media type: {format}")) + }) + } else { + RdfFormat::from_extension(format).ok_or_else(|| { + PyValueError::new_err(format!("Not supported RDF format extension: {format}")) + }) + } +} + fn to_io_err(error: PyErr) -> io::Error { io::Error::new(io::ErrorKind::Other, error) } diff --git a/python/src/store.rs b/python/src/store.rs index da20686a..d71d4936 100644 --- a/python/src/store.rs +++ b/python/src/store.rs @@ -1,9 +1,10 @@ #![allow(clippy::needless_option_as_deref)] -use crate::io::{allow_threads_unsafe, map_io_err, map_parse_error, PyReadable, PyWritable}; +use crate::io::{ + allow_threads_unsafe, map_io_err, map_parse_error, rdf_format, PyReadable, PyWritable, +}; use crate::model::*; use crate::sparql::*; -use oxigraph::io::RdfFormat; use oxigraph::model::{GraphName, GraphNameRef}; use oxigraph::sparql::Update; use oxigraph::store::{self, LoaderError, SerializerError, StorageError, Store}; @@ -348,26 +349,27 @@ impl PyStore { /// /// It currently supports the following formats: /// - /// * `N-Triples `_ (``application/n-triples``) - /// * `N-Quads `_ (``application/n-quads``) - /// * `Turtle `_ (``text/turtle``) - /// * `TriG `_ (``application/trig``) - /// * `RDF/XML `_ (``application/rdf+xml``) + /// * `N-Triples `_ (``application/n-triples`` or ``nt``) + /// * `N-Quads `_ (``application/n-quads`` or ``nq``) + /// * `Turtle `_ (``text/turtle`` or ``ttl``) + /// * `TriG `_ (``application/trig`` or ``trig``) + /// * `N3 `_ (``text/n3`` or ``n3``) + /// * `RDF/XML `_ (``application/rdf+xml`` or ``rdf``) /// - /// It supports also some MIME type aliases. + /// It supports also some media type and extension aliases. /// For example, ``application/turtle`` could also be used for `Turtle `_ - /// and ``application/xml`` for `RDF/XML `_. + /// and ``application/xml`` or ``xml`` for `RDF/XML `_. /// /// :param input: The binary I/O object or file path to read from. For example, it could be a file path as a string or a file reader opened in binary mode with ``open('my_file.ttl', 'rb')``. /// :type input: io(bytes) or io(str) or str or pathlib.Path - /// :param mime_type: the MIME type of the RDF serialization. - /// :type mime_type: str + /// :param format: the format of the RDF serialization using a media type like ``text/turtle`` or an extension like `ttl`. + /// :type format: str /// :param base_iri: the base IRI used to resolve the relative IRIs in the file or :py:const:`None` if relative IRI resolution should not be done. /// :type base_iri: str or None, optional /// :param to_graph: if it is a file composed of triples, the graph in which the triples should be stored. By default, the default graph is used. /// :type to_graph: NamedNode or BlankNode or DefaultGraph or None, optional /// :rtype: None - /// :raises ValueError: if the MIME type is not supported. + /// :raises ValueError: if the format is not supported. /// :raises SyntaxError: if the provided data is invalid. /// :raises OSError: if an error happens during a quad insertion. /// @@ -375,20 +377,16 @@ impl PyStore { /// >>> store.load(io.BytesIO(b'

"1" .'), "text/turtle", base_iri="http://example.com/", to_graph=NamedNode("http://example.com/g")) /// >>> list(store) /// [ predicate= object=> graph_name=>] - #[pyo3(signature = (input, mime_type, *, base_iri = None, to_graph = None))] + #[pyo3(signature = (input, format, *, base_iri = None, to_graph = None))] fn load( &self, input: PyObject, - mime_type: &str, + format: &str, base_iri: Option<&str>, to_graph: Option<&PyAny>, py: Python<'_>, ) -> PyResult<()> { - let Some(format) = RdfFormat::from_media_type(mime_type) else { - return Err(PyValueError::new_err(format!( - "Not supported MIME type: {mime_type}" - ))); - }; + let format = rdf_format(format)?; let to_graph_name = if let Some(graph_name) = to_graph { Some(GraphName::from(&PyGraphNameRef::try_from(graph_name)?)) } else { @@ -419,26 +417,27 @@ impl PyStore { /// /// It currently supports the following formats: /// - /// * `N-Triples `_ (``application/n-triples``) - /// * `N-Quads `_ (``application/n-quads``) - /// * `Turtle `_ (``text/turtle``) - /// * `TriG `_ (``application/trig``) - /// * `RDF/XML `_ (``application/rdf+xml``) + /// * `N-Triples `_ (``application/n-triples`` or ``nt``) + /// * `N-Quads `_ (``application/n-quads`` or ``nq``) + /// * `Turtle `_ (``text/turtle`` or ``ttl``) + /// * `TriG `_ (``application/trig`` or ``trig``) + /// * `N3 `_ (``text/n3`` or ``n3``) + /// * `RDF/XML `_ (``application/rdf+xml`` or ``rdf``) /// - /// It supports also some MIME type aliases. + /// It supports also some media type and extension aliases. /// For example, ``application/turtle`` could also be used for `Turtle `_ - /// and ``application/xml`` for `RDF/XML `_. + /// and ``application/xml`` or ``xml`` for `RDF/XML `_. /// /// :param input: The binary I/O object or file path to read from. For example, it could be a file path as a string or a file reader opened in binary mode with ``open('my_file.ttl', 'rb')``. /// :type input: io(bytes) or io(str) or str or pathlib.Path - /// :param mime_type: the MIME type of the RDF serialization. - /// :type mime_type: str + /// :param format: the format of the RDF serialization using a media type like ``text/turtle`` or an extension like `ttl`. + /// :type format: str /// :param base_iri: the base IRI used to resolve the relative IRIs in the file or :py:const:`None` if relative IRI resolution should not be done. /// :type base_iri: str or None, optional /// :param to_graph: if it is a file composed of triples, the graph in which the triples should be stored. By default, the default graph is used. /// :type to_graph: NamedNode or BlankNode or DefaultGraph or None, optional /// :rtype: None - /// :raises ValueError: if the MIME type is not supported. + /// :raises ValueError: if the format is not supported. /// :raises SyntaxError: if the provided data is invalid. /// :raises OSError: if an error happens during a quad insertion. /// @@ -446,20 +445,16 @@ impl PyStore { /// >>> store.bulk_load(io.BytesIO(b'

"1" .'), "text/turtle", base_iri="http://example.com/", to_graph=NamedNode("http://example.com/g")) /// >>> list(store) /// [ predicate= object=> graph_name=>] - #[pyo3(signature = (input, mime_type, *, base_iri = None, to_graph = None))] + #[pyo3(signature = (input, format, *, base_iri = None, to_graph = None))] fn bulk_load( &self, input: PyObject, - mime_type: &str, + format: &str, base_iri: Option<&str>, to_graph: Option<&PyAny>, py: Python<'_>, ) -> PyResult<()> { - let Some(format) = RdfFormat::from_media_type(mime_type) else { - return Err(PyValueError::new_err(format!( - "Not supported MIME type: {mime_type}" - ))); - }; + let format = rdf_format(format)?; let to_graph_name = if let Some(graph_name) = to_graph { Some(GraphName::from(&PyGraphNameRef::try_from(graph_name)?)) } else { @@ -488,24 +483,25 @@ impl PyStore { /// /// It currently supports the following formats: /// - /// * `N-Triples `_ (``application/n-triples``) - /// * `N-Quads `_ (``application/n-quads``) - /// * `Turtle `_ (``text/turtle``) - /// * `TriG `_ (``application/trig``) - /// * `RDF/XML `_ (``application/rdf+xml``) + /// * `N-Triples `_ (``application/n-triples`` or ``nt``) + /// * `N-Quads `_ (``application/n-quads`` or ``nq``) + /// * `Turtle `_ (``text/turtle`` or ``ttl``) + /// * `TriG `_ (``application/trig`` or ``trig``) + /// * `N3 `_ (``text/n3`` or ``n3``) + /// * `RDF/XML `_ (``application/rdf+xml`` or ``rdf``) /// - /// It supports also some MIME type aliases. + /// It supports also some media type and extension aliases. /// For example, ``application/turtle`` could also be used for `Turtle `_ - /// and ``application/xml`` for `RDF/XML `_. + /// and ``application/xml`` or ``xml`` for `RDF/XML `_. /// /// :param output: The binary I/O object or file path to write to. For example, it could be a file path as a string or a file writer opened in binary mode with ``open('my_file.ttl', 'wb')``. /// :type output: io(bytes) or str or pathlib.Path - /// :param mime_type: the MIME type of the RDF serialization. - /// :type mime_type: str + /// :param format: the format of the RDF serialization using a media type like ``text/turtle`` or an extension like `ttl`. + /// :type format: str /// :param from_graph: the store graph from which dump the triples. Required if the serialization format does not support named graphs. If it does supports named graphs the full dataset is written. /// :type from_graph: NamedNode or BlankNode or DefaultGraph or None, optional /// :rtype: None - /// :raises ValueError: if the MIME type is not supported or the `from_graph` parameter is not given with a syntax not supporting named graphs. + /// :raises ValueError: if the format is not supported or the `from_graph` parameter is not given with a syntax not supporting named graphs. /// :raises OSError: if an error happens during a quad lookup /// /// >>> store = Store() @@ -514,19 +510,15 @@ impl PyStore { /// >>> store.dump(output, "text/turtle", from_graph=NamedNode("http://example.com/g")) /// >>> output.getvalue() /// b' "1" .\n' - #[pyo3(signature = (output, mime_type, *, from_graph = None))] + #[pyo3(signature = (output, format, *, from_graph = None))] fn dump( &self, output: PyObject, - mime_type: &str, + format: &str, from_graph: Option<&PyAny>, py: Python<'_>, ) -> PyResult<()> { - let Some(format) = RdfFormat::from_media_type(mime_type) else { - return Err(PyValueError::new_err(format!( - "Not supported MIME type: {mime_type}" - ))); - }; + let format = rdf_format(format)?; let from_graph_name = if let Some(graph_name) = from_graph { Some(GraphName::from(&PyGraphNameRef::try_from(graph_name)?)) } else { diff --git a/python/tests/test_io.py b/python/tests/test_io.py index d3f535c4..e9069e08 100644 --- a/python/tests/test_io.py +++ b/python/tests/test_io.py @@ -69,7 +69,7 @@ class TestParse(unittest.TestCase): def test_parse_io_error(self) -> None: with self.assertRaises(UnsupportedOperation) as _, TemporaryFile("wb") as fp: - list(parse(fp, mime_type="application/n-triples")) + list(parse(fp, "nt")) def test_parse_quad(self) -> None: self.assertEqual( diff --git a/python/tests/test_store.py b/python/tests/test_store.py index ee02e830..a3ed8fe0 100644 --- a/python/tests/test_store.py +++ b/python/tests/test_store.py @@ -225,7 +225,7 @@ class TestStore(unittest.TestCase): store = Store() store.load( BytesIO(b" ."), - mime_type="application/n-triples", + "application/n-triples", ) self.assertEqual(set(store), {Quad(foo, bar, baz, DefaultGraph())}) @@ -233,7 +233,7 @@ class TestStore(unittest.TestCase): store = Store() store.load( BytesIO(b" ."), - mime_type="application/n-triples", + "application/n-triples", to_graph=graph, ) self.assertEqual(set(store), {Quad(foo, bar, baz, graph)}) @@ -242,7 +242,7 @@ class TestStore(unittest.TestCase): store = Store() store.load( BytesIO(b" <> ."), - mime_type="text/turtle", + "text/turtle", base_iri="http://baz", ) self.assertEqual(set(store), {Quad(foo, bar, baz, DefaultGraph())}) @@ -251,7 +251,7 @@ class TestStore(unittest.TestCase): store = Store() store.load( BytesIO(b" ."), - mime_type="application/n-quads", + "nq", ) self.assertEqual(set(store), {Quad(foo, bar, baz, graph)}) @@ -259,7 +259,7 @@ class TestStore(unittest.TestCase): store = Store() store.load( BytesIO(b" { <> . }"), - mime_type="application/trig", + "application/trig", base_iri="http://baz", ) self.assertEqual(set(store), {Quad(foo, bar, baz, graph)}) @@ -269,13 +269,13 @@ class TestStore(unittest.TestCase): file_name = Path(fp.name) fp.write(b" .") store = Store() - store.load(file_name, mime_type="application/n-quads") + store.load(file_name, "nq") file_name.unlink() self.assertEqual(set(store), {Quad(foo, bar, baz, graph)}) def test_load_with_io_error(self) -> None: with self.assertRaises(UnsupportedOperation) as _, TemporaryFile("wb") as fp: - Store().load(fp, mime_type="application/n-triples") + Store().load(fp, "application/n-triples") def test_dump_ntriples(self) -> None: store = Store() @@ -291,7 +291,7 @@ class TestStore(unittest.TestCase): store = Store() store.add(Quad(foo, bar, baz, graph)) output = BytesIO() - store.dump(output, "application/n-quads") + store.dump(output, "nq") self.assertEqual( output.getvalue(), b" .\n", @@ -314,7 +314,7 @@ class TestStore(unittest.TestCase): file_name = Path(fp.name) store = Store() store.add(Quad(foo, bar, baz, graph)) - store.dump(file_name, "application/n-quads") + store.dump(file_name, "nq") self.assertEqual( file_name.read_text(), " .\n", @@ -324,7 +324,7 @@ class TestStore(unittest.TestCase): store = Store() store.add(Quad(foo, bar, bar)) with self.assertRaises(OSError) as _, TemporaryFile("rb") as fp: - store.dump(fp, mime_type="application/trig") + store.dump(fp, "application/trig") def test_write_in_read(self) -> None: store = Store() From 6611b491b1d00ecb160963789bd14b73289dbf70 Mon Sep 17 00:00:00 2001 From: Tpt Date: Sun, 27 Aug 2023 17:48:38 +0200 Subject: [PATCH 066/217] Sets Rust minimum version to 1.70 --- .github/workflows/tests.yml | 8 +- Cargo.lock | 166 +++++++++++++++-------------- Cargo.toml | 2 +- lib/Cargo.toml | 2 +- lib/sparopt/src/optimizer.rs | 6 +- lib/src/storage/numeric_encoder.rs | 1 - lib/src/storage/small_string.rs | 2 +- testsuite/Cargo.toml | 2 +- 8 files changed, 98 insertions(+), 91 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5dec1690..b16ea777 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -94,7 +94,7 @@ jobs: - uses: actions/checkout@v3 with: submodules: true - - run: rustup update && rustup override set 1.65.0 && rustup component add clippy + - run: rustup update && rustup override set 1.70.0 && rustup component add clippy - uses: Swatinem/rust-cache@v2 - run: cargo clippy -- -D warnings -D clippy::all working-directory: ./lib/oxsdatatypes @@ -129,7 +129,7 @@ jobs: - uses: actions/checkout@v3 with: submodules: true - - run: rustup update && rustup override set 1.65.0 && rustup target add wasm32-unknown-unknown && rustup component add clippy + - run: rustup update && rustup override set 1.70.0 && rustup target add wasm32-unknown-unknown && rustup component add clippy - uses: Swatinem/rust-cache@v2 - run: cargo clippy --lib --tests --target wasm32-unknown-unknown -- -D warnings -D clippy::all working-directory: ./js @@ -140,7 +140,7 @@ jobs: - uses: actions/checkout@v3 with: submodules: true - - run: rustup update && rustup override set 1.65.0 && rustup target add wasm32-wasi && rustup component add clippy + - run: rustup update && rustup override set 1.70.0 && rustup target add wasm32-wasi && rustup component add clippy - uses: Swatinem/rust-cache@v2 - run: cargo clippy --lib --tests --target wasm32-wasi -- -D warnings -D clippy::all working-directory: ./lib @@ -236,7 +236,7 @@ jobs: - uses: actions/checkout@v3 with: submodules: true - - run: rustup update && rustup override set 1.65.0 + - run: rustup update && rustup override set 1.70.0 - uses: Swatinem/rust-cache@v2 - run: cargo doc --all-features working-directory: ./lib diff --git a/Cargo.lock b/Cargo.lock index aa576fdc..22aa96e4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,9 +4,9 @@ version = 3 [[package]] name = "addr2line" -version = "0.20.0" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4fa78e18c64fce05e902adecd7a5eed15a5e0a3439f7b0e169f0252214865e3" +checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" dependencies = [ "gimli", ] @@ -34,24 +34,23 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "anstream" -version = "0.3.2" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163" +checksum = "b1f58811cfac344940f1a400b6e6231ce35171f614f26439e80f8c1465c5cc0c" dependencies = [ "anstyle", "anstyle-parse", "anstyle-query", "anstyle-wincon", "colorchoice", - "is-terminal", "utf8parse", ] [[package]] name = "anstyle" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a30da5c5f2d5e72842e00bcb57657162cdabef0931f40e2deb9b4140440cecd" +checksum = "15c4c2c83f81532e5845a733998b6971faca23490340a418e9b72a3ec9de12ea" [[package]] name = "anstyle-parse" @@ -73,9 +72,9 @@ dependencies = [ [[package]] name = "anstyle-wincon" -version = "1.0.2" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c677ab05e09154296dd37acecd46420c17b9713e8366facafa8fc0885167cf4c" +checksum = "58f54d10c6dfa51283a066ceab3ec1ab78d13fae00aa49243a45e4571fb79dfd" dependencies = [ "anstyle", "windows-sys", @@ -134,9 +133,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "backtrace" -version = "0.3.68" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4319208da049c43661739c5fade2ba182f09d1dc2299b32298d3a31692b17e12" +checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" dependencies = [ "addr2line", "cc", @@ -149,9 +148,9 @@ dependencies = [ [[package]] name = "base64" -version = "0.21.2" +version = "0.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d" +checksum = "414dcefbc63d77c526a76b3afcf6fbb9b5e2791c19c3aa2297733208750c6e53" [[package]] name = "bindgen" @@ -228,9 +227,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.0.82" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "305fe645edc1442a0fa8b6726ba61d422798d37a52e12eaecf4b022ebbb88f01" +checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" dependencies = [ "jobserver", "libc", @@ -291,9 +290,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.3.22" +version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b417ae4361bca3f5de378294fc7472d3c4ed86a5ef9f49e93ae722f432aae8d2" +checksum = "1d5f1946157a96594eb2d2c10eb7ad9a2b27518cb3000209dec700c35df9197d" dependencies = [ "clap_builder", "clap_derive", @@ -302,9 +301,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.3.22" +version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c90dc0f0e42c64bff177ca9d7be6fcc9ddb0f26a6e062174a61c84dd6c644d4" +checksum = "78116e32a042dd73c2901f0dc30790d20ff3447f3e3472fad359e8c3d282bcd6" dependencies = [ "anstream", "anstyle", @@ -314,9 +313,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.3.12" +version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54a9bb5758fc5dfe728d1019941681eccaf0cf8a4189b692a0ee2f2ecf90a050" +checksum = "c9fd1a5729c4548118d7d70ff234a44868d00489a4b6597b0b020918a0e91a1a" dependencies = [ "heck", "proc-macro2", @@ -326,9 +325,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b" +checksum = "cd7cc57abe963c6d3b9d8be5b06ba7c8957a930305ca90304f24ef040aa6f961" [[package]] name = "colorchoice" @@ -469,6 +468,12 @@ dependencies = [ "typenum", ] +[[package]] +name = "deranged" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2696e8a945f658fd14dc3b87242e6b80cd0f36ff04ea560fa39082368847946" + [[package]] name = "derive_arbitrary" version = "1.3.1" @@ -615,9 +620,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.27.3" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e" +checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" [[package]] name = "glob" @@ -898,9 +903,9 @@ dependencies = [ [[package]] name = "object" -version = "0.31.1" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bda667d9f2b5051b8833f59f3bf748b28ef54f850f4fcb389a252aa383866d1" +checksum = "77ac5bbd07aea88c60a577a1ce218075ffd59208b2d7ca97adf9bfc5aeb21ebe" dependencies = [ "memchr", ] @@ -925,9 +930,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "oxhttp" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aba22f4e7dc5088def616521ad0357b33fa9bc952848e6a449e0b6843a34e628" +checksum = "64f8f4b616372a7e657a05100d1389325fc2f7d3fee5c9de05166d979cb2b729" dependencies = [ "httparse", "lazy_static", @@ -1149,9 +1154,9 @@ checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" [[package]] name = "pin-project-lite" -version = "0.2.12" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12cc1b0bf1727a77a54b6654e7b5f1af8604923edc8b81885f8ec92f9e3f0a05" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" [[package]] name = "pkg-config" @@ -1393,9 +1398,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.9.3" +version = "1.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81bc1d4caf89fac26a70747fe603c130093b53c773888797a6329091246d651a" +checksum = "12de2eff854e5fa4b1295edd650e227e9d8fb0c9e90b12e7f36d6a6811791a29" dependencies = [ "aho-corasick", "memchr", @@ -1405,9 +1410,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fed1ceff11a1dddaee50c9dc8e4938bd106e9d89ae372f192311e7da498e3b69" +checksum = "49530408a136e16e5b486e883fbb6ba058e8e4e8ae6621a77b048b314336e629" dependencies = [ "aho-corasick", "memchr", @@ -1416,9 +1421,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2" +checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" [[package]] name = "ring" @@ -1466,9 +1471,9 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "rustix" -version = "0.38.8" +version = "0.38.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19ed4fa021d81c8392ce04db050a3da9a60299050b7ae1cf482d862b54a7218f" +checksum = "9bfe0f2582b4931a45d1fa608f8a8722e8b3c7ac54dd6d5f3b3212791fedef49" dependencies = [ "bitflags 2.4.0", "errno", @@ -1479,14 +1484,14 @@ dependencies = [ [[package]] name = "rustls" -version = "0.20.8" +version = "0.21.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fff78fc74d175294f4e83b28343315ffcfb114b156f0185e9741cb5570f50e2f" +checksum = "1d1feddffcfcc0b33f5c6ce9a29e341e4cd59c3f78e7ee45f4a40c038b1d6cbb" dependencies = [ "log", "ring", + "rustls-webpki", "sct", - "webpki", ] [[package]] @@ -1510,6 +1515,16 @@ dependencies = [ "base64", ] +[[package]] +name = "rustls-webpki" +version = "0.101.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d93931baf2d282fff8d3a532bbfd7653f734643161b87e3e01e59a04439bf0d" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "ryu" version = "1.0.15" @@ -1575,18 +1590,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.183" +version = "1.0.188" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32ac8da02677876d532745a130fc9d8e6edfa81a269b107c5b00829b91d8eb3c" +checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.183" +version = "1.0.188" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aafe972d60b0b9bee71a91b92fee2d4fb3c9d7e8f6b179aa99f27203d99a4816" +checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" dependencies = [ "proc-macro2", "quote", @@ -1634,9 +1649,9 @@ checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3" [[package]] name = "siphasher" -version = "0.3.10" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de" +checksum = "54ac45299ccbd390721be55b412d41931911f654fa99e2cb8bfb57184b2061fe" [[package]] name = "smallvec" @@ -1722,9 +1737,9 @@ checksum = "9d0e916b1148c8e263850e1ebcbd046f333e0683c724876bb0da63ea4373dc8a" [[package]] name = "tempfile" -version = "3.7.1" +version = "3.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc02fddf48964c42031a0b3fe0428320ecf3a73c401040fc0096f97794310651" +checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" dependencies = [ "cfg-if", "fastrand", @@ -1771,10 +1786,11 @@ dependencies = [ [[package]] name = "time" -version = "0.3.23" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59e399c068f43a5d116fedaf73b203fa4f9c519f17e2b34f63221d3792f81446" +checksum = "17f6bb557fd245c28e6411aa56b6403c689ad95061f50e4be16c274e70a17e48" dependencies = [ + "deranged", "itoa", "serde", "time-core", @@ -1789,9 +1805,9 @@ checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" [[package]] name = "time-macros" -version = "0.2.10" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96ba15a897f3c86766b757e5ac7221554c6750054d74d5b28844fce5fb36a6c4" +checksum = "1a942f44339478ef67935ab2bbaec2fb0322496cf3cbe84b261e06ac3814c572" dependencies = [ "time-core", ] @@ -2001,16 +2017,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "webpki" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "which" version = "4.4.0" @@ -2076,9 +2082,9 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.48.4" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d92ecb8ae0317859f509f17b19adc74b0763b0fa3b085dea8ed01085c8dac222" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" dependencies = [ "windows_aarch64_gnullvm", "windows_aarch64_msvc", @@ -2091,45 +2097,45 @@ dependencies = [ [[package]] name = "windows_aarch64_gnullvm" -version = "0.48.4" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d14b0ee96970be7108701212f097ce67ca772fd84cb0ffbc86d26a94e77ba929" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_msvc" -version = "0.48.4" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1332277d49f440c8fc6014941e320ee47ededfcce10cb272728470f56cc092c9" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_i686_gnu" -version = "0.48.4" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d992130ac399d56f02c20564e9975ac5ba08cb25cb832849bbc0d736a101abe5" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_msvc" -version = "0.48.4" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "962e96d0fa4b4773c63977977ea6564f463fb10e34a6e07360428b53ae7a3f71" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_x86_64_gnu" -version = "0.48.4" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30652a53018a48a9735fbc2986ff0446c37bc8bed0d3f98a0ed4d04cdb80027e" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnullvm" -version = "0.48.4" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5bb3f0331abfe1a95af56067f1e64b3791b55b5373b03869560b6025de809bf" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_msvc" -version = "0.48.4" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd1df36d9fd0bbe4849461de9b969f765170f4e0f90497d580a235d515722b10" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "zstd" diff --git a/Cargo.toml b/Cargo.toml index 466be35b..15d79ef4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,7 +24,7 @@ authors = ["Tpt "] license = "MIT OR Apache-2.0" homepage = "https://oxigraph.org/" edition = "2021" -rust-version = "1.65" +rust-version = "1.70" [profile.release] lto = true diff --git a/lib/Cargo.toml b/lib/Cargo.toml index d3eb9c04..dd5a7c4d 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -31,7 +31,7 @@ regex = "1" oxilangtag = "0.1" oxiri = "0.2" hex = "0.4" -siphasher = "0.3" +siphasher = "1" lazy_static = "1" json-event-parser = "0.1" oxrdf = { version = "0.2.0-alpha.1-dev", path = "oxrdf", features = ["rdf-star", "oxsdatatypes"] } diff --git a/lib/sparopt/src/optimizer.rs b/lib/sparopt/src/optimizer.rs index a141bd63..344a4536 100644 --- a/lib/sparopt/src/optimizer.rs +++ b/lib/sparopt/src/optimizer.rs @@ -508,7 +508,8 @@ impl Optimizer { while let Some(next_entry_id) = not_yet_reordered_ids .iter() .enumerate() - .filter_map(|(i, v)| v.then(|| i)) + .filter(|(_, v)| **v) + .map(|(i, _)| i) .min_by_key(|i| estimate_graph_pattern_size(&to_reorder[*i], input_types)) { not_yet_reordered_ids[next_entry_id] = false; // It's now done @@ -518,7 +519,8 @@ impl Optimizer { while let Some(next_id) = not_yet_reordered_ids .iter() .enumerate() - .filter_map(|(i, v)| v.then(|| i)) + .filter(|(_, v)| **v) + .map(|(i, _)| i) .filter(|i| { has_common_variables(&output_types, &to_reorder_types[*i], input_types) }) diff --git a/lib/src/storage/numeric_encoder.rs b/lib/src/storage/numeric_encoder.rs index 19632624..94e237f6 100644 --- a/lib/src/storage/numeric_encoder.rs +++ b/lib/src/storage/numeric_encoder.rs @@ -531,7 +531,6 @@ impl From> for EncodedTerm { } "http://www.w3.org/2001/XMLSchema#boolean" => parse_boolean_str(value), "http://www.w3.org/2001/XMLSchema#string" => { - let value = value; Some(if let Ok(value) = SmallString::try_from(value) { Self::SmallStringLiteral(value) } else { diff --git a/lib/src/storage/small_string.rs b/lib/src/storage/small_string.rs index 4bb1520b..a7134bd6 100644 --- a/lib/src/storage/small_string.rs +++ b/lib/src/storage/small_string.rs @@ -112,7 +112,7 @@ impl Eq for SmallString {} impl PartialOrd for SmallString { #[inline] fn partial_cmp(&self, other: &Self) -> Option { - self.as_str().partial_cmp(other.as_str()) + Some(self.cmp(other)) } } diff --git a/testsuite/Cargo.toml b/testsuite/Cargo.toml index a32411f3..a25a27dc 100644 --- a/testsuite/Cargo.toml +++ b/testsuite/Cargo.toml @@ -18,7 +18,7 @@ oxttl = { path= "../lib/oxttl" } sparopt = { path = "../lib/sparopt" } spargebra = { path = "../lib/spargebra" } text-diff = "0.4" -time = { version = "=0.3.23", features = ["formatting"] } +time = { version = "0.3", features = ["formatting"] } [dev-dependencies] criterion = "0.5" From 024bc7b8e876eeabf19400fac26e2ea6cb5e5e6e Mon Sep 17 00:00:00 2001 From: Tpt Date: Sun, 27 Aug 2023 17:49:10 +0200 Subject: [PATCH 067/217] Simplifies Gitter link --- README.md | 2 +- js/README.md | 2 +- lib/README.md | 2 +- lib/oxrdf/README.md | 2 +- lib/oxrdfio/README.md | 2 +- lib/oxrdfxml/README.md | 2 +- lib/oxsdatatypes/README.md | 2 +- lib/oxttl/README.md | 2 +- lib/sparesults/README.md | 2 +- lib/spargebra/README.md | 2 +- lib/sparopt/README.md | 2 +- lib/sparql-smith/README.md | 2 +- python/README.md | 2 +- server/README.md | 2 +- 14 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 5313fd0f..7be322f6 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ [![npm](https://img.shields.io/npm/v/oxigraph)](https://www.npmjs.com/package/oxigraph) [![actions status](https://github.com/oxigraph/oxigraph/workflows/build/badge.svg)](https://github.com/oxigraph/oxigraph/actions) [![dependency status](https://deps.rs/repo/github/oxigraph/oxigraph/status.svg)](https://deps.rs/repo/github/oxigraph/oxigraph) -[![Gitter](https://badges.gitter.im/oxigraph/community.svg)](https://gitter.im/oxigraph/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) +[![Gitter](https://badges.gitter.im/oxigraph/community.svg)](https://gitter.im/oxigraph/community) [![Twitter URL](https://img.shields.io/twitter/url?style=social&url=https%3A%2F%2Ftwitter.com%2Foxigraph)](https://twitter.com/oxigraph) Oxigraph is a graph database implementing the [SPARQL](https://www.w3.org/TR/sparql11-overview/) standard. diff --git a/js/README.md b/js/README.md index 0ee92e9b..dafc7d71 100644 --- a/js/README.md +++ b/js/README.md @@ -3,7 +3,7 @@ Oxigraph for JavaScript [![npm](https://img.shields.io/npm/v/oxigraph)](https://www.npmjs.com/package/oxigraph) [![actions status](https://github.com/oxigraph/oxigraph/workflows/build/badge.svg)](https://github.com/oxigraph/oxigraph/actions) -[![Gitter](https://badges.gitter.im/oxigraph/community.svg)](https://gitter.im/oxigraph/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) +[![Gitter](https://badges.gitter.im/oxigraph/community.svg)](https://gitter.im/oxigraph/community) This package provides a JavaScript API on top of [Oxigraph](https://crates.io/crates/oxigraph), compiled with WebAssembly. diff --git a/lib/README.md b/lib/README.md index 80d071ff..9b7f1de3 100644 --- a/lib/README.md +++ b/lib/README.md @@ -5,7 +5,7 @@ Oxigraph [![Released API docs](https://docs.rs/oxigraph/badge.svg)](https://docs.rs/oxigraph) [![Crates.io downloads](https://img.shields.io/crates/d/oxigraph)](https://crates.io/crates/oxigraph) [![actions status](https://github.com/oxigraph/oxigraph/workflows/build/badge.svg)](https://github.com/oxigraph/oxigraph/actions) -[![Gitter](https://badges.gitter.im/oxigraph/community.svg)](https://gitter.im/oxigraph/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) +[![Gitter](https://badges.gitter.im/oxigraph/community.svg)](https://gitter.im/oxigraph/community) Oxigraph is a graph database library implementing the [SPARQL](https://www.w3.org/TR/sparql11-overview/) standard. diff --git a/lib/oxrdf/README.md b/lib/oxrdf/README.md index 5993a746..8511d95f 100644 --- a/lib/oxrdf/README.md +++ b/lib/oxrdf/README.md @@ -5,7 +5,7 @@ OxRDF [![Released API docs](https://docs.rs/oxrdf/badge.svg)](https://docs.rs/oxrdf) [![Crates.io downloads](https://img.shields.io/crates/d/oxrdf)](https://crates.io/crates/oxrdf) [![actions status](https://github.com/oxigraph/oxigraph/workflows/build/badge.svg)](https://github.com/oxigraph/oxigraph/actions) -[![Gitter](https://badges.gitter.im/oxigraph/community.svg)](https://gitter.im/oxigraph/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) +[![Gitter](https://badges.gitter.im/oxigraph/community.svg)](https://gitter.im/oxigraph/community) OxRDF is a simple library providing datastructures encoding [RDF 1.1 concepts](https://www.w3.org/TR/rdf11-concepts/). diff --git a/lib/oxrdfio/README.md b/lib/oxrdfio/README.md index 1712b8ae..2bdd68d8 100644 --- a/lib/oxrdfio/README.md +++ b/lib/oxrdfio/README.md @@ -5,7 +5,7 @@ OxRDF I/O [![Released API docs](https://docs.rs/oxrdfio/badge.svg)](https://docs.rs/oxrdfio) [![Crates.io downloads](https://img.shields.io/crates/d/oxrdfio)](https://crates.io/crates/oxrdfio) [![actions status](https://github.com/oxigraph/oxigraph/workflows/build/badge.svg)](https://github.com/oxigraph/oxigraph/actions) -[![Gitter](https://badges.gitter.im/oxigraph/community.svg)](https://gitter.im/oxigraph/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) +[![Gitter](https://badges.gitter.im/oxigraph/community.svg)](https://gitter.im/oxigraph/community) OxRDF I/O is a set of parsers and serializers for RDF. diff --git a/lib/oxrdfxml/README.md b/lib/oxrdfxml/README.md index d2c0390d..54c1e117 100644 --- a/lib/oxrdfxml/README.md +++ b/lib/oxrdfxml/README.md @@ -5,7 +5,7 @@ OxRDF/XML [![Released API docs](https://docs.rs/oxrdfxml/badge.svg)](https://docs.rs/oxrdfxml) [![Crates.io downloads](https://img.shields.io/crates/d/oxrdfxml)](https://crates.io/crates/oxrdfxml) [![actions status](https://github.com/oxigraph/oxigraph/workflows/build/badge.svg)](https://github.com/oxigraph/oxigraph/actions) -[![Gitter](https://badges.gitter.im/oxigraph/community.svg)](https://gitter.im/oxigraph/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) +[![Gitter](https://badges.gitter.im/oxigraph/community.svg)](https://gitter.im/oxigraph/community) OxRdfXml is a parser and serializer for [RDF/XML](https://www.w3.org/TR/rdf-syntax-grammar/). diff --git a/lib/oxsdatatypes/README.md b/lib/oxsdatatypes/README.md index f198d5ae..c7ae8e55 100644 --- a/lib/oxsdatatypes/README.md +++ b/lib/oxsdatatypes/README.md @@ -5,7 +5,7 @@ oxsdatatypes [![Released API docs](https://docs.rs/oxsdatatypes/badge.svg)](https://docs.rs/oxsdatatypes) [![Crates.io downloads](https://img.shields.io/crates/d/oxsdatatypes)](https://crates.io/crates/oxsdatatypes) [![actions status](https://github.com/oxigraph/oxigraph/workflows/build/badge.svg)](https://github.com/oxigraph/oxigraph/actions) -[![Gitter](https://badges.gitter.im/oxigraph/community.svg)](https://gitter.im/oxigraph/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) +[![Gitter](https://badges.gitter.im/oxigraph/community.svg)](https://gitter.im/oxigraph/community) oxsdatatypes is an implementation of some [XML Schema Definition Language Datatypes](https://www.w3.org/TR/xmlschema11-2/). Its main aim is to ease the implementation of SPARQL and XPath. diff --git a/lib/oxttl/README.md b/lib/oxttl/README.md index ace0955d..47ec03ef 100644 --- a/lib/oxttl/README.md +++ b/lib/oxttl/README.md @@ -5,7 +5,7 @@ OxTTL [![Released API docs](https://docs.rs/oxttl/badge.svg)](https://docs.rs/oxttl) [![Crates.io downloads](https://img.shields.io/crates/d/oxttl)](https://crates.io/crates/oxttl) [![actions status](https://github.com/oxigraph/oxigraph/workflows/build/badge.svg)](https://github.com/oxigraph/oxigraph/actions) -[![Gitter](https://badges.gitter.im/oxigraph/community.svg)](https://gitter.im/oxigraph/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) +[![Gitter](https://badges.gitter.im/oxigraph/community.svg)](https://gitter.im/oxigraph/community) Oxttl is a set of parsers and serializers for [Turtle](https://www.w3.org/TR/turtle/), [TriG](https://www.w3.org/TR/trig/), [N-Triples](https://www.w3.org/TR/n-triples/), [N-Quads](https://www.w3.org/TR/n-quads/) and [N3](https://w3c.github.io/N3/spec/). diff --git a/lib/sparesults/README.md b/lib/sparesults/README.md index 73ccea36..7e719065 100644 --- a/lib/sparesults/README.md +++ b/lib/sparesults/README.md @@ -5,7 +5,7 @@ Sparesults [![Released API docs](https://docs.rs/sparesults/badge.svg)](https://docs.rs/sparesults) [![Crates.io downloads](https://img.shields.io/crates/d/sparesults)](https://crates.io/crates/sparesults) [![actions status](https://github.com/oxigraph/oxigraph/workflows/build/badge.svg)](https://github.com/oxigraph/oxigraph/actions) -[![Gitter](https://badges.gitter.im/oxigraph/community.svg)](https://gitter.im/oxigraph/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) +[![Gitter](https://badges.gitter.im/oxigraph/community.svg)](https://gitter.im/oxigraph/community) Sparesults is a set of parsers and serializers for [SPARQL](https://www.w3.org/TR/sparql11-overview/) query results formats. diff --git a/lib/spargebra/README.md b/lib/spargebra/README.md index 275fb5f0..313d8758 100644 --- a/lib/spargebra/README.md +++ b/lib/spargebra/README.md @@ -5,7 +5,7 @@ Spargebra [![Released API docs](https://docs.rs/spargebra/badge.svg)](https://docs.rs/spargebra) [![Crates.io downloads](https://img.shields.io/crates/d/spargebra)](https://crates.io/crates/spargebra) [![actions status](https://github.com/oxigraph/oxigraph/workflows/build/badge.svg)](https://github.com/oxigraph/oxigraph/actions) -[![Gitter](https://badges.gitter.im/oxigraph/community.svg)](https://gitter.im/oxigraph/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) +[![Gitter](https://badges.gitter.im/oxigraph/community.svg)](https://gitter.im/oxigraph/community) Spargebra is a [SPARQL](https://www.w3.org/TR/sparql11-overview/) parser. diff --git a/lib/sparopt/README.md b/lib/sparopt/README.md index 7a0ac67b..1a6e1c64 100644 --- a/lib/sparopt/README.md +++ b/lib/sparopt/README.md @@ -5,7 +5,7 @@ sparopt [![Released API docs](https://docs.rs/sparopt/badge.svg)](https://docs.rs/sparopt) [![Crates.io downloads](https://img.shields.io/crates/d/sparopt)](https://crates.io/crates/sparopt) [![actions status](https://github.com/oxigraph/oxigraph/workflows/build/badge.svg)](https://github.com/oxigraph/oxigraph/actions) -[![Gitter](https://badges.gitter.im/oxigraph/community.svg)](https://gitter.im/oxigraph/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) +[![Gitter](https://badges.gitter.im/oxigraph/community.svg)](https://gitter.im/oxigraph/community) sparopt is a work in progress [SPARQL Query](https://www.w3.org/TR/sparql11-query/) optimizer. diff --git a/lib/sparql-smith/README.md b/lib/sparql-smith/README.md index a02586b4..f637870c 100644 --- a/lib/sparql-smith/README.md +++ b/lib/sparql-smith/README.md @@ -5,7 +5,7 @@ SPARQL smith [![Released API docs](https://docs.rs/sparql-smith/badge.svg)](https://docs.rs/sparql-smith) [![Crates.io downloads](https://img.shields.io/crates/d/sparql-smith)](https://crates.io/crates/sparql-smith) [![actions status](https://github.com/oxigraph/oxigraph/workflows/build/badge.svg)](https://github.com/oxigraph/oxigraph/actions) -[![Gitter](https://badges.gitter.im/oxigraph/community.svg)](https://gitter.im/oxigraph/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) +[![Gitter](https://badges.gitter.im/oxigraph/community.svg)](https://gitter.im/oxigraph/community) sparql-smith is a test case generator for the [SPARQL](https://www.w3.org/TR/sparql11-overview/) language. diff --git a/python/README.md b/python/README.md index 42bb8d19..8a2e8247 100644 --- a/python/README.md +++ b/python/README.md @@ -5,7 +5,7 @@ ![PyPI - Implementation](https://img.shields.io/pypi/implementation/pyoxigraph) ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/pyoxigraph) [![actions status](https://github.com/oxigraph/oxigraph/workflows/build/badge.svg)](https://github.com/oxigraph/oxigraph/actions) -[![Gitter](https://badges.gitter.im/oxigraph/community.svg)](https://gitter.im/oxigraph/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) +[![Gitter](https://badges.gitter.im/oxigraph/community.svg)](https://gitter.im/oxigraph/community) Pyoxigraph is a graph database library implementing the [SPARQL](https://www.w3.org/TR/sparql11-overview/) standard. It is a Python library written on top of [Oxigraph](https://crates.io/crates/oxigraph). diff --git a/server/README.md b/server/README.md index 31075667..28a06073 100644 --- a/server/README.md +++ b/server/README.md @@ -5,7 +5,7 @@ Oxigraph Server [![Crates.io downloads](https://img.shields.io/crates/d/oxigraph_server)](https://crates.io/crates/oxigraph_server) [![Conda](https://img.shields.io/conda/vn/conda-forge/oxigraph-server)](https://anaconda.org/conda-forge/oxigraph-server) [![actions status](https://github.com/oxigraph/oxigraph/workflows/build/badge.svg)](https://github.com/oxigraph/oxigraph/actions) -[![Gitter](https://badges.gitter.im/oxigraph/community.svg)](https://gitter.im/oxigraph/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) +[![Gitter](https://badges.gitter.im/oxigraph/community.svg)](https://gitter.im/oxigraph/community) Oxigraph Server is a standalone HTTP server providing a graph database implementing the [SPARQL](https://www.w3.org/TR/sparql11-overview/) standard. From f10e5a40a396cdd4b7b58249044d807205b7e18f Mon Sep 17 00:00:00 2001 From: Tpt Date: Sat, 26 Aug 2023 13:17:24 +0200 Subject: [PATCH 068/217] Enables more Clippy lints Auto-enable all Clippy lints that are not in nursery but a blacklist --- .cargo/build_config.py | 82 ++++++ .cargo/config.toml | 59 ++-- Cargo.lock | 1 - fuzz/Cargo.toml | 1 - fuzz/fuzz_targets/sparql_eval.rs | 15 +- lib/Cargo.toml | 3 +- lib/benches/store.rs | 2 + lib/oxrdf/src/literal.rs | 2 +- lib/oxrdf/src/parser.rs | 2 +- lib/oxrdfio/Cargo.toml | 2 +- lib/oxrdfxml/Cargo.toml | 2 +- lib/oxrdfxml/src/error.rs | 8 +- lib/oxrdfxml/src/parser.rs | 16 +- lib/oxsdatatypes/src/decimal.rs | 3 +- lib/oxsdatatypes/src/double.rs | 2 +- lib/oxsdatatypes/src/float.rs | 2 +- lib/oxttl/Cargo.toml | 2 +- lib/oxttl/src/lexer.rs | 107 ++++--- lib/oxttl/src/line_formats.rs | 29 +- lib/oxttl/src/n3.rs | 84 +++--- lib/oxttl/src/terse.rs | 181 ++++++------ lib/oxttl/src/toolkit/lexer.rs | 6 +- lib/sparesults/src/csv.rs | 2 +- lib/sparesults/src/error.rs | 4 +- lib/sparesults/src/solution.rs | 3 + lib/spargebra/src/parser.rs | 1 + lib/sparopt/src/algebra.rs | 2 +- lib/sparopt/src/optimizer.rs | 11 +- lib/sparopt/src/type_inference.rs | 1 + lib/src/sparql/eval.rs | 33 ++- lib/src/sparql/model.rs | 164 +++++------ lib/src/storage/backend/fallback.rs | 14 +- lib/src/storage/backend/rocksdb.rs | 63 +++-- lib/src/storage/mod.rs | 54 ++-- lib/src/storage/numeric_encoder.rs | 30 +- lib/src/storage/small_string.rs | 6 +- lib/src/store.rs | 419 ++++++++++++++-------------- lib/tests/store.rs | 21 +- python/src/io.rs | 7 +- server/Cargo.toml | 4 +- server/src/main.rs | 6 +- testsuite/src/lib.rs | 1 + testsuite/src/manifest.rs | 4 +- testsuite/src/sparql_evaluator.rs | 18 +- testsuite/tests/canonicalization.rs | 2 + testsuite/tests/oxigraph.rs | 2 + testsuite/tests/parser.rs | 2 + testsuite/tests/serd.rs | 2 + testsuite/tests/sparql.rs | 2 + 49 files changed, 832 insertions(+), 657 deletions(-) create mode 100644 .cargo/build_config.py diff --git a/.cargo/build_config.py b/.cargo/build_config.py new file mode 100644 index 00000000..97dd9667 --- /dev/null +++ b/.cargo/build_config.py @@ -0,0 +1,82 @@ +import json +from urllib.request import urlopen + +MSRV = "1.70.0" +DEFAULT_BUILD_FLAGS = { + "-Wtrivial-casts", + "-Wtrivial-numeric-casts", + "-Wunsafe-code", + "-Wunused-lifetimes", + "-Wunused-qualifications", +} +FLAGS_BLACKLIST = { + "-Wclippy::alloc-instead-of-core", + "-Wclippy::arithmetic-side-effects", # TODO: might be nice + "-Wclippy::as-conversions", + "-Wclippy::cargo-common-metadata", # TODO: might be nice + "-Wclippy::doc-markdown", # Too many false positives + "-Wclippy::default-numeric-fallback", + "-Wclippy::else-if-without-else", + "-Wclippy::exhaustive-enums", + "-Wclippy::exhaustive-structs", + "-Wclippy::float-arithmetic", + "-Wclippy::float-cmp", + "-Wclippy::float-cmp-const", + "-Wclippy::impl-trait-in-params", + "-Wclippy::implicit-return", + "-Wclippy::indexing-slicing", + "-Wclippy::integer-arithmetic", + "-Wclippy::integer-division", + "-Wclippy::map-err-ignore", + "-Wclippy::missing-docs-in-private-items", + "-Wclippy::missing-errors-doc", + "-Wclippy::missing-inline-in-public-items", + "-Wclippy::missing-panics-doc", + "-Wclippy::missing-trait-methods", + "-Wclippy::mixed-read-write-in-expression", + "-Wclippy::mod-module-files", + "-Wclippy::module-name-repetitions", + "-Wclippy::modulo-arithmetic", + "-Wclippy::multiple-crate-versions", + "-Wclippy::multiple-unsafe-ops-per-block", + "-Wclippy::must-use-candidate", # TODO: might be nice + "-Wclippy::option-option", + "-Wclippy::pattern-type-mismatch", + "-Wclippy::pub-use", + "-Wclippy::question-mark-used", + "-Wclippy::self-named-module-files", # TODO: might be nice + "-Wclippy::semicolon-if-nothing-returned", # TODO: might be nice + "-Wclippy::semicolon-outside-block", + "-Wclippy::similar-names", + "-Wclippy::single-char-lifetime-names", + "-Wclippy::std-instead-of-alloc", + "-Wclippy::std-instead-of-core", + "-Wclippy::shadow-reuse", + "-Wclippy::shadow-unrelated", + "-Wclippy::string-slice", # TODO: might be nice + "-Wclippy::too-many-lines", + "-Wclippy::separated-literal-suffix", + "-Wclippy::unreachable", # TODO: might be nice + "-Wclippy::unwrap-used", # TODO: might be nice to use expect instead + "-Wclippy::wildcard-enum-match-arm", # TODO: might be nice + "-Wclippy::wildcard-imports", # TODO: might be nice +} + +build_flags = set(DEFAULT_BUILD_FLAGS) +with urlopen(f"https://rust-lang.github.io/rust-clippy/rust-{MSRV}/lints.json") as response: + for lint in json.load(response): + if lint["level"] == "allow" and lint["group"] != "nursery": + build_flags.add(f"-Wclippy::{lint['id'].replace('_', '-')}") + +for flag in FLAGS_BLACKLIST: + if flag in build_flags: + build_flags.remove(flag) + else: + print(f"Unused blacklisted flag: {flag}") + +with open("./config.toml", "wt") as fp: + fp.write("[build]\n") + fp.write("rustflags = [\n") + for flag in sorted(build_flags): + fp.write(f" \"{flag}\",\n") + fp.write("]\n") diff --git a/.cargo/config.toml b/.cargo/config.toml index 09e6f6fe..3f1a93ac 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,11 +1,9 @@ [build] rustflags = [ - "-Wtrivial-casts", - "-Wtrivial-numeric-casts", - "-Wunsafe-code", - "-Wunused-lifetimes", - "-Wunused-qualifications", + "-Wclippy::allow-attributes", + "-Wclippy::allow-attributes-without-reason", "-Wclippy::as-underscore", + "-Wclippy::assertions-on-result-states", "-Wclippy::bool-to-int-with-if", "-Wclippy::borrow-as-ptr", "-Wclippy::case-sensitive-file-extension-comparisons", @@ -19,11 +17,13 @@ rustflags = [ "-Wclippy::clone-on-ref-ptr", "-Wclippy::cloned-instead-of-copied", "-Wclippy::copy-iterator", + "-Wclippy::create-dir", "-Wclippy::dbg-macro", "-Wclippy::decimal-literal-representation", "-Wclippy::default-trait-access", "-Wclippy::default-union-representation", "-Wclippy::deref-by-slicing", + "-Wclippy::disallowed-script-idents", "-Wclippy::doc-link-with-quotes", "-Wclippy::empty-drop", "-Wclippy::empty-enum", @@ -35,8 +35,10 @@ rustflags = [ "-Wclippy::explicit-deref-methods", "-Wclippy::explicit-into-iter-loop", "-Wclippy::explicit-iter-loop", + "-Wclippy::filetype-is-file", "-Wclippy::filter-map-next", "-Wclippy::flat-map-option", + "-Wclippy::fn-params-excessive-bools", "-Wclippy::fn-to-numeric-cast-any", "-Wclippy::format-push-string", "-Wclippy::from-iter-instead-of-collect", @@ -44,6 +46,7 @@ rustflags = [ "-Wclippy::if-not-else", "-Wclippy::if-then-some-else-none", "-Wclippy::implicit-clone", + "-Wclippy::implicit-hasher", "-Wclippy::inconsistent-struct-constructor", "-Wclippy::index-refutable-slice", "-Wclippy::inefficient-to-string", @@ -52,38 +55,49 @@ rustflags = [ "-Wclippy::inline-asm-x86-intel-syntax", "-Wclippy::invalid-upcast-comparisons", "-Wclippy::items-after-statements", + "-Wclippy::iter-not-returning-iterator", "-Wclippy::large-digit-groups", - # TODO: 1.68+ "-Wclippy::large-futures", + "-Wclippy::large-futures", + "-Wclippy::large-include-file", "-Wclippy::large-stack-arrays", "-Wclippy::large-types-passed-by-value", "-Wclippy::let-underscore-must-use", - "-Wclippy::let-unit-value", + "-Wclippy::let-underscore-untyped", "-Wclippy::linkedlist", "-Wclippy::lossy-float-literal", "-Wclippy::macro-use-imports", "-Wclippy::manual-assert", "-Wclippy::manual-instant-elapsed", - # TODO: 1.67+ "-Wclippy::manual-let-else", + "-Wclippy::manual-let-else", "-Wclippy::manual-ok-or", "-Wclippy::manual-string-new", "-Wclippy::many-single-char-names", "-Wclippy::map-unwrap-or", "-Wclippy::match-bool", + "-Wclippy::match-on-vec-items", "-Wclippy::match-same-arms", + "-Wclippy::match-wild-err-arm", "-Wclippy::match-wildcard-for-single-variants", "-Wclippy::maybe-infinite-iter", "-Wclippy::mem-forget", "-Wclippy::mismatching-type-param-order", + "-Wclippy::missing-assert-message", + "-Wclippy::missing-enforced-import-renames", "-Wclippy::multiple-inherent-impl", "-Wclippy::mut-mut", "-Wclippy::mutex-atomic", "-Wclippy::naive-bytecount", "-Wclippy::needless-bitwise-bool", "-Wclippy::needless-continue", + "-Wclippy::needless-for-each", "-Wclippy::needless-pass-by-value", + "-Wclippy::negative-feature-names", "-Wclippy::no-effect-underscore-binding", - # TODO: 1.69+ "-Wclippy::no-mangle-with-rust-abi", + "-Wclippy::no-mangle-with-rust-abi", "-Wclippy::non-ascii-literal", + "-Wclippy::panic", + "-Wclippy::panic-in-result-fn", + "-Wclippy::partial-pub-fields", "-Wclippy::print-stderr", "-Wclippy::print-stdout", "-Wclippy::ptr-as-ptr", @@ -100,38 +114,49 @@ rustflags = [ "-Wclippy::return-self-not-must-use", "-Wclippy::same-functions-in-if-condition", "-Wclippy::same-name-method", - # TODO: 1.68+ "-Wclippy::semicolon-outside-block", + "-Wclippy::semicolon-inside-block", + "-Wclippy::shadow-same", "-Wclippy::single-match-else", "-Wclippy::stable-sort-primitive", "-Wclippy::str-to-string", "-Wclippy::string-add", "-Wclippy::string-add-assign", - "-Wclippy::string-lit-as-bytes", "-Wclippy::string-to-string", - # TODO: 1.67+ "-Wclippy::suspicious-xor-used-as-pow", + "-Wclippy::struct-excessive-bools", + "-Wclippy::suspicious-xor-used-as-pow", + "-Wclippy::tests-outside-test-module", "-Wclippy::todo", "-Wclippy::transmute-ptr-to-ptr", "-Wclippy::trivially-copy-pass-by-ref", "-Wclippy::try-err", + "-Wclippy::unchecked-duration-subtraction", + "-Wclippy::undocumented-unsafe-blocks", "-Wclippy::unicode-not-nfc", "-Wclippy::unimplemented", - # TODO: 1.66+ "-Wclippy::uninlined-format-args", - # TODO: 1.70+ "-Wclippy::unnecessary-box-returns", + "-Wclippy::uninlined-format-args", + "-Wclippy::unnecessary-box-returns", "-Wclippy::unnecessary-join", - # TODO: 1.67+ "-Wclippy::unnecessary-safety-comment", - # TODO: 1.67+ "-Wclippy::unnecessary-safety-doc", + "-Wclippy::unnecessary-safety-comment", + "-Wclippy::unnecessary-safety-doc", "-Wclippy::unnecessary-self-imports", "-Wclippy::unnecessary-wraps", "-Wclippy::unneeded-field-pattern", "-Wclippy::unnested-or-patterns", "-Wclippy::unreadable-literal", + "-Wclippy::unsafe-derive-deserialize", "-Wclippy::unseparated-literal-suffix", "-Wclippy::unused-async", "-Wclippy::unused-self", + "-Wclippy::unwrap-in-result", "-Wclippy::use-debug", "-Wclippy::used-underscore-binding", "-Wclippy::verbose-bit-mask", "-Wclippy::verbose-file-reads", "-Wclippy::wildcard-dependencies", "-Wclippy::zero-sized-map-values", -] \ No newline at end of file + "-Wtrivial-casts", + "-Wtrivial-numeric-casts", + "-Wunsafe-code", + "-Wunused-lifetimes", + "-Wunused-qualifications", +] diff --git a/Cargo.lock b/Cargo.lock index 22aa96e4..fa61e235 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -952,7 +952,6 @@ dependencies = [ "hex", "js-sys", "json-event-parser", - "lazy_static", "libc", "md-5", "oxhttp", diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index f1524903..4e41ea60 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -9,7 +9,6 @@ cargo-fuzz = true [dependencies] anyhow = "1" -lazy_static = "1" libfuzzer-sys = "0.4" oxrdf = { path = "../lib/oxrdf", features = ["rdf-star"] } oxttl = { path = "../lib/oxttl", features = ["rdf-star"] } diff --git a/fuzz/fuzz_targets/sparql_eval.rs b/fuzz/fuzz_targets/sparql_eval.rs index 5b52f4bd..24c8a176 100644 --- a/fuzz/fuzz_targets/sparql_eval.rs +++ b/fuzz/fuzz_targets/sparql_eval.rs @@ -1,27 +1,26 @@ #![no_main] -use lazy_static::lazy_static; use libfuzzer_sys::fuzz_target; use oxigraph::io::RdfFormat; use oxigraph::sparql::{Query, QueryOptions, QueryResults, QuerySolutionIter}; use oxigraph::store::Store; +use std::sync::OnceLock; -lazy_static! { - static ref STORE: Store = { +fuzz_target!(|data: sparql_smith::Query| { + static STORE: OnceLock = OnceLock::new(); + let store = STORE.get_or_init(|| { let store = Store::new().unwrap(); store .load_dataset(sparql_smith::DATA_TRIG.as_bytes(), RdfFormat::TriG, None) .unwrap(); store - }; -} + }); -fuzz_target!(|data: sparql_smith::Query| { let query_str = data.to_string(); if let Ok(query) = Query::parse(&query_str, None) { let options = QueryOptions::default(); - let with_opt = STORE.query_opt(query.clone(), options.clone()).unwrap(); - let without_opt = STORE + let with_opt = store.query_opt(query.clone(), options.clone()).unwrap(); + let without_opt = store .query_opt(query, options.without_optimizations()) .unwrap(); match (with_opt, without_opt) { diff --git a/lib/Cargo.toml b/lib/Cargo.toml index dd5a7c4d..7b06b69a 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -32,11 +32,10 @@ oxilangtag = "0.1" oxiri = "0.2" hex = "0.4" siphasher = "1" -lazy_static = "1" json-event-parser = "0.1" oxrdf = { version = "0.2.0-alpha.1-dev", path = "oxrdf", features = ["rdf-star", "oxsdatatypes"] } oxsdatatypes = { version = "0.2.0-alpha.1-dev", path="oxsdatatypes" } -oxrdfio = { version = "0.1.0-alpha.1-dev" , path = "oxrdfio", features = ["rdf-star"] } +oxrdfio = { version = "0.1.0-alpha.1-dev", path = "oxrdfio", features = ["rdf-star"] } spargebra = { version = "0.3.0-alpha.1-dev", path = "spargebra", features = ["rdf-star", "sep-0002", "sep-0006"] } sparopt = { version = "0.1.0-alpha.1-dev", path="sparopt", features = ["rdf-star", "sep-0002", "sep-0006"] } sparesults = { version = "0.2.0-alpha.1-dev", path = "sparesults", features = ["rdf-star"] } diff --git a/lib/benches/store.rs b/lib/benches/store.rs index eaaf71e7..6f18f8d1 100644 --- a/lib/benches/store.rs +++ b/lib/benches/store.rs @@ -1,3 +1,5 @@ +#![allow(clippy::panic)] + use criterion::{criterion_group, criterion_main, Criterion, Throughput}; use oxhttp::model::{Method, Request, Status}; use oxigraph::io::RdfFormat; diff --git a/lib/oxrdf/src/literal.rs b/lib/oxrdf/src/literal.rs index de37adc8..0e0b6da8 100644 --- a/lib/oxrdf/src/literal.rs +++ b/lib/oxrdf/src/literal.rs @@ -628,7 +628,7 @@ pub fn print_quoted_str(string: &str, f: &mut impl Write) -> fmt::Result { '"' => f.write_str("\\\""), '\\' => f.write_str("\\\\"), '\0'..='\u{1f}' | '\u{7f}' => write!(f, "\\u{:04X}", u32::from(c)), - c => f.write_char(c), + _ => f.write_char(c), }?; } f.write_char('"') diff --git a/lib/oxrdf/src/parser.rs b/lib/oxrdf/src/parser.rs index e06128d3..4d95072f 100644 --- a/lib/oxrdf/src/parser.rs +++ b/lib/oxrdf/src/parser.rs @@ -232,7 +232,7 @@ fn read_literal(s: &str) -> Result<(Literal, &str), TermParseError> { return Err(TermParseError::msg("Unexpected literal end")); } } - c => value.push(c), + _ => value.push(c), } } Err(TermParseError::msg("Unexpected literal end")) diff --git a/lib/oxrdfio/Cargo.toml b/lib/oxrdfio/Cargo.toml index c04c16fa..1ca0ce17 100644 --- a/lib/oxrdfio/Cargo.toml +++ b/lib/oxrdfio/Cargo.toml @@ -22,7 +22,7 @@ rdf-star = ["oxrdf/rdf-star", "oxttl/rdf-star"] [dependencies] oxrdf = { version = "0.2.0-alpha.1-dev", path = "../oxrdf" } oxrdfxml = { version = "0.1.0-alpha.1-dev", path = "../oxrdfxml" } -oxttl = { version = "0.1.0-alpha.1-dev" , path = "../oxttl" } +oxttl = { version = "0.1.0-alpha.1-dev", path = "../oxttl" } tokio = { version = "1", optional = true, features = ["io-util"] } [dev-dependencies] diff --git a/lib/oxrdfxml/Cargo.toml b/lib/oxrdfxml/Cargo.toml index f4c4d67d..fa7ef2b1 100644 --- a/lib/oxrdfxml/Cargo.toml +++ b/lib/oxrdfxml/Cargo.toml @@ -4,7 +4,7 @@ version = "0.1.0-alpha.1-dev" authors.workspace = true license.workspace = true readme = "README.md" -keywords = ["RDF/XML", "RDF"] +keywords = ["RDFXML", "XML", "RDF"] repository = "https://github.com/oxigraph/oxigraph/tree/master/lib/oxrdfxml" homepage.workspace = true description = """ diff --git a/lib/oxrdfxml/src/error.rs b/lib/oxrdfxml/src/error.rs index 1382844e..fd561be6 100644 --- a/lib/oxrdfxml/src/error.rs +++ b/lib/oxrdfxml/src/error.rs @@ -65,7 +65,7 @@ impl From for ParseError { Ok(error) => error, Err(error) => io::Error::new(error.kind(), error), }), - error => Self::Syntax(SyntaxError { + _ => Self::Syntax(SyntaxError { inner: SyntaxErrorKind::Xml(error), }), } @@ -121,10 +121,10 @@ impl fmt::Display for SyntaxError { SyntaxErrorKind::Xml(error) => error.fmt(f), SyntaxErrorKind::XmlAttribute(error) => error.fmt(f), SyntaxErrorKind::InvalidIri { iri, error } => { - write!(f, "error while parsing IRI '{}': {}", iri, error) + write!(f, "error while parsing IRI '{iri}': {error}") } SyntaxErrorKind::InvalidLanguageTag { tag, error } => { - write!(f, "error while parsing language tag '{}': {}", tag, error) + write!(f, "error while parsing language tag '{tag}': {error}") } SyntaxErrorKind::Msg { msg } => f.write_str(msg), } @@ -156,7 +156,7 @@ impl From for io::Error { quick_xml::Error::UnexpectedEof(error) => { Self::new(io::ErrorKind::UnexpectedEof, error) } - error => Self::new(io::ErrorKind::InvalidData, error), + _ => Self::new(io::ErrorKind::InvalidData, error), }, SyntaxErrorKind::Msg { msg } => Self::new(io::ErrorKind::InvalidData, msg), _ => Self::new(io::ErrorKind::InvalidData, error), diff --git a/lib/oxrdfxml/src/parser.rs b/lib/oxrdfxml/src/parser.rs index 747b2419..22983350 100644 --- a/lib/oxrdfxml/src/parser.rs +++ b/lib/oxrdfxml/src/parser.rs @@ -421,7 +421,9 @@ impl RdfXmlReader { match event { Event::Start(event) => self.parse_start_event(&event, results), Event::End(event) => self.parse_end_event(&event, results), - Event::Empty(_) => unreachable!("The expand_empty_elements option must be enabled"), + Event::Empty(_) => { + Err(SyntaxError::msg("The expand_empty_elements option must be enabled").into()) + } Event::Text(event) => self.parse_text_event(&event), Event::CData(event) => self.parse_text_event(&event.escape()?), Event::Comment(_) | Event::PI(_) => Ok(()), @@ -672,7 +674,9 @@ impl RdfXmlReader { subject: subject.clone(), }, Some(RdfXmlState::ParseTypeLiteralPropertyElt { .. }) => { - panic!("ParseTypeLiteralPropertyElt production children should never be considered as a RDF/XML content") + return Err( + SyntaxError::msg("ParseTypeLiteralPropertyElt production children should never be considered as a RDF/XML content").into() + ); } None => { return Err( @@ -747,8 +751,7 @@ impl RdfXmlReader { }; *li_counter += 1; NamedNode::new_unchecked(format!( - "http://www.w3.org/1999/02/22-rdf-syntax-ns#_{}", - li_counter + "http://www.w3.org/1999/02/22-rdf-syntax-ns#_{li_counter}" )) } else if RESERVED_RDF_ELEMENTS.contains(&&*tag_name) || *tag_name == *RDF_DESCRIPTION @@ -881,7 +884,7 @@ impl RdfXmlReader { if event.iter().copied().all(is_whitespace) { Ok(()) } else { - Err(SyntaxError::msg(format!("Unexpected text event: '{}'", text)).into()) + Err(SyntaxError::msg(format!("Unexpected text event: '{text}'")).into()) } } } @@ -1057,8 +1060,7 @@ impl RdfXmlReader { let object = writer.into_inner(); if object.is_empty() { return Err(SyntaxError::msg(format!( - "No value found for rdf:XMLLiteral value of property {}", - iri + "No value found for rdf:XMLLiteral value of property {iri}" ))); } let triple = Triple::new( diff --git a/lib/oxsdatatypes/src/decimal.rs b/lib/oxsdatatypes/src/decimal.rs index c93ef56a..fca06e5a 100644 --- a/lib/oxsdatatypes/src/decimal.rs +++ b/lib/oxsdatatypes/src/decimal.rs @@ -121,7 +121,6 @@ impl Decimal { let mut shift_left = 0_u32; if left != 0 { while let Some(r) = left.checked_mul(10) { - assert_eq!(r / 10, left); left = r; shift_left += 1; } @@ -571,7 +570,7 @@ impl fmt::Display for Decimal { .find_map(|(i, v)| if v == b'0' { None } else { Some(i) }) .unwrap_or(40); - let decimal_part_digits = usize::try_from(DECIMAL_PART_DIGITS).unwrap(); + let decimal_part_digits = usize::try_from(DECIMAL_PART_DIGITS).map_err(|_| fmt::Error)?; if last_non_zero >= decimal_part_digits { let end = if let Some(mut width) = f.width() { if self.value.is_negative() { diff --git a/lib/oxsdatatypes/src/double.rs b/lib/oxsdatatypes/src/double.rs index b392d9cf..d0c77c8a 100644 --- a/lib/oxsdatatypes/src/double.rs +++ b/lib/oxsdatatypes/src/double.rs @@ -173,7 +173,7 @@ impl From for Double { impl From for Double { #[inline] fn from(value: Boolean) -> Self { - if bool::from(value) { 1. } else { 0. }.into() + f64::from(bool::from(value)).into() } } diff --git a/lib/oxsdatatypes/src/float.rs b/lib/oxsdatatypes/src/float.rs index 4de94913..1feac769 100644 --- a/lib/oxsdatatypes/src/float.rs +++ b/lib/oxsdatatypes/src/float.rs @@ -153,7 +153,7 @@ impl From for Float { impl From for Float { #[inline] fn from(value: Boolean) -> Self { - if bool::from(value) { 1. } else { 0. }.into() + f32::from(bool::from(value)).into() } } diff --git a/lib/oxttl/Cargo.toml b/lib/oxttl/Cargo.toml index f5dda6c6..0cc2741b 100644 --- a/lib/oxttl/Cargo.toml +++ b/lib/oxttl/Cargo.toml @@ -4,7 +4,7 @@ version = "0.1.0-alpha.1-dev" authors.workspace = true license.workspace = true readme = "README.md" -keywords = ["N-Triples", "N-Quads", "Turtle", "TriG", "N3", "RDF"] +keywords = ["N-Triples", "N-Quads", "Turtle", "TriG", "N3"] repository = "https://github.com/oxigraph/oxigraph/tree/master/lib/oxttl" homepage.workspace = true description = """ diff --git a/lib/oxttl/src/lexer.rs b/lib/oxttl/src/lexer.rs index 7480908b..f12f3b25 100644 --- a/lib/oxttl/src/lexer.rs +++ b/lib/oxttl/src/lexer.rs @@ -4,6 +4,7 @@ use oxilangtag::LanguageTag; use oxiri::Iri; use oxrdf::NamedNode; use std::borrow::Cow; +use std::cmp::min; use std::collections::HashMap; use std::ops::{Range, RangeInclusive}; use std::str; @@ -261,7 +262,7 @@ impl N3Lexer { } return Some(( i, - Ok(N3Token::PlainKeyword(str::from_utf8(&data[..i]).unwrap())), + str_from_utf8(&data[..i], 0..i).map(N3Token::PlainKeyword), )); } } @@ -279,14 +280,17 @@ impl N3Lexer { } else { ( i, - Ok(N3Token::PlainKeyword(str::from_utf8(&data[..i]).unwrap())), + str_from_utf8(&data[..i], 0..i).map(N3Token::PlainKeyword), ) }); } else { return None; } } - let pn_prefix = str::from_utf8(&data[..i - 1]).unwrap(); + let pn_prefix = match str_from_utf8(&data[..i - 1], 0..i - 1) { + Ok(pn_prefix) => pn_prefix, + Err(e) => return Some((i, Err(e))), + }; if pn_prefix.ends_with('.') { return Some(( i, @@ -387,10 +391,13 @@ impl N3Lexer { // We add the missing bytes if i - position_that_is_already_in_buffer > 1 { buffer.push_str( - str::from_utf8( + match str_from_utf8( &data[position_that_is_already_in_buffer..i - 1], - ) - .unwrap(), + position_that_is_already_in_buffer..i - 1, + ) { + Ok(data) => data, + Err(e) => return Some((i, Err(e))), + }, ) } buffer.push(a); @@ -411,8 +418,13 @@ impl N3Lexer { } else { let buffer = if let Some(mut buffer) = buffer { buffer.push_str( - str::from_utf8(&data[position_that_is_already_in_buffer..i]) - .unwrap(), + match str_from_utf8( + &data[position_that_is_already_in_buffer..i], + position_that_is_already_in_buffer..i, + ) { + Ok(data) => data, + Err(e) => return Some((i, Err(e))), + }, ); // We do not include the last dot while buffer.ends_with('.') { @@ -421,7 +433,10 @@ impl N3Lexer { } Cow::Owned(buffer) } else { - let mut data = str::from_utf8(&data[..i]).unwrap(); + let mut data = match str_from_utf8(&data[..i], 0..i) { + Ok(data) => data, + Err(e) => return Some((i, Err(e))), + }; // We do not include the last dot while let Some(d) = data.strip_suffix('.') { data = d; @@ -443,7 +458,10 @@ impl N3Lexer { } Cow::Owned(buffer) } else { - let mut data = str::from_utf8(&data[..i]).unwrap(); + let mut data = match str_from_utf8(&data[..i], 0..i) { + Ok(data) => data, + Err(e) => return Some((i, Err(e))), + }; // We do not include the last dot while let Some(d) = data.strip_suffix('.') { data = d; @@ -475,9 +493,7 @@ impl N3Lexer { i -= 1; return Some(( i, - Ok(N3Token::BlankNodeLabel( - str::from_utf8(&data[2..i]).unwrap(), - )), + str_from_utf8(&data[2..i], 2..i).map(N3Token::BlankNodeLabel), )); } } else if i == 0 { @@ -489,16 +505,12 @@ impl N3Lexer { i -= 1; return Some(( i, - Ok(N3Token::BlankNodeLabel( - str::from_utf8(&data[2..i]).unwrap(), - )), + str_from_utf8(&data[2..i], 2..i).map(N3Token::BlankNodeLabel), )); } else { return Some(( i, - Ok(N3Token::BlankNodeLabel( - str::from_utf8(&data[2..i]).unwrap(), - )), + str_from_utf8(&data[2..i], 2..i).map(N3Token::BlankNodeLabel), )); } i += consumed; @@ -537,7 +549,7 @@ impl N3Lexer { position: Range, ) -> Result, TokenRecognizerError> { Ok(N3Token::LangTag( - LanguageTag::parse(str::from_utf8(lang_tag).unwrap()) + LanguageTag::parse(str_from_utf8(lang_tag, position.clone())?) .map_err(|e| (position.clone(), e.to_string()))? .into_inner(), )) @@ -553,18 +565,9 @@ impl N3Lexer { let mut i = 1; loop { let end = memchr2(delimiter, b'\\', &data[i..])?; - match str::from_utf8(&data[i..i + end]) { - Ok(a) => string.push_str(a), - Err(e) => { - return Some(( - end, - Err(( - i..i + end, - format!("The string contains invalid UTF-8 characters: {e}"), - ) - .into()), - )) - } + match str_from_utf8(&data[i..i + end], i..i + end) { + Ok(s) => string.push_str(s), + Err(e) => return Some((end, Err(e))), }; i += end; match data[i] { @@ -600,18 +603,9 @@ impl N3Lexer { let mut i = 3; loop { let end = memchr2(delimiter, b'\\', &data[i..])?; - match str::from_utf8(&data[i..i + end]) { - Ok(a) => string.push_str(a), - Err(e) => { - return Some(( - end, - Err(( - i..i + end, - format!("The string contains invalid UTF-8 characters: {e}"), - ) - .into()), - )) - } + match str_from_utf8(&data[i..i + end], i..i + end) { + Ok(s) => string.push_str(s), + Err(e) => return Some((end, Err(e))), }; i += end; match data[i] { @@ -706,7 +700,7 @@ impl N3Lexer { } else if count_before == 0 && count_after.unwrap_or(0) == 0 { Err((0..i, "A double should not be empty").into()) } else { - Ok(N3Token::Double(str::from_utf8(&data[..i]).unwrap())) + str_from_utf8(&data[..i], 0..i).map(N3Token::Double) }, )) } else if let Some(count_after) = count_after { @@ -718,11 +712,11 @@ impl N3Lexer { if count_before == 0 { Err((0..i, "An integer should not be empty").into()) } else { - Ok(N3Token::Integer(str::from_utf8(&data[..i]).unwrap())) + str_from_utf8(&data[..i], 0..i).map(N3Token::Integer) }, )) } else { - Some((i, Ok(N3Token::Decimal(str::from_utf8(&data[..i]).unwrap())))) + Some((i, str_from_utf8(&data[..i], 0..i).map(N3Token::Decimal))) } } else { Some(( @@ -730,7 +724,7 @@ impl N3Lexer { if count_before == 0 { Err((0..i, "An integer should not be empty").into()) } else { - Ok(N3Token::Integer(str::from_utf8(&data[..i]).unwrap())) + str_from_utf8(&data[..i], 0..i).map(N3Token::Integer) }, )) } @@ -780,12 +774,7 @@ impl N3Lexer { if data.len() < len { return Ok(None); } - let val = str::from_utf8(&data[..len]).map_err(|e| { - ( - position..position + len + 2, - format!("The escape sequence contains invalid UTF-8 characters: {e}"), - ) - })?; + let val = str_from_utf8(&data[..len], position..position + len + 2)?; let codepoint = u32::from_str_radix(val, 16).map_err(|e| { ( position..position + len + 2, @@ -936,3 +925,13 @@ pub fn resolve_local_name( Err(format!("The prefix {prefix}: has not been declared")) } } + +fn str_from_utf8(data: &[u8], range: Range) -> Result<&str, TokenRecognizerError> { + str::from_utf8(data).map_err(|e| { + ( + range.start + e.valid_up_to()..min(range.end, range.start + e.valid_up_to() + 4), + format!("Invalid UTF-8: {e}"), + ) + .into() + }) +} diff --git a/lib/oxttl/src/line_formats.rs b/lib/oxttl/src/line_formats.rs index 5990c889..f95e56f3 100644 --- a/lib/oxttl/src/line_formats.rs +++ b/lib/oxttl/src/line_formats.rs @@ -74,7 +74,7 @@ impl RuleRecognizer for NQuadsRecognizer { self.stack.push(NQuadsState::ExpectSubject); self } - token => self.error( + _ => self.error( errors, format!("The subject of a triple should be an IRI or a blank node, {token:?} found"), ), @@ -86,7 +86,7 @@ impl RuleRecognizer for NQuadsRecognizer { self.stack.push(NQuadsState::ExpectedObject); self } - token => self.error( + _ => self.error( errors, format!("The predicate of a triple should be an IRI, {token:?} found"), ), @@ -116,7 +116,7 @@ impl RuleRecognizer for NQuadsRecognizer { self.stack.push(NQuadsState::ExpectSubject); self } - token => self.error( + _ => self.error( errors, format!("The object of a triple should be an IRI, a blank node or a literal, {token:?} found"), ), @@ -139,7 +139,7 @@ impl RuleRecognizer for NQuadsRecognizer { .push(NQuadsState::ExpectLiteralDatatype { value }); self } - token => { + _ => { self.objects.push(Literal::new_simple_literal(value).into()); self.stack .push(NQuadsState::ExpectPossibleGraphOrEndOfQuotedTriple); @@ -159,7 +159,7 @@ impl RuleRecognizer for NQuadsRecognizer { .push(NQuadsState::ExpectPossibleGraphOrEndOfQuotedTriple); self } - token => self.error(errors, format!("A literal datatype must be an IRI, found {token:?}")), + _ => self.error(errors, format!("A literal datatype must be an IRI, found {token:?}")), }, NQuadsState::ExpectPossibleGraphOrEndOfQuotedTriple => { if self.stack.is_empty() { @@ -177,7 +177,7 @@ impl RuleRecognizer for NQuadsRecognizer { self.stack.push(NQuadsState::ExpectDot); self } - token => { + _ => { self.emit_quad(results, GraphName::DefaultGraph); self.stack.push(NQuadsState::ExpectDot); self.recognize_next(token, results, errors) @@ -189,16 +189,13 @@ impl RuleRecognizer for NQuadsRecognizer { self.error(errors, "Expecting the end of a quoted triple '>>'") } } - NQuadsState::ExpectDot => match token { - N3Token::Punctuation(".") => { - self.stack.push(NQuadsState::ExpectSubject); - self - } - token => { - errors.push("Quads should be followed by a dot".into()); - self.stack.push(NQuadsState::ExpectSubject); - self.recognize_next(token, results, errors) - } + NQuadsState::ExpectDot => if let N3Token::Punctuation(".") = token { + self.stack.push(NQuadsState::ExpectSubject); + self + } else { + errors.push("Quads should be followed by a dot".into()); + self.stack.push(NQuadsState::ExpectSubject); + self.recognize_next(token, results, errors) }, #[cfg(feature = "rdf-star")] NQuadsState::AfterQuotedSubject => { diff --git a/lib/oxttl/src/n3.rs b/lib/oxttl/src/n3.rs index 14c58413..46274ecf 100644 --- a/lib/oxttl/src/n3.rs +++ b/lib/oxttl/src/n3.rs @@ -567,7 +567,7 @@ impl RuleRecognizer for N3Recognizer { self.stack.push(N3State::BaseExpectIri); self } - token => { + _ => { self.stack.push(N3State::N3DocExpectDot); self.stack.push(N3State::Triples); self.recognize_next(token, results, errors) @@ -712,7 +712,7 @@ impl RuleRecognizer for N3Recognizer { self.stack.push(N3State::Path); self } - token => { + _ => { self.stack.push(N3State::AfterRegularVerb); self.stack.push(N3State::Path); self.recognize_next(token, results, errors) @@ -755,7 +755,7 @@ impl RuleRecognizer for N3Recognizer { self.stack.push(N3State::PathItem); self } - token => self.recognize_next(token, results, errors) + _ => self.recognize_next(token, results, errors) }, N3State::PathAfterIndicator { is_inverse } => { let predicate = self.terms.pop().unwrap(); @@ -836,7 +836,7 @@ impl RuleRecognizer for N3Recognizer { self.stack.push(N3State::FormulaContent); self } - token => self.error(errors, format!("This is not a valid RDF value: {token:?}")) + _ => self.error(errors, format!("This is not a valid RDF value: {token:?}")) } } N3State::PropertyListMiddle => match token { @@ -848,7 +848,7 @@ impl RuleRecognizer for N3Recognizer { self.stack.push(N3State::IriPropertyList); self }, - token => { + _ => { self.terms.push(BlankNode::default().into()); self.stack.push(N3State::PropertyListEnd); self.stack.push(N3State::PredicateObjectList); @@ -881,47 +881,43 @@ impl RuleRecognizer for N3Recognizer { self.error(errors, "The '[ id' construction should be followed by an IRI") } } - N3State::CollectionBeginning => match token { - N3Token::Punctuation(")") => { - self.terms.push(rdf::NIL.into()); - self - } - token => { - let root = BlankNode::default(); - self.terms.push(root.clone().into()); - self.terms.push(root.into()); - self.stack.push(N3State::CollectionPossibleEnd); - self.stack.push(N3State::Path); - self.recognize_next(token, results, errors) - } + N3State::CollectionBeginning => if let N3Token::Punctuation(")") = token { + self.terms.push(rdf::NIL.into()); + self + } else { + let root = BlankNode::default(); + self.terms.push(root.clone().into()); + self.terms.push(root.into()); + self.stack.push(N3State::CollectionPossibleEnd); + self.stack.push(N3State::Path); + self.recognize_next(token, results, errors) }, N3State::CollectionPossibleEnd => { let value = self.terms.pop().unwrap(); let old = self.terms.pop().unwrap(); results.push(self.quad( - old.clone(), - rdf::FIRST, - value, + old.clone(), + rdf::FIRST, + value, )); - match token { - N3Token::Punctuation(")") => { - results.push(self.quad(old, - rdf::REST, - rdf::NIL - )); - self - } - token => { - let new = BlankNode::default(); - results.push(self.quad( old, - rdf::REST, - new.clone() - )); - self.terms.push(new.into()); - self.stack.push(N3State::CollectionPossibleEnd); - self.stack.push(N3State::Path); - self.recognize_next(token, results, errors) - } + if let N3Token::Punctuation(")") = token { + results.push(self.quad( + old, + rdf::REST, + rdf::NIL + )); + self + } else { + let new = BlankNode::default(); + results.push(self.quad( + old, + rdf::REST, + new.clone() + )); + self.terms.push(new.into()); + self.stack.push(N3State::CollectionPossibleEnd); + self.stack.push(N3State::Path); + self.recognize_next(token, results, errors) } } N3State::LiteralPossibleSuffix { value } => { @@ -934,7 +930,7 @@ impl RuleRecognizer for N3Recognizer { self.stack.push(N3State::LiteralExpectDatatype { value }); self } - token => { + _ => { self.terms.push(Literal::new_simple_literal(value).into()); self.recognize_next(token, results, errors) } @@ -953,7 +949,7 @@ impl RuleRecognizer for N3Recognizer { }, Err(e) => self.error(errors, e) } - token => { + _ => { self.error(errors, format!("Expecting a datatype IRI after '^^, found {token:?}")).recognize_next(token, results, errors) } } @@ -985,7 +981,7 @@ impl RuleRecognizer for N3Recognizer { self.stack.push(N3State::BaseExpectIri); self } - token => { + _ => { self.stack.push(N3State::FormulaContentExpectDot); self.stack.push(N3State::Triples); self.recognize_next(token, results, errors) @@ -1002,7 +998,7 @@ impl RuleRecognizer for N3Recognizer { self.stack.push(N3State::FormulaContent); self } - token => { + _ => { errors.push("A dot is expected at the end of N3 statements".into()); self.stack.push(N3State::FormulaContent); self.recognize_next(token, results, errors) diff --git a/lib/oxttl/src/terse.rs b/lib/oxttl/src/terse.rs index cedf089e..ecd24d4a 100644 --- a/lib/oxttl/src/terse.rs +++ b/lib/oxttl/src/terse.rs @@ -80,11 +80,11 @@ impl RuleRecognizer for TriGRecognizer { self.stack.push(TriGState::GraphName); self } - token @ N3Token::Punctuation("{") if self.with_graph_name => { + N3Token::Punctuation("{") if self.with_graph_name => { self.stack.push(TriGState::WrappedGraph); self.recognize_next(token, results, errors) } - token => { + _ => { self.stack.push(TriGState::TriplesOrGraph); self.recognize_next(token, results, errors) } @@ -166,7 +166,7 @@ impl RuleRecognizer for TriGRecognizer { self.stack.push(TriGState::QuotedSubject); self } - token => { + _ => { self.error(errors, format!("The token {token:?} is not a valid subject or graph name")) } } @@ -208,49 +208,43 @@ impl RuleRecognizer for TriGRecognizer { self.recognize_next(token, results, errors) } TriGState::SubjectCollectionBeginning => { - match token { - N3Token::Punctuation(")") => { - self.cur_subject.push(rdf::NIL.into()); - self - } - token => { - let root = BlankNode::default(); - self.cur_subject.push(root.clone().into()); - self.cur_subject.push(root.into()); - self.cur_predicate.push(rdf::FIRST.into()); - self.stack.push(TriGState::SubjectCollectionPossibleEnd); - self.stack.push(TriGState::Object); - self.recognize_next(token, results, errors) - } + if let N3Token::Punctuation(")") = token { + self.cur_subject.push(rdf::NIL.into()); + self + } else { + let root = BlankNode::default(); + self.cur_subject.push(root.clone().into()); + self.cur_subject.push(root.into()); + self.cur_predicate.push(rdf::FIRST.into()); + self.stack.push(TriGState::SubjectCollectionPossibleEnd); + self.stack.push(TriGState::Object); + self.recognize_next(token, results, errors) } }, TriGState::SubjectCollectionPossibleEnd => { let old = self.cur_subject.pop().unwrap(); self.cur_object.pop(); - match token { - N3Token::Punctuation(")") => { - self.cur_predicate.pop(); - results.push(Quad::new( - old, - rdf::REST, - rdf::NIL, - self.cur_graph.clone() - )); - self - } - token => { - let new = BlankNode::default(); - results.push(Quad::new( - old, - rdf::REST, - new.clone(), - self.cur_graph.clone() - )); - self.cur_subject.push(new.into()); - self.stack.push(TriGState::ObjectCollectionPossibleEnd); - self.stack.push(TriGState::Object); - self.recognize_next(token, results, errors) - } + if let N3Token::Punctuation(")") = token { + self.cur_predicate.pop(); + results.push(Quad::new( + old, + rdf::REST, + rdf::NIL, + self.cur_graph.clone() + )); + self + } else { + let new = BlankNode::default(); + results.push(Quad::new( + old, + rdf::REST, + new.clone(), + self.cur_graph.clone() + )); + self.cur_subject.push(new.into()); + self.stack.push(TriGState::ObjectCollectionPossibleEnd); + self.stack.push(TriGState::Object); + self.recognize_next(token, results, errors) } } // [5g] wrappedGraph ::= '{' triplesBlock? '}' @@ -273,7 +267,7 @@ impl RuleRecognizer for TriGRecognizer { self.stack.push(TriGState::Triples); self } - token => { + _ => { errors.push("A '}' or a '.' is expected at the end of a graph block".into()); self.recognize_next(token, results, errors) } @@ -322,7 +316,7 @@ impl RuleRecognizer for TriGRecognizer { self.stack.push(TriGState::QuotedSubject); self } - token => { + _ => { self.error(errors, format!("The token {token:?} is not a valid RDF subject")) } }, @@ -355,7 +349,7 @@ impl RuleRecognizer for TriGRecognizer { self.stack.push(TriGState::GraphNameAnonEnd); self } - token => { + _ => { self.error(errors, format!("The token {token:?} is not a valid graph name")) } } @@ -419,7 +413,7 @@ impl RuleRecognizer for TriGRecognizer { self.stack.push(TriGState::PredicateObjectList); self } - token => { + _ => { self.cur_object.pop(); self.recognize_next(token, results, errors) } @@ -461,7 +455,7 @@ impl RuleRecognizer for TriGRecognizer { }, Err(e) => self.error(errors, e) } - token => { + _ => { self.error(errors, format!("The token {token:?} is not a valid predicate")) } } @@ -541,7 +535,7 @@ impl RuleRecognizer for TriGRecognizer { self.stack.push(TriGState::QuotedSubject); self } - token => { + _ => { self.error(errors, format!("This is not a valid RDF object: {token:?}")) } @@ -563,48 +557,42 @@ impl RuleRecognizer for TriGRecognizer { } else { self.error(errors, "blank node property lists should end with a ']'") } - TriGState::ObjectCollectionBeginning => match token { - N3Token::Punctuation(")") => { - self.cur_object.push(rdf::NIL.into()); - self.emit_quad(results); - self - } - token => { - let root = BlankNode::default(); - self.cur_object.push(root.clone().into()); - self.emit_quad(results); - self.cur_subject.push(root.into()); - self.cur_predicate.push(rdf::FIRST.into()); - self.stack.push(TriGState::ObjectCollectionPossibleEnd); - self.stack.push(TriGState::Object); - self.recognize_next(token, results, errors) - } + TriGState::ObjectCollectionBeginning => if let N3Token::Punctuation(")") = token { + self.cur_object.push(rdf::NIL.into()); + self.emit_quad(results); + self + } else { + let root = BlankNode::default(); + self.cur_object.push(root.clone().into()); + self.emit_quad(results); + self.cur_subject.push(root.into()); + self.cur_predicate.push(rdf::FIRST.into()); + self.stack.push(TriGState::ObjectCollectionPossibleEnd); + self.stack.push(TriGState::Object); + self.recognize_next(token, results, errors) }, TriGState::ObjectCollectionPossibleEnd => { let old = self.cur_subject.pop().unwrap(); self.cur_object.pop(); - match token { - N3Token::Punctuation(")") => { - self.cur_predicate.pop(); - results.push(Quad::new(old, - rdf::REST, - rdf::NIL, - self.cur_graph.clone() - )); - self - } - token => { - let new = BlankNode::default(); - results.push(Quad::new(old, - rdf::REST, - new.clone(), - self.cur_graph.clone() - )); - self.cur_subject.push(new.into()); - self.stack.push(TriGState::ObjectCollectionPossibleEnd); - self.stack.push(TriGState::Object); - self.recognize_next(token, results, errors) - } + if let N3Token::Punctuation(")") = token { + self.cur_predicate.pop(); + results.push(Quad::new(old, + rdf::REST, + rdf::NIL, + self.cur_graph.clone() + )); + self + }else { + let new = BlankNode::default(); + results.push(Quad::new(old, + rdf::REST, + new.clone(), + self.cur_graph.clone() + )); + self.cur_subject.push(new.into()); + self.stack.push(TriGState::ObjectCollectionPossibleEnd); + self.stack.push(TriGState::Object); + self.recognize_next(token, results, errors) } } TriGState::LiteralPossibleSuffix { value, emit } => { @@ -620,7 +608,7 @@ impl RuleRecognizer for TriGRecognizer { self.stack.push(TriGState::LiteralExpectDatatype { value, emit }); self } - token => { + _ => { self.cur_object.push(Literal::new_simple_literal(value).into()); if emit { self.emit_quad(results); @@ -648,7 +636,7 @@ impl RuleRecognizer for TriGRecognizer { }, Err(e) => self.error(errors, e) } - token => { + _ => { self.error(errors, format!("Expecting a datatype IRI after '^^, found {token:?}")).recognize_next(token, results, errors) } } @@ -715,7 +703,7 @@ impl RuleRecognizer for TriGRecognizer { self.stack.push(TriGState::QuotedSubject); self } - token => self.error(errors, format!("This is not a valid RDF quoted triple subject: {token:?}")) + _ => self.error(errors, format!("This is not a valid RDF quoted triple subject: {token:?}")) } // [29t] qtObject ::= iri | BlankNode | literal | quotedTriple #[cfg(feature = "rdf-star")] @@ -771,7 +759,7 @@ impl RuleRecognizer for TriGRecognizer { self.stack.push(TriGState::QuotedSubject); self } - token => self.error(errors, format!("This is not a valid RDF quoted triple object: {token:?}")) + _ => self.error(errors, format!("This is not a valid RDF quoted triple object: {token:?}")) } #[cfg(feature = "rdf-star")] TriGState::QuotedAnonEnd => if token == N3Token::Punctuation("]") { @@ -796,9 +784,18 @@ impl RuleRecognizer for TriGRecognizer { ) { match &*self.stack { [] | [TriGState::TriGDoc] => { - debug_assert!(self.cur_subject.is_empty()); - debug_assert!(self.cur_predicate.is_empty()); - debug_assert!(self.cur_object.is_empty()); + debug_assert!( + self.cur_subject.is_empty(), + "The cur_subject stack must be empty if the state stack is empty" + ); + debug_assert!( + self.cur_predicate.is_empty(), + "The cur_predicate stack must be empty if the state stack is empty" + ); + debug_assert!( + self.cur_object.is_empty(), + "The cur_object stack must be empty if the state stack is empty" + ); } [.., TriGState::LiteralPossibleSuffix { value, emit: true }] => { self.cur_object diff --git a/lib/oxttl/src/toolkit/lexer.rs b/lib/oxttl/src/toolkit/lexer.rs index 5b980f94..34c1c01e 100644 --- a/lib/oxttl/src/toolkit/lexer.rs +++ b/lib/oxttl/src/toolkit/lexer.rs @@ -153,12 +153,10 @@ impl Lexer { options: &R::Options, ) -> Option>, LexerError>> { self.skip_whitespaces_and_comments()?; - let (consumed, result) = if let Some(r) = + let Some((consumed, result)) = self.parser .recognize_next_token(&self.data[self.start..], self.is_ending, options) - { - r - } else { + else { return if self.is_ending { if self.start == self.data.len() { None // We have finished diff --git a/lib/sparesults/src/csv.rs b/lib/sparesults/src/csv.rs index 759aea26..ff33ce30 100644 --- a/lib/sparesults/src/csv.rs +++ b/lib/sparesults/src/csv.rs @@ -198,7 +198,7 @@ fn write_tsv_quoted_str(string: &str, f: &mut impl Write) -> io::Result<()> { b'\r' => f.write_all(b"\\r"), b'"' => f.write_all(b"\\\""), b'\\' => f.write_all(b"\\\\"), - c => f.write_all(&[c]), + _ => f.write_all(&[c]), }?; } f.write_all(b"\"") diff --git a/lib/sparesults/src/error.rs b/lib/sparesults/src/error.rs index fbb45728..d150e847 100644 --- a/lib/sparesults/src/error.rs +++ b/lib/sparesults/src/error.rs @@ -64,7 +64,7 @@ impl From for ParseError { Ok(error) => error, Err(error) => io::Error::new(error.kind(), error), }), - error => Self::Syntax(SyntaxError { + _ => Self::Syntax(SyntaxError { inner: SyntaxErrorKind::Xml(error), }), } @@ -128,7 +128,7 @@ impl From for io::Error { quick_xml::Error::UnexpectedEof(error) => { Self::new(io::ErrorKind::UnexpectedEof, error) } - error => Self::new(io::ErrorKind::InvalidData, error), + _ => Self::new(io::ErrorKind::InvalidData, error), }, SyntaxErrorKind::Term(error) => Self::new(io::ErrorKind::InvalidData, error), SyntaxErrorKind::Msg { msg } => Self::new(io::ErrorKind::InvalidData, msg), diff --git a/lib/sparesults/src/solution.rs b/lib/sparesults/src/solution.rs index a8059204..842bc7d3 100644 --- a/lib/sparesults/src/solution.rs +++ b/lib/sparesults/src/solution.rs @@ -141,6 +141,7 @@ impl<'a> IntoIterator for &'a QuerySolution { impl Index for QuerySolution { type Output = Term; + #[allow(clippy::panic)] #[inline] fn index(&self, index: usize) -> &Term { self.get(index) @@ -151,6 +152,7 @@ impl Index for QuerySolution { impl Index<&str> for QuerySolution { type Output = Term; + #[allow(clippy::panic)] #[inline] fn index(&self, index: &str) -> &Term { self.get(index) @@ -161,6 +163,7 @@ impl Index<&str> for QuerySolution { impl Index> for QuerySolution { type Output = Term; + #[allow(clippy::panic)] #[inline] fn index(&self, index: VariableRef<'_>) -> &Term { self.get(index) diff --git a/lib/spargebra/src/parser.rs b/lib/spargebra/src/parser.rs index 32522c5c..60f8038c 100644 --- a/lib/spargebra/src/parser.rs +++ b/lib/spargebra/src/parser.rs @@ -1085,6 +1085,7 @@ parser! { let (delete, insert) = c; let mut delete = delete.unwrap_or_default(); let mut insert = insert.unwrap_or_default(); + #[allow(clippy::shadow_same)] let mut pattern = pattern; let mut using = if u.is_empty() { diff --git a/lib/sparopt/src/algebra.rs b/lib/sparopt/src/algebra.rs index c7d7c82a..fd7942d5 100644 --- a/lib/sparopt/src/algebra.rs +++ b/lib/sparopt/src/algebra.rs @@ -875,7 +875,7 @@ impl GraphPattern { inner, expression: expression & e2, }, - inner => Self::Filter { + _ => Self::Filter { inner: Box::new(inner), expression, }, diff --git a/lib/sparopt/src/optimizer.rs b/lib/sparopt/src/optimizer.rs index 344a4536..87902b59 100644 --- a/lib/sparopt/src/optimizer.rs +++ b/lib/sparopt/src/optimizer.rs @@ -272,13 +272,13 @@ impl Optimizer { fn push_filters( pattern: GraphPattern, - filters: Vec, + mut filters: Vec, input_types: &VariableTypes, ) -> GraphPattern { match pattern { - pattern @ (GraphPattern::QuadPattern { .. } + GraphPattern::QuadPattern { .. } | GraphPattern::Path { .. } - | GraphPattern::Values { .. }) => { + | GraphPattern::Values { .. } => { GraphPattern::filter(pattern, Expression::and_all(filters)) } GraphPattern::Join { @@ -416,7 +416,6 @@ impl Optimizer { ) } GraphPattern::Filter { inner, expression } => { - let mut filters = filters; if let Expression::And(expressions) = expression { filters.extend(expressions) } else { @@ -479,9 +478,9 @@ impl Optimizer { fn reorder_joins(pattern: GraphPattern, input_types: &VariableTypes) -> GraphPattern { match pattern { - pattern @ (GraphPattern::QuadPattern { .. } + GraphPattern::QuadPattern { .. } | GraphPattern::Path { .. } - | GraphPattern::Values { .. }) => pattern, + | GraphPattern::Values { .. } => pattern, GraphPattern::Join { left, right, .. } => { // We flatten the join operation let mut to_reorder = Vec::new(); diff --git a/lib/sparopt/src/type_inference.rs b/lib/sparopt/src/type_inference.rs index d52476bd..161ba58a 100644 --- a/lib/sparopt/src/type_inference.rs +++ b/lib/sparopt/src/type_inference.rs @@ -340,6 +340,7 @@ impl VariableTypes { } } +#[allow(clippy::struct_excessive_bools)] #[derive(Clone, Copy, Eq, PartialEq, Debug, Default)] pub struct VariableType { pub undef: bool, diff --git a/lib/src/sparql/eval.rs b/lib/src/sparql/eval.rs index ad50cd2f..fb1737d6 100644 --- a/lib/src/sparql/eval.rs +++ b/lib/src/sparql/eval.rs @@ -326,11 +326,11 @@ impl SimpleEvaluator { inner, silent, } => { + #[allow(clippy::shadow_same)] let silent = *silent; let service_name = TupleSelector::from_named_node_pattern(name, encoded_variables, &self.dataset); - let _ = - self.build_graph_pattern_evaluator(inner, encoded_variables, &mut Vec::new()); // We call recursively to fill "encoded_variables" + self.build_graph_pattern_evaluator(inner, encoded_variables, &mut Vec::new()); // We call recursively to fill "encoded_variables" let graph_pattern = spargebra::algebra::GraphPattern::from(inner.as_ref()); let variables = Rc::from(encoded_variables.as_slice()); let eval = self.clone(); @@ -907,6 +907,7 @@ impl SimpleEvaluator { let (mut child, child_stats) = self.graph_pattern_evaluator(inner, encoded_variables); stat_children.push(child_stats); + #[allow(clippy::shadow_same)] let start = *start; if start > 0 { child = Rc::new(move |from| Box::new(child(from).skip(start))); @@ -3416,10 +3417,10 @@ fn cmp_terms(dataset: &DatasetView, a: Option<&EncodedTerm>, b: Option<&EncodedT } _ => Ordering::Greater, }, - a => match b { + _ => match b { _ if b.is_named_node() || b.is_blank_node() => Ordering::Greater, _ if b.is_triple() => Ordering::Less, - b => { + _ => { if let Some(ord) = partial_cmp_literals(dataset, a, b) { ord } else if let (Ok(Term::Literal(a)), Ok(Term::Literal(b))) = @@ -5872,14 +5873,18 @@ impl Timer { } } -#[test] -fn uuid() { - let mut buffer = String::default(); - generate_uuid(&mut buffer); - assert!( - Regex::new("^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$") - .unwrap() - .is_match(&buffer), - "{buffer} is not a valid UUID" - ); +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn uuid() { + let mut buffer = String::default(); + generate_uuid(&mut buffer); + assert!( + Regex::new("^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$") + .unwrap() + .is_match(&buffer), + "{buffer} is not a valid UUID" + ); + } } diff --git a/lib/src/sparql/model.rs b/lib/src/sparql/model.rs index 1cd64158..d076baec 100644 --- a/lib/src/sparql/model.rs +++ b/lib/src/sparql/model.rs @@ -259,87 +259,95 @@ impl Iterator for QueryTripleIter { } } -#[test] -fn test_serialization_roundtrip() -> Result<(), EvaluationError> { - use std::io::Cursor; - use std::str; +#[cfg(test)] +mod tests { + #![allow(clippy::panic_in_result_fn)] - for format in [ - QueryResultsFormat::Json, - QueryResultsFormat::Xml, - QueryResultsFormat::Tsv, - ] { - let results = vec![ - QueryResults::Boolean(true), - QueryResults::Boolean(false), - QueryResults::Solutions(QuerySolutionIter::new( - Rc::new(vec![ - Variable::new_unchecked("foo"), - Variable::new_unchecked("bar"), - ]), - Box::new( - vec![ - Ok(vec![None, None]), - Ok(vec![ - Some(NamedNode::new_unchecked("http://example.com").into()), - None, - ]), - Ok(vec![ - None, - Some(NamedNode::new_unchecked("http://example.com").into()), - ]), - Ok(vec![ - Some(BlankNode::new_unchecked("foo").into()), - Some(BlankNode::new_unchecked("bar").into()), - ]), - Ok(vec![Some(Literal::new_simple_literal("foo").into()), None]), - Ok(vec![ - Some( - Literal::new_language_tagged_literal_unchecked("foo", "fr").into(), - ), - None, - ]), - Ok(vec![ - Some(Literal::from(1).into()), - Some(Literal::from(true).into()), - ]), - Ok(vec![ - Some(Literal::from(1.33).into()), - Some(Literal::from(false).into()), - ]), - Ok(vec![ - Some( - Triple::new( - NamedNode::new_unchecked("http://example.com/s"), - NamedNode::new_unchecked("http://example.com/p"), + use super::*; + + #[test] + fn test_serialization_roundtrip() -> Result<(), EvaluationError> { + use std::io::Cursor; + use std::str; + + for format in [ + QueryResultsFormat::Json, + QueryResultsFormat::Xml, + QueryResultsFormat::Tsv, + ] { + let results = vec![ + QueryResults::Boolean(true), + QueryResults::Boolean(false), + QueryResults::Solutions(QuerySolutionIter::new( + Rc::new(vec![ + Variable::new_unchecked("foo"), + Variable::new_unchecked("bar"), + ]), + Box::new( + vec![ + Ok(vec![None, None]), + Ok(vec![ + Some(NamedNode::new_unchecked("http://example.com").into()), + None, + ]), + Ok(vec![ + None, + Some(NamedNode::new_unchecked("http://example.com").into()), + ]), + Ok(vec![ + Some(BlankNode::new_unchecked("foo").into()), + Some(BlankNode::new_unchecked("bar").into()), + ]), + Ok(vec![Some(Literal::new_simple_literal("foo").into()), None]), + Ok(vec![ + Some( + Literal::new_language_tagged_literal_unchecked("foo", "fr") + .into(), + ), + None, + ]), + Ok(vec![ + Some(Literal::from(1).into()), + Some(Literal::from(true).into()), + ]), + Ok(vec![ + Some(Literal::from(1.33).into()), + Some(Literal::from(false).into()), + ]), + Ok(vec![ + Some( Triple::new( - NamedNode::new_unchecked("http://example.com/os"), - NamedNode::new_unchecked("http://example.com/op"), - NamedNode::new_unchecked("http://example.com/oo"), - ), - ) - .into(), - ), - None, - ]), - ] - .into_iter(), - ), - )), - ]; + NamedNode::new_unchecked("http://example.com/s"), + NamedNode::new_unchecked("http://example.com/p"), + Triple::new( + NamedNode::new_unchecked("http://example.com/os"), + NamedNode::new_unchecked("http://example.com/op"), + NamedNode::new_unchecked("http://example.com/oo"), + ), + ) + .into(), + ), + None, + ]), + ] + .into_iter(), + ), + )), + ]; - for ex in results { - let mut buffer = Vec::new(); - ex.write(&mut buffer, format)?; - let ex2 = QueryResults::read(Cursor::new(buffer.clone()), format)?; - let mut buffer2 = Vec::new(); - ex2.write(&mut buffer2, format)?; - assert_eq!( - str::from_utf8(&buffer).unwrap(), - str::from_utf8(&buffer2).unwrap() - ); + for ex in results { + let mut buffer = Vec::new(); + ex.write(&mut buffer, format)?; + let ex2 = QueryResults::read(Cursor::new(buffer.clone()), format)?; + let mut buffer2 = Vec::new(); + ex2.write(&mut buffer2, format)?; + assert_eq!( + str::from_utf8(&buffer).unwrap(), + str::from_utf8(&buffer2).unwrap() + ); + } } - } - Ok(()) + Ok(()) + } } diff --git a/lib/src/storage/backend/fallback.rs b/lib/src/storage/backend/fallback.rs index 6bdd5673..c2eb17ec 100644 --- a/lib/src/storage/backend/fallback.rs +++ b/lib/src/storage/backend/fallback.rs @@ -1,6 +1,7 @@ //! TODO: This storage is dramatically naive. use crate::storage::StorageError; +use crate::store::CorruptionError; use std::cell::RefCell; use std::collections::{BTreeMap, HashMap}; use std::error::Error; @@ -30,9 +31,13 @@ impl Db { } #[allow(clippy::unwrap_in_result)] - pub fn column_family(&self, name: &'static str) -> Option { - let name = ColumnFamily(name); - self.0.read().unwrap().contains_key(&name).then_some(name) + pub fn column_family(&self, name: &'static str) -> Result { + let column_family = ColumnFamily(name); + if self.0.read().unwrap().contains_key(&column_family) { + Ok(column_family) + } else { + Err(CorruptionError::msg(format!("Column family {name} does not exist")).into()) + } } #[must_use] @@ -116,6 +121,7 @@ impl Reader { } } + #[allow(clippy::iter_not_returning_iterator)] pub fn iter(&self, column_family: &ColumnFamily) -> Result { self.scan_prefix(column_family, &[]) } @@ -226,7 +232,7 @@ pub struct Transaction<'a>( impl Transaction<'_> { #[allow(unsafe_code, clippy::useless_transmute)] pub fn reader(&self) -> Reader { - // This transmute is safe because we take a weak reference and the only Rc reference used is guarded by the lifetime. + // SAFETY: This transmute is safe because we take a weak reference and the only Rc reference used is guarded by the lifetime. Reader(InnerReader::Transaction(Rc::downgrade(unsafe { transmute(&self.0) }))) diff --git a/lib/src/storage/backend/rocksdb.rs b/lib/src/storage/backend/rocksdb.rs index 68e88744..6d065059 100644 --- a/lib/src/storage/backend/rocksdb.rs +++ b/lib/src/storage/backend/rocksdb.rs @@ -1,9 +1,14 @@ //! Code inspired by [Rust RocksDB](https://github.com/rust-rocksdb/rust-rocksdb) under Apache License 2.0. -#![allow(unsafe_code, trivial_casts)] +#![allow( + unsafe_code, + trivial_casts, + clippy::undocumented_unsafe_blocks, + clippy::panic_in_result_fn, + clippy::unwrap_in_result +)] use crate::storage::error::{CorruptionError, StorageError}; -use lazy_static::lazy_static; use libc::{self, c_void, free}; use oxrocksdb_sys::*; use rand::random; @@ -20,7 +25,7 @@ use std::marker::PhantomData; use std::ops::Deref; use std::path::{Path, PathBuf}; use std::rc::{Rc, Weak}; -use std::sync::Arc; +use std::sync::{Arc, OnceLock}; use std::thread::{available_parallelism, yield_now}; use std::{ptr, slice}; @@ -51,23 +56,6 @@ macro_rules! ffi_result_impl { }} } -lazy_static! { - static ref ROCKSDB_ENV: UnsafeEnv = { - unsafe { - let env = rocksdb_create_default_env(); - assert!(!env.is_null(), "rocksdb_create_default_env returned null"); - UnsafeEnv(env) - } - }; - static ref ROCKSDB_MEM_ENV: UnsafeEnv = { - unsafe { - let env = rocksdb_create_mem_env(); - assert!(!env.is_null(), "rocksdb_create_mem_env returned null"); - UnsafeEnv(env) - } - }; -} - pub struct ColumnFamilyDefinition { pub name: &'static str, pub use_iter: bool, @@ -132,7 +120,7 @@ impl Drop for RwDbHandler { } if self.in_memory { #[allow(clippy::let_underscore_must_use)] - let _ = remove_dir_all(&self.path); + let _: io::Result<()> = remove_dir_all(&self.path); } } } @@ -167,7 +155,7 @@ impl Drop for RoDbHandler { } if let Some(path) = &self.path_to_remove { #[allow(clippy::let_underscore_must_use)] - let _ = remove_dir_all(path); + let _: io::Result<()> = remove_dir_all(path); } } } @@ -466,6 +454,9 @@ impl Db { limit_max_open_files: bool, in_memory: bool, ) -> Result<*mut rocksdb_options_t, StorageError> { + static ROCKSDB_ENV: OnceLock = OnceLock::new(); + static ROCKSDB_MEM_ENV: OnceLock = OnceLock::new(); + unsafe { let options = rocksdb_options_create(); assert!(!options.is_null(), "rocksdb_options_create returned null"); @@ -502,10 +493,19 @@ impl Db { rocksdb_options_set_env( options, if in_memory { - ROCKSDB_MEM_ENV.0 + ROCKSDB_MEM_ENV.get_or_init(|| { + let env = rocksdb_create_mem_env(); + assert!(!env.is_null(), "rocksdb_create_mem_env returned null"); + UnsafeEnv(env) + }) } else { - ROCKSDB_ENV.0 - }, + ROCKSDB_ENV.get_or_init(|| { + let env = rocksdb_create_default_env(); + assert!(!env.is_null(), "rocksdb_create_default_env returned null"); + UnsafeEnv(env) + }) + } + .0, ); Ok(options) } @@ -551,17 +551,17 @@ impl Db { (column_family_names, c_column_family_names, cf_options) } - pub fn column_family(&self, name: &'static str) -> Option { + pub fn column_family(&self, name: &'static str) -> Result { let (column_family_names, cf_handles) = match &self.inner { DbKind::ReadOnly(db) => (&db.column_family_names, &db.cf_handles), DbKind::ReadWrite(db) => (&db.column_family_names, &db.cf_handles), }; for (cf, cf_handle) in column_family_names.iter().zip(cf_handles) { if *cf == name { - return Some(ColumnFamily(*cf_handle)); + return Ok(ColumnFamily(*cf_handle)); } } - None + Err(CorruptionError::msg(format!("Column family {name} does not exist")).into()) } #[must_use] @@ -572,7 +572,8 @@ impl Db { if db.is_secondary { // We try to refresh (and ignore the errors) #[allow(clippy::let_underscore_must_use)] - let _ = ffi_result!(rocksdb_try_catch_up_with_primary_with_status(db.db)); + let _: Result<(), ErrorStatus> = + ffi_result!(rocksdb_try_catch_up_with_primary_with_status(db.db)); } let options = rocksdb_readoptions_create_copy(db.read_options); Reader { @@ -980,6 +981,7 @@ impl Reader { Ok(self.get(column_family, key)?.is_some()) //TODO: optimize } + #[allow(clippy::iter_not_returning_iterator)] pub fn iter(&self, column_family: &ColumnFamily) -> Result { self.scan_prefix(column_family, &[]) } @@ -1392,7 +1394,8 @@ impl From for StorageError { struct UnsafeEnv(*mut rocksdb_env_t); -// Hack for lazy_static. OK because only written in lazy static and used in a thread-safe way by RocksDB +// Hack for OnceCell. OK because only written in OnceCell and used in a thread-safe way by RocksDB +unsafe impl Send for UnsafeEnv {} unsafe impl Sync for UnsafeEnv {} fn path_to_cstring(path: &Path) -> Result { diff --git a/lib/src/storage/mod.rs b/lib/src/storage/mod.rs index 6591ac47..126f5fcc 100644 --- a/lib/src/storage/mod.rs +++ b/lib/src/storage/mod.rs @@ -28,7 +28,7 @@ use std::path::{Path, PathBuf}; #[cfg(not(target_family = "wasm"))] use std::sync::atomic::{AtomicU64, Ordering}; #[cfg(not(target_family = "wasm"))] -use std::thread; +use std::{io, thread}; mod backend; mod binary_encoder; @@ -178,22 +178,21 @@ impl Storage { ] } - #[allow(clippy::unnecessary_wraps, clippy::unwrap_in_result)] fn setup(db: Db) -> Result { let this = Self { #[cfg(not(target_family = "wasm"))] - default_cf: db.column_family(DEFAULT_CF).unwrap(), - id2str_cf: db.column_family(ID2STR_CF).unwrap(), - spog_cf: db.column_family(SPOG_CF).unwrap(), - posg_cf: db.column_family(POSG_CF).unwrap(), - ospg_cf: db.column_family(OSPG_CF).unwrap(), - gspo_cf: db.column_family(GSPO_CF).unwrap(), - gpos_cf: db.column_family(GPOS_CF).unwrap(), - gosp_cf: db.column_family(GOSP_CF).unwrap(), - dspo_cf: db.column_family(DSPO_CF).unwrap(), - dpos_cf: db.column_family(DPOS_CF).unwrap(), - dosp_cf: db.column_family(DOSP_CF).unwrap(), - graphs_cf: db.column_family(GRAPHS_CF).unwrap(), + default_cf: db.column_family(DEFAULT_CF)?, + id2str_cf: db.column_family(ID2STR_CF)?, + spog_cf: db.column_family(SPOG_CF)?, + posg_cf: db.column_family(POSG_CF)?, + ospg_cf: db.column_family(OSPG_CF)?, + gspo_cf: db.column_family(GSPO_CF)?, + gpos_cf: db.column_family(GPOS_CF)?, + gosp_cf: db.column_family(GOSP_CF)?, + dspo_cf: db.column_family(DSPO_CF)?, + dpos_cf: db.column_family(DPOS_CF)?, + dosp_cf: db.column_family(DOSP_CF)?, + graphs_cf: db.column_family(GRAPHS_CF)?, db, }; #[cfg(not(target_family = "wasm"))] @@ -1282,7 +1281,7 @@ impl StorageBulkLoader { batch_size, )?; for thread in threads { - thread.join().unwrap()?; + map_thread_result(thread.join()).map_err(StorageError::Io)??; self.on_possible_progress(&done_counter, &mut done_and_displayed_counter); } Ok(()) @@ -1303,7 +1302,7 @@ impl StorageBulkLoader { // We avoid to have too many threads if threads.len() >= num_threads { if let Some(thread) = threads.pop_front() { - thread.join().unwrap()?; + map_thread_result(thread.join()).map_err(StorageError::Io)??; self.on_possible_progress(done_counter, done_and_displayed_counter); } } @@ -1353,7 +1352,7 @@ impl<'a> FileBulkLoader<'a> { self.encode(quads)?; let size = self.triples.len() + self.quads.len(); self.save()?; - counter.fetch_add(size.try_into().unwrap(), Ordering::Relaxed); + counter.fetch_add(size.try_into().unwrap_or(u64::MAX), Ordering::Relaxed); Ok(()) } @@ -1376,7 +1375,12 @@ impl<'a> FileBulkLoader<'a> { match quad.graph_name.as_ref() { GraphNameRef::NamedNode(n) => n.into(), GraphNameRef::BlankNode(n) => n.into(), - GraphNameRef::DefaultGraph => unreachable!(), + GraphNameRef::DefaultGraph => { + return Err(CorruptionError::new( + "Default graph this not the default graph", + ) + .into()) + } }, &encoded.graph_name, )?; @@ -1534,3 +1538,17 @@ impl<'a> FileBulkLoader<'a> { sst.finish() } } + +#[cfg(not(target_family = "wasm"))] +fn map_thread_result(result: thread::Result) -> io::Result { + result.map_err(|e| { + io::Error::new( + io::ErrorKind::Other, + if let Ok(e) = e.downcast::<&dyn std::fmt::Display>() { + format!("A loader processed crashed with {e}") + } else { + "A loader processed crashed with and unknown error".into() + }, + ) + }) +} diff --git a/lib/src/storage/numeric_encoder.rs b/lib/src/storage/numeric_encoder.rs index 94e237f6..59f7532f 100644 --- a/lib/src/storage/numeric_encoder.rs +++ b/lib/src/storage/numeric_encoder.rs @@ -716,13 +716,19 @@ pub fn insert_term Result<(), StorageError>>( if let EncodedTerm::NamedNode { iri_id } = encoded { insert_str(iri_id, node.as_str()) } else { - unreachable!("Invalid term encoding {encoded:?} for {term}") + Err( + CorruptionError::new(format!("Invalid term encoding {encoded:?} for {term}")) + .into(), + ) } } TermRef::BlankNode(node) => match encoded { EncodedTerm::BigBlankNode { id_id } => insert_str(id_id, node.as_str()), EncodedTerm::SmallBlankNode(..) | EncodedTerm::NumericalBlankNode { .. } => Ok(()), - _ => unreachable!("Invalid term encoding {encoded:?} for {term}"), + _ => Err( + CorruptionError::new(format!("Invalid term encoding {encoded:?} for {term}")) + .into(), + ), }, TermRef::Literal(literal) => match encoded { EncodedTerm::BigStringLiteral { value_id } @@ -733,7 +739,10 @@ pub fn insert_term Result<(), StorageError>>( if let Some(language) = literal.language() { insert_str(language_id, language) } else { - unreachable!("Invalid term encoding {encoded:?} for {term}") + Err(CorruptionError::new(format!( + "Invalid term encoding {encoded:?} for {term}" + )) + .into()) } } EncodedTerm::BigBigLangStringLiteral { @@ -744,7 +753,10 @@ pub fn insert_term Result<(), StorageError>>( if let Some(language) = literal.language() { insert_str(language_id, language) } else { - unreachable!("Invalid term encoding {encoded:?} for {term}") + Err(CorruptionError::new(format!( + "Invalid term encoding {encoded:?} for {term}" + )) + .into()) } } EncodedTerm::SmallTypedLiteral { datatype_id, .. } => { @@ -775,7 +787,10 @@ pub fn insert_term Result<(), StorageError>>( | EncodedTerm::DurationLiteral(..) | EncodedTerm::YearMonthDurationLiteral(..) | EncodedTerm::DayTimeDurationLiteral(..) => Ok(()), - _ => unreachable!("Invalid term encoding {encoded:?} for {term}"), + _ => Err( + CorruptionError::new(format!("Invalid term encoding {encoded:?} for {term}")) + .into(), + ), }, TermRef::Triple(triple) => { if let EncodedTerm::Triple(encoded) = encoded { @@ -787,7 +802,10 @@ pub fn insert_term Result<(), StorageError>>( )?; insert_term(triple.object.as_ref(), &encoded.object, insert_str) } else { - unreachable!("Invalid term encoding {encoded:?} for {term}") + Err( + CorruptionError::new(format!("Invalid term encoding {encoded:?} for {term}")) + .into(), + ) } } } diff --git a/lib/src/storage/small_string.rs b/lib/src/storage/small_string.rs index a7134bd6..d5d18987 100644 --- a/lib/src/storage/small_string.rs +++ b/lib/src/storage/small_string.rs @@ -46,10 +46,8 @@ impl SmallString { #[inline] #[allow(unsafe_code)] pub fn as_str(&self) -> &str { - unsafe { - // safe because we ensured it in constructors - str::from_utf8_unchecked(self.as_bytes()) - } + // SAFETY: safe because we ensured it in constructors + unsafe { str::from_utf8_unchecked(self.as_bytes()) } } #[inline] diff --git a/lib/src/store.rs b/lib/src/store.rs index f8ea0af5..9bba3c3b 100644 --- a/lib/src/store.rs +++ b/lib/src/store.rs @@ -1598,215 +1598,222 @@ impl BulkLoader { } } -#[test] -fn store() -> Result<(), StorageError> { - use crate::model::*; - - let main_s = Subject::from(BlankNode::default()); - let main_p = NamedNode::new("http://example.com").unwrap(); - let main_o = Term::from(Literal::from(1)); - let main_g = GraphName::from(BlankNode::default()); - - let default_quad = Quad::new( - main_s.clone(), - main_p.clone(), - main_o.clone(), - GraphName::DefaultGraph, - ); - let named_quad = Quad::new( - main_s.clone(), - main_p.clone(), - main_o.clone(), - main_g.clone(), - ); - let default_quads = vec![ - Quad::new( - main_s.clone(), - main_p.clone(), - Literal::from(0), - GraphName::DefaultGraph, - ), - default_quad.clone(), - Quad::new( - main_s.clone(), - main_p.clone(), - Literal::from(200_000_000), - GraphName::DefaultGraph, - ), - ]; - let all_quads = vec![ - Quad::new( +#[cfg(test)] +mod tests { + #![allow(clippy::panic_in_result_fn)] + + use super::*; + + #[test] + fn store() -> Result<(), StorageError> { + use crate::model::*; + + let main_s = Subject::from(BlankNode::default()); + let main_p = NamedNode::new("http://example.com").unwrap(); + let main_o = Term::from(Literal::from(1)); + let main_g = GraphName::from(BlankNode::default()); + + let default_quad = Quad::new( main_s.clone(), main_p.clone(), - Literal::from(0), + main_o.clone(), GraphName::DefaultGraph, - ), - default_quad.clone(), - Quad::new( + ); + let named_quad = Quad::new( main_s.clone(), main_p.clone(), - Literal::from(200_000_000), - GraphName::DefaultGraph, - ), - named_quad.clone(), - ]; - - let store = Store::new()?; - for t in &default_quads { - assert!(store.insert(t)?); - } - assert!(!store.insert(&default_quad)?); - - assert!(store.remove(&default_quad)?); - assert!(!store.remove(&default_quad)?); - assert!(store.insert(&named_quad)?); - assert!(!store.insert(&named_quad)?); - assert!(store.insert(&default_quad)?); - assert!(!store.insert(&default_quad)?); - - assert_eq!(store.len()?, 4); - assert_eq!(store.iter().collect::, _>>()?, all_quads); - assert_eq!( - store - .quads_for_pattern(Some(main_s.as_ref()), None, None, None) - .collect::, _>>()?, - all_quads - ); - assert_eq!( - store - .quads_for_pattern(Some(main_s.as_ref()), Some(main_p.as_ref()), None, None) - .collect::, _>>()?, - all_quads - ); - assert_eq!( - store - .quads_for_pattern( - Some(main_s.as_ref()), - Some(main_p.as_ref()), - Some(main_o.as_ref()), - None - ) - .collect::, _>>()?, - vec![default_quad.clone(), named_quad.clone()] - ); - assert_eq!( - store - .quads_for_pattern( - Some(main_s.as_ref()), - Some(main_p.as_ref()), - Some(main_o.as_ref()), - Some(GraphNameRef::DefaultGraph) - ) - .collect::, _>>()?, - vec![default_quad.clone()] - ); - assert_eq!( - store - .quads_for_pattern( - Some(main_s.as_ref()), - Some(main_p.as_ref()), - Some(main_o.as_ref()), - Some(main_g.as_ref()) - ) - .collect::, _>>()?, - vec![named_quad.clone()] - ); - assert_eq!( - store - .quads_for_pattern( - Some(main_s.as_ref()), - Some(main_p.as_ref()), - None, - Some(GraphNameRef::DefaultGraph) - ) - .collect::, _>>()?, - default_quads - ); - assert_eq!( - store - .quads_for_pattern(Some(main_s.as_ref()), None, Some(main_o.as_ref()), None) - .collect::, _>>()?, - vec![default_quad.clone(), named_quad.clone()] - ); - assert_eq!( - store - .quads_for_pattern( - Some(main_s.as_ref()), - None, - Some(main_o.as_ref()), - Some(GraphNameRef::DefaultGraph) - ) - .collect::, _>>()?, - vec![default_quad.clone()] - ); - assert_eq!( - store - .quads_for_pattern( - Some(main_s.as_ref()), - None, - Some(main_o.as_ref()), - Some(main_g.as_ref()) - ) - .collect::, _>>()?, - vec![named_quad.clone()] - ); - assert_eq!( - store - .quads_for_pattern( - Some(main_s.as_ref()), - None, - None, - Some(GraphNameRef::DefaultGraph) - ) - .collect::, _>>()?, - default_quads - ); - assert_eq!( - store - .quads_for_pattern(None, Some(main_p.as_ref()), None, None) - .collect::, _>>()?, - all_quads - ); - assert_eq!( - store - .quads_for_pattern(None, Some(main_p.as_ref()), Some(main_o.as_ref()), None) - .collect::, _>>()?, - vec![default_quad.clone(), named_quad.clone()] - ); - assert_eq!( - store - .quads_for_pattern(None, None, Some(main_o.as_ref()), None) - .collect::, _>>()?, - vec![default_quad.clone(), named_quad.clone()] - ); - assert_eq!( - store - .quads_for_pattern(None, None, None, Some(GraphNameRef::DefaultGraph)) - .collect::, _>>()?, - default_quads - ); - assert_eq!( - store - .quads_for_pattern( - None, - Some(main_p.as_ref()), - Some(main_o.as_ref()), - Some(GraphNameRef::DefaultGraph) - ) - .collect::, _>>()?, - vec![default_quad] - ); - assert_eq!( - store - .quads_for_pattern( - None, - Some(main_p.as_ref()), - Some(main_o.as_ref()), - Some(main_g.as_ref()) - ) - .collect::, _>>()?, - vec![named_quad] - ); - - Ok(()) + main_o.clone(), + main_g.clone(), + ); + let default_quads = vec![ + Quad::new( + main_s.clone(), + main_p.clone(), + Literal::from(0), + GraphName::DefaultGraph, + ), + default_quad.clone(), + Quad::new( + main_s.clone(), + main_p.clone(), + Literal::from(200_000_000), + GraphName::DefaultGraph, + ), + ]; + let all_quads = vec![ + Quad::new( + main_s.clone(), + main_p.clone(), + Literal::from(0), + GraphName::DefaultGraph, + ), + default_quad.clone(), + Quad::new( + main_s.clone(), + main_p.clone(), + Literal::from(200_000_000), + GraphName::DefaultGraph, + ), + named_quad.clone(), + ]; + + let store = Store::new()?; + for t in &default_quads { + assert!(store.insert(t)?); + } + assert!(!store.insert(&default_quad)?); + + assert!(store.remove(&default_quad)?); + assert!(!store.remove(&default_quad)?); + assert!(store.insert(&named_quad)?); + assert!(!store.insert(&named_quad)?); + assert!(store.insert(&default_quad)?); + assert!(!store.insert(&default_quad)?); + + assert_eq!(store.len()?, 4); + assert_eq!(store.iter().collect::, _>>()?, all_quads); + assert_eq!( + store + .quads_for_pattern(Some(main_s.as_ref()), None, None, None) + .collect::, _>>()?, + all_quads + ); + assert_eq!( + store + .quads_for_pattern(Some(main_s.as_ref()), Some(main_p.as_ref()), None, None) + .collect::, _>>()?, + all_quads + ); + assert_eq!( + store + .quads_for_pattern( + Some(main_s.as_ref()), + Some(main_p.as_ref()), + Some(main_o.as_ref()), + None + ) + .collect::, _>>()?, + vec![default_quad.clone(), named_quad.clone()] + ); + assert_eq!( + store + .quads_for_pattern( + Some(main_s.as_ref()), + Some(main_p.as_ref()), + Some(main_o.as_ref()), + Some(GraphNameRef::DefaultGraph) + ) + .collect::, _>>()?, + vec![default_quad.clone()] + ); + assert_eq!( + store + .quads_for_pattern( + Some(main_s.as_ref()), + Some(main_p.as_ref()), + Some(main_o.as_ref()), + Some(main_g.as_ref()) + ) + .collect::, _>>()?, + vec![named_quad.clone()] + ); + assert_eq!( + store + .quads_for_pattern( + Some(main_s.as_ref()), + Some(main_p.as_ref()), + None, + Some(GraphNameRef::DefaultGraph) + ) + .collect::, _>>()?, + default_quads + ); + assert_eq!( + store + .quads_for_pattern(Some(main_s.as_ref()), None, Some(main_o.as_ref()), None) + .collect::, _>>()?, + vec![default_quad.clone(), named_quad.clone()] + ); + assert_eq!( + store + .quads_for_pattern( + Some(main_s.as_ref()), + None, + Some(main_o.as_ref()), + Some(GraphNameRef::DefaultGraph) + ) + .collect::, _>>()?, + vec![default_quad.clone()] + ); + assert_eq!( + store + .quads_for_pattern( + Some(main_s.as_ref()), + None, + Some(main_o.as_ref()), + Some(main_g.as_ref()) + ) + .collect::, _>>()?, + vec![named_quad.clone()] + ); + assert_eq!( + store + .quads_for_pattern( + Some(main_s.as_ref()), + None, + None, + Some(GraphNameRef::DefaultGraph) + ) + .collect::, _>>()?, + default_quads + ); + assert_eq!( + store + .quads_for_pattern(None, Some(main_p.as_ref()), None, None) + .collect::, _>>()?, + all_quads + ); + assert_eq!( + store + .quads_for_pattern(None, Some(main_p.as_ref()), Some(main_o.as_ref()), None) + .collect::, _>>()?, + vec![default_quad.clone(), named_quad.clone()] + ); + assert_eq!( + store + .quads_for_pattern(None, None, Some(main_o.as_ref()), None) + .collect::, _>>()?, + vec![default_quad.clone(), named_quad.clone()] + ); + assert_eq!( + store + .quads_for_pattern(None, None, None, Some(GraphNameRef::DefaultGraph)) + .collect::, _>>()?, + default_quads + ); + assert_eq!( + store + .quads_for_pattern( + None, + Some(main_p.as_ref()), + Some(main_o.as_ref()), + Some(GraphNameRef::DefaultGraph) + ) + .collect::, _>>()?, + vec![default_quad] + ); + assert_eq!( + store + .quads_for_pattern( + None, + Some(main_p.as_ref()), + Some(main_o.as_ref()), + Some(main_g.as_ref()) + ) + .collect::, _>>()?, + vec![named_quad] + ); + + Ok(()) + } } diff --git a/lib/tests/store.rs b/lib/tests/store.rs index 3deeeaac..e17fb69c 100644 --- a/lib/tests/store.rs +++ b/lib/tests/store.rs @@ -1,3 +1,6 @@ +#![cfg(test)] +#![allow(clippy::panic_in_result_fn)] + use oxigraph::io::RdfFormat; use oxigraph::model::vocab::{rdf, xsd}; use oxigraph::model::*; @@ -8,7 +11,7 @@ use rand::random; use std::env::temp_dir; use std::error::Error; #[cfg(not(target_family = "wasm"))] -use std::fs::{create_dir, remove_dir_all, File}; +use std::fs::{create_dir_all, remove_dir_all, File}; #[cfg(not(target_family = "wasm"))] use std::io::Write; #[cfg(target_os = "linux")] @@ -237,10 +240,10 @@ fn test_dump_dataset() -> Result<(), Box> { #[test] fn test_snapshot_isolation_iterator() -> Result<(), Box> { let quad = QuadRef::new( - NamedNodeRef::new_unchecked("http://example.com/s"), - NamedNodeRef::new_unchecked("http://example.com/p"), - NamedNodeRef::new_unchecked("http://example.com/o"), - NamedNodeRef::new_unchecked("http://www.wikidata.org/wiki/Special:EntityData/Q90"), + NamedNodeRef::new("http://example.com/s")?, + NamedNodeRef::new("http://example.com/p")?, + NamedNodeRef::new("http://example.com/o")?, + NamedNodeRef::new("http://www.wikidata.org/wiki/Special:EntityData/Q90")?, ); let store = Store::new()?; store.insert(quad)?; @@ -274,7 +277,7 @@ fn test_bulk_load_on_existing_delete_overrides_the_delete() -> Result<(), Box Result<(), Box> { let dir = TempDir::default(); - create_dir(&dir.0)?; + create_dir_all(&dir.0)?; { File::create(dir.0.join("CURRENT"))?.write_all(b"foo")?; } @@ -346,7 +349,7 @@ fn test_bad_backup() -> Result<(), Box> { let store_dir = TempDir::default(); let backup_dir = TempDir::default(); - create_dir(&backup_dir.0)?; + create_dir_all(&backup_dir.0)?; assert!(Store::open(&store_dir)?.backup(&backup_dir.0).is_err()); Ok(()) } @@ -430,7 +433,7 @@ fn test_secondary() -> Result<(), Box> { #[cfg(not(target_family = "wasm"))] fn test_open_secondary_bad_dir() -> Result<(), Box> { let primary_dir = TempDir::default(); - create_dir(&primary_dir.0)?; + create_dir_all(&primary_dir.0)?; { File::create(primary_dir.0.join("CURRENT"))?.write_all(b"foo")?; } @@ -491,7 +494,7 @@ fn test_read_only() -> Result<(), Box> { #[cfg(not(target_family = "wasm"))] fn test_open_read_only_bad_dir() -> Result<(), Box> { let dir = TempDir::default(); - create_dir(&dir.0)?; + create_dir_all(&dir.0)?; { File::create(dir.0.join("CURRENT"))?.write_all(b"foo")?; } diff --git a/python/src/io.rs b/python/src/io.rs index f365af50..0b705c11 100644 --- a/python/src/io.rs +++ b/python/src/io.rs @@ -333,14 +333,15 @@ pub fn allow_threads_unsafe(f: impl FnOnce() -> T) -> T { impl Drop for RestoreGuard { fn drop(&mut self) { + // SAFETY: not cloned so called once unsafe { pyo3::ffi::PyEval_RestoreThread(self.tstate); } } } - let _guard = RestoreGuard { - tstate: unsafe { pyo3::ffi::PyEval_SaveThread() }, - }; + // SAFETY: we have the restore part in Drop to make sure it's properly executed + let tstate = unsafe { pyo3::ffi::PyEval_SaveThread() }; + let _guard = RestoreGuard { tstate }; f() } diff --git a/server/Cargo.toml b/server/Cargo.toml index 9e61e54d..01ed8a77 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -4,6 +4,8 @@ version.workspace = true authors.workspace = true license.workspace = true readme = "README.md" +keywords = ["RDF", "SPARQL", "graph-database", "database"] +categories = ["command-line-utilities", "database"] repository = "https://github.com/oxigraph/oxigraph/tree/main/server" homepage = "https://oxigraph.org/server/" description = """ @@ -15,7 +17,7 @@ rust-version.workspace = true [dependencies] anyhow = "1" oxhttp = { version = "0.1", features = ["rayon"] } -clap = { version = "4.0", features = ["derive"] } +clap = { version = "4", features = ["derive"] } oxigraph = { version = "0.4.0-alpha.1-dev", path = "../lib", features = ["http_client"] } sparesults = { version = "0.2.0-alpha.1-dev", path = "../lib/sparesults", features = ["rdf-star"] } rand = "0.8" diff --git a/server/src/main.rs b/server/src/main.rs index 4082ed52..038f5d05 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -642,13 +642,13 @@ pub fn main() -> anyhow::Result<()> { explanation.write_in_json(&mut file)?; }, Some("txt") => { - write!(file, "{:?}", explanation)?; + write!(file, "{explanation:?}")?; }, _ => bail!("The given explanation file {} must have an extension that is .json or .txt", explain_file.display()) } close_file_writer(file)?; } else if explain || stats { - eprintln!("{:#?}", explanation); + eprintln!("{explanation:#?}"); } print_result } @@ -1753,7 +1753,7 @@ impl io::Result>) + 'static> ReadForWrite Result<()> { let mut evaluator = TestEvaluator::default(); register_parser_tests(&mut evaluator); diff --git a/testsuite/src/manifest.rs b/testsuite/src/manifest.rs index fd450fe2..3a70442e 100644 --- a/testsuite/src/manifest.rs +++ b/testsuite/src/manifest.rs @@ -87,9 +87,7 @@ impl TestManifest { let Some(test_node) = self.tests_to_do.pop_front() else { return Ok(None); }; - let test_node = if let Term::NamedNode(test_node) = test_node { - test_node - } else { + let Term::NamedNode(test_node) = test_node else { bail!("Invalid test identifier. Got {test_node}"); }; diff --git a/testsuite/src/sparql_evaluator.rs b/testsuite/src/sparql_evaluator.rs index f440b6ad..318da172 100644 --- a/testsuite/src/sparql_evaluator.rs +++ b/testsuite/src/sparql_evaluator.rs @@ -505,11 +505,12 @@ impl StaticQueryResults { // Hack to normalize literals let store = Store::new()?; for t in graph { - store - .insert(t.in_graph(GraphNameRef::DefaultGraph)) - .unwrap(); + store.insert(t.in_graph(GraphNameRef::DefaultGraph))?; } - let mut graph: Graph = store.iter().map(|q| Triple::from(q.unwrap())).collect(); + let mut graph = store + .iter() + .map(|q| Ok(Triple::from(q?))) + .collect::>()?; if let Some(result_set) = graph.subject_for_predicate_object(rdf::TYPE, rs::RESULT_SET) { if let Some(bool) = graph.object_for_subject_predicate(result_set, rs::BOOLEAN) { @@ -737,11 +738,10 @@ fn evaluate_query_optimization_test(test: &Test) -> Result<()> { .result .as_ref() .ok_or_else(|| anyhow!("No tests result found"))?; - let expected = if let spargebra::Query::Select { pattern, .. } = - spargebra::Query::parse(&read_file_to_string(result)?, Some(result))? - { - pattern - } else { + let spargebra::Query::Select { + pattern: expected, .. + } = spargebra::Query::parse(&read_file_to_string(result)?, Some(result))? + else { bail!("Only SELECT queries are supported in query sparql-optimization tests") }; if expected == actual { diff --git a/testsuite/tests/canonicalization.rs b/testsuite/tests/canonicalization.rs index c6e5f0e2..978902d2 100644 --- a/testsuite/tests/canonicalization.rs +++ b/testsuite/tests/canonicalization.rs @@ -1,3 +1,5 @@ +#![cfg(test)] + use anyhow::Result; use oxigraph_testsuite::check_testsuite; diff --git a/testsuite/tests/oxigraph.rs b/testsuite/tests/oxigraph.rs index a5bc7e0f..b76e5a2a 100644 --- a/testsuite/tests/oxigraph.rs +++ b/testsuite/tests/oxigraph.rs @@ -1,3 +1,5 @@ +#![cfg(test)] + use anyhow::Result; use oxigraph_testsuite::check_testsuite; diff --git a/testsuite/tests/parser.rs b/testsuite/tests/parser.rs index 2846b8f0..9e7141b5 100644 --- a/testsuite/tests/parser.rs +++ b/testsuite/tests/parser.rs @@ -1,3 +1,5 @@ +#![cfg(test)] + use anyhow::Result; use oxigraph_testsuite::check_testsuite; diff --git a/testsuite/tests/serd.rs b/testsuite/tests/serd.rs index ae40d752..eb4910ad 100644 --- a/testsuite/tests/serd.rs +++ b/testsuite/tests/serd.rs @@ -1,3 +1,5 @@ +#![cfg(test)] + use anyhow::Result; use oxigraph_testsuite::check_testsuite; diff --git a/testsuite/tests/sparql.rs b/testsuite/tests/sparql.rs index 74e8c437..eafb80fd 100644 --- a/testsuite/tests/sparql.rs +++ b/testsuite/tests/sparql.rs @@ -1,3 +1,5 @@ +#![cfg(test)] + use anyhow::Result; use oxigraph_testsuite::check_testsuite; From 9da26c6f95e1adbc0b2b68ac6d5c391e1d7b058d Mon Sep 17 00:00:00 2001 From: Tpt Date: Thu, 31 Aug 2023 22:41:12 +0200 Subject: [PATCH 069/217] Makes media type to format more robust Allows more combinations --- lib/oxrdfio/src/format.rs | 43 ++++++---- lib/sparesults/src/lib.rs | 78 ++++++++++++++--- server/src/main.rs | 173 ++++++++++++++++++++++++-------------- 3 files changed, 199 insertions(+), 95 deletions(-) diff --git a/lib/oxrdfio/src/format.rs b/lib/oxrdfio/src/format.rs index 8c4ce230..cb03a3eb 100644 --- a/lib/oxrdfio/src/format.rs +++ b/lib/oxrdfio/src/format.rs @@ -140,25 +140,32 @@ impl RdfFormat { /// ``` #[inline] pub fn from_media_type(media_type: &str) -> Option { - const MEDIA_TYPES: [(&str, RdfFormat); 14] = [ - ("application/n-quads", RdfFormat::NQuads), - ("application/n-triples", RdfFormat::NTriples), - ("application/rdf+xml", RdfFormat::RdfXml), - ("application/trig", RdfFormat::TriG), - ("application/turtle", RdfFormat::Turtle), - ("application/xml", RdfFormat::RdfXml), - ("application/x-trig", RdfFormat::TriG), - ("application/x-turtle", RdfFormat::Turtle), - ("text/n3", RdfFormat::N3), - ("text/nquads", RdfFormat::NQuads), - ("text/plain", RdfFormat::NTriples), - ("text/turtle", RdfFormat::Turtle), - ("text/xml", RdfFormat::RdfXml), - ("text/x-nquads", RdfFormat::NQuads), + const MEDIA_SUBTYPES: [(&str, RdfFormat); 10] = [ + ("n-quads", RdfFormat::NQuads), + ("n-triples", RdfFormat::NTriples), + ("n3", RdfFormat::N3), + ("nquads", RdfFormat::NQuads), + ("ntriples", RdfFormat::NTriples), + ("plain", RdfFormat::NTriples), + ("rdf+xml", RdfFormat::RdfXml), + ("trig", RdfFormat::TriG), + ("turtle", RdfFormat::Turtle), + ("xml", RdfFormat::RdfXml), ]; - let media_type = media_type.split(';').next()?.trim(); - for (candidate_media_type, candidate_id) in MEDIA_TYPES { - if candidate_media_type.eq_ignore_ascii_case(media_type) { + + let (r#type, subtype) = media_type + .split_once(';') + .unwrap_or((media_type, "")) + .0 + .split_once('/')?; + let r#type = r#type.trim(); + if !r#type.eq_ignore_ascii_case("application") && !r#type.eq_ignore_ascii_case("text") { + return None; + } + let subtype = subtype.trim(); + let subtype = subtype.strip_prefix("x-").unwrap_or(subtype); + for (candidate_subtype, candidate_id) in MEDIA_SUBTYPES { + if candidate_subtype.eq_ignore_ascii_case(subtype) { return Some(candidate_id); } } diff --git a/lib/sparesults/src/lib.rs b/lib/sparesults/src/lib.rs index 223dcaa8..7d1a6729 100644 --- a/lib/sparesults/src/lib.rs +++ b/lib/sparesults/src/lib.rs @@ -16,6 +16,7 @@ use crate::json::*; pub use crate::solution::QuerySolution; use crate::xml::*; use oxrdf::{TermRef, Variable, VariableRef}; +use std::fmt; use std::io::{self, BufRead, Write}; use std::rc::Rc; @@ -84,6 +85,23 @@ impl QueryResultsFormat { } } + /// The format name. + /// + /// ``` + /// use sparesults::QueryResultsFormat; + /// + /// assert_eq!(QueryResultsFormat::Json.name(), "SPARQL Results in JSON") + /// ``` + #[inline] + pub const fn name(self) -> &'static str { + match self { + Self::Xml => "SPARQL Results in XML", + Self::Json => "SPARQL Results in JSON", + Self::Csv => "SPARQL Results in CSV", + Self::Tsv => "SPARQL Results in TSV", + } + } + /// Looks for a known format from a media type. /// /// It supports some media type aliases. @@ -97,15 +115,35 @@ impl QueryResultsFormat { /// ``` #[inline] pub fn from_media_type(media_type: &str) -> Option { - match media_type.split(';').next()?.trim() { - "application/sparql-results+xml" | "application/xml" | "text/xml" => Some(Self::Xml), - "application/sparql-results+json" | "application/json" | "text/json" => { - Some(Self::Json) + const MEDIA_SUBTYPES: [(&str, QueryResultsFormat); 8] = [ + ("csv", QueryResultsFormat::Csv), + ("json", QueryResultsFormat::Json), + ("plain", QueryResultsFormat::Csv), + ("sparql-results+json", QueryResultsFormat::Json), + ("sparql-results+xml", QueryResultsFormat::Xml), + ("tab-separated-values", QueryResultsFormat::Tsv), + ("tsv", QueryResultsFormat::Tsv), + ("xml", QueryResultsFormat::Xml), + ]; + + let (r#type, subtype) = media_type + .split_once(';') + .unwrap_or((media_type, "")) + .0 + .trim() + .split_once('/')?; + let r#type = r#type.trim(); + if !r#type.eq_ignore_ascii_case("application") && !r#type.eq_ignore_ascii_case("text") { + return None; + } + let subtype = subtype.trim(); + let subtype = subtype.strip_prefix("x-").unwrap_or(subtype); + for (candidate_subtype, candidate_id) in MEDIA_SUBTYPES { + if candidate_subtype.eq_ignore_ascii_case(subtype) { + return Some(candidate_id); } - "text/csv" => Some(Self::Csv), - "text/tab-separated-values" | "text/tsv" => Some(Self::Tsv), - _ => None, } + None } /// Looks for a known format from an extension. @@ -120,13 +158,27 @@ impl QueryResultsFormat { /// ``` #[inline] pub fn from_extension(extension: &str) -> Option { - match extension { - "srx" | "xml" => Some(Self::Xml), - "srj" | "json" => Some(Self::Json), - "csv" | "txt" => Some(Self::Csv), - "tsv" => Some(Self::Tsv), - _ => None, + const MEDIA_TYPES: [(&str, QueryResultsFormat); 7] = [ + ("csv", QueryResultsFormat::Csv), + ("json", QueryResultsFormat::Json), + ("srj", QueryResultsFormat::Json), + ("srx", QueryResultsFormat::Xml), + ("tsv", QueryResultsFormat::Tsv), + ("txt", QueryResultsFormat::Csv), + ("xml", QueryResultsFormat::Xml), + ]; + for (candidate_extension, candidate_id) in MEDIA_TYPES { + if candidate_extension.eq_ignore_ascii_case(extension) { + return Some(candidate_id); + } } + None + } +} + +impl fmt::Display for QueryResultsFormat { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.name()) } } diff --git a/server/src/main.rs b/server/src/main.rs index 038f5d05..e32a1fe7 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -1525,108 +1525,96 @@ impl From for GraphName { fn rdf_content_negotiation(request: &Request) -> Result { content_negotiation( request, + RdfFormat::from_media_type, + RdfFormat::NQuads, &[ - "application/n-quads", - "application/n-triples", - "application/rdf+xml", - "application/trig", - "application/turtle", - "application/xml", - "application/x-trig", - "application/x-turtle", - "text/n3", - "text/nquads", - "text/plain", - "text/turtle", - "text/xml", - "text/x-nquads", + ("application", RdfFormat::NQuads), + ("text", RdfFormat::NQuads), ], - RdfFormat::from_media_type, + "application/n-quads or text/turtle", ) } fn query_results_content_negotiation(request: &Request) -> Result { content_negotiation( request, + QueryResultsFormat::from_media_type, + QueryResultsFormat::Json, &[ - "application/json", - "application/sparql-results+json", - "application/sparql-results+xml", - "application/xml", - "text/csv", - "text/json", - "text/tab-separated-values", - "text/tsv", - "text/xml", + ("application", QueryResultsFormat::Json), + ("text", QueryResultsFormat::Json), ], - QueryResultsFormat::from_media_type, + "application/sparql-results+json or text/tsv", ) } -fn content_negotiation( +fn content_negotiation( request: &Request, - supported: &[&str], parse: impl Fn(&str) -> Option, + default: F, + default_by_base: &[(&str, F)], + example: &str, ) -> Result { - let default = HeaderValue::default(); + let default_value = HeaderValue::default(); let header = request .header(&HeaderName::ACCEPT) - .unwrap_or(&default) + .unwrap_or(&default_value) .to_str() .map_err(|_| bad_request("The Accept header should be a valid ASCII string"))?; if header.is_empty() { - return parse(supported.first().unwrap()) - .ok_or_else(|| internal_server_error("Unknown media type")); + return Ok(default); } let mut result = None; let mut result_score = 0_f32; - - for possible in header.split(',') { - let (possible, parameters) = possible.split_once(';').unwrap_or((possible, "")); + for mut possible in header.split(',') { + let mut score = 1.; + if let Some((possible_type, last_parameter)) = possible.rsplit_once(';') { + if let Some((name, value)) = last_parameter.split_once('=') { + if name.trim().eq_ignore_ascii_case("q") { + score = f32::from_str(value.trim()).map_err(|_| { + bad_request(format!("Invalid Accept media type score: {value}")) + })?; + possible = possible_type; + } + } + } + if score <= result_score { + continue; + } let (possible_base, possible_sub) = possible + .split_once(';') + .unwrap_or((possible, "")) + .0 .split_once('/') .ok_or_else(|| bad_request(format!("Invalid media type: '{possible}'")))?; let possible_base = possible_base.trim(); let possible_sub = possible_sub.trim(); - let mut score = 1.; - for parameter in parameters.split(';') { - let parameter = parameter.trim(); - if let Some(s) = parameter.strip_prefix("q=") { - score = f32::from_str(s.trim()) - .map_err(|_| bad_request(format!("Invalid Accept media type score: {s}")))? + let mut format = None; + if possible_base == "*" && possible_sub == "*" { + format = Some(default); + } else if possible_sub == "*" { + for (base, sub_format) in default_by_base { + if *base == possible_base { + format = Some(*sub_format); + } } + } else { + format = parse(possible); } - if score <= result_score { - continue; - } - for candidate in supported { - let (candidate_base, candidate_sub) = candidate - .split_once(';') - .map_or(*candidate, |(p, _)| p) - .split_once('/') - .ok_or_else(|| { - internal_server_error(format!("Invalid media type: '{possible}'")) - })?; - if (possible_base == candidate_base || possible_base == "*") - && (possible_sub == candidate_sub || possible_sub == "*") - { - result = Some(candidate); - result_score = score; - break; - } + if let Some(format) = format { + result = Some(format); + result_score = score; } } - let result = result.ok_or_else(|| { + result.ok_or_else(|| { ( Status::NOT_ACCEPTABLE, - format!("The available Content-Types are {}", supported.join(", "),), + format!("The accept header does not provide any accepted format like {example}"), ) - })?; - - parse(result).ok_or_else(|| internal_server_error("Unknown media type")) + }) } fn content_type(request: &Request) -> Option { @@ -2344,6 +2332,21 @@ mod tests { ) } + #[test] + fn get_query_accept_substar() -> Result<()> { + let request = Request::builder( + Method::GET, + "http://localhost/query?query=SELECT%20?s%20?p%20?o%20WHERE%20{%20?s%20?p%20?o%20}" + .parse()?, + ) + .with_header(HeaderName::ACCEPT, "text/*")? + .build(); + ServerTest::new()?.test_body( + request, + "{\"head\":{\"vars\":[\"s\",\"p\",\"o\"]},\"results\":{\"bindings\":[]}}", + ) + } + #[test] fn get_query_accept_good() -> Result<()> { let request = Request::builder( @@ -2366,13 +2369,55 @@ mod tests { fn get_query_accept_bad() -> Result<()> { let request = Request::builder( Method::GET, - "http://localhost/query?query=SELECT%20*%20WHERE%20{%20?s%20?p%20?o%20}".parse()?, + "http://localhost/query?query=SELECT%20?s%20?p%20?o%20WHERE%20{%20?s%20?p%20?o%20}" + .parse()?, ) .with_header(HeaderName::ACCEPT, "application/foo")? .build(); ServerTest::new()?.test_status(request, Status::NOT_ACCEPTABLE) } + #[test] + fn get_query_accept_explicit_priority() -> Result<()> { + let request = Request::builder( + Method::GET, + "http://localhost/query?query=SELECT%20?s%20?p%20?o%20WHERE%20{%20?s%20?p%20?o%20}" + .parse()?, + ) + .with_header(HeaderName::ACCEPT, "text/foo;q=0.5 , text/json ; q = 0.7")? + .build(); + ServerTest::new()?.test_body( + request, + "{\"head\":{\"vars\":[\"s\",\"p\",\"o\"]},\"results\":{\"bindings\":[]}}", + ) + } + + #[test] + fn get_query_accept_implicit_priority() -> Result<()> { + let request = Request::builder( + Method::GET, + "http://localhost/query?query=SELECT%20?s%20?p%20?o%20WHERE%20{%20?s%20?p%20?o%20}" + .parse()?, + ) + .with_header(HeaderName::ACCEPT, "text/json,text/foo")? + .build(); + ServerTest::new()?.test_body( + request, + "{\"head\":{\"vars\":[\"s\",\"p\",\"o\"]},\"results\":{\"bindings\":[]}}", + ) + } + #[test] + fn get_query_accept_implicit_and_explicit_priority() -> Result<()> { + let request = Request::builder( + Method::GET, + "http://localhost/query?query=SELECT%20?s%20?p%20?o%20WHERE%20{%20?s%20?p%20?o%20}" + .parse()?, + ) + .with_header(HeaderName::ACCEPT, "text/foo;q=0.9,text/csv")? + .build(); + ServerTest::new()?.test_body(request, "s,p,o\r\n") + } + #[test] fn get_bad_query() -> Result<()> { ServerTest::new()?.test_status( From 7fe055d2b45690159af41bb0f1b49ed58f099e25 Mon Sep 17 00:00:00 2001 From: Tpt Date: Thu, 31 Aug 2023 18:45:13 +0200 Subject: [PATCH 070/217] Exposes SPARQL results I/O in Oxigraph and improve EvaluationError --- Cargo.lock | 1 - lib/oxrdfio/README.md | 25 ++--- lib/src/io/mod.rs | 26 ++++- lib/src/lib.rs | 12 +-- lib/src/model.rs | 25 +++++ lib/src/sparql/error.rs | 159 ++++++++++++++---------------- lib/src/sparql/eval.rs | 6 +- lib/src/sparql/mod.rs | 4 +- lib/src/sparql/model.rs | 60 ++++++----- lib/src/sparql/results.rs | 47 +++++++++ lib/src/sparql/service.rs | 32 +++--- lib/src/sparql/update.rs | 46 ++++----- python/src/sparql.rs | 18 +++- server/Cargo.toml | 1 - server/src/main.rs | 2 +- testsuite/src/sparql_evaluator.rs | 5 +- 16 files changed, 280 insertions(+), 189 deletions(-) create mode 100644 lib/src/model.rs create mode 100644 lib/src/sparql/results.rs diff --git a/Cargo.lock b/Cargo.lock index fa61e235..50c7f71e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -998,7 +998,6 @@ dependencies = [ "predicates", "rand", "rayon-core", - "sparesults", "url", ] diff --git a/lib/oxrdfio/README.md b/lib/oxrdfio/README.md index 2bdd68d8..318c718c 100644 --- a/lib/oxrdfio/README.md +++ b/lib/oxrdfio/README.md @@ -21,27 +21,28 @@ Support for [SPARQL-star](https://w3c.github.io/rdf-star/cg-spec/2021-12-17.html It is designed as a low level parser compatible with both synchronous and asynchronous I/O (behind the `async-tokio` feature). -Usage example counting the number of people in a Turtle file: +Usage example converting a Turtle file to a N-Triples file: ```rust -use oxrdf::{NamedNodeRef, vocab::rdf}; -use oxrdfio::{RdfFormat, RdfParser}; +use oxrdfio::{RdfFormat, RdfParser, RdfSerializer}; -let file = b"@base . +let turtle_file = b"@base . @prefix schema: . a schema:Person ; schema:name \"Foo\" . a schema:Person ; schema:name \"Bar\" ."; -let schema_person = NamedNodeRef::new("http://schema.org/Person").unwrap(); -let mut count = 0; -for quad in RdfParser::from_format(RdfFormat::Turtle).parse_read(file.as_ref()) { - let quad = quad.unwrap(); - if quad.predicate == rdf::TYPE && quad.object == schema_person.into() { - count += 1; - } +let ntriples_file = b" . + \"Foo\" . + . + \"Bar\" . +"; + +let mut writer = RdfSerializer::from_format(RdfFormat::NTriples).serialize_to_write(Vec::new()); +for quad in RdfParser::from_format(RdfFormat::Turtle).parse_read(turtle_file.as_ref()) { + writer.write_quad(&quad.unwrap()).unwrap(); } -assert_eq!(2, count); +assert_eq!(writer.finish().unwrap(), ntriples_file); ``` ## License diff --git a/lib/src/io/mod.rs b/lib/src/io/mod.rs index f183157d..2e4dd2f4 100644 --- a/lib/src/io/mod.rs +++ b/lib/src/io/mod.rs @@ -1,4 +1,28 @@ -//! Utilities to read and write RDF graphs and datasets. +//! Utilities to read and write RDF graphs and datasets using [OxRDF I/O](https://crates.io/crates/oxrdfio). +//! +//! Usage example converting a Turtle file to a N-Triples file: +//! ``` +//! use oxigraph::io::{RdfFormat, RdfParser, RdfSerializer}; +//! +//! let turtle_file = b"@base . +//! @prefix schema: . +//! a schema:Person ; +//! schema:name \"Foo\" . +//! a schema:Person ; +//! schema:name \"Bar\" ."; +//! +//! let ntriples_file = b" . +//! \"Foo\" . +//! . +//! \"Bar\" . +//! "; +//! +//! let mut writer = RdfSerializer::from_format(RdfFormat::NTriples).serialize_to_write(Vec::new()); +//! for quad in RdfParser::from_format(RdfFormat::Turtle).parse_read(turtle_file.as_ref()) { +//! writer.write_quad(&quad.unwrap()).unwrap(); +//! } +//! assert_eq!(writer.finish().unwrap(), ntriples_file); +//! ``` mod format; pub mod read; diff --git a/lib/src/lib.rs b/lib/src/lib.rs index e5a680d8..b36c4d6c 100644 --- a/lib/src/lib.rs +++ b/lib/src/lib.rs @@ -6,17 +6,7 @@ #![doc(html_logo_url = "https://raw.githubusercontent.com/oxigraph/oxigraph/main/logo.svg")] pub mod io; +pub mod model; pub mod sparql; mod storage; pub mod store; - -pub mod model { - //! Implements data structures for [RDF 1.1 Concepts](https://www.w3.org/TR/rdf11-concepts/) using [OxRDF](https://crates.io/crates/oxrdf). - - pub use oxrdf::{ - dataset, graph, vocab, BlankNode, BlankNodeIdParseError, BlankNodeRef, Dataset, Graph, - GraphName, GraphNameRef, IriParseError, LanguageTagParseError, Literal, LiteralRef, - NamedNode, NamedNodeRef, NamedOrBlankNode, NamedOrBlankNodeRef, Quad, QuadRef, Subject, - SubjectRef, Term, TermParseError, TermRef, Triple, TripleRef, - }; -} diff --git a/lib/src/model.rs b/lib/src/model.rs new file mode 100644 index 00000000..ef8cbd1d --- /dev/null +++ b/lib/src/model.rs @@ -0,0 +1,25 @@ +//! Implements data structures for [RDF 1.1 Concepts](https://www.w3.org/TR/rdf11-concepts/) using [OxRDF](https://crates.io/crates/oxrdf). +//! +//! Usage example: +//! +//! ``` +//! use oxigraph::model::*; +//! +//! let mut graph = Graph::default(); +//! +//! // insertion +//! let ex = NamedNodeRef::new("http://example.com").unwrap(); +//! let triple = TripleRef::new(ex, ex, ex); +//! graph.insert(triple); +//! +//! // simple filter +//! let results: Vec<_> = graph.triples_for_subject(ex).collect(); +//! assert_eq!(vec![triple], results); +//! ``` + +pub use oxrdf::{ + dataset, graph, vocab, BlankNode, BlankNodeIdParseError, BlankNodeRef, Dataset, Graph, + GraphName, GraphNameRef, IriParseError, LanguageTagParseError, Literal, LiteralRef, NamedNode, + NamedNodeRef, NamedOrBlankNode, NamedOrBlankNodeRef, Quad, QuadRef, Subject, SubjectRef, Term, + TermParseError, TermRef, Triple, TripleRef, +}; diff --git a/lib/src/sparql/error.rs b/lib/src/sparql/error.rs index 62e5d821..4728efb7 100644 --- a/lib/src/sparql/error.rs +++ b/lib/src/sparql/error.rs @@ -1,7 +1,10 @@ -use crate::io::{ParseError, SyntaxError}; +use crate::io::ParseError as RdfParseError; +use crate::model::NamedNode; +use crate::sparql::results::ParseError as ResultsParseError; +use crate::sparql::ParseError; use crate::storage::StorageError; use std::convert::Infallible; -use std::error; +use std::error::Error; use std::fmt; use std::io; @@ -10,29 +13,31 @@ use std::io; #[non_exhaustive] pub enum EvaluationError { /// An error in SPARQL parsing. - Parsing(spargebra::ParseError), + Parsing(ParseError), /// An error from the storage. Storage(StorageError), /// An error while parsing an external RDF file. - GraphParsing(SyntaxError), + GraphParsing(RdfParseError), /// An error while parsing an external result file (likely from a federated query). - ResultsParsing(sparesults::ParseError), - /// An error returned during store or results I/Os. - Io(io::Error), - /// An error returned during the query evaluation itself (not supported custom function...). - Query(QueryError), -} - -/// An error returned during the query evaluation itself (not supported custom function...). -#[derive(Debug)] -pub struct QueryError { - inner: QueryErrorKind, -} - -#[derive(Debug)] -enum QueryErrorKind { - Msg { msg: String }, - Other(Box), + ResultsParsing(ResultsParseError), + /// An error returned during results serialization. + ResultsSerialization(io::Error), + /// Error during `SERVICE` evaluation + Service(Box), + /// Error when `CREATE` tries to create an already existing graph + GraphAlreadyExists(NamedNode), + /// Error when `DROP` or `CLEAR` tries to remove a not existing graph + GraphDoesNotExist(NamedNode), + /// The variable storing the `SERVICE` name is unbound + UnboundService, + /// The given `SERVICE` is not supported + UnsupportedService(NamedNode), + /// The given content media type returned from an HTTP response is not supported (`SERVICE` and `LOAD`) + UnsupportedContentType(String), + /// The `SERVICE` call has not returns solutions + ServiceDoesNotReturnSolutions, + /// The results are not a RDF graph + NotAGraph, } impl fmt::Display for EvaluationError { @@ -43,64 +48,50 @@ impl fmt::Display for EvaluationError { Self::Storage(error) => error.fmt(f), Self::GraphParsing(error) => error.fmt(f), Self::ResultsParsing(error) => error.fmt(f), - Self::Io(error) => error.fmt(f), - Self::Query(error) => error.fmt(f), + Self::ResultsSerialization(error) => error.fmt(f), + Self::Service(error) => error.fmt(f), + Self::GraphAlreadyExists(graph) => write!(f, "The graph {graph} already exists"), + Self::GraphDoesNotExist(graph) => write!(f, "The graph {graph} does not exist"), + Self::UnboundService => write!(f, "The variable encoding the service name is unbound"), + Self::UnsupportedService(service) => { + write!(f, "The service {service} is not supported") + } + Self::UnsupportedContentType(content_type) => { + write!(f, "The content media type {content_type} is not supported") + } + Self::ServiceDoesNotReturnSolutions => write!( + f, + "The service is not returning solutions but a boolean or a graph" + ), + Self::NotAGraph => write!(f, "The query results are not a RDF graph"), } } } -impl fmt::Display for QueryError { +impl Error for EvaluationError { #[inline] - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match &self.inner { - QueryErrorKind::Msg { msg } => write!(f, "{msg}"), - QueryErrorKind::Other(error) => error.fmt(f), - } - } -} - -impl error::Error for EvaluationError { - #[inline] - fn source(&self) -> Option<&(dyn error::Error + 'static)> { + fn source(&self) -> Option<&(dyn Error + 'static)> { match self { Self::Parsing(e) => Some(e), Self::Storage(e) => Some(e), Self::GraphParsing(e) => Some(e), Self::ResultsParsing(e) => Some(e), - Self::Io(e) => Some(e), - Self::Query(e) => Some(e), - } - } -} - -impl error::Error for QueryError { - #[inline] - fn source(&self) -> Option<&(dyn error::Error + 'static)> { - match &self.inner { - QueryErrorKind::Msg { .. } => None, - QueryErrorKind::Other(e) => Some(e.as_ref()), + Self::ResultsSerialization(e) => Some(e), + Self::Service(e) => { + let e = Box::as_ref(e); + Some(e) + } + Self::GraphAlreadyExists(_) + | Self::GraphDoesNotExist(_) + | Self::UnboundService + | Self::UnsupportedService(_) + | Self::UnsupportedContentType(_) + | Self::ServiceDoesNotReturnSolutions + | Self::NotAGraph => None, } } } -impl EvaluationError { - /// Wraps another error. - #[inline] - pub(crate) fn wrap(error: impl error::Error + Send + Sync + 'static) -> Self { - Self::Query(QueryError { - inner: QueryErrorKind::Other(Box::new(error)), - }) - } - - /// Builds an error from a printable error message. - #[inline] - pub(crate) fn msg(msg: impl Into) -> Self { - Self::Query(QueryError { - inner: QueryErrorKind::Msg { msg: msg.into() }, - }) - } -} - impl From for EvaluationError { #[inline] fn from(error: Infallible) -> Self { @@ -108,9 +99,9 @@ impl From for EvaluationError { } } -impl From for EvaluationError { +impl From for EvaluationError { #[inline] - fn from(error: spargebra::ParseError) -> Self { + fn from(error: ParseError) -> Self { Self::Parsing(error) } } @@ -122,26 +113,16 @@ impl From for EvaluationError { } } -impl From for EvaluationError { +impl From for EvaluationError { #[inline] - fn from(error: io::Error) -> Self { - Self::Io(error) - } -} - -impl From for EvaluationError { - #[inline] - fn from(error: ParseError) -> Self { - match error { - ParseError::Syntax(error) => Self::GraphParsing(error), - ParseError::Io(error) => Self::Io(error), - } + fn from(error: RdfParseError) -> Self { + Self::GraphParsing(error) } } -impl From for EvaluationError { +impl From for EvaluationError { #[inline] - fn from(error: sparesults::ParseError) -> Self { + fn from(error: ResultsParseError) -> Self { Self::ResultsParsing(error) } } @@ -153,9 +134,19 @@ impl From for io::Error { EvaluationError::Parsing(error) => Self::new(io::ErrorKind::InvalidData, error), EvaluationError::GraphParsing(error) => error.into(), EvaluationError::ResultsParsing(error) => error.into(), - EvaluationError::Io(error) => error, + EvaluationError::ResultsSerialization(error) => error, EvaluationError::Storage(error) => error.into(), - EvaluationError::Query(error) => Self::new(io::ErrorKind::Other, error), + EvaluationError::Service(error) => match error.downcast() { + Ok(error) => *error, + Err(error) => Self::new(io::ErrorKind::Other, error), + }, + EvaluationError::GraphAlreadyExists(_) + | EvaluationError::GraphDoesNotExist(_) + | EvaluationError::UnboundService + | EvaluationError::UnsupportedService(_) + | EvaluationError::UnsupportedContentType(_) + | EvaluationError::ServiceDoesNotReturnSolutions + | EvaluationError::NotAGraph => Self::new(io::ErrorKind::InvalidInput, error), } } } diff --git a/lib/src/sparql/eval.rs b/lib/src/sparql/eval.rs index fb1737d6..ed9feba7 100644 --- a/lib/src/sparql/eval.rs +++ b/lib/src/sparql/eval.rs @@ -1077,7 +1077,7 @@ impl SimpleEvaluator { ) -> Result { let service_name = service_name .get_pattern_value(from) - .ok_or_else(|| EvaluationError::msg("The SERVICE name is not bound"))?; + .ok_or(EvaluationError::UnboundService)?; if let QueryResults::Solutions(iter) = self.service_handler.handle( self.dataset.decode_named_node(&service_name)?, Query { @@ -1092,9 +1092,7 @@ impl SimpleEvaluator { )? { Ok(encode_bindings(Rc::clone(&self.dataset), variables, iter)) } else { - Err(EvaluationError::msg( - "The service call has not returned a set of solutions", - )) + Err(EvaluationError::ServiceDoesNotReturnSolutions) } } diff --git a/lib/src/sparql/mod.rs b/lib/src/sparql/mod.rs index 95249203..6d3ef593 100644 --- a/lib/src/sparql/mod.rs +++ b/lib/src/sparql/mod.rs @@ -8,13 +8,14 @@ mod error; mod eval; mod http; mod model; +pub mod results; mod service; mod update; use crate::model::{NamedNode, Term}; pub use crate::sparql::algebra::{Query, QueryDataset, Update}; use crate::sparql::dataset::DatasetView; -pub use crate::sparql::error::{EvaluationError, QueryError}; +pub use crate::sparql::error::EvaluationError; use crate::sparql::eval::{EvalNodeWithStats, SimpleEvaluator, Timer}; pub use crate::sparql::model::{QueryResults, QuerySolution, QuerySolutionIter, QueryTripleIter}; pub use crate::sparql::service::ServiceHandler; @@ -24,7 +25,6 @@ use crate::storage::StorageReader; use json_event_parser::{JsonEvent, JsonWriter}; pub use oxrdf::{Variable, VariableNameParseError}; use oxsdatatypes::{DayTimeDuration, Float}; -pub use sparesults::QueryResultsFormat; pub use spargebra::ParseError; use sparopt::algebra::GraphPattern; use sparopt::Optimizer; diff --git a/lib/src/sparql/model.rs b/lib/src/sparql/model.rs index d076baec..71a25ea9 100644 --- a/lib/src/sparql/model.rs +++ b/lib/src/sparql/model.rs @@ -1,12 +1,12 @@ use crate::io::{RdfFormat, RdfSerializer}; use crate::model::*; use crate::sparql::error::EvaluationError; -use oxrdf::{Variable, VariableRef}; -pub use sparesults::QuerySolution; -use sparesults::{ +use crate::sparql::results::{ ParseError, QueryResultsFormat, QueryResultsParser, QueryResultsReader, QueryResultsSerializer, SolutionsReader, }; +use oxrdf::{Variable, VariableRef}; +pub use sparesults::QuerySolution; use std::io::{BufRead, Write}; use std::rc::Rc; @@ -38,7 +38,7 @@ impl QueryResults { /// ``` /// use oxigraph::store::Store; /// use oxigraph::model::*; - /// use oxigraph::sparql::QueryResultsFormat; + /// use oxigraph::sparql::results::QueryResultsFormat; /// /// let store = Store::new()?; /// let ex = NamedNodeRef::new("http://example.com")?; @@ -57,33 +57,43 @@ impl QueryResults { let serializer = QueryResultsSerializer::from_format(format); match self { Self::Boolean(value) => { - serializer.write_boolean_result(writer, value)?; + serializer + .write_boolean_result(writer, value) + .map_err(EvaluationError::ResultsSerialization)?; } Self::Solutions(solutions) => { - let mut writer = - serializer.solutions_writer(writer, solutions.variables().to_vec())?; + let mut writer = serializer + .solutions_writer(writer, solutions.variables().to_vec()) + .map_err(EvaluationError::ResultsSerialization)?; for solution in solutions { - writer.write(&solution?)?; + writer + .write(&solution?) + .map_err(EvaluationError::ResultsSerialization)?; } - writer.finish()?; + writer + .finish() + .map_err(EvaluationError::ResultsSerialization)?; } Self::Graph(triples) => { let s = VariableRef::new_unchecked("subject"); let p = VariableRef::new_unchecked("predicate"); let o = VariableRef::new_unchecked("object"); - let mut writer = serializer.solutions_writer( - writer, - vec![s.into_owned(), p.into_owned(), o.into_owned()], - )?; + let mut writer = serializer + .solutions_writer(writer, vec![s.into_owned(), p.into_owned(), o.into_owned()]) + .map_err(EvaluationError::ResultsSerialization)?; for triple in triples { let triple = triple?; - writer.write([ - (s, &triple.subject.into()), - (p, &triple.predicate.into()), - (o, &triple.object), - ])?; + writer + .write([ + (s, &triple.subject.into()), + (p, &triple.predicate.into()), + (o, &triple.object), + ]) + .map_err(EvaluationError::ResultsSerialization)?; } - writer.finish()?; + writer + .finish() + .map_err(EvaluationError::ResultsSerialization)?; } } Ok(()) @@ -116,14 +126,16 @@ impl QueryResults { if let Self::Graph(triples) = self { let mut writer = RdfSerializer::from_format(format.into()).serialize_to_write(write); for triple in triples { - writer.write_triple(&triple?)?; + writer + .write_triple(&triple?) + .map_err(EvaluationError::ResultsSerialization)?; } - writer.finish()?; + writer + .finish() + .map_err(EvaluationError::ResultsSerialization)?; Ok(()) } else { - Err(EvaluationError::msg( - "Bindings or booleans could not be formatted as an RDF graph", - )) + Err(EvaluationError::NotAGraph) } } } diff --git a/lib/src/sparql/results.rs b/lib/src/sparql/results.rs new file mode 100644 index 00000000..26fa287a --- /dev/null +++ b/lib/src/sparql/results.rs @@ -0,0 +1,47 @@ +//! Utilities to read and write RDF results formats using [sparesults](https://crates.io/crates/sparesults). +//! +//! It supports [SPARQL Query Results XML Format (Second Edition)](https://www.w3.org/TR/rdf-sparql-XMLres/), [SPARQL 1.1 Query Results JSON Format](https://www.w3.org/TR/sparql11-results-json/) and [SPARQL 1.1 Query Results CSV and TSV Formats](https://www.w3.org/TR/sparql11-results-csv-tsv/). +//! +//! Usage example converting a JSON result file into a TSV result file: +//! +//! ``` +//! use oxigraph::sparql::results::{QueryResultsFormat, QueryResultsParser, QueryResultsReader, QueryResultsSerializer}; +//! use std::io::Result; +//! +//! fn convert_json_to_tsv(json_file: &[u8]) -> Result> { +//! let json_parser = QueryResultsParser::from_format(QueryResultsFormat::Json); +//! let tsv_serializer = QueryResultsSerializer::from_format(QueryResultsFormat::Tsv); +//! // We start to read the JSON file and see which kind of results it is +//! match json_parser.read_results(json_file)? { +//! QueryResultsReader::Boolean(value) => { +//! // it's a boolean result, we copy it in TSV to the output buffer +//! tsv_serializer.write_boolean_result(Vec::new(), value) +//! }, +//! QueryResultsReader::Solutions(solutions_reader) => { +//! // it's a set of solutions, we create a writer and we write to it while reading in streaming from the JSON file +//! let mut solutions_writer = tsv_serializer.solutions_writer(Vec::new(), solutions_reader.variables().to_vec())?; +//! for solution in solutions_reader { +//! solutions_writer.write(&solution?)?; +//! } +//! solutions_writer.finish() +//! } +//! } +//! } +//! +//! // Let's test with a boolean +//! assert_eq!( +//! convert_json_to_tsv(b"{\"boolean\":true}".as_slice()).unwrap(), +//! b"true" +//! ); +//! +//! // And with a set of solutions +//! assert_eq!( +//! convert_json_to_tsv(b"{\"head\":{\"vars\":[\"foo\",\"bar\"]},\"results\":{\"bindings\":[{\"foo\":{\"type\":\"literal\",\"value\":\"test\"}}]}}".as_slice()).unwrap(), +//! b"?foo\t?bar\n\"test\"\t\n" +//! ); +//! ``` + +pub use sparesults::{ + ParseError, QueryResultsFormat, QueryResultsParser, QueryResultsReader, QueryResultsSerializer, + SolutionsReader, SyntaxError, +}; diff --git a/lib/src/sparql/service.rs b/lib/src/sparql/service.rs index 3d6576e2..ae397ee2 100644 --- a/lib/src/sparql/service.rs +++ b/lib/src/sparql/service.rs @@ -3,7 +3,7 @@ use crate::sparql::algebra::Query; use crate::sparql::error::EvaluationError; use crate::sparql::http::Client; use crate::sparql::model::QueryResults; -use crate::sparql::QueryResultsFormat; +use crate::sparql::results::QueryResultsFormat; use std::error::Error; use std::io::BufReader; use std::time::Duration; @@ -61,10 +61,8 @@ pub struct EmptyServiceHandler; impl ServiceHandler for EmptyServiceHandler { type Error = EvaluationError; - fn handle(&self, _: NamedNode, _: Query) -> Result { - Err(EvaluationError::msg( - "The SERVICE feature is not implemented", - )) + fn handle(&self, name: NamedNode, _: Query) -> Result { + Err(EvaluationError::UnsupportedService(name)) } } @@ -88,7 +86,7 @@ impl ServiceHandler for ErrorConversionServiceHandler { ) -> Result { self.handler .handle(service_name, query) - .map_err(EvaluationError::wrap) + .map_err(|e| EvaluationError::Service(Box::new(e))) } } @@ -112,17 +110,17 @@ impl ServiceHandler for SimpleServiceHandler { service_name: NamedNode, query: Query, ) -> Result { - let (content_type, body) = self.client.post( - service_name.as_str(), - query.to_string().into_bytes(), - "application/sparql-query", - "application/sparql-results+json, application/sparql-results+xml", - )?; - let format = QueryResultsFormat::from_media_type(&content_type).ok_or_else(|| { - EvaluationError::msg(format!( - "Unsupported Content-Type returned by {service_name}: {content_type}" - )) - })?; + let (content_type, body) = self + .client + .post( + service_name.as_str(), + query.to_string().into_bytes(), + "application/sparql-query", + "application/sparql-results+json, application/sparql-results+xml", + ) + .map_err(|e| EvaluationError::Service(Box::new(e)))?; + let format = QueryResultsFormat::from_media_type(&content_type) + .ok_or_else(|| EvaluationError::UnsupportedContentType(content_type))?; Ok(QueryResults::read(BufReader::new(body), format)?) } } diff --git a/lib/src/sparql/update.rs b/lib/src/sparql/update.rs index ef4f9c54..1744c464 100644 --- a/lib/src/sparql/update.rs +++ b/lib/src/sparql/update.rs @@ -17,6 +17,7 @@ use spargebra::term::{ use spargebra::GraphUpdateOperation; use sparopt::Optimizer; use std::collections::HashMap; +use std::io; use std::rc::Rc; pub fn evaluate_update<'a, 'b: 'a>( @@ -71,9 +72,7 @@ impl<'a, 'b: 'a> SimpleUpdateEvaluator<'a, 'b> { } => self.eval_delete_insert( delete, insert, - using_dataset - .as_ref() - .ok_or_else(|| EvaluationError::msg("No dataset"))?, + using_dataset.as_ref().unwrap_or(&QueryDataset::new()), pattern, ), GraphUpdateOperation::Load { @@ -161,15 +160,15 @@ impl<'a, 'b: 'a> SimpleUpdateEvaluator<'a, 'b> { } fn eval_load(&mut self, from: &NamedNode, to: &GraphName) -> Result<(), EvaluationError> { - let (content_type, body) = self.client.get( - from.as_str(), - "application/n-triples, text/turtle, application/rdf+xml", - )?; - let format = RdfFormat::from_media_type(&content_type).ok_or_else(|| { - EvaluationError::msg(format!( - "Unsupported Content-Type returned by {from}: {content_type}" - )) - })?; + let (content_type, body) = self + .client + .get( + from.as_str(), + "application/n-triples, text/turtle, application/rdf+xml", + ) + .map_err(|e| EvaluationError::Service(Box::new(e)))?; + let format = RdfFormat::from_media_type(&content_type) + .ok_or_else(|| EvaluationError::UnsupportedContentType(content_type))?; let to_graph_name = match to { GraphName::NamedNode(graph_name) => graph_name.into(), GraphName::DefaultGraph => GraphNameRef::DefaultGraph, @@ -178,11 +177,12 @@ impl<'a, 'b: 'a> SimpleUpdateEvaluator<'a, 'b> { .rename_blank_nodes() .without_named_graphs() .with_default_graph(to_graph_name); - if let Some(base_iri) = &self.base_iri { - parser = parser.with_base_iri(base_iri.as_str()).map_err(|e| { - EvaluationError::msg(format!("The LOAD IRI '{base_iri}' is invalid: {e}")) - })?; - } + parser = parser.with_base_iri(from.as_str()).map_err(|e| { + EvaluationError::Service(Box::new(io::Error::new( + io::ErrorKind::InvalidInput, + format!("Invalid URL: {from}: {e}"), + ))) + })?; for q in parser.parse_read(body) { self.transaction.insert(q?.as_ref())?; } @@ -193,9 +193,7 @@ impl<'a, 'b: 'a> SimpleUpdateEvaluator<'a, 'b> { if self.transaction.insert_named_graph(graph_name.into())? || silent { Ok(()) } else { - Err(EvaluationError::msg(format!( - "The graph {graph_name} already exists" - ))) + Err(EvaluationError::GraphAlreadyExists(graph_name.clone())) } } @@ -211,9 +209,7 @@ impl<'a, 'b: 'a> SimpleUpdateEvaluator<'a, 'b> { } else if silent { Ok(()) } else { - Err(EvaluationError::msg(format!( - "The graph {graph} does not exists" - ))) + Err(EvaluationError::GraphDoesNotExist(graph_name.clone())) } } GraphTarget::DefaultGraph => { @@ -231,9 +227,7 @@ impl<'a, 'b: 'a> SimpleUpdateEvaluator<'a, 'b> { if self.transaction.remove_named_graph(graph_name.into())? || silent { Ok(()) } else { - Err(EvaluationError::msg(format!( - "The graph {graph_name} does not exists" - ))) + Err(EvaluationError::GraphDoesNotExist(graph_name.clone())) } } GraphTarget::DefaultGraph => { diff --git a/python/src/sparql.rs b/python/src/sparql.rs index 408f6f0c..7c99707b 100644 --- a/python/src/sparql.rs +++ b/python/src/sparql.rs @@ -233,9 +233,21 @@ pub fn map_evaluation_error(error: EvaluationError) -> PyErr { match error { EvaluationError::Parsing(error) => PySyntaxError::new_err(error.to_string()), EvaluationError::Storage(error) => map_storage_error(error), - EvaluationError::Io(error) => map_io_err(error), - EvaluationError::GraphParsing(error) => PySyntaxError::new_err(error.to_string()), - EvaluationError::Query(error) => PyValueError::new_err(error.to_string()), + EvaluationError::GraphParsing(error) => match error { + oxigraph::io::ParseError::Syntax(error) => PySyntaxError::new_err(error.to_string()), + oxigraph::io::ParseError::Io(error) => map_io_err(error), + }, + EvaluationError::ResultsParsing(error) => match error { + oxigraph::sparql::results::ParseError::Syntax(error) => { + PySyntaxError::new_err(error.to_string()) + } + oxigraph::sparql::results::ParseError::Io(error) => map_io_err(error), + }, + EvaluationError::ResultsSerialization(error) => map_io_err(error), + EvaluationError::Service(error) => match error.downcast() { + Ok(error) => map_io_err(*error), + Err(error) => PyRuntimeError::new_err(error.to_string()), + }, _ => PyRuntimeError::new_err(error.to_string()), } } diff --git a/server/Cargo.toml b/server/Cargo.toml index 01ed8a77..350863f9 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -19,7 +19,6 @@ anyhow = "1" oxhttp = { version = "0.1", features = ["rayon"] } clap = { version = "4", features = ["derive"] } oxigraph = { version = "0.4.0-alpha.1-dev", path = "../lib", features = ["http_client"] } -sparesults = { version = "0.2.0-alpha.1-dev", path = "../lib/sparesults", features = ["rdf-star"] } rand = "0.8" url = "2" oxiri = "0.2" diff --git a/server/src/main.rs b/server/src/main.rs index e32a1fe7..817fed04 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -8,12 +8,12 @@ use oxigraph::io::{RdfFormat, RdfParser, RdfSerializer}; use oxigraph::model::{ GraphName, GraphNameRef, IriParseError, NamedNode, NamedNodeRef, NamedOrBlankNode, }; +use oxigraph::sparql::results::{QueryResultsFormat, QueryResultsSerializer}; use oxigraph::sparql::{Query, QueryOptions, QueryResults, Update}; use oxigraph::store::{BulkLoader, LoaderError, Store}; use oxiri::Iri; use rand::random; use rayon_core::ThreadPoolBuilder; -use sparesults::{QueryResultsFormat, QueryResultsSerializer}; use std::borrow::Cow; use std::cell::RefCell; use std::cmp::{max, min}; diff --git a/testsuite/src/sparql_evaluator.rs b/testsuite/src/sparql_evaluator.rs index 318da172..543f0538 100644 --- a/testsuite/src/sparql_evaluator.rs +++ b/testsuite/src/sparql_evaluator.rs @@ -6,6 +6,7 @@ use crate::vocab::*; use anyhow::{anyhow, bail, Result}; use oxigraph::model::vocab::*; use oxigraph::model::*; +use oxigraph::sparql::results::QueryResultsFormat; use oxigraph::sparql::*; use oxigraph::store::Store; use sparopt::Optimizer; @@ -339,10 +340,10 @@ impl ServiceHandler for StaticServiceHandler { self.services .get(&service_name) .ok_or_else(|| { - io::Error::new( + EvaluationError::Service(Box::new(io::Error::new( io::ErrorKind::InvalidInput, format!("Service {service_name} not found"), - ) + ))) })? .query_opt( query, From 1eaa77ad93416f912d21bbf589545705072ee1b7 Mon Sep 17 00:00:00 2001 From: Tpt Date: Fri, 1 Sep 2023 17:10:05 +0200 Subject: [PATCH 071/217] Uses new rustdoc

style --- lib/oxrdf/src/dataset.rs | 5 +++-- lib/oxrdf/src/graph.rs | 5 +++-- lib/oxrdfio/src/serializer.rs | 16 ++++++++-------- lib/oxsdatatypes/src/double.rs | 2 +- lib/oxsdatatypes/src/float.rs | 2 +- lib/sparesults/src/lib.rs | 2 +- lib/src/io/write.rs | 4 ++-- lib/src/sparql/algebra.rs | 2 -- lib/src/store.rs | 34 +++++++++++++++++----------------- 9 files changed, 36 insertions(+), 36 deletions(-) diff --git a/lib/oxrdf/src/dataset.rs b/lib/oxrdf/src/dataset.rs index e5688cb9..017825b0 100644 --- a/lib/oxrdf/src/dataset.rs +++ b/lib/oxrdf/src/dataset.rs @@ -39,8 +39,9 @@ use std::hash::{Hash, Hasher}; /// An in-memory [RDF dataset](https://www.w3.org/TR/rdf11-concepts/#dfn-rdf-dataset). /// /// It can accommodate a fairly large number of quads (in the few millions). -/// Beware: it interns the string and does not do any garbage collection yet: -/// if you insert and remove a lot of different terms, memory will grow without any reduction. +/// +///
It interns the strings and does not do any garbage collection yet: +/// if you insert and remove a lot of different terms, memory will grow without any reduction.
/// /// Usage example: /// ``` diff --git a/lib/oxrdf/src/graph.rs b/lib/oxrdf/src/graph.rs index a40d1293..980e3ebf 100644 --- a/lib/oxrdf/src/graph.rs +++ b/lib/oxrdf/src/graph.rs @@ -29,8 +29,9 @@ use std::fmt; /// An in-memory [RDF graph](https://www.w3.org/TR/rdf11-concepts/#dfn-rdf-graph). /// /// It can accommodate a fairly large number of triples (in the few millions). -/// Beware: it interns the string and does not do any garbage collection yet: -/// if you insert and remove a lot of different terms, memory will grow without any reduction. +/// +///
It interns the string and does not do any garbage collection yet: +/// if you insert and remove a lot of different terms, memory will grow without any reduction.
/// /// Usage example: /// ``` diff --git a/lib/oxrdfio/src/serializer.rs b/lib/oxrdfio/src/serializer.rs index 6289ed28..0d931d81 100644 --- a/lib/oxrdfio/src/serializer.rs +++ b/lib/oxrdfio/src/serializer.rs @@ -62,9 +62,9 @@ impl RdfSerializer { /// Writes to a [`Write`] implementation. /// - /// Warning: Do not forget to run the [`finish`](ToWriteQuadWriter::finish()) method to properly write the last bytes of the file. + ///
Do not forget to run the [`finish`](ToWriteQuadWriter::finish()) method to properly write the last bytes of the file.
/// - /// Warning: This writer does unbuffered writes. You might want to use [`BufWriter`](io::BufWriter) to avoid that. + ///
This writer does unbuffered writes. You might want to use [`BufWriter`](io::BufWriter) to avoid that.
/// /// ``` /// use oxrdfio::{RdfFormat, RdfSerializer}; @@ -107,9 +107,9 @@ impl RdfSerializer { /// Writes to a Tokio [`AsyncWrite`] implementation. /// - /// Warning: Do not forget to run the [`finish`](ToTokioAsyncWriteQuadWriter::finish()) method to properly write the last bytes of the file. + ///
Do not forget to run the [`finish`](ToTokioAsyncWriteQuadWriter::finish()) method to properly write the last bytes of the file.
/// - /// Warning: This writer does unbuffered writes. You might want to use [`BufWriter`](tokio::io::BufWriter) to avoid that. + ///
This writer does unbuffered writes. You might want to use [`BufWriter`](tokio::io::BufWriter) to avoid that.
/// /// ``` /// use oxrdfio::{RdfFormat, RdfSerializer}; @@ -163,9 +163,9 @@ impl RdfSerializer { /// /// Can be built using [`RdfSerializer::serialize_to_write`]. /// -/// Warning: Do not forget to run the [`finish`](ToWriteQuadWriter::finish()) method to properly write the last bytes of the file. +///
Do not forget to run the [`finish`](ToWriteQuadWriter::finish()) method to properly write the last bytes of the file.
/// -/// Warning: This writer does unbuffered writes. You might want to use [`BufWriter`](io::BufWriter) to avoid that. +///
This writer does unbuffered writes. You might want to use [`BufWriter`](io::BufWriter) to avoid that.
/// /// ``` /// use oxrdfio::{RdfFormat, RdfSerializer}; @@ -232,9 +232,9 @@ impl ToWriteQuadWriter { /// /// Can be built using [`RdfSerializer::serialize_to_write`]. /// -/// Warning: Do not forget to run the [`finish`](ToWriteQuadWriter::finish()) method to properly write the last bytes of the file. +///
Do not forget to run the [`finish`](ToWriteQuadWriter::finish()) method to properly write the last bytes of the file.
/// -/// Warning: This writer does unbuffered writes. You might want to use [`BufWriter`](io::BufWriter) to avoid that. +///
This writer does unbuffered writes. You might want to use [`BufWriter`](io::BufWriter) to avoid that.
/// /// ``` /// use oxrdfio::{RdfFormat, RdfSerializer}; diff --git a/lib/oxsdatatypes/src/double.rs b/lib/oxsdatatypes/src/double.rs index d0c77c8a..bc040738 100644 --- a/lib/oxsdatatypes/src/double.rs +++ b/lib/oxsdatatypes/src/double.rs @@ -9,7 +9,7 @@ use std::str::FromStr; /// /// Uses internally a [`f64`]. /// -/// Beware: serialization is currently buggy and do not follow the canonical mapping yet. +///
Serialization does not follow the canonical mapping.
#[derive(Debug, Clone, Copy, Default, PartialEq)] #[repr(transparent)] pub struct Double { diff --git a/lib/oxsdatatypes/src/float.rs b/lib/oxsdatatypes/src/float.rs index 1feac769..996a6401 100644 --- a/lib/oxsdatatypes/src/float.rs +++ b/lib/oxsdatatypes/src/float.rs @@ -9,7 +9,7 @@ use std::str::FromStr; /// /// Uses internally a [`f32`]. /// -/// Beware: serialization is currently buggy and do not follow the canonical mapping yet. +///
Serialization does not follow the canonical mapping.
#[derive(Debug, Clone, Copy, Default, PartialEq)] #[repr(transparent)] pub struct Float { diff --git a/lib/sparesults/src/lib.rs b/lib/sparesults/src/lib.rs index 7d1a6729..5daa2bd9 100644 --- a/lib/sparesults/src/lib.rs +++ b/lib/sparesults/src/lib.rs @@ -479,7 +479,7 @@ impl QueryResultsSerializer { /// Allows writing query results. /// Could be built using a [`QueryResultsSerializer`]. /// -/// Warning: Do not forget to run the [`finish`](SolutionsWriter::finish()) method to properly write the last bytes of the file. +///
Do not forget to run the [`finish`](SolutionsWriter::finish()) method to properly write the last bytes of the file.
/// /// Example in TSV (the API is the same for JSON and XML): /// ``` diff --git a/lib/src/io/write.rs b/lib/src/io/write.rs index 26a8c441..df1cab61 100644 --- a/lib/src/io/write.rs +++ b/lib/src/io/write.rs @@ -55,7 +55,7 @@ impl GraphSerializer { /// Allows writing triples. /// Could be built using a [`GraphSerializer`]. /// -/// Warning: Do not forget to run the [`finish`](TripleWriter::finish()) method to properly write the last bytes of the file. +///
Do not forget to run the [`finish`](TripleWriter::finish()) method to properly write the last bytes of the file.
/// /// ``` /// use oxigraph::io::{GraphFormat, GraphSerializer}; @@ -138,7 +138,7 @@ impl DatasetSerializer { /// Allows writing triples. /// Could be built using a [`DatasetSerializer`]. /// -/// Warning: Do not forget to run the [`finish`](QuadWriter::finish()) method to properly write the last bytes of the file. +///
Do not forget to run the [`finish`](QuadWriter::finish()) method to properly write the last bytes of the file.
/// /// ``` /// use oxigraph::io::{DatasetFormat, DatasetSerializer}; diff --git a/lib/src/sparql/algebra.rs b/lib/src/sparql/algebra.rs index f468e69e..17b6cc05 100644 --- a/lib/src/sparql/algebra.rs +++ b/lib/src/sparql/algebra.rs @@ -1,8 +1,6 @@ //! [SPARQL 1.1 Query Algebra](https://www.w3.org/TR/sparql11-query/#sparqlQuery) //! //! The root type for SPARQL queries is [`Query`] and the root type for updates is [`Update`]. -//! -//! Warning: this implementation is an unstable work in progress use crate::model::*; use crate::sparql::eval::Timer; diff --git a/lib/src/store.rs b/lib/src/store.rs index 9bba3c3b..a6215707 100644 --- a/lib/src/store.rs +++ b/lib/src/store.rs @@ -213,7 +213,7 @@ impl Store { /// Executes a [SPARQL 1.1 query](https://www.w3.org/TR/sparql11-query/) with some options and /// returns a query explanation with some statistics (if enabled with the `with_stats` parameter). /// - /// Beware: if you want to compute statistics you need to exhaust the results iterator before having a look at them. + ///
If you want to compute statistics you need to exhaust the results iterator before having a look at them.
/// /// Usage example serialising the explanation with statistics in JSON: /// ``` @@ -324,7 +324,7 @@ impl Store { /// Returns the number of quads in the store. /// - /// Warning: this function executes a full scan. + ///
This function executes a full scan.
/// /// Usage example: /// ``` @@ -565,7 +565,7 @@ impl Store { /// Adds atomically a set of quads to this store. /// - /// Warning: This operation uses a memory heavy transaction internally, use the [`bulk_loader`](Store::bulk_loader) if you plan to add ten of millions of triples. + ///
This operation uses a memory heavy transaction internally, use the [`bulk_loader`](Store::bulk_loader) if you plan to add ten of millions of triples.
pub fn extend( &self, quads: impl IntoIterator>, @@ -815,7 +815,7 @@ impl Store { /// /// Useful to call after a batch upload or another similar operation. /// - /// Warning: Can take hours on huge databases. + ///
Can take hours on huge databases.
#[cfg(not(target_family = "wasm"))] pub fn optimize(&self) -> Result<(), StorageError> { self.storage.compact() @@ -826,10 +826,10 @@ impl Store { /// After its creation, the backup is usable using [`Store::open`] /// like a regular Oxigraph database and operates independently from the original database. /// - /// Warning: Backups are only possible for on-disk databases created using [`Store::open`]. + ///
Backups are only possible for on-disk databases created using [`Store::open`].
/// Temporary in-memory databases created using [`Store::new`] are not compatible with RocksDB backup system. /// - /// Warning: An error is raised if the `target_directory` already exists. + ///
An error is raised if the `target_directory` already exists.
/// /// If the target directory is in the same file system as the current database, /// the database content will not be fully copied @@ -1012,7 +1012,7 @@ impl<'a> Transaction<'a> { /// Returns the number of quads in the store. /// - /// Warning: this function executes a full scan. + ///
this function executes a full scan.
pub fn len(&self) -> Result { self.writer.reader().len() } @@ -1364,11 +1364,11 @@ impl Iterator for GraphNameIter { /// A bulk loader allowing to load at lot of data quickly into the store. /// -/// Warning: The operations provided here are not atomic. +///
The operations provided here are not atomic.
/// If the operation fails in the middle, only a part of the data may be written to the store. /// Results might get weird if you delete data during the loading process. /// -/// Warning: It is optimized for speed. +///
It is optimized for speed.
/// Memory usage is configurable using [`BulkLoader::set_max_memory_size_in_megabytes`] /// and the number of used threads with [`BulkLoader::set_num_threads`]. /// By default the memory consumption target (excluding the system and RocksDB internal consumption) @@ -1452,11 +1452,11 @@ impl BulkLoader { /// /// This function is optimized for large dataset loading speed. For small files, [`Store::load_dataset`] might be more convenient. /// - /// Warning: This method is not atomic. + ///
This method is not atomic.
/// If the parsing fails in the middle of the file, only a part of it may be written to the store. /// Results might get weird if you delete data during the loading process. /// - /// Warning: This method is optimized for speed. See [the struct](BulkLoader) documentation for more details. + ///
This method is optimized for speed. See [the struct](BulkLoader) documentation for more details.
/// /// Usage example: /// ``` @@ -1510,11 +1510,11 @@ impl BulkLoader { /// /// This function is optimized for large graph loading speed. For small files, [`Store::load_graph`] might be more convenient. /// - /// Warning: This method is not atomic. + ///
This method is not atomic.
/// If the parsing fails in the middle of the file, only a part of it may be written to the store. /// Results might get weird if you delete data during the loading process. /// - /// Warning: This method is optimized for speed. See [the struct](BulkLoader) documentation for more details. + ///
This method is optimized for speed. See [the struct](BulkLoader) documentation for more details.
/// /// Usage example: /// ``` @@ -1570,11 +1570,11 @@ impl BulkLoader { /// Adds a set of quads using the bulk loader. /// - /// Warning: This method is not atomic. + ///
This method is not atomic.
/// If the process fails in the middle of the file, only a part of the data may be written to the store. /// Results might get weird if you delete data during the loading process. /// - /// Warning: This method is optimized for speed. See [the struct](BulkLoader) documentation for more details. + ///
This method is optimized for speed. See [the struct](BulkLoader) documentation for more details.
pub fn load_quads( &self, quads: impl IntoIterator>, @@ -1584,11 +1584,11 @@ impl BulkLoader { /// Adds a set of quads using the bulk loader while breaking in the middle of the process in case of error. /// - /// Warning: This method is not atomic. + ///
This method is not atomic.
/// If the process fails in the middle of the file, only a part of the data may be written to the store. /// Results might get weird if you delete data during the loading process. /// - /// Warning: This method is optimized for speed. See [the struct](BulkLoader) documentation for more details. + ///
This method is optimized for speed. See [the struct](BulkLoader) documentation for more details.
pub fn load_ok_quads + From>( &self, quads: impl IntoIterator, EI>>, From 1d55635fe2bdc248847e6db1abf5a624cf4335d6 Mon Sep 17 00:00:00 2001 From: Tpt Date: Sun, 3 Sep 2023 08:55:44 +0200 Subject: [PATCH 072/217] Migrates from Rome to Biome --- js/README.md | 12 +++++++++++- js/{rome.json => biome.json} | 1 + js/package.json | 6 +++--- js/test/model.mjs | 2 +- js/test/store.mjs | 4 ++-- 5 files changed, 18 insertions(+), 7 deletions(-) rename js/{rome.json => biome.json} (77%) diff --git a/js/README.md b/js/README.md index dafc7d71..dee80ddc 100644 --- a/js/README.md +++ b/js/README.md @@ -252,7 +252,17 @@ The Oxigraph bindings are written in Rust using [the Rust WASM toolkit](https:// The [The Rust Wasm Book](https://rustwasm.github.io/docs/book/) is a great tutorial to get started. -To run the tests of the JS bindings written in JS run `npm test`. +To setup a dev environment: +- ensure to have a Rust toolchain with `rustup` and `cargo` installed ([possible instructions](https://www.rust-lang.org/tools/install)). +- install [`wasm-pack`](https://rustwasm.github.io/wasm-pack/): `cargo install wasm-pack` (it is also in some Linux distribution repositories). +- `npm install` to install pure JS dependencies. +- you are good to go! + +Testing and linting: +- Rust code is formatted with [rustfmt](https://github.com/rust-lang/rustfmt) and linted with [clippy](https://github.com/rust-lang/rust-clippy). + You can execute them with `cargo fmt` and `cargo clippy`. +- JS code is formatted and linted with [Biome](https://biomejs.dev/). `npm run fmt` to auto-format and `npm test` to lint and test. +- Tests are written in JavaScript using [Mocha](https://mochajs.org/) in the `test` directory. `npm test` to run them. ## License diff --git a/js/rome.json b/js/biome.json similarity index 77% rename from js/rome.json rename to js/biome.json index 272422a5..72ae2ce8 100644 --- a/js/rome.json +++ b/js/biome.json @@ -1,4 +1,5 @@ { + "$schema": "https://biomejs.dev/schemas/1.0.0/schema.json", "formatter": { "indentStyle": "space", "indentSize": 4, diff --git a/js/package.json b/js/package.json index e8744994..02577037 100644 --- a/js/package.json +++ b/js/package.json @@ -5,11 +5,11 @@ "devDependencies": { "@rdfjs/data-model": "^2.0.1", "mocha": "^10.0.0", - "rome": "^12.0.0" + "@biomejs/biome": "^1.0.0" }, "scripts": { - "fmt": "rome format . --write && rome check . --apply-unsafe", - "test": "rome ci . && wasm-pack build --debug --target nodejs && mocha", + "fmt": "biome format . --write && biome check . --apply-unsafe", + "test": "biome ci . && wasm-pack build --debug --target nodejs && mocha", "build": "rm -rf pkg && wasm-pack build --release --target web --out-name web && mv pkg pkg-web && wasm-pack build --release --target nodejs --out-name node && mv pkg pkg-node && node build_package.js && rm -r pkg-web && rm -r pkg-node", "release": "npm run build && npm publish ./pkg", "pack": "npm run build && npm pack ./pkg" diff --git a/js/test/model.mjs b/js/test/model.mjs index 37f83199..9e816cfe 100644 --- a/js/test/model.mjs +++ b/js/test/model.mjs @@ -1,8 +1,8 @@ /* global describe, it */ +import assert from "assert"; import runTests from "../node_modules/@rdfjs/data-model/test/index.js"; import oxigraph from "../pkg/oxigraph.js"; -import assert from "assert"; runTests({ factory: oxigraph }); diff --git a/js/test/store.mjs b/js/test/store.mjs index ed50cb40..c73560a0 100644 --- a/js/test/store.mjs +++ b/js/test/store.mjs @@ -1,8 +1,8 @@ /* global describe, it */ -import { Store } from "../pkg/oxigraph.js"; -import dataModel from "@rdfjs/data-model"; import assert from "assert"; +import dataModel from "@rdfjs/data-model"; +import { Store } from "../pkg/oxigraph.js"; const ex = dataModel.namedNode("http://example.com"); const triple = dataModel.quad( From b1c90b599bde4645668342e974ebafd4358091fb Mon Sep 17 00:00:00 2001 From: Tpt Date: Sun, 3 Sep 2023 15:49:01 +0200 Subject: [PATCH 073/217] Testsuite: simplifies error handling --- testsuite/src/parser_evaluator.rs | 60 +++---- testsuite/src/sparql_evaluator.rs | 290 +++++++++++++----------------- 2 files changed, 158 insertions(+), 192 deletions(-) diff --git a/testsuite/src/parser_evaluator.rs b/testsuite/src/parser_evaluator.rs index dbf660da..a32ab084 100644 --- a/testsuite/src/parser_evaluator.rs +++ b/testsuite/src/parser_evaluator.rs @@ -2,7 +2,7 @@ use crate::evaluator::TestEvaluator; use crate::files::{guess_rdf_format, load_dataset, load_n3}; use crate::manifest::Test; use crate::report::dataset_diff; -use anyhow::{anyhow, bail, Result}; +use anyhow::{anyhow, ensure, Result}; use oxigraph::io::RdfFormat; use oxigraph::model::{BlankNode, Dataset, Quad}; use oxttl::n3::{N3Quad, N3Term}; @@ -97,7 +97,7 @@ fn evaluate_positive_syntax_test(test: &Test, format: RdfFormat) -> Result<()> { let action = test .action .as_deref() - .ok_or_else(|| anyhow!("No action found for test {test}"))?; + .ok_or_else(|| anyhow!("No action found"))?; load_dataset(action, format, false).map_err(|e| anyhow!("Parse error: {e}"))?; Ok(()) } @@ -106,7 +106,7 @@ fn evaluate_positive_n3_syntax_test(test: &Test) -> Result<()> { let action = test .action .as_deref() - .ok_or_else(|| anyhow!("No action found for test {test}"))?; + .ok_or_else(|| anyhow!("No action found"))?; load_n3(action, false).map_err(|e| anyhow!("Parse error: {e}"))?; Ok(()) } @@ -115,29 +115,31 @@ fn evaluate_negative_syntax_test(test: &Test, format: RdfFormat) -> Result<()> { let action = test .action .as_deref() - .ok_or_else(|| anyhow!("No action found for test {test}"))?; - match load_dataset(action, format, false) { - Ok(_) => bail!("File parsed without errors even if it should not"), - Err(_) => Ok(()), - } + .ok_or_else(|| anyhow!("No action found"))?; + ensure!( + load_dataset(action, format, false).is_err(), + "File parsed without errors even if it should not" + ); + Ok(()) } fn evaluate_negative_n3_syntax_test(test: &Test) -> Result<()> { let action = test .action .as_deref() - .ok_or_else(|| anyhow!("No action found for test {test}"))?; - match load_n3(action, false) { - Ok(_) => bail!("File parsed without errors even if it should not"), - Err(_) => Ok(()), - } + .ok_or_else(|| anyhow!("No action found"))?; + ensure!( + load_n3(action, false).is_err(), + "File parsed without errors even if it should not" + ); + Ok(()) } fn evaluate_eval_test(test: &Test, format: RdfFormat, ignore_errors: bool) -> Result<()> { let action = test .action .as_deref() - .ok_or_else(|| anyhow!("No action found for test {test}"))?; + .ok_or_else(|| anyhow!("No action found"))?; let mut actual_dataset = load_dataset(action, format, ignore_errors) .map_err(|e| anyhow!("Parse error on file {action}: {e}"))?; actual_dataset.canonicalize(); @@ -148,21 +150,19 @@ fn evaluate_eval_test(test: &Test, format: RdfFormat, ignore_errors: bool) -> Re let mut expected_dataset = load_dataset(results, guess_rdf_format(results)?, false) .map_err(|e| anyhow!("Parse error on file {results}: {e}"))?; expected_dataset.canonicalize(); - if expected_dataset == actual_dataset { - Ok(()) - } else { - bail!( - "The two files are not isomorphic. Diff:\n{}", - dataset_diff(&expected_dataset, &actual_dataset) - ) - } + ensure!( + expected_dataset == actual_dataset, + "The two files are not isomorphic. Diff:\n{}", + dataset_diff(&expected_dataset, &actual_dataset) + ); + Ok(()) } fn evaluate_n3_eval_test(test: &Test, ignore_errors: bool) -> Result<()> { let action = test .action .as_deref() - .ok_or_else(|| anyhow!("No action found for test {test}"))?; + .ok_or_else(|| anyhow!("No action found"))?; let mut actual_dataset = n3_to_dataset( load_n3(action, ignore_errors).map_err(|e| anyhow!("Parse error on file {action}: {e}"))?, ); @@ -175,14 +175,12 @@ fn evaluate_n3_eval_test(test: &Test, ignore_errors: bool) -> Result<()> { load_n3(results, false).map_err(|e| anyhow!("Parse error on file {results}: {e}"))?, ); expected_dataset.canonicalize(); - if expected_dataset == actual_dataset { - Ok(()) - } else { - bail!( - "The two files are not isomorphic. Diff:\n{}", - dataset_diff(&expected_dataset, &actual_dataset) - ) - } + ensure!( + expected_dataset == actual_dataset, + "The two files are not isomorphic. Diff:\n{}", + dataset_diff(&expected_dataset, &actual_dataset) + ); + Ok(()) } fn n3_to_dataset(quads: Vec) -> Dataset { diff --git a/testsuite/src/sparql_evaluator.rs b/testsuite/src/sparql_evaluator.rs index 543f0538..4d01725d 100644 --- a/testsuite/src/sparql_evaluator.rs +++ b/testsuite/src/sparql_evaluator.rs @@ -3,7 +3,7 @@ use crate::files::*; use crate::manifest::*; use crate::report::{dataset_diff, format_diff}; use crate::vocab::*; -use anyhow::{anyhow, bail, Result}; +use anyhow::{anyhow, bail, ensure, Error, Result}; use oxigraph::model::vocab::*; use oxigraph::model::*; use oxigraph::sparql::results::QueryResultsFormat; @@ -51,23 +51,23 @@ pub fn register_sparql_tests(evaluator: &mut TestEvaluator) { ); evaluator.register( "https://github.com/oxigraph/oxigraph/tests#PositiveJsonResultsSyntaxTest", - evaluate_positive_json_result_syntax_test, + |t| evaluate_positive_result_syntax_test(t, QueryResultsFormat::Json), ); evaluator.register( "https://github.com/oxigraph/oxigraph/tests#NegativeJsonResultsSyntaxTest", - evaluate_negative_json_result_syntax_test, + |t| evaluate_negative_result_syntax_test(t, QueryResultsFormat::Json), ); evaluator.register( "https://github.com/oxigraph/oxigraph/tests#PositiveXmlResultsSyntaxTest", - evaluate_positive_xml_result_syntax_test, + |t| evaluate_positive_result_syntax_test(t, QueryResultsFormat::Xml), ); evaluator.register( "https://github.com/oxigraph/oxigraph/tests#NegativeXmlResultsSyntaxTest", - evaluate_negative_xml_result_syntax_test, + |t| evaluate_negative_result_syntax_test(t, QueryResultsFormat::Xml), ); evaluator.register( "https://github.com/oxigraph/oxigraph/tests#NegativeTsvResultsSyntaxTest", - evaluate_negative_tsv_result_syntax_test, + |t| evaluate_negative_result_syntax_test(t, QueryResultsFormat::Tsv), ); evaluator.register( "https://github.com/oxigraph/oxigraph/tests#QueryOptimizationTest", @@ -79,11 +79,11 @@ fn evaluate_positive_syntax_test(test: &Test) -> Result<()> { let query_file = test .action .as_deref() - .ok_or_else(|| anyhow!("No action found for test {test}"))?; + .ok_or_else(|| anyhow!("No action found"))?; let query = Query::parse(&read_file_to_string(query_file)?, Some(query_file)) - .map_err(|e| anyhow!("Not able to parse {test} with error: {e}"))?; + .map_err(|e| anyhow!("Not able to parse with error: {e}"))?; Query::parse(&query.to_string(), None) - .map_err(|e| anyhow!("Failure to deserialize \"{query}\" of {test} with error: {e}"))?; + .map_err(|e| anyhow!("Failure to deserialize \"{query}\" with error: {e}"))?; Ok(()) } @@ -91,49 +91,19 @@ fn evaluate_negative_syntax_test(test: &Test) -> Result<()> { let query_file = test .action .as_deref() - .ok_or_else(|| anyhow!("No action found for test {test}"))?; - match Query::parse(&read_file_to_string(query_file)?, Some(query_file)) { - Ok(result) => { - bail!("Oxigraph parses even if it should not {test}. The output tree is: {result}") - } - Err(_) => Ok(()), - } -} - -fn evaluate_positive_json_result_syntax_test(test: &Test) -> Result<()> { - result_syntax_check(test, QueryResultsFormat::Json) -} - -fn evaluate_negative_json_result_syntax_test(test: &Test) -> Result<()> { - if result_syntax_check(test, QueryResultsFormat::Json).is_ok() { - bail!("Oxigraph parses even if it should not {test}.") - } - Ok(()) -} - -fn evaluate_positive_xml_result_syntax_test(test: &Test) -> Result<()> { - result_syntax_check(test, QueryResultsFormat::Xml) -} - -fn evaluate_negative_xml_result_syntax_test(test: &Test) -> Result<()> { - if result_syntax_check(test, QueryResultsFormat::Xml).is_ok() { - bail!("Oxigraph parses even if it should not {test}.") - } - Ok(()) -} - -fn evaluate_negative_tsv_result_syntax_test(test: &Test) -> Result<()> { - if result_syntax_check(test, QueryResultsFormat::Tsv).is_ok() { - bail!("Oxigraph parses even if it should not {test}.") - } + .ok_or_else(|| anyhow!("No action found"))?; + ensure!( + Query::parse(&read_file_to_string(query_file)?, Some(query_file)).is_err(), + "Oxigraph parses even if it should not." + ); Ok(()) } -fn result_syntax_check(test: &Test, format: QueryResultsFormat) -> Result<()> { +fn evaluate_positive_result_syntax_test(test: &Test, format: QueryResultsFormat) -> Result<()> { let action_file = test .action .as_deref() - .ok_or_else(|| anyhow!("No action found for test {test}"))?; + .ok_or_else(|| anyhow!("No action found"))?; let actual_results = StaticQueryResults::from_query_results( QueryResults::read(Cursor::new(read_file_to_string(action_file)?), format)?, true, @@ -143,16 +113,30 @@ fn result_syntax_check(test: &Test, format: QueryResultsFormat) -> Result<()> { QueryResults::read(Cursor::new(read_file_to_string(result_file)?), format)?, true, )?; - if !are_query_results_isomorphic(&expected_results, &actual_results) { - bail!( - "Failure on {test}.\n{}\n", - results_diff(expected_results, actual_results), - ); - } + ensure!( + are_query_results_isomorphic(&expected_results, &actual_results), + "Not isomorphic results:\n{}\n", + results_diff(expected_results, actual_results), + ); } Ok(()) } +fn evaluate_negative_result_syntax_test(test: &Test, format: QueryResultsFormat) -> Result<()> { + let action_file = test + .action + .as_deref() + .ok_or_else(|| anyhow!("No action found"))?; + ensure!( + QueryResults::read(Cursor::new(read_file_to_string(action_file)?), format) + .map_err(Error::from) + .and_then(|r| { StaticQueryResults::from_query_results(r, true) }) + .is_err(), + "Oxigraph parses even if it should not." + ); + Ok(()) +} + fn evaluate_evaluation_test(test: &Test) -> Result<()> { let store = Store::new()?; if let Some(data) = &test.data { @@ -164,36 +148,34 @@ fn evaluate_evaluation_test(test: &Test) -> Result<()> { let query_file = test .query .as_deref() - .ok_or_else(|| anyhow!("No action found for test {test}"))?; + .ok_or_else(|| anyhow!("No action found"))?; let options = QueryOptions::default() .with_service_handler(StaticServiceHandler::new(&test.service_data)?); let query = Query::parse(&read_file_to_string(query_file)?, Some(query_file)) - .map_err(|e| anyhow!("Failure to parse query of {test} with error: {e}"))?; + .map_err(|e| anyhow!("Failure to parse query with error: {e}"))?; // We check parsing roundtrip Query::parse(&query.to_string(), None) - .map_err(|e| anyhow!("Failure to deserialize \"{query}\" of {test} with error: {e}"))?; + .map_err(|e| anyhow!("Failure to deserialize \"{query}\" with error: {e}"))?; // FROM and FROM NAMED support. We make sure the data is in the store if !query.dataset().is_default_dataset() { for graph_name in query.dataset().default_graph_graphs().unwrap_or(&[]) { - if let GraphName::NamedNode(graph_name) = graph_name { - load_graph_to_store(graph_name.as_str(), &store, graph_name.as_ref())?; - } else { - bail!("Invalid FROM in query {query} for test {test}"); - } + let GraphName::NamedNode(graph_name) = graph_name else { + bail!("Invalid FROM in query {query}"); + }; + load_graph_to_store(graph_name.as_str(), &store, graph_name.as_ref())?; } for graph_name in query.dataset().available_named_graphs().unwrap_or(&[]) { - if let NamedOrBlankNode::NamedNode(graph_name) = graph_name { - load_graph_to_store(graph_name.as_str(), &store, graph_name.as_ref())?; - } else { - bail!("Invalid FROM NAMED in query {query} for test {test}"); - } + let NamedOrBlankNode::NamedNode(graph_name) = graph_name else { + bail!("Invalid FROM NAMED in query {query}"); + }; + load_graph_to_store(graph_name.as_str(), &store, graph_name.as_ref())?; } } let expected_results = load_sparql_query_result(test.result.as_ref().unwrap()) - .map_err(|e| anyhow!("Error constructing expected graph for {test}: {e}"))?; + .map_err(|e| anyhow!("Error constructing expected graph: {e}"))?; let with_order = if let StaticQueryResults::Solutions { ordered, .. } = &expected_results { *ordered } else { @@ -207,16 +189,15 @@ fn evaluate_evaluation_test(test: &Test) -> Result<()> { } let actual_results = store .query_opt(query.clone(), options) - .map_err(|e| anyhow!("Failure to execute query of {test} with error: {e}"))?; + .map_err(|e| anyhow!("Failure to execute query with error: {e}"))?; let actual_results = StaticQueryResults::from_query_results(actual_results, with_order)?; - if !are_query_results_isomorphic(&expected_results, &actual_results) { - bail!( - "Failure on {test}.\n{}\nParsed query:\n{}\nData:\n{store}\n", - results_diff(expected_results, actual_results), - Query::parse(&read_file_to_string(query_file)?, Some(query_file)).unwrap() - ); - } + ensure!( + are_query_results_isomorphic(&expected_results, &actual_results), + "Not isomorphic results.\n{}\nParsed query:\n{}\nData:\n{store}\n", + results_diff(expected_results, actual_results), + Query::parse(&read_file_to_string(query_file)?, Some(query_file)).unwrap() + ); } Ok(()) } @@ -225,11 +206,11 @@ fn evaluate_positive_update_syntax_test(test: &Test) -> Result<()> { let update_file = test .action .as_deref() - .ok_or_else(|| anyhow!("No action found for test {test}"))?; + .ok_or_else(|| anyhow!("No action found"))?; let update = Update::parse(&read_file_to_string(update_file)?, Some(update_file)) - .map_err(|e| anyhow!("Not able to parse {test} with error: {e}"))?; + .map_err(|e| anyhow!("Not able to parse with error: {e}"))?; Update::parse(&update.to_string(), None) - .map_err(|e| anyhow!("Failure to deserialize \"{update}\" of {test} with error: {e}"))?; + .map_err(|e| anyhow!("Failure to deserialize \"{update}\" with error: {e}"))?; Ok(()) } @@ -237,13 +218,12 @@ fn evaluate_negative_update_syntax_test(test: &Test) -> Result<()> { let update_file = test .action .as_deref() - .ok_or_else(|| anyhow!("No action found for test {test}"))?; - match Update::parse(&read_file_to_string(update_file)?, Some(update_file)) { - Ok(result) => { - bail!("Oxigraph parses even if it should not {test}. The output tree is: {result}") - } - Err(_) => Ok(()), - } + .ok_or_else(|| anyhow!("No action found"))?; + ensure!( + Update::parse(&read_file_to_string(update_file)?, Some(update_file)).is_err(), + "Oxigraph parses even if it should not." + ); + Ok(()) } fn evaluate_update_evaluation_test(test: &Test) -> Result<()> { @@ -266,30 +246,28 @@ fn evaluate_update_evaluation_test(test: &Test) -> Result<()> { let update_file = test .update .as_deref() - .ok_or_else(|| anyhow!("No action found for test {test}"))?; + .ok_or_else(|| anyhow!("No action found"))?; let update = Update::parse(&read_file_to_string(update_file)?, Some(update_file)) - .map_err(|e| anyhow!("Failure to parse update of {test} with error: {e}"))?; + .map_err(|e| anyhow!("Failure to parse update with error: {e}"))?; // We check parsing roundtrip Update::parse(&update.to_string(), None) - .map_err(|e| anyhow!("Failure to deserialize \"{update}\" of {test} with error: {e}"))?; + .map_err(|e| anyhow!("Failure to deserialize \"{update}\" with error: {e}"))?; store .update(update) - .map_err(|e| anyhow!("Failure to execute update of {test} with error: {e}"))?; + .map_err(|e| anyhow!("Failure to execute update with error: {e}"))?; let mut store_dataset: Dataset = store.iter().collect::>()?; store_dataset.canonicalize(); let mut result_store_dataset: Dataset = result_store.iter().collect::>()?; result_store_dataset.canonicalize(); - if store_dataset == result_store_dataset { - Ok(()) - } else { - bail!( - "Failure on {test}.\nDiff:\n{}\nParsed update:\n{}\n", - dataset_diff(&result_store_dataset, &store_dataset), - Update::parse(&read_file_to_string(update_file)?, Some(update_file)).unwrap(), - ) - } + ensure!( + store_dataset == result_store_dataset, + "Not isomorphic result dataset.\nDiff:\n{}\nParsed update:\n{}\n", + dataset_diff(&result_store_dataset, &store_dataset), + Update::parse(&read_file_to_string(update_file)?, Some(update_file)).unwrap(), + ); + Ok(()) } fn load_sparql_query_result(url: &str) -> Result { @@ -522,11 +500,10 @@ impl StaticQueryResults { let mut variables: Vec = graph .objects_for_subject_predicate(result_set, rs::RESULT_VARIABLE) .map(|object| { - if let TermRef::Literal(l) = object { - Ok(Variable::new_unchecked(l.value())) - } else { + let TermRef::Literal(l) = object else { bail!("Invalid rs:resultVariable: {object}") - } + }; + Ok(Variable::new_unchecked(l.value())) }) .collect::>>()?; variables.sort(); @@ -534,45 +511,38 @@ impl StaticQueryResults { let mut solutions = graph .objects_for_subject_predicate(result_set, rs::SOLUTION) .map(|object| { - if let TermRef::BlankNode(solution) = object { - let mut bindings = graph - .objects_for_subject_predicate(solution, rs::BINDING) - .map(|object| { - if let TermRef::BlankNode(binding) = object { - if let (Some(TermRef::Literal(variable)), Some(value)) = ( - graph.object_for_subject_predicate( - binding, - rs::VARIABLE, - ), - graph.object_for_subject_predicate(binding, rs::VALUE), - ) { - Ok(( - Variable::new_unchecked(variable.value()), - value.into_owned(), - )) - } else { - bail!("Invalid rs:binding: {binding}") - } - } else { - bail!("Invalid rs:binding: {object}") - } - }) - .collect::>>()?; - bindings.sort_by(|(a, _), (b, _)| a.cmp(b)); - let index = graph - .object_for_subject_predicate(solution, rs::INDEX) - .map(|object| { - if let TermRef::Literal(l) = object { - Ok(u64::from_str(l.value())?) - } else { - bail!("Invalid rs:index: {object}") - } - }) - .transpose()?; - Ok((bindings, index)) - } else { + let TermRef::BlankNode(solution) = object else { bail!("Invalid rs:solution: {object}") - } + }; + let mut bindings = graph + .objects_for_subject_predicate(solution, rs::BINDING) + .map(|object| { + let TermRef::BlankNode(binding) = object else { + bail!("Invalid rs:binding: {object}") + }; + let (Some(TermRef::Literal(variable)), Some(value)) = ( + graph.object_for_subject_predicate(binding, rs::VARIABLE), + graph.object_for_subject_predicate(binding, rs::VALUE), + ) else { + bail!("Invalid rs:binding: {binding}") + }; + Ok(( + Variable::new_unchecked(variable.value()), + value.into_owned(), + )) + }) + .collect::>>()?; + bindings.sort_by(|(a, _), (b, _)| a.cmp(b)); + let index = graph + .object_for_subject_predicate(solution, rs::INDEX) + .map(|object| { + let TermRef::Literal(l) = object else { + bail!("Invalid rs:index: {object}") + }; + Ok(u64::from_str(l.value())?) + }) + .transpose()?; + Ok((bindings, index)) }) .collect::>>()?; solutions.sort_by(|(_, index_a), (_, index_b)| index_a.cmp(index_b)); @@ -723,7 +693,7 @@ fn evaluate_query_optimization_test(test: &Test) -> Result<()> { let action = test .action .as_deref() - .ok_or_else(|| anyhow!("No action found for test {test}"))?; + .ok_or_else(|| anyhow!("No action found"))?; let actual = (&Optimizer::optimize_graph_pattern( (&if let spargebra::Query::Select { pattern, .. } = spargebra::Query::parse(&read_file_to_string(action)?, Some(action))? @@ -745,26 +715,24 @@ fn evaluate_query_optimization_test(test: &Test) -> Result<()> { else { bail!("Only SELECT queries are supported in query sparql-optimization tests") }; - if expected == actual { - Ok(()) - } else { - bail!( - "Failure on {test}.\nDiff:\n{}\n", - format_diff( - &spargebra::Query::Select { - pattern: expected, - dataset: None, - base_iri: None - } - .to_sse(), - &spargebra::Query::Select { - pattern: actual, - dataset: None, - base_iri: None - } - .to_sse(), - "query" - ) + ensure!( + expected == actual, + "Not equal queries.\nDiff:\n{}\n", + format_diff( + &spargebra::Query::Select { + pattern: expected, + dataset: None, + base_iri: None + } + .to_sse(), + &spargebra::Query::Select { + pattern: actual, + dataset: None, + base_iri: None + } + .to_sse(), + "query" ) - } + ); + Ok(()) } From 8193cac86dcd72c2ce23a601834bf1974c21de94 Mon Sep 17 00:00:00 2001 From: Tpt Date: Sun, 3 Sep 2023 16:07:18 +0200 Subject: [PATCH 074/217] Testsuite: avoid creating and dropping Stores --- testsuite/src/sparql_evaluator.rs | 56 ++++++++++++++++++++++++++----- 1 file changed, 47 insertions(+), 9 deletions(-) diff --git a/testsuite/src/sparql_evaluator.rs b/testsuite/src/sparql_evaluator.rs index 4d01725d..67081be6 100644 --- a/testsuite/src/sparql_evaluator.rs +++ b/testsuite/src/sparql_evaluator.rs @@ -13,8 +13,9 @@ use sparopt::Optimizer; use std::collections::HashMap; use std::fmt::Write; use std::io::{self, BufReader, Cursor}; +use std::ops::Deref; use std::str::FromStr; -use std::sync::Arc; +use std::sync::{Arc, Mutex, OnceLock}; pub fn register_sparql_tests(evaluator: &mut TestEvaluator) { evaluator.register( @@ -138,7 +139,7 @@ fn evaluate_negative_result_syntax_test(test: &Test, format: QueryResultsFormat) } fn evaluate_evaluation_test(test: &Test) -> Result<()> { - let store = Store::new()?; + let store = get_store()?; if let Some(data) = &test.data { load_dataset_to_store(data, &store)?; } @@ -194,9 +195,10 @@ fn evaluate_evaluation_test(test: &Test) -> Result<()> { ensure!( are_query_results_isomorphic(&expected_results, &actual_results), - "Not isomorphic results.\n{}\nParsed query:\n{}\nData:\n{store}\n", + "Not isomorphic results.\n{}\nParsed query:\n{}\nData:\n{}\n", results_diff(expected_results, actual_results), - Query::parse(&read_file_to_string(query_file)?, Some(query_file)).unwrap() + Query::parse(&read_file_to_string(query_file)?, Some(query_file)).unwrap(), + &*store ); } Ok(()) @@ -227,7 +229,7 @@ fn evaluate_negative_update_syntax_test(test: &Test) -> Result<()> { } fn evaluate_update_evaluation_test(test: &Test) -> Result<()> { - let store = Store::new()?; + let store = get_store()?; if let Some(data) = &test.data { load_dataset_to_store(data, &store)?; } @@ -235,7 +237,7 @@ fn evaluate_update_evaluation_test(test: &Test) -> Result<()> { load_graph_to_store(value, &store, name.clone())?; } - let result_store = Store::new()?; + let result_store = get_store()?; if let Some(data) = &test.result { load_dataset_to_store(data, &result_store)?; } @@ -286,7 +288,7 @@ fn load_sparql_query_result(url: &str) -> Result { #[derive(Clone)] struct StaticServiceHandler { - services: Arc>, + services: Arc>, } impl StaticServiceHandler { @@ -297,7 +299,7 @@ impl StaticServiceHandler { .iter() .map(|(name, data)| { let name = NamedNode::new(name)?; - let store = Store::new()?; + let store = get_store()?; load_dataset_to_store(data, &store)?; Ok((name, store)) }) @@ -482,7 +484,7 @@ impl StaticQueryResults { fn from_graph(graph: &Graph) -> Result { // Hack to normalize literals - let store = Store::new()?; + let store = get_store()?; for t in graph { store.insert(t.in_graph(GraphNameRef::DefaultGraph))?; } @@ -736,3 +738,39 @@ fn evaluate_query_optimization_test(test: &Test) -> Result<()> { ); Ok(()) } + +// Pool of stores to avoid allocating/deallocating them a lot +static STORE_POOL: OnceLock>> = OnceLock::new(); + +fn get_store() -> Result { + let store = if let Some(store) = STORE_POOL.get_or_init(Mutex::default).lock().unwrap().pop() { + store + } else { + Store::new()? + }; + Ok(StoreRef { store }) +} + +struct StoreRef { + store: Store, +} + +impl Drop for StoreRef { + fn drop(&mut self) { + if self.store.clear().is_ok() { + STORE_POOL + .get_or_init(Mutex::default) + .lock() + .unwrap() + .push(self.store.clone()) + } + } +} + +impl Deref for StoreRef { + type Target = Store; + + fn deref(&self) -> &Store { + &self.store + } +} From 13c3515d7b1b537dd08ef0e4391afa3e347d14f9 Mon Sep 17 00:00:00 2001 From: Tpt Date: Sat, 2 Sep 2023 17:12:43 +0200 Subject: [PATCH 075/217] OxTTL: return file position in errors --- fuzz/fuzz_targets/nquads.rs | 10 +- fuzz/fuzz_targets/trig.rs | 10 +- lib/oxrdfio/src/error.rs | 34 +- lib/oxrdfxml/src/error.rs | 12 - lib/oxrdfxml/src/parser.rs | 6 +- lib/oxttl/src/lexer.rs | 6 +- lib/oxttl/src/lib.rs | 2 +- lib/oxttl/src/line_formats.rs | 8 +- lib/oxttl/src/n3.rs | 4 +- lib/oxttl/src/terse.rs | 20 +- lib/oxttl/src/toolkit/error.rs | 132 +++++++ lib/oxttl/src/toolkit/lexer.rs | 344 ++++++++++++------ lib/oxttl/src/toolkit/mod.rs | 8 +- lib/oxttl/src/toolkit/parser.rs | 142 +------- .../parser-error/invalid_iri.nt | 2 + .../parser-error/invalid_iri_comment.nt | 2 + .../parser-error/invalid_iri_comment_crlf.nt | 2 + .../parser-error/invalid_iri_crlf.nt | 2 + .../parser-error/invalid_iri_error.txt | 1 + .../parser-error/invalid_predicate.nt | 2 + .../parser-error/invalid_predicate_error.txt | 1 + .../parser-error/invalid_string_escape.nt | 1 + .../invalid_string_escape_error.txt | 1 + .../oxigraph-tests/parser-error/manifest.ttl | 66 ++++ .../parser-error/unexpected_eof.nt | 2 + .../parser-error/unexpected_eof_crlf.nt | 2 + .../parser-error/unexpected_eof_error.txt | 1 + testsuite/src/parser_evaluator.rs | 21 +- testsuite/tests/oxigraph.rs | 8 + 29 files changed, 552 insertions(+), 300 deletions(-) create mode 100644 lib/oxttl/src/toolkit/error.rs create mode 100644 testsuite/oxigraph-tests/parser-error/invalid_iri.nt create mode 100644 testsuite/oxigraph-tests/parser-error/invalid_iri_comment.nt create mode 100644 testsuite/oxigraph-tests/parser-error/invalid_iri_comment_crlf.nt create mode 100644 testsuite/oxigraph-tests/parser-error/invalid_iri_crlf.nt create mode 100644 testsuite/oxigraph-tests/parser-error/invalid_iri_error.txt create mode 100644 testsuite/oxigraph-tests/parser-error/invalid_predicate.nt create mode 100644 testsuite/oxigraph-tests/parser-error/invalid_predicate_error.txt create mode 100644 testsuite/oxigraph-tests/parser-error/invalid_string_escape.nt create mode 100644 testsuite/oxigraph-tests/parser-error/invalid_string_escape_error.txt create mode 100644 testsuite/oxigraph-tests/parser-error/manifest.ttl create mode 100644 testsuite/oxigraph-tests/parser-error/unexpected_eof.nt create mode 100644 testsuite/oxigraph-tests/parser-error/unexpected_eof_crlf.nt create mode 100644 testsuite/oxigraph-tests/parser-error/unexpected_eof_error.txt diff --git a/fuzz/fuzz_targets/nquads.rs b/fuzz/fuzz_targets/nquads.rs index a7de4913..c964e229 100644 --- a/fuzz/fuzz_targets/nquads.rs +++ b/fuzz/fuzz_targets/nquads.rs @@ -2,9 +2,9 @@ use libfuzzer_sys::fuzz_target; use oxrdf::Quad; -use oxttl::{NQuadsParser, NQuadsSerializer, SyntaxError}; +use oxttl::{NQuadsParser, NQuadsSerializer}; -fn parse<'a>(chunks: impl IntoIterator) -> (Vec, Vec) { +fn parse<'a>(chunks: impl IntoIterator) -> (Vec, Vec) { let mut quads = Vec::new(); let mut errors = Vec::new(); let mut parser = NQuadsParser::new().with_quoted_triples().parse(); @@ -13,7 +13,7 @@ fn parse<'a>(chunks: impl IntoIterator) -> (Vec, Vec quads.push(quad), - Err(error) => errors.push(error), + Err(error) => errors.push(error.to_string()), } } } @@ -21,7 +21,7 @@ fn parse<'a>(chunks: impl IntoIterator) -> (Vec, Vec quads.push(quad), - Err(error) => errors.push(error), + Err(error) => errors.push(error.to_string()), } } assert!(parser.is_end()); @@ -39,7 +39,7 @@ fuzz_target!(|data: &[u8]| { .collect::>() .as_slice()]); assert_eq!(quads, quads_without_split); - assert_eq!(errors.len(), errors_without_split.len()); + assert_eq!(errors, errors_without_split); // We serialize let mut writer = NQuadsSerializer::new().serialize_to_write(Vec::new()); diff --git a/fuzz/fuzz_targets/trig.rs b/fuzz/fuzz_targets/trig.rs index 6a930a97..c0713e69 100644 --- a/fuzz/fuzz_targets/trig.rs +++ b/fuzz/fuzz_targets/trig.rs @@ -2,9 +2,9 @@ use libfuzzer_sys::fuzz_target; use oxrdf::{Dataset, GraphName, Quad, Subject, Term, Triple}; -use oxttl::{SyntaxError, TriGParser, TriGSerializer}; +use oxttl::{TriGParser, TriGSerializer}; -fn parse<'a>(chunks: impl IntoIterator) -> (Vec, Vec) { +fn parse<'a>(chunks: impl IntoIterator) -> (Vec, Vec) { let mut quads = Vec::new(); let mut errors = Vec::new(); let mut parser = TriGParser::new() @@ -17,7 +17,7 @@ fn parse<'a>(chunks: impl IntoIterator) -> (Vec, Vec quads.push(quad), - Err(error) => errors.push(error), + Err(error) => errors.push(error.to_string()), } } } @@ -25,7 +25,7 @@ fn parse<'a>(chunks: impl IntoIterator) -> (Vec, Vec quads.push(quad), - Err(error) => errors.push(error), + Err(error) => errors.push(error.to_string()), } } assert!(parser.is_end()); @@ -96,7 +96,7 @@ fuzz_target!(|data: &[u8]| { String::from_utf8_lossy(&serialize_quads(&quads_without_split)) ); } - assert_eq!(errors.len(), errors_without_split.len()); + assert_eq!(errors, errors_without_split); // We serialize let new_serialization = serialize_quads(&quads); diff --git a/lib/oxrdfio/src/error.rs b/lib/oxrdfio/src/error.rs index ac8173a7..235ba1b7 100644 --- a/lib/oxrdfio/src/error.rs +++ b/lib/oxrdfio/src/error.rs @@ -1,4 +1,5 @@ use std::error::Error; +use std::ops::Range; use std::{fmt, io}; /// Error returned during RDF format parsing. @@ -110,10 +111,33 @@ pub struct SyntaxError { enum SyntaxErrorKind { Turtle(oxttl::SyntaxError), RdfXml(oxrdfxml::SyntaxError), - Msg { msg: &'static str }, } +impl SyntaxError { + /// The location of the error inside of the file. + #[inline] + pub fn location(&self) -> Option> { + match &self.inner { + SyntaxErrorKind::Turtle(e) => { + let location = e.location(); + Some( + TextPosition { + line: location.start.line, + column: location.start.column, + offset: location.start.offset, + }..TextPosition { + line: location.end.line, + column: location.end.column, + offset: location.end.offset, + }, + ) + } + SyntaxErrorKind::RdfXml(_) | SyntaxErrorKind::Msg { .. } => None, + } + } +} + impl fmt::Display for SyntaxError { #[inline] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { @@ -146,3 +170,11 @@ impl From for io::Error { } } } + +/// A position in a text i.e. a `line` number starting from 0, a `column` number starting from 0 (in number of code points) and a global file `offset` starting from 0 (in number of bytes). +#[derive(Eq, PartialEq, Debug, Clone, Copy)] +pub struct TextPosition { + pub line: u64, + pub column: u64, + pub offset: u64, +} diff --git a/lib/oxrdfxml/src/error.rs b/lib/oxrdfxml/src/error.rs index fd561be6..cb9eb9c4 100644 --- a/lib/oxrdfxml/src/error.rs +++ b/lib/oxrdfxml/src/error.rs @@ -72,15 +72,6 @@ impl From for ParseError { } } -impl From for ParseError { - #[inline] - fn from(error: quick_xml::events::attributes::AttrError) -> Self { - Self::Syntax(SyntaxError { - inner: SyntaxErrorKind::XmlAttribute(error), - }) - } -} - /// An error in the syntax of the parsed file. #[derive(Debug)] pub struct SyntaxError { @@ -90,7 +81,6 @@ pub struct SyntaxError { #[derive(Debug)] pub enum SyntaxErrorKind { Xml(quick_xml::Error), - XmlAttribute(quick_xml::events::attributes::AttrError), InvalidIri { iri: String, error: IriParseError, @@ -119,7 +109,6 @@ impl fmt::Display for SyntaxError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match &self.inner { SyntaxErrorKind::Xml(error) => error.fmt(f), - SyntaxErrorKind::XmlAttribute(error) => error.fmt(f), SyntaxErrorKind::InvalidIri { iri, error } => { write!(f, "error while parsing IRI '{iri}': {error}") } @@ -136,7 +125,6 @@ impl Error for SyntaxError { fn source(&self) -> Option<&(dyn Error + 'static)> { match &self.inner { SyntaxErrorKind::Xml(error) => Some(error), - SyntaxErrorKind::XmlAttribute(error) => Some(error), SyntaxErrorKind::InvalidIri { error, .. } => Some(error), SyntaxErrorKind::InvalidLanguageTag { error, .. } => Some(error), SyntaxErrorKind::Msg { .. } => None, diff --git a/lib/oxrdfxml/src/parser.rs b/lib/oxrdfxml/src/parser.rs index 22983350..dcd216a4 100644 --- a/lib/oxrdfxml/src/parser.rs +++ b/lib/oxrdfxml/src/parser.rs @@ -8,7 +8,7 @@ use quick_xml::escape::unescape_with; use quick_xml::events::attributes::Attribute; use quick_xml::events::*; use quick_xml::name::{LocalName, QName, ResolveResult}; -use quick_xml::{NsReader, Writer}; +use quick_xml::{Error, NsReader, Writer}; use std::collections::{HashMap, HashSet}; use std::io::{BufReader, Read}; use std::str; @@ -515,7 +515,7 @@ impl RdfXmlReader { .to_string(), ); for attr in event.attributes() { - clean_event.push_attribute(attr?); + clean_event.push_attribute(attr.map_err(Error::InvalidAttr)?); } writer.write_event(Event::Start(clean_event))?; self.in_literal_depth += 1; @@ -544,7 +544,7 @@ impl RdfXmlReader { let mut type_attr = None; for attribute in event.attributes() { - let attribute = attribute?; + let attribute = attribute.map_err(Error::InvalidAttr)?; if attribute.key.as_ref().starts_with(b"xml") { if attribute.key.as_ref() == b"xml:lang" { let tag = self.convert_attribute(&attribute)?; diff --git a/lib/oxttl/src/lexer.rs b/lib/oxttl/src/lexer.rs index f12f3b25..65dba56e 100644 --- a/lib/oxttl/src/lexer.rs +++ b/lib/oxttl/src/lexer.rs @@ -266,7 +266,7 @@ impl N3Lexer { )); } } - Err(e) => return Some((e.position.end, Err(e))), + Err(e) => return Some((e.location.end, Err(e))), } } else if is_ending { while data[..i].ends_with(b".") { @@ -447,7 +447,7 @@ impl N3Lexer { return Some((i, Ok((buffer, might_be_invalid_iri)))); } } - Err(e) => return Some((e.position.end, Err(e))), + Err(e) => return Some((e.location.end, Err(e))), } } else if is_ending { let buffer = if let Some(mut buffer) = buffer { @@ -515,7 +515,7 @@ impl N3Lexer { } i += consumed; } - Err(e) => return Some((e.position.end, Err(e))), + Err(e) => return Some((e.location.end, Err(e))), } } } diff --git a/lib/oxttl/src/lib.rs b/lib/oxttl/src/lib.rs index ac96515e..0e04e243 100644 --- a/lib/oxttl/src/lib.rs +++ b/lib/oxttl/src/lib.rs @@ -17,7 +17,7 @@ pub mod turtle; pub use crate::n3::N3Parser; pub use crate::nquads::{NQuadsParser, NQuadsSerializer}; pub use crate::ntriples::{NTriplesParser, NTriplesSerializer}; -pub use crate::toolkit::{ParseError, SyntaxError}; +pub use crate::toolkit::{ParseError, SyntaxError, TextPosition}; pub use crate::trig::{TriGParser, TriGSerializer}; pub use crate::turtle::{TurtleParser, TurtleSerializer}; diff --git a/lib/oxttl/src/line_formats.rs b/lib/oxttl/src/line_formats.rs index f95e56f3..1b4c31e6 100644 --- a/lib/oxttl/src/line_formats.rs +++ b/lib/oxttl/src/line_formats.rs @@ -76,7 +76,7 @@ impl RuleRecognizer for NQuadsRecognizer { } _ => self.error( errors, - format!("The subject of a triple should be an IRI or a blank node, {token:?} found"), + "The subject of a triple should be an IRI or a blank node, TOKEN found", ), }, NQuadsState::ExpectPredicate => match token { @@ -88,7 +88,7 @@ impl RuleRecognizer for NQuadsRecognizer { } _ => self.error( errors, - format!("The predicate of a triple should be an IRI, {token:?} found"), + "The predicate of a triple should be an IRI, TOKEN found", ), }, NQuadsState::ExpectedObject => match token { @@ -118,7 +118,7 @@ impl RuleRecognizer for NQuadsRecognizer { } _ => self.error( errors, - format!("The object of a triple should be an IRI, a blank node or a literal, {token:?} found"), + "The object of a triple should be an IRI, a blank node or a literal, TOKEN found", ), }, NQuadsState::ExpectLiteralAnnotationOrGraphNameOrDot { value } => match token { @@ -159,7 +159,7 @@ impl RuleRecognizer for NQuadsRecognizer { .push(NQuadsState::ExpectPossibleGraphOrEndOfQuotedTriple); self } - _ => self.error(errors, format!("A literal datatype must be an IRI, found {token:?}")), + _ => self.error(errors, "A literal datatype must be an IRI, found TOKEN"), }, NQuadsState::ExpectPossibleGraphOrEndOfQuotedTriple => { if self.stack.is_empty() { diff --git a/lib/oxttl/src/n3.rs b/lib/oxttl/src/n3.rs index 46274ecf..84e36235 100644 --- a/lib/oxttl/src/n3.rs +++ b/lib/oxttl/src/n3.rs @@ -836,7 +836,7 @@ impl RuleRecognizer for N3Recognizer { self.stack.push(N3State::FormulaContent); self } - _ => self.error(errors, format!("This is not a valid RDF value: {token:?}")) + _ => self.error(errors, "TOKEN is not a valid RDF value") } } N3State::PropertyListMiddle => match token { @@ -950,7 +950,7 @@ impl RuleRecognizer for N3Recognizer { Err(e) => self.error(errors, e) } _ => { - self.error(errors, format!("Expecting a datatype IRI after '^^, found {token:?}")).recognize_next(token, results, errors) + self.error(errors, "Expecting a datatype IRI after '^^, found TOKEN").recognize_next(token, results, errors) } } } diff --git a/lib/oxttl/src/terse.rs b/lib/oxttl/src/terse.rs index ecd24d4a..f99aaf4d 100644 --- a/lib/oxttl/src/terse.rs +++ b/lib/oxttl/src/terse.rs @@ -167,7 +167,7 @@ impl RuleRecognizer for TriGRecognizer { self } _ => { - self.error(errors, format!("The token {token:?} is not a valid subject or graph name")) + self.error(errors, "TOKEN is not a valid subject or graph name") } } TriGState::WrappedGraphOrPredicateObjectList { term } => { @@ -317,7 +317,7 @@ impl RuleRecognizer for TriGRecognizer { self } _ => { - self.error(errors, format!("The token {token:?} is not a valid RDF subject")) + self.error(errors, "TOKEN is not a valid RDF subject") } }, TriGState::TriplesBlankNodePropertyListCurrent => if token == N3Token::Punctuation("]") { @@ -350,7 +350,7 @@ impl RuleRecognizer for TriGRecognizer { self } _ => { - self.error(errors, format!("The token {token:?} is not a valid graph name")) + self.error(errors, "TOKEN is not a valid graph name") } } TriGState::GraphNameAnonEnd => if token == N3Token::Punctuation("]") { @@ -456,7 +456,7 @@ impl RuleRecognizer for TriGRecognizer { Err(e) => self.error(errors, e) } _ => { - self.error(errors, format!("The token {token:?} is not a valid predicate")) + self.error(errors, "TOKEN is not a valid predicate") } } // [12] object ::= iri | BlankNode | collection | blankNodePropertyList | literal | quotedTriple @@ -536,7 +536,7 @@ impl RuleRecognizer for TriGRecognizer { self } _ => { - self.error(errors, format!("This is not a valid RDF object: {token:?}")) + self.error(errors, "TOKEN is not a valid RDF object") } } @@ -637,7 +637,7 @@ impl RuleRecognizer for TriGRecognizer { Err(e) => self.error(errors, e) } _ => { - self.error(errors, format!("Expecting a datatype IRI after '^^, found {token:?}")).recognize_next(token, results, errors) + self.error(errors, "Expecting a datatype IRI after ^^, found TOKEN").recognize_next(token, results, errors) } } } @@ -653,7 +653,7 @@ impl RuleRecognizer for TriGRecognizer { if token == N3Token::Punctuation(">>") { self } else { - self.error(errors, format!("Expecting '>>' to close a quoted triple, found {token:?}")) + self.error(errors, "Expecting '>>' to close a quoted triple, found TOKEN") } } #[cfg(feature = "rdf-star")] @@ -670,7 +670,7 @@ impl RuleRecognizer for TriGRecognizer { if token == N3Token::Punctuation(">>") { self } else { - self.error(errors, format!("Expecting '>>' to close a quoted triple, found {token:?}")) + self.error(errors, "Expecting '>>' to close a quoted triple, found TOKEN") } } // [28t] qtSubject ::= iri | BlankNode | quotedTriple @@ -703,7 +703,7 @@ impl RuleRecognizer for TriGRecognizer { self.stack.push(TriGState::QuotedSubject); self } - _ => self.error(errors, format!("This is not a valid RDF quoted triple subject: {token:?}")) + _ => self.error(errors, "TOKEN is not a valid RDF quoted triple subject: TOKEN") } // [29t] qtObject ::= iri | BlankNode | literal | quotedTriple #[cfg(feature = "rdf-star")] @@ -759,7 +759,7 @@ impl RuleRecognizer for TriGRecognizer { self.stack.push(TriGState::QuotedSubject); self } - _ => self.error(errors, format!("This is not a valid RDF quoted triple object: {token:?}")) + _ => self.error(errors, "TOKEN is not a valid RDF quoted triple object") } #[cfg(feature = "rdf-star")] TriGState::QuotedAnonEnd => if token == N3Token::Punctuation("]") { diff --git a/lib/oxttl/src/toolkit/error.rs b/lib/oxttl/src/toolkit/error.rs new file mode 100644 index 00000000..df50b950 --- /dev/null +++ b/lib/oxttl/src/toolkit/error.rs @@ -0,0 +1,132 @@ +use std::error::Error; +use std::ops::Range; +use std::{fmt, io}; + +/// A position in a text i.e. a `line` number starting from 0, a `column` number starting from 0 (in number of code points) and a global file `offset` starting from 0 (in number of bytes). +#[derive(Eq, PartialEq, Debug, Clone, Copy)] +pub struct TextPosition { + pub line: u64, + pub column: u64, + pub offset: u64, +} + +/// An error in the syntax of the parsed file. +/// +/// It is composed of a message and a byte range in the input. +#[derive(Debug)] +pub struct SyntaxError { + pub(super) location: Range, + pub(super) message: String, +} + +impl SyntaxError { + /// The location of the error inside of the file. + #[inline] + pub fn location(&self) -> Range { + self.location.clone() + } + + /// The error message. + #[inline] + pub fn message(&self) -> &str { + &self.message + } +} + +impl fmt::Display for SyntaxError { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if self.location.start.offset + 1 >= self.location.end.offset { + write!( + f, + "Parser error at line {} column {}: {}", + self.location.start.line + 1, + self.location.start.column + 1, + self.message + ) + } else if self.location.start.line == self.location.end.line { + write!( + f, + "Parser error between at line {} between columns {} and column {}: {}", + self.location.start.line + 1, + self.location.start.column + 1, + self.location.end.column + 1, + self.message + ) + } else { + write!( + f, + "Parser error between line {} column {} and line {} column {}: {}", + self.location.start.line + 1, + self.location.start.column + 1, + self.location.end.line + 1, + self.location.end.column + 1, + self.message + ) + } + } +} + +impl Error for SyntaxError {} + +impl From for io::Error { + #[inline] + fn from(error: SyntaxError) -> Self { + io::Error::new(io::ErrorKind::InvalidData, error) + } +} + +/// A parsing error. +/// +/// It is the union of [`SyntaxError`] and [`std::io::Error`]. +#[derive(Debug)] +pub enum ParseError { + /// I/O error during parsing (file not found...). + Io(io::Error), + /// An error in the file syntax. + Syntax(SyntaxError), +} + +impl fmt::Display for ParseError { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Io(e) => e.fmt(f), + Self::Syntax(e) => e.fmt(f), + } + } +} + +impl Error for ParseError { + #[inline] + fn source(&self) -> Option<&(dyn Error + 'static)> { + Some(match self { + Self::Io(e) => e, + Self::Syntax(e) => e, + }) + } +} + +impl From for ParseError { + #[inline] + fn from(error: SyntaxError) -> Self { + Self::Syntax(error) + } +} + +impl From for ParseError { + #[inline] + fn from(error: io::Error) -> Self { + Self::Io(error) + } +} + +impl From for io::Error { + #[inline] + fn from(error: ParseError) -> Self { + match error { + ParseError::Syntax(e) => e.into(), + ParseError::Io(e) => e, + } + } +} diff --git a/lib/oxttl/src/toolkit/lexer.rs b/lib/oxttl/src/toolkit/lexer.rs index 34c1c01e..0f7373c2 100644 --- a/lib/oxttl/src/toolkit/lexer.rs +++ b/lib/oxttl/src/toolkit/lexer.rs @@ -1,9 +1,10 @@ -use memchr::memchr2; +use crate::toolkit::error::{SyntaxError, TextPosition}; +use memchr::{memchr2, memchr2_iter}; +use std::borrow::Cow; use std::cmp::min; -use std::error::Error; -use std::fmt; use std::io::{self, Read}; use std::ops::{Range, RangeInclusive}; +use std::str; #[cfg(feature = "async-tokio")] use tokio::io::{AsyncRead, AsyncReadExt}; @@ -22,14 +23,14 @@ pub trait TokenRecognizer { } pub struct TokenRecognizerError { - pub position: Range, + pub location: Range, pub message: String, } impl> From<(Range, S)> for TokenRecognizerError { - fn from((position, message): (Range, S)) -> Self { + fn from((location, message): (Range, S)) -> Self { Self { - position, + location, message: message.into(), } } @@ -37,34 +38,37 @@ impl> From<(Range, S)> for TokenRecognizerError { #[allow(clippy::range_plus_one)] impl> From<(RangeInclusive, S)> for TokenRecognizerError { - fn from((position, message): (RangeInclusive, S)) -> Self { - (*position.start()..*position.end() + 1, message).into() + fn from((location, message): (RangeInclusive, S)) -> Self { + (*location.start()..*location.end() + 1, message).into() } } impl> From<(usize, S)> for TokenRecognizerError { - fn from((position, message): (usize, S)) -> Self { - (position..=position, message).into() + fn from((location, message): (usize, S)) -> Self { + (location..=location, message).into() } } -pub struct TokenWithPosition { - pub token: T, - pub position: Range, -} - pub struct Lexer { parser: R, data: Vec, - start: usize, + position: Position, + previous_position: Position, // Lexer position before the last emitted token is_ending: bool, - position: usize, min_buffer_size: usize, max_buffer_size: usize, is_line_jump_whitespace: bool, line_comment_start: Option<&'static [u8]>, } +#[derive(Clone, Copy)] +struct Position { + line_start_buffer_offset: usize, + buffer_offset: usize, + global_offset: u64, + global_line: u64, +} + impl Lexer { pub fn new( parser: R, @@ -76,9 +80,19 @@ impl Lexer { Self { parser, data: Vec::new(), - start: 0, + position: Position { + line_start_buffer_offset: 0, + buffer_offset: 0, + global_offset: 0, + global_line: 0, + }, + previous_position: Position { + line_start_buffer_offset: 0, + buffer_offset: 0, + global_offset: 0, + global_line: 0, + }, is_ending: false, - position: 0, min_buffer_size, max_buffer_size, is_line_jump_whitespace, @@ -148,24 +162,43 @@ impl Lexer { Ok(()) } - pub fn read_next( - &mut self, - options: &R::Options, - ) -> Option>, LexerError>> { + #[allow(clippy::unwrap_in_result)] + pub fn read_next(&mut self, options: &R::Options) -> Option, SyntaxError>> { self.skip_whitespaces_and_comments()?; - let Some((consumed, result)) = - self.parser - .recognize_next_token(&self.data[self.start..], self.is_ending, options) - else { + self.previous_position = self.position; + let Some((consumed, result)) = self.parser.recognize_next_token( + &self.data[self.position.buffer_offset..], + self.is_ending, + options, + ) else { return if self.is_ending { - if self.start == self.data.len() { + if self.position.buffer_offset == self.data.len() { None // We have finished } else { - let error = LexerError { - position: self.position..self.position + (self.data.len() - self.start), + let (new_line_jumps, new_line_start) = + Self::find_number_of_line_jumps_and_start_of_last_line( + &self.data[self.position.buffer_offset..], + ); + if new_line_jumps > 0 { + self.position.line_start_buffer_offset = + self.position.buffer_offset + new_line_start; + } + self.position.global_offset += + u64::try_from(self.data.len() - self.position.buffer_offset).unwrap(); + self.position.buffer_offset = self.data.len(); + self.position.global_line += new_line_jumps; + let new_position = TextPosition { + line: self.position.global_line, + column: Self::column_from_bytes( + &self.data[self.position.line_start_buffer_offset..], + ), + offset: self.position.global_offset, + }; + let error = SyntaxError { + location: new_position..new_position, message: "Unexpected end of file".into(), }; - self.start = self.data.len(); // We consume everything + self.position.buffer_offset = self.data.len(); // We consume everything Some(Err(error)) } } else { @@ -177,44 +210,119 @@ impl Lexer { "The lexer must consume at least one byte each time" ); debug_assert!( - self.start + consumed <= self.data.len(), + self.position.buffer_offset + consumed <= self.data.len(), "The lexer tried to consumed {consumed} bytes but only {} bytes are readable", - self.data.len() - self.start + self.data.len() - self.position.buffer_offset ); - let old_position = self.position; - self.start += consumed; - self.position += consumed; - Some(match result { - Ok(token) => Ok(TokenWithPosition { - token, - position: old_position..self.position, - }), - Err(e) => Err(LexerError { - position: e.position.start + self.position..e.position.end + self.position, - message: e.message, - }), - }) + let (new_line_jumps, new_line_start) = + Self::find_number_of_line_jumps_and_start_of_last_line( + &self.data[self.position.buffer_offset..self.position.buffer_offset + consumed], + ); + if new_line_jumps > 0 { + self.position.line_start_buffer_offset = self.position.buffer_offset + new_line_start; + } + self.position.buffer_offset += consumed; + self.position.global_offset += u64::try_from(consumed).unwrap(); + self.position.global_line += new_line_jumps; + Some(result.map_err(|e| SyntaxError { + location: self.location_from_buffer_offset_range(e.location), + message: e.message, + })) + } + + pub fn location_from_buffer_offset_range( + &self, + offset_range: Range, + ) -> Range { + let start_offset = self.previous_position.buffer_offset + offset_range.start; + let (start_extra_line_jumps, start_line_start) = + Self::find_number_of_line_jumps_and_start_of_last_line( + &self.data[self.previous_position.buffer_offset..start_offset], + ); + let start_line_start = if start_extra_line_jumps > 0 { + start_line_start + self.previous_position.buffer_offset + } else { + self.previous_position.line_start_buffer_offset + }; + let end_offset = self.previous_position.buffer_offset + offset_range.end; + let (end_extra_line_jumps, end_line_start) = + Self::find_number_of_line_jumps_and_start_of_last_line( + &self.data[self.previous_position.buffer_offset..end_offset], + ); + let end_line_start = if end_extra_line_jumps > 0 { + end_line_start + self.previous_position.buffer_offset + } else { + self.previous_position.line_start_buffer_offset + }; + TextPosition { + line: self.previous_position.global_line + start_extra_line_jumps, + column: Self::column_from_bytes(&self.data[start_line_start..start_offset]), + offset: self.previous_position.global_offset + + u64::try_from(offset_range.start).unwrap(), + }..TextPosition { + line: self.previous_position.global_line + end_extra_line_jumps, + column: Self::column_from_bytes(&self.data[end_line_start..end_offset]), + offset: self.previous_position.global_offset + u64::try_from(offset_range.end).unwrap(), + } + } + + pub fn last_token_location(&self) -> Range { + TextPosition { + line: self.previous_position.global_line, + column: Self::column_from_bytes( + &self.data[self.previous_position.line_start_buffer_offset + ..self.previous_position.buffer_offset], + ), + offset: self.previous_position.global_offset, + }..TextPosition { + line: self.position.global_line, + column: Self::column_from_bytes( + &self.data[self.position.line_start_buffer_offset..self.position.buffer_offset], + ), + offset: self.position.global_offset, + } + } + + pub fn last_token_source(&self) -> Cow<'_, str> { + String::from_utf8_lossy( + &self.data[self.previous_position.buffer_offset..self.position.buffer_offset], + ) } pub fn is_end(&self) -> bool { - self.is_ending && self.data.len() == self.start + self.is_ending && self.data.len() == self.position.buffer_offset } + #[allow(clippy::unwrap_in_result)] fn skip_whitespaces_and_comments(&mut self) -> Option<()> { loop { - self.skip_whitespaces(); + self.skip_whitespaces()?; - let buf = &self.data[self.start..]; + let buf = &self.data[self.position.buffer_offset..]; if let Some(line_comment_start) = self.line_comment_start { if buf.starts_with(line_comment_start) { // Comment if let Some(end) = memchr2(b'\r', b'\n', &buf[line_comment_start.len()..]) { - self.start += end + line_comment_start.len(); - self.position += end + line_comment_start.len(); + let mut end_position = line_comment_start.len() + end; + if buf.get(end_position).copied() == Some(b'\r') { + // We look for \n for Windows line end style + if let Some(c) = buf.get(end_position + 1) { + if *c == b'\n' { + end_position += 1; + } + } else if !self.is_ending { + return None; // We need to read more + } + } + let comment_size = end_position + 1; + self.position.buffer_offset += comment_size; + self.position.line_start_buffer_offset = self.position.buffer_offset; + self.position.global_offset += u64::try_from(comment_size).unwrap(); + self.position.global_line += 1; continue; } if self.is_ending { - self.start = self.data.len(); // EOF + self.position.buffer_offset = self.data.len(); // EOF return Some(()); } return None; // We need more data @@ -224,80 +332,98 @@ impl Lexer { } } - fn skip_whitespaces(&mut self) { + fn skip_whitespaces(&mut self) -> Option<()> { if self.is_line_jump_whitespace { - for (i, c) in self.data[self.start..].iter().enumerate() { - if !matches!(c, b' ' | b'\t' | b'\r' | b'\n') { - self.start += i; - self.position += i; - return; + let mut i = self.position.buffer_offset; + while let Some(c) = self.data.get(i) { + match c { + b' ' | b'\t' => { + self.position.buffer_offset += 1; + self.position.global_offset += 1; + } + b'\r' => { + // We look for \n for Windows line end style + let mut increment: u8 = 1; + if let Some(c) = self.data.get(i + 1) { + if *c == b'\n' { + increment += 1; + i += 1; + } + } else if !self.is_ending { + return None; // We need to read more + } + self.position.buffer_offset += usize::from(increment); + self.position.line_start_buffer_offset = self.position.buffer_offset; + self.position.global_offset += u64::from(increment); + self.position.global_line += 1; + } + b'\n' => { + self.position.buffer_offset += 1; + self.position.line_start_buffer_offset = self.position.buffer_offset; + self.position.global_offset += 1; + self.position.global_line += 1; + } + _ => return Some(()), } + i += 1; //TODO: SIMD } } else { - for (i, c) in self.data[self.start..].iter().enumerate() { - if !matches!(c, b' ' | b'\t') { - self.start += i; - self.position += i; - return; + for c in &self.data[self.position.buffer_offset..] { + if matches!(c, b' ' | b'\t') { + self.position.buffer_offset += 1; + self.position.global_offset += 1; + } else { + return Some(()); } //TODO: SIMD } } - // We only have whitespaces - self.position += self.data.len() - self.start; - self.start = self.data.len(); + Some(()) } fn shrink_data(&mut self) { - if self.start > 0 { - self.data.copy_within(self.start.., 0); - self.data.truncate(self.data.len() - self.start); - self.start = 0; + if self.position.line_start_buffer_offset > 0 { + self.data + .copy_within(self.position.line_start_buffer_offset.., 0); + self.data + .truncate(self.data.len() - self.position.line_start_buffer_offset); + self.position.buffer_offset -= self.position.line_start_buffer_offset; + self.position.line_start_buffer_offset = 0; + self.previous_position = self.position; } } -} -#[derive(Debug)] -pub struct LexerError { - position: Range, - message: String, -} - -impl LexerError { - pub fn position(&self) -> Range { - self.position.clone() - } - - pub fn message(&self) -> &str { - &self.message - } - - pub fn into_message(self) -> String { - self.message - } -} - -impl fmt::Display for LexerError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - if self.position.start + 1 == self.position.end { - write!( - f, - "Lexer error at byte {}: {}", - self.position.start, self.message - ) - } else { - write!( - f, - "Lexer error between bytes {} and {}: {}", - self.position.start, self.position.end, self.message - ) + fn find_number_of_line_jumps_and_start_of_last_line(bytes: &[u8]) -> (u64, usize) { + let mut num_of_jumps = 0; + let mut last_jump_pos = 0; + let mut previous_cr = 0; + for pos in memchr2_iter(b'\r', b'\n', bytes) { + if bytes[pos] == b'\r' { + previous_cr = pos; + num_of_jumps += 1; + last_jump_pos = pos + 1; + } else { + if previous_cr < pos - 1 { + // We count \r\n as a single line jump + num_of_jumps += 1; + } + last_jump_pos = pos + 1; + } } + (num_of_jumps, last_jump_pos) } -} -impl Error for LexerError { - fn description(&self) -> &str { - self.message() + fn column_from_bytes(bytes: &[u8]) -> u64 { + match str::from_utf8(bytes) { + Ok(s) => u64::try_from(s.chars().count()).unwrap(), + Err(e) => { + if e.valid_up_to() == 0 { + 0 + } else { + Self::column_from_bytes(&bytes[..e.valid_up_to()]) + } + } + } } } diff --git a/lib/oxttl/src/toolkit/mod.rs b/lib/oxttl/src/toolkit/mod.rs index 300b9c2c..cc8e3624 100644 --- a/lib/oxttl/src/toolkit/mod.rs +++ b/lib/oxttl/src/toolkit/mod.rs @@ -2,12 +2,12 @@ //! //! Provides the basic code to write plain Rust lexers and parsers able to read files chunk by chunk. +mod error; mod lexer; mod parser; -pub use self::lexer::{Lexer, LexerError, TokenRecognizer, TokenRecognizerError}; +pub use self::error::{ParseError, SyntaxError, TextPosition}; +pub use self::lexer::{Lexer, TokenRecognizer, TokenRecognizerError}; #[cfg(feature = "async-tokio")] pub use self::parser::FromTokioAsyncReadIterator; -pub use self::parser::{ - FromReadIterator, ParseError, Parser, RuleRecognizer, RuleRecognizerError, SyntaxError, -}; +pub use self::parser::{FromReadIterator, Parser, RuleRecognizer, RuleRecognizerError}; diff --git a/lib/oxttl/src/toolkit/parser.rs b/lib/oxttl/src/toolkit/parser.rs index 38419477..7a9ba8bf 100644 --- a/lib/oxttl/src/toolkit/parser.rs +++ b/lib/oxttl/src/toolkit/parser.rs @@ -1,9 +1,6 @@ -use crate::toolkit::lexer::TokenWithPosition; -use crate::toolkit::{Lexer, LexerError, TokenRecognizer}; -use std::error::Error; +use crate::toolkit::error::{ParseError, SyntaxError}; +use crate::toolkit::lexer::{Lexer, TokenRecognizer}; use std::io::Read; -use std::ops::Range; -use std::{fmt, io}; #[cfg(feature = "async-tokio")] use tokio::io::AsyncRead; @@ -42,7 +39,6 @@ pub struct Parser { state: Option, results: Vec, errors: Vec, - position: Range, default_lexer_options: ::Options, } @@ -53,7 +49,6 @@ impl Parser { state: Some(recognizer), results: vec![], errors: vec![], - position: 0..0, default_lexer_options: ::Options::default(), } } @@ -76,8 +71,10 @@ impl Parser { loop { if let Some(error) = self.errors.pop() { return Some(Err(SyntaxError { - position: self.position.clone(), - message: error.message, + location: self.lexer.last_token_location(), + message: error + .message + .replace("TOKEN", &self.lexer.last_token_source()), })); } if let Some(result) = self.results.pop() { @@ -89,8 +86,7 @@ impl Parser { .map_or(&self.default_lexer_options, |p| p.lexer_options()), ) { match result { - Ok(TokenWithPosition { token, position }) => { - self.position = position; + Ok(token) => { self.state = self.state.take().map(|state| { state.recognize_next(token, &mut self.results, &mut self.errors) }); @@ -98,7 +94,7 @@ impl Parser { } Err(e) => { self.state = self.state.take().map(RR::error_recovery_state); - return Some(Err(e.into())); + return Some(Err(e)); } } } @@ -126,128 +122,6 @@ impl Parser { } } -/// An error in the syntax of the parsed file. -/// -/// It is composed of a message and a byte range in the input. -#[derive(Debug)] -pub struct SyntaxError { - position: Range, - message: String, -} - -impl SyntaxError { - /// The invalid byte range in the input. - #[inline] - pub fn position(&self) -> Range { - self.position.clone() - } - - /// The error message. - #[inline] - pub fn message(&self) -> &str { - &self.message - } - - /// Converts this error to an error message. - #[inline] - pub fn into_message(self) -> String { - self.message - } -} - -impl fmt::Display for SyntaxError { - #[inline] - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - if self.position.start + 1 == self.position.end { - write!( - f, - "Parser error at byte {}: {}", - self.position.start, self.message - ) - } else { - write!( - f, - "Parser error between bytes {} and {}: {}", - self.position.start, self.position.end, self.message - ) - } - } -} - -impl Error for SyntaxError {} - -impl From for io::Error { - #[inline] - fn from(error: SyntaxError) -> Self { - io::Error::new(io::ErrorKind::InvalidData, error) - } -} - -impl From for SyntaxError { - #[inline] - fn from(e: LexerError) -> Self { - Self { - position: e.position(), - message: e.into_message(), - } - } -} - -/// A parsing error. -/// -/// It is the union of [`SyntaxError`] and [`std::io::Error`]. -#[derive(Debug)] -pub enum ParseError { - /// I/O error during parsing (file not found...). - Io(io::Error), - /// An error in the file syntax. - Syntax(SyntaxError), -} - -impl fmt::Display for ParseError { - #[inline] - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Io(e) => e.fmt(f), - Self::Syntax(e) => e.fmt(f), - } - } -} - -impl Error for ParseError { - #[inline] - fn source(&self) -> Option<&(dyn Error + 'static)> { - Some(match self { - Self::Io(e) => e, - Self::Syntax(e) => e, - }) - } -} - -impl From for ParseError { - #[inline] - fn from(error: SyntaxError) -> Self { - Self::Syntax(error) - } -} - -impl From for ParseError { - #[inline] - fn from(error: io::Error) -> Self { - Self::Io(error) - } -} - -impl From for io::Error { - #[inline] - fn from(error: ParseError) -> Self { - match error { - ParseError::Syntax(e) => e.into(), - ParseError::Io(e) => e, - } - } -} - pub struct FromReadIterator { read: R, parser: Parser, diff --git a/testsuite/oxigraph-tests/parser-error/invalid_iri.nt b/testsuite/oxigraph-tests/parser-error/invalid_iri.nt new file mode 100644 index 00000000..021c7911 --- /dev/null +++ b/testsuite/oxigraph-tests/parser-error/invalid_iri.nt @@ -0,0 +1,2 @@ + . + . diff --git a/testsuite/oxigraph-tests/parser-error/invalid_iri_comment.nt b/testsuite/oxigraph-tests/parser-error/invalid_iri_comment.nt new file mode 100644 index 00000000..7c8d2120 --- /dev/null +++ b/testsuite/oxigraph-tests/parser-error/invalid_iri_comment.nt @@ -0,0 +1,2 @@ + . # foo + . diff --git a/testsuite/oxigraph-tests/parser-error/invalid_iri_comment_crlf.nt b/testsuite/oxigraph-tests/parser-error/invalid_iri_comment_crlf.nt new file mode 100644 index 00000000..7c8d2120 --- /dev/null +++ b/testsuite/oxigraph-tests/parser-error/invalid_iri_comment_crlf.nt @@ -0,0 +1,2 @@ + . # foo + . diff --git a/testsuite/oxigraph-tests/parser-error/invalid_iri_crlf.nt b/testsuite/oxigraph-tests/parser-error/invalid_iri_crlf.nt new file mode 100644 index 00000000..021c7911 --- /dev/null +++ b/testsuite/oxigraph-tests/parser-error/invalid_iri_crlf.nt @@ -0,0 +1,2 @@ + . + . diff --git a/testsuite/oxigraph-tests/parser-error/invalid_iri_error.txt b/testsuite/oxigraph-tests/parser-error/invalid_iri_error.txt new file mode 100644 index 00000000..26729063 --- /dev/null +++ b/testsuite/oxigraph-tests/parser-error/invalid_iri_error.txt @@ -0,0 +1 @@ +Parser error between at line 2 between columns 24 and column 36: Invalid IRI code point ' ' \ No newline at end of file diff --git a/testsuite/oxigraph-tests/parser-error/invalid_predicate.nt b/testsuite/oxigraph-tests/parser-error/invalid_predicate.nt new file mode 100644 index 00000000..63e6fd7a --- /dev/null +++ b/testsuite/oxigraph-tests/parser-error/invalid_predicate.nt @@ -0,0 +1,2 @@ + . + "p" . diff --git a/testsuite/oxigraph-tests/parser-error/invalid_predicate_error.txt b/testsuite/oxigraph-tests/parser-error/invalid_predicate_error.txt new file mode 100644 index 00000000..469dd19f --- /dev/null +++ b/testsuite/oxigraph-tests/parser-error/invalid_predicate_error.txt @@ -0,0 +1 @@ +Parser error between at line 2 between columns 24 and column 27: "p" is not a valid predicate \ No newline at end of file diff --git a/testsuite/oxigraph-tests/parser-error/invalid_string_escape.nt b/testsuite/oxigraph-tests/parser-error/invalid_string_escape.nt new file mode 100644 index 00000000..8a625fce --- /dev/null +++ b/testsuite/oxigraph-tests/parser-error/invalid_string_escape.nt @@ -0,0 +1 @@ + "fooé \a baré" . diff --git a/testsuite/oxigraph-tests/parser-error/invalid_string_escape_error.txt b/testsuite/oxigraph-tests/parser-error/invalid_string_escape_error.txt new file mode 100644 index 00000000..f5e45857 --- /dev/null +++ b/testsuite/oxigraph-tests/parser-error/invalid_string_escape_error.txt @@ -0,0 +1 @@ +Parser error between at line 1 between columns 53 and column 55: Unexpected escape character '\a' \ No newline at end of file diff --git a/testsuite/oxigraph-tests/parser-error/manifest.ttl b/testsuite/oxigraph-tests/parser-error/manifest.ttl new file mode 100644 index 00000000..86159b38 --- /dev/null +++ b/testsuite/oxigraph-tests/parser-error/manifest.ttl @@ -0,0 +1,66 @@ +@prefix mf: . +@prefix rdf: . +@prefix rdfs: . +@prefix rdft: . + +<> + rdf:type mf:Manifest ; + rdfs:comment "Oxigraph parser error test cases" ; + mf:entries ( + <#invalid_iri> + <#invalid_iri_crlf> + <#invalid_iri_comment> + <#invalid_iri_comment_crlf> + <#invalid_string_escape> + <#unexpected_eof> + <#unexpected_eof_crlf> + <#invalid_predicate> + ) . + +<#invalid_iri> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad IRI" ; + mf:action ; + mf:result . + +<#invalid_iri_crlf> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad IRI" ; + mf:action ; + mf:result . + +<#invalid_iri_comment> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad IRI" ; + mf:action ; + mf:result . + +<#invalid_iri_comment_crlf> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad IRI" ; + mf:action ; + mf:result . + +<#invalid_string_escape> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "bad string escape" ; + mf:action ; + mf:result . + +<#unexpected_eof> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "unexpected end of file" ; + mf:action ; + mf:result . + +<#unexpected_eof_crlf> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "unexpected end of file" ; + mf:action ; + mf:result . + +<#invalid_predicate> + rdf:type rdft:TestTurtleNegativeSyntax ; + mf:name "invalid predicate" ; + mf:action ; + mf:result . diff --git a/testsuite/oxigraph-tests/parser-error/unexpected_eof.nt b/testsuite/oxigraph-tests/parser-error/unexpected_eof.nt new file mode 100644 index 00000000..8c0a4ca2 --- /dev/null +++ b/testsuite/oxigraph-tests/parser-error/unexpected_eof.nt @@ -0,0 +1,2 @@ + Result<()> { .action .as_deref() .ok_or_else(|| anyhow!("No action found"))?; - ensure!( - load_dataset(action, format, false).is_err(), - "File parsed without errors even if it should not" - ); + let Err(error) = load_dataset(action, format, false) else { + bail!("File parsed without errors even if it should not"); + }; + if let Some(result) = &test.result { + let expected = read_file_to_string(result)?; + ensure!( + expected == error.to_string(), + "Not expected error message:\n{}", + format_diff(&expected, &error.to_string(), "message") + ); + } Ok(()) } diff --git a/testsuite/tests/oxigraph.rs b/testsuite/tests/oxigraph.rs index b76e5a2a..238b57c7 100644 --- a/testsuite/tests/oxigraph.rs +++ b/testsuite/tests/oxigraph.rs @@ -20,6 +20,14 @@ fn oxigraph_parser_recovery_testsuite() -> Result<()> { ) } +#[test] +fn oxigraph_parser_error_testsuite() -> Result<()> { + check_testsuite( + "https://github.com/oxigraph/oxigraph/tests/parser-error/manifest.ttl", + &[], + ) +} + #[test] fn oxigraph_sparql_testsuite() -> Result<()> { check_testsuite( From 6edfb7a2f4624bf9f907142602c382400174c525 Mon Sep 17 00:00:00 2001 From: Tpt Date: Mon, 4 Sep 2023 21:32:58 +0200 Subject: [PATCH 076/217] Python: Adds location data to SyntaxError --- .cargo/build_config.py | 12 +++++---- python/src/io.rs | 56 +++++++++++++++++++++++++++++++++++++---- python/src/sparql.rs | 7 ++---- python/src/store.rs | 18 +++++++------ python/tests/test_io.py | 15 +++++++++++ 5 files changed, 85 insertions(+), 23 deletions(-) diff --git a/.cargo/build_config.py b/.cargo/build_config.py index 97dd9667..44079d9c 100644 --- a/.cargo/build_config.py +++ b/.cargo/build_config.py @@ -53,17 +53,19 @@ FLAGS_BLACKLIST = { "-Wclippy::std-instead-of-core", "-Wclippy::shadow-reuse", "-Wclippy::shadow-unrelated", - "-Wclippy::string-slice", # TODO: might be nice + "-Wclippy::string-slice", # TODO: might be nice "-Wclippy::too-many-lines", "-Wclippy::separated-literal-suffix", - "-Wclippy::unreachable", # TODO: might be nice - "-Wclippy::unwrap-used", # TODO: might be nice to use expect instead + "-Wclippy::unreachable", # TODO: might be nice + "-Wclippy::unwrap-used", # TODO: might be nice to use expect instead "-Wclippy::wildcard-enum-match-arm", # TODO: might be nice "-Wclippy::wildcard-imports", # TODO: might be nice } build_flags = set(DEFAULT_BUILD_FLAGS) -with urlopen(f"https://rust-lang.github.io/rust-clippy/rust-{MSRV}/lints.json") as response: +with urlopen( + f"https://rust-lang.github.io/rust-clippy/rust-{MSRV}/lints.json" +) as response: for lint in json.load(response): if lint["level"] == "allow" and lint["group"] != "nursery": build_flags.add(f"-Wclippy::{lint['id'].replace('_', '-')}") @@ -78,5 +80,5 @@ with open("./config.toml", "wt") as fp: fp.write("[build]\n") fp.write("rustflags = [\n") for flag in sorted(build_flags): - fp.write(f" \"{flag}\",\n") + fp.write(f' "{flag}",\n') fp.write("]\n") diff --git a/python/src/io.rs b/python/src/io.rs index 0b705c11..86f39992 100644 --- a/python/src/io.rs +++ b/python/src/io.rs @@ -12,6 +12,7 @@ use std::error::Error; use std::fs::File; use std::io::{self, BufWriter, Cursor, Read, Write}; use std::path::{Path, PathBuf}; +use std::sync::OnceLock; pub fn add_to_module(module: &PyModule) -> PyResult<()> { module.add_wrapped(wrap_pyfunction!(parse))?; @@ -62,8 +63,9 @@ pub fn parse( py: Python<'_>, ) -> PyResult { let format = rdf_format(format)?; - let input = if let Ok(path) = input.extract::(py) { - PyReadable::from_file(&path, py).map_err(map_io_err)? + let file_path = input.extract::(py).ok(); + let input = if let Some(file_path) = &file_path { + PyReadable::from_file(file_path, py).map_err(map_io_err)? } else { PyReadable::from_data(input, py) }; @@ -81,6 +83,7 @@ pub fn parse( } Ok(PyQuadReader { inner: parser.parse_read(input), + file_path, } .into_py(py)) } @@ -151,6 +154,7 @@ pub fn serialize(input: &PyAny, output: PyObject, format: &str, py: Python<'_>) #[pyclass(name = "QuadReader", module = "pyoxigraph")] pub struct PyQuadReader { inner: FromReadQuadReader, + file_path: Option, } #[pymethods] @@ -163,7 +167,10 @@ impl PyQuadReader { py.allow_threads(|| { self.inner .next() - .map(|q| Ok(q.map_err(map_parse_error)?.into())) + .map(|q| { + Ok(q.map_err(|e| map_parse_error(e, self.file_path.clone()))? + .into()) + }) .transpose() }) } @@ -314,9 +321,38 @@ pub fn map_io_err(error: io::Error) -> PyErr { } } -pub fn map_parse_error(error: ParseError) -> PyErr { +pub fn map_parse_error(error: ParseError, file_path: Option) -> PyErr { match error { - ParseError::Syntax(error) => PySyntaxError::new_err(error.to_string()), + ParseError::Syntax(error) => { + // Python 3.9 does not support end line and end column + if python_version() >= (3, 10, 0) { + let params = if let Some(location) = error.location() { + ( + file_path, + Some(location.start.line + 1), + Some(location.start.column + 1), + None::>, + Some(location.end.line + 1), + Some(location.end.column + 1), + ) + } else { + (None, None, None, None, None, None) + }; + PySyntaxError::new_err((error.to_string(), params)) + } else { + let params = if let Some(location) = error.location() { + ( + file_path, + Some(location.start.line + 1), + Some(location.start.column + 1), + None::>, + ) + } else { + (None, None, None, None) + }; + PySyntaxError::new_err((error.to_string(), params)) + } + } ParseError::Io(error) => map_io_err(error), } } @@ -345,3 +381,13 @@ pub fn allow_threads_unsafe(f: impl FnOnce() -> T) -> T { let _guard = RestoreGuard { tstate }; f() } + +fn python_version() -> (u8, u8, u8) { + static VERSION: OnceLock<(u8, u8, u8)> = OnceLock::new(); + *VERSION.get_or_init(|| { + Python::with_gil(|py| { + let v = py.version_info(); + (v.major, v.minor, v.patch) + }) + }) +} diff --git a/python/src/sparql.rs b/python/src/sparql.rs index 7c99707b..c7ad693b 100644 --- a/python/src/sparql.rs +++ b/python/src/sparql.rs @@ -1,4 +1,4 @@ -use crate::io::{allow_threads_unsafe, map_io_err}; +use crate::io::{allow_threads_unsafe, map_io_err, map_parse_error}; use crate::map_storage_error; use crate::model::*; use oxigraph::model::Term; @@ -233,10 +233,7 @@ pub fn map_evaluation_error(error: EvaluationError) -> PyErr { match error { EvaluationError::Parsing(error) => PySyntaxError::new_err(error.to_string()), EvaluationError::Storage(error) => map_storage_error(error), - EvaluationError::GraphParsing(error) => match error { - oxigraph::io::ParseError::Syntax(error) => PySyntaxError::new_err(error.to_string()), - oxigraph::io::ParseError::Io(error) => map_io_err(error), - }, + EvaluationError::GraphParsing(error) => map_parse_error(error, None), EvaluationError::ResultsParsing(error) => match error { oxigraph::sparql::results::ParseError::Syntax(error) => { PySyntaxError::new_err(error.to_string()) diff --git a/python/src/store.rs b/python/src/store.rs index d71d4936..70dda4cd 100644 --- a/python/src/store.rs +++ b/python/src/store.rs @@ -392,8 +392,9 @@ impl PyStore { } else { None }; - let input = if let Ok(path) = input.extract::(py) { - PyReadable::from_file(&path, py).map_err(map_io_err)? + let file_path = input.extract::(py).ok(); + let input = if let Some(file_path) = &file_path { + PyReadable::from_file(file_path, py).map_err(map_io_err)? } else { PyReadable::from_data(input, py) }; @@ -404,7 +405,7 @@ impl PyStore { } else { self.inner.load_dataset(input, format, base_iri) } - .map_err(map_loader_error) + .map_err(|e| map_loader_error(e, file_path)) }) } @@ -460,8 +461,9 @@ impl PyStore { } else { None }; - let input = if let Ok(path) = input.extract::(py) { - PyReadable::from_file(&path, py).map_err(map_io_err)? + let file_path = input.extract::(py).ok(); + let input = if let Some(file_path) = &file_path { + PyReadable::from_file(file_path, py).map_err(map_io_err)? } else { PyReadable::from_data(input, py) }; @@ -475,7 +477,7 @@ impl PyStore { .bulk_loader() .load_dataset(input, format, base_iri) } - .map_err(map_loader_error) + .map_err(|e| map_loader_error(e, file_path)) }) } @@ -833,10 +835,10 @@ pub fn map_storage_error(error: StorageError) -> PyErr { } } -pub fn map_loader_error(error: LoaderError) -> PyErr { +pub fn map_loader_error(error: LoaderError, file_path: Option) -> PyErr { match error { LoaderError::Storage(error) => map_storage_error(error), - LoaderError::Parsing(error) => map_parse_error(error), + LoaderError::Parsing(error) => map_parse_error(error, file_path), LoaderError::InvalidBaseIri { .. } => PyValueError::new_err(error.to_string()), } } diff --git a/python/tests/test_io.py b/python/tests/test_io.py index e9069e08..d70179e7 100644 --- a/python/tests/test_io.py +++ b/python/tests/test_io.py @@ -1,3 +1,4 @@ +import sys import unittest from io import BytesIO, StringIO, UnsupportedOperation from tempfile import NamedTemporaryFile, TemporaryFile @@ -83,6 +84,20 @@ class TestParse(unittest.TestCase): [EXAMPLE_QUAD], ) + def test_parse_syntax_error(self) -> None: + with NamedTemporaryFile() as fp: + fp.write(b"@base .\n") + fp.write(b' "p" "1"') + fp.flush() + with self.assertRaises(SyntaxError) as ctx: + list(parse(fp.name, "text/turtle")) + self.assertEqual(ctx.exception.filename, fp.name) + self.assertEqual(ctx.exception.lineno, 2) + self.assertEqual(ctx.exception.offset, 7) + if sys.version_info >= (3, 10): + self.assertEqual(ctx.exception.end_lineno, 2) + self.assertEqual(ctx.exception.end_offset, 10) + def test_parse_without_named_graphs(self) -> None: with self.assertRaises(SyntaxError) as _: list( From be002dd51e162679884b31ca58d0f0f5389d1272 Mon Sep 17 00:00:00 2001 From: Tpt Date: Fri, 8 Sep 2023 17:02:17 +0200 Subject: [PATCH 077/217] Migrates to new sha1 name --- Cargo.lock | 8 ++++---- lib/Cargo.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 50c7f71e..b4db7cbd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -963,7 +963,7 @@ dependencies = [ "oxsdatatypes", "rand", "regex", - "sha-1", + "sha1", "sha2", "siphasher", "sparesults", @@ -1618,10 +1618,10 @@ dependencies = [ ] [[package]] -name = "sha-1" -version = "0.10.1" +name = "sha1" +version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5058ada175748e33390e40e872bd0fe59a19f265d0158daa551c5a88a76009c" +checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" dependencies = [ "cfg-if", "cpufeatures", diff --git a/lib/Cargo.toml b/lib/Cargo.toml index 7b06b69a..2c37165f 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -24,7 +24,7 @@ rocksdb_debug = [] [dependencies] rand = "0.8" md-5 = "0.10" -sha-1 = "0.10" +sha1 = "0.10" sha2 = "0.10" digest = "0.10" regex = "1" From 99abe69ba1eb0c8b06600b3b28c1aa066112da52 Mon Sep 17 00:00:00 2001 From: Tpt Date: Tue, 5 Sep 2023 21:30:57 +0200 Subject: [PATCH 078/217] oxttl: Exposes prefixes --- lib/oxrdfio/src/parser.rs | 40 +++--- lib/oxrdfio/src/serializer.rs | 58 ++++---- lib/oxrdfxml/src/parser.rs | 80 +++++------ lib/oxrdfxml/src/serializer.rs | 58 ++++---- lib/oxttl/src/line_formats.rs | 47 ++++--- lib/oxttl/src/n3.rs | 241 ++++++++++++++++++++++---------- lib/oxttl/src/nquads.rs | 74 +++++----- lib/oxttl/src/ntriples.rs | 114 ++++++++------- lib/oxttl/src/terse.rs | 115 ++++++++------- lib/oxttl/src/toolkit/parser.rs | 42 ++++-- lib/oxttl/src/trig.rs | 225 +++++++++++++++++++---------- lib/oxttl/src/turtle.rs | 221 +++++++++++++++++++---------- 12 files changed, 792 insertions(+), 523 deletions(-) diff --git a/lib/oxrdfio/src/parser.rs b/lib/oxrdfio/src/parser.rs index 25f86598..bc88cfc4 100644 --- a/lib/oxrdfio/src/parser.rs +++ b/lib/oxrdfio/src/parser.rs @@ -272,19 +272,19 @@ impl RdfParser { /// Reads are buffered. /// /// ``` - /// use oxrdfio::{RdfFormat, RdfParser, ParseError}; + /// use oxrdfio::{RdfFormat, RdfParser}; /// - /// #[tokio::main(flavor = "current_thread")] - /// async fn main() -> Result<(), ParseError> { - /// let file = " ."; + /// # #[tokio::main(flavor = "current_thread")] + /// # async fn main() -> Result<(), oxrdfio::ParseError> { + /// let file = " ."; /// - /// let parser = RdfParser::from_format(RdfFormat::NTriples); - /// let mut reader = parser.parse_tokio_async_read(file.as_bytes()); - /// if let Some(quad) = reader.next().await { - /// assert_eq!(quad?.subject.to_string(), ""); - /// } - /// Ok(()) + /// let parser = RdfParser::from_format(RdfFormat::NTriples); + /// let mut reader = parser.parse_tokio_async_read(file.as_bytes()); + /// if let Some(quad) = reader.next().await { + /// assert_eq!(quad?.subject.to_string(), ""); /// } + /// # Ok(()) + /// # } /// ``` #[cfg(feature = "async-tokio")] pub fn parse_tokio_async_read( @@ -390,19 +390,19 @@ impl Iterator for FromReadQuadReader { /// Reads are buffered. /// /// ``` -/// use oxrdfio::{RdfFormat, RdfParser, ParseError}; +/// use oxrdfio::{RdfFormat, RdfParser}; /// -/// #[tokio::main(flavor = "current_thread")] -/// async fn main() -> Result<(), ParseError> { -/// let file = " ."; +/// # #[tokio::main(flavor = "current_thread")] +/// # async fn main() -> Result<(), oxrdfio::ParseError> { +/// let file = " ."; /// -/// let parser = RdfParser::from_format(RdfFormat::NTriples); -/// let mut reader = parser.parse_tokio_async_read(file.as_bytes()); -/// if let Some(quad) = reader.next().await { -/// assert_eq!(quad?.subject.to_string(), ""); -/// } -/// Ok(()) +/// let parser = RdfParser::from_format(RdfFormat::NTriples); +/// let mut reader = parser.parse_tokio_async_read(file.as_bytes()); +/// if let Some(quad) = reader.next().await { +/// assert_eq!(quad?.subject.to_string(), ""); /// } +/// # Ok(()) +/// # } /// ``` #[must_use] #[cfg(feature = "async-tokio")] diff --git a/lib/oxrdfio/src/serializer.rs b/lib/oxrdfio/src/serializer.rs index 0d931d81..9bfaceec 100644 --- a/lib/oxrdfio/src/serializer.rs +++ b/lib/oxrdfio/src/serializer.rs @@ -114,23 +114,22 @@ impl RdfSerializer { /// ``` /// use oxrdfio::{RdfFormat, RdfSerializer}; /// use oxrdf::{Quad, NamedNode}; - /// use std::io; /// - /// #[tokio::main(flavor = "current_thread")] - /// async fn main() -> io::Result<()> { - /// let mut buffer = Vec::new(); - /// let mut writer = RdfSerializer::from_format(RdfFormat::NQuads).serialize_to_tokio_async_write(&mut buffer); - /// writer.write_quad(&Quad { - /// subject: NamedNode::new_unchecked("http://example.com/s").into(), - /// predicate: NamedNode::new_unchecked("http://example.com/p"), - /// object: NamedNode::new_unchecked("http://example.com/o").into(), - /// graph_name: NamedNode::new_unchecked("http://example.com/g").into() - /// }).await?; - /// writer.finish().await?; + /// # #[tokio::main(flavor = "current_thread")] + /// # async fn main() -> std::io::Result<()> { + /// let mut buffer = Vec::new(); + /// let mut writer = RdfSerializer::from_format(RdfFormat::NQuads).serialize_to_tokio_async_write(&mut buffer); + /// writer.write_quad(&Quad { + /// subject: NamedNode::new_unchecked("http://example.com/s").into(), + /// predicate: NamedNode::new_unchecked("http://example.com/p"), + /// object: NamedNode::new_unchecked("http://example.com/o").into(), + /// graph_name: NamedNode::new_unchecked("http://example.com/g").into() + /// }).await?; + /// writer.finish().await?; /// - /// assert_eq!(buffer.as_slice(), " .\n".as_bytes()); - /// Ok(()) - /// } + /// assert_eq!(buffer.as_slice(), " .\n".as_bytes()); + /// # Ok(()) + /// # } /// ``` #[cfg(feature = "async-tokio")] pub fn serialize_to_tokio_async_write( @@ -239,23 +238,22 @@ impl ToWriteQuadWriter { /// ``` /// use oxrdfio::{RdfFormat, RdfSerializer}; /// use oxrdf::{Quad, NamedNode}; -/// use std::io; /// -/// #[tokio::main(flavor = "current_thread")] -/// async fn main() -> io::Result<()> { -/// let mut buffer = Vec::new(); -/// let mut writer = RdfSerializer::from_format(RdfFormat::NQuads).serialize_to_tokio_async_write(&mut buffer); -/// writer.write_quad(&Quad { -/// subject: NamedNode::new_unchecked("http://example.com/s").into(), -/// predicate: NamedNode::new_unchecked("http://example.com/p"), -/// object: NamedNode::new_unchecked("http://example.com/o").into(), -/// graph_name: NamedNode::new_unchecked("http://example.com/g").into() -/// }).await?; -/// writer.finish().await?; +/// # #[tokio::main(flavor = "current_thread")] +/// # async fn main() -> std::io::Result<()> { +/// let mut buffer = Vec::new(); +/// let mut writer = RdfSerializer::from_format(RdfFormat::NQuads).serialize_to_tokio_async_write(&mut buffer); +/// writer.write_quad(&Quad { +/// subject: NamedNode::new_unchecked("http://example.com/s").into(), +/// predicate: NamedNode::new_unchecked("http://example.com/p"), +/// object: NamedNode::new_unchecked("http://example.com/o").into(), +/// graph_name: NamedNode::new_unchecked("http://example.com/g").into() +/// }).await?; +/// writer.finish().await?; /// -/// assert_eq!(buffer.as_slice(), " .\n".as_bytes()); -/// Ok(()) -/// } +/// assert_eq!(buffer.as_slice(), " .\n".as_bytes()); +/// # Ok(()) +/// # } /// ``` #[must_use] #[cfg(feature = "async-tokio")] diff --git a/lib/oxrdfxml/src/parser.rs b/lib/oxrdfxml/src/parser.rs index dcd216a4..086d094a 100644 --- a/lib/oxrdfxml/src/parser.rs +++ b/lib/oxrdfxml/src/parser.rs @@ -108,31 +108,31 @@ impl RdfXmlParser { /// Count the number of people: /// ``` /// use oxrdf::{NamedNodeRef, vocab::rdf}; - /// use oxrdfxml::{RdfXmlParser, ParseError}; + /// use oxrdfxml::RdfXmlParser; /// - /// #[tokio::main(flavor = "current_thread")] - /// async fn main() -> Result<(), ParseError> { + /// # #[tokio::main(flavor = "current_thread")] + /// # async fn main() -> Result<(), oxrdfxml::ParseError> { /// let file = b" - /// - /// - /// - /// Foo - /// - /// - /// "; + /// + /// + /// + /// Foo + /// + /// + /// "; /// - /// let schema_person = NamedNodeRef::new_unchecked("http://schema.org/Person"); - /// let mut count = 0; - /// let mut parser = RdfXmlParser::new().parse_tokio_async_read(file.as_ref()); - /// while let Some(triple) = parser.next().await { - /// let triple = triple?; - /// if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { - /// count += 1; - /// } + /// let schema_person = NamedNodeRef::new_unchecked("http://schema.org/Person"); + /// let mut count = 0; + /// let mut parser = RdfXmlParser::new().parse_tokio_async_read(file.as_ref()); + /// while let Some(triple) = parser.next().await { + /// let triple = triple?; + /// if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { + /// count += 1; /// } - /// assert_eq!(2, count); - /// Ok(()) /// } + /// assert_eq!(2, count); + /// # Ok(()) + /// # } /// ``` #[cfg(feature = "async-tokio")] pub fn parse_tokio_async_read( @@ -234,31 +234,31 @@ impl FromReadRdfXmlReader { /// Count the number of people: /// ``` /// use oxrdf::{NamedNodeRef, vocab::rdf}; -/// use oxrdfxml::{RdfXmlParser, ParseError}; +/// use oxrdfxml::RdfXmlParser; /// -/// #[tokio::main(flavor = "current_thread")] -/// async fn main() -> Result<(), ParseError> { +/// # #[tokio::main(flavor = "current_thread")] +/// # async fn main() -> Result<(), oxrdfxml::ParseError> { /// let file = b" -/// -/// -/// -/// Foo -/// -/// -/// "; +/// +/// +/// +/// Foo +/// +/// +/// "; /// -/// let schema_person = NamedNodeRef::new_unchecked("http://schema.org/Person"); -/// let mut count = 0; -/// let mut parser = RdfXmlParser::new().parse_tokio_async_read(file.as_ref()); -/// while let Some(triple) = parser.next().await { -/// let triple = triple?; -/// if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { -/// count += 1; -/// } +/// let schema_person = NamedNodeRef::new_unchecked("http://schema.org/Person"); +/// let mut count = 0; +/// let mut parser = RdfXmlParser::new().parse_tokio_async_read(file.as_ref()); +/// while let Some(triple) = parser.next().await { +/// let triple = triple?; +/// if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { +/// count += 1; /// } -/// assert_eq!(2, count); -/// Ok(()) /// } +/// assert_eq!(2, count); +/// # Ok(()) +/// # } /// ``` #[cfg(feature = "async-tokio")] #[must_use] diff --git a/lib/oxrdfxml/src/serializer.rs b/lib/oxrdfxml/src/serializer.rs index d3e9949f..01e5b4b4 100644 --- a/lib/oxrdfxml/src/serializer.rs +++ b/lib/oxrdfxml/src/serializer.rs @@ -74,22 +74,21 @@ impl RdfXmlSerializer { /// ``` /// use oxrdf::{NamedNodeRef, TripleRef}; /// use oxrdfxml::RdfXmlSerializer; - /// use std::io::Result; /// - /// #[tokio::main(flavor = "current_thread")] - /// async fn main() -> Result<()> { - /// let mut writer = RdfXmlSerializer::new().serialize_to_tokio_async_write(Vec::new()); - /// writer.write_triple(TripleRef::new( - /// NamedNodeRef::new_unchecked("http://example.com#me"), - /// NamedNodeRef::new_unchecked("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), - /// NamedNodeRef::new_unchecked("http://schema.org/Person"), - /// )).await?; - /// assert_eq!( - /// b"\n\n\t\n\t\t\n\t\n", - /// writer.finish().await?.as_slice() - /// ); - /// Ok(()) - /// } + /// # #[tokio::main(flavor = "current_thread")] + /// # async fn main() -> std::io::Result<()> { + /// let mut writer = RdfXmlSerializer::new().serialize_to_tokio_async_write(Vec::new()); + /// writer.write_triple(TripleRef::new( + /// NamedNodeRef::new_unchecked("http://example.com#me"), + /// NamedNodeRef::new_unchecked("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), + /// NamedNodeRef::new_unchecked("http://schema.org/Person"), + /// )).await?; + /// assert_eq!( + /// b"\n\n\t\n\t\t\n\t\n", + /// writer.finish().await?.as_slice() + /// ); + /// # Ok(()) + /// # } /// ``` #[allow(clippy::unused_self)] #[cfg(feature = "async-tokio")] @@ -160,22 +159,21 @@ impl ToWriteRdfXmlWriter { /// ``` /// use oxrdf::{NamedNodeRef, TripleRef}; /// use oxrdfxml::RdfXmlSerializer; -/// use std::io::Result; /// -/// #[tokio::main(flavor = "current_thread")] -/// async fn main() -> Result<()> { -/// let mut writer = RdfXmlSerializer::new().serialize_to_tokio_async_write(Vec::new()); -/// writer.write_triple(TripleRef::new( -/// NamedNodeRef::new_unchecked("http://example.com#me"), -/// NamedNodeRef::new_unchecked("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), -/// NamedNodeRef::new_unchecked("http://schema.org/Person"), -/// )).await?; -/// assert_eq!( -/// b"\n\n\t\n\t\t\n\t\n", -/// writer.finish().await?.as_slice() -/// ); -/// Ok(()) -/// } +/// # #[tokio::main(flavor = "current_thread")] +/// # async fn main() -> std::io::Result<()> { +/// let mut writer = RdfXmlSerializer::new().serialize_to_tokio_async_write(Vec::new()); +/// writer.write_triple(TripleRef::new( +/// NamedNodeRef::new_unchecked("http://example.com#me"), +/// NamedNodeRef::new_unchecked("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), +/// NamedNodeRef::new_unchecked("http://schema.org/Person"), +/// )).await?; +/// assert_eq!( +/// b"\n\n\t\n\t\t\n\t\n", +/// writer.finish().await?.as_slice() +/// ); +/// # Ok(()) +/// # } /// ``` #[cfg(feature = "async-tokio")] #[must_use] diff --git a/lib/oxttl/src/line_formats.rs b/lib/oxttl/src/line_formats.rs index 1b4c31e6..4d6919a5 100644 --- a/lib/oxttl/src/line_formats.rs +++ b/lib/oxttl/src/line_formats.rs @@ -9,13 +9,15 @@ use oxrdf::{BlankNode, GraphName, Literal, NamedNode, Quad, Subject, Term}; pub struct NQuadsRecognizer { stack: Vec, + subjects: Vec, + predicates: Vec, + objects: Vec, +} +pub struct NQuadsRecognizerContext { with_graph_name: bool, #[cfg(feature = "rdf-star")] with_quoted_triples: bool, lexer_options: N3LexerOptions, - subjects: Vec, - predicates: Vec, - objects: Vec, } enum NQuadsState { @@ -39,6 +41,7 @@ enum NQuadsState { impl RuleRecognizer for NQuadsRecognizer { type TokenRecognizer = N3Lexer; type Output = Quad; + type Context = NQuadsRecognizerContext; fn error_recovery_state(mut self) -> Self { self.stack.clear(); @@ -51,6 +54,7 @@ impl RuleRecognizer for NQuadsRecognizer { fn recognize_next( mut self, token: N3Token, + context: &mut NQuadsRecognizerContext, results: &mut Vec, errors: &mut Vec, ) -> Self { @@ -69,7 +73,7 @@ impl RuleRecognizer for NQuadsRecognizer { self } #[cfg(feature = "rdf-star")] - N3Token::Punctuation("<<") if self.with_quoted_triples => { + N3Token::Punctuation("<<") if context.with_quoted_triples => { self.stack.push(NQuadsState::AfterQuotedSubject); self.stack.push(NQuadsState::ExpectSubject); self @@ -111,7 +115,7 @@ impl RuleRecognizer for NQuadsRecognizer { self } #[cfg(feature = "rdf-star")] - N3Token::Punctuation("<<") if self.with_quoted_triples => { + N3Token::Punctuation("<<") if context.with_quoted_triples => { self.stack.push(NQuadsState::AfterQuotedObject); self.stack.push(NQuadsState::ExpectSubject); self @@ -143,7 +147,7 @@ impl RuleRecognizer for NQuadsRecognizer { self.objects.push(Literal::new_simple_literal(value).into()); self.stack .push(NQuadsState::ExpectPossibleGraphOrEndOfQuotedTriple); - self.recognize_next(token, results, errors) + self.recognize_next(token, context, results, errors) } }, NQuadsState::ExpectLiteralDatatype { value } => match token { @@ -164,7 +168,7 @@ impl RuleRecognizer for NQuadsRecognizer { NQuadsState::ExpectPossibleGraphOrEndOfQuotedTriple => { if self.stack.is_empty() { match token { - N3Token::IriRef(g) if self.with_graph_name => { + N3Token::IriRef(g) if context.with_graph_name => { self.emit_quad( results, NamedNode::from(g).into(), @@ -172,7 +176,7 @@ impl RuleRecognizer for NQuadsRecognizer { self.stack.push(NQuadsState::ExpectDot); self } - N3Token::BlankNodeLabel(g) if self.with_graph_name => { + N3Token::BlankNodeLabel(g) if context.with_graph_name => { self.emit_quad(results, BlankNode::new_unchecked(g).into()); self.stack.push(NQuadsState::ExpectDot); self @@ -180,7 +184,7 @@ impl RuleRecognizer for NQuadsRecognizer { _ => { self.emit_quad(results, GraphName::DefaultGraph); self.stack.push(NQuadsState::ExpectDot); - self.recognize_next(token, results, errors) + self.recognize_next(token, context, results, errors) } } } else if token == N3Token::Punctuation(">>") { @@ -195,7 +199,7 @@ impl RuleRecognizer for NQuadsRecognizer { } else { errors.push("Quads should be followed by a dot".into()); self.stack.push(NQuadsState::ExpectSubject); - self.recognize_next(token, results, errors) + self.recognize_next(token, context, results, errors) }, #[cfg(feature = "rdf-star")] NQuadsState::AfterQuotedSubject => { @@ -206,7 +210,7 @@ impl RuleRecognizer for NQuadsRecognizer { }; self.subjects.push(triple.into()); self.stack.push(NQuadsState::ExpectPredicate); - self.recognize_next(token, results, errors) + self.recognize_next(token,context, results, errors) } #[cfg(feature = "rdf-star")] NQuadsState::AfterQuotedObject => { @@ -218,7 +222,7 @@ impl RuleRecognizer for NQuadsRecognizer { self.objects.push(triple.into()); self.stack .push(NQuadsState::ExpectPossibleGraphOrEndOfQuotedTriple); - self.recognize_next(token, results, errors) + self.recognize_next(token, context, results, errors) } } } else if token == N3Token::Punctuation(".") { @@ -229,7 +233,12 @@ impl RuleRecognizer for NQuadsRecognizer { } } - fn recognize_end(mut self, results: &mut Vec, errors: &mut Vec) { + fn recognize_end( + mut self, + _context: &mut NQuadsRecognizerContext, + results: &mut Vec, + errors: &mut Vec, + ) { match &*self.stack { [NQuadsState::ExpectSubject] | [] => (), [NQuadsState::ExpectDot] => errors.push("Triples should be followed by a dot".into()), @@ -246,8 +255,8 @@ impl RuleRecognizer for NQuadsRecognizer { } } - fn lexer_options(&self) -> &N3LexerOptions { - &self.lexer_options + fn lexer_options(context: &NQuadsRecognizerContext) -> &N3LexerOptions { + &context.lexer_options } } @@ -266,13 +275,15 @@ impl NQuadsRecognizer { ), NQuadsRecognizer { stack: vec![NQuadsState::ExpectSubject], + subjects: Vec::new(), + predicates: Vec::new(), + objects: Vec::new(), + }, + NQuadsRecognizerContext { with_graph_name, #[cfg(feature = "rdf-star")] with_quoted_triples, lexer_options: N3LexerOptions::default(), - subjects: Vec::new(), - predicates: Vec::new(), - objects: Vec::new(), }, ) } diff --git a/lib/oxttl/src/n3.rs b/lib/oxttl/src/n3.rs index 84e36235..c4f02613 100644 --- a/lib/oxttl/src/n3.rs +++ b/lib/oxttl/src/n3.rs @@ -272,30 +272,29 @@ impl N3Parser { /// ``` /// use oxrdf::{NamedNode, vocab::rdf}; /// use oxttl::n3::{N3Parser, N3Term}; - /// use oxttl::ParseError; /// - /// #[tokio::main(flavor = "current_thread")] - /// async fn main() -> Result<(), ParseError> { - /// let file = b"@base . - /// @prefix schema: . - /// a schema:Person ; - /// schema:name \"Foo\" . - /// a schema:Person ; - /// schema:name \"Bar\" ."; + /// # #[tokio::main(flavor = "current_thread")] + /// # async fn main() -> Result<(), oxttl::ParseError> { + /// let file = b"@base . + /// @prefix schema: . + /// a schema:Person ; + /// schema:name \"Foo\" . + /// a schema:Person ; + /// schema:name \"Bar\" ."; /// - /// let rdf_type = N3Term::NamedNode(rdf::TYPE.into_owned()); - /// let schema_person = N3Term::NamedNode(NamedNode::new_unchecked("http://schema.org/Person")); - /// let mut count = 0; - /// let mut parser = N3Parser::new().parse_tokio_async_read(file.as_ref()); - /// while let Some(triple) = parser.next().await { - /// let triple = triple?; - /// if triple.predicate == rdf_type && triple.object == schema_person { - /// count += 1; - /// } + /// let rdf_type = N3Term::NamedNode(rdf::TYPE.into_owned()); + /// let schema_person = N3Term::NamedNode(NamedNode::new_unchecked("http://schema.org/Person")); + /// let mut count = 0; + /// let mut parser = N3Parser::new().parse_tokio_async_read(file.as_ref()); + /// while let Some(triple) = parser.next().await { + /// let triple = triple?; + /// if triple.predicate == rdf_type && triple.object == schema_person { + /// count += 1; /// } - /// assert_eq!(2, count); - /// Ok(()) /// } + /// assert_eq!(2, count); + /// # Ok(()) + /// # } /// ``` #[cfg(feature = "async-tokio")] pub fn parse_tokio_async_read( @@ -382,6 +381,33 @@ pub struct FromReadN3Reader { inner: FromReadIterator, } +impl FromReadN3Reader { + /// The list of IRI prefixes considered at the current step of the parsing. + /// + /// This method returns the mapping from prefix name to prefix value. + /// It is empty at the beginning of the parsing and gets updated when prefixes are encountered. + /// It should be full at the end of the parsing (but if a prefix is overridden, only the latest version will be returned). + /// + /// ``` + /// use oxttl::N3Parser; + /// + /// let file = b"@base . + /// @prefix schema: . + /// a schema:Person ; + /// schema:name \"Foo\" ."; + /// + /// let mut reader = N3Parser::new().parse_read(file.as_ref()); + /// assert!(reader.prefixes().is_empty()); // No prefix at the beginning + /// + /// reader.next().unwrap()?; // We read the first triple + /// assert_eq!(reader.prefixes()["schema"], "http://schema.org/"); // There are now prefixes + /// # Result::<_,Box>::Ok(()) + /// ``` + pub fn prefixes(&self) -> &HashMap> { + &self.inner.parser.context.prefixes + } +} + impl Iterator for FromReadN3Reader { type Item = Result; @@ -396,30 +422,29 @@ impl Iterator for FromReadN3Reader { /// ``` /// use oxrdf::{NamedNode, vocab::rdf}; /// use oxttl::n3::{N3Parser, N3Term}; -/// use oxttl::ParseError; /// -/// #[tokio::main(flavor = "current_thread")] -/// async fn main() -> Result<(), ParseError> { -/// let file = b"@base . -/// @prefix schema: . -/// a schema:Person ; -/// schema:name \"Foo\" . -/// a schema:Person ; -/// schema:name \"Bar\" ."; +/// # #[tokio::main(flavor = "current_thread")] +/// # async fn main() -> Result<(), oxttl::ParseError> { +/// let file = b"@base . +/// @prefix schema: . +/// a schema:Person ; +/// schema:name \"Foo\" . +/// a schema:Person ; +/// schema:name \"Bar\" ."; /// -/// let rdf_type = N3Term::NamedNode(rdf::TYPE.into_owned()); -/// let schema_person = N3Term::NamedNode(NamedNode::new_unchecked("http://schema.org/Person")); -/// let mut count = 0; -/// let mut parser = N3Parser::new().parse_tokio_async_read(file.as_ref()); -/// while let Some(triple) = parser.next().await { -/// let triple = triple?; -/// if triple.predicate == rdf_type && triple.object == schema_person { -/// count += 1; -/// } +/// let rdf_type = N3Term::NamedNode(rdf::TYPE.into_owned()); +/// let schema_person = N3Term::NamedNode(NamedNode::new_unchecked("http://schema.org/Person")); +/// let mut count = 0; +/// let mut parser = N3Parser::new().parse_tokio_async_read(file.as_ref()); +/// while let Some(triple) = parser.next().await { +/// let triple = triple?; +/// if triple.predicate == rdf_type && triple.object == schema_person { +/// count += 1; /// } -/// assert_eq!(2, count); -/// Ok(()) /// } +/// assert_eq!(2, count); +/// # Ok(()) +/// # } /// ``` #[cfg(feature = "async-tokio")] #[must_use] @@ -433,6 +458,34 @@ impl FromTokioAsyncReadN3Reader { pub async fn next(&mut self) -> Option> { Some(self.inner.next().await?.map(Into::into)) } + + /// The list of IRI prefixes considered at the current step of the parsing. + /// + /// This method returns the mapping from prefix name to prefix value. + /// It is empty at the beginning of the parsing and gets updated when prefixes are encountered. + /// It should be full at the end of the parsing (but if a prefix is overridden, only the latest version will be returned). + /// + /// ``` + /// use oxttl::N3Parser; + /// + /// # #[tokio::main(flavor = "current_thread")] + /// # async fn main() -> Result<(), oxttl::ParseError> { + /// let file = b"@base . + /// @prefix schema: . + /// a schema:Person ; + /// schema:name \"Foo\" ."; + /// + /// let mut reader = N3Parser::new().parse_tokio_async_read(file.as_ref()); + /// assert!(reader.prefixes().is_empty()); // No prefix at the beginning + /// + /// reader.next().await.unwrap()?; // We read the first triple + /// assert_eq!(reader.prefixes()["schema"], "http://schema.org/"); // There are now prefixes + /// # Ok(()) + /// # } + /// ``` + pub fn prefixes(&self) -> &HashMap> { + &self.inner.parser.context.prefixes + } } /// Parses a N3 file by using a low-level API. Can be built using [`N3Parser::parse`]. @@ -501,6 +554,32 @@ impl LowLevelN3Reader { pub fn read_next(&mut self) -> Option> { self.parser.read_next() } + + /// The list of IRI prefixes considered at the current step of the parsing. + /// + /// This method returns the mapping from prefix name to prefix value. + /// It is empty at the beginning of the parsing and gets updated when prefixes are encountered. + /// It should be full at the end of the parsing (but if a prefix is overridden, only the latest version will be returned). + /// + /// ``` + /// use oxttl::N3Parser; + /// + /// let file = b"@base . + /// @prefix schema: . + /// a schema:Person ; + /// schema:name \"Foo\" ."; + /// + /// let mut reader = N3Parser::new().parse(); + /// reader.extend_from_slice(file); + /// assert!(reader.prefixes().is_empty()); // No prefix at the beginning + /// + /// reader.read_next().unwrap()?; // We read the first triple + /// assert_eq!(reader.prefixes()["schema"], "http://schema.org/"); // There are now prefixes + /// # Result::<_,Box>::Ok(()) + /// ``` + pub fn prefixes(&self) -> &HashMap> { + &self.parser.context.prefixes + } } #[derive(Clone)] @@ -511,16 +590,20 @@ enum Predicate { struct N3Recognizer { stack: Vec, - lexer_options: N3LexerOptions, - prefixes: HashMap>, terms: Vec, predicates: Vec, contexts: Vec, } +struct N3RecognizerContext { + lexer_options: N3LexerOptions, + prefixes: HashMap>, +} + impl RuleRecognizer for N3Recognizer { type TokenRecognizer = N3Lexer; type Output = N3Quad; + type Context = N3RecognizerContext; fn error_recovery_state(mut self) -> Self { self.stack.clear(); @@ -533,6 +616,7 @@ impl RuleRecognizer for N3Recognizer { fn recognize_next( mut self, token: N3Token, + context: &mut N3RecognizerContext, results: &mut Vec, errors: &mut Vec, ) -> Self { @@ -570,7 +654,7 @@ impl RuleRecognizer for N3Recognizer { _ => { self.stack.push(N3State::N3DocExpectDot); self.stack.push(N3State::Triples); - self.recognize_next(token, results, errors) + self.recognize_next(token, context, results, errors) } } }, @@ -579,12 +663,12 @@ impl RuleRecognizer for N3Recognizer { self } else { errors.push("A dot is expected at the end of N3 statements".into()); - self.recognize_next(token, results, errors) + self.recognize_next(token, context, results, errors) } }, N3State::BaseExpectIri => match token { N3Token::IriRef(iri) => { - self.lexer_options.base_iri = Some(iri); + context.lexer_options.base_iri = Some(iri); self } _ => self.error(errors, "The BASE keyword should be followed by an IRI"), @@ -600,7 +684,7 @@ impl RuleRecognizer for N3Recognizer { }, N3State::PrefixExpectIri { name } => match token { N3Token::IriRef(iri) => { - self.prefixes.insert(name, iri); + context.prefixes.insert(name, iri); self } _ => self.error(errors, "The PREFIX declaration should be followed by a prefix and its value as an IRI"), @@ -609,25 +693,25 @@ impl RuleRecognizer for N3Recognizer { N3State::Triples => { self.stack.push(N3State::TriplesMiddle); self.stack.push(N3State::Path); - self.recognize_next(token, results, errors) + self.recognize_next(token, context, results, errors) }, N3State::TriplesMiddle => if matches!(token, N3Token::Punctuation("." | "]" | "}" | ")")) { - self.recognize_next(token, results, errors) + self.recognize_next(token, context, results, errors) } else { self.stack.push(N3State::TriplesEnd); self.stack.push(N3State::PredicateObjectList); - self.recognize_next(token, results, errors) + self.recognize_next(token, context, results, errors) }, N3State::TriplesEnd => { self.terms.pop(); - self.recognize_next(token, results, errors) + self.recognize_next(token, context, results, errors) }, // [10] predicateObjectList ::= verb objectList ( ";" ( verb objectList) ? ) * N3State::PredicateObjectList => { self.stack.push(N3State::PredicateObjectListEnd); self.stack.push(N3State::ObjectsList); self.stack.push(N3State::Verb); - self.recognize_next(token, results, errors) + self.recognize_next(token, context, results, errors) }, N3State::PredicateObjectListEnd => { self.predicates.pop(); @@ -635,25 +719,25 @@ impl RuleRecognizer for N3Recognizer { self.stack.push(N3State::PredicateObjectListPossibleContinuation); self } else { - self.recognize_next(token, results, errors) + self.recognize_next(token, context, results, errors) } }, N3State::PredicateObjectListPossibleContinuation => if token == N3Token::Punctuation(";") { self.stack.push(N3State::PredicateObjectListPossibleContinuation); self } else if matches!(token, N3Token::Punctuation(";" | "." | "}" | "]" | ")")) { - self.recognize_next(token, results, errors) + self.recognize_next(token, context, results, errors) } else { self.stack.push(N3State::PredicateObjectListEnd); self.stack.push(N3State::ObjectsList); self.stack.push(N3State::Verb); - self.recognize_next(token, results, errors) + self.recognize_next(token, context, results, errors) }, // [11] objectList ::= object ( "," object) * N3State::ObjectsList => { self.stack.push(N3State::ObjectsListEnd); self.stack.push(N3State::Path); - self.recognize_next(token, results, errors) + self.recognize_next(token, context, results, errors) } N3State::ObjectsListEnd => { let object = self.terms.pop().unwrap(); @@ -675,7 +759,7 @@ impl RuleRecognizer for N3Recognizer { self.stack.push(N3State::Path); self } else { - self.recognize_next(token, results, errors) + self.recognize_next(token, context, results, errors) } }, // [12] verb ::= predicate | "a" | ( "has" expression) | ( "is" expression "of") | "=" | "<=" | "=>" @@ -715,16 +799,16 @@ impl RuleRecognizer for N3Recognizer { _ => { self.stack.push(N3State::AfterRegularVerb); self.stack.push(N3State::Path); - self.recognize_next(token, results, errors) + self.recognize_next(token, context, results, errors) } } N3State::AfterRegularVerb => { self.predicates.push(Predicate::Regular(self.terms.pop().unwrap())); - self.recognize_next(token, results, errors) + self.recognize_next(token, context, results, errors) } N3State::AfterInvertedVerb => { self.predicates.push(Predicate::Inverted(self.terms.pop().unwrap())); - self.recognize_next(token, results, errors) + self.recognize_next(token, context, results, errors) } N3State::AfterVerbIs => match token { N3Token::PlainKeyword("of") => { @@ -742,7 +826,7 @@ impl RuleRecognizer for N3Recognizer { N3State::Path => { self.stack.push(N3State::PathFollowUp); self.stack.push(N3State::PathItem); - self.recognize_next(token, results, errors) + self.recognize_next(token, context, results, errors) } N3State::PathFollowUp => match token { N3Token::Punctuation("!") => { @@ -755,7 +839,7 @@ impl RuleRecognizer for N3Recognizer { self.stack.push(N3State::PathItem); self } - _ => self.recognize_next(token, results, errors) + _ => self.recognize_next(token, context, results, errors) }, N3State::PathAfterIndicator { is_inverse } => { let predicate = self.terms.pop().unwrap(); @@ -764,7 +848,7 @@ impl RuleRecognizer for N3Recognizer { results.push(if is_inverse { self.quad(current.clone(), predicate, previous) } else { self.quad(previous, predicate, current.clone())}); self.terms.push(current.into()); self.stack.push(N3State::PathFollowUp); - self.recognize_next(token, results, errors) + self.recognize_next(token, context, results, errors) }, // [18] pathItem ::= iri | blankNode | quickVar | collection | blankNodePropertyList | iriPropertyList | literal | formula // [19] literal ::= rdfLiteral | numericLiteral | BOOLEAN_LITERAL @@ -784,7 +868,7 @@ impl RuleRecognizer for N3Recognizer { self.terms.push(NamedNode::from(iri).into()); self } - N3Token::PrefixedName { prefix, local, might_be_invalid_iri } => match resolve_local_name(prefix, &local, might_be_invalid_iri, &self.prefixes) { + N3Token::PrefixedName { prefix, local, might_be_invalid_iri } => match resolve_local_name(prefix, &local, might_be_invalid_iri, &context.prefixes) { Ok(t) => { self.terms.push(t.into()); self @@ -852,14 +936,14 @@ impl RuleRecognizer for N3Recognizer { self.terms.push(BlankNode::default().into()); self.stack.push(N3State::PropertyListEnd); self.stack.push(N3State::PredicateObjectList); - self.recognize_next(token, results, errors) + self.recognize_next(token, context, results, errors) } } N3State::PropertyListEnd => if token == N3Token::Punctuation("]") { self } else { errors.push("blank node property lists should end with a ']'".into()); - self.recognize_next(token, results, errors) + self.recognize_next(token, context, results, errors) } N3State::IriPropertyList => match token { N3Token::IriRef(id) => { @@ -868,7 +952,7 @@ impl RuleRecognizer for N3Recognizer { self.stack.push(N3State::PredicateObjectList); self } - N3Token::PrefixedName { prefix, local, might_be_invalid_iri } => match resolve_local_name(prefix, &local, might_be_invalid_iri, &self.prefixes) { + N3Token::PrefixedName { prefix, local, might_be_invalid_iri } => match resolve_local_name(prefix, &local, might_be_invalid_iri, &context.prefixes) { Ok(t) => { self.terms.push(t.into()); self.stack.push(N3State::PropertyListEnd); @@ -890,7 +974,7 @@ impl RuleRecognizer for N3Recognizer { self.terms.push(root.into()); self.stack.push(N3State::CollectionPossibleEnd); self.stack.push(N3State::Path); - self.recognize_next(token, results, errors) + self.recognize_next(token, context, results, errors) }, N3State::CollectionPossibleEnd => { let value = self.terms.pop().unwrap(); @@ -917,7 +1001,7 @@ impl RuleRecognizer for N3Recognizer { self.terms.push(new.into()); self.stack.push(N3State::CollectionPossibleEnd); self.stack.push(N3State::Path); - self.recognize_next(token, results, errors) + self.recognize_next(token, context, results, errors) } } N3State::LiteralPossibleSuffix { value } => { @@ -932,7 +1016,7 @@ impl RuleRecognizer for N3Recognizer { } _ => { self.terms.push(Literal::new_simple_literal(value).into()); - self.recognize_next(token, results, errors) + self.recognize_next(token, context, results, errors) } } } @@ -942,7 +1026,7 @@ impl RuleRecognizer for N3Recognizer { self.terms.push(Literal::new_typed_literal(value, datatype).into()); self }, - N3Token::PrefixedName { prefix, local, might_be_invalid_iri } => match resolve_local_name(prefix, &local, might_be_invalid_iri, &self.prefixes) { + N3Token::PrefixedName { prefix, local, might_be_invalid_iri } => match resolve_local_name(prefix, &local, might_be_invalid_iri, &context.prefixes) { Ok(datatype) =>{ self.terms.push(Literal::new_typed_literal(value, datatype).into()); self @@ -950,7 +1034,7 @@ impl RuleRecognizer for N3Recognizer { Err(e) => self.error(errors, e) } _ => { - self.error(errors, "Expecting a datatype IRI after '^^, found TOKEN").recognize_next(token, results, errors) + self.error(errors, "Expecting a datatype IRI after '^^, found TOKEN").recognize_next(token, context, results, errors) } } } @@ -984,7 +1068,7 @@ impl RuleRecognizer for N3Recognizer { _ => { self.stack.push(N3State::FormulaContentExpectDot); self.stack.push(N3State::Triples); - self.recognize_next(token, results, errors) + self.recognize_next(token, context, results, errors) } } } @@ -1001,7 +1085,7 @@ impl RuleRecognizer for N3Recognizer { _ => { errors.push("A dot is expected at the end of N3 statements".into()); self.stack.push(N3State::FormulaContent); - self.recognize_next(token, results, errors) + self.recognize_next(token, context, results, errors) } } } @@ -1016,6 +1100,7 @@ impl RuleRecognizer for N3Recognizer { fn recognize_end( self, + _state: &mut N3RecognizerContext, _results: &mut Vec, errors: &mut Vec, ) { @@ -1025,8 +1110,8 @@ impl RuleRecognizer for N3Recognizer { } } - fn lexer_options(&self) -> &N3LexerOptions { - &self.lexer_options + fn lexer_options(context: &N3RecognizerContext) -> &N3LexerOptions { + &context.lexer_options } } @@ -1045,12 +1130,14 @@ impl N3Recognizer { ), N3Recognizer { stack: vec![N3State::N3Doc], - lexer_options: N3LexerOptions { base_iri }, - prefixes, terms: Vec::new(), predicates: Vec::new(), contexts: Vec::new(), }, + N3RecognizerContext { + lexer_options: N3LexerOptions { base_iri }, + prefixes, + }, ) } diff --git a/lib/oxttl/src/nquads.rs b/lib/oxttl/src/nquads.rs index caa7c642..c2bf35cd 100644 --- a/lib/oxttl/src/nquads.rs +++ b/lib/oxttl/src/nquads.rs @@ -90,27 +90,27 @@ impl NQuadsParser { /// Count the number of people: /// ``` /// use oxrdf::{NamedNodeRef, vocab::rdf}; - /// use oxttl::{ParseError, NQuadsParser}; + /// use oxttl::NQuadsParser; /// - /// #[tokio::main(flavor = "current_thread")] - /// async fn main() -> Result<(), ParseError> { - /// let file = b" . + /// # #[tokio::main(flavor = "current_thread")] + /// # async fn main() -> Result<(), oxttl::ParseError> { + /// let file = b" . /// \"Foo\" . /// . /// \"Bar\" ."; /// - /// let schema_person = NamedNodeRef::new_unchecked("http://schema.org/Person"); - /// let mut count = 0; - /// let mut parser = NQuadsParser::new().parse_tokio_async_read(file.as_ref()); - /// while let Some(triple) = parser.next().await { - /// let triple = triple?; - /// if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { - /// count += 1; - /// } + /// let schema_person = NamedNodeRef::new_unchecked("http://schema.org/Person"); + /// let mut count = 0; + /// let mut parser = NQuadsParser::new().parse_tokio_async_read(file.as_ref()); + /// while let Some(triple) = parser.next().await { + /// let triple = triple?; + /// if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { + /// count += 1; /// } - /// assert_eq!(2, count); - /// Ok(()) /// } + /// assert_eq!(2, count); + /// # Ok(()) + /// # } /// ``` #[cfg(feature = "async-tokio")] pub fn parse_tokio_async_read( @@ -211,27 +211,27 @@ impl Iterator for FromReadNQuadsReader { /// Count the number of people: /// ``` /// use oxrdf::{NamedNodeRef, vocab::rdf}; -/// use oxttl::{ParseError, NQuadsParser}; +/// use oxttl::NQuadsParser; /// -/// #[tokio::main(flavor = "current_thread")] -/// async fn main() -> Result<(), ParseError> { -/// let file = b" . +/// # #[tokio::main(flavor = "current_thread")] +/// # async fn main() -> Result<(), oxttl::ParseError> { +/// let file = b" . /// \"Foo\" . /// . /// \"Bar\" ."; /// -/// let schema_person = NamedNodeRef::new_unchecked("http://schema.org/Person"); -/// let mut count = 0; -/// let mut parser = NQuadsParser::new().parse_tokio_async_read(file.as_ref()); -/// while let Some(triple) = parser.next().await { -/// let triple = triple?; -/// if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { -/// count += 1; -/// } +/// let schema_person = NamedNodeRef::new_unchecked("http://schema.org/Person"); +/// let mut count = 0; +/// let mut parser = NQuadsParser::new().parse_tokio_async_read(file.as_ref()); +/// while let Some(triple) = parser.next().await { +/// let triple = triple?; +/// if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { +/// count += 1; /// } -/// assert_eq!(2, count); -/// Ok(()) /// } +/// assert_eq!(2, count); +/// # Ok(()) +/// # } /// ``` #[cfg(feature = "async-tokio")] #[must_use] @@ -377,10 +377,9 @@ impl NQuadsSerializer { /// ``` /// use oxrdf::{NamedNodeRef, QuadRef}; /// use oxttl::NQuadsSerializer; - /// use std::io::Result; /// - /// #[tokio::main(flavor = "current_thread")] - /// async fn main() -> Result<()> { + /// # #[tokio::main(flavor = "current_thread")] + /// # async fn main() -> std::io::Result<()> { /// let mut writer = NQuadsSerializer::new().serialize_to_tokio_async_write(Vec::new()); /// writer.write_quad(QuadRef::new( /// NamedNodeRef::new_unchecked("http://example.com#me"), @@ -392,8 +391,8 @@ impl NQuadsSerializer { /// b" .\n", /// writer.finish().as_slice() /// ); - /// Ok(()) - /// } + /// # Ok(()) + /// # } /// ``` #[cfg(feature = "async-tokio")] pub fn serialize_to_tokio_async_write( @@ -475,10 +474,9 @@ impl ToWriteNQuadsWriter { /// ``` /// use oxrdf::{NamedNodeRef, QuadRef}; /// use oxttl::NQuadsSerializer; -/// use std::io::Result; /// -/// #[tokio::main(flavor = "current_thread")] -/// async fn main() -> Result<()> { +/// # #[tokio::main(flavor = "current_thread")] +/// # async fn main() -> std::io::Result<()> { /// let mut writer = NQuadsSerializer::new().serialize_to_tokio_async_write(Vec::new()); /// writer.write_quad(QuadRef::new( /// NamedNodeRef::new_unchecked("http://example.com#me"), @@ -490,8 +488,8 @@ impl ToWriteNQuadsWriter { /// b" .\n", /// writer.finish().as_slice() /// ); -/// Ok(()) -/// } +/// # Ok(()) +/// # } /// ``` #[cfg(feature = "async-tokio")] #[must_use] diff --git a/lib/oxttl/src/ntriples.rs b/lib/oxttl/src/ntriples.rs index f8c32b12..4fd06227 100644 --- a/lib/oxttl/src/ntriples.rs +++ b/lib/oxttl/src/ntriples.rs @@ -91,27 +91,27 @@ impl NTriplesParser { /// Count the number of people: /// ``` /// use oxrdf::{NamedNodeRef, vocab::rdf}; - /// use oxttl::{ParseError, NTriplesParser}; + /// use oxttl::NTriplesParser; /// - /// #[tokio::main(flavor = "current_thread")] - /// async fn main() -> Result<(), ParseError> { - /// let file = b" . + /// # #[tokio::main(flavor = "current_thread")] + /// # async fn main() -> Result<(), oxttl::ParseError> { + /// let file = b" . /// \"Foo\" . /// . /// \"Bar\" ."; /// - /// let schema_person = NamedNodeRef::new_unchecked("http://schema.org/Person"); - /// let mut count = 0; - /// let mut parser = NTriplesParser::new().parse_tokio_async_read(file.as_ref()); - /// while let Some(triple) = parser.next().await { - /// let triple = triple?; - /// if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { - /// count += 1; - /// } + /// let schema_person = NamedNodeRef::new_unchecked("http://schema.org/Person"); + /// let mut count = 0; + /// let mut parser = NTriplesParser::new().parse_tokio_async_read(file.as_ref()); + /// while let Some(triple) = parser.next().await { + /// let triple = triple?; + /// if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { + /// count += 1; /// } - /// assert_eq!(2, count); - /// Ok(()) /// } + /// assert_eq!(2, count); + /// # Ok(()) + /// # } /// ``` #[cfg(feature = "async-tokio")] pub fn parse_tokio_async_read( @@ -212,27 +212,27 @@ impl Iterator for FromReadNTriplesReader { /// Count the number of people: /// ``` /// use oxrdf::{NamedNodeRef, vocab::rdf}; -/// use oxttl::{ParseError, NTriplesParser}; +/// use oxttl::NTriplesParser; /// -/// #[tokio::main(flavor = "current_thread")] -/// async fn main() -> Result<(), ParseError> { -/// let file = b" . +/// # #[tokio::main(flavor = "current_thread")] +/// # async fn main() -> Result<(), oxttl::ParseError> { +/// let file = b" . /// \"Foo\" . /// . /// \"Bar\" ."; /// -/// let schema_person = NamedNodeRef::new_unchecked("http://schema.org/Person"); -/// let mut count = 0; -/// let mut parser = NTriplesParser::new().parse_tokio_async_read(file.as_ref()); -/// while let Some(triple) = parser.next().await { -/// let triple = triple?; -/// if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { -/// count += 1; -/// } +/// let schema_person = NamedNodeRef::new_unchecked("http://schema.org/Person"); +/// let mut count = 0; +/// let mut parser = NTriplesParser::new().parse_tokio_async_read(file.as_ref()); +/// while let Some(triple) = parser.next().await { +/// let triple = triple?; +/// if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { +/// count += 1; /// } -/// assert_eq!(2, count); -/// Ok(()) /// } +/// assert_eq!(2, count); +/// # Ok(()) +/// # } /// ``` #[cfg(feature = "async-tokio")] #[must_use] @@ -376,22 +376,21 @@ impl NTriplesSerializer { /// ``` /// use oxrdf::{NamedNodeRef, TripleRef}; /// use oxttl::NTriplesSerializer; - /// use std::io::Result; /// - /// #[tokio::main(flavor = "current_thread")] - /// async fn main() -> Result<()> { - /// let mut writer = NTriplesSerializer::new().serialize_to_tokio_async_write(Vec::new()); - /// writer.write_triple(TripleRef::new( - /// NamedNodeRef::new_unchecked("http://example.com#me"), - /// NamedNodeRef::new_unchecked("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), - /// NamedNodeRef::new_unchecked("http://schema.org/Person"), - /// )).await?; - /// assert_eq!( - /// b" .\n", - /// writer.finish().as_slice() - /// ); - /// Ok(()) - /// } + /// # #[tokio::main(flavor = "current_thread")] + /// # async fn main() -> std::io::Result<()> { + /// let mut writer = NTriplesSerializer::new().serialize_to_tokio_async_write(Vec::new()); + /// writer.write_triple(TripleRef::new( + /// NamedNodeRef::new_unchecked("http://example.com#me"), + /// NamedNodeRef::new_unchecked("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), + /// NamedNodeRef::new_unchecked("http://schema.org/Person"), + /// )).await?; + /// assert_eq!( + /// b" .\n", + /// writer.finish().as_slice() + /// ); + /// # Ok(()) + /// # } /// ``` #[cfg(feature = "async-tokio")] pub fn serialize_to_tokio_async_write( @@ -471,22 +470,21 @@ impl ToWriteNTriplesWriter { /// ``` /// use oxrdf::{NamedNodeRef, TripleRef}; /// use oxttl::NTriplesSerializer; -/// use std::io::Result; /// -/// #[tokio::main(flavor = "current_thread")] -/// async fn main() -> Result<()> { -/// let mut writer = NTriplesSerializer::new().serialize_to_tokio_async_write(Vec::new()); -/// writer.write_triple(TripleRef::new( -/// NamedNodeRef::new_unchecked("http://example.com#me"), -/// NamedNodeRef::new_unchecked("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), -/// NamedNodeRef::new_unchecked("http://schema.org/Person") -/// )).await?; -/// assert_eq!( -/// b" .\n", -/// writer.finish().as_slice() -/// ); -/// Ok(()) -/// } +/// # #[tokio::main(flavor = "current_thread")] +/// # async fn main() -> std::io::Result<()> { +/// let mut writer = NTriplesSerializer::new().serialize_to_tokio_async_write(Vec::new()); +/// writer.write_triple(TripleRef::new( +/// NamedNodeRef::new_unchecked("http://example.com#me"), +/// NamedNodeRef::new_unchecked("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), +/// NamedNodeRef::new_unchecked("http://schema.org/Person") +/// )).await?; +/// assert_eq!( +/// b" .\n", +/// writer.finish().as_slice() +/// ); +/// # Ok(()) +/// # } /// ``` #[cfg(feature = "async-tokio")] #[must_use] diff --git a/lib/oxttl/src/terse.rs b/lib/oxttl/src/terse.rs index f99aaf4d..ecce10b6 100644 --- a/lib/oxttl/src/terse.rs +++ b/lib/oxttl/src/terse.rs @@ -14,20 +14,25 @@ use std::collections::HashMap; pub struct TriGRecognizer { stack: Vec, - with_graph_name: bool, - #[cfg(feature = "rdf-star")] - with_quoted_triples: bool, - lexer_options: N3LexerOptions, - prefixes: HashMap>, cur_subject: Vec, cur_predicate: Vec, cur_object: Vec, cur_graph: GraphName, } +#[allow(clippy::partial_pub_fields)] +pub struct TriGRecognizerContext { + lexer_options: N3LexerOptions, + pub with_graph_name: bool, + #[cfg(feature = "rdf-star")] + pub with_quoted_triples: bool, + pub prefixes: HashMap>, +} + impl RuleRecognizer for TriGRecognizer { type TokenRecognizer = N3Lexer; type Output = Quad; + type Context = TriGRecognizerContext; fn error_recovery_state(mut self) -> Self { self.stack.clear(); @@ -41,6 +46,7 @@ impl RuleRecognizer for TriGRecognizer { fn recognize_next( mut self, token: N3Token, + context: &mut TriGRecognizerContext, results: &mut Vec, errors: &mut Vec, ) -> Self { @@ -75,18 +81,18 @@ impl RuleRecognizer for TriGRecognizer { self.stack.push(TriGState::BaseExpectIri); self } - N3Token::PlainKeyword(k) if k.eq_ignore_ascii_case("graph") && self.with_graph_name => { + N3Token::PlainKeyword(k) if k.eq_ignore_ascii_case("graph") && context.with_graph_name => { self.stack.push(TriGState::WrappedGraph); self.stack.push(TriGState::GraphName); self } - N3Token::Punctuation("{") if self.with_graph_name => { + N3Token::Punctuation("{") if context.with_graph_name => { self.stack.push(TriGState::WrappedGraph); - self.recognize_next(token, results, errors) + self.recognize_next(token, context,results, errors) } _ => { self.stack.push(TriGState::TriplesOrGraph); - self.recognize_next(token, results, errors) + self.recognize_next(token, context,results, errors) } } }, @@ -96,12 +102,12 @@ impl RuleRecognizer for TriGRecognizer { self } else { errors.push("A dot is expected at the end of statements".into()); - self.recognize_next(token, results, errors) + self.recognize_next(token, context,results, errors) } }, TriGState::BaseExpectIri => match token { N3Token::IriRef(iri) => { - self.lexer_options.base_iri = Some(iri); + context.lexer_options.base_iri = Some(iri); self } _ => self.error(errors, "The BASE keyword should be followed by an IRI"), @@ -117,7 +123,7 @@ impl RuleRecognizer for TriGRecognizer { }, TriGState::PrefixExpectIri { name } => match token { N3Token::IriRef(iri) => { - self.prefixes.insert(name, iri); + context.prefixes.insert(name, iri); self } _ => self.error(errors, "The PREFIX declaration should be followed by a prefix and its value as an IRI"), @@ -131,7 +137,7 @@ impl RuleRecognizer for TriGRecognizer { }); self } - N3Token::PrefixedName { prefix, local, might_be_invalid_iri } => match resolve_local_name(prefix, &local, might_be_invalid_iri, &self.prefixes) { + N3Token::PrefixedName { prefix, local, might_be_invalid_iri } => match resolve_local_name(prefix, &local, might_be_invalid_iri, &context.prefixes) { Ok(t) => { self.stack.push(TriGState::WrappedGraphOrPredicateObjectList { term: t.into() @@ -157,7 +163,7 @@ impl RuleRecognizer for TriGRecognizer { self } #[cfg(feature = "rdf-star")] - N3Token::Punctuation("<<") if self.with_quoted_triples => { + N3Token::Punctuation("<<") if context.with_quoted_triples => { self.stack.push(TriGState::ExpectDot); self.stack.push(TriGState::PredicateObjectList); self.stack.push(TriGState::SubjectQuotedTripleEnd); @@ -171,7 +177,7 @@ impl RuleRecognizer for TriGRecognizer { } } TriGState::WrappedGraphOrPredicateObjectList { term } => { - if token == N3Token::Punctuation("{") && self.with_graph_name { + if token == N3Token::Punctuation("{") && context.with_graph_name { self.cur_graph = term.into(); self.stack.push(TriGState::WrappedGraph); } else { @@ -179,7 +185,7 @@ impl RuleRecognizer for TriGRecognizer { self.stack.push(TriGState::ExpectDot); self.stack.push(TriGState::PredicateObjectList); } - self.recognize_next(token, results, errors) + self.recognize_next(token, context,results, errors) } TriGState::WrappedGraphBlankNodePropertyListCurrent => if token == N3Token::Punctuation("]") { self.stack.push(TriGState::WrappedGraphOrPredicateObjectList { @@ -191,7 +197,7 @@ impl RuleRecognizer for TriGRecognizer { self.stack.push(TriGState::ExpectDot); self.stack.push(TriGState::SubjectBlankNodePropertyListEnd); self.stack.push(TriGState::PredicateObjectList); - self.recognize_next(token, results, errors) + self.recognize_next(token, context,results, errors) } TriGState::SubjectBlankNodePropertyListEnd => if token == N3Token::Punctuation("]") { self.stack.push(TriGState::SubjectBlankNodePropertyListAfter ); @@ -199,13 +205,13 @@ impl RuleRecognizer for TriGRecognizer { } else { errors.push("blank node property lists should end with a ']'".into()); self.stack.push(TriGState::SubjectBlankNodePropertyListAfter ); - self.recognize_next(token, results, errors) + self.recognize_next(token, context,results, errors) } TriGState::SubjectBlankNodePropertyListAfter => if matches!(token, N3Token::Punctuation("." | "}")) { - self.recognize_next(token, results, errors) + self.recognize_next(token, context,results, errors) } else { self.stack.push(TriGState::PredicateObjectList); - self.recognize_next(token, results, errors) + self.recognize_next(token, context,results, errors) } TriGState::SubjectCollectionBeginning => { if let N3Token::Punctuation(")") = token { @@ -218,7 +224,7 @@ impl RuleRecognizer for TriGRecognizer { self.cur_predicate.push(rdf::FIRST.into()); self.stack.push(TriGState::SubjectCollectionPossibleEnd); self.stack.push(TriGState::Object); - self.recognize_next(token, results, errors) + self.recognize_next(token, context,results, errors) } }, TriGState::SubjectCollectionPossibleEnd => { @@ -244,7 +250,7 @@ impl RuleRecognizer for TriGRecognizer { self.cur_subject.push(new.into()); self.stack.push(TriGState::ObjectCollectionPossibleEnd); self.stack.push(TriGState::Object); - self.recognize_next(token, results, errors) + self.recognize_next(token, context,results, errors) } } // [5g] wrappedGraph ::= '{' triplesBlock? '}' @@ -269,7 +275,7 @@ impl RuleRecognizer for TriGRecognizer { } _ => { errors.push("A '}' or a '.' is expected at the end of a graph block".into()); - self.recognize_next(token, results, errors) + self.recognize_next(token, context,results, errors) } } } @@ -277,7 +283,7 @@ impl RuleRecognizer for TriGRecognizer { // [10] subject ::= iri | BlankNode | collection | quotedTriple TriGState::Triples => match token { N3Token::Punctuation("}") => { - self.recognize_next(token, results, errors) // Early end + self.recognize_next(token, context,results, errors) // Early end }, N3Token::Punctuation("[") => { self.cur_subject.push(BlankNode::default().into()); @@ -289,7 +295,7 @@ impl RuleRecognizer for TriGRecognizer { self.stack.push(TriGState::PredicateObjectList); self } - N3Token::PrefixedName { prefix, local, might_be_invalid_iri } => match resolve_local_name(prefix, &local, might_be_invalid_iri, &self.prefixes) { + N3Token::PrefixedName { prefix, local, might_be_invalid_iri } => match resolve_local_name(prefix, &local, might_be_invalid_iri, &context.prefixes) { Ok(t) => { self.cur_subject.push(t.into()); self.stack.push(TriGState::PredicateObjectList); @@ -308,7 +314,7 @@ impl RuleRecognizer for TriGRecognizer { self } #[cfg(feature = "rdf-star")] - N3Token::Punctuation("<<") if self.with_quoted_triples => { + N3Token::Punctuation("<<") if context.with_quoted_triples => { self.stack.push(TriGState::PredicateObjectList); self.stack.push(TriGState::SubjectQuotedTripleEnd); self.stack.push(TriGState::QuotedObject); @@ -326,7 +332,7 @@ impl RuleRecognizer for TriGRecognizer { } else { self.stack.push(TriGState::SubjectBlankNodePropertyListEnd); self.stack.push(TriGState::PredicateObjectList); - self.recognize_next(token, results, errors) + self.recognize_next(token, context,results, errors) } // [7g] labelOrSubject ::= iri | BlankNode TriGState::GraphName => match token { @@ -334,7 +340,7 @@ impl RuleRecognizer for TriGRecognizer { self.cur_graph = NamedNode::from(iri).into(); self } - N3Token::PrefixedName { prefix, local, might_be_invalid_iri } => match resolve_local_name(prefix, &local, might_be_invalid_iri, &self.prefixes) { + N3Token::PrefixedName { prefix, local, might_be_invalid_iri } => match resolve_local_name(prefix, &local, might_be_invalid_iri, &context.prefixes) { Ok(t) => { self.cur_graph = t.into(); self @@ -364,7 +370,7 @@ impl RuleRecognizer for TriGRecognizer { self.stack.push(TriGState::PredicateObjectListEnd); self.stack.push(TriGState::ObjectsList); self.stack.push(TriGState::Verb); - self.recognize_next(token, results, errors) + self.recognize_next(token, context,results, errors) }, TriGState::PredicateObjectListEnd => { self.cur_predicate.pop(); @@ -372,26 +378,26 @@ impl RuleRecognizer for TriGRecognizer { self.stack.push(TriGState::PredicateObjectListPossibleContinuation); self } else { - self.recognize_next(token, results, errors) + self.recognize_next(token, context,results, errors) } }, TriGState::PredicateObjectListPossibleContinuation => if token == N3Token::Punctuation(";") { self.stack.push(TriGState::PredicateObjectListPossibleContinuation); self } else if matches!(token, N3Token::Punctuation("." | "}" | "]")) { - self.recognize_next(token, results, errors) + self.recognize_next(token, context,results, errors) } else { self.stack.push(TriGState::PredicateObjectListEnd); self.stack.push(TriGState::ObjectsList); self.stack.push(TriGState::Verb); - self.recognize_next(token, results, errors) + self.recognize_next(token, context,results, errors) }, // [8] objectList ::= object annotation? ( ',' object annotation? )* // [30t] annotation ::= '{|' predicateObjectList '|}' TriGState::ObjectsList => { self.stack.push(TriGState::ObjectsListEnd); self.stack.push(TriGState::Object); - self.recognize_next(token, results, errors) + self.recognize_next(token, context,results, errors) } TriGState::ObjectsListEnd => { match token { @@ -415,7 +421,7 @@ impl RuleRecognizer for TriGRecognizer { } _ => { self.cur_object.pop(); - self.recognize_next(token, results, errors) + self.recognize_next(token, context,results, errors) } } }, @@ -435,7 +441,7 @@ impl RuleRecognizer for TriGRecognizer { self.stack.push(TriGState::Object); self } else { - self.recognize_next(token, results, errors) + self.recognize_next(token, context,results, errors) }, // [9] verb ::= predicate | 'a' // [11] predicate ::= iri @@ -448,7 +454,7 @@ impl RuleRecognizer for TriGRecognizer { self.cur_predicate.push(NamedNode::from(iri)); self } - N3Token::PrefixedName { prefix, local, might_be_invalid_iri } => match resolve_local_name(prefix, &local, might_be_invalid_iri, &self.prefixes) { + N3Token::PrefixedName { prefix, local, might_be_invalid_iri } => match resolve_local_name(prefix, &local, might_be_invalid_iri, &context.prefixes) { Ok(t) => { self.cur_predicate.push(t); self @@ -477,7 +483,7 @@ impl RuleRecognizer for TriGRecognizer { self.emit_quad(results); self } - N3Token::PrefixedName { prefix, local, might_be_invalid_iri } => match resolve_local_name(prefix, &local, might_be_invalid_iri, &self.prefixes) { + N3Token::PrefixedName { prefix, local, might_be_invalid_iri } => match resolve_local_name(prefix, &local, might_be_invalid_iri, &context.prefixes) { Ok(t) => { self.cur_object.push(t.into()); self.emit_quad(results); @@ -528,7 +534,7 @@ impl RuleRecognizer for TriGRecognizer { self } #[cfg(feature = "rdf-star")] - N3Token::Punctuation("<<") if self.with_quoted_triples => { + N3Token::Punctuation("<<") if context.with_quoted_triples => { self.stack.push(TriGState::ObjectQuotedTripleEnd { emit: true }); self.stack.push(TriGState::QuotedObject); self.stack.push(TriGState::Verb); @@ -548,7 +554,7 @@ impl RuleRecognizer for TriGRecognizer { self.cur_subject.push(BlankNode::default().into()); self.stack.push(TriGState::ObjectBlankNodePropertyListEnd); self.stack.push(TriGState::PredicateObjectList); - self.recognize_next(token, results, errors) + self.recognize_next(token, context,results, errors) } TriGState::ObjectBlankNodePropertyListEnd => if token == N3Token::Punctuation("]") { self.cur_object.push(self.cur_subject.pop().unwrap().into()); @@ -569,7 +575,7 @@ impl RuleRecognizer for TriGRecognizer { self.cur_predicate.push(rdf::FIRST.into()); self.stack.push(TriGState::ObjectCollectionPossibleEnd); self.stack.push(TriGState::Object); - self.recognize_next(token, results, errors) + self.recognize_next(token, context,results, errors) }, TriGState::ObjectCollectionPossibleEnd => { let old = self.cur_subject.pop().unwrap(); @@ -592,7 +598,7 @@ impl RuleRecognizer for TriGRecognizer { self.cur_subject.push(new.into()); self.stack.push(TriGState::ObjectCollectionPossibleEnd); self.stack.push(TriGState::Object); - self.recognize_next(token, results, errors) + self.recognize_next(token, context,results, errors) } } TriGState::LiteralPossibleSuffix { value, emit } => { @@ -613,7 +619,7 @@ impl RuleRecognizer for TriGRecognizer { if emit { self.emit_quad(results); } - self.recognize_next(token, results, errors) + self.recognize_next(token, context,results, errors) } } } @@ -626,7 +632,7 @@ impl RuleRecognizer for TriGRecognizer { } self }, - N3Token::PrefixedName { prefix, local, might_be_invalid_iri } => match resolve_local_name(prefix, &local, might_be_invalid_iri, &self.prefixes) { + N3Token::PrefixedName { prefix, local, might_be_invalid_iri } => match resolve_local_name(prefix, &local, might_be_invalid_iri, &context.prefixes) { Ok(t) => { self.cur_object.push(Literal::new_typed_literal(value, t).into()); if emit { @@ -637,7 +643,7 @@ impl RuleRecognizer for TriGRecognizer { Err(e) => self.error(errors, e) } _ => { - self.error(errors, "Expecting a datatype IRI after ^^, found TOKEN").recognize_next(token, results, errors) + self.error(errors, "Expecting a datatype IRI after ^^, found TOKEN").recognize_next(token, context, results, errors) } } } @@ -685,7 +691,7 @@ impl RuleRecognizer for TriGRecognizer { self.cur_subject.push(NamedNode::from(iri).into()); self } - N3Token::PrefixedName { prefix, local, might_be_invalid_iri } => match resolve_local_name(prefix, &local, might_be_invalid_iri, &self.prefixes) { + N3Token::PrefixedName { prefix, local, might_be_invalid_iri } => match resolve_local_name(prefix, &local, might_be_invalid_iri, &context.prefixes) { Ok(t) => { self.cur_subject.push(t.into()); self @@ -717,7 +723,7 @@ impl RuleRecognizer for TriGRecognizer { self.cur_object.push(NamedNode::from(iri).into()); self } - N3Token::PrefixedName { prefix, local, might_be_invalid_iri } => match resolve_local_name(prefix, &local, might_be_invalid_iri, &self.prefixes) { + N3Token::PrefixedName { prefix, local, might_be_invalid_iri } => match resolve_local_name(prefix, &local, might_be_invalid_iri, &context.prefixes) { Ok(t) => { self.cur_object.push(t.into()); self @@ -779,6 +785,7 @@ impl RuleRecognizer for TriGRecognizer { fn recognize_end( mut self, + _context: &mut TriGRecognizerContext, results: &mut Vec, errors: &mut Vec, ) { @@ -807,8 +814,8 @@ impl RuleRecognizer for TriGRecognizer { } } - fn lexer_options(&self) -> &N3LexerOptions { - &self.lexer_options + fn lexer_options(context: &TriGRecognizerContext) -> &N3LexerOptions { + &context.lexer_options } } @@ -829,16 +836,18 @@ impl TriGRecognizer { ), TriGRecognizer { stack: vec![TriGState::TriGDoc], - with_graph_name, - #[cfg(feature = "rdf-star")] - with_quoted_triples, - lexer_options: N3LexerOptions { base_iri }, - prefixes, cur_subject: Vec::new(), cur_predicate: Vec::new(), cur_object: Vec::new(), cur_graph: GraphName::DefaultGraph, }, + TriGRecognizerContext { + with_graph_name, + #[cfg(feature = "rdf-star")] + with_quoted_triples, + prefixes, + lexer_options: N3LexerOptions { base_iri }, + }, ) } diff --git a/lib/oxttl/src/toolkit/parser.rs b/lib/oxttl/src/toolkit/parser.rs index 7a9ba8bf..7af93752 100644 --- a/lib/oxttl/src/toolkit/parser.rs +++ b/lib/oxttl/src/toolkit/parser.rs @@ -7,19 +7,28 @@ use tokio::io::AsyncRead; pub trait RuleRecognizer: Sized { type TokenRecognizer: TokenRecognizer; type Output; + type Context; fn error_recovery_state(self) -> Self; fn recognize_next( self, token: ::Token<'_>, + context: &mut Self::Context, results: &mut Vec, errors: &mut Vec, ) -> Self; - fn recognize_end(self, results: &mut Vec, errors: &mut Vec); + fn recognize_end( + self, + context: &mut Self::Context, + results: &mut Vec, + errors: &mut Vec, + ); - fn lexer_options(&self) -> &::Options; + fn lexer_options( + context: &Self::Context, + ) -> &::Options; } pub struct RuleRecognizerError { @@ -34,22 +43,23 @@ impl> From for RuleRecognizerError { } } +#[allow(clippy::partial_pub_fields)] pub struct Parser { lexer: Lexer, state: Option, + pub context: RR::Context, results: Vec, errors: Vec, - default_lexer_options: ::Options, } impl Parser { - pub fn new(lexer: Lexer, recognizer: RR) -> Self { + pub fn new(lexer: Lexer, recognizer: RR, context: RR::Context) -> Self { Self { lexer, state: Some(recognizer), + context, results: vec![], errors: vec![], - default_lexer_options: ::Options::default(), } } @@ -80,15 +90,16 @@ impl Parser { if let Some(result) = self.results.pop() { return Some(Ok(result)); } - if let Some(result) = self.lexer.read_next( - self.state - .as_ref() - .map_or(&self.default_lexer_options, |p| p.lexer_options()), - ) { + if let Some(result) = self.lexer.read_next(RR::lexer_options(&self.context)) { match result { Ok(token) => { self.state = self.state.take().map(|state| { - state.recognize_next(token, &mut self.results, &mut self.errors) + state.recognize_next( + token, + &mut self.context, + &mut self.results, + &mut self.errors, + ) }); continue; } @@ -102,7 +113,7 @@ impl Parser { let Some(state) = self.state.take() else { return None; }; - state.recognize_end(&mut self.results, &mut self.errors) + state.recognize_end(&mut self.context, &mut self.results, &mut self.errors) } else { return None; } @@ -122,9 +133,10 @@ impl Parser { } } +#[allow(clippy::partial_pub_fields)] pub struct FromReadIterator { read: R, - parser: Parser, + pub parser: Parser, } impl Iterator for FromReadIterator { @@ -145,8 +157,8 @@ impl Iterator for FromReadIterator { #[cfg(feature = "async-tokio")] pub struct FromTokioAsyncReadIterator { - read: R, - parser: Parser, + pub read: R, + pub parser: Parser, } #[cfg(feature = "async-tokio")] diff --git a/lib/oxttl/src/trig.rs b/lib/oxttl/src/trig.rs index 6dfb2b43..1b5e04a3 100644 --- a/lib/oxttl/src/trig.rs +++ b/lib/oxttl/src/trig.rs @@ -116,29 +116,29 @@ impl TriGParser { /// Count the number of people: /// ``` /// use oxrdf::{NamedNodeRef, vocab::rdf}; - /// use oxttl::{ParseError, TriGParser}; - /// - /// #[tokio::main(flavor = "current_thread")] - /// async fn main() -> Result<(), ParseError> { - /// let file = b"@base . - /// @prefix schema: . - /// a schema:Person ; - /// schema:name \"Foo\" . - /// a schema:Person ; - /// schema:name \"Bar\" ."; - /// - /// let schema_person = NamedNodeRef::new_unchecked("http://schema.org/Person"); - /// let mut count = 0; - /// let mut parser = TriGParser::new().parse_tokio_async_read(file.as_ref()); - /// while let Some(triple) = parser.next().await { - /// let triple = triple?; - /// if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { - /// count += 1; - /// } + /// use oxttl::TriGParser; + /// + /// # #[tokio::main(flavor = "current_thread")] + /// # async fn main() -> Result<(), oxttl::ParseError> { + /// let file = b"@base . + /// @prefix schema: . + /// a schema:Person ; + /// schema:name \"Foo\" . + /// a schema:Person ; + /// schema:name \"Bar\" ."; + /// + /// let schema_person = NamedNodeRef::new_unchecked("http://schema.org/Person"); + /// let mut count = 0; + /// let mut parser = TriGParser::new().parse_tokio_async_read(file.as_ref()); + /// while let Some(triple) = parser.next().await { + /// let triple = triple?; + /// if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { + /// count += 1; /// } - /// assert_eq!(2, count); - /// Ok(()) /// } + /// assert_eq!(2, count); + /// # Ok(()) + /// # } /// ``` #[cfg(feature = "async-tokio")] pub fn parse_tokio_async_read( @@ -229,6 +229,33 @@ pub struct FromReadTriGReader { inner: FromReadIterator, } +impl FromReadTriGReader { + /// The list of IRI prefixes considered at the current step of the parsing. + /// + /// This method returns the mapping from prefix name to prefix value. + /// It is empty at the beginning of the parsing and gets updated when prefixes are encountered. + /// It should be full at the end of the parsing (but if a prefix is overridden, only the latest version will be returned). + /// + /// ``` + /// use oxttl::TriGParser; + /// + /// let file = b"@base . + /// @prefix schema: . + /// a schema:Person ; + /// schema:name \"Foo\" ."; + /// + /// let mut reader = TriGParser::new().parse_read(file.as_ref()); + /// assert!(reader.prefixes().is_empty()); // No prefix at the beginning + /// + /// reader.next().unwrap()?; // We read the first triple + /// assert_eq!(reader.prefixes()["schema"], "http://schema.org/"); // There are now prefixes + /// # Result::<_,Box>::Ok(()) + /// ``` + pub fn prefixes(&self) -> &HashMap> { + &self.inner.parser.context.prefixes + } +} + impl Iterator for FromReadTriGReader { type Item = Result; @@ -242,29 +269,29 @@ impl Iterator for FromReadTriGReader { /// Count the number of people: /// ``` /// use oxrdf::{NamedNodeRef, vocab::rdf}; -/// use oxttl::{ParseError, TriGParser}; +/// use oxttl::TriGParser; /// -/// #[tokio::main(flavor = "current_thread")] -/// async fn main() -> Result<(), ParseError> { -/// let file = b"@base . -/// @prefix schema: . -/// a schema:Person ; -/// schema:name \"Foo\" . -/// a schema:Person ; -/// schema:name \"Bar\" ."; +/// # #[tokio::main(flavor = "current_thread")] +/// # async fn main() -> Result<(), oxttl::ParseError> { +/// let file = b"@base . +/// @prefix schema: . +/// a schema:Person ; +/// schema:name \"Foo\" . +/// a schema:Person ; +/// schema:name \"Bar\" ."; /// -/// let schema_person = NamedNodeRef::new_unchecked("http://schema.org/Person"); -/// let mut count = 0; -/// let mut parser = TriGParser::new().parse_tokio_async_read(file.as_ref()); -/// while let Some(triple) = parser.next().await { -/// let triple = triple?; -/// if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { -/// count += 1; -/// } +/// let schema_person = NamedNodeRef::new_unchecked("http://schema.org/Person"); +/// let mut count = 0; +/// let mut parser = TriGParser::new().parse_tokio_async_read(file.as_ref()); +/// while let Some(triple) = parser.next().await { +/// let triple = triple?; +/// if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { +/// count += 1; /// } -/// assert_eq!(2, count); -/// Ok(()) /// } +/// assert_eq!(2, count); +/// # Ok(()) +/// # } /// ``` #[cfg(feature = "async-tokio")] #[must_use] @@ -278,6 +305,34 @@ impl FromTokioAsyncReadTriGReader { pub async fn next(&mut self) -> Option> { Some(self.inner.next().await?.map(Into::into)) } + + /// The list of IRI prefixes considered at the current step of the parsing. + /// + /// This method returns the mapping from prefix name to prefix value. + /// It is empty at the beginning of the parsing and gets updated when prefixes are encountered. + /// It should be full at the end of the parsing (but if a prefix is overridden, only the latest version will be returned). + /// + /// ``` + /// use oxttl::TriGParser; + /// + /// # #[tokio::main(flavor = "current_thread")] + /// # async fn main() -> Result<(), oxttl::ParseError> { + /// let file = b"@base . + /// @prefix schema: . + /// a schema:Person ; + /// schema:name \"Foo\" ."; + /// + /// let mut reader = TriGParser::new().parse_tokio_async_read(file.as_ref()); + /// assert!(reader.prefixes().is_empty()); // No prefix at the beginning + /// + /// reader.next().await.unwrap()?; // We read the first triple + /// assert_eq!(reader.prefixes()["schema"], "http://schema.org/"); // There are now prefixes + /// # Ok(()) + /// # } + /// ``` + pub fn prefixes(&self) -> &HashMap> { + &self.inner.parser.context.prefixes + } } /// Parses a TriG file by using a low-level API. Can be built using [`TriGParser::parse`]. @@ -345,6 +400,32 @@ impl LowLevelTriGReader { pub fn read_next(&mut self) -> Option> { self.parser.read_next() } + + /// The list of IRI prefixes considered at the current step of the parsing. + /// + /// This method returns the mapping from prefix name to prefix value. + /// It is empty at the beginning of the parsing and gets updated when prefixes are encountered. + /// It should be full at the end of the parsing (but if a prefix is overridden, only the latest version will be returned). + /// + /// ``` + /// use oxttl::TriGParser; + /// + /// let file = b"@base . + /// @prefix schema: . + /// a schema:Person ; + /// schema:name \"Foo\" ."; + /// + /// let mut reader = TriGParser::new().parse(); + /// reader.extend_from_slice(file); + /// assert!(reader.prefixes().is_empty()); // No prefix at the beginning + /// + /// reader.read_next().unwrap()?; // We read the first triple + /// assert_eq!(reader.prefixes()["schema"], "http://schema.org/"); // There are now prefixes + /// # Result::<_,Box>::Ok(()) + /// ``` + pub fn prefixes(&self) -> &HashMap> { + &self.parser.context.prefixes + } } /// A [TriG](https://www.w3.org/TR/trig/) serializer. @@ -410,23 +491,22 @@ impl TriGSerializer { /// ``` /// use oxrdf::{NamedNodeRef, QuadRef}; /// use oxttl::TriGSerializer; - /// use std::io::Result; - /// - /// #[tokio::main(flavor = "current_thread")] - /// async fn main() -> Result<()> { - /// let mut writer = TriGSerializer::new().serialize_to_tokio_async_write(Vec::new()); - /// writer.write_quad(QuadRef::new( - /// NamedNodeRef::new_unchecked("http://example.com#me"), - /// NamedNodeRef::new_unchecked("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), - /// NamedNodeRef::new_unchecked("http://schema.org/Person"), - /// NamedNodeRef::new_unchecked("http://example.com"), - /// )).await?; - /// assert_eq!( - /// b" {\n\t .\n}\n", - /// writer.finish().await?.as_slice() - /// ); - /// Ok(()) - /// } + /// + /// # #[tokio::main(flavor = "current_thread")] + /// # async fn main() -> std::io::Result<()> { + /// let mut writer = TriGSerializer::new().serialize_to_tokio_async_write(Vec::new()); + /// writer.write_quad(QuadRef::new( + /// NamedNodeRef::new_unchecked("http://example.com#me"), + /// NamedNodeRef::new_unchecked("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), + /// NamedNodeRef::new_unchecked("http://schema.org/Person"), + /// NamedNodeRef::new_unchecked("http://example.com"), + /// )).await?; + /// assert_eq!( + /// b" {\n\t .\n}\n", + /// writer.finish().await?.as_slice() + /// ); + /// # Ok(()) + /// # } /// ``` #[cfg(feature = "async-tokio")] pub fn serialize_to_tokio_async_write( @@ -513,23 +593,22 @@ impl ToWriteTriGWriter { /// ``` /// use oxrdf::{NamedNodeRef, QuadRef}; /// use oxttl::TriGSerializer; -/// use std::io::Result; /// -/// #[tokio::main(flavor = "current_thread")] -/// async fn main() -> Result<()> { -/// let mut writer = TriGSerializer::new().serialize_to_tokio_async_write(Vec::new()); -/// writer.write_quad(QuadRef::new( -/// NamedNodeRef::new_unchecked("http://example.com#me"), -/// NamedNodeRef::new_unchecked("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), -/// NamedNodeRef::new_unchecked("http://schema.org/Person"), -/// NamedNodeRef::new_unchecked("http://example.com"), -/// )).await?; -/// assert_eq!( -/// b" {\n\t .\n}\n", -/// writer.finish().await?.as_slice() -/// ); -/// Ok(()) -/// } +/// # #[tokio::main(flavor = "current_thread")] +/// # async fn main() -> std::io::Result<()> { +/// let mut writer = TriGSerializer::new().serialize_to_tokio_async_write(Vec::new()); +/// writer.write_quad(QuadRef::new( +/// NamedNodeRef::new_unchecked("http://example.com#me"), +/// NamedNodeRef::new_unchecked("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), +/// NamedNodeRef::new_unchecked("http://schema.org/Person"), +/// NamedNodeRef::new_unchecked("http://example.com"), +/// )).await?; +/// assert_eq!( +/// b" {\n\t .\n}\n", +/// writer.finish().await?.as_slice() +/// ); +/// # Ok(()) +/// # } /// ``` #[cfg(feature = "async-tokio")] #[must_use] diff --git a/lib/oxttl/src/turtle.rs b/lib/oxttl/src/turtle.rs index 133c9cca..7106758d 100644 --- a/lib/oxttl/src/turtle.rs +++ b/lib/oxttl/src/turtle.rs @@ -118,29 +118,29 @@ impl TurtleParser { /// Count the number of people: /// ``` /// use oxrdf::{NamedNodeRef, vocab::rdf}; - /// use oxttl::{ParseError, TurtleParser}; - /// - /// #[tokio::main(flavor = "current_thread")] - /// async fn main() -> Result<(), ParseError> { - /// let file = b"@base . - /// @prefix schema: . - /// a schema:Person ; - /// schema:name \"Foo\" . - /// a schema:Person ; - /// schema:name \"Bar\" ."; - /// - /// let schema_person = NamedNodeRef::new_unchecked("http://schema.org/Person"); - /// let mut count = 0; - /// let mut parser = TurtleParser::new().parse_tokio_async_read(file.as_ref()); - /// while let Some(triple) = parser.next().await { - /// let triple = triple?; - /// if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { - /// count += 1; - /// } + /// use oxttl::TurtleParser; + /// + /// # #[tokio::main(flavor = "current_thread")] + /// # async fn main() -> Result<(), oxttl::ParseError> { + /// let file = b"@base . + /// @prefix schema: . + /// a schema:Person ; + /// schema:name \"Foo\" . + /// a schema:Person ; + /// schema:name \"Bar\" ."; + /// + /// let schema_person = NamedNodeRef::new_unchecked("http://schema.org/Person"); + /// let mut count = 0; + /// let mut parser = TurtleParser::new().parse_tokio_async_read(file.as_ref()); + /// while let Some(triple) = parser.next().await { + /// let triple = triple?; + /// if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { + /// count += 1; /// } - /// assert_eq!(2, count); - /// Ok(()) /// } + /// assert_eq!(2, count); + /// # Ok(()) + /// # } /// ``` #[cfg(feature = "async-tokio")] pub fn parse_tokio_async_read( @@ -231,6 +231,33 @@ pub struct FromReadTurtleReader { inner: FromReadIterator, } +impl FromReadTurtleReader { + /// The list of IRI prefixes considered at the current step of the parsing. + /// + /// This method returns the mapping from prefix name to prefix value. + /// It is empty at the beginning of the parsing and gets updated when prefixes are encountered. + /// It should be full at the end of the parsing (but if a prefix is overridden, only the latest version will be returned). + /// + /// ``` + /// use oxttl::TurtleParser; + /// + /// let file = b"@base . + /// @prefix schema: . + /// a schema:Person ; + /// schema:name \"Foo\" ."; + /// + /// let mut reader = TurtleParser::new().parse_read(file.as_ref()); + /// assert!(reader.prefixes().is_empty()); // No prefix at the beginning + /// + /// reader.next().unwrap()?; // We read the first triple + /// assert_eq!(reader.prefixes()["schema"], "http://schema.org/"); // There are now prefixes + /// # Result::<_,Box>::Ok(()) + /// ``` + pub fn prefixes(&self) -> &HashMap> { + &self.inner.parser.context.prefixes + } +} + impl Iterator for FromReadTurtleReader { type Item = Result; @@ -244,29 +271,29 @@ impl Iterator for FromReadTurtleReader { /// Count the number of people: /// ``` /// use oxrdf::{NamedNodeRef, vocab::rdf}; -/// use oxttl::{ParseError, TurtleParser}; +/// use oxttl::TurtleParser; /// -/// #[tokio::main(flavor = "current_thread")] -/// async fn main() -> Result<(), ParseError> { -/// let file = b"@base . -/// @prefix schema: . -/// a schema:Person ; -/// schema:name \"Foo\" . -/// a schema:Person ; -/// schema:name \"Bar\" ."; +/// # #[tokio::main(flavor = "current_thread")] +/// # async fn main() -> Result<(), oxttl::ParseError> { +/// let file = b"@base . +/// @prefix schema: . +/// a schema:Person ; +/// schema:name \"Foo\" . +/// a schema:Person ; +/// schema:name \"Bar\" ."; /// -/// let schema_person = NamedNodeRef::new_unchecked("http://schema.org/Person"); -/// let mut count = 0; -/// let mut parser = TurtleParser::new().parse_tokio_async_read(file.as_ref()); -/// while let Some(triple) = parser.next().await { -/// let triple = triple?; -/// if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { -/// count += 1; -/// } +/// let schema_person = NamedNodeRef::new_unchecked("http://schema.org/Person"); +/// let mut count = 0; +/// let mut parser = TurtleParser::new().parse_tokio_async_read(file.as_ref()); +/// while let Some(triple) = parser.next().await { +/// let triple = triple?; +/// if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { +/// count += 1; /// } -/// assert_eq!(2, count); -/// Ok(()) /// } +/// assert_eq!(2, count); +/// # Ok(()) +/// # } /// ``` #[cfg(feature = "async-tokio")] #[must_use] @@ -280,6 +307,34 @@ impl FromTokioAsyncReadTurtleReader { pub async fn next(&mut self) -> Option> { Some(self.inner.next().await?.map(Into::into)) } + + /// The list of IRI prefixes considered at the current step of the parsing. + /// + /// This method returns the mapping from prefix name to prefix value. + /// It is empty at the beginning of the parsing and gets updated when prefixes are encountered. + /// It should be full at the end of the parsing (but if a prefix is overridden, only the latest version will be returned). + /// + /// ``` + /// use oxttl::TurtleParser; + /// + /// # #[tokio::main(flavor = "current_thread")] + /// # async fn main() -> Result<(), oxttl::ParseError> { + /// let file = b"@base . + /// @prefix schema: . + /// a schema:Person ; + /// schema:name \"Foo\" ."; + /// + /// let mut reader = TurtleParser::new().parse_tokio_async_read(file.as_ref()); + /// assert!(reader.prefixes().is_empty()); // No prefix at the beginning + /// + /// reader.next().await.unwrap()?; // We read the first triple + /// assert_eq!(reader.prefixes()["schema"], "http://schema.org/"); // There are now prefixes + /// # Ok(()) + /// # } + /// ``` + pub fn prefixes(&self) -> &HashMap> { + &self.inner.parser.context.prefixes + } } /// Parses a Turtle file by using a low-level API. Can be built using [`TurtleParser::parse`]. @@ -347,6 +402,32 @@ impl LowLevelTurtleReader { pub fn read_next(&mut self) -> Option> { Some(self.parser.read_next()?.map(Into::into)) } + + /// The list of IRI prefixes considered at the current step of the parsing. + /// + /// This method returns the mapping from prefix name to prefix value. + /// It is empty at the beginning of the parsing and gets updated when prefixes are encountered. + /// It should be full at the end of the parsing (but if a prefix is overridden, only the latest version will be returned). + /// + /// ``` + /// use oxttl::TurtleParser; + /// + /// let file = b"@base . + /// @prefix schema: . + /// a schema:Person ; + /// schema:name \"Foo\" ."; + /// + /// let mut reader = TurtleParser::new().parse(); + /// reader.extend_from_slice(file); + /// assert!(reader.prefixes().is_empty()); // No prefix at the beginning + /// + /// reader.read_next().unwrap()?; // We read the first triple + /// assert_eq!(reader.prefixes()["schema"], "http://schema.org/"); // There are now prefixes + /// # Result::<_,Box>::Ok(()) + /// ``` + pub fn prefixes(&self) -> &HashMap> { + &self.parser.context.prefixes + } } /// A [Turtle](https://www.w3.org/TR/turtle/) serializer. @@ -411,22 +492,21 @@ impl TurtleSerializer { /// ``` /// use oxrdf::{NamedNodeRef, TripleRef}; /// use oxttl::TurtleSerializer; - /// use std::io::Result; - /// - /// #[tokio::main(flavor = "current_thread")] - /// async fn main() -> Result<()> { - /// let mut writer = TurtleSerializer::new().serialize_to_tokio_async_write(Vec::new()); - /// writer.write_triple(TripleRef::new( - /// NamedNodeRef::new_unchecked("http://example.com#me"), - /// NamedNodeRef::new_unchecked("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), - /// NamedNodeRef::new_unchecked("http://schema.org/Person"), - /// )).await?; - /// assert_eq!( - /// b" .\n", - /// writer.finish().await?.as_slice() - /// ); - /// Ok(()) - /// } + /// + /// # #[tokio::main(flavor = "current_thread")] + /// # async fn main() -> std::io::Result<()> { + /// let mut writer = TurtleSerializer::new().serialize_to_tokio_async_write(Vec::new()); + /// writer.write_triple(TripleRef::new( + /// NamedNodeRef::new_unchecked("http://example.com#me"), + /// NamedNodeRef::new_unchecked("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), + /// NamedNodeRef::new_unchecked("http://schema.org/Person"), + /// )).await?; + /// assert_eq!( + /// b" .\n", + /// writer.finish().await?.as_slice() + /// ); + /// # Ok(()) + /// # } /// ``` #[cfg(feature = "async-tokio")] pub fn serialize_to_tokio_async_write( @@ -506,22 +586,21 @@ impl ToWriteTurtleWriter { /// ``` /// use oxrdf::{NamedNodeRef, TripleRef}; /// use oxttl::TurtleSerializer; -/// use std::io::Result; /// -/// #[tokio::main(flavor = "current_thread")] -/// async fn main() -> Result<()> { -/// let mut writer = TurtleSerializer::new().serialize_to_tokio_async_write(Vec::new()); -/// writer.write_triple(TripleRef::new( -/// NamedNodeRef::new_unchecked("http://example.com#me"), -/// NamedNodeRef::new_unchecked("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), -/// NamedNodeRef::new_unchecked("http://schema.org/Person") -/// )).await?; -/// assert_eq!( -/// b" .\n", -/// writer.finish().await?.as_slice() -/// ); -/// Ok(()) -/// } +/// # #[tokio::main(flavor = "current_thread")] +/// # async fn main() -> std::io::Result<()> { +/// let mut writer = TurtleSerializer::new().serialize_to_tokio_async_write(Vec::new()); +/// writer.write_triple(TripleRef::new( +/// NamedNodeRef::new_unchecked("http://example.com#me"), +/// NamedNodeRef::new_unchecked("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), +/// NamedNodeRef::new_unchecked("http://schema.org/Person") +/// )).await?; +/// assert_eq!( +/// b" .\n", +/// writer.finish().await?.as_slice() +/// ); +/// # Ok(()) +/// # } /// ``` #[cfg(feature = "async-tokio")] #[must_use] From bdedcc47e3eeee4613c840acd25f4016ee27adce Mon Sep 17 00:00:00 2001 From: Tpt Date: Mon, 4 Sep 2023 22:18:27 +0200 Subject: [PATCH 079/217] Renames oxigraph-server to oxigraph(-cli) --- .github/workflows/artifacts.yml | 52 ++++++++++++++-------------- .github/workflows/tests.yml | 10 +++--- Cargo.lock | 24 ++++++------- Cargo.toml | 4 +-- README.md | 4 +-- bench/bsbm_oxigraph.sh | 8 ++--- {server => cli}/Cargo.toml | 12 ++++--- {server => cli}/Dockerfile | 8 ++--- {server => cli}/README.md | 38 ++++++++++---------- {server => cli}/logo.svg | 0 {server => cli}/src/main.rs | 6 ++-- {server => cli}/templates/query.html | 0 js/Cargo.toml | 2 +- lib/README.md | 2 +- server | 1 + testsuite/Cargo.toml | 2 +- 16 files changed, 88 insertions(+), 85 deletions(-) rename {server => cli}/Cargo.toml (74%) rename {server => cli}/Dockerfile (80%) rename {server => cli}/README.md (84%) rename {server => cli}/logo.svg (100%) rename {server => cli}/src/main.rs (99%) rename {server => cli}/templates/query.html (100%) create mode 120000 server diff --git a/.github/workflows/artifacts.yml b/.github/workflows/artifacts.yml index ddcc158d..d963f2d0 100644 --- a/.github/workflows/artifacts.yml +++ b/.github/workflows/artifacts.yml @@ -26,28 +26,28 @@ jobs: echo -e "\n\n[target.aarch64-unknown-linux-gnu]\nlinker = \"aarch64-linux-gnu-gcc\"" >> .cargo/config.toml - uses: Swatinem/rust-cache@v2 - run: cargo build --release - working-directory: ./server + working-directory: ./cli - run: cargo build --release --target aarch64-unknown-linux-gnu - working-directory: ./server + working-directory: ./cli env: BINDGEN_EXTRA_CLANG_ARGS: --sysroot /usr/aarch64-linux-gnu - uses: actions/upload-artifact@v3 with: - name: oxigraph_server_x86_64_linux_gnu - path: target/release/oxigraph_server + name: oxigraph_x86_64_linux_gnu + path: target/release/oxigraph - uses: actions/upload-artifact@v3 with: - name: oxigraph_server_aarch64_linux_gnu - path: target/aarch64-unknown-linux-gnu/release/oxigraph_server - - run: mv target/release/oxigraph_server oxigraph_server_${{ github.event.release.tag_name }}_x86_64_linux_gnu + name: oxigraph_aarch64_linux_gnu + path: target/aarch64-unknown-linux-gnu/release/oxigraph + - run: mv target/release/oxigraph oxigraph_${{ github.event.release.tag_name }}_x86_64_linux_gnu if: github.event_name == 'release' - - run: mv target/aarch64-unknown-linux-gnu/release/oxigraph_server oxigraph_server_${{ github.event.release.tag_name }}_aarch64_linux_gnu + - run: mv target/aarch64-unknown-linux-gnu/release/oxigraph oxigraph_${{ github.event.release.tag_name }}_aarch64_linux_gnu if: github.event_name == 'release' - uses: softprops/action-gh-release@v1 with: files: | - oxigraph_server_${{ github.event.release.tag_name }}_x86_64_linux_gnu - oxigraph_server_${{ github.event.release.tag_name }}_aarch64_linux_gnu + oxigraph_${{ github.event.release.tag_name }}_x86_64_linux_gnu + oxigraph_${{ github.event.release.tag_name }}_aarch64_linux_gnu if: github.event_name == 'release' binary_mac: @@ -63,26 +63,26 @@ jobs: - run: rustup update && rustup target add aarch64-apple-darwin - uses: Swatinem/rust-cache@v2 - run: cargo build --release - working-directory: ./server + working-directory: ./cli - run: cargo build --release --target aarch64-apple-darwin - working-directory: ./server + working-directory: ./cli - uses: actions/upload-artifact@v3 with: - name: oxigraph_server_x86_64_apple - path: target/release/oxigraph_server + name: oxigraph_x86_64_apple + path: target/release/oxigraph - uses: actions/upload-artifact@v3 with: - name: oxigraph_server_aarch64_apple - path: target/aarch64-apple-darwin/release/oxigraph_server - - run: mv target/release/oxigraph_server oxigraph_server_${{ github.event.release.tag_name }}_x86_64_apple + name: oxigraph_aarch64_apple + path: target/aarch64-apple-darwin/release/oxigraph + - run: mv target/release/oxigraph oxigraph_${{ github.event.release.tag_name }}_x86_64_apple if: github.event_name == 'release' - - run: mv target/aarch64-apple-darwin/release/oxigraph_server oxigraph_server_${{ github.event.release.tag_name }}_aarch64_apple + - run: mv target/aarch64-apple-darwin/release/oxigraph oxigraph_${{ github.event.release.tag_name }}_aarch64_apple if: github.event_name == 'release' - uses: softprops/action-gh-release@v1 with: files: | - oxigraph_server_${{ github.event.release.tag_name }}_x86_64_apple - oxigraph_server_${{ github.event.release.tag_name }}_aarch64_apple + oxigraph_${{ github.event.release.tag_name }}_x86_64_apple + oxigraph_${{ github.event.release.tag_name }}_aarch64_apple if: github.event_name == 'release' binary_windows: @@ -95,16 +95,16 @@ jobs: - uses: Swatinem/rust-cache@v2 - run: Remove-Item -LiteralPath "C:\msys64\" -Force -Recurse - run: cargo build --release - working-directory: ./server + working-directory: ./cli - uses: actions/upload-artifact@v3 with: - name: oxigraph_server_x86_64_windows_msvc - path: target/release/oxigraph_server.exe - - run: mv target/release/oxigraph_server.exe oxigraph_server_${{ github.event.release.tag_name }}_x86_64_windows_msvc.exe + name: oxigraph_x86_64_windows_msvc + path: target/release/oxigraph.exe + - run: mv target/release/oxigraph.exe oxigraph_${{ github.event.release.tag_name }}_x86_64_windows_msvc.exe if: github.event_name == 'release' - uses: softprops/action-gh-release@v1 with: - files: oxigraph_server_${{ github.event.release.tag_name }}_x86_64_windows_msvc.exe + files: oxigraph_${{ github.event.release.tag_name }}_x86_64_windows_msvc.exe if: github.event_name == 'release' wheel_linux: @@ -337,7 +337,7 @@ jobs: working-directory: ./lib continue-on-error: true - run: cargo publish - working-directory: ./server + working-directory: ./cli homebrew: if: "github.event_name == 'release' && !contains('-', github.event.release.tag_name)" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b16ea777..edf5cd42 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -51,7 +51,7 @@ jobs: - run: cargo clippy working-directory: ./python - run: cargo clippy - working-directory: ./server + working-directory: ./cli - run: cargo clippy working-directory: ./testsuite @@ -119,7 +119,7 @@ jobs: - run: cargo clippy -- -D warnings -D clippy::all working-directory: ./python - run: cargo clippy -- -D warnings -D clippy::all - working-directory: ./server + working-directory: ./cli - run: cargo clippy -- -D warnings -D clippy::all working-directory: ./testsuite @@ -164,7 +164,7 @@ jobs: - run: rustup update - uses: Swatinem/rust-cache@v2 - run: cargo install cargo-semver-checks || true - - run: cargo semver-checks check-release --exclude oxrocksdb-sys --exclude oxigraph_js --exclude pyoxigraph --exclude oxigraph_testsuite --exclude oxigraph_server + - run: cargo semver-checks check-release --exclude oxrocksdb-sys --exclude oxigraph-js --exclude pyoxigraph --exclude oxigraph-testsuite --exclude oxigraph-cli test_linux: runs-on: ubuntu-latest @@ -187,7 +187,7 @@ jobs: - run: rustup update && rustup override set nightly - run: sudo apt install -y llvm - uses: Swatinem/rust-cache@v2 - - run: cargo test --tests --target x86_64-unknown-linux-gnu --workspace --exclude pyoxigraph --exclude oxigraph_testsuite --exclude oxigraph_server + - run: cargo test --tests --target x86_64-unknown-linux-gnu --workspace --exclude pyoxigraph --exclude oxigraph-testsuite --exclude oxigraph-cli env: RUST_BACKTRACE: 1 RUSTFLAGS: -Z sanitizer=address @@ -215,7 +215,7 @@ jobs: - uses: Swatinem/rust-cache@v2 - uses: taiki-e/install-action@wasmtime - run: cargo install cargo-wasi || true - - run: cargo wasi test --workspace --exclude oxigraph_js --exclude oxigraph_server --exclude oxigraph_testsuite --exclude oxrocksdb-sys --exclude pyoxigraph + - run: cargo wasi test --workspace --exclude oxigraph-js --exclude oxigraph-cli --exclude oxigraph-testsuite --exclude oxrocksdb-sys --exclude pyoxigraph env: RUST_BACKTRACE: 1 diff --git a/Cargo.lock b/Cargo.lock index b4db7cbd..63a066a1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -973,17 +973,7 @@ dependencies = [ ] [[package]] -name = "oxigraph_js" -version = "0.4.0-alpha.1-dev" -dependencies = [ - "console_error_panic_hook", - "js-sys", - "oxigraph", - "wasm-bindgen", -] - -[[package]] -name = "oxigraph_server" +name = "oxigraph-cli" version = "0.4.0-alpha.1-dev" dependencies = [ "anyhow", @@ -1002,7 +992,17 @@ dependencies = [ ] [[package]] -name = "oxigraph_testsuite" +name = "oxigraph-js" +version = "0.4.0-alpha.1-dev" +dependencies = [ + "console_error_panic_hook", + "js-sys", + "oxigraph", + "wasm-bindgen", +] + +[[package]] +name = "oxigraph-testsuite" version = "0.4.0-alpha.1-dev" dependencies = [ "anyhow", diff --git a/Cargo.toml b/Cargo.toml index 15d79ef4..8264afa9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,7 @@ members = [ "lib/sparql-smith", "oxrocksdb-sys", "python", - "server", + "cli", "testsuite" ] resolver = "2" @@ -30,5 +30,5 @@ rust-version = "1.70" lto = true codegen-units = 1 -[profile.release.package.oxigraph_js] +[profile.release.package.oxigraph-js] opt-level = "z" diff --git a/README.md b/README.md index 7be322f6..55a39b10 100644 --- a/README.md +++ b/README.md @@ -29,8 +29,8 @@ It is split into multiple parts: - [`pyoxigraph` that exposes Oxigraph to the Python world](https://pyoxigraph.readthedocs.io/). Its source code is in the `python` directory. [![PyPI](https://img.shields.io/pypi/v/pyoxigraph)](https://pypi.org/project/pyoxigraph/) - [JavaScript bindings for Oxigraph](https://www.npmjs.com/package/oxigraph). WebAssembly is used to package Oxigraph into a NodeJS compatible NPM package. Its source code is in the `js` directory. [![npm](https://img.shields.io/npm/v/oxigraph)](https://www.npmjs.com/package/oxigraph) -- [Oxigraph server](https://crates.io/crates/oxigraph_server) that provides a standalone binary of a web server implementing the [SPARQL 1.1 Protocol](https://www.w3.org/TR/sparql11-protocol/) and the [SPARQL 1.1 Graph Store Protocol](https://www.w3.org/TR/sparql11-http-rdf-update/). Its source code is in the `server` directory. - [![Latest Version](https://img.shields.io/crates/v/oxigraph_server.svg)](https://crates.io/crates/oxigraph_server) +- [Oxigraph binary](https://crates.io/crates/oxigraph-cli) that provides a standalone command line tool allowing to manipulate RDF data and spawn a a web server implementing the [SPARQL 1.1 Protocol](https://www.w3.org/TR/sparql11-protocol/) and the [SPARQL 1.1 Graph Store Protocol](https://www.w3.org/TR/sparql11-http-rdf-update/). Its source code is in the `cli` directory. + [![Latest Version](https://img.shields.io/crates/v/oxigraph-cli.svg)](https://crates.io/crates/oxigraph-cli) Oxigraph implements the following specifications: diff --git a/bench/bsbm_oxigraph.sh b/bench/bsbm_oxigraph.sh index 7ee691cb..a4b15201 100755 --- a/bench/bsbm_oxigraph.sh +++ b/bench/bsbm_oxigraph.sh @@ -6,10 +6,10 @@ PARALLELISM=16 set -eu cd bsbm-tools ./generate -fc -pc ${DATASET_SIZE} -s nt -fn "explore-${DATASET_SIZE}" -ud -ufn "explore-update-${DATASET_SIZE}" -cargo build --release --manifest-path="../../server/Cargo.toml" -VERSION=$(./../../target/release/oxigraph_server --version | sed 's/oxigraph_server //g') -./../../target/release/oxigraph_server --location oxigraph_data load --file "explore-${DATASET_SIZE}.nt" -./../../target/release/oxigraph_server --location oxigraph_data serve --bind 127.0.0.1:7878 & +cargo build --release --manifest-path="../../cli/Cargo.toml" +VERSION=$(./../../target/release/oxigraph --version | sed 's/oxigraph //g') +./../../target/release/oxigraph --location oxigraph_data load --file "explore-${DATASET_SIZE}.nt" +./../../target/release/oxigraph --location oxigraph_data serve --bind 127.0.0.1:7878 & sleep 1 ./testdriver -mt ${PARALLELISM} -ucf usecases/explore/sparql.txt -o "../bsbm.explore.oxigraph.${VERSION}.${DATASET_SIZE}.${PARALLELISM}.xml" http://127.0.0.1:7878/query ./testdriver -mt ${PARALLELISM} -ucf usecases/exploreAndUpdate/sparql.txt -o "../bsbm.exploreAndUpdate.oxigraph.${VERSION}.${DATASET_SIZE}.${PARALLELISM}.xml" http://127.0.0.1:7878/query -u http://127.0.0.1:7878/update -udataset "explore-update-${DATASET_SIZE}.nt" diff --git a/server/Cargo.toml b/cli/Cargo.toml similarity index 74% rename from server/Cargo.toml rename to cli/Cargo.toml index 350863f9..62c6b40b 100644 --- a/server/Cargo.toml +++ b/cli/Cargo.toml @@ -1,19 +1,23 @@ [package] -name = "oxigraph_server" +name = "oxigraph-cli" version.workspace = true authors.workspace = true license.workspace = true readme = "README.md" keywords = ["RDF", "SPARQL", "graph-database", "database"] categories = ["command-line-utilities", "database"] -repository = "https://github.com/oxigraph/oxigraph/tree/main/server" -homepage = "https://oxigraph.org/server/" +repository = "https://github.com/oxigraph/oxigraph/tree/main/cli" +homepage = "https://oxigraph.org/cli/" description = """ -Oxigraph SPARQL HTTP server +Oxigraph command line toolkit and SPARQL HTTP server """ edition.workspace = true rust-version.workspace = true +[[bin]] +name = "oxigraph" +path = "src/main.rs" + [dependencies] anyhow = "1" oxhttp = { version = "0.1", features = ["rayon"] } diff --git a/server/Dockerfile b/cli/Dockerfile similarity index 80% rename from server/Dockerfile rename to cli/Dockerfile index 92a6f4d4..abbd6aad 100644 --- a/server/Dockerfile +++ b/cli/Dockerfile @@ -8,18 +8,18 @@ RUN apt-get update && \ rustup target add aarch64-unknown-linux-gnu ; \ fi COPY . /oxigraph -WORKDIR /oxigraph/server +WORKDIR /oxigraph/cli RUN if [ "$BUILDARCH" != "$TARGETARCH" ] && [ "$TARGETARCH" = "arm64" ] ; \ then \ export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc && \ export BINDGEN_EXTRA_CLANG_ARGS="--sysroot /usr/aarch64-linux-gnu" && \ cargo build --release --target aarch64-unknown-linux-gnu && \ - mv /oxigraph/target/aarch64-unknown-linux-gnu/release/oxigraph_server /oxigraph/target/release/oxigraph_server ; \ + mv /oxigraph/target/aarch64-unknown-linux-gnu/release/oxigraph /oxigraph/target/release/oxigraph ; \ else \ cargo build --release ; \ fi FROM --platform=$TARGETPLATFORM gcr.io/distroless/cc-debian11 -COPY --from=builder /oxigraph/target/release/oxigraph_server /usr/local/bin/oxigraph_server -ENTRYPOINT [ "/usr/local/bin/oxigraph_server" ] +COPY --from=builder /oxigraph/target/release/oxigraph /usr/local/bin/oxigraph +ENTRYPOINT [ "/usr/local/bin/oxigraph" ] CMD [ "serve", "--location", "/data", "--bind", "0.0.0.0:7878" ] diff --git a/server/README.md b/cli/README.md similarity index 84% rename from server/README.md rename to cli/README.md index 28a06073..8c650b4f 100644 --- a/server/README.md +++ b/cli/README.md @@ -1,17 +1,15 @@ -Oxigraph Server -=============== +Oxigraph CLI +============ -[![Latest Version](https://img.shields.io/crates/v/oxigraph_server.svg)](https://crates.io/crates/oxigraph_server) -[![Crates.io downloads](https://img.shields.io/crates/d/oxigraph_server)](https://crates.io/crates/oxigraph_server) +[![Latest Version](https://img.shields.io/crates/v/oxigraph-cli.svg)](https://crates.io/crates/oxigraph-cli) +[![Crates.io downloads](https://img.shields.io/crates/d/oxigraph-cli)](https://crates.io/crates/oxigraph-cli) [![Conda](https://img.shields.io/conda/vn/conda-forge/oxigraph-server)](https://anaconda.org/conda-forge/oxigraph-server) [![actions status](https://github.com/oxigraph/oxigraph/workflows/build/badge.svg)](https://github.com/oxigraph/oxigraph/actions) [![Gitter](https://badges.gitter.im/oxigraph/community.svg)](https://gitter.im/oxigraph/community) -Oxigraph Server is a standalone HTTP server providing a graph database implementing the [SPARQL](https://www.w3.org/TR/sparql11-overview/) standard. - -Its goal is to provide a compliant, safe, and fast graph database based on the [RocksDB](https://rocksdb.org/) key-value store. -It is written in Rust. -It also provides a set of utility functions for reading, writing, and processing RDF files. +Oxigraph CLI is a graph database implementing the [SPARQL](https://www.w3.org/TR/sparql11-overview/) standard. +It is packaged as a command line tool allowing to manipulate an RDF files, query them using SPARQL... +It also allows to spawn a HTTP server on top of the database. Oxigraph is in heavy development and SPARQL query evaluation has not been optimized yet. @@ -36,15 +34,15 @@ A preliminary benchmark [is provided](../bench/README.md). You need to have [a recent stable version of Rust and Cargo installed](https://www.rust-lang.org/tools/install). -To download, build and install the latest released version run `cargo install oxigraph_server`. +To download, build and install the latest released version run `cargo install oxigraph-cli`. There is no need to clone the git repository. -To compile the server from source, clone this git repository including its submodules (`git clone --recursive https://github.com/oxigraph/oxigraph.git`), and execute `cargo build --release` in the `server` directory to compile the full server after having downloaded its dependencies. -It will create a fat binary in `target/release/oxigraph_server`. +To compile the command line tool from source, clone this git repository including its submodules (`git clone --recursive https://github.com/oxigraph/oxigraph.git`), and execute `cargo build --release` in the `cli` directory to compile the full binary after having downloaded its dependencies. +It will create a fat binary in `target/release/oxigraph`. ## Usage -Run `oxigraph_server serve --location my_data_storage_directory` to start the server where `my_data_storage_directory` is the directory where you want Oxigraph data to be stored. It listens by default on `localhost:7878`. +Run `oxigraph serve --location my_data_storage_directory` to start the server where `my_data_storage_directory` is the directory where you want Oxigraph data to be stored. It listens by default on `localhost:7878`. The server provides an HTML UI, based on [YASGUI](https://yasgui.triply.cc), with a form to execute SPARQL requests. @@ -78,10 +76,10 @@ It provides the following REST actions: ``` will add the N-Quads file `MY_FILE.nq` to the server dataset. -Use `oxigraph_server --help` to see the possible options when starting the server. +Use `oxigraph --help` to see the possible options when starting the server. It is also possible to load RDF data offline using bulk loading: -`oxigraph_server load --location my_data_storage_directory --file my_file.nq` +`oxigraph load --location my_data_storage_directory --file my_file.nq` ## Using a Docker image @@ -217,14 +215,14 @@ To install Oxigraph server using Homebrew do: brew tap oxigraph/oxigraph brew install oxigraph ``` -It installs the `oxigraph_server` binary. [See the usage documentation to know how to use it](#usage). +It installs the `oxigraph` binary. [See the usage documentation to know how to use it](#usage). ## Systemd It is possible to run Oxigraph in the background using systemd. -For that, you can use the following `oxigraph_server.service` file (it might be inserted into `/etc/systemd/system/` or `$HOME/.config/systemd/user`): +For that, you can use the following `oxigraph.service` file (it might be inserted into `/etc/systemd/system/` or `$HOME/.config/systemd/user`): ```ini [Unit] Description=Oxigraph database server @@ -233,7 +231,7 @@ Wants=network-online.target [Service] Type=notify -ExecStart=/PATH/TO/oxigraph_server serve --location /PATH/TO/OXIGRAPH/DATA +ExecStart=/PATH/TO/oxigraph serve --location /PATH/TO/OXIGRAPH/DATA [Install] WantedBy=multi-user.target @@ -242,8 +240,8 @@ WantedBy=multi-user.target ## Migration guide ### From 0.2 to 0.3 -* The cli API has been completely rewritten. To start the server run `oxigraph_server serve --location MY_STORAGE` instead of `oxigraph_server --file MY_STORAGE`. -* Fast data bulk loading is not supported using `oxigraph_server load --location MY_STORAGE --file MY_FILE`. The file format is guessed from the extension (`.nt`, `.ttl`, `.nq`...). +* The cli API has been completely rewritten. To start the server run `oxigraph serve --location MY_STORAGE` instead of `oxigraph --file MY_STORAGE`. +* Fast data bulk loading is not supported using `oxigraph load --location MY_STORAGE --file MY_FILE`. The file format is guessed from the extension (`.nt`, `.ttl`, `.nq`...). * [RDF-star](https://w3c.github.io/rdf-star/cg-spec/2021-12-17.html) is now implemented. * All operations are now transactional using the "repeatable read" isolation level: the store only exposes changes that have been "committed" (i.e. no partial writes) diff --git a/server/logo.svg b/cli/logo.svg similarity index 100% rename from server/logo.svg rename to cli/logo.svg diff --git a/server/src/main.rs b/cli/src/main.rs similarity index 99% rename from server/src/main.rs rename to cli/src/main.rs index 817fed04..5095076d 100644 --- a/server/src/main.rs +++ b/cli/src/main.rs @@ -38,8 +38,8 @@ const HTML_ROOT_PAGE: &str = include_str!("../templates/query.html"); const LOGO: &str = include_str!("../logo.svg"); #[derive(Parser)] -#[command(about, version)] -/// Oxigraph SPARQL server. +#[command(about, version, name = "oxigraph")] +/// Oxigraph command line toolkit and SPARQL HTTP server. struct Args { #[command(subcommand)] command: Command, @@ -1830,7 +1830,7 @@ mod tests { fn cli_command() -> Result { Ok(Command::from_std( escargot::CargoBuild::new() - .bin(env!("CARGO_PKG_NAME")) + .bin("oxigraph") .manifest_path(format!("{}/Cargo.toml", env!("CARGO_MANIFEST_DIR"))) .run()? .command(), diff --git a/server/templates/query.html b/cli/templates/query.html similarity index 100% rename from server/templates/query.html rename to cli/templates/query.html diff --git a/js/Cargo.toml b/js/Cargo.toml index 0b3f5738..0fa31d8f 100644 --- a/js/Cargo.toml +++ b/js/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "oxigraph_js" +name = "oxigraph-js" version.workspace = true authors.workspace = true license.workspace = true diff --git a/lib/README.md b/lib/README.md index 9b7f1de3..d2afa1bc 100644 --- a/lib/README.md +++ b/lib/README.md @@ -14,7 +14,7 @@ It also provides a set of utility functions for reading, writing, and processing Oxigraph is in heavy development and SPARQL query evaluation has not been optimized yet. -Oxigraph also provides [a standalone HTTP server](https://crates.io/crates/oxigraph_server) and [a Python library](https://pyoxigraph.readthedocs.io/) based on this library. +Oxigraph also provides [a standalone HTTP server](https://crates.io/crates/oxigraph-cli) and [a Python library](https://pyoxigraph.readthedocs.io/) based on this library. Oxigraph implements the following specifications: diff --git a/server b/server new file mode 120000 index 00000000..76ec9f59 --- /dev/null +++ b/server @@ -0,0 +1 @@ +cli \ No newline at end of file diff --git a/testsuite/Cargo.toml b/testsuite/Cargo.toml index a25a27dc..3a473001 100644 --- a/testsuite/Cargo.toml +++ b/testsuite/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "oxigraph_testsuite" +name = "oxigraph-testsuite" version.workspace = true authors.workspace = true license.workspace = true From 555f6b8d7c7cc6e745580b24cfaa4869ff742c96 Mon Sep 17 00:00:00 2001 From: Tpt Date: Fri, 8 Sep 2023 21:33:37 +0200 Subject: [PATCH 080/217] xsd:duration: properly fails when building not-serializable durations P1M1D - P3D is giving 1M and -3D. This is not serializable with xsd:duration formatting --- lib/oxsdatatypes/src/date_time.rs | 4 +- lib/oxsdatatypes/src/duration.rs | 92 +++++++++++++++++++++++++------ lib/oxsdatatypes/src/lib.rs | 3 +- 3 files changed, 80 insertions(+), 19 deletions(-) diff --git a/lib/oxsdatatypes/src/date_time.rs b/lib/oxsdatatypes/src/date_time.rs index 671d00a8..00920b43 100644 --- a/lib/oxsdatatypes/src/date_time.rs +++ b/lib/oxsdatatypes/src/date_time.rs @@ -1877,11 +1877,11 @@ pub fn since_unix_epoch() -> Duration { target_os = "unknown" ))] fn since_unix_epoch() -> Duration { - Duration::new( - 0, + DayTimeDuration::new( Decimal::try_from(crate::Double::from(js_sys::Date::now() / 1000.)) .expect("The current time seems way in the future, it's strange"), ) + .into() } #[cfg(not(any( diff --git a/lib/oxsdatatypes/src/duration.rs b/lib/oxsdatatypes/src/duration.rs index 87be5a22..f8162272 100644 --- a/lib/oxsdatatypes/src/duration.rs +++ b/lib/oxsdatatypes/src/duration.rs @@ -16,12 +16,30 @@ pub struct Duration { impl Duration { #[inline] - #[must_use] - pub fn new(months: impl Into, seconds: impl Into) -> Self { - Self { - year_month: YearMonthDuration::new(months), - day_time: DayTimeDuration::new(seconds), + pub fn new( + months: impl Into, + seconds: impl Into, + ) -> Result { + Self::construct( + YearMonthDuration::new(months), + DayTimeDuration::new(seconds), + ) + } + + #[inline] + fn construct( + year_month: YearMonthDuration, + day_time: DayTimeDuration, + ) -> Result { + if (year_month > YearMonthDuration::default() && day_time < DayTimeDuration::default()) + || (year_month < YearMonthDuration::default() && day_time > DayTimeDuration::default()) + { + return Err(OppositeSignInDurationComponentsError); } + Ok(Self { + year_month, + day_time, + }) } #[inline] @@ -103,10 +121,11 @@ impl Duration { #[must_use] pub fn checked_add(self, rhs: impl Into) -> Option { let rhs = rhs.into(); - Some(Self { - year_month: self.year_month.checked_add(rhs.year_month)?, - day_time: self.day_time.checked_add(rhs.day_time)?, - }) + Self::construct( + self.year_month.checked_add(rhs.year_month)?, + self.day_time.checked_add(rhs.day_time)?, + ) + .ok() } /// [op:subtract-yearMonthDurations](https://www.w3.org/TR/xpath-functions-31/#func-subtract-yearMonthDurations) and [op:subtract-dayTimeDurations](https://www.w3.org/TR/xpath-functions-31/#func-subtract-dayTimeDurations) @@ -116,10 +135,11 @@ impl Duration { #[must_use] pub fn checked_sub(self, rhs: impl Into) -> Option { let rhs = rhs.into(); - Some(Self { - year_month: self.year_month.checked_sub(rhs.year_month)?, - day_time: self.day_time.checked_sub(rhs.day_time)?, - }) + Self::construct( + self.year_month.checked_sub(rhs.year_month)?, + self.day_time.checked_sub(rhs.day_time)?, + ) + .ok() } /// Unary negation. @@ -172,7 +192,7 @@ impl FromStr for Duration { Ok(Self::new( parts.year_month.unwrap_or(0), parts.day_time.unwrap_or_default(), - )) + )?) } } @@ -183,6 +203,9 @@ impl fmt::Display for Duration { let ym = self.year_month.months; let ss = self.day_time.seconds; + if (ym < 0 && ss > 0.into()) || (ym > 0 && ss < 0.into()) { + return Err(fmt::Error); // Not able to format with only a part of the duration that is negative + } if ym < 0 || ss < 0.into() { write!(f, "-")?; } @@ -950,14 +973,35 @@ impl fmt::Display for DurationOverflowError { impl Error for DurationOverflowError {} +/// The year-month and the day-time components of a [`Duration\] have an opposite sign. +#[derive(Debug, Clone, Copy)] +pub struct OppositeSignInDurationComponentsError; + +impl fmt::Display for OppositeSignInDurationComponentsError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "The xsd:yearMonthDuration and xsd:dayTimeDuration components of a xsd:duration can't have opposite sign") + } +} + +impl Error for OppositeSignInDurationComponentsError {} + +impl From for ParseDurationError { + #[inline] + fn from(_: OppositeSignInDurationComponentsError) -> Self { + Self { + msg: "The xsd:yearMonthDuration and xsd:dayTimeDuration components of a xsd:duration can't have opposite sign" + } + } +} + #[cfg(test)] mod tests { use super::*; #[test] fn from_str() -> Result<(), ParseDurationError> { - let min = Duration::new(i64::MIN, Decimal::MIN); - let max = Duration::new(i64::MAX, Decimal::MAX); + let min = Duration::new(i64::MIN, Decimal::MIN)?; + let max = Duration::new(i64::MAX, Decimal::MAX)?; assert_eq!(YearMonthDuration::from_str("P1Y")?.to_string(), "P1Y"); assert_eq!(Duration::from_str("P1Y")?.to_string(), "P1Y"); @@ -1164,6 +1208,14 @@ mod tests { Duration::from_str("P2DT12H5M")?.checked_add(Duration::from_str("P5DT12H")?), Some(Duration::from_str("P8DT5M")?) ); + assert_eq!( + Duration::from_str("P1M2D")?.checked_add(Duration::from_str("-P3D")?), + None + ); + assert_eq!( + Duration::from_str("P1M2D")?.checked_add(Duration::from_str("-P2M")?), + None + ); Ok(()) } @@ -1177,6 +1229,14 @@ mod tests { Duration::from_str("P2DT12H")?.checked_sub(Duration::from_str("P1DT10H30M")?), Some(Duration::from_str("P1DT1H30M")?) ); + assert_eq!( + Duration::from_str("P1M2D")?.checked_sub(Duration::from_str("P3D")?), + None + ); + assert_eq!( + Duration::from_str("P1M2D")?.checked_sub(Duration::from_str("P2M")?), + None + ); Ok(()) } diff --git a/lib/oxsdatatypes/src/lib.rs b/lib/oxsdatatypes/src/lib.rs index a31caf61..336bdd20 100644 --- a/lib/oxsdatatypes/src/lib.rs +++ b/lib/oxsdatatypes/src/lib.rs @@ -20,7 +20,8 @@ pub use self::date_time::{ pub use self::decimal::{Decimal, ParseDecimalError, TooLargeForDecimalError}; pub use self::double::Double; pub use self::duration::{ - DayTimeDuration, Duration, DurationOverflowError, ParseDurationError, YearMonthDuration, + DayTimeDuration, Duration, DurationOverflowError, OppositeSignInDurationComponentsError, + ParseDurationError, YearMonthDuration, }; pub use self::float::Float; pub use self::integer::{Integer, TooLargeForIntegerError}; From 3c51dd31bc99fac24282484cc7c17bc6849e9a46 Mon Sep 17 00:00:00 2001 From: Tpt Date: Sat, 9 Sep 2023 09:44:13 +0200 Subject: [PATCH 081/217] Move back MSRV to 1.67 but keep Cargo.lock to 1.70 Allows to build on Debian unstable while using latest dependencies --- .github/workflows/artifacts.yml | 2 +- .github/workflows/tests.yml | 137 ++++++------------ Cargo.lock | 111 +++++++------- Cargo.toml | 2 +- cli/Cargo.toml | 18 +-- fuzz/Cargo.toml | 3 +- fuzz/fuzz_targets/sparql_eval.rs | 15 +- js/Cargo.toml | 6 +- lib/Cargo.toml | 29 ++-- lib/oxrdfio/Cargo.toml | 4 +- lib/oxrdfxml/Cargo.toml | 6 +- lib/oxsdatatypes/Cargo.toml | 2 +- lib/oxsdatatypes/src/date_time.rs | 1 + lib/oxsdatatypes/src/double.rs | 2 +- lib/oxsdatatypes/src/float.rs | 2 +- lib/oxttl/Cargo.toml | 6 +- lib/sparesults/Cargo.toml | 2 +- lib/sparql-smith/Cargo.toml | 2 +- lib/src/storage/backend/rocksdb.rs | 41 +++--- lints/test_debian_compatibility.py | 116 +++++++++++++++ .../test_spec_links.py | 4 +- oxrocksdb-sys/Cargo.toml | 6 +- python/src/io.rs | 14 +- testsuite/Cargo.toml | 5 +- testsuite/src/sparql_evaluator.rs | 17 +-- 25 files changed, 317 insertions(+), 236 deletions(-) create mode 100644 lints/test_debian_compatibility.py rename test_spec_links.py => lints/test_spec_links.py (95%) diff --git a/.github/workflows/artifacts.yml b/.github/workflows/artifacts.yml index d963f2d0..3951704e 100644 --- a/.github/workflows/artifacts.yml +++ b/.github/workflows/artifacts.yml @@ -22,7 +22,7 @@ jobs: submodules: true - run: rustup update && rustup target add aarch64-unknown-linux-gnu - run: | - sudo apt update && sudo apt install -y g++-aarch64-linux-gnu + sudo apt update && sudo apt-get install -y g++-aarch64-linux-gnu echo -e "\n\n[target.aarch64-unknown-linux-gnu]\nlinker = \"aarch64-linux-gnu-gcc\"" >> .cargo/config.toml - uses: Swatinem/rust-cache@v2 - run: cargo build --release diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index edf5cd42..f811899f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -26,75 +26,7 @@ jobs: - uses: actions/checkout@v3 with: submodules: true - - run: rustup update && rustup component add clippy - - uses: Swatinem/rust-cache@v2 - - run: cargo clippy - working-directory: ./lib/oxsdatatypes - - run: cargo clippy - working-directory: ./lib/oxrdf - - run: cargo clippy - working-directory: ./lib/oxrdfxml - - run: cargo clippy - working-directory: ./lib/oxttl - - run: cargo clippy - working-directory: ./lib/oxrdfio - - run: cargo clippy - working-directory: ./lib/sparesults - - run: cargo clippy - working-directory: ./lib/spargebra - - run: cargo clippy - working-directory: ./lib/sparopt - - run: cargo clippy - working-directory: ./lib - - run: cargo clippy --all-targets --all-features - working-directory: ./lib - - run: cargo clippy - working-directory: ./python - - run: cargo clippy - working-directory: ./cli - - run: cargo clippy - working-directory: ./testsuite - - clippy_wasm_js: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - run: rustup update && rustup target add wasm32-unknown-unknown && rustup component add clippy - - uses: Swatinem/rust-cache@v2 - - run: cargo clippy --lib --tests --target wasm32-unknown-unknown - working-directory: ./js - - clippy_wasi: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - run: rustup update && rustup target add wasm32-wasi && rustup component add clippy - - uses: Swatinem/rust-cache@v2 - - run: cargo clippy --lib --tests --target wasm32-wasi - working-directory: ./lib - - clippy_wasm_unknown: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - run: rustup update && rustup target add wasm32-unknown-unknown && rustup component add clippy - - uses: Swatinem/rust-cache@v2 - - run: cargo clippy --lib --tests --target wasm32-unknown-unknown --features getrandom/custom --features oxsdatatypes/custom-now - working-directory: ./lib - - clippy_msv: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - run: rustup update && rustup override set 1.70.0 && rustup component add clippy + - run: rustup update && rustup default 1.70.0 && rustup component add clippy - uses: Swatinem/rust-cache@v2 - run: cargo clippy -- -D warnings -D clippy::all working-directory: ./lib/oxsdatatypes @@ -123,28 +55,39 @@ jobs: - run: cargo clippy -- -D warnings -D clippy::all working-directory: ./testsuite - clippy_msv_wasm_js: + clippy_wasm_js: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 with: submodules: true - - run: rustup update && rustup override set 1.70.0 && rustup target add wasm32-unknown-unknown && rustup component add clippy + - run: rustup update && rustup default 1.70.0 && rustup target add wasm32-unknown-unknown && rustup component add clippy - uses: Swatinem/rust-cache@v2 - run: cargo clippy --lib --tests --target wasm32-unknown-unknown -- -D warnings -D clippy::all working-directory: ./js - clippy_msv_wasi: + clippy_wasi: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 with: submodules: true - - run: rustup update && rustup override set 1.70.0 && rustup target add wasm32-wasi && rustup component add clippy + - run: rustup update && rustup default 1.70.0 && rustup target add wasm32-wasi && rustup component add clippy - uses: Swatinem/rust-cache@v2 - run: cargo clippy --lib --tests --target wasm32-wasi -- -D warnings -D clippy::all working-directory: ./lib + clippy_wasm_unknown: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + submodules: true + - run: rustup update && rustup default 1.70.0 && rustup target add wasm32-unknown-unknown && rustup component add clippy + - uses: Swatinem/rust-cache@v2 + - run: cargo clippy --lib --tests --target wasm32-unknown-unknown --features getrandom/custom --features oxsdatatypes/custom-now -- -D warnings -D clippy::all + working-directory: ./lib + deny: runs-on: ubuntu-latest steps: @@ -175,8 +118,17 @@ jobs: - run: rustup update - uses: Swatinem/rust-cache@v2 - run: cargo test - env: - RUST_BACKTRACE: 1 + + test_linux_msv: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + submodules: true + - run: rustup update && rustup toolchain install nightly && rustup default 1.67.0 + - uses: Swatinem/rust-cache@v2 + - run: rm Cargo.lock && cargo +nightly update -Z direct-minimal-versions + - run: cargo test address_sanitizer: runs-on: ubuntu-latest @@ -184,12 +136,11 @@ jobs: - uses: actions/checkout@v3 with: submodules: true - - run: rustup update && rustup override set nightly - - run: sudo apt install -y llvm + - run: rustup update && rustup default nightly + - run: sudo apt-get install -y llvm - uses: Swatinem/rust-cache@v2 - run: cargo test --tests --target x86_64-unknown-linux-gnu --workspace --exclude pyoxigraph --exclude oxigraph-testsuite --exclude oxigraph-cli env: - RUST_BACKTRACE: 1 RUSTFLAGS: -Z sanitizer=address test_windows: @@ -225,18 +176,7 @@ jobs: - uses: actions/checkout@v3 with: submodules: true - - run: rustup update - - uses: Swatinem/rust-cache@v2 - - run: cargo doc --all-features - working-directory: ./lib - - rustdoc_msrv: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - run: rustup update && rustup override set 1.70.0 + - run: rustup update && rustup default 1.70.0 - uses: Swatinem/rust-cache@v2 - run: cargo doc --all-features working-directory: ./lib @@ -300,7 +240,7 @@ jobs: - uses: actions/checkout@v3 with: submodules: true - - run: rustup update + - run: rustup update && rustup toolchain install nightly && rustup default 1.67.0 - uses: Swatinem/rust-cache@v2 - uses: actions/setup-python@v4 with: @@ -308,6 +248,7 @@ jobs: cache: pip cache-dependency-path: '**/requirements.dev.txt' - run: pip install -r python/requirements.dev.txt + - run: rm Cargo.lock && cargo +nightly update -Z direct-minimal-versions - run: maturin build -m python/Cargo.toml - run: pip install --no-index --find-links=target/wheels/ pyoxigraph - run: rm -r target/wheels @@ -346,7 +287,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - run: sudo apt install -y clang-format + - run: sudo apt-get install -y clang-format - run: clang-format --Werror --dry-run oxrocksdb-sys/api/* fuzz_changes: @@ -423,11 +364,19 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - run: sudo apt install -y shellcheck + - run: sudo apt-get install -y shellcheck - run: git grep -l '^#\( *shellcheck \|!\(/bin/\|/usr/bin/env \)\(sh\|bash\|dash\|ksh\)\)' | xargs shellcheck spec_links: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - run: python test_spec_links.py + - run: python lints/test_spec_links.py + + debian_compatibility: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - run: rustup update + - uses: Swatinem/rust-cache@v2 + - run: python lints/test_debian_compatibility.py diff --git a/Cargo.lock b/Cargo.lock index 63a066a1..e9b6b056 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -19,9 +19,9 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "aho-corasick" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6748e8def348ed4d14996fa801f4122cd763fff530258cdc03f64b25f89d3a5a" +checksum = "0c378d78423fdad8089616f827526ee33c19f2fddbd5de1629152c9593ba4783" dependencies = [ "memchr", ] @@ -154,9 +154,9 @@ checksum = "414dcefbc63d77c526a76b3afcf6fbb9b5e2791c19c3aa2297733208750c6e53" [[package]] name = "bindgen" -version = "0.66.1" +version = "0.68.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2b84e06fc203107bfbad243f4aba2af864eb7db3b1cf46ea0a023b0b433d2a7" +checksum = "726e4313eb6ec35d2730258ad4e15b547ee75d6afaa1361a922e78e59b7d8078" dependencies = [ "bitflags 2.4.0", "cexpr", @@ -171,7 +171,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.29", + "syn 2.0.31", "which", ] @@ -198,9 +198,9 @@ dependencies = [ [[package]] name = "bstr" -version = "1.6.0" +version = "1.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6798148dccfbff0fae41c7574d2fa8f1ef3492fba0face179de5d8d447d67b05" +checksum = "4c2f7349907b712260e64b0afe2f84692af14a454be26187d9df565c7f69266a" dependencies = [ "memchr", "regex-automata", @@ -215,9 +215,9 @@ checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" [[package]] name = "bytes" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" +checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" [[package]] name = "cast" @@ -290,20 +290,19 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.0" +version = "4.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d5f1946157a96594eb2d2c10eb7ad9a2b27518cb3000209dec700c35df9197d" +checksum = "6a13b88d2c62ff462f88e4a121f17a82c1af05693a2f192b5c38d14de73c19f6" dependencies = [ "clap_builder", "clap_derive", - "once_cell", ] [[package]] name = "clap_builder" -version = "4.4.0" +version = "4.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78116e32a042dd73c2901f0dc30790d20ff3447f3e3472fad359e8c3d282bcd6" +checksum = "2bb9faaa7c2ef94b2743a21f5a29e6f0010dff4caa69ac8e9d6cf8b6fa74da08" dependencies = [ "anstream", "anstyle", @@ -313,14 +312,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.4.0" +version = "4.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9fd1a5729c4548118d7d70ff234a44868d00489a4b6597b0b020918a0e91a1a" +checksum = "0862016ff20d69b84ef8247369fabf5c008a7417002411897d40ee1f4532b873" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.29", + "syn 2.0.31", ] [[package]] @@ -482,7 +481,7 @@ checksum = "53e0efad4403bfc52dc201159c4b842a246a14b98c64b55dfd0f2d89729dfeb8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.29", + "syn 2.0.31", ] [[package]] @@ -515,9 +514,9 @@ checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" [[package]] name = "errno" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b30f669a7961ef1631673d2766cc92f52d64f7ef354d4fe0ddfd30ed52f0f4f" +checksum = "136526188508e25c6fef639d7927dfb3e0e3084488bf202267829cf7fc23dbdd" dependencies = [ "errno-dragonfly", "libc", @@ -678,6 +677,15 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "home" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" +dependencies = [ + "windows-sys", +] + [[package]] name = "httparse" version = "1.8.0" @@ -838,9 +846,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.5.0" +version = "2.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c" [[package]] name = "memoffset" @@ -903,9 +911,9 @@ dependencies = [ [[package]] name = "object" -version = "0.32.0" +version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77ac5bbd07aea88c60a577a1ce218075ffd59208b2d7ca97adf9bfc5aeb21ebe" +checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" dependencies = [ "memchr", ] @@ -952,6 +960,7 @@ dependencies = [ "hex", "js-sys", "json-event-parser", + "lazy_static", "libc", "md-5", "oxhttp", @@ -1008,6 +1017,7 @@ dependencies = [ "anyhow", "clap", "criterion", + "lazy_static", "oxigraph", "oxttl", "rio_api", @@ -1229,12 +1239,12 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.2.12" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c64d9ba0963cdcea2e1b2230fbae2bab30eb25a174be395c41e764bfb65dd62" +checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d" dependencies = [ "proc-macro2", - "syn 2.0.29", + "syn 2.0.31", ] [[package]] @@ -1396,9 +1406,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.9.4" +version = "1.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12de2eff854e5fa4b1295edd650e227e9d8fb0c9e90b12e7f36d6a6811791a29" +checksum = "697061221ea1b4a94a624f67d0ae2bfe4e22b8a17b6a192afb11046542cc8c47" dependencies = [ "aho-corasick", "memchr", @@ -1408,9 +1418,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49530408a136e16e5b486e883fbb6ba058e8e4e8ae6621a77b048b314336e629" +checksum = "c2f401f4955220693b56f8ec66ee9c78abffd8d1c4f23dc41a23839eb88f0795" dependencies = [ "aho-corasick", "memchr", @@ -1469,9 +1479,9 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "rustix" -version = "0.38.9" +version = "0.38.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bfe0f2582b4931a45d1fa608f8a8722e8b3c7ac54dd6d5f3b3212791fedef49" +checksum = "c0c3dde1fc030af041adc40e79c0e7fbcf431dd24870053d187d7c66e4b87453" dependencies = [ "bitflags 2.4.0", "errno", @@ -1482,9 +1492,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.6" +version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d1feddffcfcc0b33f5c6ce9a29e341e4cd59c3f78e7ee45f4a40c038b1d6cbb" +checksum = "cd8d6c9f025a446bc4d18ad9632e69aec8f287aa84499ee335599fabd20c3fd8" dependencies = [ "log", "ring", @@ -1603,7 +1613,7 @@ checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.29", + "syn 2.0.31", ] [[package]] @@ -1641,9 +1651,9 @@ dependencies = [ [[package]] name = "shlex" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3" +checksum = "a7cee0529a6d40f580e7a5e6c495c8fbfe21b7b52795ed4bb5e62cdf92bc6380" [[package]] name = "siphasher" @@ -1718,9 +1728,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.29" +version = "2.0.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c324c494eba9d92503e6f1ef2e6df781e78f6a7705a0202d9801b198807d518a" +checksum = "718fa2415bcb8d8bd775917a1bf12a7931b6dfa890753378538118181e0cb398" dependencies = [ "proc-macro2", "quote", @@ -1855,7 +1865,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.29", + "syn 2.0.31", ] [[package]] @@ -1905,9 +1915,9 @@ checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" [[package]] name = "url" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50bff7831e19200a85b17131d085c25d7811bc4e186efdaf54bbd132994a88cb" +checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" dependencies = [ "form_urlencoded", "idna", @@ -1937,9 +1947,9 @@ dependencies = [ [[package]] name = "walkdir" -version = "2.3.3" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36df944cda56c7d8d8b7496af378e6b16de9284591917d307c9b4d313c44e698" +checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee" dependencies = [ "same-file", "winapi-util", @@ -1972,7 +1982,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.29", + "syn 2.0.31", "wasm-bindgen-shared", ] @@ -1994,7 +2004,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.29", + "syn 2.0.31", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -2017,13 +2027,14 @@ dependencies = [ [[package]] name = "which" -version = "4.4.0" +version = "4.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2441c784c52b289a054b7201fc93253e288f094e2f4be9058343127c4226a269" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" dependencies = [ "either", - "libc", + "home", "once_cell", + "rustix", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 8264afa9..0d6bc83e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,7 +24,7 @@ authors = ["Tpt "] license = "MIT OR Apache-2.0" homepage = "https://oxigraph.org/" edition = "2021" -rust-version = "1.70" +rust-version = "1.67" [profile.release] lto = true diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 62c6b40b..486a46bf 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -19,18 +19,18 @@ name = "oxigraph" path = "src/main.rs" [dependencies] -anyhow = "1" -oxhttp = { version = "0.1", features = ["rayon"] } -clap = { version = "4", features = ["derive"] } +anyhow = "1.0.72" +oxhttp = { version = "0.1.7", features = ["rayon"] } +clap = { version = ">=4.0, <5.0", features = ["derive"] } oxigraph = { version = "0.4.0-alpha.1-dev", path = "../lib", features = ["http_client"] } rand = "0.8" -url = "2" +url = "2.4" oxiri = "0.2" -flate2 = "1" -rayon-core = "1" +flate2 = "1.0" +rayon-core = "1.11" [dev-dependencies] -assert_cmd = "2" -assert_fs = "1" +assert_cmd = "2.0" +assert_fs = "1.0" escargot = "0.5" -predicates = "3" +predicates = ">=2.0, <4.0" diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index 4e41ea60..900b0b92 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -8,7 +8,8 @@ edition = "2021" cargo-fuzz = true [dependencies] -anyhow = "1" +anyhow = "1.0.72" +lazy_static = "1.4" libfuzzer-sys = "0.4" oxrdf = { path = "../lib/oxrdf", features = ["rdf-star"] } oxttl = { path = "../lib/oxttl", features = ["rdf-star"] } diff --git a/fuzz/fuzz_targets/sparql_eval.rs b/fuzz/fuzz_targets/sparql_eval.rs index 24c8a176..5b52f4bd 100644 --- a/fuzz/fuzz_targets/sparql_eval.rs +++ b/fuzz/fuzz_targets/sparql_eval.rs @@ -1,26 +1,27 @@ #![no_main] +use lazy_static::lazy_static; use libfuzzer_sys::fuzz_target; use oxigraph::io::RdfFormat; use oxigraph::sparql::{Query, QueryOptions, QueryResults, QuerySolutionIter}; use oxigraph::store::Store; -use std::sync::OnceLock; -fuzz_target!(|data: sparql_smith::Query| { - static STORE: OnceLock = OnceLock::new(); - let store = STORE.get_or_init(|| { +lazy_static! { + static ref STORE: Store = { let store = Store::new().unwrap(); store .load_dataset(sparql_smith::DATA_TRIG.as_bytes(), RdfFormat::TriG, None) .unwrap(); store - }); + }; +} +fuzz_target!(|data: sparql_smith::Query| { let query_str = data.to_string(); if let Ok(query) = Query::parse(&query_str, None) { let options = QueryOptions::default(); - let with_opt = store.query_opt(query.clone(), options.clone()).unwrap(); - let without_opt = store + let with_opt = STORE.query_opt(query.clone(), options.clone()).unwrap(); + let without_opt = STORE .query_opt(query, options.without_optimizations()) .unwrap(); match (with_opt, without_opt) { diff --git a/js/Cargo.toml b/js/Cargo.toml index 0fa31d8f..bfc2ffd5 100644 --- a/js/Cargo.toml +++ b/js/Cargo.toml @@ -17,6 +17,6 @@ name = "oxigraph" [dependencies] oxigraph = { version = "0.4.0-alpha.1-dev", path="../lib", features = ["js"] } -wasm-bindgen = "0.2" -js-sys = "0.3" -console_error_panic_hook = "0.1" +wasm-bindgen = "0.2.83" +js-sys = "0.3.60" +console_error_panic_hook = "0.1.7" diff --git a/lib/Cargo.toml b/lib/Cargo.toml index 2c37165f..0ed6e9a9 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -22,36 +22,37 @@ http_client = ["oxhttp", "oxhttp/rustls"] rocksdb_debug = [] [dependencies] -rand = "0.8" -md-5 = "0.10" -sha1 = "0.10" -sha2 = "0.10" digest = "0.10" -regex = "1" -oxilangtag = "0.1" -oxiri = "0.2" hex = "0.4" -siphasher = "1" json-event-parser = "0.1" +lazy_static = "1.4" +md-5 = "0.10" +oxilangtag = "0.1" +oxiri = "0.2" oxrdf = { version = "0.2.0-alpha.1-dev", path = "oxrdf", features = ["rdf-star", "oxsdatatypes"] } -oxsdatatypes = { version = "0.2.0-alpha.1-dev", path="oxsdatatypes" } oxrdfio = { version = "0.1.0-alpha.1-dev", path = "oxrdfio", features = ["rdf-star"] } +oxsdatatypes = { version = "0.2.0-alpha.1-dev", path="oxsdatatypes" } +rand = "0.8" +regex = "1.7" +sha1 = "0.10" +sha2 = "0.10" +siphasher = ">=0.3, <2.0" +sparesults = { version = "0.2.0-alpha.1-dev", path = "sparesults", features = ["rdf-star"] } spargebra = { version = "0.3.0-alpha.1-dev", path = "spargebra", features = ["rdf-star", "sep-0002", "sep-0006"] } sparopt = { version = "0.1.0-alpha.1-dev", path="sparopt", features = ["rdf-star", "sep-0002", "sep-0006"] } -sparesults = { version = "0.2.0-alpha.1-dev", path = "sparesults", features = ["rdf-star"] } [target.'cfg(not(target_family = "wasm"))'.dependencies] -libc = "0.2" +libc = "0.2.147" oxrocksdb-sys = { version = "0.4.0-alpha.1-dev", path="../oxrocksdb-sys" } -oxhttp = { version = "0.1", optional = true } +oxhttp = { version = "0.1.7", optional = true } [target.'cfg(all(target_family = "wasm", target_os = "unknown"))'.dependencies] getrandom = "0.2" -js-sys = { version = "0.3", optional = true } +js-sys = { version = "0.3.60", optional = true } [target.'cfg(not(target_family = "wasm"))'.dev-dependencies] criterion = "0.5" -oxhttp = "0.1" +oxhttp = "0.1.7" zstd = "0.12" [package.metadata.docs.rs] diff --git a/lib/oxrdfio/Cargo.toml b/lib/oxrdfio/Cargo.toml index 1ca0ce17..45683a5e 100644 --- a/lib/oxrdfio/Cargo.toml +++ b/lib/oxrdfio/Cargo.toml @@ -23,10 +23,10 @@ rdf-star = ["oxrdf/rdf-star", "oxttl/rdf-star"] oxrdf = { version = "0.2.0-alpha.1-dev", path = "../oxrdf" } oxrdfxml = { version = "0.1.0-alpha.1-dev", path = "../oxrdfxml" } oxttl = { version = "0.1.0-alpha.1-dev", path = "../oxttl" } -tokio = { version = "1", optional = true, features = ["io-util"] } +tokio = { version = "1.29", optional = true, features = ["io-util"] } [dev-dependencies] -tokio = { version = "1", features = ["rt", "macros"] } +tokio = { version = "1.29", features = ["rt", "macros"] } [package.metadata.docs.rs] all-features = true diff --git a/lib/oxrdfxml/Cargo.toml b/lib/oxrdfxml/Cargo.toml index fa7ef2b1..83739b35 100644 --- a/lib/oxrdfxml/Cargo.toml +++ b/lib/oxrdfxml/Cargo.toml @@ -22,11 +22,11 @@ async-tokio = ["dep:tokio", "quick-xml/async-tokio"] oxrdf = { version = "0.2.0-alpha.1-dev", path = "../oxrdf" } oxilangtag = "0.1" oxiri = "0.2" -quick-xml = "0.30" -tokio = { version = "1", optional = true, features = ["io-util"] } +quick-xml = ">=0.29, <0.31" +tokio = { version = "1.29", optional = true, features = ["io-util"] } [dev-dependencies] -tokio = { version = "1", features = ["rt", "macros"] } +tokio = { version = "1.29", features = ["rt", "macros"] } [package.metadata.docs.rs] all-features = true diff --git a/lib/oxsdatatypes/Cargo.toml b/lib/oxsdatatypes/Cargo.toml index 64b974a9..10f9f4f2 100644 --- a/lib/oxsdatatypes/Cargo.toml +++ b/lib/oxsdatatypes/Cargo.toml @@ -19,7 +19,7 @@ js = ["js-sys"] custom-now = [] [target.'cfg(all(target_family = "wasm", target_os = "unknown"))'.dependencies] -js-sys = { version = "0.3", optional = true } +js-sys = { version = "0.3.60", optional = true } [package.metadata.docs.rs] all-features = true diff --git a/lib/oxsdatatypes/src/date_time.rs b/lib/oxsdatatypes/src/date_time.rs index 00920b43..6c525794 100644 --- a/lib/oxsdatatypes/src/date_time.rs +++ b/lib/oxsdatatypes/src/date_time.rs @@ -1867,6 +1867,7 @@ pub fn since_unix_epoch() -> Duration { fn custom_ox_now() -> Duration; } + // SAFETY: Must be defined, if not compilation fails unsafe { custom_ox_now() } } diff --git a/lib/oxsdatatypes/src/double.rs b/lib/oxsdatatypes/src/double.rs index bc040738..6e3f890e 100644 --- a/lib/oxsdatatypes/src/double.rs +++ b/lib/oxsdatatypes/src/double.rs @@ -173,7 +173,7 @@ impl From for Double { impl From for Double { #[inline] fn from(value: Boolean) -> Self { - f64::from(bool::from(value)).into() + if bool::from(value) { 1. } else { 0. }.into() } } diff --git a/lib/oxsdatatypes/src/float.rs b/lib/oxsdatatypes/src/float.rs index 996a6401..ce88c0e4 100644 --- a/lib/oxsdatatypes/src/float.rs +++ b/lib/oxsdatatypes/src/float.rs @@ -153,7 +153,7 @@ impl From for Float { impl From for Float { #[inline] fn from(value: Boolean) -> Self { - f32::from(bool::from(value)).into() + if bool::from(value) { 1. } else { 0. }.into() } } diff --git a/lib/oxttl/Cargo.toml b/lib/oxttl/Cargo.toml index 0cc2741b..98b583bf 100644 --- a/lib/oxttl/Cargo.toml +++ b/lib/oxttl/Cargo.toml @@ -20,14 +20,14 @@ rdf-star = ["oxrdf/rdf-star"] async-tokio = ["dep:tokio"] [dependencies] -memchr = "2" +memchr = "2.5" oxrdf = { version = "0.2.0-alpha.1-dev", path = "../oxrdf" } oxiri = "0.2" oxilangtag = "0.1" -tokio = { version = "1", optional = true, features = ["io-util"] } +tokio = { version = "1.29", optional = true, features = ["io-util"] } [dev-dependencies] -tokio = { version = "1", features = ["rt", "macros"] } +tokio = { version = "1.29", features = ["rt", "macros"] } [package.metadata.docs.rs] all-features = true diff --git a/lib/sparesults/Cargo.toml b/lib/sparesults/Cargo.toml index 8cee3164..0a7904db 100644 --- a/lib/sparesults/Cargo.toml +++ b/lib/sparesults/Cargo.toml @@ -21,7 +21,7 @@ rdf-star = ["oxrdf/rdf-star"] [dependencies] json-event-parser = "0.1" oxrdf = { version = "0.2.0-alpha.1-dev", path="../oxrdf" } -quick-xml = "0.30" +quick-xml = ">=0.29, <0.31" [package.metadata.docs.rs] all-features = true diff --git a/lib/sparql-smith/Cargo.toml b/lib/sparql-smith/Cargo.toml index 5fe5e33c..22d361d6 100644 --- a/lib/sparql-smith/Cargo.toml +++ b/lib/sparql-smith/Cargo.toml @@ -21,4 +21,4 @@ order = [] sep-0006 = [] [dependencies] -arbitrary = { version = "1", features = ["derive"] } +arbitrary = { version = "1.3", features = ["derive"] } diff --git a/lib/src/storage/backend/rocksdb.rs b/lib/src/storage/backend/rocksdb.rs index 6d065059..4360e3e6 100644 --- a/lib/src/storage/backend/rocksdb.rs +++ b/lib/src/storage/backend/rocksdb.rs @@ -9,6 +9,7 @@ )] use crate::storage::error::{CorruptionError, StorageError}; +use lazy_static::lazy_static; use libc::{self, c_void, free}; use oxrocksdb_sys::*; use rand::random; @@ -25,7 +26,7 @@ use std::marker::PhantomData; use std::ops::Deref; use std::path::{Path, PathBuf}; use std::rc::{Rc, Weak}; -use std::sync::{Arc, OnceLock}; +use std::sync::Arc; use std::thread::{available_parallelism, yield_now}; use std::{ptr, slice}; @@ -56,6 +57,23 @@ macro_rules! ffi_result_impl { }} } +lazy_static! { + static ref ROCKSDB_ENV: UnsafeEnv = { + unsafe { + let env = rocksdb_create_default_env(); + assert!(!env.is_null(), "rocksdb_create_default_env returned null"); + UnsafeEnv(env) + } + }; + static ref ROCKSDB_MEM_ENV: UnsafeEnv = { + unsafe { + let env = rocksdb_create_mem_env(); + assert!(!env.is_null(), "rocksdb_create_mem_env returned null"); + UnsafeEnv(env) + } + }; +} + pub struct ColumnFamilyDefinition { pub name: &'static str, pub use_iter: bool, @@ -454,9 +472,6 @@ impl Db { limit_max_open_files: bool, in_memory: bool, ) -> Result<*mut rocksdb_options_t, StorageError> { - static ROCKSDB_ENV: OnceLock = OnceLock::new(); - static ROCKSDB_MEM_ENV: OnceLock = OnceLock::new(); - unsafe { let options = rocksdb_options_create(); assert!(!options.is_null(), "rocksdb_options_create returned null"); @@ -493,19 +508,10 @@ impl Db { rocksdb_options_set_env( options, if in_memory { - ROCKSDB_MEM_ENV.get_or_init(|| { - let env = rocksdb_create_mem_env(); - assert!(!env.is_null(), "rocksdb_create_mem_env returned null"); - UnsafeEnv(env) - }) + ROCKSDB_MEM_ENV.0 } else { - ROCKSDB_ENV.get_or_init(|| { - let env = rocksdb_create_default_env(); - assert!(!env.is_null(), "rocksdb_create_default_env returned null"); - UnsafeEnv(env) - }) - } - .0, + ROCKSDB_ENV.0 + }, ); Ok(options) } @@ -1394,8 +1400,7 @@ impl From for StorageError { struct UnsafeEnv(*mut rocksdb_env_t); -// Hack for OnceCell. OK because only written in OnceCell and used in a thread-safe way by RocksDB -unsafe impl Send for UnsafeEnv {} +// Hack for lazy_static. OK because only written in lazy static and used in a thread-safe way by RocksDB unsafe impl Sync for UnsafeEnv {} fn path_to_cstring(path: &Path) -> Result { diff --git a/lints/test_debian_compatibility.py b/lints/test_debian_compatibility.py new file mode 100644 index 00000000..ec11b319 --- /dev/null +++ b/lints/test_debian_compatibility.py @@ -0,0 +1,116 @@ +import json +import subprocess +from pathlib import Path +from urllib.request import urlopen + +TARGET_DEBIAN_VERSIONS = ["sid"] +IGNORE_PACKAGES = {"oxigraph-js", "oxigraph-testsuite", "pyoxigraph", "sparql-smith"} +ALLOWED_MISSING_PACKAGES = {"escargot", "quick-xml"} + +base_path = Path(__file__).parent.parent + +cargo_metadata = json.loads( + subprocess.check_output(["cargo", "metadata", "--format-version", "1"]) +) +package_by_id = {package["id"]: package for package in cargo_metadata["packages"]} +workspace_packages = { + package_id.split(" ")[0] + for package_id in cargo_metadata["workspace_default_members"] +} +debian_cache = {} +errors = set() + + +def parse_version(version): + return tuple(int(e) for e in version.split("-")[0].split(".")) + + +for package_id in cargo_metadata["workspace_default_members"]: + package = package_by_id[package_id] + if package["name"] in IGNORE_PACKAGES: + continue + for dependency in package["dependencies"]: + if ( + dependency["name"] in workspace_packages + or dependency["name"] in ALLOWED_MISSING_PACKAGES + ): + continue + candidate_debian_name = f"rust-{dependency['name'].replace('_', '-')}" + if dependency["name"] not in debian_cache: + with urlopen( + f"https://sources.debian.org/api/src/{candidate_debian_name}/" + ) as response: + debian_package = json.loads(response.read().decode()) + debian_cache[candidate_debian_name] = debian_package + debian_package = debian_cache[candidate_debian_name] + if "error" in debian_package: + errors.add(f"No Debian package found for {dependency['name']}") + continue + for target_debian_suite in TARGET_DEBIAN_VERSIONS: + debian_version = next( + ( + debian_version + for debian_version in debian_package["versions"] + if target_debian_suite in debian_version["suites"] + ), + None, + ) + if debian_version is None: + errors.add( + f"The debian package {debian_package['package']} does not support {target_debian_suite}" + ) + continue + + # We check the debian version is compatible with the req version + parsed_debian_version = parse_version(debian_version["version"]) + for range_element in dependency["req"].split(","): + range_element = range_element.strip() + if range_element.startswith("^"): + first_found = False + for expected, actual in zip( + parse_version(range_element[1:]), parsed_debian_version + ): + if first_found: + if actual > expected: + break # Done + if actual < expected: + errors.add( + f"The debian package {debian_package['package']} version {debian_version['version']} is not compatible with requirement {range_element}" + ) + break + else: + if actual != expected: + errors.add( + f"The debian package {debian_package['package']} version {debian_version['version']} is not compatible with requirement {range_element}" + ) + if expected != 0: + first_found = True + elif range_element.startswith(">="): + if not parsed_debian_version >= parse_version(range_element[2:]): + errors.add( + f"The debian package {debian_package['package']} version {debian_version['version']} is not compatible with requirement {range_element}" + ) + elif range_element.startswith(">"): + if not parsed_debian_version > parse_version(range_element[1:]): + errors.add( + f"The debian package {debian_package['package']} version {debian_version['version']} is not compatible with requirement {range_element}" + ) + elif range_element.startswith("<="): + if not parsed_debian_version <= parse_version(range_element[2:]): + errors.add( + f"The debian package {debian_package['package']} version {debian_version['version']} is not compatible with requirement {range_element}" + ) + elif range_element.startswith("<"): + if not parsed_debian_version < parse_version(range_element[1:]): + errors.add( + f"The debian package {debian_package['package']} version {debian_version['version']} is not compatible with requirement {range_element}" + ) + else: + errors.add( + f"The requirement {range_element} of {dependency['name']} is not supported by this script" + ) + +for error in sorted(errors): + print(error) +if errors: + exit(1) diff --git a/test_spec_links.py b/lints/test_spec_links.py similarity index 95% rename from test_spec_links.py rename to lints/test_spec_links.py index 76fb2e88..a0d2c2e9 100644 --- a/test_spec_links.py +++ b/lints/test_spec_links.py @@ -10,12 +10,12 @@ LINK_REGEXES = { r"<(https?://(w3c.github.io|www.w3.org)/[^>]+)>`_", # reStructuredText } -base_path = Path(__file__).parent +base_path = Path(__file__).parent.parent spec_cache = {} errors = set() for ext in ("md", "rs", "rst"): - for file in Path(__file__).parent.rglob(f"*.{ext}"): + for file in base_path.rglob(f"*.{ext}"): content = file.read_text() for link_regex in LINK_REGEXES: for m in re.finditer(link_regex, content): diff --git a/oxrocksdb-sys/Cargo.toml b/oxrocksdb-sys/Cargo.toml index e91881e9..70e03ff3 100644 --- a/oxrocksdb-sys/Cargo.toml +++ b/oxrocksdb-sys/Cargo.toml @@ -15,8 +15,8 @@ build = "build.rs" links = "rocksdb" [dependencies] -libc = "0.2" +libc = "0.2.147" [build-dependencies] -bindgen = "0.66" -cc = { version = "1", features = ["parallel"] } +bindgen = ">=0.60, <0.69" +cc = { version = "1.0.73", features = ["parallel"] } diff --git a/python/src/io.rs b/python/src/io.rs index 86f39992..9bd957c6 100644 --- a/python/src/io.rs +++ b/python/src/io.rs @@ -12,7 +12,6 @@ use std::error::Error; use std::fs::File; use std::io::{self, BufWriter, Cursor, Read, Write}; use std::path::{Path, PathBuf}; -use std::sync::OnceLock; pub fn add_to_module(module: &PyModule) -> PyResult<()> { module.add_wrapped(wrap_pyfunction!(parse))?; @@ -325,7 +324,7 @@ pub fn map_parse_error(error: ParseError, file_path: Option) -> PyErr { match error { ParseError::Syntax(error) => { // Python 3.9 does not support end line and end column - if python_version() >= (3, 10, 0) { + if python_version() >= (3, 10) { let params = if let Some(location) = error.location() { ( file_path, @@ -382,12 +381,9 @@ pub fn allow_threads_unsafe(f: impl FnOnce() -> T) -> T { f() } -fn python_version() -> (u8, u8, u8) { - static VERSION: OnceLock<(u8, u8, u8)> = OnceLock::new(); - *VERSION.get_or_init(|| { - Python::with_gil(|py| { - let v = py.version_info(); - (v.major, v.minor, v.patch) - }) +fn python_version() -> (u8, u8) { + Python::with_gil(|py| { + let v = py.version_info(); + (v.major, v.minor) }) } diff --git a/testsuite/Cargo.toml b/testsuite/Cargo.toml index 3a473001..47c3ce2e 100644 --- a/testsuite/Cargo.toml +++ b/testsuite/Cargo.toml @@ -11,8 +11,9 @@ rust-version.workspace = true publish = false [dependencies] -anyhow = "1" -clap = { version = "4", features = ["derive"] } +anyhow = "1.0.72" +clap = { version = ">=4.0, <5.0", features = ["derive"] } +lazy_static = "1.4" oxigraph = { path = "../lib" } oxttl = { path= "../lib/oxttl" } sparopt = { path = "../lib/sparopt" } diff --git a/testsuite/src/sparql_evaluator.rs b/testsuite/src/sparql_evaluator.rs index 67081be6..d5b4d265 100644 --- a/testsuite/src/sparql_evaluator.rs +++ b/testsuite/src/sparql_evaluator.rs @@ -4,6 +4,7 @@ use crate::manifest::*; use crate::report::{dataset_diff, format_diff}; use crate::vocab::*; use anyhow::{anyhow, bail, ensure, Error, Result}; +use lazy_static::lazy_static; use oxigraph::model::vocab::*; use oxigraph::model::*; use oxigraph::sparql::results::QueryResultsFormat; @@ -15,7 +16,7 @@ use std::fmt::Write; use std::io::{self, BufReader, Cursor}; use std::ops::Deref; use std::str::FromStr; -use std::sync::{Arc, Mutex, OnceLock}; +use std::sync::{Arc, Mutex}; pub fn register_sparql_tests(evaluator: &mut TestEvaluator) { evaluator.register( @@ -739,11 +740,13 @@ fn evaluate_query_optimization_test(test: &Test) -> Result<()> { Ok(()) } -// Pool of stores to avoid allocating/deallocating them a lot -static STORE_POOL: OnceLock>> = OnceLock::new(); +lazy_static! { + // Pool of stores to avoid allocating/deallocating them a lot + static ref STORE_POOL: Mutex> = Mutex::new(Vec::new()); +} fn get_store() -> Result { - let store = if let Some(store) = STORE_POOL.get_or_init(Mutex::default).lock().unwrap().pop() { + let store = if let Some(store) = STORE_POOL.lock().unwrap().pop() { store } else { Store::new()? @@ -758,11 +761,7 @@ struct StoreRef { impl Drop for StoreRef { fn drop(&mut self) { if self.store.clear().is_ok() { - STORE_POOL - .get_or_init(Mutex::default) - .lock() - .unwrap() - .push(self.store.clone()) + STORE_POOL.lock().unwrap().push(self.store.clone()) } } } From be074000ccf9b5c00c3de243f697cffa5fde3088 Mon Sep 17 00:00:00 2001 From: Tpt Date: Sun, 10 Sep 2023 16:28:54 +0200 Subject: [PATCH 082/217] Allows dynamic RocksDB linking --- .github/workflows/install_rocksdb.sh | 11 +++ .github/workflows/tests.yml | 17 +++- Cargo.lock | 1 + oxrocksdb-sys/Cargo.toml | 4 + oxrocksdb-sys/README.md | 5 ++ oxrocksdb-sys/api/c.cc | 111 ++++++++++++++++++++++++--- oxrocksdb-sys/api/c.h | 2 +- oxrocksdb-sys/build.rs | 56 +++++++++++--- 8 files changed, 186 insertions(+), 21 deletions(-) create mode 100644 .github/workflows/install_rocksdb.sh diff --git a/.github/workflows/install_rocksdb.sh b/.github/workflows/install_rocksdb.sh new file mode 100644 index 00000000..ac2b712b --- /dev/null +++ b/.github/workflows/install_rocksdb.sh @@ -0,0 +1,11 @@ +if [ -f "rocksdb" ] +then + cd rocksdb || exit +else + git clone https://github.com/facebook/rocksdb.git + cd rocksdb || exit + git checkout v8.0.0 + make shared_lib +fi +sudo make install-shared +sudo ldconfig /usr/local/lib diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f811899f..d570e796 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -130,7 +130,7 @@ jobs: - run: rm Cargo.lock && cargo +nightly update -Z direct-minimal-versions - run: cargo test - address_sanitizer: + test_linux_address_sanitizer: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -143,6 +143,21 @@ jobs: env: RUSTFLAGS: -Z sanitizer=address + test_linux_dynamic_linking: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + submodules: true + - run: rustup update + - uses: Swatinem/rust-cache@v2 + - uses: actions/cache@v3 + with: + path: rocksdb + key: ${{ runner.os }}-rocksdb-8.0.0 + - run: bash .github/workflows/install_rocksdb.sh + - run: cargo test --tests --features oxrocksdb-sys/pkg-config + test_windows: runs-on: windows-latest steps: diff --git a/Cargo.lock b/Cargo.lock index e9b6b056..a958c167 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1078,6 +1078,7 @@ dependencies = [ "bindgen", "cc", "libc", + "pkg-config", ] [[package]] diff --git a/oxrocksdb-sys/Cargo.toml b/oxrocksdb-sys/Cargo.toml index 70e03ff3..849a8ab7 100644 --- a/oxrocksdb-sys/Cargo.toml +++ b/oxrocksdb-sys/Cargo.toml @@ -14,9 +14,13 @@ rust-version.workspace = true build = "build.rs" links = "rocksdb" +[features] +pkg-config = ["dep:pkg-config"] + [dependencies] libc = "0.2.147" [build-dependencies] +pkg-config = { version = "0.3.25", optional = true } bindgen = ">=0.60, <0.69" cc = { version = "1.0.73", features = ["parallel"] } diff --git a/oxrocksdb-sys/README.md b/oxrocksdb-sys/README.md index f4587db6..15b9cdbd 100644 --- a/oxrocksdb-sys/README.md +++ b/oxrocksdb-sys/README.md @@ -3,4 +3,9 @@ Oxigraph RocksDB bindings [RocksDB](http://rocksdb.org/) bindings for [Oxigraph](https://oxigraph.org). +By default it builds RocksDB as part of this crate. +It is also possible to dynamically link to RocksDB using the disabled by default `pkg-config` feature. +In this case [pkg-config](https://crates.io/crates/pkg-config) will be used to link to RocksDB. +Refer to this crate documentation if you want to configure the library lookup. + Based on [librocksdb-sys](https://crates.io/crates/librocksdb-sys) under Apache v2 license. diff --git a/oxrocksdb-sys/api/c.cc b/oxrocksdb-sys/api/c.cc index 49c5e55a..9a948ea2 100644 --- a/oxrocksdb-sys/api/c.cc +++ b/oxrocksdb-sys/api/c.cc @@ -1,7 +1,98 @@ -#include "../rocksdb/db/c.cc" - #include "c.h" +#include +#include +#include + +#include + +using ROCKSDB_NAMESPACE::Checkpoint; +using ROCKSDB_NAMESPACE::ColumnFamilyDescriptor; +using ROCKSDB_NAMESPACE::ColumnFamilyHandle; +using ROCKSDB_NAMESPACE::ColumnFamilyOptions; +using ROCKSDB_NAMESPACE::CompactRangeOptions; +using ROCKSDB_NAMESPACE::DB; +using ROCKSDB_NAMESPACE::DBOptions; +using ROCKSDB_NAMESPACE::FlushOptions; +using ROCKSDB_NAMESPACE::IngestExternalFileOptions; +using ROCKSDB_NAMESPACE::Iterator; +using ROCKSDB_NAMESPACE::Options; +using ROCKSDB_NAMESPACE::PinnableSlice; +using ROCKSDB_NAMESPACE::ReadOptions; +using ROCKSDB_NAMESPACE::Slice; +using ROCKSDB_NAMESPACE::SstFileWriter; +using ROCKSDB_NAMESPACE::Status; +using ROCKSDB_NAMESPACE::Transaction; +using ROCKSDB_NAMESPACE::TransactionDB; +using ROCKSDB_NAMESPACE::TransactionDBOptions; +using ROCKSDB_NAMESPACE::WriteOptions; +using std::vector; + +// From RocksDB +extern "C" { +struct rocksdb_t { + DB* rep; +}; + +struct rocksdb_column_family_handle_t { + ColumnFamilyHandle* rep; +}; + +struct rocksdb_compactoptions_t { + CompactRangeOptions rep; + Slice full_history_ts_low; +}; + +struct rocksdb_flushoptions_t { + FlushOptions rep; +}; + +struct rocksdb_ingestexternalfileoptions_t { + IngestExternalFileOptions rep; +}; + +struct rocksdb_iterator_t { + Iterator* rep; +}; + +struct rocksdb_options_t { + Options rep; +}; + +struct rocksdb_pinnableslice_t { + PinnableSlice rep; +}; + +struct rocksdb_readoptions_t { + ReadOptions rep; + // stack variables to set pointers to in ReadOptions + Slice upper_bound; + Slice lower_bound; + Slice timestamp; + Slice iter_start_ts; +}; + +struct rocksdb_sstfilewriter_t { + SstFileWriter* rep; +}; + +struct rocksdb_transaction_t { + Transaction* rep; +}; + +struct rocksdb_transactiondb_t { + TransactionDB* rep; +}; + +struct rocksdb_transactiondb_options_t { + TransactionDBOptions rep; +}; + +struct rocksdb_writeoptions_t { + WriteOptions rep; +}; +} + static bool SaveStatus(rocksdb_status_t* target, const Status source) { target->code = static_cast(source.code()); target->subcode = static_cast(source.subcode()); @@ -42,7 +133,7 @@ rocksdb_t* rocksdb_open_for_read_only_column_families_with_status( const rocksdb_options_t* const* column_family_options, rocksdb_column_family_handle_t** column_family_handles, unsigned char error_if_wal_file_exists, rocksdb_status_t* statusptr) { - std::vector column_families; + vector column_families; for (int i = 0; i < num_column_families; i++) { column_families.emplace_back(ColumnFamilyDescriptor( std::string(column_family_names[i]), @@ -50,7 +141,7 @@ rocksdb_t* rocksdb_open_for_read_only_column_families_with_status( } DB* db; - std::vector handles; + vector handles; if (SaveStatus(statusptr, DB::OpenForReadOnly(DBOptions(db_options->rep), std::string(name), column_families, &handles, &db, @@ -81,14 +172,14 @@ rocksdb_t* rocksdb_open_as_secondary_column_families_with_status( const rocksdb_options_t* const* column_family_options, rocksdb_column_family_handle_t** column_family_handles, rocksdb_status_t* statusptr) { - std::vector column_families; + vector column_families; for (int i = 0; i != num_column_families; ++i) { column_families.emplace_back( std::string(column_family_names[i]), ColumnFamilyOptions(column_family_options[i]->rep)); } DB* db; - std::vector handles; + vector handles; if (SaveStatus(statusptr, DB::OpenAsSecondary( DBOptions(db_options->rep), std::string(name), std::string(secondary_path), column_families, @@ -127,7 +218,7 @@ rocksdb_transactiondb_t* rocksdb_transactiondb_open_column_families_with_status( const rocksdb_options_t* const* column_family_options, rocksdb_column_family_handle_t** column_family_handles, rocksdb_status_t* statusptr) { - std::vector column_families; + vector column_families; for (int i = 0; i < num_column_families; i++) { column_families.emplace_back(ColumnFamilyDescriptor( std::string(column_family_names[i]), @@ -135,7 +226,7 @@ rocksdb_transactiondb_t* rocksdb_transactiondb_open_column_families_with_status( } TransactionDB* txn_db; - std::vector handles; + vector handles; if (SaveStatus(statusptr, TransactionDB::Open(options->rep, txn_db_options->rep, std::string(name), column_families, @@ -205,10 +296,10 @@ void rocksdb_transactiondb_compact_range_cf_opt_with_status( void rocksdb_transactiondb_ingest_external_files_with_status( rocksdb_transactiondb_t* db, const rocksdb_ingestexternalfilearg_t* list, const size_t list_len, rocksdb_status_t* statusptr) { - std::vector args(list_len); + vector args(list_len); for (size_t i = 0; i < list_len; ++i) { args[i].column_family = list[i].column_family->rep; - std::vector files(list[i].external_files_len); + vector files(list[i].external_files_len); for (size_t j = 0; j < list[i].external_files_len; ++j) { files[j] = std::string(list[i].external_files[j]); } diff --git a/oxrocksdb-sys/api/c.h b/oxrocksdb-sys/api/c.h index c0b2887d..022e8f97 100644 --- a/oxrocksdb-sys/api/c.h +++ b/oxrocksdb-sys/api/c.h @@ -1,6 +1,6 @@ #pragma once -#include "../rocksdb/include/rocksdb/c.h" +#include #ifdef __cplusplus extern "C" { diff --git a/oxrocksdb-sys/build.rs b/oxrocksdb-sys/build.rs index ff8a633b..abae57b5 100644 --- a/oxrocksdb-sys/build.rs +++ b/oxrocksdb-sys/build.rs @@ -1,9 +1,14 @@ // Code from https://github.com/rust-rocksdb/rust-rocksdb/blob/eb2d302682418b361a80ad8f4dcf335ade60dcf5/librocksdb-sys/build.rs // License: https://github.com/rust-rocksdb/rust-rocksdb/blob/master/LICENSE -use std::env::{remove_var, set_var, var}; +use std::env::var; +#[cfg(not(feature = "pkg-config"))] +use std::env::{remove_var, set_var}; +#[cfg(not(feature = "pkg-config"))] +use std::path::Path; use std::path::PathBuf; +#[cfg(not(feature = "pkg-config"))] fn link(name: &str, bundled: bool) { let target = var("TARGET").unwrap(); let target: Vec<_> = target.split('-').collect(); @@ -16,8 +21,14 @@ fn link(name: &str, bundled: bool) { } } -fn bindgen_rocksdb() { - bindgen::Builder::default() +fn bindgen_rocksdb_api(includes: &[PathBuf]) { + println!("cargo:rerun-if-changed=api/"); + + let mut builder = bindgen::Builder::default(); + for include in includes { + builder = builder.clang_arg(format!("-I{}", include.display())); + } + builder .header("api/c.h") .ctypes_prefix("libc") .size_t_is_usize(true) @@ -30,6 +41,22 @@ fn bindgen_rocksdb() { .unwrap(); } +fn build_rocksdb_api(includes: &[PathBuf]) { + let target = var("TARGET").unwrap(); + let mut config = cc::Build::new(); + for include in includes { + config.include(include); + } + if target.contains("msvc") { + config.flag("-EHsc").flag("-std:c++17"); + } else { + config.flag("-std=c++17"); + } + config.cpp(true).file("api/c.cc").compile("oxrocksdb_api"); +} + +#[cfg(not(feature = "pkg-config"))] + fn build_rocksdb() { let target = var("TARGET").unwrap(); @@ -38,7 +65,6 @@ fn build_rocksdb() { .cpp(true) .include("rocksdb/include/") .include("rocksdb/") - .file("api/c.cc") .file("api/build_version.cc") .define("NDEBUG", Some("1")) .define("LZ4", Some("1")) @@ -174,15 +200,15 @@ fn build_rocksdb() { } for file in lib_sources { - if file == "db/c.cc" || file == "util/build_version.cc" { - continue; + if file != "util/build_version.cc" { + config.file(&format!("rocksdb/{file}")); } - config.file(&format!("rocksdb/{file}")); } config.compile("rocksdb"); } +#[cfg(not(feature = "pkg-config"))] fn build_lz4() { let mut config = cc::Build::new(); config @@ -196,9 +222,21 @@ fn build_lz4() { config.compile("lz4"); } +#[cfg(not(feature = "pkg-config"))] fn main() { - println!("cargo:rerun-if-changed=api/"); - bindgen_rocksdb(); + let includes = [Path::new("rocksdb/include").to_path_buf()]; build_lz4(); build_rocksdb(); + build_rocksdb_api(&includes); + bindgen_rocksdb_api(&includes); +} + +#[cfg(feature = "pkg-config")] +fn main() { + let library = pkg_config::Config::new() + .atleast_version("8.0.0") + .probe("rocksdb") + .unwrap(); + build_rocksdb_api(&library.include_paths); + bindgen_rocksdb_api(&library.include_paths); } From 87d2006b6ee4d7eb8124f5fff2de9285f79d2979 Mon Sep 17 00:00:00 2001 From: Tpt Date: Mon, 11 Sep 2023 21:34:01 +0200 Subject: [PATCH 083/217] RocksDB: adds a feature for dynamic linking directly in the oxigraph crates --- .github/workflows/tests.yml | 4 +--- cli/Cargo.toml | 3 +++ lib/Cargo.toml | 1 + python/Cargo.toml | 1 + 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d570e796..c06b05d2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -46,8 +46,6 @@ jobs: working-directory: ./lib/sparopt - run: cargo clippy -- -D warnings -D clippy::all working-directory: ./lib - - run: cargo clippy --all-targets --all-features -- -D warnings -D clippy::all - working-directory: ./lib - run: cargo clippy -- -D warnings -D clippy::all working-directory: ./python - run: cargo clippy -- -D warnings -D clippy::all @@ -193,7 +191,7 @@ jobs: submodules: true - run: rustup update && rustup default 1.70.0 - uses: Swatinem/rust-cache@v2 - - run: cargo doc --all-features + - run: cargo doc working-directory: ./lib env: RUSTDOCFLAGS: -D warnings diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 486a46bf..ef9f42de 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -18,6 +18,9 @@ rust-version.workspace = true name = "oxigraph" path = "src/main.rs" +[features] +rocksdb-pkg-config = ["oxigraph/rocksdb-pkg-config"] + [dependencies] anyhow = "1.0.72" oxhttp = { version = "0.1.7", features = ["rayon"] } diff --git a/lib/Cargo.toml b/lib/Cargo.toml index 0ed6e9a9..f05e8a56 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -19,6 +19,7 @@ rust-version.workspace = true default = [] js = ["getrandom/js", "oxsdatatypes/js", "js-sys"] http_client = ["oxhttp", "oxhttp/rustls"] +rocksdb-pkg-config = ["oxrocksdb-sys/pkg-config"] rocksdb_debug = [] [dependencies] diff --git a/python/Cargo.toml b/python/Cargo.toml index a8f610e1..ce5df20a 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -19,6 +19,7 @@ doctest = false [features] abi3 = ["pyo3/abi3-py38"] +rocksdb-pkg-config = ["oxigraph/rocksdb-pkg-config"] [dependencies] oxigraph = { version = "0.4.0-alpha.1-dev", path="../lib", features = ["http_client"] } From 1b511ed018e4e85cd9ccd39eac6a936ec20ec42e Mon Sep 17 00:00:00 2001 From: Tpt Date: Mon, 11 Sep 2023 21:30:54 +0200 Subject: [PATCH 084/217] Python: guess file type from file extension --- python/src/io.rs | 67 +++++++++++++++++++++++++------------- python/src/store.rs | 53 +++++++++++++++--------------- python/tests/test_io.py | 8 ++--- python/tests/test_store.py | 25 +++++++------- 4 files changed, 88 insertions(+), 65 deletions(-) diff --git a/python/src/io.rs b/python/src/io.rs index 9bd957c6..97ca9b3d 100644 --- a/python/src/io.rs +++ b/python/src/io.rs @@ -9,6 +9,7 @@ use pyo3::types::PyBytes; use pyo3::{intern, wrap_pyfunction}; use std::cmp::max; use std::error::Error; +use std::ffi::OsStr; use std::fs::File; use std::io::{self, BufWriter, Cursor, Read, Write}; use std::path::{Path, PathBuf}; @@ -33,10 +34,10 @@ pub fn add_to_module(module: &PyModule) -> PyResult<()> { /// For example, ``application/turtle`` could also be used for `Turtle `_ /// and ``application/xml`` or ``xml`` for `RDF/XML `_. /// -/// :param input: The binary I/O object or file path to read from. For example, it could be a file path as a string or a file reader opened in binary mode with ``open('my_file.ttl', 'rb')``. +/// :param input: The I/O object or file path to read from. For example, it could be a file path as a string or a file reader opened in binary mode with ``open('my_file.ttl', 'rb')``. /// :type input: io(bytes) or io(str) or str or pathlib.Path -/// :param format: the format of the RDF serialization using a media type like ``text/turtle`` or an extension like `ttl`. -/// :type format: str +/// :param format: the format of the RDF serialization using a media type like ``text/turtle`` or an extension like `ttl`. If :py:const:`None`, the format is guessed from the file name extension. +/// :type format: str or None, optional /// :param base_iri: the base IRI used to resolve the relative IRIs in the file or :py:const:`None` if relative IRI resolution should not be done. /// :type base_iri: str or None, optional /// :param without_named_graphs: Sets that the parser must fail if parsing a named graph. @@ -52,21 +53,21 @@ pub fn add_to_module(module: &PyModule) -> PyResult<()> { /// >>> list(parse(input, "text/turtle", base_iri="http://example.com/")) /// [ predicate= object=> graph_name=>] #[pyfunction] -#[pyo3(signature = (input, format, *, base_iri = None, without_named_graphs = false, rename_blank_nodes = false))] +#[pyo3(signature = (input, /, format = None, *, base_iri = None, without_named_graphs = false, rename_blank_nodes = false))] pub fn parse( - input: PyObject, - format: &str, + input: &PyAny, + format: Option<&str>, base_iri: Option<&str>, without_named_graphs: bool, rename_blank_nodes: bool, py: Python<'_>, ) -> PyResult { - let format = rdf_format(format)?; - let file_path = input.extract::(py).ok(); + let file_path = input.extract::().ok(); + let format = rdf_format(format, file_path.as_deref())?; let input = if let Some(file_path) = &file_path { PyReadable::from_file(file_path, py).map_err(map_io_err)? } else { - PyReadable::from_data(input, py) + PyReadable::from_data(input) }; let mut parser = RdfParser::from_format(format); if let Some(base_iri) = base_iri { @@ -106,8 +107,8 @@ pub fn parse( /// :type input: iterable(Triple) or iterable(Quad) /// :param output: The binary I/O object or file path to write to. For example, it could be a file path as a string or a file writer opened in binary mode with ``open('my_file.ttl', 'wb')``. /// :type output: io(bytes) or str or pathlib.Path -/// :param format: the format of the RDF serialization using a media type like ``text/turtle`` or an extension like `ttl`. -/// :type format: str +/// :param format: the format of the RDF serialization using a media type like ``text/turtle`` or an extension like `ttl`. If :py:const:`None`, the format is guessed from the file name extension. +/// :type format: str or None, optional /// :rtype: None /// :raises ValueError: if the format is not supported. /// :raises TypeError: if a triple is given during a quad format serialization or reverse. @@ -117,10 +118,16 @@ pub fn parse( /// >>> output.getvalue() /// b' "1" .\n' #[pyfunction] -pub fn serialize(input: &PyAny, output: PyObject, format: &str, py: Python<'_>) -> PyResult<()> { - let format = rdf_format(format)?; - let output = if let Ok(path) = output.extract::(py) { - PyWritable::from_file(&path, py).map_err(map_io_err)? +pub fn serialize( + input: &PyAny, + output: &PyAny, + format: Option<&str>, + py: Python<'_>, +) -> PyResult<()> { + let file_path = output.extract::().ok(); + let format = rdf_format(format, file_path.as_deref())?; + let output = if let Some(file_path) = &file_path { + PyWritable::from_file(file_path, py).map_err(map_io_err)? } else { PyWritable::from_data(output) }; @@ -186,13 +193,13 @@ impl PyReadable { Ok(Self::File(py.allow_threads(|| File::open(file))?)) } - pub fn from_data(data: PyObject, py: Python<'_>) -> Self { - if let Ok(bytes) = data.extract::>(py) { + pub fn from_data(data: &PyAny) -> Self { + if let Ok(bytes) = data.extract::>() { Self::Bytes(Cursor::new(bytes)) - } else if let Ok(string) = data.extract::(py) { + } else if let Ok(string) = data.extract::() { Self::Bytes(Cursor::new(string.into_bytes())) } else { - Self::Io(PyIo(data)) + Self::Io(PyIo(data.into())) } } } @@ -217,8 +224,8 @@ impl PyWritable { Ok(Self::File(py.allow_threads(|| File::create(file))?)) } - pub fn from_data(data: PyObject) -> Self { - Self::Io(PyIo(data)) + pub fn from_data(data: &PyAny) -> Self { + Self::Io(PyIo(data.into())) } pub fn close(mut self) -> io::Result<()> { @@ -293,7 +300,23 @@ impl Write for PyIo { } } -pub fn rdf_format(format: &str) -> PyResult { +pub fn rdf_format(format: Option<&str>, path: Option<&Path>) -> PyResult { + let format = if let Some(format) = format { + format + } else if let Some(path) = path { + if let Some(ext) = path.extension().and_then(OsStr::to_str) { + ext + } else { + return Err(PyValueError::new_err(format!( + "The file name {} has no extension to guess a file format from", + path.display() + ))); + } + } else { + return Err(PyValueError::new_err( + "The format parameter is required when a file path is not given", + )); + }; if format.contains('/') { RdfFormat::from_media_type(format).ok_or_else(|| { PyValueError::new_err(format!("Not supported RDF format media type: {format}")) diff --git a/python/src/store.rs b/python/src/store.rs index 70dda4cd..42dbc533 100644 --- a/python/src/store.rs +++ b/python/src/store.rs @@ -360,10 +360,10 @@ impl PyStore { /// For example, ``application/turtle`` could also be used for `Turtle `_ /// and ``application/xml`` or ``xml`` for `RDF/XML `_. /// - /// :param input: The binary I/O object or file path to read from. For example, it could be a file path as a string or a file reader opened in binary mode with ``open('my_file.ttl', 'rb')``. + /// :param input: The I/O object or file path to read from. For example, it could be a file path as a string or a file reader opened in binary mode with ``open('my_file.ttl', 'rb')``. /// :type input: io(bytes) or io(str) or str or pathlib.Path - /// :param format: the format of the RDF serialization using a media type like ``text/turtle`` or an extension like `ttl`. - /// :type format: str + /// :param format: the format of the RDF serialization using a media type like ``text/turtle`` or an extension like `ttl`. If :py:const:`None`, the format is guessed from the file name extension. + /// :type format: str or None, optional /// :param base_iri: the base IRI used to resolve the relative IRIs in the file or :py:const:`None` if relative IRI resolution should not be done. /// :type base_iri: str or None, optional /// :param to_graph: if it is a file composed of triples, the graph in which the triples should be stored. By default, the default graph is used. @@ -377,26 +377,26 @@ impl PyStore { /// >>> store.load(io.BytesIO(b'

"1" .'), "text/turtle", base_iri="http://example.com/", to_graph=NamedNode("http://example.com/g")) /// >>> list(store) /// [ predicate= object=> graph_name=>] - #[pyo3(signature = (input, format, *, base_iri = None, to_graph = None))] + #[pyo3(signature = (input, /, format = None, *, base_iri = None, to_graph = None))] fn load( &self, - input: PyObject, - format: &str, + input: &PyAny, + format: Option<&str>, base_iri: Option<&str>, to_graph: Option<&PyAny>, py: Python<'_>, ) -> PyResult<()> { - let format = rdf_format(format)?; let to_graph_name = if let Some(graph_name) = to_graph { Some(GraphName::from(&PyGraphNameRef::try_from(graph_name)?)) } else { None }; - let file_path = input.extract::(py).ok(); + let file_path = input.extract::().ok(); + let format = rdf_format(format, file_path.as_deref())?; let input = if let Some(file_path) = &file_path { PyReadable::from_file(file_path, py).map_err(map_io_err)? } else { - PyReadable::from_data(input, py) + PyReadable::from_data(input) }; py.allow_threads(|| { if let Some(to_graph_name) = to_graph_name { @@ -429,10 +429,10 @@ impl PyStore { /// For example, ``application/turtle`` could also be used for `Turtle `_ /// and ``application/xml`` or ``xml`` for `RDF/XML `_. /// - /// :param input: The binary I/O object or file path to read from. For example, it could be a file path as a string or a file reader opened in binary mode with ``open('my_file.ttl', 'rb')``. + /// :param input: The I/O object or file path to read from. For example, it could be a file path as a string or a file reader opened in binary mode with ``open('my_file.ttl', 'rb')``. /// :type input: io(bytes) or io(str) or str or pathlib.Path - /// :param format: the format of the RDF serialization using a media type like ``text/turtle`` or an extension like `ttl`. - /// :type format: str + /// :param format: the format of the RDF serialization using a media type like ``text/turtle`` or an extension like `ttl`. If :py:const:`None`, the format is guessed from the file name extension. + /// :type format: str or None, optional /// :param base_iri: the base IRI used to resolve the relative IRIs in the file or :py:const:`None` if relative IRI resolution should not be done. /// :type base_iri: str or None, optional /// :param to_graph: if it is a file composed of triples, the graph in which the triples should be stored. By default, the default graph is used. @@ -446,26 +446,26 @@ impl PyStore { /// >>> store.bulk_load(io.BytesIO(b'

"1" .'), "text/turtle", base_iri="http://example.com/", to_graph=NamedNode("http://example.com/g")) /// >>> list(store) /// [ predicate= object=> graph_name=>] - #[pyo3(signature = (input, format, *, base_iri = None, to_graph = None))] + #[pyo3(signature = (input, /, format = None, *, base_iri = None, to_graph = None))] fn bulk_load( &self, - input: PyObject, - format: &str, + input: &PyAny, + format: Option<&str>, base_iri: Option<&str>, to_graph: Option<&PyAny>, py: Python<'_>, ) -> PyResult<()> { - let format = rdf_format(format)?; let to_graph_name = if let Some(graph_name) = to_graph { Some(GraphName::from(&PyGraphNameRef::try_from(graph_name)?)) } else { None }; - let file_path = input.extract::(py).ok(); + let file_path = input.extract::().ok(); + let format = rdf_format(format, file_path.as_deref())?; let input = if let Some(file_path) = &file_path { PyReadable::from_file(file_path, py).map_err(map_io_err)? } else { - PyReadable::from_data(input, py) + PyReadable::from_data(input) }; py.allow_threads(|| { if let Some(to_graph_name) = to_graph_name { @@ -498,8 +498,8 @@ impl PyStore { /// /// :param output: The binary I/O object or file path to write to. For example, it could be a file path as a string or a file writer opened in binary mode with ``open('my_file.ttl', 'wb')``. /// :type output: io(bytes) or str or pathlib.Path - /// :param format: the format of the RDF serialization using a media type like ``text/turtle`` or an extension like `ttl`. - /// :type format: str + /// :param format: the format of the RDF serialization using a media type like ``text/turtle`` or an extension like `ttl`. If :py:const:`None`, the format is guessed from the file name extension. + /// :type format: str or None, optional /// :param from_graph: the store graph from which dump the triples. Required if the serialization format does not support named graphs. If it does supports named graphs the full dataset is written. /// :type from_graph: NamedNode or BlankNode or DefaultGraph or None, optional /// :rtype: None @@ -512,22 +512,23 @@ impl PyStore { /// >>> store.dump(output, "text/turtle", from_graph=NamedNode("http://example.com/g")) /// >>> output.getvalue() /// b' "1" .\n' - #[pyo3(signature = (output, format, *, from_graph = None))] + #[pyo3(signature = (output, /, format = None, *, from_graph = None))] fn dump( &self, - output: PyObject, - format: &str, + output: &PyAny, + format: Option<&str>, from_graph: Option<&PyAny>, py: Python<'_>, ) -> PyResult<()> { - let format = rdf_format(format)?; let from_graph_name = if let Some(graph_name) = from_graph { Some(GraphName::from(&PyGraphNameRef::try_from(graph_name)?)) } else { None }; - let output = if let Ok(path) = output.extract::(py) { - PyWritable::from_file(&path, py).map_err(map_io_err)? + let file_path = output.extract::().ok(); + let format = rdf_format(format, file_path.as_deref())?; + let output = if let Some(file_path) = &file_path { + PyWritable::from_file(file_path, py).map_err(map_io_err)? } else { PyWritable::from_data(output) }; diff --git a/python/tests/test_io.py b/python/tests/test_io.py index d70179e7..006fc436 100644 --- a/python/tests/test_io.py +++ b/python/tests/test_io.py @@ -20,11 +20,11 @@ EXAMPLE_QUAD = Quad( class TestParse(unittest.TestCase): def test_parse_file(self) -> None: - with NamedTemporaryFile() as fp: + with NamedTemporaryFile(suffix=".ttl") as fp: fp.write('

"éù" .'.encode()) fp.flush() self.assertEqual( - list(parse(fp.name, "text/turtle", base_iri="http://example.com/")), + list(parse(fp.name, base_iri="http://example.com/")), [EXAMPLE_TRIPLE], ) @@ -138,8 +138,8 @@ class TestSerialize(unittest.TestCase): ) def test_serialize_to_file(self) -> None: - with NamedTemporaryFile() as fp: - serialize([EXAMPLE_TRIPLE], fp.name, "text/turtle") + with NamedTemporaryFile(suffix=".ttl") as fp: + serialize([EXAMPLE_TRIPLE], fp.name) self.assertEqual( fp.read().decode(), ' "éù" .\n', diff --git a/python/tests/test_store.py b/python/tests/test_store.py index a3ed8fe0..56f30b4b 100644 --- a/python/tests/test_store.py +++ b/python/tests/test_store.py @@ -265,13 +265,12 @@ class TestStore(unittest.TestCase): self.assertEqual(set(store), {Quad(foo, bar, baz, graph)}) def test_load_file(self) -> None: - with NamedTemporaryFile(delete=False) as fp: - file_name = Path(fp.name) + with NamedTemporaryFile(suffix=".nq") as fp: fp.write(b" .") - store = Store() - store.load(file_name, "nq") - file_name.unlink() - self.assertEqual(set(store), {Quad(foo, bar, baz, graph)}) + fp.flush() + store = Store() + store.load(fp.name) + self.assertEqual(set(store), {Quad(foo, bar, baz, graph)}) def test_load_with_io_error(self) -> None: with self.assertRaises(UnsupportedOperation) as _, TemporaryFile("wb") as fp: @@ -311,14 +310,14 @@ class TestStore(unittest.TestCase): def test_dump_file(self) -> None: with NamedTemporaryFile(delete=False) as fp: + store = Store() + store.add(Quad(foo, bar, baz, graph)) file_name = Path(fp.name) - store = Store() - store.add(Quad(foo, bar, baz, graph)) - store.dump(file_name, "nq") - self.assertEqual( - file_name.read_text(), - " .\n", - ) + store.dump(file_name, "nq") + self.assertEqual( + file_name.read_text(), + " .\n", + ) def test_dump_with_io_error(self) -> None: store = Store() From a8f98a00560166a9326442e3d4c1183433440247 Mon Sep 17 00:00:00 2001 From: Tpt Date: Tue, 12 Sep 2023 08:26:13 +0200 Subject: [PATCH 085/217] Python: makes serialization method output bytes if no output is specified --- python/src/io.rs | 56 ++++++++++++++++++++++++++------------ python/src/store.rs | 41 +++++++++++++++++----------- python/tests/test_io.py | 6 ++++ python/tests/test_store.py | 4 +-- 4 files changed, 70 insertions(+), 37 deletions(-) diff --git a/python/src/io.rs b/python/src/io.rs index 97ca9b3d..28e7d81b 100644 --- a/python/src/io.rs +++ b/python/src/io.rs @@ -105,31 +105,39 @@ pub fn parse( /// /// :param input: the RDF triples and quads to serialize. /// :type input: iterable(Triple) or iterable(Quad) -/// :param output: The binary I/O object or file path to write to. For example, it could be a file path as a string or a file writer opened in binary mode with ``open('my_file.ttl', 'wb')``. -/// :type output: io(bytes) or str or pathlib.Path +/// :param output: The binary I/O object or file path to write to. For example, it could be a file path as a string or a file writer opened in binary mode with ``open('my_file.ttl', 'wb')``. If :py:const:`None`, a :py:func:`bytes` buffer is returned with the serialized content. +/// :type output: io(bytes) or str or pathlib.Path or None, optional /// :param format: the format of the RDF serialization using a media type like ``text/turtle`` or an extension like `ttl`. If :py:const:`None`, the format is guessed from the file name extension. /// :type format: str or None, optional -/// :rtype: None +/// :rtype: bytes or None /// :raises ValueError: if the format is not supported. /// :raises TypeError: if a triple is given during a quad format serialization or reverse. /// +/// >>> serialize([Triple(NamedNode('http://example.com'), NamedNode('http://example.com/p'), Literal('1'))], format="ttl") +/// b' "1" .\n' +/// /// >>> output = io.BytesIO() /// >>> serialize([Triple(NamedNode('http://example.com'), NamedNode('http://example.com/p'), Literal('1'))], output, "text/turtle") /// >>> output.getvalue() /// b' "1" .\n' #[pyfunction] -pub fn serialize( +#[pyo3(signature = (input, output = None, /, format = None))] +pub fn serialize<'a>( input: &PyAny, - output: &PyAny, + output: Option<&PyAny>, format: Option<&str>, - py: Python<'_>, -) -> PyResult<()> { - let file_path = output.extract::().ok(); + py: Python<'a>, +) -> PyResult> { + let file_path = output.and_then(|output| output.extract::().ok()); let format = rdf_format(format, file_path.as_deref())?; - let output = if let Some(file_path) = &file_path { - PyWritable::from_file(file_path, py).map_err(map_io_err)? + let output = if let Some(output) = output { + if let Some(file_path) = &file_path { + PyWritable::from_file(file_path, py).map_err(map_io_err)? + } else { + PyWritable::from_data(output) + } } else { - PyWritable::from_data(output) + PyWritable::Bytes(Vec::new()) }; let mut writer = RdfSerializer::from_format(format).serialize_to_write(BufWriter::new(output)); for i in input.iter()? { @@ -153,8 +161,7 @@ pub fn serialize( .map_err(map_io_err)? .into_inner() .map_err(|e| map_io_err(e.into_error()))? - .close() - .map_err(map_io_err) + .close(py) } #[pyclass(name = "QuadReader", module = "pyoxigraph")] @@ -215,6 +222,7 @@ impl Read for PyReadable { } pub enum PyWritable { + Bytes(Vec), Io(PyIo), File(File), } @@ -228,18 +236,29 @@ impl PyWritable { Self::Io(PyIo(data.into())) } - pub fn close(mut self) -> io::Result<()> { - self.flush()?; - if let Self::File(file) = self { - file.sync_all()?; + pub fn close(self, py: Python<'_>) -> PyResult> { + match self { + Self::Bytes(bytes) => Ok(Some(PyBytes::new(py, &bytes))), + Self::File(mut file) => { + py.allow_threads(|| { + file.flush()?; + file.sync_all() + }) + .map_err(map_io_err)?; + Ok(None) + } + Self::Io(mut io) => { + py.allow_threads(|| io.flush()).map_err(map_io_err)?; + Ok(None) + } } - Ok(()) } } impl Write for PyWritable { fn write(&mut self, buf: &[u8]) -> io::Result { match self { + Self::Bytes(bytes) => bytes.write(buf), Self::Io(io) => io.write(buf), Self::File(file) => file.write(buf), } @@ -247,6 +266,7 @@ impl Write for PyWritable { fn flush(&mut self) -> io::Result<()> { match self { + Self::Bytes(_) => Ok(()), Self::Io(io) => io.flush(), Self::File(file) => file.flush(), } diff --git a/python/src/store.rs b/python/src/store.rs index 42dbc533..80a04709 100644 --- a/python/src/store.rs +++ b/python/src/store.rs @@ -10,6 +10,7 @@ use oxigraph::sparql::Update; use oxigraph::store::{self, LoaderError, SerializerError, StorageError, Store}; use pyo3::exceptions::{PyRuntimeError, PyValueError}; use pyo3::prelude::*; +use pyo3::types::PyBytes; use std::io::BufWriter; use std::path::PathBuf; @@ -496,41 +497,50 @@ impl PyStore { /// For example, ``application/turtle`` could also be used for `Turtle `_ /// and ``application/xml`` or ``xml`` for `RDF/XML `_. /// - /// :param output: The binary I/O object or file path to write to. For example, it could be a file path as a string or a file writer opened in binary mode with ``open('my_file.ttl', 'wb')``. - /// :type output: io(bytes) or str or pathlib.Path + /// :param output: The binary I/O object or file path to write to. For example, it could be a file path as a string or a file writer opened in binary mode with ``open('my_file.ttl', 'wb')``. If :py:const:`None`, a :py:func:`bytes` buffer is returned with the serialized content. + /// :type output: io(bytes) or str or pathlib.Path or None, optional /// :param format: the format of the RDF serialization using a media type like ``text/turtle`` or an extension like `ttl`. If :py:const:`None`, the format is guessed from the file name extension. /// :type format: str or None, optional /// :param from_graph: the store graph from which dump the triples. Required if the serialization format does not support named graphs. If it does supports named graphs the full dataset is written. /// :type from_graph: NamedNode or BlankNode or DefaultGraph or None, optional - /// :rtype: None + /// :rtype: bytes or None /// :raises ValueError: if the format is not supported or the `from_graph` parameter is not given with a syntax not supporting named graphs. /// :raises OSError: if an error happens during a quad lookup /// /// >>> store = Store() + /// >>> store.add(Quad(NamedNode('http://example.com'), NamedNode('http://example.com/p'), Literal('1'))) + /// >>> store.dump(format="trig") + /// b' "1" .\n' + /// + /// >>> store = Store() /// >>> store.add(Quad(NamedNode('http://example.com'), NamedNode('http://example.com/p'), Literal('1'), NamedNode('http://example.com/g'))) /// >>> output = io.BytesIO() /// >>> store.dump(output, "text/turtle", from_graph=NamedNode("http://example.com/g")) /// >>> output.getvalue() /// b' "1" .\n' - #[pyo3(signature = (output, /, format = None, *, from_graph = None))] - fn dump( + #[pyo3(signature = (output = None, /, format = None, *, from_graph = None))] + fn dump<'a>( &self, - output: &PyAny, + output: Option<&PyAny>, format: Option<&str>, from_graph: Option<&PyAny>, - py: Python<'_>, - ) -> PyResult<()> { + py: Python<'a>, + ) -> PyResult> { let from_graph_name = if let Some(graph_name) = from_graph { Some(GraphName::from(&PyGraphNameRef::try_from(graph_name)?)) } else { None }; - let file_path = output.extract::().ok(); + let file_path = output.and_then(|output| output.extract::().ok()); let format = rdf_format(format, file_path.as_deref())?; - let output = if let Some(file_path) = &file_path { - PyWritable::from_file(file_path, py).map_err(map_io_err)? + let output = if let Some(output) = output { + if let Some(file_path) = &file_path { + PyWritable::from_file(file_path, py).map_err(map_io_err)? + } else { + PyWritable::from_data(output) + } } else { - PyWritable::from_data(output) + PyWritable::Bytes(Vec::new()) }; py.allow_threads(|| { let output = BufWriter::new(output); @@ -541,10 +551,9 @@ impl PyStore { } .map_err(map_serializer_error)? .into_inner() - .map_err(|e| map_io_err(e.into_error()))? - .close() - .map_err(map_io_err) - }) + .map_err(|e| map_io_err(e.into_error())) + })? + .close(py) } /// Returns an iterator over all the store named graphs. diff --git a/python/tests/test_io.py b/python/tests/test_io.py index 006fc436..851b66b5 100644 --- a/python/tests/test_io.py +++ b/python/tests/test_io.py @@ -129,6 +129,12 @@ class TestParse(unittest.TestCase): class TestSerialize(unittest.TestCase): + def test_serialize_to_bytes(self) -> None: + self.assertEqual( + serialize([EXAMPLE_TRIPLE.triple], None, "text/turtle").decode(), + ' "éù" .\n', + ) + def test_serialize_to_bytes_io(self) -> None: output = BytesIO() serialize([EXAMPLE_TRIPLE.triple], output, "text/turtle") diff --git a/python/tests/test_store.py b/python/tests/test_store.py index 56f30b4b..4df54406 100644 --- a/python/tests/test_store.py +++ b/python/tests/test_store.py @@ -289,10 +289,8 @@ class TestStore(unittest.TestCase): def test_dump_nquads(self) -> None: store = Store() store.add(Quad(foo, bar, baz, graph)) - output = BytesIO() - store.dump(output, "nq") self.assertEqual( - output.getvalue(), + store.dump(format="nq"), b" .\n", ) From 180ae2229376f9f96692b0921334f90d4efa5d3e Mon Sep 17 00:00:00 2001 From: Tpt Date: Tue, 12 Sep 2023 09:00:13 +0200 Subject: [PATCH 086/217] Python: introduces QueryBoolean class --- python/src/lib.rs | 1 + python/src/sparql.rs | 34 +++++++++++++++++++++++++++++++++- python/src/store.rs | 6 +++--- 3 files changed, 37 insertions(+), 4 deletions(-) diff --git a/python/src/lib.rs b/python/src/lib.rs index bdc38681..474b0abc 100644 --- a/python/src/lib.rs +++ b/python/src/lib.rs @@ -32,6 +32,7 @@ fn pyoxigraph(_py: Python<'_>, module: &PyModule) -> PyResult<()> { module.add_class::()?; module.add_class::()?; module.add_class::()?; + module.add_class::()?; module.add_class::()?; io::add_to_module(module) } diff --git a/python/src/sparql.rs b/python/src/sparql.rs index c7ad693b..4310652a 100644 --- a/python/src/sparql.rs +++ b/python/src/sparql.rs @@ -65,7 +65,7 @@ pub fn query_results_to_python(py: Python<'_>, results: QueryResults) -> PyObjec match results { QueryResults::Solutions(inner) => PyQuerySolutions { inner }.into_py(py), QueryResults::Graph(inner) => PyQueryTriples { inner }.into_py(py), - QueryResults::Boolean(b) => b.into_py(py), + QueryResults::Boolean(inner) => PyQueryBoolean { inner }.into_py(py), } } @@ -204,6 +204,38 @@ impl PyQuerySolutions { } } +/// A boolean returned by a SPARQL ``ASK`` query. +/// +/// It can be easily casted to a regular boolean using the :py:func:`bool` function. +/// +/// >>> store = Store() +/// >>> store.add(Quad(NamedNode('http://example.com'), NamedNode('http://example.com/p'), Literal('1'))) +/// >>> bool(store.query('ASK { ?s ?p ?o }')) +/// True +#[pyclass(unsendable, name = "QueryBoolean", module = "pyoxigraph")] +pub struct PyQueryBoolean { + inner: bool, +} + +#[pymethods] +impl PyQueryBoolean { + fn __bool__(&self) -> bool { + self.inner + } + + fn __richcmp__(&self, other: &Self, op: CompareOp) -> bool { + op.matches(self.inner.cmp(&other.inner)) + } + + fn __hash__(&self) -> u64 { + self.inner.into() + } + + fn __repr__(&self) -> String { + format!("", self.inner) + } +} + /// An iterator of :py:class:`Triple` returned by a SPARQL ``CONSTRUCT`` or ``DESCRIBE`` query /// /// >>> store = Store() diff --git a/python/src/store.rs b/python/src/store.rs index 80a04709..6e5329de 100644 --- a/python/src/store.rs +++ b/python/src/store.rs @@ -251,8 +251,8 @@ impl PyStore { /// :type default_graph: NamedNode or BlankNode or DefaultGraph or list(NamedNode or BlankNode or DefaultGraph) or None, optional /// :param named_graphs: list of the named graphs that could be used in SPARQL `GRAPH` clause. By default, all the store named graphs are available. /// :type named_graphs: list(NamedNode or BlankNode) or None, optional - /// :return: a :py:class:`bool` for ``ASK`` queries, an iterator of :py:class:`Triple` for ``CONSTRUCT`` and ``DESCRIBE`` queries and an iterator of :py:class:`QuerySolution` for ``SELECT`` queries. - /// :rtype: QuerySolutions or QueryTriples or bool + /// :return: a :py:class:`QueryBoolean` for ``ASK`` queries, an iterator of :py:class:`Triple` for ``CONSTRUCT`` and ``DESCRIBE`` queries and an iterator of :py:class:`QuerySolution` for ``SELECT`` queries. + /// :rtype: QuerySolutions or QueryBoolean or QueryTriples /// :raises SyntaxError: if the provided query is invalid. /// :raises OSError: if an error happens while reading the store. /// @@ -274,7 +274,7 @@ impl PyStore { /// /// >>> store = Store() /// >>> store.add(Quad(NamedNode('http://example.com'), NamedNode('http://example.com/p'), Literal('1'))) - /// >>> store.query('ASK { ?s ?p ?o }') + /// >>> bool(store.query('ASK { ?s ?p ?o }')) /// True #[pyo3(signature = (query, *, base_iri = None, use_default_graph_as_union = false, default_graph = None, named_graphs = None))] fn query( From a6f32390df9ae44067fc3b0a7b7269cae635a83e Mon Sep 17 00:00:00 2001 From: Tpt Date: Tue, 12 Sep 2023 18:05:52 +0200 Subject: [PATCH 087/217] Python: query results I/O --- python/docs/sparql.rst | 11 +- python/src/io.rs | 132 ++++++++++------- python/src/lib.rs | 6 +- python/src/sparql.rs | 297 +++++++++++++++++++++++++++++++++---- python/src/store.rs | 53 +++---- python/tests/test_io.py | 47 +++++- python/tests/test_store.py | 33 +++++ 7 files changed, 469 insertions(+), 110 deletions(-) diff --git a/python/docs/sparql.rst b/python/docs/sparql.rst index 4c6a0479..1e87d179 100644 --- a/python/docs/sparql.rst +++ b/python/docs/sparql.rst @@ -9,7 +9,6 @@ Variable .. autoclass:: pyoxigraph.Variable :members: - ``SELECT`` solutions """""""""""""""""""" .. autoclass:: pyoxigraph.QuerySolutions @@ -17,7 +16,17 @@ Variable .. autoclass:: pyoxigraph.QuerySolution :members: +``ASK`` results +""""""""""""""" +.. autoclass:: pyoxigraph.QueryBoolean + :members: + ``CONSTRUCT`` results """"""""""""""""""""" .. autoclass:: pyoxigraph.QueryTriples :members: + +Query results parsing +""""""""""""""""""""" +.. autoclass:: pyoxigraph.parse_query_results + :members: diff --git a/python/src/io.rs b/python/src/io.rs index 28e7d81b..9a88ec9a 100644 --- a/python/src/io.rs +++ b/python/src/io.rs @@ -3,10 +3,11 @@ use crate::model::{PyQuad, PyTriple}; use oxigraph::io::{FromReadQuadReader, ParseError, RdfFormat, RdfParser, RdfSerializer}; use oxigraph::model::QuadRef; +use oxigraph::sparql::results::QueryResultsFormat; use pyo3::exceptions::{PySyntaxError, PyValueError}; +use pyo3::intern; use pyo3::prelude::*; use pyo3::types::PyBytes; -use pyo3::{intern, wrap_pyfunction}; use std::cmp::max; use std::error::Error; use std::ffi::OsStr; @@ -14,11 +15,6 @@ use std::fs::File; use std::io::{self, BufWriter, Cursor, Read, Write}; use std::path::{Path, PathBuf}; -pub fn add_to_module(module: &PyModule) -> PyResult<()> { - module.add_wrapped(wrap_pyfunction!(parse))?; - module.add_wrapped(wrap_pyfunction!(serialize)) -} - /// Parses RDF graph and dataset serialization formats. /// /// It currently supports the following formats: @@ -63,7 +59,7 @@ pub fn parse( py: Python<'_>, ) -> PyResult { let file_path = input.extract::().ok(); - let format = rdf_format(format, file_path.as_deref())?; + let format = parse_format(format, file_path.as_deref())?; let input = if let Some(file_path) = &file_path { PyReadable::from_file(file_path, py).map_err(map_io_err)? } else { @@ -128,40 +124,31 @@ pub fn serialize<'a>( format: Option<&str>, py: Python<'a>, ) -> PyResult> { - let file_path = output.and_then(|output| output.extract::().ok()); - let format = rdf_format(format, file_path.as_deref())?; - let output = if let Some(output) = output { - if let Some(file_path) = &file_path { - PyWritable::from_file(file_path, py).map_err(map_io_err)? - } else { - PyWritable::from_data(output) - } - } else { - PyWritable::Bytes(Vec::new()) - }; - let mut writer = RdfSerializer::from_format(format).serialize_to_write(BufWriter::new(output)); - for i in input.iter()? { - let i = i?; - if let Ok(triple) = i.extract::>() { - writer.write_triple(&*triple) - } else { - let quad = i.extract::>()?; - let quad = QuadRef::from(&*quad); - if !quad.graph_name.is_default_graph() && !format.supports_datasets() { - return Err(PyValueError::new_err( - "The {format} format does not support named graphs", - )); + PyWritable::do_write( + |output, format| { + let mut writer = RdfSerializer::from_format(format).serialize_to_write(output); + for i in input.iter()? { + let i = i?; + if let Ok(triple) = i.extract::>() { + writer.write_triple(&*triple) + } else { + let quad = i.extract::>()?; + let quad = QuadRef::from(&*quad); + if !quad.graph_name.is_default_graph() && !format.supports_datasets() { + return Err(PyValueError::new_err( + "The {format} format does not support named graphs", + )); + } + writer.write_quad(quad) + } + .map_err(map_io_err)?; } - writer.write_quad(quad) - } - .map_err(map_io_err)?; - } - writer - .finish() - .map_err(map_io_err)? - .into_inner() - .map_err(|e| map_io_err(e.into_error()))? - .close(py) + writer.finish().map_err(map_io_err) + }, + output, + format, + py, + ) } #[pyclass(name = "QuadReader", module = "pyoxigraph")] @@ -228,15 +215,33 @@ pub enum PyWritable { } impl PyWritable { - pub fn from_file(file: &Path, py: Python<'_>) -> io::Result { - Ok(Self::File(py.allow_threads(|| File::create(file))?)) - } - - pub fn from_data(data: &PyAny) -> Self { - Self::Io(PyIo(data.into())) + pub fn do_write<'a, F: Format>( + write: impl FnOnce(BufWriter, F) -> PyResult>, + output: Option<&PyAny>, + format: Option<&str>, + py: Python<'a>, + ) -> PyResult> { + let file_path = output.and_then(|output| output.extract::().ok()); + let format = parse_format::(format, file_path.as_deref())?; + let output = if let Some(output) = output { + if let Some(file_path) = &file_path { + Self::File( + py.allow_threads(|| File::create(file_path)) + .map_err(map_io_err)?, + ) + } else { + Self::Io(PyIo(output.into())) + } + } else { + PyWritable::Bytes(Vec::new()) + }; + let writer = write(BufWriter::new(output), format)?; + py.allow_threads(|| writer.into_inner()) + .map_err(|e| map_io_err(e.into_error()))? + .close(py) } - pub fn close(self, py: Python<'_>) -> PyResult> { + fn close(self, py: Python<'_>) -> PyResult> { match self { Self::Bytes(bytes) => Ok(Some(PyBytes::new(py, &bytes))), Self::File(mut file) => { @@ -320,7 +325,32 @@ impl Write for PyIo { } } -pub fn rdf_format(format: Option<&str>, path: Option<&Path>) -> PyResult { +pub trait Format: Sized { + fn from_media_type(media_type: &str) -> Option; + fn from_extension(extension: &str) -> Option; +} + +impl Format for RdfFormat { + fn from_media_type(media_type: &str) -> Option { + Self::from_media_type(media_type) + } + + fn from_extension(extension: &str) -> Option { + Self::from_extension(extension) + } +} + +impl Format for QueryResultsFormat { + fn from_media_type(media_type: &str) -> Option { + Self::from_media_type(media_type) + } + + fn from_extension(extension: &str) -> Option { + Self::from_extension(extension) + } +} + +pub fn parse_format(format: Option<&str>, path: Option<&Path>) -> PyResult { let format = if let Some(format) = format { format } else if let Some(path) = path { @@ -338,11 +368,11 @@ pub fn rdf_format(format: Option<&str>, path: Option<&Path>) -> PyResult) -> PyErr { /// /// Code from pyo3: https://github.com/PyO3/pyo3/blob/a67180c8a42a0bc0fdc45b651b62c0644130cf47/src/python.rs#L366 #[allow(unsafe_code)] -pub fn allow_threads_unsafe(f: impl FnOnce() -> T) -> T { +pub fn allow_threads_unsafe(_py: Python<'_>, f: impl FnOnce() -> T) -> T { struct RestoreGuard { tstate: *mut pyo3::ffi::PyThreadState, } diff --git a/python/src/lib.rs b/python/src/lib.rs index 474b0abc..d1b76022 100644 --- a/python/src/lib.rs +++ b/python/src/lib.rs @@ -10,6 +10,7 @@ mod model; mod sparql; mod store; +use crate::io::*; use crate::model::*; use crate::sparql::*; use crate::store::*; @@ -34,5 +35,8 @@ fn pyoxigraph(_py: Python<'_>, module: &PyModule) -> PyResult<()> { module.add_class::()?; module.add_class::()?; module.add_class::()?; - io::add_to_module(module) + module.add_wrapped(wrap_pyfunction!(parse))?; + module.add_wrapped(wrap_pyfunction!(parse_query_results))?; + module.add_wrapped(wrap_pyfunction!(serialize))?; + Ok(()) } diff --git a/python/src/sparql.rs b/python/src/sparql.rs index 4310652a..039c2628 100644 --- a/python/src/sparql.rs +++ b/python/src/sparql.rs @@ -1,14 +1,23 @@ -use crate::io::{allow_threads_unsafe, map_io_err, map_parse_error}; -use crate::map_storage_error; +use crate::io::*; use crate::model::*; +use crate::store::map_storage_error; +use oxigraph::io::RdfSerializer; use oxigraph::model::Term; -use oxigraph::sparql::*; +use oxigraph::sparql::results::{ + ParseError, QueryResultsParser, QueryResultsReader, QueryResultsSerializer, SolutionsReader, +}; +use oxigraph::sparql::{ + EvaluationError, Query, QueryResults, QuerySolution, QuerySolutionIter, QueryTripleIter, + Variable, +}; use pyo3::basic::CompareOp; use pyo3::exceptions::{ PyNotImplementedError, PyRuntimeError, PySyntaxError, PyTypeError, PyValueError, }; - use pyo3::prelude::*; +use pyo3::types::PyBytes; +use std::io::BufReader; +use std::path::PathBuf; use std::vec::IntoIter; pub fn parse_query( @@ -17,8 +26,9 @@ pub fn parse_query( use_default_graph_as_union: bool, default_graph: Option<&PyAny>, named_graphs: Option<&PyAny>, + py: Python<'_>, ) -> PyResult { - let mut query = allow_threads_unsafe(|| Query::parse(query, base_iri)) + let mut query = allow_threads_unsafe(py, || Query::parse(query, base_iri)) .map_err(|e| map_evaluation_error(e.into()))?; if use_default_graph_as_union && default_graph.is_some() { @@ -63,7 +73,10 @@ pub fn parse_query( pub fn query_results_to_python(py: Python<'_>, results: QueryResults) -> PyObject { match results { - QueryResults::Solutions(inner) => PyQuerySolutions { inner }.into_py(py), + QueryResults::Solutions(inner) => PyQuerySolutions { + inner: PyQuerySolutionsVariant::Query(inner), + } + .into_py(py), QueryResults::Graph(inner) => PyQueryTriples { inner }.into_py(py), QueryResults::Boolean(inner) => PyQueryBoolean { inner }.into_py(py), } @@ -172,7 +185,11 @@ impl SolutionValueIter { /// [>] #[pyclass(unsendable, name = "QuerySolutions", module = "pyoxigraph")] pub struct PyQuerySolutions { - inner: QuerySolutionIter, + inner: PyQuerySolutionsVariant, +} +enum PyQuerySolutionsVariant { + Query(QuerySolutionIter), + Reader(SolutionsReader>), } #[pymethods] @@ -185,22 +202,99 @@ impl PyQuerySolutions { /// [] #[getter] fn variables(&self) -> Vec { - self.inner - .variables() - .iter() - .map(|v| v.clone().into()) - .collect() + match &self.inner { + PyQuerySolutionsVariant::Query(inner) => { + inner.variables().iter().map(|v| v.clone().into()).collect() + } + PyQuerySolutionsVariant::Reader(inner) => { + inner.variables().iter().map(|v| v.clone().into()).collect() + } + } + } + + /// Writes the query results into a file. + /// + /// It currently supports the following formats: + /// + /// * `XML `_ (``application/sparql-results+xml`` or ``srx``) + /// * `JSON `_ (``application/sparql-results+json`` or ``srj``) + /// * `CSV `_ (``text/csv`` or ``csv``) + /// * `TSV `_ (``text/tab-separated-values`` or ``tsv``) + /// + /// It supports also some media type and extension aliases. + /// For example, ``application/json`` could also be used for `JSON `_. + /// + /// :param output: The binary I/O object or file path to write to. For example, it could be a file path as a string or a file writer opened in binary mode with ``open('my_file.ttl', 'wb')``. If :py:const:`None`, a :py:func:`bytes` buffer is returned with the serialized content. + /// :type output: io(bytes) or str or pathlib.Path or None, optional + /// :param format: the format of the query results serialization using a media type like ``text/csv`` or an extension like `csv`. If :py:const:`None`, the format is guessed from the file name extension. + /// :type format: str or None, optional + /// :rtype: bytes or None + /// :raises ValueError: if the format is not supported. + /// :raises OSError: if an error happens during a file writing. + /// + /// >>> store = Store() + /// >>> store.add(Quad(NamedNode('http://example.com'), NamedNode('http://example.com/p'), Literal('1'))) + /// >>> results = store.query("SELECT ?s ?p ?o WHERE { ?s ?p ?o }") + /// >>> results.serialize(format="json") + /// b'{"head":{"vars":["s","p","o"]},"results":{"bindings":[{"s":{"type":"uri","value":"http://example.com"},"p":{"type":"uri","value":"http://example.com/p"},"o":{"type":"literal","value":"1"}}]}}' + #[pyo3(signature = (output = None, /, format = None))] + fn serialize<'a>( + &mut self, + output: Option<&PyAny>, + format: Option<&str>, + py: Python<'a>, + ) -> PyResult> { + PyWritable::do_write( + |output, format| { + let mut writer = QueryResultsSerializer::from_format(format) + .solutions_writer( + output, + match &self.inner { + PyQuerySolutionsVariant::Query(inner) => inner.variables().to_vec(), + PyQuerySolutionsVariant::Reader(inner) => inner.variables().to_vec(), + }, + ) + .map_err(map_io_err)?; + match &mut self.inner { + PyQuerySolutionsVariant::Query(inner) => { + for solution in inner { + writer + .write(&solution.map_err(map_evaluation_error)?) + .map_err(map_io_err)?; + } + } + PyQuerySolutionsVariant::Reader(inner) => { + for solution in inner { + writer + .write(&solution.map_err(map_query_results_parse_error)?) + .map_err(map_io_err)?; + } + } + } + + writer.finish().map_err(map_io_err) + }, + output, + format, + py, + ) } fn __iter__(slf: PyRef<'_, Self>) -> PyRef { slf } - fn __next__(&mut self) -> PyResult> { - Ok(allow_threads_unsafe(|| self.inner.next()) - .transpose() - .map_err(map_evaluation_error)? - .map(move |inner| PyQuerySolution { inner })) + fn __next__(&mut self, py: Python<'_>) -> PyResult> { + Ok(match &mut self.inner { + PyQuerySolutionsVariant::Query(inner) => allow_threads_unsafe(py, || { + inner.next().transpose().map_err(map_evaluation_error) + }), + PyQuerySolutionsVariant::Reader(inner) => inner + .next() + .transpose() + .map_err(map_query_results_parse_error), + }? + .map(move |inner| PyQuerySolution { inner })) } } @@ -219,6 +313,52 @@ pub struct PyQueryBoolean { #[pymethods] impl PyQueryBoolean { + /// Writes the query results into a file. + /// + /// It currently supports the following formats: + /// + /// * `XML `_ (``application/sparql-results+xml`` or ``srx``) + /// * `JSON `_ (``application/sparql-results+json`` or ``srj``) + /// * `CSV `_ (``text/csv`` or ``csv``) + /// * `TSV `_ (``text/tab-separated-values`` or ``tsv``) + /// + /// It supports also some media type and extension aliases. + /// For example, ``application/json`` could also be used for `JSON `_. + /// + /// :param output: The binary I/O object or file path to write to. For example, it could be a file path as a string or a file writer opened in binary mode with ``open('my_file.ttl', 'wb')``. If :py:const:`None`, a :py:func:`bytes` buffer is returned with the serialized content. + /// :type output: io(bytes) or str or pathlib.Path or None, optional + /// :param format: the format of the query results serialization using a media type like ``text/csv`` or an extension like `csv`. If :py:const:`None`, the format is guessed from the file name extension. + /// :type format: str or None, optional + /// :rtype: bytes or None + /// :raises ValueError: if the format is not supported. + /// :raises OSError: if an error happens during a file writing. + /// + /// >>> store = Store() + /// >>> store.add(Quad(NamedNode('http://example.com'), NamedNode('http://example.com/p'), Literal('1'))) + /// >>> results = store.query("ASK { ?s ?p ?o }") + /// >>> results.serialize(format="json") + /// b'{"head":{},"boolean":true}' + #[pyo3(signature = (output = None, /, format = None))] + fn serialize<'a>( + &mut self, + output: Option<&PyAny>, + format: Option<&str>, + py: Python<'a>, + ) -> PyResult> { + PyWritable::do_write( + |output, format| { + py.allow_threads(|| { + QueryResultsSerializer::from_format(format) + .write_boolean_result(output, self.inner) + .map_err(map_io_err) + }) + }, + output, + format, + py, + ) + } + fn __bool__(&self) -> bool { self.inner } @@ -249,29 +389,129 @@ pub struct PyQueryTriples { #[pymethods] impl PyQueryTriples { + /// Writes the query results into a file. + /// + /// It currently supports the following formats: + /// + /// * `N-Triples `_ (``application/n-triples`` or ``nt``) + /// * `N-Quads `_ (``application/n-quads`` or ``nq``) + /// * `Turtle `_ (``text/turtle`` or ``ttl``) + /// * `TriG `_ (``application/trig`` or ``trig``) + /// * `N3 `_ (``text/n3`` or ``n3``) + /// * `RDF/XML `_ (``application/rdf+xml`` or ``rdf``) + /// + /// It supports also some media type and extension aliases. + /// For example, ``application/turtle`` could also be used for `Turtle `_ + /// and ``application/xml`` or ``xml`` for `RDF/XML `_. + /// + /// :param output: The binary I/O object or file path to write to. For example, it could be a file path as a string or a file writer opened in binary mode with ``open('my_file.ttl', 'wb')``. If :py:const:`None`, a :py:func:`bytes` buffer is returned with the serialized content. + /// :type output: io(bytes) or str or pathlib.Path or None, optional + /// :param format: the format of the RDF serialization using a media type like ``text/turtle`` or an extension like `ttl`. If :py:const:`None`, the format is guessed from the file name extension. + /// :type format: str or None, optional + /// :rtype: bytes or None + /// :raises ValueError: if the format is not supported. + /// :raises OSError: if an error happens during a file writing. + /// + /// >>> store = Store() + /// >>> store.add(Quad(NamedNode('http://example.com'), NamedNode('http://example.com/p'), Literal('1'))) + /// >>> results = store.query("CONSTRUCT { ?s ?p ?o } WHERE { ?s ?p ?o }") + /// >>> results.serialize(format="nt") + /// b' "1" .\n' + #[pyo3(signature = (output = None, /, format = None))] + fn serialize<'a>( + &mut self, + output: Option<&PyAny>, + format: Option<&str>, + py: Python<'a>, + ) -> PyResult> { + PyWritable::do_write( + |output, format| { + let mut writer = RdfSerializer::from_format(format).serialize_to_write(output); + for triple in &mut self.inner { + writer + .write_triple(&triple.map_err(map_evaluation_error)?) + .map_err(map_io_err)?; + } + writer.finish().map_err(map_io_err) + }, + output, + format, + py, + ) + } + fn __iter__(slf: PyRef<'_, Self>) -> PyRef { slf } - fn __next__(&mut self) -> PyResult> { - Ok(allow_threads_unsafe(|| self.inner.next()) + fn __next__(&mut self, py: Python<'_>) -> PyResult> { + Ok(allow_threads_unsafe(py, || self.inner.next()) .transpose() .map_err(map_evaluation_error)? .map(Into::into)) } } +/// Parses SPARQL query results. +/// +/// It currently supports the following formats: +/// +/// * `XML `_ (``application/sparql-results+xml`` or ``srx``) +/// * `JSON `_ (``application/sparql-results+json`` or ``srj``) +/// * `CSV `_ (``text/csv`` or ``csv``) +/// * `TSV `_ (``text/tab-separated-values`` or ``tsv``) +/// +/// It supports also some media type and extension aliases. +/// For example, ``application/json`` could also be used for `JSON `_. +/// +/// :param input: The I/O object or file path to read from. For example, it could be a file path as a string or a file reader opened in binary mode with ``open('my_file.ttl', 'rb')``. +/// :type input: io(bytes) or io(str) or str or pathlib.Path +/// :param format: the format of the RDF serialization using a media type like ``text/turtle`` or an extension like `ttl`. If :py:const:`None`, the format is guessed from the file name extension. +/// :type format: str or None, optional +/// :return: an iterator of :py:class:`QuerySolution` or a :py:func:`bool`. +/// :rtype: QuerySolutions or QueryBoolean +/// :raises ValueError: if the format is not supported. +/// :raises SyntaxError: if the provided data is invalid. +/// +/// >>> input = io.BytesIO(b'?s\t?p\t?o\n\t\t1\n') +/// >>> list(parse_query_results(input, "text/tsv")) +/// [ p= o=>>] +/// +/// >>> input = io.BytesIO(b'{"head":{},"boolean":true}') +/// >>> parse_query_results(input, "application/sparql-results+json") +/// +#[pyfunction] +#[pyo3(signature = (input, /, format = None))] +pub fn parse_query_results( + input: &PyAny, + format: Option<&str>, + py: Python<'_>, +) -> PyResult { + let file_path = input.extract::().ok(); + let format = parse_format(format, file_path.as_deref())?; + let input = if let Some(file_path) = &file_path { + PyReadable::from_file(file_path, py).map_err(map_io_err)? + } else { + PyReadable::from_data(input) + }; + let results = QueryResultsParser::from_format(format) + .read_results(BufReader::new(input)) + .map_err(map_query_results_parse_error)?; + Ok(match results { + QueryResultsReader::Solutions(inner) => PyQuerySolutions { + inner: PyQuerySolutionsVariant::Reader(inner), + } + .into_py(py), + QueryResultsReader::Boolean(inner) => PyQueryBoolean { inner }.into_py(py), + }) +} + pub fn map_evaluation_error(error: EvaluationError) -> PyErr { match error { EvaluationError::Parsing(error) => PySyntaxError::new_err(error.to_string()), EvaluationError::Storage(error) => map_storage_error(error), EvaluationError::GraphParsing(error) => map_parse_error(error, None), - EvaluationError::ResultsParsing(error) => match error { - oxigraph::sparql::results::ParseError::Syntax(error) => { - PySyntaxError::new_err(error.to_string()) - } - oxigraph::sparql::results::ParseError::Io(error) => map_io_err(error), - }, + EvaluationError::ResultsParsing(error) => map_query_results_parse_error(error), EvaluationError::ResultsSerialization(error) => map_io_err(error), EvaluationError::Service(error) => match error.downcast() { Ok(error) => map_io_err(*error), @@ -280,3 +520,10 @@ pub fn map_evaluation_error(error: EvaluationError) -> PyErr { _ => PyRuntimeError::new_err(error.to_string()), } } + +pub fn map_query_results_parse_error(error: ParseError) -> PyErr { + match error { + ParseError::Syntax(error) => PySyntaxError::new_err(error.to_string()), + ParseError::Io(error) => map_io_err(error), + } +} diff --git a/python/src/store.rs b/python/src/store.rs index 6e5329de..874e48aa 100644 --- a/python/src/store.rs +++ b/python/src/store.rs @@ -1,17 +1,17 @@ #![allow(clippy::needless_option_as_deref)] use crate::io::{ - allow_threads_unsafe, map_io_err, map_parse_error, rdf_format, PyReadable, PyWritable, + allow_threads_unsafe, map_io_err, map_parse_error, parse_format, PyReadable, PyWritable, }; use crate::model::*; use crate::sparql::*; +use oxigraph::io::RdfFormat; use oxigraph::model::{GraphName, GraphNameRef}; use oxigraph::sparql::Update; use oxigraph::store::{self, LoaderError, SerializerError, StorageError, Store}; use pyo3::exceptions::{PyRuntimeError, PyValueError}; use pyo3::prelude::*; use pyo3::types::PyBytes; -use std::io::BufWriter; use std::path::PathBuf; /// RDF store. @@ -251,7 +251,7 @@ impl PyStore { /// :type default_graph: NamedNode or BlankNode or DefaultGraph or list(NamedNode or BlankNode or DefaultGraph) or None, optional /// :param named_graphs: list of the named graphs that could be used in SPARQL `GRAPH` clause. By default, all the store named graphs are available. /// :type named_graphs: list(NamedNode or BlankNode) or None, optional - /// :return: a :py:class:`QueryBoolean` for ``ASK`` queries, an iterator of :py:class:`Triple` for ``CONSTRUCT`` and ``DESCRIBE`` queries and an iterator of :py:class:`QuerySolution` for ``SELECT`` queries. + /// :return: a :py:func:`bool` for ``ASK`` queries, an iterator of :py:class:`Triple` for ``CONSTRUCT`` and ``DESCRIBE`` queries and an iterator of :py:class:`QuerySolution` for ``SELECT`` queries. /// :rtype: QuerySolutions or QueryBoolean or QueryTriples /// :raises SyntaxError: if the provided query is invalid. /// :raises OSError: if an error happens while reading the store. @@ -292,9 +292,10 @@ impl PyStore { use_default_graph_as_union, default_graph, named_graphs, + py, )?; let results = - allow_threads_unsafe(|| self.inner.query(query)).map_err(map_evaluation_error)?; + allow_threads_unsafe(py, || self.inner.query(query)).map_err(map_evaluation_error)?; Ok(query_results_to_python(py, results)) } @@ -393,7 +394,7 @@ impl PyStore { None }; let file_path = input.extract::().ok(); - let format = rdf_format(format, file_path.as_deref())?; + let format = parse_format::(format, file_path.as_deref())?; let input = if let Some(file_path) = &file_path { PyReadable::from_file(file_path, py).map_err(map_io_err)? } else { @@ -462,7 +463,7 @@ impl PyStore { None }; let file_path = input.extract::().ok(); - let format = rdf_format(format, file_path.as_deref())?; + let format = parse_format::(format, file_path.as_deref())?; let input = if let Some(file_path) = &file_path { PyReadable::from_file(file_path, py).map_err(map_io_err)? } else { @@ -505,7 +506,7 @@ impl PyStore { /// :type from_graph: NamedNode or BlankNode or DefaultGraph or None, optional /// :rtype: bytes or None /// :raises ValueError: if the format is not supported or the `from_graph` parameter is not given with a syntax not supporting named graphs. - /// :raises OSError: if an error happens during a quad lookup + /// :raises OSError: if an error happens during a quad lookup or file writing. /// /// >>> store = Store() /// >>> store.add(Quad(NamedNode('http://example.com'), NamedNode('http://example.com/p'), Literal('1'))) @@ -531,29 +532,21 @@ impl PyStore { } else { None }; - let file_path = output.and_then(|output| output.extract::().ok()); - let format = rdf_format(format, file_path.as_deref())?; - let output = if let Some(output) = output { - if let Some(file_path) = &file_path { - PyWritable::from_file(file_path, py).map_err(map_io_err)? - } else { - PyWritable::from_data(output) - } - } else { - PyWritable::Bytes(Vec::new()) - }; - py.allow_threads(|| { - let output = BufWriter::new(output); - if let Some(from_graph_name) = &from_graph_name { - self.inner.dump_graph(output, format, from_graph_name) - } else { - self.inner.dump_dataset(output, format) - } - .map_err(map_serializer_error)? - .into_inner() - .map_err(|e| map_io_err(e.into_error())) - })? - .close(py) + PyWritable::do_write::( + |output, format| { + py.allow_threads(|| { + if let Some(from_graph_name) = &from_graph_name { + self.inner.dump_graph(output, format, from_graph_name) + } else { + self.inner.dump_dataset(output, format) + } + .map_err(map_serializer_error) + }) + }, + output, + format, + py, + ) } /// Returns an iterator over all the store named graphs. diff --git a/python/tests/test_io.py b/python/tests/test_io.py index 851b66b5..9d4a1326 100644 --- a/python/tests/test_io.py +++ b/python/tests/test_io.py @@ -3,7 +3,16 @@ import unittest from io import BytesIO, StringIO, UnsupportedOperation from tempfile import NamedTemporaryFile, TemporaryFile -from pyoxigraph import Literal, NamedNode, Quad, parse, serialize +from pyoxigraph import ( + Literal, + NamedNode, + Quad, + QueryBoolean, + QuerySolutions, + parse, + parse_query_results, + serialize, +) EXAMPLE_TRIPLE = Quad( NamedNode("http://example.com/foo"), @@ -131,7 +140,7 @@ class TestParse(unittest.TestCase): class TestSerialize(unittest.TestCase): def test_serialize_to_bytes(self) -> None: self.assertEqual( - serialize([EXAMPLE_TRIPLE.triple], None, "text/turtle").decode(), + (serialize([EXAMPLE_TRIPLE.triple], None, "text/turtle") or b"").decode(), ' "éù" .\n', ) @@ -162,3 +171,37 @@ class TestSerialize(unittest.TestCase): output.getvalue(), b' {\n\t "1" .\n}\n', ) + + +class TestParseQuerySolutions(unittest.TestCase): + def test_parse_file(self) -> None: + with NamedTemporaryFile(suffix=".tsv") as fp: + fp.write( + b'?s\t?p\t?o\n\t\t"1"\n' + ) + fp.flush() + r = parse_query_results(fp.name) + self.assertIsInstance(r, QuerySolutions) + results = list(r) # type: ignore[arg-type] + self.assertEqual(results[0]["s"], NamedNode("http://example.com/s")) + self.assertEqual(results[0][2], Literal("1")) + + def test_parse_not_existing_file(self) -> None: + with self.assertRaises(IOError) as _: + parse_query_results( + "/tmp/not-existing-oxigraph-file.ttl", "application/json" + ) + + def test_parse_str_io(self) -> None: + result = parse_query_results(StringIO("true"), "tsv") + self.assertIsInstance(result, QueryBoolean) + self.assertTrue(result) + + def test_parse_bytes_io(self) -> None: + result = parse_query_results(BytesIO(b"false"), "tsv") + self.assertIsInstance(result, QueryBoolean) + self.assertFalse(result) + + def test_parse_io_error(self) -> None: + with self.assertRaises(UnsupportedOperation) as _, TemporaryFile("wb") as fp: + parse_query_results(fp, "srx") diff --git a/python/tests/test_store.py b/python/tests/test_store.py index 4df54406..84107d84 100644 --- a/python/tests/test_store.py +++ b/python/tests/test_store.py @@ -189,6 +189,39 @@ class TestStore(unittest.TestCase): ) self.assertEqual(len(list(results)), 2) + def test_select_query_dump(self) -> None: + store = Store() + store.add(Quad(foo, bar, baz)) + results = store.query("SELECT ?s WHERE { ?s ?p ?o }") + output = BytesIO() + results.serialize(output, "csv") + self.assertEqual( + output.getvalue().decode(), + "s\r\nhttp://foo\r\n", + ) + + def test_ask_query_dump(self) -> None: + store = Store() + store.add(Quad(foo, bar, baz)) + results = store.query("ASK { ?s ?p ?o }") + output = BytesIO() + results.serialize(output, "csv") + self.assertEqual( + output.getvalue().decode(), + "true", + ) + + def test_construct_query_dump(self) -> None: + store = Store() + store.add(Quad(foo, bar, baz)) + results = store.query("CONSTRUCT WHERE { ?s ?p ?o }") + output = BytesIO() + results.serialize(output, "nt") + self.assertEqual( + output.getvalue().decode(), + " .\n", + ) + def test_update_insert_data(self) -> None: store = Store() store.update("INSERT DATA { }") From 4c97637e4b8abc651f545cd5e8b80ebeba0f58c4 Mon Sep 17 00:00:00 2001 From: Tpt Date: Tue, 12 Sep 2023 21:34:12 +0200 Subject: [PATCH 088/217] Python: improves documentation --- python/src/io.rs | 9 ++++++--- python/src/model.rs | 18 +++++++++--------- python/src/sparql.rs | 15 ++++++++------- python/src/store.rs | 11 ++++++----- 4 files changed, 29 insertions(+), 24 deletions(-) diff --git a/python/src/io.rs b/python/src/io.rs index 9a88ec9a..be494e22 100644 --- a/python/src/io.rs +++ b/python/src/io.rs @@ -36,14 +36,15 @@ use std::path::{Path, PathBuf}; /// :type format: str or None, optional /// :param base_iri: the base IRI used to resolve the relative IRIs in the file or :py:const:`None` if relative IRI resolution should not be done. /// :type base_iri: str or None, optional -/// :param without_named_graphs: Sets that the parser must fail if parsing a named graph. +/// :param without_named_graphs: Sets that the parser must fail when parsing a named graph. /// :type without_named_graphs: bool, optional -/// :param rename_blank_nodes: Renames the blank nodes ids from the ones set in the serialization to random ids. This allows to avoid id conflicts when merging graphs together. +/// :param rename_blank_nodes: Renames the blank nodes identifiers from the ones set in the serialization to random ids. This allows to avoid identifier conflicts when merging graphs together. /// :type rename_blank_nodes: bool, optional /// :return: an iterator of RDF triples or quads depending on the format. /// :rtype: iterator(Quad) /// :raises ValueError: if the format is not supported. /// :raises SyntaxError: if the provided data is invalid. +/// :raises OSError: if a system error happens while reading the file. /// /// >>> input = io.BytesIO(b'

"1" .') /// >>> list(parse(input, "text/turtle", base_iri="http://example.com/")) @@ -101,13 +102,15 @@ pub fn parse( /// /// :param input: the RDF triples and quads to serialize. /// :type input: iterable(Triple) or iterable(Quad) -/// :param output: The binary I/O object or file path to write to. For example, it could be a file path as a string or a file writer opened in binary mode with ``open('my_file.ttl', 'wb')``. If :py:const:`None`, a :py:func:`bytes` buffer is returned with the serialized content. +/// :param output: The binary I/O object or file path to write to. For example, it could be a file path as a string or a file writer opened in binary mode with ``open('my_file.ttl', 'wb')``. If :py:const:`None`, a :py:class:`bytes` buffer is returned with the serialized content. /// :type output: io(bytes) or str or pathlib.Path or None, optional /// :param format: the format of the RDF serialization using a media type like ``text/turtle`` or an extension like `ttl`. If :py:const:`None`, the format is guessed from the file name extension. /// :type format: str or None, optional +/// :return: py:class:`bytes` with the serialization if the ``output`` parameter is :py:const:`None`, :py:const:`None` if ``output`` is set. /// :rtype: bytes or None /// :raises ValueError: if the format is not supported. /// :raises TypeError: if a triple is given during a quad format serialization or reverse. +/// :raises OSError: if a system error happens while writing the file. /// /// >>> serialize([Triple(NamedNode('http://example.com'), NamedNode('http://example.com/p'), Literal('1'))], format="ttl") /// b' "1" .\n' diff --git a/python/src/model.rs b/python/src/model.rs index e2694c61..61f327a4 100644 --- a/python/src/model.rs +++ b/python/src/model.rs @@ -16,7 +16,7 @@ use std::vec::IntoIter; /// :type value: str /// :raises ValueError: if the IRI is not valid according to `RFC 3987 `_. /// -/// The :py:func:`str` function provides a serialization compatible with NTriples, Turtle, and SPARQL: +/// The :py:class:`str` function provides a serialization compatible with NTriples, Turtle, and SPARQL: /// /// >>> str(NamedNode('http://example.com')) /// '' @@ -135,11 +135,11 @@ impl PyNamedNode { /// An RDF `blank node `_. /// -/// :param value: the `blank node ID `_ (if not present, a random blank node ID is automatically generated). +/// :param value: the `blank node identifier `_ (if not present, a random blank node identifier is automatically generated). /// :type value: str or None, optional -/// :raises ValueError: if the blank node ID is invalid according to NTriples, Turtle, and SPARQL grammars. +/// :raises ValueError: if the blank node identifier is invalid according to NTriples, Turtle, and SPARQL grammars. /// -/// The :py:func:`str` function provides a serialization compatible with NTriples, Turtle, and SPARQL: +/// The :py:class:`str` function provides a serialization compatible with NTriples, Turtle, and SPARQL: /// /// >>> str(BlankNode('ex')) /// '_:ex' @@ -198,7 +198,7 @@ impl PyBlankNode { .into()) } - /// :return: the `blank node ID `_. + /// :return: the `blank node identifier `_. /// :rtype: str /// /// >>> BlankNode("ex").value @@ -270,7 +270,7 @@ impl PyBlankNode { /// :type language: str or None, optional /// :raises ValueError: if the language tag is not valid according to `RFC 5646 `_ (`BCP 47 `_). /// -/// The :py:func:`str` function provides a serialization compatible with NTriples, Turtle, and SPARQL: +/// The :py:class:`str` function provides a serialization compatible with NTriples, Turtle, and SPARQL: /// /// >>> str(Literal('example')) /// '"example"' @@ -606,7 +606,7 @@ impl IntoPy for PyTerm { /// :param object: the triple object. /// :type object: NamedNode or BlankNode or Literal or Triple /// -/// The :py:func:`str` function provides a serialization compatible with NTriples, Turtle, and SPARQL: +/// The :py:class:`str` function provides a serialization compatible with NTriples, Turtle, and SPARQL: /// /// >>> str(Triple(NamedNode('http://example.com'), NamedNode('http://example.com/p'), Literal('1'))) /// ' "1"' @@ -801,7 +801,7 @@ impl IntoPy for PyGraphName { /// :param graph_name: the quad graph name. If not present, the default graph is assumed. /// :type graph_name: NamedNode or BlankNode or DefaultGraph or None, optional /// -/// The :py:func:`str` function provides a serialization compatible with NTriples, Turtle, and SPARQL: +/// The :py:class:`str` function provides a serialization compatible with NTriples, Turtle, and SPARQL: /// /// >>> str(Quad(NamedNode('http://example.com'), NamedNode('http://example.com/p'), Literal('1'), NamedNode('http://example.com/g'))) /// ' "1" ' @@ -995,7 +995,7 @@ impl PyQuad { /// :type value: str /// :raises ValueError: if the variable name is invalid according to the SPARQL grammar. /// -/// The :py:func:`str` function provides a serialization compatible with SPARQL: +/// The :py:class:`str` function provides a serialization compatible with SPARQL: /// /// >>> str(Variable('foo')) /// '?foo' diff --git a/python/src/sparql.rs b/python/src/sparql.rs index 039c2628..ef544e2d 100644 --- a/python/src/sparql.rs +++ b/python/src/sparql.rs @@ -224,13 +224,13 @@ impl PyQuerySolutions { /// It supports also some media type and extension aliases. /// For example, ``application/json`` could also be used for `JSON `_. /// - /// :param output: The binary I/O object or file path to write to. For example, it could be a file path as a string or a file writer opened in binary mode with ``open('my_file.ttl', 'wb')``. If :py:const:`None`, a :py:func:`bytes` buffer is returned with the serialized content. + /// :param output: The binary I/O object or file path to write to. For example, it could be a file path as a string or a file writer opened in binary mode with ``open('my_file.ttl', 'wb')``. If :py:const:`None`, a :py:class:`bytes` buffer is returned with the serialized content. /// :type output: io(bytes) or str or pathlib.Path or None, optional /// :param format: the format of the query results serialization using a media type like ``text/csv`` or an extension like `csv`. If :py:const:`None`, the format is guessed from the file name extension. /// :type format: str or None, optional /// :rtype: bytes or None /// :raises ValueError: if the format is not supported. - /// :raises OSError: if an error happens during a file writing. + /// :raises OSError: if a system error happens while writing the file. /// /// >>> store = Store() /// >>> store.add(Quad(NamedNode('http://example.com'), NamedNode('http://example.com/p'), Literal('1'))) @@ -325,13 +325,13 @@ impl PyQueryBoolean { /// It supports also some media type and extension aliases. /// For example, ``application/json`` could also be used for `JSON `_. /// - /// :param output: The binary I/O object or file path to write to. For example, it could be a file path as a string or a file writer opened in binary mode with ``open('my_file.ttl', 'wb')``. If :py:const:`None`, a :py:func:`bytes` buffer is returned with the serialized content. + /// :param output: The binary I/O object or file path to write to. For example, it could be a file path as a string or a file writer opened in binary mode with ``open('my_file.ttl', 'wb')``. If :py:const:`None`, a :py:class:`bytes` buffer is returned with the serialized content. /// :type output: io(bytes) or str or pathlib.Path or None, optional /// :param format: the format of the query results serialization using a media type like ``text/csv`` or an extension like `csv`. If :py:const:`None`, the format is guessed from the file name extension. /// :type format: str or None, optional /// :rtype: bytes or None /// :raises ValueError: if the format is not supported. - /// :raises OSError: if an error happens during a file writing. + /// :raises OSError: if a system error happens while writing the file. /// /// >>> store = Store() /// >>> store.add(Quad(NamedNode('http://example.com'), NamedNode('http://example.com/p'), Literal('1'))) @@ -404,13 +404,13 @@ impl PyQueryTriples { /// For example, ``application/turtle`` could also be used for `Turtle `_ /// and ``application/xml`` or ``xml`` for `RDF/XML `_. /// - /// :param output: The binary I/O object or file path to write to. For example, it could be a file path as a string or a file writer opened in binary mode with ``open('my_file.ttl', 'wb')``. If :py:const:`None`, a :py:func:`bytes` buffer is returned with the serialized content. + /// :param output: The binary I/O object or file path to write to. For example, it could be a file path as a string or a file writer opened in binary mode with ``open('my_file.ttl', 'wb')``. If :py:const:`None`, a :py:class:`bytes` buffer is returned with the serialized content. /// :type output: io(bytes) or str or pathlib.Path or None, optional /// :param format: the format of the RDF serialization using a media type like ``text/turtle`` or an extension like `ttl`. If :py:const:`None`, the format is guessed from the file name extension. /// :type format: str or None, optional /// :rtype: bytes or None /// :raises ValueError: if the format is not supported. - /// :raises OSError: if an error happens during a file writing. + /// :raises OSError: if a system error happens while writing the file. /// /// >>> store = Store() /// >>> store.add(Quad(NamedNode('http://example.com'), NamedNode('http://example.com/p'), Literal('1'))) @@ -468,10 +468,11 @@ impl PyQueryTriples { /// :type input: io(bytes) or io(str) or str or pathlib.Path /// :param format: the format of the RDF serialization using a media type like ``text/turtle`` or an extension like `ttl`. If :py:const:`None`, the format is guessed from the file name extension. /// :type format: str or None, optional -/// :return: an iterator of :py:class:`QuerySolution` or a :py:func:`bool`. +/// :return: an iterator of :py:class:`QuerySolution` or a :py:class:`bool`. /// :rtype: QuerySolutions or QueryBoolean /// :raises ValueError: if the format is not supported. /// :raises SyntaxError: if the provided data is invalid. +/// :raises OSError: if a system error happens while reading the file. /// /// >>> input = io.BytesIO(b'?s\t?p\t?o\n\t\t1\n') /// >>> list(parse_query_results(input, "text/tsv")) diff --git a/python/src/store.rs b/python/src/store.rs index 874e48aa..1befc92a 100644 --- a/python/src/store.rs +++ b/python/src/store.rs @@ -33,7 +33,7 @@ use std::path::PathBuf; /// :type path: str or pathlib.Path or None, optional /// :raises OSError: if the target directory contains invalid data or could not be accessed. /// -/// The :py:func:`str` function provides a serialization of the store in NQuads: +/// The :py:class:`str` function provides a serialization of the store in NQuads: /// /// >>> store = Store() /// >>> store.add(Quad(NamedNode('http://example.com'), NamedNode('http://example.com/p'), Literal('1'), NamedNode('http://example.com/g'))) @@ -251,7 +251,7 @@ impl PyStore { /// :type default_graph: NamedNode or BlankNode or DefaultGraph or list(NamedNode or BlankNode or DefaultGraph) or None, optional /// :param named_graphs: list of the named graphs that could be used in SPARQL `GRAPH` clause. By default, all the store named graphs are available. /// :type named_graphs: list(NamedNode or BlankNode) or None, optional - /// :return: a :py:func:`bool` for ``ASK`` queries, an iterator of :py:class:`Triple` for ``CONSTRUCT`` and ``DESCRIBE`` queries and an iterator of :py:class:`QuerySolution` for ``SELECT`` queries. + /// :return: a :py:class:`bool` for ``ASK`` queries, an iterator of :py:class:`Triple` for ``CONSTRUCT`` and ``DESCRIBE`` queries and an iterator of :py:class:`QuerySolution` for ``SELECT`` queries. /// :rtype: QuerySolutions or QueryBoolean or QueryTriples /// :raises SyntaxError: if the provided query is invalid. /// :raises OSError: if an error happens while reading the store. @@ -373,7 +373,7 @@ impl PyStore { /// :rtype: None /// :raises ValueError: if the format is not supported. /// :raises SyntaxError: if the provided data is invalid. - /// :raises OSError: if an error happens during a quad insertion. + /// :raises OSError: if an error happens during a quad insertion or if a system error happens while reading the file. /// /// >>> store = Store() /// >>> store.load(io.BytesIO(b'

"1" .'), "text/turtle", base_iri="http://example.com/", to_graph=NamedNode("http://example.com/g")) @@ -442,7 +442,7 @@ impl PyStore { /// :rtype: None /// :raises ValueError: if the format is not supported. /// :raises SyntaxError: if the provided data is invalid. - /// :raises OSError: if an error happens during a quad insertion. + /// :raises OSError: if an error happens during a quad insertion or if a system error happens while reading the file. /// /// >>> store = Store() /// >>> store.bulk_load(io.BytesIO(b'

"1" .'), "text/turtle", base_iri="http://example.com/", to_graph=NamedNode("http://example.com/g")) @@ -498,12 +498,13 @@ impl PyStore { /// For example, ``application/turtle`` could also be used for `Turtle `_ /// and ``application/xml`` or ``xml`` for `RDF/XML `_. /// - /// :param output: The binary I/O object or file path to write to. For example, it could be a file path as a string or a file writer opened in binary mode with ``open('my_file.ttl', 'wb')``. If :py:const:`None`, a :py:func:`bytes` buffer is returned with the serialized content. + /// :param output: The binary I/O object or file path to write to. For example, it could be a file path as a string or a file writer opened in binary mode with ``open('my_file.ttl', 'wb')``. If :py:const:`None`, a :py:class:`bytes` buffer is returned with the serialized content. /// :type output: io(bytes) or str or pathlib.Path or None, optional /// :param format: the format of the RDF serialization using a media type like ``text/turtle`` or an extension like `ttl`. If :py:const:`None`, the format is guessed from the file name extension. /// :type format: str or None, optional /// :param from_graph: the store graph from which dump the triples. Required if the serialization format does not support named graphs. If it does supports named graphs the full dataset is written. /// :type from_graph: NamedNode or BlankNode or DefaultGraph or None, optional + /// :return: py:class:`bytes` with the serialization if the ``output`` parameter is :py:const:`None`, :py:const:`None` if ``output`` is set. /// :rtype: bytes or None /// :raises ValueError: if the format is not supported or the `from_graph` parameter is not given with a syntax not supporting named graphs. /// :raises OSError: if an error happens during a quad lookup or file writing. From 7c4578f5f56b088722255e56b8a46079ce4d98f3 Mon Sep 17 00:00:00 2001 From: Tpt Date: Tue, 12 Sep 2023 21:39:16 +0200 Subject: [PATCH 089/217] ReadTheDocs: updates Rust version --- .readthedocs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 57d4f630..da4c7a80 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -7,7 +7,7 @@ build: os: "ubuntu-22.04" tools: python: "3" - rust: "1.64" + rust: "1.70" apt_packages: - clang From 8c8ca5459615f44aabd9b414969c9b070fa1961b Mon Sep 17 00:00:00 2001 From: Tpt Date: Fri, 15 Sep 2023 08:42:43 +0200 Subject: [PATCH 090/217] CI: Increases fuzzing time We have a lot of fuzzers now, let's give them a bit of time --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c06b05d2..03c038bb 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -319,7 +319,7 @@ jobs: - uses: google/clusterfuzzlite/actions/run_fuzzers@v1 with: github-token: ${{ secrets.GITHUB_TOKEN }} - fuzz-seconds: 600 + fuzz-seconds: 900 mode: code-change sanitizer: address minimize-crashes: true @@ -341,7 +341,7 @@ jobs: - uses: google/clusterfuzzlite/actions/run_fuzzers@v1 with: github-token: ${{ secrets.GITHUB_TOKEN }} - fuzz-seconds: 3600 + fuzz-seconds: 7200 mode: batch sanitizer: address minimize-crashes: true From 8ee30cf0016377ee6b7fe1cacc98f1f899bc857a Mon Sep 17 00:00:00 2001 From: Tpt Date: Fri, 15 Sep 2023 17:39:11 +0200 Subject: [PATCH 091/217] Removes .devcontainer Unused and unmaintained --- .devcontainer/Dockerfile | 21 ---------- .devcontainer/devcontainer.json | 69 --------------------------------- 2 files changed, 90 deletions(-) delete mode 100644 .devcontainer/Dockerfile delete mode 100644 .devcontainer/devcontainer.json diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile deleted file mode 100644 index a7a58796..00000000 --- a/.devcontainer/Dockerfile +++ /dev/null @@ -1,21 +0,0 @@ -# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.241.1/containers/rust/.devcontainer/base.Dockerfile - -# [Choice] Debian OS version (use bullseye on local arm64/Apple Silicon): buster, bullseye -ARG VARIANT="bullseye" -FROM mcr.microsoft.com/vscode/devcontainers/rust:0-${VARIANT} - -# [Optional] Uncomment this section to install additional packages. -RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ - && apt-get -y install --no-install-recommends \ - python3 \ - python3-venv \ - python-is-python3 \ - libclang-dev - -ENV VIRTUAL_ENV=/opt/venv -RUN python -m venv $VIRTUAL_ENV -ENV PATH="$VIRTUAL_ENV/bin:$PATH" -RUN pip install --no-cache-dir -r python/requirements.dev.txt - -# Change owner to the devcontainer user -RUN chown -R 1000:1000 $VIRTUAL_ENV diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json deleted file mode 100644 index e8978483..00000000 --- a/.devcontainer/devcontainer.json +++ /dev/null @@ -1,69 +0,0 @@ -// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at: -// https://github.com/microsoft/vscode-dev-containers/tree/v0.241.1/containers/rust -{ - "name": "Rust", - "build": { - "dockerfile": "Dockerfile", - "args": { - // Use the VARIANT arg to pick a Debian OS version: buster, bullseye - // Use bullseye when on local on arm64/Apple Silicon. - "VARIANT": "bullseye" - } - }, - "runArgs": ["--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined"], - - // Configure tool-specific properties. - "customizations": { - // Configure properties specific to VS Code. - "vscode": { - // Set *default* container specific settings.json values on container create. - "settings": { - "lldb.executable": "/usr/bin/lldb", - // VS Code don't watch files under ./target - "files.watcherExclude": { - "**/target/**": true - }, - "rust-analyzer.checkOnSave.command": "clippy", - - "python.defaultInterpreterPath": "/opt/venv/bin/python", - "python.linting.enabled": true, - "python.linting.pylintEnabled": true, - "python.formatting.autopep8Path": "/usr/local/py-utils/bin/autopep8", - "python.formatting.blackPath": "/usr/local/py-utils/bin/black", - "python.formatting.yapfPath": "/usr/local/py-utils/bin/yapf", - "python.linting.banditPath": "/usr/local/py-utils/bin/bandit", - "python.linting.flake8Path": "/usr/local/py-utils/bin/flake8", - "python.linting.mypyPath": "/usr/local/py-utils/bin/mypy", - "python.linting.pycodestylePath": "/usr/local/py-utils/bin/pycodestyle", - "python.linting.pydocstylePath": "/usr/local/py-utils/bin/pydocstyle", - "python.linting.pylintPath": "/opt/venv/bin/pylint", - "python.testing.pytestPath": "/opt/venv/bin/pytest" - }, - - // Add the IDs of extensions you want installed when the container is created. - "extensions": [ - "vadimcn.vscode-lldb", - "mutantdino.resourcemonitor", - "rust-lang.rust-analyzer", - "tamasfe.even-better-toml", - "serayuzgur.crates", - "ms-python.python", - "ms-python.vscode-pylance", - "esbenp.prettier-vscode", - "stardog-union.stardog-rdf-grammars" - ] - } - }, - - // Use 'forwardPorts' to make a list of ports inside the container available locally. - // "forwardPorts": [], - - // Use 'postCreateCommand' to run commands after the container is created. - "postCreateCommand": "git submodule update --init && cargo build", - - // Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. - "remoteUser": "vscode", - "features": { - "python": "3.10" - } -} From 0783d1dcdaa28815a4d329596c45affaae5077e2 Mon Sep 17 00:00:00 2001 From: Tpt Date: Fri, 15 Sep 2023 22:09:48 +0200 Subject: [PATCH 092/217] Splits sparesults lib.rs Avoids a big file --- lib/sparesults/src/format.rs | 163 ++++++++++ lib/sparesults/src/lib.rs | 542 +------------------------------ lib/sparesults/src/parser.rs | 202 ++++++++++++ lib/sparesults/src/serializer.rs | 182 +++++++++++ 4 files changed, 553 insertions(+), 536 deletions(-) create mode 100644 lib/sparesults/src/format.rs create mode 100644 lib/sparesults/src/parser.rs create mode 100644 lib/sparesults/src/serializer.rs diff --git a/lib/sparesults/src/format.rs b/lib/sparesults/src/format.rs new file mode 100644 index 00000000..e7eba74a --- /dev/null +++ b/lib/sparesults/src/format.rs @@ -0,0 +1,163 @@ +use std::fmt; + +/// [SPARQL query](https://www.w3.org/TR/sparql11-query/) results serialization formats. +#[derive(Eq, PartialEq, Debug, Clone, Copy, Hash)] +#[non_exhaustive] +pub enum QueryResultsFormat { + /// [SPARQL Query Results XML Format](https://www.w3.org/TR/rdf-sparql-XMLres/) + Xml, + /// [SPARQL Query Results JSON Format](https://www.w3.org/TR/sparql11-results-json/) + Json, + /// [SPARQL Query Results CSV Format](https://www.w3.org/TR/sparql11-results-csv-tsv/) + Csv, + /// [SPARQL Query Results TSV Format](https://www.w3.org/TR/sparql11-results-csv-tsv/) + Tsv, +} + +impl QueryResultsFormat { + /// The format canonical IRI according to the [Unique URIs for file formats registry](https://www.w3.org/ns/formats/). + /// + /// ``` + /// use sparesults::QueryResultsFormat; + /// + /// assert_eq!(QueryResultsFormat::Json.iri(), "http://www.w3.org/ns/formats/SPARQL_Results_JSON") + /// ``` + #[inline] + pub fn iri(self) -> &'static str { + match self { + Self::Xml => "http://www.w3.org/ns/formats/SPARQL_Results_XML", + Self::Json => "http://www.w3.org/ns/formats/SPARQL_Results_JSON", + Self::Csv => "http://www.w3.org/ns/formats/SPARQL_Results_CSV", + Self::Tsv => "http://www.w3.org/ns/formats/SPARQL_Results_TSV", + } + } + /// The format [IANA media type](https://tools.ietf.org/html/rfc2046). + /// + /// ``` + /// use sparesults::QueryResultsFormat; + /// + /// assert_eq!(QueryResultsFormat::Json.media_type(), "application/sparql-results+json") + /// ``` + #[inline] + pub fn media_type(self) -> &'static str { + match self { + Self::Xml => "application/sparql-results+xml", + Self::Json => "application/sparql-results+json", + Self::Csv => "text/csv; charset=utf-8", + Self::Tsv => "text/tab-separated-values; charset=utf-8", + } + } + + /// The format [IANA-registered](https://tools.ietf.org/html/rfc2046) file extension. + /// + /// ``` + /// use sparesults::QueryResultsFormat; + /// + /// assert_eq!(QueryResultsFormat::Json.file_extension(), "srj") + /// ``` + #[inline] + pub fn file_extension(self) -> &'static str { + match self { + Self::Xml => "srx", + Self::Json => "srj", + Self::Csv => "csv", + Self::Tsv => "tsv", + } + } + + /// The format name. + /// + /// ``` + /// use sparesults::QueryResultsFormat; + /// + /// assert_eq!(QueryResultsFormat::Json.name(), "SPARQL Results in JSON") + /// ``` + #[inline] + pub const fn name(self) -> &'static str { + match self { + Self::Xml => "SPARQL Results in XML", + Self::Json => "SPARQL Results in JSON", + Self::Csv => "SPARQL Results in CSV", + Self::Tsv => "SPARQL Results in TSV", + } + } + + /// Looks for a known format from a media type. + /// + /// It supports some media type aliases. + /// For example, "application/xml" is going to return `Xml` even if it is not its canonical media type. + /// + /// Example: + /// ``` + /// use sparesults::QueryResultsFormat; + /// + /// assert_eq!(QueryResultsFormat::from_media_type("application/sparql-results+json; charset=utf-8"), Some(QueryResultsFormat::Json)) + /// ``` + #[inline] + pub fn from_media_type(media_type: &str) -> Option { + const MEDIA_SUBTYPES: [(&str, QueryResultsFormat); 8] = [ + ("csv", QueryResultsFormat::Csv), + ("json", QueryResultsFormat::Json), + ("plain", QueryResultsFormat::Csv), + ("sparql-results+json", QueryResultsFormat::Json), + ("sparql-results+xml", QueryResultsFormat::Xml), + ("tab-separated-values", QueryResultsFormat::Tsv), + ("tsv", QueryResultsFormat::Tsv), + ("xml", QueryResultsFormat::Xml), + ]; + + let (r#type, subtype) = media_type + .split_once(';') + .unwrap_or((media_type, "")) + .0 + .trim() + .split_once('/')?; + let r#type = r#type.trim(); + if !r#type.eq_ignore_ascii_case("application") && !r#type.eq_ignore_ascii_case("text") { + return None; + } + let subtype = subtype.trim(); + let subtype = subtype.strip_prefix("x-").unwrap_or(subtype); + for (candidate_subtype, candidate_id) in MEDIA_SUBTYPES { + if candidate_subtype.eq_ignore_ascii_case(subtype) { + return Some(candidate_id); + } + } + None + } + + /// Looks for a known format from an extension. + /// + /// It supports some aliases. + /// + /// Example: + /// ``` + /// use sparesults::QueryResultsFormat; + /// + /// assert_eq!(QueryResultsFormat::from_extension("json"), Some(QueryResultsFormat::Json)) + /// ``` + #[inline] + pub fn from_extension(extension: &str) -> Option { + const MEDIA_TYPES: [(&str, QueryResultsFormat); 7] = [ + ("csv", QueryResultsFormat::Csv), + ("json", QueryResultsFormat::Json), + ("srj", QueryResultsFormat::Json), + ("srx", QueryResultsFormat::Xml), + ("tsv", QueryResultsFormat::Tsv), + ("txt", QueryResultsFormat::Csv), + ("xml", QueryResultsFormat::Xml), + ]; + for (candidate_extension, candidate_id) in MEDIA_TYPES { + if candidate_extension.eq_ignore_ascii_case(extension) { + return Some(candidate_id); + } + } + None + } +} + +impl fmt::Display for QueryResultsFormat { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.name()) + } +} diff --git a/lib/sparesults/src/lib.rs b/lib/sparesults/src/lib.rs index 5daa2bd9..fc2aee67 100644 --- a/lib/sparesults/src/lib.rs +++ b/lib/sparesults/src/lib.rs @@ -6,545 +6,15 @@ mod csv; mod error; +mod format; mod json; +mod parser; +mod serializer; pub mod solution; mod xml; -use crate::csv::*; pub use crate::error::{ParseError, SyntaxError}; -use crate::json::*; +pub use crate::format::QueryResultsFormat; +pub use crate::parser::{QueryResultsParser, QueryResultsReader, SolutionsReader}; +pub use crate::serializer::{QueryResultsSerializer, SolutionsWriter}; pub use crate::solution::QuerySolution; -use crate::xml::*; -use oxrdf::{TermRef, Variable, VariableRef}; -use std::fmt; -use std::io::{self, BufRead, Write}; -use std::rc::Rc; - -/// [SPARQL query](https://www.w3.org/TR/sparql11-query/) results serialization formats. -#[derive(Eq, PartialEq, Debug, Clone, Copy, Hash)] -#[non_exhaustive] -pub enum QueryResultsFormat { - /// [SPARQL Query Results XML Format](https://www.w3.org/TR/rdf-sparql-XMLres/) - Xml, - /// [SPARQL Query Results JSON Format](https://www.w3.org/TR/sparql11-results-json/) - Json, - /// [SPARQL Query Results CSV Format](https://www.w3.org/TR/sparql11-results-csv-tsv/) - Csv, - /// [SPARQL Query Results TSV Format](https://www.w3.org/TR/sparql11-results-csv-tsv/) - Tsv, -} - -impl QueryResultsFormat { - /// The format canonical IRI according to the [Unique URIs for file formats registry](https://www.w3.org/ns/formats/). - /// - /// ``` - /// use sparesults::QueryResultsFormat; - /// - /// assert_eq!(QueryResultsFormat::Json.iri(), "http://www.w3.org/ns/formats/SPARQL_Results_JSON") - /// ``` - #[inline] - pub fn iri(self) -> &'static str { - match self { - Self::Xml => "http://www.w3.org/ns/formats/SPARQL_Results_XML", - Self::Json => "http://www.w3.org/ns/formats/SPARQL_Results_JSON", - Self::Csv => "http://www.w3.org/ns/formats/SPARQL_Results_CSV", - Self::Tsv => "http://www.w3.org/ns/formats/SPARQL_Results_TSV", - } - } - /// The format [IANA media type](https://tools.ietf.org/html/rfc2046). - /// - /// ``` - /// use sparesults::QueryResultsFormat; - /// - /// assert_eq!(QueryResultsFormat::Json.media_type(), "application/sparql-results+json") - /// ``` - #[inline] - pub fn media_type(self) -> &'static str { - match self { - Self::Xml => "application/sparql-results+xml", - Self::Json => "application/sparql-results+json", - Self::Csv => "text/csv; charset=utf-8", - Self::Tsv => "text/tab-separated-values; charset=utf-8", - } - } - - /// The format [IANA-registered](https://tools.ietf.org/html/rfc2046) file extension. - /// - /// ``` - /// use sparesults::QueryResultsFormat; - /// - /// assert_eq!(QueryResultsFormat::Json.file_extension(), "srj") - /// ``` - #[inline] - pub fn file_extension(self) -> &'static str { - match self { - Self::Xml => "srx", - Self::Json => "srj", - Self::Csv => "csv", - Self::Tsv => "tsv", - } - } - - /// The format name. - /// - /// ``` - /// use sparesults::QueryResultsFormat; - /// - /// assert_eq!(QueryResultsFormat::Json.name(), "SPARQL Results in JSON") - /// ``` - #[inline] - pub const fn name(self) -> &'static str { - match self { - Self::Xml => "SPARQL Results in XML", - Self::Json => "SPARQL Results in JSON", - Self::Csv => "SPARQL Results in CSV", - Self::Tsv => "SPARQL Results in TSV", - } - } - - /// Looks for a known format from a media type. - /// - /// It supports some media type aliases. - /// For example, "application/xml" is going to return `Xml` even if it is not its canonical media type. - /// - /// Example: - /// ``` - /// use sparesults::QueryResultsFormat; - /// - /// assert_eq!(QueryResultsFormat::from_media_type("application/sparql-results+json; charset=utf-8"), Some(QueryResultsFormat::Json)) - /// ``` - #[inline] - pub fn from_media_type(media_type: &str) -> Option { - const MEDIA_SUBTYPES: [(&str, QueryResultsFormat); 8] = [ - ("csv", QueryResultsFormat::Csv), - ("json", QueryResultsFormat::Json), - ("plain", QueryResultsFormat::Csv), - ("sparql-results+json", QueryResultsFormat::Json), - ("sparql-results+xml", QueryResultsFormat::Xml), - ("tab-separated-values", QueryResultsFormat::Tsv), - ("tsv", QueryResultsFormat::Tsv), - ("xml", QueryResultsFormat::Xml), - ]; - - let (r#type, subtype) = media_type - .split_once(';') - .unwrap_or((media_type, "")) - .0 - .trim() - .split_once('/')?; - let r#type = r#type.trim(); - if !r#type.eq_ignore_ascii_case("application") && !r#type.eq_ignore_ascii_case("text") { - return None; - } - let subtype = subtype.trim(); - let subtype = subtype.strip_prefix("x-").unwrap_or(subtype); - for (candidate_subtype, candidate_id) in MEDIA_SUBTYPES { - if candidate_subtype.eq_ignore_ascii_case(subtype) { - return Some(candidate_id); - } - } - None - } - - /// Looks for a known format from an extension. - /// - /// It supports some aliases. - /// - /// Example: - /// ``` - /// use sparesults::QueryResultsFormat; - /// - /// assert_eq!(QueryResultsFormat::from_extension("json"), Some(QueryResultsFormat::Json)) - /// ``` - #[inline] - pub fn from_extension(extension: &str) -> Option { - const MEDIA_TYPES: [(&str, QueryResultsFormat); 7] = [ - ("csv", QueryResultsFormat::Csv), - ("json", QueryResultsFormat::Json), - ("srj", QueryResultsFormat::Json), - ("srx", QueryResultsFormat::Xml), - ("tsv", QueryResultsFormat::Tsv), - ("txt", QueryResultsFormat::Csv), - ("xml", QueryResultsFormat::Xml), - ]; - for (candidate_extension, candidate_id) in MEDIA_TYPES { - if candidate_extension.eq_ignore_ascii_case(extension) { - return Some(candidate_id); - } - } - None - } -} - -impl fmt::Display for QueryResultsFormat { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(self.name()) - } -} - -/// Parsers for [SPARQL query](https://www.w3.org/TR/sparql11-query/) results serialization formats. -/// -/// It currently supports the following formats: -/// * [SPARQL Query Results XML Format](https://www.w3.org/TR/rdf-sparql-XMLres/) ([`QueryResultsFormat::Xml`](QueryResultsFormat::Xml)). -/// * [SPARQL Query Results JSON Format](https://www.w3.org/TR/sparql11-results-json/) ([`QueryResultsFormat::Json`](QueryResultsFormat::Json)). -/// * [SPARQL Query Results TSV Format](https://www.w3.org/TR/sparql11-results-csv-tsv/) ([`QueryResultsFormat::Tsv`](QueryResultsFormat::Tsv)). -/// -/// Example in JSON (the API is the same for XML and TSV): -/// ``` -/// use sparesults::{QueryResultsFormat, QueryResultsParser, QueryResultsReader}; -/// use oxrdf::{Literal, Variable}; -/// -/// let json_parser = QueryResultsParser::from_format(QueryResultsFormat::Json); -/// // boolean -/// if let QueryResultsReader::Boolean(v) = json_parser.read_results(b"{\"boolean\":true}".as_slice())? { -/// assert_eq!(v, true); -/// } -/// // solutions -/// if let QueryResultsReader::Solutions(solutions) = json_parser.read_results(b"{\"head\":{\"vars\":[\"foo\",\"bar\"]},\"results\":{\"bindings\":[{\"foo\":{\"type\":\"literal\",\"value\":\"test\"}}]}}".as_slice())? { -/// assert_eq!(solutions.variables(), &[Variable::new_unchecked("foo"), Variable::new_unchecked("bar")]); -/// for solution in solutions { -/// assert_eq!(solution?.iter().collect::>(), vec![(&Variable::new_unchecked("foo"), &Literal::from("test").into())]); -/// } -/// } -/// # Result::<(),sparesults::ParseError>::Ok(()) -/// ``` -pub struct QueryResultsParser { - format: QueryResultsFormat, -} - -impl QueryResultsParser { - /// Builds a parser for the given format. - #[inline] - pub fn from_format(format: QueryResultsFormat) -> Self { - Self { format } - } - - /// Reads a result file. - /// - /// Example in XML (the API is the same for JSON and TSV): - /// ``` - /// use sparesults::{QueryResultsFormat, QueryResultsParser, QueryResultsReader}; - /// use oxrdf::{Literal, Variable}; - /// - /// let json_parser = QueryResultsParser::from_format(QueryResultsFormat::Xml); - /// - /// // boolean - /// if let QueryResultsReader::Boolean(v) = json_parser.read_results(b"true".as_slice())? { - /// assert_eq!(v, true); - /// } - /// - /// // solutions - /// if let QueryResultsReader::Solutions(solutions) = json_parser.read_results(b"test".as_slice())? { - /// assert_eq!(solutions.variables(), &[Variable::new_unchecked("foo"), Variable::new_unchecked("bar")]); - /// for solution in solutions { - /// assert_eq!(solution?.iter().collect::>(), vec![(&Variable::new_unchecked("foo"), &Literal::from("test").into())]); - /// } - /// } - /// # Result::<(),sparesults::ParseError>::Ok(()) - /// ``` - pub fn read_results(&self, reader: R) -> Result, ParseError> { - Ok(match self.format { - QueryResultsFormat::Xml => match XmlQueryResultsReader::read(reader)? { - XmlQueryResultsReader::Boolean(r) => QueryResultsReader::Boolean(r), - XmlQueryResultsReader::Solutions { - solutions, - variables, - } => QueryResultsReader::Solutions(SolutionsReader { - variables: Rc::new(variables), - solutions: SolutionsReaderKind::Xml(solutions), - }), - }, - QueryResultsFormat::Json => match JsonQueryResultsReader::read(reader)? { - JsonQueryResultsReader::Boolean(r) => QueryResultsReader::Boolean(r), - JsonQueryResultsReader::Solutions { - solutions, - variables, - } => QueryResultsReader::Solutions(SolutionsReader { - variables: Rc::new(variables), - solutions: SolutionsReaderKind::Json(solutions), - }), - }, - QueryResultsFormat::Csv => return Err(SyntaxError::msg("CSV SPARQL results syntax is lossy and can't be parsed to a proper RDF representation").into()), - QueryResultsFormat::Tsv => match TsvQueryResultsReader::read(reader)? { - TsvQueryResultsReader::Boolean(r) => QueryResultsReader::Boolean(r), - TsvQueryResultsReader::Solutions { - solutions, - variables, - } => QueryResultsReader::Solutions(SolutionsReader { - variables: Rc::new(variables), - solutions: SolutionsReaderKind::Tsv(solutions), - }), - }, - }) - } -} - -/// The reader for a given read of a results file. -/// -/// It is either a read boolean ([`bool`]) or a streaming reader of a set of solutions ([`SolutionsReader`]). -/// -/// Example in TSV (the API is the same for JSON and XML): -/// ``` -/// use sparesults::{QueryResultsFormat, QueryResultsParser, QueryResultsReader}; -/// use oxrdf::{Literal, Variable}; -/// -/// let json_parser = QueryResultsParser::from_format(QueryResultsFormat::Tsv); -/// -/// // boolean -/// if let QueryResultsReader::Boolean(v) = json_parser.read_results(b"true".as_slice())? { -/// assert_eq!(v, true); -/// } -/// -/// // solutions -/// if let QueryResultsReader::Solutions(solutions) = json_parser.read_results(b"?foo\t?bar\n\"test\"\t".as_slice())? { -/// assert_eq!(solutions.variables(), &[Variable::new_unchecked("foo"), Variable::new_unchecked("bar")]); -/// for solution in solutions { -/// assert_eq!(solution?.iter().collect::>(), vec![(&Variable::new_unchecked("foo"), &Literal::from("test").into())]); -/// } -/// } -/// # Result::<(),sparesults::ParseError>::Ok(()) -/// ``` -pub enum QueryResultsReader { - Solutions(SolutionsReader), - Boolean(bool), -} - -/// A streaming reader of a set of [`QuerySolution`] solutions. -/// -/// It implements the [`Iterator`] API to iterate over the solutions. -/// -/// Example in JSON (the API is the same for XML and TSV): -/// ``` -/// use sparesults::{QueryResultsFormat, QueryResultsParser, QueryResultsReader}; -/// use oxrdf::{Literal, Variable}; -/// -/// let json_parser = QueryResultsParser::from_format(QueryResultsFormat::Json); -/// if let QueryResultsReader::Solutions(solutions) = json_parser.read_results(b"{\"head\":{\"vars\":[\"foo\",\"bar\"]},\"results\":{\"bindings\":[{\"foo\":{\"type\":\"literal\",\"value\":\"test\"}}]}}".as_slice())? { -/// assert_eq!(solutions.variables(), &[Variable::new_unchecked("foo"), Variable::new_unchecked("bar")]); -/// for solution in solutions { -/// assert_eq!(solution?.iter().collect::>(), vec![(&Variable::new_unchecked("foo"), &Literal::from("test").into())]); -/// } -/// } -/// # Result::<(),sparesults::ParseError>::Ok(()) -/// ``` -#[allow(clippy::rc_buffer)] -pub struct SolutionsReader { - variables: Rc>, - solutions: SolutionsReaderKind, -} - -enum SolutionsReaderKind { - Xml(XmlSolutionsReader), - Json(JsonSolutionsReader), - Tsv(TsvSolutionsReader), -} - -impl SolutionsReader { - /// Ordered list of the declared variables at the beginning of the results. - /// - /// Example in TSV (the API is the same for JSON and XML): - /// ``` - /// use sparesults::{QueryResultsFormat, QueryResultsParser, QueryResultsReader}; - /// use oxrdf::Variable; - /// - /// let json_parser = QueryResultsParser::from_format(QueryResultsFormat::Tsv); - /// if let QueryResultsReader::Solutions(solutions) = json_parser.read_results(b"?foo\t?bar\n\"ex1\"\t\"ex2\"".as_slice())? { - /// assert_eq!(solutions.variables(), &[Variable::new_unchecked("foo"), Variable::new_unchecked("bar")]); - /// } - /// # Result::<(),sparesults::ParseError>::Ok(()) - /// ``` - #[inline] - pub fn variables(&self) -> &[Variable] { - &self.variables - } -} - -impl Iterator for SolutionsReader { - type Item = Result; - - fn next(&mut self) -> Option> { - Some( - match &mut self.solutions { - SolutionsReaderKind::Xml(reader) => reader.read_next(), - SolutionsReaderKind::Json(reader) => reader.read_next(), - SolutionsReaderKind::Tsv(reader) => reader.read_next(), - } - .transpose()? - .map(|values| (Rc::clone(&self.variables), values).into()), - ) - } -} - -/// A serializer for [SPARQL query](https://www.w3.org/TR/sparql11-query/) results serialization formats. -/// -/// It currently supports the following formats: -/// * [SPARQL Query Results XML Format](https://www.w3.org/TR/rdf-sparql-XMLres/) ([`QueryResultsFormat::Xml`](QueryResultsFormat::Xml)) -/// * [SPARQL Query Results JSON Format](https://www.w3.org/TR/sparql11-results-json/) ([`QueryResultsFormat::Json`](QueryResultsFormat::Json)) -/// * [SPARQL Query Results CSV Format](https://www.w3.org/TR/sparql11-results-csv-tsv/) ([`QueryResultsFormat::Csv`](QueryResultsFormat::Csv)) -/// * [SPARQL Query Results TSV Format](https://www.w3.org/TR/sparql11-results-csv-tsv/) ([`QueryResultsFormat::Tsv`](QueryResultsFormat::Tsv)) -/// -/// Example in JSON (the API is the same for XML and TSV): -/// ``` -/// use sparesults::{QueryResultsFormat, QueryResultsSerializer}; -/// use oxrdf::{LiteralRef, Variable, VariableRef}; -/// use std::iter::once; -/// -/// let json_serializer = QueryResultsSerializer::from_format(QueryResultsFormat::Json); -/// -/// // boolean -/// let mut buffer = Vec::new(); -/// json_serializer.write_boolean_result(&mut buffer, true)?; -/// assert_eq!(buffer, b"{\"head\":{},\"boolean\":true}"); -/// -/// // solutions -/// let mut buffer = Vec::new(); -/// let mut writer = json_serializer.solutions_writer(&mut buffer, vec![Variable::new_unchecked("foo"), Variable::new_unchecked("bar")])?; -/// writer.write(once((VariableRef::new_unchecked("foo"), LiteralRef::from("test"))))?; -/// writer.finish()?; -/// assert_eq!(buffer, b"{\"head\":{\"vars\":[\"foo\",\"bar\"]},\"results\":{\"bindings\":[{\"foo\":{\"type\":\"literal\",\"value\":\"test\"}}]}}"); -/// # std::io::Result::Ok(()) -/// ``` -pub struct QueryResultsSerializer { - format: QueryResultsFormat, -} - -impl QueryResultsSerializer { - /// Builds a serializer for the given format. - #[inline] - pub fn from_format(format: QueryResultsFormat) -> Self { - Self { format } - } - - /// Write a boolean query result (from an `ASK` query) into the given [`Write`] implementation. - /// - /// Example in XML (the API is the same for JSON and TSV): - /// ``` - /// use sparesults::{QueryResultsFormat, QueryResultsSerializer}; - /// - /// let json_serializer = QueryResultsSerializer::from_format(QueryResultsFormat::Xml); - /// let mut buffer = Vec::new(); - /// json_serializer.write_boolean_result(&mut buffer, true)?; - /// assert_eq!(buffer, b"true"); - /// # std::io::Result::Ok(()) - /// ``` - pub fn write_boolean_result(&self, writer: W, value: bool) -> io::Result { - match self.format { - QueryResultsFormat::Xml => write_boolean_xml_result(writer, value), - QueryResultsFormat::Json => write_boolean_json_result(writer, value), - QueryResultsFormat::Csv => write_boolean_csv_result(writer, value), - QueryResultsFormat::Tsv => write_boolean_tsv_result(writer, value), - } - } - - /// Returns a `SolutionsWriter` allowing writing query solutions into the given [`Write`] implementation. - /// - /// Example in XML (the API is the same for JSON and TSV): - /// ``` - /// use sparesults::{QueryResultsFormat, QueryResultsSerializer}; - /// use oxrdf::{LiteralRef, Variable, VariableRef}; - /// use std::iter::once; - /// - /// let json_serializer = QueryResultsSerializer::from_format(QueryResultsFormat::Xml); - /// let mut buffer = Vec::new(); - /// let mut writer = json_serializer.solutions_writer(&mut buffer, vec![Variable::new_unchecked("foo"), Variable::new_unchecked("bar")])?; - /// writer.write(once((VariableRef::new_unchecked("foo"), LiteralRef::from("test"))))?; - /// writer.finish()?; - /// assert_eq!(buffer, b"test"); - /// # std::io::Result::Ok(()) - /// ``` - pub fn solutions_writer( - &self, - writer: W, - variables: Vec, - ) -> io::Result> { - Ok(SolutionsWriter { - formatter: match self.format { - QueryResultsFormat::Xml => { - SolutionsWriterKind::Xml(XmlSolutionsWriter::start(writer, &variables)?) - } - QueryResultsFormat::Json => { - SolutionsWriterKind::Json(JsonSolutionsWriter::start(writer, &variables)?) - } - QueryResultsFormat::Csv => { - SolutionsWriterKind::Csv(CsvSolutionsWriter::start(writer, variables)?) - } - QueryResultsFormat::Tsv => { - SolutionsWriterKind::Tsv(TsvSolutionsWriter::start(writer, variables)?) - } - }, - }) - } -} - -/// Allows writing query results. -/// Could be built using a [`QueryResultsSerializer`]. -/// -///

Do not forget to run the [`finish`](SolutionsWriter::finish()) method to properly write the last bytes of the file.
-/// -/// Example in TSV (the API is the same for JSON and XML): -/// ``` -/// use sparesults::{QueryResultsFormat, QueryResultsSerializer}; -/// use oxrdf::{LiteralRef, Variable, VariableRef}; -/// use std::iter::once; -/// -/// let json_serializer = QueryResultsSerializer::from_format(QueryResultsFormat::Tsv); -/// let mut buffer = Vec::new(); -/// let mut writer = json_serializer.solutions_writer(&mut buffer, vec![Variable::new_unchecked("foo"), Variable::new_unchecked("bar")])?; -/// writer.write(once((VariableRef::new_unchecked("foo"), LiteralRef::from("test"))))?; -/// writer.finish()?; -/// assert_eq!(buffer, b"?foo\t?bar\n\"test\"\t\n"); -/// # std::io::Result::Ok(()) -/// ``` -#[must_use] -pub struct SolutionsWriter { - formatter: SolutionsWriterKind, -} - -enum SolutionsWriterKind { - Xml(XmlSolutionsWriter), - Json(JsonSolutionsWriter), - Csv(CsvSolutionsWriter), - Tsv(TsvSolutionsWriter), -} - -impl SolutionsWriter { - /// Writes a solution. - /// - /// Example in JSON (the API is the same for XML and TSV): - /// ``` - /// use sparesults::{QueryResultsFormat, QueryResultsSerializer, QuerySolution}; - /// use oxrdf::{Literal, LiteralRef, Variable, VariableRef}; - /// use std::iter::once; - /// - /// let json_serializer = QueryResultsSerializer::from_format(QueryResultsFormat::Json); - /// let mut buffer = Vec::new(); - /// let mut writer = json_serializer.solutions_writer(&mut buffer, vec![Variable::new_unchecked("foo"), Variable::new_unchecked("bar")])?; - /// writer.write(once((VariableRef::new_unchecked("foo"), LiteralRef::from("test"))))?; - /// writer.write(&QuerySolution::from((vec![Variable::new_unchecked("bar")], vec![Some(Literal::from("test").into())])))?; - /// writer.finish()?; - /// assert_eq!(buffer, b"{\"head\":{\"vars\":[\"foo\",\"bar\"]},\"results\":{\"bindings\":[{\"foo\":{\"type\":\"literal\",\"value\":\"test\"}},{\"bar\":{\"type\":\"literal\",\"value\":\"test\"}}]}}"); - /// # std::io::Result::Ok(()) - /// ``` - pub fn write<'a>( - &mut self, - solution: impl IntoIterator>, impl Into>)>, - ) -> io::Result<()> { - let solution = solution.into_iter().map(|(v, s)| (v.into(), s.into())); - match &mut self.formatter { - SolutionsWriterKind::Xml(writer) => writer.write(solution), - SolutionsWriterKind::Json(writer) => writer.write(solution), - SolutionsWriterKind::Csv(writer) => writer.write(solution), - SolutionsWriterKind::Tsv(writer) => writer.write(solution), - } - } - - /// Writes the last bytes of the file. - pub fn finish(self) -> io::Result { - match self.formatter { - SolutionsWriterKind::Xml(write) => write.finish(), - SolutionsWriterKind::Json(write) => write.finish(), - SolutionsWriterKind::Csv(write) => write.finish(), - SolutionsWriterKind::Tsv(write) => write.finish(), - } - } -} diff --git a/lib/sparesults/src/parser.rs b/lib/sparesults/src/parser.rs new file mode 100644 index 00000000..32ffd62e --- /dev/null +++ b/lib/sparesults/src/parser.rs @@ -0,0 +1,202 @@ +use crate::csv::{TsvQueryResultsReader, TsvSolutionsReader}; +use crate::error::{ParseError, SyntaxError}; +use crate::format::QueryResultsFormat; +use crate::json::{JsonQueryResultsReader, JsonSolutionsReader}; +use crate::solution::QuerySolution; +use crate::xml::{XmlQueryResultsReader, XmlSolutionsReader}; +use oxrdf::Variable; +use std::io::BufRead; +use std::rc::Rc; + +/// Parsers for [SPARQL query](https://www.w3.org/TR/sparql11-query/) results serialization formats. +/// +/// It currently supports the following formats: +/// * [SPARQL Query Results XML Format](https://www.w3.org/TR/rdf-sparql-XMLres/) ([`QueryResultsFormat::Xml`](QueryResultsFormat::Xml)). +/// * [SPARQL Query Results JSON Format](https://www.w3.org/TR/sparql11-results-json/) ([`QueryResultsFormat::Json`](QueryResultsFormat::Json)). +/// * [SPARQL Query Results TSV Format](https://www.w3.org/TR/sparql11-results-csv-tsv/) ([`QueryResultsFormat::Tsv`](QueryResultsFormat::Tsv)). +/// +/// Example in JSON (the API is the same for XML and TSV): +/// ``` +/// use sparesults::{QueryResultsFormat, QueryResultsParser, QueryResultsReader}; +/// use oxrdf::{Literal, Variable}; +/// +/// let json_parser = QueryResultsParser::from_format(QueryResultsFormat::Json); +/// // boolean +/// if let QueryResultsReader::Boolean(v) = json_parser.read_results(b"{\"boolean\":true}".as_slice())? { +/// assert_eq!(v, true); +/// } +/// // solutions +/// if let QueryResultsReader::Solutions(solutions) = json_parser.read_results(b"{\"head\":{\"vars\":[\"foo\",\"bar\"]},\"results\":{\"bindings\":[{\"foo\":{\"type\":\"literal\",\"value\":\"test\"}}]}}".as_slice())? { +/// assert_eq!(solutions.variables(), &[Variable::new_unchecked("foo"), Variable::new_unchecked("bar")]); +/// for solution in solutions { +/// assert_eq!(solution?.iter().collect::>(), vec![(&Variable::new_unchecked("foo"), &Literal::from("test").into())]); +/// } +/// } +/// # Result::<(),sparesults::ParseError>::Ok(()) +/// ``` +pub struct QueryResultsParser { + format: QueryResultsFormat, +} + +impl QueryResultsParser { + /// Builds a parser for the given format. + #[inline] + pub fn from_format(format: QueryResultsFormat) -> Self { + Self { format } + } + + /// Reads a result file. + /// + /// Example in XML (the API is the same for JSON and TSV): + /// ``` + /// use sparesults::{QueryResultsFormat, QueryResultsParser, QueryResultsReader}; + /// use oxrdf::{Literal, Variable}; + /// + /// let json_parser = QueryResultsParser::from_format(QueryResultsFormat::Xml); + /// + /// // boolean + /// if let QueryResultsReader::Boolean(v) = json_parser.read_results(b"true".as_slice())? { + /// assert_eq!(v, true); + /// } + /// + /// // solutions + /// if let QueryResultsReader::Solutions(solutions) = json_parser.read_results(b"test".as_slice())? { + /// assert_eq!(solutions.variables(), &[Variable::new_unchecked("foo"), Variable::new_unchecked("bar")]); + /// for solution in solutions { + /// assert_eq!(solution?.iter().collect::>(), vec![(&Variable::new_unchecked("foo"), &Literal::from("test").into())]); + /// } + /// } + /// # Result::<(),sparesults::ParseError>::Ok(()) + /// ``` + pub fn read_results(&self, reader: R) -> Result, ParseError> { + Ok(match self.format { + QueryResultsFormat::Xml => match XmlQueryResultsReader::read(reader)? { + XmlQueryResultsReader::Boolean(r) => QueryResultsReader::Boolean(r), + XmlQueryResultsReader::Solutions { + solutions, + variables, + } => QueryResultsReader::Solutions(SolutionsReader { + variables: Rc::new(variables), + solutions: SolutionsReaderKind::Xml(solutions), + }), + }, + QueryResultsFormat::Json => match JsonQueryResultsReader::read(reader)? { + JsonQueryResultsReader::Boolean(r) => QueryResultsReader::Boolean(r), + JsonQueryResultsReader::Solutions { + solutions, + variables, + } => QueryResultsReader::Solutions(SolutionsReader { + variables: Rc::new(variables), + solutions: SolutionsReaderKind::Json(solutions), + }), + }, + QueryResultsFormat::Csv => return Err(SyntaxError::msg("CSV SPARQL results syntax is lossy and can't be parsed to a proper RDF representation").into()), + QueryResultsFormat::Tsv => match TsvQueryResultsReader::read(reader)? { + TsvQueryResultsReader::Boolean(r) => QueryResultsReader::Boolean(r), + TsvQueryResultsReader::Solutions { + solutions, + variables, + } => QueryResultsReader::Solutions(SolutionsReader { + variables: Rc::new(variables), + solutions: SolutionsReaderKind::Tsv(solutions), + }), + }, + }) + } +} + +/// The reader for a given read of a results file. +/// +/// It is either a read boolean ([`bool`]) or a streaming reader of a set of solutions ([`SolutionsReader`]). +/// +/// Example in TSV (the API is the same for JSON and XML): +/// ``` +/// use sparesults::{QueryResultsFormat, QueryResultsParser, QueryResultsReader}; +/// use oxrdf::{Literal, Variable}; +/// +/// let json_parser = QueryResultsParser::from_format(QueryResultsFormat::Tsv); +/// +/// // boolean +/// if let QueryResultsReader::Boolean(v) = json_parser.read_results(b"true".as_slice())? { +/// assert_eq!(v, true); +/// } +/// +/// // solutions +/// if let QueryResultsReader::Solutions(solutions) = json_parser.read_results(b"?foo\t?bar\n\"test\"\t".as_slice())? { +/// assert_eq!(solutions.variables(), &[Variable::new_unchecked("foo"), Variable::new_unchecked("bar")]); +/// for solution in solutions { +/// assert_eq!(solution?.iter().collect::>(), vec![(&Variable::new_unchecked("foo"), &Literal::from("test").into())]); +/// } +/// } +/// # Result::<(),sparesults::ParseError>::Ok(()) +/// ``` +pub enum QueryResultsReader { + Solutions(SolutionsReader), + Boolean(bool), +} + +/// A streaming reader of a set of [`QuerySolution`] solutions. +/// +/// It implements the [`Iterator`] API to iterate over the solutions. +/// +/// Example in JSON (the API is the same for XML and TSV): +/// ``` +/// use sparesults::{QueryResultsFormat, QueryResultsParser, QueryResultsReader}; +/// use oxrdf::{Literal, Variable}; +/// +/// let json_parser = QueryResultsParser::from_format(QueryResultsFormat::Json); +/// if let QueryResultsReader::Solutions(solutions) = json_parser.read_results(b"{\"head\":{\"vars\":[\"foo\",\"bar\"]},\"results\":{\"bindings\":[{\"foo\":{\"type\":\"literal\",\"value\":\"test\"}}]}}".as_slice())? { +/// assert_eq!(solutions.variables(), &[Variable::new_unchecked("foo"), Variable::new_unchecked("bar")]); +/// for solution in solutions { +/// assert_eq!(solution?.iter().collect::>(), vec![(&Variable::new_unchecked("foo"), &Literal::from("test").into())]); +/// } +/// } +/// # Result::<(),sparesults::ParseError>::Ok(()) +/// ``` +#[allow(clippy::rc_buffer)] +pub struct SolutionsReader { + variables: Rc>, + solutions: SolutionsReaderKind, +} + +enum SolutionsReaderKind { + Xml(XmlSolutionsReader), + Json(JsonSolutionsReader), + Tsv(TsvSolutionsReader), +} + +impl SolutionsReader { + /// Ordered list of the declared variables at the beginning of the results. + /// + /// Example in TSV (the API is the same for JSON and XML): + /// ``` + /// use sparesults::{QueryResultsFormat, QueryResultsParser, QueryResultsReader}; + /// use oxrdf::Variable; + /// + /// let json_parser = QueryResultsParser::from_format(QueryResultsFormat::Tsv); + /// if let QueryResultsReader::Solutions(solutions) = json_parser.read_results(b"?foo\t?bar\n\"ex1\"\t\"ex2\"".as_slice())? { + /// assert_eq!(solutions.variables(), &[Variable::new_unchecked("foo"), Variable::new_unchecked("bar")]); + /// } + /// # Result::<(),sparesults::ParseError>::Ok(()) + /// ``` + #[inline] + pub fn variables(&self) -> &[Variable] { + &self.variables + } +} + +impl Iterator for SolutionsReader { + type Item = Result; + + fn next(&mut self) -> Option> { + Some( + match &mut self.solutions { + SolutionsReaderKind::Xml(reader) => reader.read_next(), + SolutionsReaderKind::Json(reader) => reader.read_next(), + SolutionsReaderKind::Tsv(reader) => reader.read_next(), + } + .transpose()? + .map(|values| (Rc::clone(&self.variables), values).into()), + ) + } +} diff --git a/lib/sparesults/src/serializer.rs b/lib/sparesults/src/serializer.rs new file mode 100644 index 00000000..f804a295 --- /dev/null +++ b/lib/sparesults/src/serializer.rs @@ -0,0 +1,182 @@ +use crate::csv::{ + write_boolean_csv_result, write_boolean_tsv_result, CsvSolutionsWriter, TsvSolutionsWriter, +}; +use crate::format::QueryResultsFormat; +use crate::json::{write_boolean_json_result, JsonSolutionsWriter}; +use crate::xml::{write_boolean_xml_result, XmlSolutionsWriter}; +use oxrdf::{TermRef, Variable, VariableRef}; +use std::io::{self, Write}; + +/// A serializer for [SPARQL query](https://www.w3.org/TR/sparql11-query/) results serialization formats. +/// +/// It currently supports the following formats: +/// * [SPARQL Query Results XML Format](https://www.w3.org/TR/rdf-sparql-XMLres/) ([`QueryResultsFormat::Xml`](QueryResultsFormat::Xml)) +/// * [SPARQL Query Results JSON Format](https://www.w3.org/TR/sparql11-results-json/) ([`QueryResultsFormat::Json`](QueryResultsFormat::Json)) +/// * [SPARQL Query Results CSV Format](https://www.w3.org/TR/sparql11-results-csv-tsv/) ([`QueryResultsFormat::Csv`](QueryResultsFormat::Csv)) +/// * [SPARQL Query Results TSV Format](https://www.w3.org/TR/sparql11-results-csv-tsv/) ([`QueryResultsFormat::Tsv`](QueryResultsFormat::Tsv)) +/// +/// Example in JSON (the API is the same for XML and TSV): +/// ``` +/// use sparesults::{QueryResultsFormat, QueryResultsSerializer}; +/// use oxrdf::{LiteralRef, Variable, VariableRef}; +/// use std::iter::once; +/// +/// let json_serializer = QueryResultsSerializer::from_format(QueryResultsFormat::Json); +/// +/// // boolean +/// let mut buffer = Vec::new(); +/// json_serializer.write_boolean_result(&mut buffer, true)?; +/// assert_eq!(buffer, b"{\"head\":{},\"boolean\":true}"); +/// +/// // solutions +/// let mut buffer = Vec::new(); +/// let mut writer = json_serializer.solutions_writer(&mut buffer, vec![Variable::new_unchecked("foo"), Variable::new_unchecked("bar")])?; +/// writer.write(once((VariableRef::new_unchecked("foo"), LiteralRef::from("test"))))?; +/// writer.finish()?; +/// assert_eq!(buffer, b"{\"head\":{\"vars\":[\"foo\",\"bar\"]},\"results\":{\"bindings\":[{\"foo\":{\"type\":\"literal\",\"value\":\"test\"}}]}}"); +/// # std::io::Result::Ok(()) +/// ``` +pub struct QueryResultsSerializer { + format: QueryResultsFormat, +} + +impl QueryResultsSerializer { + /// Builds a serializer for the given format. + #[inline] + pub fn from_format(format: QueryResultsFormat) -> Self { + Self { format } + } + + /// Write a boolean query result (from an `ASK` query) into the given [`Write`] implementation. + /// + /// Example in XML (the API is the same for JSON and TSV): + /// ``` + /// use sparesults::{QueryResultsFormat, QueryResultsSerializer}; + /// + /// let json_serializer = QueryResultsSerializer::from_format(QueryResultsFormat::Xml); + /// let mut buffer = Vec::new(); + /// json_serializer.write_boolean_result(&mut buffer, true)?; + /// assert_eq!(buffer, b"true"); + /// # std::io::Result::Ok(()) + /// ``` + pub fn write_boolean_result(&self, writer: W, value: bool) -> io::Result { + match self.format { + QueryResultsFormat::Xml => write_boolean_xml_result(writer, value), + QueryResultsFormat::Json => write_boolean_json_result(writer, value), + QueryResultsFormat::Csv => write_boolean_csv_result(writer, value), + QueryResultsFormat::Tsv => write_boolean_tsv_result(writer, value), + } + } + + /// Returns a `SolutionsWriter` allowing writing query solutions into the given [`Write`] implementation. + /// + /// Example in XML (the API is the same for JSON and TSV): + /// ``` + /// use sparesults::{QueryResultsFormat, QueryResultsSerializer}; + /// use oxrdf::{LiteralRef, Variable, VariableRef}; + /// use std::iter::once; + /// + /// let json_serializer = QueryResultsSerializer::from_format(QueryResultsFormat::Xml); + /// let mut buffer = Vec::new(); + /// let mut writer = json_serializer.solutions_writer(&mut buffer, vec![Variable::new_unchecked("foo"), Variable::new_unchecked("bar")])?; + /// writer.write(once((VariableRef::new_unchecked("foo"), LiteralRef::from("test"))))?; + /// writer.finish()?; + /// assert_eq!(buffer, b"test"); + /// # std::io::Result::Ok(()) + /// ``` + pub fn solutions_writer( + &self, + writer: W, + variables: Vec, + ) -> io::Result> { + Ok(SolutionsWriter { + formatter: match self.format { + QueryResultsFormat::Xml => { + SolutionsWriterKind::Xml(XmlSolutionsWriter::start(writer, &variables)?) + } + QueryResultsFormat::Json => { + SolutionsWriterKind::Json(JsonSolutionsWriter::start(writer, &variables)?) + } + QueryResultsFormat::Csv => { + SolutionsWriterKind::Csv(CsvSolutionsWriter::start(writer, variables)?) + } + QueryResultsFormat::Tsv => { + SolutionsWriterKind::Tsv(TsvSolutionsWriter::start(writer, variables)?) + } + }, + }) + } +} + +/// Allows writing query results. +/// Could be built using a [`QueryResultsSerializer`]. +/// +///
Do not forget to run the [`finish`](SolutionsWriter::finish()) method to properly write the last bytes of the file.
+/// +/// Example in TSV (the API is the same for JSON and XML): +/// ``` +/// use sparesults::{QueryResultsFormat, QueryResultsSerializer}; +/// use oxrdf::{LiteralRef, Variable, VariableRef}; +/// use std::iter::once; +/// +/// let json_serializer = QueryResultsSerializer::from_format(QueryResultsFormat::Tsv); +/// let mut buffer = Vec::new(); +/// let mut writer = json_serializer.solutions_writer(&mut buffer, vec![Variable::new_unchecked("foo"), Variable::new_unchecked("bar")])?; +/// writer.write(once((VariableRef::new_unchecked("foo"), LiteralRef::from("test"))))?; +/// writer.finish()?; +/// assert_eq!(buffer, b"?foo\t?bar\n\"test\"\t\n"); +/// # std::io::Result::Ok(()) +/// ``` +#[must_use] +pub struct SolutionsWriter { + formatter: SolutionsWriterKind, +} + +enum SolutionsWriterKind { + Xml(XmlSolutionsWriter), + Json(JsonSolutionsWriter), + Csv(CsvSolutionsWriter), + Tsv(TsvSolutionsWriter), +} + +impl SolutionsWriter { + /// Writes a solution. + /// + /// Example in JSON (the API is the same for XML and TSV): + /// ``` + /// use sparesults::{QueryResultsFormat, QueryResultsSerializer, QuerySolution}; + /// use oxrdf::{Literal, LiteralRef, Variable, VariableRef}; + /// use std::iter::once; + /// + /// let json_serializer = QueryResultsSerializer::from_format(QueryResultsFormat::Json); + /// let mut buffer = Vec::new(); + /// let mut writer = json_serializer.solutions_writer(&mut buffer, vec![Variable::new_unchecked("foo"), Variable::new_unchecked("bar")])?; + /// writer.write(once((VariableRef::new_unchecked("foo"), LiteralRef::from("test"))))?; + /// writer.write(&QuerySolution::from((vec![Variable::new_unchecked("bar")], vec![Some(Literal::from("test").into())])))?; + /// writer.finish()?; + /// assert_eq!(buffer, b"{\"head\":{\"vars\":[\"foo\",\"bar\"]},\"results\":{\"bindings\":[{\"foo\":{\"type\":\"literal\",\"value\":\"test\"}},{\"bar\":{\"type\":\"literal\",\"value\":\"test\"}}]}}"); + /// # std::io::Result::Ok(()) + /// ``` + pub fn write<'a>( + &mut self, + solution: impl IntoIterator>, impl Into>)>, + ) -> io::Result<()> { + let solution = solution.into_iter().map(|(v, s)| (v.into(), s.into())); + match &mut self.formatter { + SolutionsWriterKind::Xml(writer) => writer.write(solution), + SolutionsWriterKind::Json(writer) => writer.write(solution), + SolutionsWriterKind::Csv(writer) => writer.write(solution), + SolutionsWriterKind::Tsv(writer) => writer.write(solution), + } + } + + /// Writes the last bytes of the file. + pub fn finish(self) -> io::Result { + match self.formatter { + SolutionsWriterKind::Xml(write) => write.finish(), + SolutionsWriterKind::Json(write) => write.finish(), + SolutionsWriterKind::Csv(write) => write.finish(), + SolutionsWriterKind::Tsv(write) => write.finish(), + } + } +} From 6a21cb0625083a9785958d318d86f48db8cd5457 Mon Sep 17 00:00:00 2001 From: Tpt Date: Sat, 16 Sep 2023 15:15:16 +0200 Subject: [PATCH 093/217] Makes sparesults formatter API closer to oxrdfio --- cli/src/main.rs | 19 ++++---- fuzz/src/result_format.rs | 4 +- lib/sparesults/README.md | 8 +-- lib/sparesults/src/lib.rs | 2 +- lib/sparesults/src/serializer.rs | 84 ++++++++++++++++++++------------ lib/src/sparql/model.rs | 9 ++-- lib/src/sparql/results.rs | 8 +-- python/src/sparql.rs | 4 +- 8 files changed, 81 insertions(+), 57 deletions(-) diff --git a/cli/src/main.rs b/cli/src/main.rs index 5095076d..54f79299 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -555,7 +555,7 @@ pub fn main() -> anyhow::Result<()> { }; if let Some(results_file) = results_file { let mut writer = QueryResultsSerializer::from_format(format) - .solutions_writer( + .serialize_solutions_to_write( BufWriter::new(File::create(results_file)?), solutions.variables().to_vec(), )?; @@ -565,7 +565,7 @@ pub fn main() -> anyhow::Result<()> { close_file_writer(writer.finish()?)?; } else { let mut writer = QueryResultsSerializer::from_format(format) - .solutions_writer( + .serialize_solutions_to_write( stdout().lock(), solutions.variables().to_vec(), )?; @@ -595,14 +595,15 @@ pub fn main() -> anyhow::Result<()> { }; if let Some(results_file) = results_file { close_file_writer( - QueryResultsSerializer::from_format(format).write_boolean_result( - BufWriter::new(File::create(results_file)?), - result, - )?, + QueryResultsSerializer::from_format(format) + .serialize_boolean_to_write( + BufWriter::new(File::create(results_file)?), + result, + )?, )?; } else { QueryResultsSerializer::from_format(format) - .write_boolean_result(stdout().lock(), result)? + .serialize_boolean_to_write(stdout().lock(), result)? .flush()?; } } @@ -1318,7 +1319,7 @@ fn evaluate_sparql_query( move |w| { Ok(( QueryResultsSerializer::from_format(format) - .solutions_writer(w, solutions.variables().to_vec())?, + .serialize_solutions_to_write(w, solutions.variables().to_vec())?, solutions, )) }, @@ -1338,7 +1339,7 @@ fn evaluate_sparql_query( let format = query_results_content_negotiation(request)?; let mut body = Vec::new(); QueryResultsSerializer::from_format(format) - .write_boolean_result(&mut body, result) + .serialize_boolean_to_write(&mut body, result) .map_err(internal_server_error)?; Ok(Response::builder(Status::OK) .with_header(HeaderName::CONTENT_TYPE, format.media_type()) diff --git a/fuzz/src/result_format.rs b/fuzz/src/result_format.rs index cc00f9be..139d3c9a 100644 --- a/fuzz/src/result_format.rs +++ b/fuzz/src/result_format.rs @@ -18,7 +18,7 @@ pub fn fuzz_result_format(format: QueryResultsFormat, data: &[u8]) { // We try to write again let mut writer = serializer - .solutions_writer( + .serialize_solutions_to_write( Vec::new(), solutions .get(0) @@ -49,7 +49,7 @@ pub fn fuzz_result_format(format: QueryResultsFormat, data: &[u8]) { // We try to write again let mut serialized = Vec::new(); serializer - .write_boolean_result(&mut serialized, value) + .serialize_boolean_to_write(&mut serialized, value) .unwrap(); // And to parse again diff --git a/lib/sparesults/README.md b/lib/sparesults/README.md index 7e719065..f733d37b 100644 --- a/lib/sparesults/README.md +++ b/lib/sparesults/README.md @@ -28,15 +28,15 @@ fn convert_json_to_tsv(json_file: &[u8]) -> Result> { match json_parser.read_results(json_file)? { QueryResultsReader::Boolean(value) => { // it's a boolean result, we copy it in TSV to the output buffer - tsv_serializer.write_boolean_result(Vec::new(), value) + tsv_serializer.serialize_boolean_to_write(Vec::new(), value) }, QueryResultsReader::Solutions(solutions_reader) => { // it's a set of solutions, we create a writer and we write to it while reading in streaming from the JSON file - let mut solutions_writer = tsv_serializer.solutions_writer(Vec::new(), solutions_reader.variables().to_vec())?; + let mut serialize_solutions_to_write = tsv_serializer.serialize_solutions_to_write(Vec::new(), solutions_reader.variables().to_vec())?; for solution in solutions_reader { - solutions_writer.write(&solution?)?; + serialize_solutions_to_write.write(&solution?)?; } - solutions_writer.finish() + serialize_solutions_to_write.finish() } } } diff --git a/lib/sparesults/src/lib.rs b/lib/sparesults/src/lib.rs index fc2aee67..21cdce12 100644 --- a/lib/sparesults/src/lib.rs +++ b/lib/sparesults/src/lib.rs @@ -16,5 +16,5 @@ mod xml; pub use crate::error::{ParseError, SyntaxError}; pub use crate::format::QueryResultsFormat; pub use crate::parser::{QueryResultsParser, QueryResultsReader, SolutionsReader}; -pub use crate::serializer::{QueryResultsSerializer, SolutionsWriter}; +pub use crate::serializer::{QueryResultsSerializer, ToWriteSolutionsWriter}; pub use crate::solution::QuerySolution; diff --git a/lib/sparesults/src/serializer.rs b/lib/sparesults/src/serializer.rs index f804a295..2ce77f4d 100644 --- a/lib/sparesults/src/serializer.rs +++ b/lib/sparesults/src/serializer.rs @@ -25,12 +25,12 @@ use std::io::{self, Write}; /// /// // boolean /// let mut buffer = Vec::new(); -/// json_serializer.write_boolean_result(&mut buffer, true)?; +/// json_serializer.serialize_boolean_to_write(&mut buffer, true)?; /// assert_eq!(buffer, b"{\"head\":{},\"boolean\":true}"); /// /// // solutions /// let mut buffer = Vec::new(); -/// let mut writer = json_serializer.solutions_writer(&mut buffer, vec![Variable::new_unchecked("foo"), Variable::new_unchecked("bar")])?; +/// let mut writer = json_serializer.serialize_solutions_to_write(&mut buffer, vec![Variable::new_unchecked("foo"), Variable::new_unchecked("bar")])?; /// writer.write(once((VariableRef::new_unchecked("foo"), LiteralRef::from("test"))))?; /// writer.finish()?; /// assert_eq!(buffer, b"{\"head\":{\"vars\":[\"foo\",\"bar\"]},\"results\":{\"bindings\":[{\"foo\":{\"type\":\"literal\",\"value\":\"test\"}}]}}"); @@ -55,21 +55,30 @@ impl QueryResultsSerializer { /// /// let json_serializer = QueryResultsSerializer::from_format(QueryResultsFormat::Xml); /// let mut buffer = Vec::new(); - /// json_serializer.write_boolean_result(&mut buffer, true)?; + /// json_serializer.serialize_boolean_to_write(&mut buffer, true)?; /// assert_eq!(buffer, b"true"); /// # std::io::Result::Ok(()) /// ``` - pub fn write_boolean_result(&self, writer: W, value: bool) -> io::Result { + pub fn serialize_boolean_to_write(&self, write: W, value: bool) -> io::Result { match self.format { - QueryResultsFormat::Xml => write_boolean_xml_result(writer, value), - QueryResultsFormat::Json => write_boolean_json_result(writer, value), - QueryResultsFormat::Csv => write_boolean_csv_result(writer, value), - QueryResultsFormat::Tsv => write_boolean_tsv_result(writer, value), + QueryResultsFormat::Xml => write_boolean_xml_result(write, value), + QueryResultsFormat::Json => write_boolean_json_result(write, value), + QueryResultsFormat::Csv => write_boolean_csv_result(write, value), + QueryResultsFormat::Tsv => write_boolean_tsv_result(write, value), } } + #[deprecated(note = "Use serialize_boolean_to_write")] + pub fn write_boolean_result(&self, writer: W, value: bool) -> io::Result { + self.serialize_boolean_to_write(writer, value) + } + /// Returns a `SolutionsWriter` allowing writing query solutions into the given [`Write`] implementation. /// + ///
Do not forget to run the [`finish`](ToWriteSolutionsWriter::finish()) method to properly write the last bytes of the file.
+ /// + ///
This writer does unbuffered writes. You might want to use [`BufWriter`](io::BufWriter) to avoid that.
+ /// /// Example in XML (the API is the same for JSON and TSV): /// ``` /// use sparesults::{QueryResultsFormat, QueryResultsSerializer}; @@ -78,40 +87,51 @@ impl QueryResultsSerializer { /// /// let json_serializer = QueryResultsSerializer::from_format(QueryResultsFormat::Xml); /// let mut buffer = Vec::new(); - /// let mut writer = json_serializer.solutions_writer(&mut buffer, vec![Variable::new_unchecked("foo"), Variable::new_unchecked("bar")])?; + /// let mut writer = json_serializer.serialize_solutions_to_write(&mut buffer, vec![Variable::new_unchecked("foo"), Variable::new_unchecked("bar")])?; /// writer.write(once((VariableRef::new_unchecked("foo"), LiteralRef::from("test"))))?; /// writer.finish()?; /// assert_eq!(buffer, b"test"); /// # std::io::Result::Ok(()) /// ``` - pub fn solutions_writer( + pub fn serialize_solutions_to_write( &self, - writer: W, + write: W, variables: Vec, - ) -> io::Result> { - Ok(SolutionsWriter { + ) -> io::Result> { + Ok(ToWriteSolutionsWriter { formatter: match self.format { QueryResultsFormat::Xml => { - SolutionsWriterKind::Xml(XmlSolutionsWriter::start(writer, &variables)?) + ToWriteSolutionsWriterKind::Xml(XmlSolutionsWriter::start(write, &variables)?) } QueryResultsFormat::Json => { - SolutionsWriterKind::Json(JsonSolutionsWriter::start(writer, &variables)?) + ToWriteSolutionsWriterKind::Json(JsonSolutionsWriter::start(write, &variables)?) } QueryResultsFormat::Csv => { - SolutionsWriterKind::Csv(CsvSolutionsWriter::start(writer, variables)?) + ToWriteSolutionsWriterKind::Csv(CsvSolutionsWriter::start(write, variables)?) } QueryResultsFormat::Tsv => { - SolutionsWriterKind::Tsv(TsvSolutionsWriter::start(writer, variables)?) + ToWriteSolutionsWriterKind::Tsv(TsvSolutionsWriter::start(write, variables)?) } }, }) } + + #[deprecated(note = "Use serialize_solutions_to_write")] + pub fn solutions_writer( + &self, + writer: W, + variables: Vec, + ) -> io::Result> { + self.serialize_solutions_to_write(writer, variables) + } } /// Allows writing query results. /// Could be built using a [`QueryResultsSerializer`]. /// -///
Do not forget to run the [`finish`](SolutionsWriter::finish()) method to properly write the last bytes of the file.
+///
Do not forget to run the [`finish`](ToWriteSolutionsWriter::finish()) method to properly write the last bytes of the file.
+/// +///
This writer does unbuffered writes. You might want to use [`BufWriter`](io::BufWriter) to avoid that.
/// /// Example in TSV (the API is the same for JSON and XML): /// ``` @@ -121,25 +141,25 @@ impl QueryResultsSerializer { /// /// let json_serializer = QueryResultsSerializer::from_format(QueryResultsFormat::Tsv); /// let mut buffer = Vec::new(); -/// let mut writer = json_serializer.solutions_writer(&mut buffer, vec![Variable::new_unchecked("foo"), Variable::new_unchecked("bar")])?; +/// let mut writer = json_serializer.serialize_solutions_to_write(&mut buffer, vec![Variable::new_unchecked("foo"), Variable::new_unchecked("bar")])?; /// writer.write(once((VariableRef::new_unchecked("foo"), LiteralRef::from("test"))))?; /// writer.finish()?; /// assert_eq!(buffer, b"?foo\t?bar\n\"test\"\t\n"); /// # std::io::Result::Ok(()) /// ``` #[must_use] -pub struct SolutionsWriter { - formatter: SolutionsWriterKind, +pub struct ToWriteSolutionsWriter { + formatter: ToWriteSolutionsWriterKind, } -enum SolutionsWriterKind { +enum ToWriteSolutionsWriterKind { Xml(XmlSolutionsWriter), Json(JsonSolutionsWriter), Csv(CsvSolutionsWriter), Tsv(TsvSolutionsWriter), } -impl SolutionsWriter { +impl ToWriteSolutionsWriter { /// Writes a solution. /// /// Example in JSON (the API is the same for XML and TSV): @@ -150,7 +170,7 @@ impl SolutionsWriter { /// /// let json_serializer = QueryResultsSerializer::from_format(QueryResultsFormat::Json); /// let mut buffer = Vec::new(); - /// let mut writer = json_serializer.solutions_writer(&mut buffer, vec![Variable::new_unchecked("foo"), Variable::new_unchecked("bar")])?; + /// let mut writer = json_serializer.serialize_solutions_to_write(&mut buffer, vec![Variable::new_unchecked("foo"), Variable::new_unchecked("bar")])?; /// writer.write(once((VariableRef::new_unchecked("foo"), LiteralRef::from("test"))))?; /// writer.write(&QuerySolution::from((vec![Variable::new_unchecked("bar")], vec![Some(Literal::from("test").into())])))?; /// writer.finish()?; @@ -163,20 +183,20 @@ impl SolutionsWriter { ) -> io::Result<()> { let solution = solution.into_iter().map(|(v, s)| (v.into(), s.into())); match &mut self.formatter { - SolutionsWriterKind::Xml(writer) => writer.write(solution), - SolutionsWriterKind::Json(writer) => writer.write(solution), - SolutionsWriterKind::Csv(writer) => writer.write(solution), - SolutionsWriterKind::Tsv(writer) => writer.write(solution), + ToWriteSolutionsWriterKind::Xml(writer) => writer.write(solution), + ToWriteSolutionsWriterKind::Json(writer) => writer.write(solution), + ToWriteSolutionsWriterKind::Csv(writer) => writer.write(solution), + ToWriteSolutionsWriterKind::Tsv(writer) => writer.write(solution), } } /// Writes the last bytes of the file. pub fn finish(self) -> io::Result { match self.formatter { - SolutionsWriterKind::Xml(write) => write.finish(), - SolutionsWriterKind::Json(write) => write.finish(), - SolutionsWriterKind::Csv(write) => write.finish(), - SolutionsWriterKind::Tsv(write) => write.finish(), + ToWriteSolutionsWriterKind::Xml(write) => write.finish(), + ToWriteSolutionsWriterKind::Json(write) => write.finish(), + ToWriteSolutionsWriterKind::Csv(write) => write.finish(), + ToWriteSolutionsWriterKind::Tsv(write) => write.finish(), } } } diff --git a/lib/src/sparql/model.rs b/lib/src/sparql/model.rs index 71a25ea9..d02f7ea2 100644 --- a/lib/src/sparql/model.rs +++ b/lib/src/sparql/model.rs @@ -58,12 +58,12 @@ impl QueryResults { match self { Self::Boolean(value) => { serializer - .write_boolean_result(writer, value) + .serialize_boolean_to_write(writer, value) .map_err(EvaluationError::ResultsSerialization)?; } Self::Solutions(solutions) => { let mut writer = serializer - .solutions_writer(writer, solutions.variables().to_vec()) + .serialize_solutions_to_write(writer, solutions.variables().to_vec()) .map_err(EvaluationError::ResultsSerialization)?; for solution in solutions { writer @@ -79,7 +79,10 @@ impl QueryResults { let p = VariableRef::new_unchecked("predicate"); let o = VariableRef::new_unchecked("object"); let mut writer = serializer - .solutions_writer(writer, vec![s.into_owned(), p.into_owned(), o.into_owned()]) + .serialize_solutions_to_write( + writer, + vec![s.into_owned(), p.into_owned(), o.into_owned()], + ) .map_err(EvaluationError::ResultsSerialization)?; for triple in triples { let triple = triple?; diff --git a/lib/src/sparql/results.rs b/lib/src/sparql/results.rs index 26fa287a..511461cc 100644 --- a/lib/src/sparql/results.rs +++ b/lib/src/sparql/results.rs @@ -15,15 +15,15 @@ //! match json_parser.read_results(json_file)? { //! QueryResultsReader::Boolean(value) => { //! // it's a boolean result, we copy it in TSV to the output buffer -//! tsv_serializer.write_boolean_result(Vec::new(), value) +//! tsv_serializer.serialize_boolean_to_write(Vec::new(), value) //! }, //! QueryResultsReader::Solutions(solutions_reader) => { //! // it's a set of solutions, we create a writer and we write to it while reading in streaming from the JSON file -//! let mut solutions_writer = tsv_serializer.solutions_writer(Vec::new(), solutions_reader.variables().to_vec())?; +//! let mut serialize_solutions_to_write = tsv_serializer.serialize_solutions_to_write(Vec::new(), solutions_reader.variables().to_vec())?; //! for solution in solutions_reader { -//! solutions_writer.write(&solution?)?; +//! serialize_solutions_to_write.write(&solution?)?; //! } -//! solutions_writer.finish() +//! serialize_solutions_to_write.finish() //! } //! } //! } diff --git a/python/src/sparql.rs b/python/src/sparql.rs index ef544e2d..a2964e0f 100644 --- a/python/src/sparql.rs +++ b/python/src/sparql.rs @@ -247,7 +247,7 @@ impl PyQuerySolutions { PyWritable::do_write( |output, format| { let mut writer = QueryResultsSerializer::from_format(format) - .solutions_writer( + .serialize_solutions_to_write( output, match &self.inner { PyQuerySolutionsVariant::Query(inner) => inner.variables().to_vec(), @@ -349,7 +349,7 @@ impl PyQueryBoolean { |output, format| { py.allow_threads(|| { QueryResultsSerializer::from_format(format) - .write_boolean_result(output, self.inner) + .serialize_boolean_to_write(output, self.inner) .map_err(map_io_err) }) }, From 7a3e07d98ddb2a828deac7c5d1a51423cdc7c53f Mon Sep 17 00:00:00 2001 From: Tpt Date: Sat, 16 Sep 2023 15:19:14 +0200 Subject: [PATCH 094/217] sparesults: no more hidden flush Let the user decide to do it or not --- lib/sparesults/src/csv.rs | 22 ++++++++++------------ lib/sparesults/src/json.rs | 4 +--- lib/sparesults/src/serializer.rs | 4 ++-- lib/sparesults/src/xml.rs | 4 +--- 4 files changed, 14 insertions(+), 20 deletions(-) diff --git a/lib/sparesults/src/csv.rs b/lib/sparesults/src/csv.rs index ff33ce30..72adac29 100644 --- a/lib/sparesults/src/csv.rs +++ b/lib/sparesults/src/csv.rs @@ -55,9 +55,8 @@ impl CsvSolutionsWriter { self.sink.write_all(b"\r\n") } - pub fn finish(mut self) -> io::Result { - self.sink.flush()?; - Ok(self.sink) + pub fn finish(self) -> W { + self.sink } } @@ -146,9 +145,8 @@ impl TsvSolutionsWriter { self.sink.write_all(b"\n") } - pub fn finish(mut self) -> io::Result { - self.sink.flush()?; - Ok(self.sink) + pub fn finish(self) -> W { + self.sink } } @@ -436,7 +434,7 @@ mod tests { .filter_map(|(v, s)| s.as_ref().map(|s| (v.as_ref(), s.as_ref()))), )?; } - let result = writer.finish()?; + let result = writer.finish(); assert_eq!(str::from_utf8(&result).unwrap(), "x,literal\r\nhttp://example/x,String\r\nhttp://example/x,\"String-with-dquote\"\"\"\r\n_:b0,Blank node\r\n,Missing 'x'\r\n,\r\nhttp://example/x,\r\n_:b1,String-with-lang\r\n_:b1,123\r\n,\"escape,\t\r\n\"\r\n"); Ok(()) } @@ -456,7 +454,7 @@ mod tests { .filter_map(|(v, s)| s.as_ref().map(|s| (v.as_ref(), s.as_ref()))), )?; } - let result = writer.finish()?; + let result = writer.finish(); assert_eq!(str::from_utf8(&result).unwrap(), "?x\t?literal\n\t\"String\"\n\t\"String-with-dquote\\\"\"\n_:b0\t\"Blank node\"\n\t\"Missing 'x'\"\n\t\n\t\n_:b1\t\"String-with-lang\"@en\n_:b1\t123\n\t\"escape,\\t\\r\\n\"\n"); // Read @@ -507,7 +505,7 @@ mod tests { fn test_no_columns_csv_serialization() -> io::Result<()> { let mut writer = CsvSolutionsWriter::start(Vec::new(), Vec::new())?; writer.write([])?; - let result = writer.finish()?; + let result = writer.finish(); assert_eq!(str::from_utf8(&result).unwrap(), "\r\n\r\n"); Ok(()) } @@ -516,7 +514,7 @@ mod tests { fn test_no_columns_tsv_serialization() -> io::Result<()> { let mut writer = TsvSolutionsWriter::start(Vec::new(), Vec::new())?; writer.write([])?; - let result = writer.finish()?; + let result = writer.finish(); assert_eq!(str::from_utf8(&result).unwrap(), "\n\n"); Ok(()) } @@ -540,7 +538,7 @@ mod tests { #[test] fn test_no_results_csv_serialization() -> io::Result<()> { let result = - CsvSolutionsWriter::start(Vec::new(), vec![Variable::new_unchecked("a")])?.finish()?; + CsvSolutionsWriter::start(Vec::new(), vec![Variable::new_unchecked("a")])?.finish(); assert_eq!(str::from_utf8(&result).unwrap(), "a\r\n"); Ok(()) } @@ -548,7 +546,7 @@ mod tests { #[test] fn test_no_results_tsv_serialization() -> io::Result<()> { let result = - TsvSolutionsWriter::start(Vec::new(), vec![Variable::new_unchecked("a")])?.finish()?; + TsvSolutionsWriter::start(Vec::new(), vec![Variable::new_unchecked("a")])?.finish(); assert_eq!(str::from_utf8(&result).unwrap(), "?a\n"); Ok(()) } diff --git a/lib/sparesults/src/json.rs b/lib/sparesults/src/json.rs index cacc4c59..67e2d494 100644 --- a/lib/sparesults/src/json.rs +++ b/lib/sparesults/src/json.rs @@ -67,9 +67,7 @@ impl JsonSolutionsWriter { self.writer.write_event(JsonEvent::EndArray)?; self.writer.write_event(JsonEvent::EndObject)?; self.writer.write_event(JsonEvent::EndObject)?; - let mut inner = self.writer.into_inner(); - inner.flush()?; - Ok(inner) + Ok(self.writer.into_inner()) } } diff --git a/lib/sparesults/src/serializer.rs b/lib/sparesults/src/serializer.rs index 2ce77f4d..6f5bc7dc 100644 --- a/lib/sparesults/src/serializer.rs +++ b/lib/sparesults/src/serializer.rs @@ -195,8 +195,8 @@ impl ToWriteSolutionsWriter { match self.formatter { ToWriteSolutionsWriterKind::Xml(write) => write.finish(), ToWriteSolutionsWriterKind::Json(write) => write.finish(), - ToWriteSolutionsWriterKind::Csv(write) => write.finish(), - ToWriteSolutionsWriterKind::Tsv(write) => write.finish(), + ToWriteSolutionsWriterKind::Csv(write) => Ok(write.finish()), + ToWriteSolutionsWriterKind::Tsv(write) => Ok(write.finish()), } } } diff --git a/lib/sparesults/src/xml.rs b/lib/sparesults/src/xml.rs index f7e0bb41..84e7e099 100644 --- a/lib/sparesults/src/xml.rs +++ b/lib/sparesults/src/xml.rs @@ -92,9 +92,7 @@ impl XmlSolutionsWriter { } pub fn finish(self) -> io::Result { - let mut inner = self.do_finish().map_err(map_xml_error)?; - inner.flush()?; - Ok(inner) + self.do_finish().map_err(map_xml_error) } fn do_finish(mut self) -> Result { From 412ca37b3c67d9b68c4a05c5c7a4214829f82861 Mon Sep 17 00:00:00 2001 From: Tpt Date: Sat, 16 Sep 2023 16:07:45 +0200 Subject: [PATCH 095/217] Makes sparesults parser API closer to oxrdfio --- fuzz/src/result_format.rs | 16 ++++----- lib/sparesults/README.md | 8 ++--- lib/sparesults/src/lib.rs | 2 +- lib/sparesults/src/parser.rs | 63 +++++++++++++++++++++--------------- lib/src/sparql/model.rs | 18 +++++------ lib/src/sparql/results.rs | 14 ++++---- python/src/sparql.rs | 11 ++++--- 7 files changed, 72 insertions(+), 60 deletions(-) diff --git a/fuzz/src/result_format.rs b/fuzz/src/result_format.rs index 139d3c9a..5c6b59ae 100644 --- a/fuzz/src/result_format.rs +++ b/fuzz/src/result_format.rs @@ -1,17 +1,17 @@ use anyhow::Context; use sparesults::{ - QueryResultsFormat, QueryResultsParser, QueryResultsReader, QueryResultsSerializer, + FromReadQueryResultsReader, QueryResultsFormat, QueryResultsParser, QueryResultsSerializer, }; pub fn fuzz_result_format(format: QueryResultsFormat, data: &[u8]) { let parser = QueryResultsParser::from_format(format); let serializer = QueryResultsSerializer::from_format(format); - let Ok(reader) = parser.read_results(data) else { + let Ok(reader) = parser.parse_read(data) else { return; }; match reader { - QueryResultsReader::Solutions(solutions) => { + FromReadQueryResultsReader::Solutions(solutions) => { let Ok(solutions) = solutions.collect::, _>>() else { return; }; @@ -31,8 +31,8 @@ pub fn fuzz_result_format(format: QueryResultsFormat, data: &[u8]) { let serialized = String::from_utf8(writer.finish().unwrap()).unwrap(); // And to parse again - if let QueryResultsReader::Solutions(roundtrip_solutions) = parser - .read_results(serialized.as_bytes()) + if let FromReadQueryResultsReader::Solutions(roundtrip_solutions) = parser + .parse_read(serialized.as_bytes()) .with_context(|| format!("Parsing {:?}", &serialized)) .unwrap() { @@ -45,7 +45,7 @@ pub fn fuzz_result_format(format: QueryResultsFormat, data: &[u8]) { ) } } - QueryResultsReader::Boolean(value) => { + FromReadQueryResultsReader::Boolean(value) => { // We try to write again let mut serialized = Vec::new(); serializer @@ -53,8 +53,8 @@ pub fn fuzz_result_format(format: QueryResultsFormat, data: &[u8]) { .unwrap(); // And to parse again - if let QueryResultsReader::Boolean(roundtrip_value) = - parser.read_results(serialized.as_slice()).unwrap() + if let FromReadQueryResultsReader::Boolean(roundtrip_value) = + parser.parse_read(serialized.as_slice()).unwrap() { assert_eq!(roundtrip_value, value) } diff --git a/lib/sparesults/README.md b/lib/sparesults/README.md index f733d37b..0c61ded1 100644 --- a/lib/sparesults/README.md +++ b/lib/sparesults/README.md @@ -18,19 +18,19 @@ This crate is intended to be a building piece for SPARQL client and server imple Usage example converting a JSON result file into a TSV result file: ```rust -use sparesults::{QueryResultsFormat, QueryResultsParser, QueryResultsReader, QueryResultsSerializer}; +use sparesults::{QueryResultsFormat, QueryResultsParser, FromReadQueryResultsReader, QueryResultsSerializer}; use std::io::Result; fn convert_json_to_tsv(json_file: &[u8]) -> Result> { let json_parser = QueryResultsParser::from_format(QueryResultsFormat::Json); let tsv_serializer = QueryResultsSerializer::from_format(QueryResultsFormat::Tsv); // We start to read the JSON file and see which kind of results it is - match json_parser.read_results(json_file)? { - QueryResultsReader::Boolean(value) => { + match json_parser.parse_read(json_file)? { + FromReadQueryResultsReader::Boolean(value) => { // it's a boolean result, we copy it in TSV to the output buffer tsv_serializer.serialize_boolean_to_write(Vec::new(), value) }, - QueryResultsReader::Solutions(solutions_reader) => { + FromReadQueryResultsReader::Solutions(solutions_reader) => { // it's a set of solutions, we create a writer and we write to it while reading in streaming from the JSON file let mut serialize_solutions_to_write = tsv_serializer.serialize_solutions_to_write(Vec::new(), solutions_reader.variables().to_vec())?; for solution in solutions_reader { diff --git a/lib/sparesults/src/lib.rs b/lib/sparesults/src/lib.rs index 21cdce12..95bcef93 100644 --- a/lib/sparesults/src/lib.rs +++ b/lib/sparesults/src/lib.rs @@ -15,6 +15,6 @@ mod xml; pub use crate::error::{ParseError, SyntaxError}; pub use crate::format::QueryResultsFormat; -pub use crate::parser::{QueryResultsParser, QueryResultsReader, SolutionsReader}; +pub use crate::parser::{FromReadQueryResultsReader, FromReadSolutionsReader, QueryResultsParser}; pub use crate::serializer::{QueryResultsSerializer, ToWriteSolutionsWriter}; pub use crate::solution::QuerySolution; diff --git a/lib/sparesults/src/parser.rs b/lib/sparesults/src/parser.rs index 32ffd62e..fd7a403d 100644 --- a/lib/sparesults/src/parser.rs +++ b/lib/sparesults/src/parser.rs @@ -17,16 +17,16 @@ use std::rc::Rc; /// /// Example in JSON (the API is the same for XML and TSV): /// ``` -/// use sparesults::{QueryResultsFormat, QueryResultsParser, QueryResultsReader}; +/// use sparesults::{QueryResultsFormat, QueryResultsParser, FromReadQueryResultsReader}; /// use oxrdf::{Literal, Variable}; /// /// let json_parser = QueryResultsParser::from_format(QueryResultsFormat::Json); /// // boolean -/// if let QueryResultsReader::Boolean(v) = json_parser.read_results(b"{\"boolean\":true}".as_slice())? { +/// if let FromReadQueryResultsReader::Boolean(v) = json_parser.parse_read(b"{\"boolean\":true}".as_slice())? { /// assert_eq!(v, true); /// } /// // solutions -/// if let QueryResultsReader::Solutions(solutions) = json_parser.read_results(b"{\"head\":{\"vars\":[\"foo\",\"bar\"]},\"results\":{\"bindings\":[{\"foo\":{\"type\":\"literal\",\"value\":\"test\"}}]}}".as_slice())? { +/// if let FromReadQueryResultsReader::Solutions(solutions) = json_parser.parse_read(b"{\"head\":{\"vars\":[\"foo\",\"bar\"]},\"results\":{\"bindings\":[{\"foo\":{\"type\":\"literal\",\"value\":\"test\"}}]}}".as_slice())? { /// assert_eq!(solutions.variables(), &[Variable::new_unchecked("foo"), Variable::new_unchecked("bar")]); /// for solution in solutions { /// assert_eq!(solution?.iter().collect::>(), vec![(&Variable::new_unchecked("foo"), &Literal::from("test").into())]); @@ -49,18 +49,18 @@ impl QueryResultsParser { /// /// Example in XML (the API is the same for JSON and TSV): /// ``` - /// use sparesults::{QueryResultsFormat, QueryResultsParser, QueryResultsReader}; + /// use sparesults::{QueryResultsFormat, QueryResultsParser, FromReadQueryResultsReader}; /// use oxrdf::{Literal, Variable}; /// /// let json_parser = QueryResultsParser::from_format(QueryResultsFormat::Xml); /// /// // boolean - /// if let QueryResultsReader::Boolean(v) = json_parser.read_results(b"true".as_slice())? { + /// if let FromReadQueryResultsReader::Boolean(v) = json_parser.parse_read(b"true".as_slice())? { /// assert_eq!(v, true); /// } /// /// // solutions - /// if let QueryResultsReader::Solutions(solutions) = json_parser.read_results(b"test".as_slice())? { + /// if let FromReadQueryResultsReader::Solutions(solutions) = json_parser.parse_read(b"test".as_slice())? { /// assert_eq!(solutions.variables(), &[Variable::new_unchecked("foo"), Variable::new_unchecked("bar")]); /// for solution in solutions { /// assert_eq!(solution?.iter().collect::>(), vec![(&Variable::new_unchecked("foo"), &Literal::from("test").into())]); @@ -68,61 +68,72 @@ impl QueryResultsParser { /// } /// # Result::<(),sparesults::ParseError>::Ok(()) /// ``` - pub fn read_results(&self, reader: R) -> Result, ParseError> { + pub fn parse_read( + &self, + reader: R, + ) -> Result, ParseError> { Ok(match self.format { QueryResultsFormat::Xml => match XmlQueryResultsReader::read(reader)? { - XmlQueryResultsReader::Boolean(r) => QueryResultsReader::Boolean(r), + XmlQueryResultsReader::Boolean(r) => FromReadQueryResultsReader::Boolean(r), XmlQueryResultsReader::Solutions { solutions, variables, - } => QueryResultsReader::Solutions(SolutionsReader { + } => FromReadQueryResultsReader::Solutions(FromReadSolutionsReader { variables: Rc::new(variables), solutions: SolutionsReaderKind::Xml(solutions), }), }, QueryResultsFormat::Json => match JsonQueryResultsReader::read(reader)? { - JsonQueryResultsReader::Boolean(r) => QueryResultsReader::Boolean(r), + JsonQueryResultsReader::Boolean(r) => FromReadQueryResultsReader::Boolean(r), JsonQueryResultsReader::Solutions { solutions, variables, - } => QueryResultsReader::Solutions(SolutionsReader { + } => FromReadQueryResultsReader::Solutions(FromReadSolutionsReader { variables: Rc::new(variables), solutions: SolutionsReaderKind::Json(solutions), }), }, QueryResultsFormat::Csv => return Err(SyntaxError::msg("CSV SPARQL results syntax is lossy and can't be parsed to a proper RDF representation").into()), QueryResultsFormat::Tsv => match TsvQueryResultsReader::read(reader)? { - TsvQueryResultsReader::Boolean(r) => QueryResultsReader::Boolean(r), + TsvQueryResultsReader::Boolean(r) => FromReadQueryResultsReader::Boolean(r), TsvQueryResultsReader::Solutions { solutions, variables, - } => QueryResultsReader::Solutions(SolutionsReader { + } => FromReadQueryResultsReader::Solutions(FromReadSolutionsReader { variables: Rc::new(variables), solutions: SolutionsReaderKind::Tsv(solutions), }), }, }) } + + #[deprecated(note = "Use parse_read")] + pub fn read_results( + &self, + reader: R, + ) -> Result, ParseError> { + self.parse_read(reader) + } } /// The reader for a given read of a results file. /// -/// It is either a read boolean ([`bool`]) or a streaming reader of a set of solutions ([`SolutionsReader`]). +/// It is either a read boolean ([`bool`]) or a streaming reader of a set of solutions ([`FromReadSolutionsReader`]). /// /// Example in TSV (the API is the same for JSON and XML): /// ``` -/// use sparesults::{QueryResultsFormat, QueryResultsParser, QueryResultsReader}; +/// use sparesults::{QueryResultsFormat, QueryResultsParser, FromReadQueryResultsReader}; /// use oxrdf::{Literal, Variable}; /// /// let json_parser = QueryResultsParser::from_format(QueryResultsFormat::Tsv); /// /// // boolean -/// if let QueryResultsReader::Boolean(v) = json_parser.read_results(b"true".as_slice())? { +/// if let FromReadQueryResultsReader::Boolean(v) = json_parser.parse_read(b"true".as_slice())? { /// assert_eq!(v, true); /// } /// /// // solutions -/// if let QueryResultsReader::Solutions(solutions) = json_parser.read_results(b"?foo\t?bar\n\"test\"\t".as_slice())? { +/// if let FromReadQueryResultsReader::Solutions(solutions) = json_parser.parse_read(b"?foo\t?bar\n\"test\"\t".as_slice())? { /// assert_eq!(solutions.variables(), &[Variable::new_unchecked("foo"), Variable::new_unchecked("bar")]); /// for solution in solutions { /// assert_eq!(solution?.iter().collect::>(), vec![(&Variable::new_unchecked("foo"), &Literal::from("test").into())]); @@ -130,8 +141,8 @@ impl QueryResultsParser { /// } /// # Result::<(),sparesults::ParseError>::Ok(()) /// ``` -pub enum QueryResultsReader { - Solutions(SolutionsReader), +pub enum FromReadQueryResultsReader { + Solutions(FromReadSolutionsReader), Boolean(bool), } @@ -141,11 +152,11 @@ pub enum QueryResultsReader { /// /// Example in JSON (the API is the same for XML and TSV): /// ``` -/// use sparesults::{QueryResultsFormat, QueryResultsParser, QueryResultsReader}; +/// use sparesults::{QueryResultsFormat, QueryResultsParser, FromReadQueryResultsReader}; /// use oxrdf::{Literal, Variable}; /// /// let json_parser = QueryResultsParser::from_format(QueryResultsFormat::Json); -/// if let QueryResultsReader::Solutions(solutions) = json_parser.read_results(b"{\"head\":{\"vars\":[\"foo\",\"bar\"]},\"results\":{\"bindings\":[{\"foo\":{\"type\":\"literal\",\"value\":\"test\"}}]}}".as_slice())? { +/// if let FromReadQueryResultsReader::Solutions(solutions) = json_parser.parse_read(b"{\"head\":{\"vars\":[\"foo\",\"bar\"]},\"results\":{\"bindings\":[{\"foo\":{\"type\":\"literal\",\"value\":\"test\"}}]}}".as_slice())? { /// assert_eq!(solutions.variables(), &[Variable::new_unchecked("foo"), Variable::new_unchecked("bar")]); /// for solution in solutions { /// assert_eq!(solution?.iter().collect::>(), vec![(&Variable::new_unchecked("foo"), &Literal::from("test").into())]); @@ -154,7 +165,7 @@ pub enum QueryResultsReader { /// # Result::<(),sparesults::ParseError>::Ok(()) /// ``` #[allow(clippy::rc_buffer)] -pub struct SolutionsReader { +pub struct FromReadSolutionsReader { variables: Rc>, solutions: SolutionsReaderKind, } @@ -165,16 +176,16 @@ enum SolutionsReaderKind { Tsv(TsvSolutionsReader), } -impl SolutionsReader { +impl FromReadSolutionsReader { /// Ordered list of the declared variables at the beginning of the results. /// /// Example in TSV (the API is the same for JSON and XML): /// ``` - /// use sparesults::{QueryResultsFormat, QueryResultsParser, QueryResultsReader}; + /// use sparesults::{QueryResultsFormat, QueryResultsParser, FromReadQueryResultsReader}; /// use oxrdf::Variable; /// /// let json_parser = QueryResultsParser::from_format(QueryResultsFormat::Tsv); - /// if let QueryResultsReader::Solutions(solutions) = json_parser.read_results(b"?foo\t?bar\n\"ex1\"\t\"ex2\"".as_slice())? { + /// if let FromReadQueryResultsReader::Solutions(solutions) = json_parser.parse_read(b"?foo\t?bar\n\"ex1\"\t\"ex2\"".as_slice())? { /// assert_eq!(solutions.variables(), &[Variable::new_unchecked("foo"), Variable::new_unchecked("bar")]); /// } /// # Result::<(),sparesults::ParseError>::Ok(()) @@ -185,7 +196,7 @@ impl SolutionsReader { } } -impl Iterator for SolutionsReader { +impl Iterator for FromReadSolutionsReader { type Item = Result; fn next(&mut self) -> Option> { diff --git a/lib/src/sparql/model.rs b/lib/src/sparql/model.rs index d02f7ea2..8ef06ecb 100644 --- a/lib/src/sparql/model.rs +++ b/lib/src/sparql/model.rs @@ -2,8 +2,8 @@ use crate::io::{RdfFormat, RdfSerializer}; use crate::model::*; use crate::sparql::error::EvaluationError; use crate::sparql::results::{ - ParseError, QueryResultsFormat, QueryResultsParser, QueryResultsReader, QueryResultsSerializer, - SolutionsReader, + FromReadQueryResultsReader, FromReadSolutionsReader, ParseError, QueryResultsFormat, + QueryResultsParser, QueryResultsSerializer, }; use oxrdf::{Variable, VariableRef}; pub use sparesults::QuerySolution; @@ -27,7 +27,7 @@ impl QueryResults { format: QueryResultsFormat, ) -> Result { Ok(QueryResultsParser::from_format(format) - .read_results(reader)? + .parse_read(reader)? .into()) } @@ -150,11 +150,11 @@ impl From for QueryResults { } } -impl From> for QueryResults { - fn from(reader: QueryResultsReader) -> Self { +impl From> for QueryResults { + fn from(reader: FromReadQueryResultsReader) -> Self { match reader { - QueryResultsReader::Solutions(s) => Self::Solutions(s.into()), - QueryResultsReader::Boolean(v) => Self::Boolean(v), + FromReadQueryResultsReader::Solutions(s) => Self::Solutions(s.into()), + FromReadQueryResultsReader::Boolean(v) => Self::Boolean(v), } } } @@ -211,8 +211,8 @@ impl QuerySolutionIter { } } -impl From> for QuerySolutionIter { - fn from(reader: SolutionsReader) -> Self { +impl From> for QuerySolutionIter { + fn from(reader: FromReadSolutionsReader) -> Self { Self { variables: Rc::new(reader.variables().to_vec()), iter: Box::new(reader.map(|t| t.map_err(EvaluationError::from))), diff --git a/lib/src/sparql/results.rs b/lib/src/sparql/results.rs index 511461cc..0716752b 100644 --- a/lib/src/sparql/results.rs +++ b/lib/src/sparql/results.rs @@ -5,19 +5,19 @@ //! Usage example converting a JSON result file into a TSV result file: //! //! ``` -//! use oxigraph::sparql::results::{QueryResultsFormat, QueryResultsParser, QueryResultsReader, QueryResultsSerializer}; +//! use oxigraph::sparql::results::{QueryResultsFormat, QueryResultsParser, FromReadQueryResultsReader, QueryResultsSerializer}; //! use std::io::Result; //! //! fn convert_json_to_tsv(json_file: &[u8]) -> Result> { //! let json_parser = QueryResultsParser::from_format(QueryResultsFormat::Json); //! let tsv_serializer = QueryResultsSerializer::from_format(QueryResultsFormat::Tsv); //! // We start to read the JSON file and see which kind of results it is -//! match json_parser.read_results(json_file)? { -//! QueryResultsReader::Boolean(value) => { +//! match json_parser.parse_read(json_file)? { +//! FromReadQueryResultsReader::Boolean(value) => { //! // it's a boolean result, we copy it in TSV to the output buffer //! tsv_serializer.serialize_boolean_to_write(Vec::new(), value) -//! }, -//! QueryResultsReader::Solutions(solutions_reader) => { +//! } +//! FromReadQueryResultsReader::Solutions(solutions_reader) => { //! // it's a set of solutions, we create a writer and we write to it while reading in streaming from the JSON file //! let mut serialize_solutions_to_write = tsv_serializer.serialize_solutions_to_write(Vec::new(), solutions_reader.variables().to_vec())?; //! for solution in solutions_reader { @@ -42,6 +42,6 @@ //! ``` pub use sparesults::{ - ParseError, QueryResultsFormat, QueryResultsParser, QueryResultsReader, QueryResultsSerializer, - SolutionsReader, SyntaxError, + FromReadQueryResultsReader, FromReadSolutionsReader, ParseError, QueryResultsFormat, + QueryResultsParser, QueryResultsSerializer, SyntaxError, }; diff --git a/python/src/sparql.rs b/python/src/sparql.rs index a2964e0f..1316eecf 100644 --- a/python/src/sparql.rs +++ b/python/src/sparql.rs @@ -4,7 +4,8 @@ use crate::store::map_storage_error; use oxigraph::io::RdfSerializer; use oxigraph::model::Term; use oxigraph::sparql::results::{ - ParseError, QueryResultsParser, QueryResultsReader, QueryResultsSerializer, SolutionsReader, + FromReadQueryResultsReader, FromReadSolutionsReader, ParseError, QueryResultsParser, + QueryResultsSerializer, }; use oxigraph::sparql::{ EvaluationError, Query, QueryResults, QuerySolution, QuerySolutionIter, QueryTripleIter, @@ -189,7 +190,7 @@ pub struct PyQuerySolutions { } enum PyQuerySolutionsVariant { Query(QuerySolutionIter), - Reader(SolutionsReader>), + Reader(FromReadSolutionsReader>), } #[pymethods] @@ -496,14 +497,14 @@ pub fn parse_query_results( PyReadable::from_data(input) }; let results = QueryResultsParser::from_format(format) - .read_results(BufReader::new(input)) + .parse_read(BufReader::new(input)) .map_err(map_query_results_parse_error)?; Ok(match results { - QueryResultsReader::Solutions(inner) => PyQuerySolutions { + FromReadQueryResultsReader::Solutions(inner) => PyQuerySolutions { inner: PyQuerySolutionsVariant::Reader(inner), } .into_py(py), - QueryResultsReader::Boolean(inner) => PyQueryBoolean { inner }.into_py(py), + FromReadQueryResultsReader::Boolean(inner) => PyQueryBoolean { inner }.into_py(py), }) } From 67fd726f9d8eb2766aba1a93ff570e99388181ca Mon Sep 17 00:00:00 2001 From: Tpt Date: Sat, 16 Sep 2023 16:11:22 +0200 Subject: [PATCH 096/217] Makes SPARQL results TSV work with a Read implementation --- Cargo.lock | 1 + lib/sparesults/Cargo.toml | 1 + lib/sparesults/src/csv.rs | 100 +++++++++++++++++++++++++++--------- lib/sparesults/src/error.rs | 12 +++-- 4 files changed, 84 insertions(+), 30 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a958c167..93c81493 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1673,6 +1673,7 @@ name = "sparesults" version = "0.2.0-alpha.1-dev" dependencies = [ "json-event-parser", + "memchr", "oxrdf", "quick-xml", ] diff --git a/lib/sparesults/Cargo.toml b/lib/sparesults/Cargo.toml index 0a7904db..3fa9e27e 100644 --- a/lib/sparesults/Cargo.toml +++ b/lib/sparesults/Cargo.toml @@ -20,6 +20,7 @@ rdf-star = ["oxrdf/rdf-star"] [dependencies] json-event-parser = "0.1" +memchr = "2.5" oxrdf = { version = "0.2.0-alpha.1-dev", path="../oxrdf" } quick-xml = ">=0.29, <0.31" diff --git a/lib/sparesults/src/csv.rs b/lib/sparesults/src/csv.rs index 72adac29..6fbfeb40 100644 --- a/lib/sparesults/src/csv.rs +++ b/lib/sparesults/src/csv.rs @@ -1,10 +1,13 @@ //! Implementation of [SPARQL 1.1 Query Results CSV and TSV Formats](https://www.w3.org/TR/sparql11-results-csv-tsv/) use crate::error::{ParseError, SyntaxError, SyntaxErrorKind}; +use memchr::memchr; use oxrdf::Variable; use oxrdf::{vocab::xsd, *}; -use std::io::{self, BufRead, Write}; -use std::str::FromStr; +use std::io::{self, BufRead, Read, Write}; +use std::str::{self, FromStr}; + +const MAX_BUFFER_SIZE: usize = 4096 * 4096; pub fn write_boolean_csv_result(mut sink: W, value: bool) -> io::Result { sink.write_all(if value { b"true" } else { b"false" })?; @@ -271,7 +274,7 @@ fn is_turtle_double(value: &str) -> bool { (with_before || with_after) && !value.is_empty() && value.iter().all(u8::is_ascii_digit) } -pub enum TsvQueryResultsReader { +pub enum TsvQueryResultsReader { Solutions { variables: Vec, solutions: TsvSolutionsReader, @@ -279,14 +282,13 @@ pub enum TsvQueryResultsReader { Boolean(bool), } -impl TsvQueryResultsReader { - pub fn read(mut source: R) -> Result { - let mut buffer = String::new(); +impl TsvQueryResultsReader { + pub fn read(read: R) -> Result { + let mut reader = LineReader::new(read); // We read the header - source.read_line(&mut buffer)?; - let line = buffer - .as_str() + let line = reader + .next_line()? .trim_matches(|c| matches!(c, ' ' | '\r' | '\n')); if line.eq_ignore_ascii_case("true") { return Ok(Self::Boolean(true)); @@ -316,29 +318,23 @@ impl TsvQueryResultsReader { let column_len = variables.len(); Ok(Self::Solutions { variables, - solutions: TsvSolutionsReader { - source, - buffer, - column_len, - }, + solutions: TsvSolutionsReader { reader, column_len }, }) } } -pub struct TsvSolutionsReader { - source: R, - buffer: String, +pub struct TsvSolutionsReader { + reader: LineReader, column_len: usize, } impl TsvSolutionsReader { pub fn read_next(&mut self) -> Result>>, ParseError> { - self.buffer.clear(); - if self.source.read_line(&mut self.buffer)? == 0 { - return Ok(None); + let line = self.reader.next_line()?; + if line.is_empty() { + return Ok(None); // EOF } - let elements = self - .buffer + let elements = line .split('\t') .map(|v| { let v = v.trim(); @@ -346,7 +342,10 @@ impl TsvSolutionsReader { Ok(None) } else { Ok(Some(Term::from_str(v).map_err(|e| SyntaxError { - inner: SyntaxErrorKind::Term(e), + inner: SyntaxErrorKind::Term { + error: e, + term: v.into(), + }, })?)) } }) @@ -357,16 +356,67 @@ impl TsvSolutionsReader { Ok(Some(Vec::new())) // Zero columns case } else { Err(SyntaxError::msg(format!( - "This TSV files has {} columns but we found a row with {} columns: {:?}", + "This TSV files has {} columns but we found a row with {} columns: {}", self.column_len, elements.len(), - self.buffer + line )) .into()) } } } +struct LineReader { + read: R, + buffer: Vec, + start: usize, + end: usize, +} + +impl LineReader { + fn new(read: R) -> Self { + Self { + read, + buffer: Vec::new(), + start: 0, + end: 0, + } + } + + fn next_line(&mut self) -> io::Result<&str> { + self.buffer.copy_within(self.start..self.end, 0); + self.end -= self.start; + self.start = 0; + let line_end = loop { + if let Some(eol) = memchr(b'\n', &self.buffer[self.start..self.end]) { + break self.start + eol + 1; + } + if self.end + 1024 > self.buffer.len() { + if self.end + 1024 > MAX_BUFFER_SIZE { + return Err(io::Error::new( + io::ErrorKind::OutOfMemory, + format!("Reached the buffer maximal size of {MAX_BUFFER_SIZE}"), + )); + } + self.buffer.resize(self.end + 1024, b'\0'); + } + let read = self.read.read(&mut self.buffer[self.end..])?; + if read == 0 { + break self.end; + } + self.end += read; + }; + let result = str::from_utf8(&self.buffer[self.start..line_end]).map_err(|e| { + io::Error::new( + io::ErrorKind::InvalidData, + format!("Invalid UTF-8 in the TSV file: {e}"), + ) + }); + self.start = line_end; + result + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/lib/sparesults/src/error.rs b/lib/sparesults/src/error.rs index d150e847..40510663 100644 --- a/lib/sparesults/src/error.rs +++ b/lib/sparesults/src/error.rs @@ -78,9 +78,9 @@ pub struct SyntaxError { } #[derive(Debug)] -pub enum SyntaxErrorKind { +pub(crate) enum SyntaxErrorKind { Xml(quick_xml::Error), - Term(TermParseError), + Term { error: TermParseError, term: String }, Msg { msg: String }, } @@ -99,7 +99,7 @@ impl fmt::Display for SyntaxError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match &self.inner { SyntaxErrorKind::Xml(e) => e.fmt(f), - SyntaxErrorKind::Term(e) => e.fmt(f), + SyntaxErrorKind::Term { error, term } => write!(f, "{error}: {term}"), SyntaxErrorKind::Msg { msg } => f.write_str(msg), } } @@ -110,7 +110,7 @@ impl Error for SyntaxError { fn source(&self) -> Option<&(dyn Error + 'static)> { match &self.inner { SyntaxErrorKind::Xml(e) => Some(e), - SyntaxErrorKind::Term(e) => Some(e), + SyntaxErrorKind::Term { error, .. } => Some(error), SyntaxErrorKind::Msg { .. } => None, } } @@ -130,7 +130,9 @@ impl From for io::Error { } _ => Self::new(io::ErrorKind::InvalidData, error), }, - SyntaxErrorKind::Term(error) => Self::new(io::ErrorKind::InvalidData, error), + SyntaxErrorKind::Term { .. } => { + Self::new(io::ErrorKind::InvalidData, error.to_string()) + } SyntaxErrorKind::Msg { msg } => Self::new(io::ErrorKind::InvalidData, msg), } } From 108721624ff46183404927d867b8db4de98d2869 Mon Sep 17 00:00:00 2001 From: Tpt Date: Sat, 16 Sep 2023 18:51:38 +0200 Subject: [PATCH 097/217] Improves bulk loader doc comments --- lib/src/store.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/src/store.rs b/lib/src/store.rs index a6215707..fa1a243f 100644 --- a/lib/src/store.rs +++ b/lib/src/store.rs @@ -1452,9 +1452,9 @@ impl BulkLoader { /// /// This function is optimized for large dataset loading speed. For small files, [`Store::load_dataset`] might be more convenient. /// - ///
This method is not atomic.
+ ///
This method is not atomic. /// If the parsing fails in the middle of the file, only a part of it may be written to the store. - /// Results might get weird if you delete data during the loading process. + /// Results might get weird if you delete data during the loading process.
/// ///
This method is optimized for speed. See [the struct](BulkLoader) documentation for more details.
/// @@ -1510,9 +1510,9 @@ impl BulkLoader { /// /// This function is optimized for large graph loading speed. For small files, [`Store::load_graph`] might be more convenient. /// - ///
This method is not atomic.
+ ///
This method is not atomic. /// If the parsing fails in the middle of the file, only a part of it may be written to the store. - /// Results might get weird if you delete data during the loading process. + /// Results might get weird if you delete data during the loading process.
/// ///
This method is optimized for speed. See [the struct](BulkLoader) documentation for more details.
/// @@ -1570,9 +1570,9 @@ impl BulkLoader { /// Adds a set of quads using the bulk loader. /// - ///
This method is not atomic.
+ ///
This method is not atomic. /// If the process fails in the middle of the file, only a part of the data may be written to the store. - /// Results might get weird if you delete data during the loading process. + /// Results might get weird if you delete data during the loading process.
/// ///
This method is optimized for speed. See [the struct](BulkLoader) documentation for more details.
pub fn load_quads( @@ -1584,9 +1584,9 @@ impl BulkLoader { /// Adds a set of quads using the bulk loader while breaking in the middle of the process in case of error. /// - ///
This method is not atomic.
+ ///
This method is not atomic. /// If the process fails in the middle of the file, only a part of the data may be written to the store. - /// Results might get weird if you delete data during the loading process. + /// Results might get weird if you delete data during the loading process.
/// ///
This method is optimized for speed. See [the struct](BulkLoader) documentation for more details.
pub fn load_ok_quads + From>( From 9b985295ae6366b96ef7986cf1bc7df50e6ee087 Mon Sep 17 00:00:00 2001 From: Tpt Date: Thu, 21 Sep 2023 08:29:41 +0200 Subject: [PATCH 098/217] Drops Musl linux 1.1 support Follows Rust move --- .github/workflows/artifacts.yml | 2 +- .github/workflows/musllinux_build.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/artifacts.yml b/.github/workflows/artifacts.yml index 3951704e..c0dd2abf 100644 --- a/.github/workflows/artifacts.yml +++ b/.github/workflows/artifacts.yml @@ -152,7 +152,7 @@ jobs: if: github.event_name == 'release' && matrix.architecture != 'x86_64' - uses: Swatinem/rust-cache@v2 - run: sed 's/%arch%/${{ matrix.architecture }}/g' .github/workflows/musllinux_build.sh | sed 's/%for_each_version%/${{ github.event_name == 'release' || '' }}/g' > .github/workflows/musllinux_build_script.sh - - run: docker run -v "$(pwd)":/workdir --platform linux/${{ matrix.architecture }} quay.io/pypa/musllinux_1_1_${{ matrix.architecture }} /bin/bash /workdir/.github/workflows/musllinux_build_script.sh + - run: docker run -v "$(pwd)":/workdir --platform linux/${{ matrix.architecture }} quay.io/pypa/musllinux_1_2_${{ matrix.architecture }} /bin/bash /workdir/.github/workflows/musllinux_build_script.sh if: github.event_name == 'release' || matrix.architecture == 'x86_64' - uses: actions/upload-artifact@v3 with: diff --git a/.github/workflows/musllinux_build.sh b/.github/workflows/musllinux_build.sh index 8abc5b0d..674be63a 100644 --- a/.github/workflows/musllinux_build.sh +++ b/.github/workflows/musllinux_build.sh @@ -11,9 +11,9 @@ source venv/bin/activate pip install -r requirements.dev.txt maturin develop --release python generate_stubs.py pyoxigraph pyoxigraph.pyi --black -maturin build --release --features abi3 --compatibility musllinux_1_1 +maturin build --release --features abi3 --compatibility musllinux_1_2 if [ %for_each_version% ]; then for VERSION in 7 8 9 10 11; do - maturin build --release --interpreter "python3.$VERSION" --compatibility musllinux_1_1 + maturin build --release --interpreter "python3.$VERSION" --compatibility musllinux_1_2 done fi From 90b7b128f25b1a4861d3c034b256ce3bce704815 Mon Sep 17 00:00:00 2001 From: Tpt Date: Sun, 24 Sep 2023 21:27:57 +0200 Subject: [PATCH 099/217] Upgrades MSRV to 1.70 --- .github/workflows/tests.yml | 4 +-- Cargo.lock | 2 -- Cargo.toml | 2 +- cli/Cargo.toml | 2 +- fuzz/Cargo.toml | 1 - fuzz/fuzz_targets/sparql_eval.rs | 15 +++++------ lib/Cargo.toml | 1 - lib/oxsdatatypes/src/double.rs | 2 +- lib/oxsdatatypes/src/float.rs | 2 +- lib/src/storage/backend/rocksdb.rs | 41 +++++++++++++----------------- python/src/io.rs | 14 ++++++---- testsuite/Cargo.toml | 3 +-- testsuite/src/sparql_evaluator.rs | 17 +++++++------ 13 files changed, 50 insertions(+), 56 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 03c038bb..953b9195 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -123,7 +123,7 @@ jobs: - uses: actions/checkout@v3 with: submodules: true - - run: rustup update && rustup toolchain install nightly && rustup default 1.67.0 + - run: rustup update && rustup toolchain install nightly && rustup default 1.70.0 - uses: Swatinem/rust-cache@v2 - run: rm Cargo.lock && cargo +nightly update -Z direct-minimal-versions - run: cargo test @@ -253,7 +253,7 @@ jobs: - uses: actions/checkout@v3 with: submodules: true - - run: rustup update && rustup toolchain install nightly && rustup default 1.67.0 + - run: rustup update && rustup toolchain install nightly && rustup default 1.70.0 - uses: Swatinem/rust-cache@v2 - uses: actions/setup-python@v4 with: diff --git a/Cargo.lock b/Cargo.lock index 93c81493..8fea3245 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -960,7 +960,6 @@ dependencies = [ "hex", "js-sys", "json-event-parser", - "lazy_static", "libc", "md-5", "oxhttp", @@ -1017,7 +1016,6 @@ dependencies = [ "anyhow", "clap", "criterion", - "lazy_static", "oxigraph", "oxttl", "rio_api", diff --git a/Cargo.toml b/Cargo.toml index 0d6bc83e..8264afa9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,7 +24,7 @@ authors = ["Tpt "] license = "MIT OR Apache-2.0" homepage = "https://oxigraph.org/" edition = "2021" -rust-version = "1.67" +rust-version = "1.70" [profile.release] lto = true diff --git a/cli/Cargo.toml b/cli/Cargo.toml index ef9f42de..05c7df74 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -24,7 +24,7 @@ rocksdb-pkg-config = ["oxigraph/rocksdb-pkg-config"] [dependencies] anyhow = "1.0.72" oxhttp = { version = "0.1.7", features = ["rayon"] } -clap = { version = ">=4.0, <5.0", features = ["derive"] } +clap = { version = "4.0", features = ["derive"] } oxigraph = { version = "0.4.0-alpha.1-dev", path = "../lib", features = ["http_client"] } rand = "0.8" url = "2.4" diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index 900b0b92..de97438f 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -9,7 +9,6 @@ cargo-fuzz = true [dependencies] anyhow = "1.0.72" -lazy_static = "1.4" libfuzzer-sys = "0.4" oxrdf = { path = "../lib/oxrdf", features = ["rdf-star"] } oxttl = { path = "../lib/oxttl", features = ["rdf-star"] } diff --git a/fuzz/fuzz_targets/sparql_eval.rs b/fuzz/fuzz_targets/sparql_eval.rs index 5b52f4bd..24c8a176 100644 --- a/fuzz/fuzz_targets/sparql_eval.rs +++ b/fuzz/fuzz_targets/sparql_eval.rs @@ -1,27 +1,26 @@ #![no_main] -use lazy_static::lazy_static; use libfuzzer_sys::fuzz_target; use oxigraph::io::RdfFormat; use oxigraph::sparql::{Query, QueryOptions, QueryResults, QuerySolutionIter}; use oxigraph::store::Store; +use std::sync::OnceLock; -lazy_static! { - static ref STORE: Store = { +fuzz_target!(|data: sparql_smith::Query| { + static STORE: OnceLock = OnceLock::new(); + let store = STORE.get_or_init(|| { let store = Store::new().unwrap(); store .load_dataset(sparql_smith::DATA_TRIG.as_bytes(), RdfFormat::TriG, None) .unwrap(); store - }; -} + }); -fuzz_target!(|data: sparql_smith::Query| { let query_str = data.to_string(); if let Ok(query) = Query::parse(&query_str, None) { let options = QueryOptions::default(); - let with_opt = STORE.query_opt(query.clone(), options.clone()).unwrap(); - let without_opt = STORE + let with_opt = store.query_opt(query.clone(), options.clone()).unwrap(); + let without_opt = store .query_opt(query, options.without_optimizations()) .unwrap(); match (with_opt, without_opt) { diff --git a/lib/Cargo.toml b/lib/Cargo.toml index f05e8a56..e4b4b92a 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -26,7 +26,6 @@ rocksdb_debug = [] digest = "0.10" hex = "0.4" json-event-parser = "0.1" -lazy_static = "1.4" md-5 = "0.10" oxilangtag = "0.1" oxiri = "0.2" diff --git a/lib/oxsdatatypes/src/double.rs b/lib/oxsdatatypes/src/double.rs index 6e3f890e..bc040738 100644 --- a/lib/oxsdatatypes/src/double.rs +++ b/lib/oxsdatatypes/src/double.rs @@ -173,7 +173,7 @@ impl From for Double { impl From for Double { #[inline] fn from(value: Boolean) -> Self { - if bool::from(value) { 1. } else { 0. }.into() + f64::from(bool::from(value)).into() } } diff --git a/lib/oxsdatatypes/src/float.rs b/lib/oxsdatatypes/src/float.rs index ce88c0e4..996a6401 100644 --- a/lib/oxsdatatypes/src/float.rs +++ b/lib/oxsdatatypes/src/float.rs @@ -153,7 +153,7 @@ impl From for Float { impl From for Float { #[inline] fn from(value: Boolean) -> Self { - if bool::from(value) { 1. } else { 0. }.into() + f32::from(bool::from(value)).into() } } diff --git a/lib/src/storage/backend/rocksdb.rs b/lib/src/storage/backend/rocksdb.rs index 4360e3e6..6d065059 100644 --- a/lib/src/storage/backend/rocksdb.rs +++ b/lib/src/storage/backend/rocksdb.rs @@ -9,7 +9,6 @@ )] use crate::storage::error::{CorruptionError, StorageError}; -use lazy_static::lazy_static; use libc::{self, c_void, free}; use oxrocksdb_sys::*; use rand::random; @@ -26,7 +25,7 @@ use std::marker::PhantomData; use std::ops::Deref; use std::path::{Path, PathBuf}; use std::rc::{Rc, Weak}; -use std::sync::Arc; +use std::sync::{Arc, OnceLock}; use std::thread::{available_parallelism, yield_now}; use std::{ptr, slice}; @@ -57,23 +56,6 @@ macro_rules! ffi_result_impl { }} } -lazy_static! { - static ref ROCKSDB_ENV: UnsafeEnv = { - unsafe { - let env = rocksdb_create_default_env(); - assert!(!env.is_null(), "rocksdb_create_default_env returned null"); - UnsafeEnv(env) - } - }; - static ref ROCKSDB_MEM_ENV: UnsafeEnv = { - unsafe { - let env = rocksdb_create_mem_env(); - assert!(!env.is_null(), "rocksdb_create_mem_env returned null"); - UnsafeEnv(env) - } - }; -} - pub struct ColumnFamilyDefinition { pub name: &'static str, pub use_iter: bool, @@ -472,6 +454,9 @@ impl Db { limit_max_open_files: bool, in_memory: bool, ) -> Result<*mut rocksdb_options_t, StorageError> { + static ROCKSDB_ENV: OnceLock = OnceLock::new(); + static ROCKSDB_MEM_ENV: OnceLock = OnceLock::new(); + unsafe { let options = rocksdb_options_create(); assert!(!options.is_null(), "rocksdb_options_create returned null"); @@ -508,10 +493,19 @@ impl Db { rocksdb_options_set_env( options, if in_memory { - ROCKSDB_MEM_ENV.0 + ROCKSDB_MEM_ENV.get_or_init(|| { + let env = rocksdb_create_mem_env(); + assert!(!env.is_null(), "rocksdb_create_mem_env returned null"); + UnsafeEnv(env) + }) } else { - ROCKSDB_ENV.0 - }, + ROCKSDB_ENV.get_or_init(|| { + let env = rocksdb_create_default_env(); + assert!(!env.is_null(), "rocksdb_create_default_env returned null"); + UnsafeEnv(env) + }) + } + .0, ); Ok(options) } @@ -1400,7 +1394,8 @@ impl From for StorageError { struct UnsafeEnv(*mut rocksdb_env_t); -// Hack for lazy_static. OK because only written in lazy static and used in a thread-safe way by RocksDB +// Hack for OnceCell. OK because only written in OnceCell and used in a thread-safe way by RocksDB +unsafe impl Send for UnsafeEnv {} unsafe impl Sync for UnsafeEnv {} fn path_to_cstring(path: &Path) -> Result { diff --git a/python/src/io.rs b/python/src/io.rs index be494e22..e8da9a8c 100644 --- a/python/src/io.rs +++ b/python/src/io.rs @@ -14,6 +14,7 @@ use std::ffi::OsStr; use std::fs::File; use std::io::{self, BufWriter, Cursor, Read, Write}; use std::path::{Path, PathBuf}; +use std::sync::OnceLock; /// Parses RDF graph and dataset serialization formats. /// @@ -400,7 +401,7 @@ pub fn map_parse_error(error: ParseError, file_path: Option) -> PyErr { match error { ParseError::Syntax(error) => { // Python 3.9 does not support end line and end column - if python_version() >= (3, 10) { + if python_version() >= (3, 10, 0) { let params = if let Some(location) = error.location() { ( file_path, @@ -457,9 +458,12 @@ pub fn allow_threads_unsafe(_py: Python<'_>, f: impl FnOnce() -> T) -> T { f() } -fn python_version() -> (u8, u8) { - Python::with_gil(|py| { - let v = py.version_info(); - (v.major, v.minor) +fn python_version() -> (u8, u8, u8) { + static VERSION: OnceLock<(u8, u8, u8)> = OnceLock::new(); + *VERSION.get_or_init(|| { + Python::with_gil(|py| { + let v = py.version_info(); + (v.major, v.minor, v.patch) + }) }) } diff --git a/testsuite/Cargo.toml b/testsuite/Cargo.toml index 47c3ce2e..59b751c2 100644 --- a/testsuite/Cargo.toml +++ b/testsuite/Cargo.toml @@ -12,8 +12,7 @@ publish = false [dependencies] anyhow = "1.0.72" -clap = { version = ">=4.0, <5.0", features = ["derive"] } -lazy_static = "1.4" +clap = { version = "4.0", features = ["derive"] } oxigraph = { path = "../lib" } oxttl = { path= "../lib/oxttl" } sparopt = { path = "../lib/sparopt" } diff --git a/testsuite/src/sparql_evaluator.rs b/testsuite/src/sparql_evaluator.rs index d5b4d265..67081be6 100644 --- a/testsuite/src/sparql_evaluator.rs +++ b/testsuite/src/sparql_evaluator.rs @@ -4,7 +4,6 @@ use crate::manifest::*; use crate::report::{dataset_diff, format_diff}; use crate::vocab::*; use anyhow::{anyhow, bail, ensure, Error, Result}; -use lazy_static::lazy_static; use oxigraph::model::vocab::*; use oxigraph::model::*; use oxigraph::sparql::results::QueryResultsFormat; @@ -16,7 +15,7 @@ use std::fmt::Write; use std::io::{self, BufReader, Cursor}; use std::ops::Deref; use std::str::FromStr; -use std::sync::{Arc, Mutex}; +use std::sync::{Arc, Mutex, OnceLock}; pub fn register_sparql_tests(evaluator: &mut TestEvaluator) { evaluator.register( @@ -740,13 +739,11 @@ fn evaluate_query_optimization_test(test: &Test) -> Result<()> { Ok(()) } -lazy_static! { - // Pool of stores to avoid allocating/deallocating them a lot - static ref STORE_POOL: Mutex> = Mutex::new(Vec::new()); -} +// Pool of stores to avoid allocating/deallocating them a lot +static STORE_POOL: OnceLock>> = OnceLock::new(); fn get_store() -> Result { - let store = if let Some(store) = STORE_POOL.lock().unwrap().pop() { + let store = if let Some(store) = STORE_POOL.get_or_init(Mutex::default).lock().unwrap().pop() { store } else { Store::new()? @@ -761,7 +758,11 @@ struct StoreRef { impl Drop for StoreRef { fn drop(&mut self) { if self.store.clear().is_ok() { - STORE_POOL.lock().unwrap().push(self.store.clone()) + STORE_POOL + .get_or_init(Mutex::default) + .lock() + .unwrap() + .push(self.store.clone()) } } } From c5f02d926387d5338e1aaeede8ac339275638219 Mon Sep 17 00:00:00 2001 From: Tpt Date: Sat, 23 Sep 2023 22:26:34 +0200 Subject: [PATCH 100/217] Upgrades oxhttp - Disables HTTPs support by default, the TLS stack is opt-in - Renames "http_client" feature to "http-client" - Uses native TLS by default in pyoxigraph and cli - Uses Rustls for Linux Python wheels and Docker images --- .github/workflows/manylinux_build.sh | 4 +- .github/workflows/musllinux_build.sh | 4 +- Cargo.lock | 99 +++++++++++++++++++++++++--- cli/Cargo.toml | 8 ++- cli/Dockerfile | 4 +- cli/README.md | 7 ++ cli/src/main.rs | 11 ++-- lib/Cargo.toml | 11 ++-- lib/README.md | 11 +++- lib/benches/store.rs | 3 +- lib/src/sparql/http/dummy.rs | 10 +-- lib/src/sparql/http/mod.rs | 8 +-- lib/src/sparql/http/simple.rs | 17 +++-- lib/src/sparql/mod.rs | 8 +-- lib/src/storage/backend/rocksdb.rs | 2 +- lints/test_debian_compatibility.py | 2 +- python/Cargo.toml | 5 +- python/README.md | 3 + 18 files changed, 162 insertions(+), 55 deletions(-) diff --git a/.github/workflows/manylinux_build.sh b/.github/workflows/manylinux_build.sh index ae72000d..e7ee8ec1 100644 --- a/.github/workflows/manylinux_build.sh +++ b/.github/workflows/manylinux_build.sh @@ -13,9 +13,9 @@ source venv/bin/activate pip install -r requirements.dev.txt maturin develop --release python generate_stubs.py pyoxigraph pyoxigraph.pyi --black -maturin build --release --features abi3 --compatibility manylinux2014 +maturin build --release --no-default-features --features abi3 --features rustls --compatibility manylinux2014 if [ %for_each_version% ]; then for VERSION in 7 8 9 10 11; do - maturin build --release --interpreter "python3.$VERSION" --compatibility manylinux2014 + maturin build --release --no-default-features --features rustls --interpreter "python3.$VERSION" --compatibility manylinux2014 done fi diff --git a/.github/workflows/musllinux_build.sh b/.github/workflows/musllinux_build.sh index 674be63a..ec6c54e1 100644 --- a/.github/workflows/musllinux_build.sh +++ b/.github/workflows/musllinux_build.sh @@ -11,9 +11,9 @@ source venv/bin/activate pip install -r requirements.dev.txt maturin develop --release python generate_stubs.py pyoxigraph pyoxigraph.pyi --black -maturin build --release --features abi3 --compatibility musllinux_1_2 +maturin build --release --no-default-features --features abi3 --features rustls --compatibility musllinux_1_2 if [ %for_each_version% ]; then for VERSION in 7 8 9 10 11; do - maturin build --release --interpreter "python3.$VERSION" --compatibility musllinux_1_2 + maturin build --release --no-default-features --features rustls --interpreter "python3.$VERSION" --compatibility musllinux_1_2 done fi diff --git a/Cargo.lock b/Cargo.lock index 8fea3245..f8f74cc6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -148,9 +148,9 @@ dependencies = [ [[package]] name = "base64" -version = "0.21.3" +version = "0.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "414dcefbc63d77c526a76b3afcf6fbb9b5e2791c19c3aa2297733208750c6e53" +checksum = "9ba43ea6f343b788c8764558649e08df62f86c6ef251fdaeb1ffd010a9ae50a2" [[package]] name = "bindgen" @@ -576,6 +576,21 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.2.0" @@ -874,6 +889,24 @@ dependencies = [ "adler", ] +[[package]] +name = "native-tls" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "nom" version = "7.1.3" @@ -930,24 +963,62 @@ version = "11.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" +[[package]] +name = "openssl" +version = "0.10.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bac25ee399abb46215765b1cb35bc0212377e58a061560d8b29b024fd0430e7c" +dependencies = [ + "bitflags 2.4.0", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.31", +] + [[package]] name = "openssl-probe" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +[[package]] +name = "openssl-sys" +version = "0.9.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db4d56a4c0478783083cfafcc42493dd4a981d41669da64b4572a2a089b51b1d" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "oxhttp" -version = "0.1.7" +version = "0.2.0-alpha.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64f8f4b616372a7e657a05100d1389325fc2f7d3fee5c9de05166d979cb2b729" +checksum = "525e4922eaf3a36d3d8534226910b6f5b878446d2f7f20b31da7889eb969f984" dependencies = [ "httparse", - "lazy_static", - "rayon-core", + "native-tls", "rustls", "rustls-native-certs", "url", + "webpki-roots", ] [[package]] @@ -1524,9 +1595,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.101.4" +version = "0.101.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d93931baf2d282fff8d3a532bbfd7653f734643161b87e3e01e59a04439bf0d" +checksum = "3c7d5dece342910d9ba34d259310cae3e0154b873b35408b787b59bce53d34fe" dependencies = [ "ring", "untrusted", @@ -1930,6 +2001,12 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.4" @@ -2025,6 +2102,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-roots" +version = "0.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14247bb57be4f377dfb94c72830b8ce8fc6beac03cf4bf7b9732eadd414123fc" + [[package]] name = "which" version = "4.4.2" diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 05c7df74..6b84f16b 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -19,13 +19,17 @@ name = "oxigraph" path = "src/main.rs" [features] +default = ["native-tls"] +native-tls = ["oxigraph/http-client-native-tls"] rocksdb-pkg-config = ["oxigraph/rocksdb-pkg-config"] +rustls-native = ["oxigraph/http-client-rustls-native"] +rustls-webpki = ["oxigraph/http-client-rustls-webpki"] [dependencies] anyhow = "1.0.72" -oxhttp = { version = "0.1.7", features = ["rayon"] } +oxhttp = { version = "0.2.0-alpha.1" } clap = { version = "4.0", features = ["derive"] } -oxigraph = { version = "0.4.0-alpha.1-dev", path = "../lib", features = ["http_client"] } +oxigraph = { version = "0.4.0-alpha.1-dev", path = "../lib" } rand = "0.8" url = "2.4" oxiri = "0.2" diff --git a/cli/Dockerfile b/cli/Dockerfile index abbd6aad..9454410f 100644 --- a/cli/Dockerfile +++ b/cli/Dockerfile @@ -13,10 +13,10 @@ RUN if [ "$BUILDARCH" != "$TARGETARCH" ] && [ "$TARGETARCH" = "arm64" ] ; \ then \ export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc && \ export BINDGEN_EXTRA_CLANG_ARGS="--sysroot /usr/aarch64-linux-gnu" && \ - cargo build --release --target aarch64-unknown-linux-gnu && \ + cargo build --release --target aarch64-unknown-linux-gnu --no-default-features --features rustls-webpki && \ mv /oxigraph/target/aarch64-unknown-linux-gnu/release/oxigraph /oxigraph/target/release/oxigraph ; \ else \ - cargo build --release ; \ + cargo build --release --no-default-features --features rustls-webpki ; \ fi FROM --platform=$TARGETPLATFORM gcr.io/distroless/cc-debian11 diff --git a/cli/README.md b/cli/README.md index 8c650b4f..aaa1f3e4 100644 --- a/cli/README.md +++ b/cli/README.md @@ -40,6 +40,13 @@ There is no need to clone the git repository. To compile the command line tool from source, clone this git repository including its submodules (`git clone --recursive https://github.com/oxigraph/oxigraph.git`), and execute `cargo build --release` in the `cli` directory to compile the full binary after having downloaded its dependencies. It will create a fat binary in `target/release/oxigraph`. +Some build options (cargo features) are available: +- `rocksdb-pkg-config`: links against an already compiled rocksdb shared library found using [pkg-config](https://crates.io/crates/pkg-config). +- `native-tls`: Enables Oxigraph HTTP client for query federation using the host OS TLS stack (enabled by default). +- `rustls-native` Enables Oxigraph HTTP client for query federation using [Rustls](https://crates.io/crates/rustls) and the native certificates. +- `rustls-webpki` Enables Oxigraph HTTP client for query federation using [Rustls](https://crates.io/crates/rustls) and the [Common CA Database](https://www.ccadb.org/) certificates. + + ## Usage Run `oxigraph serve --location my_data_storage_directory` to start the server where `my_data_storage_directory` is the directory where you want Oxigraph data to be stored. It listens by default on `localhost:7878`. diff --git a/cli/src/main.rs b/cli/src/main.rs index 54f79299..16371325 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -879,7 +879,7 @@ fn rdf_format_from_name(name: &str) -> anyhow::Result { } fn serve(store: Store, bind: String, read_only: bool, cors: bool) -> anyhow::Result<()> { - let mut server = if cors { + let server = if cors { Server::new(cors_middleware(move |request| { handle_request(request, store.clone(), read_only) .unwrap_or_else(|(status, message)| error(status, message)) @@ -889,9 +889,10 @@ fn serve(store: Store, bind: String, read_only: bool, cors: bool) -> anyhow::Res handle_request(request, store.clone(), read_only) .unwrap_or_else(|(status, message)| error(status, message)) }) - }; - server.set_global_timeout(HTTP_TIMEOUT); - server.set_server_name(concat!("Oxigraph/", env!("CARGO_PKG_VERSION")))?; + } + .with_global_timeout(HTTP_TIMEOUT) + .with_server_name(concat!("Oxigraph/", env!("CARGO_PKG_VERSION")))? + .with_max_num_threads(available_parallelism()?.get() * 128); #[cfg(target_os = "linux")] systemd_notify_ready()?; eprintln!("Listening for requests at http://{}", &bind); @@ -1191,7 +1192,7 @@ fn handle_request( resolve_with_base(request, &format!("/store/{:x}", random::()))?; web_load_graph(&store, request, format, &graph.clone().into())?; Ok(Response::builder(Status::CREATED) - .with_header(HeaderName::LOCATION, graph.as_str()) + .with_header(HeaderName::LOCATION, graph.into_string()) .unwrap() .build()) } diff --git a/lib/Cargo.toml b/lib/Cargo.toml index e4b4b92a..87b0715d 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -18,9 +18,12 @@ rust-version.workspace = true [features] default = [] js = ["getrandom/js", "oxsdatatypes/js", "js-sys"] -http_client = ["oxhttp", "oxhttp/rustls"] +http-client = ["oxhttp"] +http-client-native-tls = ["http-client", "oxhttp/native-tls"] +http-client-rustls-webpki = ["http-client", "oxhttp/rustls-webpki"] +http-client-rustls-native = ["http-client", "oxhttp/rustls-native"] rocksdb-pkg-config = ["oxrocksdb-sys/pkg-config"] -rocksdb_debug = [] +rocksdb-debug = [] [dependencies] digest = "0.10" @@ -44,7 +47,7 @@ sparopt = { version = "0.1.0-alpha.1-dev", path="sparopt", features = ["rdf-star [target.'cfg(not(target_family = "wasm"))'.dependencies] libc = "0.2.147" oxrocksdb-sys = { version = "0.4.0-alpha.1-dev", path="../oxrocksdb-sys" } -oxhttp = { version = "0.1.7", optional = true } +oxhttp = { version = "0.2.0-alpha.1", optional = true } [target.'cfg(all(target_family = "wasm", target_os = "unknown"))'.dependencies] getrandom = "0.2" @@ -52,7 +55,7 @@ js-sys = { version = "0.3.60", optional = true } [target.'cfg(not(target_family = "wasm"))'.dev-dependencies] criterion = "0.5" -oxhttp = "0.1.7" +oxhttp = "0.2.0-alpha.1" zstd = "0.12" [package.metadata.docs.rs] diff --git a/lib/README.md b/lib/README.md index d2afa1bc..31a3e510 100644 --- a/lib/README.md +++ b/lib/README.md @@ -48,9 +48,14 @@ if let QueryResults::Solutions(mut solutions) = store.query("SELECT ?s WHERE { ``` Some parts of this library are available as standalone crates: -* [`oxrdf`](https://crates.io/crates/oxrdf) provides datastructures encoding RDF basic concepts (the `oxigraph::model` module). -* [`spargebra`](https://crates.io/crates/spargebra) provides a SPARQL parser. -* [`sparesults`](https://crates.io/crates/sparesults) provides parsers and serializers for SPARQL result formats. +* [`oxrdf`](https://crates.io/crates/oxrdf), datastructures encoding RDF basic concepts (the `oxigraph::model` module). +* [`oxrdfio`](https://crates.io/crates/oxrdfio), a unified parser and serializer API for RDF formats. It itself relies on: + * [`oxttl`](https://crates.io/crates/oxttl), N-Triple, N-Quad, Turtle, TriG and N3 parsing and serialization. + * [`oxrdfxml`](https://crates.io/crates/oxrdfxml), RDF/XML parsing and serialization. +* [`spargebra`](https://crates.io/crates/spargebra), a SPARQL parser. +* [`sparesults`](https://crates.io/crates/sparesults), parsers and serializers for SPARQL result formats. +* [`sparopt`](https://crates.io/crates/sparesults), a SPARQL optimizer. +* [`oxsdatatypes`](https://crates.io/crates/oxsdatatypes), an implementation of some XML Schema datatypes. To build the library, don't forget to clone the submodules using `git clone --recursive https://github.com/oxigraph/oxigraph.git` to clone the repository including submodules or `git submodule update --init` to add submodules to the already cloned repository. diff --git a/lib/benches/store.rs b/lib/benches/store.rs index 6f18f8d1..20826041 100644 --- a/lib/benches/store.rs +++ b/lib/benches/store.rs @@ -207,8 +207,7 @@ criterion_main!(store); fn read_data(file: &str) -> impl Read { if !Path::new(file).exists() { - let mut client = oxhttp::Client::new(); - client.set_redirection_limit(5); + let client = oxhttp::Client::new().with_redirection_limit(5); let url = format!("https://github.com/Tpt/bsbm-tools/releases/download/v0.2/{file}"); let request = Request::builder(Method::GET, url.parse().unwrap()).build(); let response = client.request(request).unwrap(); diff --git a/lib/src/sparql/http/dummy.rs b/lib/src/sparql/http/dummy.rs index dc8516a6..7b3a551f 100644 --- a/lib/src/sparql/http/dummy.rs +++ b/lib/src/sparql/http/dummy.rs @@ -11,10 +11,10 @@ impl Client { } #[allow(clippy::unused_self)] - pub fn get(&self, _url: &str, _accept: &str) -> Result<(String, Empty)> { + pub fn get(&self, _url: &str, _accept: &'static str) -> Result<(String, Empty)> { Err(Error::new( ErrorKind::Unsupported, - "HTTP client is not available. Enable the feature 'http_client'", + "HTTP client is not available. Enable the feature 'http-client'", )) } @@ -23,12 +23,12 @@ impl Client { &self, _url: &str, _payload: Vec, - _content_type: &str, - _accept: &str, + _content_type: &'static str, + _accept: &'static str, ) -> Result<(String, Empty)> { Err(Error::new( ErrorKind::Unsupported, - "HTTP client is not available. Enable the feature 'http_client'", + "HTTP client is not available. Enable the feature 'http-client'", )) } } diff --git a/lib/src/sparql/http/mod.rs b/lib/src/sparql/http/mod.rs index 0266d0b3..b309cf55 100644 --- a/lib/src/sparql/http/mod.rs +++ b/lib/src/sparql/http/mod.rs @@ -1,9 +1,9 @@ -#[cfg(not(feature = "http_client"))] +#[cfg(not(feature = "http-client"))] mod dummy; -#[cfg(feature = "http_client")] +#[cfg(feature = "http-client")] mod simple; -#[cfg(not(feature = "http_client"))] +#[cfg(not(feature = "http-client"))] pub use dummy::Client; -#[cfg(feature = "http_client")] +#[cfg(feature = "http-client")] pub use simple::Client; diff --git a/lib/src/sparql/http/simple.rs b/lib/src/sparql/http/simple.rs index 57b9e540..bd81d7ce 100644 --- a/lib/src/sparql/http/simple.rs +++ b/lib/src/sparql/http/simple.rs @@ -8,18 +8,17 @@ pub struct Client { impl Client { pub fn new(timeout: Option, redirection_limit: usize) -> Self { - let mut client = oxhttp::Client::new(); + let mut client = oxhttp::Client::new() + .with_redirection_limit(redirection_limit) + .with_user_agent(concat!("Oxigraph/", env!("CARGO_PKG_VERSION"))) + .unwrap(); if let Some(timeout) = timeout { - client.set_global_timeout(timeout); + client = client.with_global_timeout(timeout); } - client.set_redirection_limit(redirection_limit); - client - .set_user_agent(concat!("Oxigraph/", env!("CARGO_PKG_VERSION"))) - .unwrap(); Self { client } } - pub fn get(&self, url: &str, accept: &str) -> Result<(String, Body)> { + pub fn get(&self, url: &str, accept: &'static str) -> Result<(String, Body)> { let request = Request::builder(Method::GET, url.parse().map_err(invalid_input_error)?) .with_header(HeaderName::ACCEPT, accept) .map_err(invalid_input_error)? @@ -50,8 +49,8 @@ impl Client { &self, url: &str, payload: Vec, - content_type: &str, - accept: &str, + content_type: &'static str, + accept: &'static str, ) -> Result<(String, Body)> { let request = Request::builder(Method::POST, url.parse().map_err(invalid_input_error)?) .with_header(HeaderName::ACCEPT, accept) diff --git a/lib/src/sparql/mod.rs b/lib/src/sparql/mod.rs index 6d3ef593..82944f95 100644 --- a/lib/src/sparql/mod.rs +++ b/lib/src/sparql/mod.rs @@ -138,7 +138,7 @@ pub(crate) fn evaluate_query( /// Options for SPARQL query evaluation. /// /// -/// If the `"http_client"` optional feature is enabled, +/// If the `"http-client"` optional feature is enabled, /// a simple HTTP 1.1 client is used to execute [SPARQL 1.1 Federated Query](https://www.w3.org/TR/sparql11-federated-query/) SERVICE calls. /// /// Usage example disabling the federated query support: @@ -182,7 +182,7 @@ impl QueryOptions { } /// Sets a timeout for HTTP requests done during SPARQL evaluation. - #[cfg(feature = "http_client")] + #[cfg(feature = "http-client")] #[inline] #[must_use] pub fn with_http_timeout(mut self, timeout: Duration) -> Self { @@ -193,7 +193,7 @@ impl QueryOptions { /// Sets an upper bound of the number of HTTP redirection followed per HTTP request done during SPARQL evaluation. /// /// By default this value is `0`. - #[cfg(feature = "http_client")] + #[cfg(feature = "http-client")] #[inline] #[must_use] pub fn with_http_redirection_limit(mut self, redirection_limit: usize) -> Self { @@ -235,7 +235,7 @@ impl QueryOptions { fn service_handler(&self) -> Rc> { self.service_handler.clone().unwrap_or_else(|| { - if cfg!(feature = "http_client") { + if cfg!(feature = "http-client") { Rc::new(service::SimpleServiceHandler::new( self.http_timeout, self.http_redirection_limit, diff --git a/lib/src/storage/backend/rocksdb.rs b/lib/src/storage/backend/rocksdb.rs index 6d065059..5e6f4103 100644 --- a/lib/src/storage/backend/rocksdb.rs +++ b/lib/src/storage/backend/rocksdb.rs @@ -200,7 +200,7 @@ impl Db { 16, ); rocksdb_options_set_block_based_table_factory(options, block_based_table_options); - #[cfg(feature = "rocksdb_debug")] + #[cfg(feature = "rocksdb-debug")] { rocksdb_options_set_info_log_level(options, 0); rocksdb_options_enable_statistics(options); diff --git a/lints/test_debian_compatibility.py b/lints/test_debian_compatibility.py index ec11b319..10f6596a 100644 --- a/lints/test_debian_compatibility.py +++ b/lints/test_debian_compatibility.py @@ -5,7 +5,7 @@ from urllib.request import urlopen TARGET_DEBIAN_VERSIONS = ["sid"] IGNORE_PACKAGES = {"oxigraph-js", "oxigraph-testsuite", "pyoxigraph", "sparql-smith"} -ALLOWED_MISSING_PACKAGES = {"escargot", "quick-xml"} +ALLOWED_MISSING_PACKAGES = {"escargot", "oxhttp", "quick-xml"} base_path = Path(__file__).parent.parent diff --git a/python/Cargo.toml b/python/Cargo.toml index ce5df20a..9ecee7af 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -18,9 +18,12 @@ name = "pyoxigraph" doctest = false [features] +default = ["native-tls"] abi3 = ["pyo3/abi3-py38"] +native-tls = ["oxigraph/http-client-native-tls"] rocksdb-pkg-config = ["oxigraph/rocksdb-pkg-config"] +rustls = ["oxigraph/http-client-rustls-native"] [dependencies] -oxigraph = { version = "0.4.0-alpha.1-dev", path="../lib", features = ["http_client"] } +oxigraph = { version = "0.4.0-alpha.1-dev", path="../lib" } pyo3 = { version = "0.19", features = ["extension-module"] } diff --git a/python/README.md b/python/README.md index 8a2e8247..c8478a60 100644 --- a/python/README.md +++ b/python/README.md @@ -32,6 +32,9 @@ Pyoxigraph documentation is [available on the Oxigraph website](https://pyoxigra To build and install the development version of pyoxigraph you need to clone this git repository including submodules (`git clone --recursive https://github.com/oxigraph/oxigraph.git`) and to run `pip install .` in the `python` directory (the one this README is in). +Note that by default the installation will not use [cpython stable ABI](https://docs.python.org/3/c-api/stable.html) and will rely on the host TLS implementation. +Use `--features abi3` feature to use cpython stable ABI and use `--no-default-features --features rustls` to use [rustls](https://github.com/rustls/rustls) with the system certificates. + ## Help Feel free to use [GitHub discussions](https://github.com/oxigraph/oxigraph/discussions) or [the Gitter chat](https://gitter.im/oxigraph/community) to ask questions or talk about Oxigraph. From 5e3a2fc89d8c633fb1e5b84f5805a4ea59fbcb36 Mon Sep 17 00:00:00 2001 From: Tpt Date: Mon, 25 Sep 2023 21:19:33 +0200 Subject: [PATCH 101/217] Uses Rustls for portable Linux binaries --- .github/workflows/artifacts.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/artifacts.yml b/.github/workflows/artifacts.yml index c0dd2abf..a9d6eac9 100644 --- a/.github/workflows/artifacts.yml +++ b/.github/workflows/artifacts.yml @@ -25,9 +25,9 @@ jobs: sudo apt update && sudo apt-get install -y g++-aarch64-linux-gnu echo -e "\n\n[target.aarch64-unknown-linux-gnu]\nlinker = \"aarch64-linux-gnu-gcc\"" >> .cargo/config.toml - uses: Swatinem/rust-cache@v2 - - run: cargo build --release + - run: cargo build --release --no-default-features --features rustls-native working-directory: ./cli - - run: cargo build --release --target aarch64-unknown-linux-gnu + - run: cargo build --release --target aarch64-unknown-linux-gnu --no-default-features --features rustls-native working-directory: ./cli env: BINDGEN_EXTRA_CLANG_ARGS: --sysroot /usr/aarch64-linux-gnu From dbb39d867a662953028ae5fbc43f6310f40540d0 Mon Sep 17 00:00:00 2001 From: Tpt Date: Mon, 25 Sep 2023 18:23:42 +0200 Subject: [PATCH 102/217] Upgrades json-event-parser --- Cargo.lock | 4 +- lib/Cargo.toml | 2 +- lib/sparesults/Cargo.toml | 2 +- lib/sparesults/src/error.rs | 25 +++- lib/sparesults/src/json.rs | 205 ++++++++++++++--------------- lib/src/sparql/eval.rs | 18 +-- lib/src/sparql/mod.rs | 16 +-- lints/test_debian_compatibility.py | 2 +- 8 files changed, 144 insertions(+), 130 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f8f74cc6..d293c2c1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -786,9 +786,9 @@ dependencies = [ [[package]] name = "json-event-parser" -version = "0.1.1" +version = "0.2.0-alpha.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32f12e624eaeb74accb9bb48f01cb071427f68115aaafa5689acb372d7e22977" +checksum = "20a2ad11b373ee8f1d5f9b0632b148a6dc65cf1faa8b2a99c89cbe70411e31a2" [[package]] name = "kernel32-sys" diff --git a/lib/Cargo.toml b/lib/Cargo.toml index 87b0715d..1ea01640 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -28,7 +28,7 @@ rocksdb-debug = [] [dependencies] digest = "0.10" hex = "0.4" -json-event-parser = "0.1" +json-event-parser = "0.2.0-alpha.1" md-5 = "0.10" oxilangtag = "0.1" oxiri = "0.2" diff --git a/lib/sparesults/Cargo.toml b/lib/sparesults/Cargo.toml index 3fa9e27e..2516cdfa 100644 --- a/lib/sparesults/Cargo.toml +++ b/lib/sparesults/Cargo.toml @@ -19,7 +19,7 @@ default = [] rdf-star = ["oxrdf/rdf-star"] [dependencies] -json-event-parser = "0.1" +json-event-parser = "0.2.0-alpha.1" memchr = "2.5" oxrdf = { version = "0.2.0-alpha.1-dev", path="../oxrdf" } quick-xml = ">=0.29, <0.31" diff --git a/lib/sparesults/src/error.rs b/lib/sparesults/src/error.rs index 40510663..1096b393 100644 --- a/lib/sparesults/src/error.rs +++ b/lib/sparesults/src/error.rs @@ -56,6 +56,15 @@ impl From for io::Error { } } +impl From for ParseError { + fn from(error: json_event_parser::ParseError) -> Self { + match error { + json_event_parser::ParseError::Syntax(error) => SyntaxError::from(error).into(), + json_event_parser::ParseError::Io(error) => error.into(), + } + } +} + impl From for ParseError { #[inline] fn from(error: quick_xml::Error) -> Self { @@ -79,6 +88,7 @@ pub struct SyntaxError { #[derive(Debug)] pub(crate) enum SyntaxErrorKind { + Json(json_event_parser::SyntaxError), Xml(quick_xml::Error), Term { error: TermParseError, term: String }, Msg { msg: String }, @@ -98,6 +108,7 @@ impl fmt::Display for SyntaxError { #[inline] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match &self.inner { + SyntaxErrorKind::Json(e) => e.fmt(f), SyntaxErrorKind::Xml(e) => e.fmt(f), SyntaxErrorKind::Term { error, term } => write!(f, "{error}: {term}"), SyntaxErrorKind::Msg { msg } => f.write_str(msg), @@ -109,6 +120,7 @@ impl Error for SyntaxError { #[inline] fn source(&self) -> Option<&(dyn Error + 'static)> { match &self.inner { + SyntaxErrorKind::Json(e) => Some(e), SyntaxErrorKind::Xml(e) => Some(e), SyntaxErrorKind::Term { error, .. } => Some(error), SyntaxErrorKind::Msg { .. } => None, @@ -120,6 +132,7 @@ impl From for io::Error { #[inline] fn from(error: SyntaxError) -> Self { match error.inner { + SyntaxErrorKind::Json(error) => Self::new(io::ErrorKind::InvalidData, error), SyntaxErrorKind::Xml(error) => match error { quick_xml::Error::Io(error) => match Arc::try_unwrap(error) { Ok(error) => error, @@ -130,10 +143,16 @@ impl From for io::Error { } _ => Self::new(io::ErrorKind::InvalidData, error), }, - SyntaxErrorKind::Term { .. } => { - Self::new(io::ErrorKind::InvalidData, error.to_string()) - } + SyntaxErrorKind::Term { .. } => Self::new(io::ErrorKind::InvalidData, error), SyntaxErrorKind::Msg { msg } => Self::new(io::ErrorKind::InvalidData, msg), } } } + +impl From for SyntaxError { + fn from(error: json_event_parser::SyntaxError) -> Self { + Self { + inner: SyntaxErrorKind::Json(error), + } + } +} diff --git a/lib/sparesults/src/json.rs b/lib/sparesults/src/json.rs index 67e2d494..3dc9be11 100644 --- a/lib/sparesults/src/json.rs +++ b/lib/sparesults/src/json.rs @@ -1,50 +1,50 @@ //! Implementation of [SPARQL Query Results JSON Format](https://www.w3.org/TR/sparql11-results-json/) use crate::error::{ParseError, SyntaxError}; -use json_event_parser::{JsonEvent, JsonReader, JsonWriter}; +use json_event_parser::{FromReadJsonReader, JsonEvent, ToWriteJsonWriter}; use oxrdf::vocab::rdf; use oxrdf::Variable; use oxrdf::*; use std::collections::BTreeMap; -use std::io::{self, BufRead, Write}; +use std::io::{self, Read, Write}; use std::mem::take; /// This limit is set in order to avoid stack overflow error when parsing nested triples due to too many recursive calls. /// The actual limit value is a wet finger compromise between not failing to parse valid files and avoiding to trigger stack overflow errors. const MAX_NUMBER_OF_NESTED_TRIPLES: usize = 128; -pub fn write_boolean_json_result(sink: W, value: bool) -> io::Result { - let mut writer = JsonWriter::from_writer(sink); +pub fn write_boolean_json_result(write: W, value: bool) -> io::Result { + let mut writer = ToWriteJsonWriter::new(write); writer.write_event(JsonEvent::StartObject)?; - writer.write_event(JsonEvent::ObjectKey("head"))?; + writer.write_event(JsonEvent::ObjectKey("head".into()))?; writer.write_event(JsonEvent::StartObject)?; writer.write_event(JsonEvent::EndObject)?; - writer.write_event(JsonEvent::ObjectKey("boolean"))?; + writer.write_event(JsonEvent::ObjectKey("boolean".into()))?; writer.write_event(JsonEvent::Boolean(value))?; writer.write_event(JsonEvent::EndObject)?; - Ok(writer.into_inner()) + writer.finish() } pub struct JsonSolutionsWriter { - writer: JsonWriter, + writer: ToWriteJsonWriter, } impl JsonSolutionsWriter { - pub fn start(sink: W, variables: &[Variable]) -> io::Result { - let mut writer = JsonWriter::from_writer(sink); + pub fn start(write: W, variables: &[Variable]) -> io::Result { + let mut writer = ToWriteJsonWriter::new(write); writer.write_event(JsonEvent::StartObject)?; - writer.write_event(JsonEvent::ObjectKey("head"))?; + writer.write_event(JsonEvent::ObjectKey("head".into()))?; writer.write_event(JsonEvent::StartObject)?; - writer.write_event(JsonEvent::ObjectKey("vars"))?; + writer.write_event(JsonEvent::ObjectKey("vars".into()))?; writer.write_event(JsonEvent::StartArray)?; for variable in variables { - writer.write_event(JsonEvent::String(variable.as_str()))?; + writer.write_event(JsonEvent::String(variable.as_str().into()))?; } writer.write_event(JsonEvent::EndArray)?; writer.write_event(JsonEvent::EndObject)?; - writer.write_event(JsonEvent::ObjectKey("results"))?; + writer.write_event(JsonEvent::ObjectKey("results".into()))?; writer.write_event(JsonEvent::StartObject)?; - writer.write_event(JsonEvent::ObjectKey("bindings"))?; + writer.write_event(JsonEvent::ObjectKey("bindings".into()))?; writer.write_event(JsonEvent::StartArray)?; Ok(Self { writer }) } @@ -56,7 +56,7 @@ impl JsonSolutionsWriter { self.writer.write_event(JsonEvent::StartObject)?; for (variable, value) in solution { self.writer - .write_event(JsonEvent::ObjectKey(variable.as_str()))?; + .write_event(JsonEvent::ObjectKey(variable.as_str().into()))?; write_json_term(value, &mut self.writer)?; } self.writer.write_event(JsonEvent::EndObject)?; @@ -67,55 +67,58 @@ impl JsonSolutionsWriter { self.writer.write_event(JsonEvent::EndArray)?; self.writer.write_event(JsonEvent::EndObject)?; self.writer.write_event(JsonEvent::EndObject)?; - Ok(self.writer.into_inner()) + self.writer.finish() } } -fn write_json_term(term: TermRef<'_>, writer: &mut JsonWriter) -> io::Result<()> { +fn write_json_term( + term: TermRef<'_>, + writer: &mut ToWriteJsonWriter, +) -> io::Result<()> { match term { TermRef::NamedNode(uri) => { writer.write_event(JsonEvent::StartObject)?; - writer.write_event(JsonEvent::ObjectKey("type"))?; - writer.write_event(JsonEvent::String("uri"))?; - writer.write_event(JsonEvent::ObjectKey("value"))?; - writer.write_event(JsonEvent::String(uri.as_str()))?; + writer.write_event(JsonEvent::ObjectKey("type".into()))?; + writer.write_event(JsonEvent::String("uri".into()))?; + writer.write_event(JsonEvent::ObjectKey("value".into()))?; + writer.write_event(JsonEvent::String(uri.as_str().into()))?; writer.write_event(JsonEvent::EndObject)?; } TermRef::BlankNode(bnode) => { writer.write_event(JsonEvent::StartObject)?; - writer.write_event(JsonEvent::ObjectKey("type"))?; - writer.write_event(JsonEvent::String("bnode"))?; - writer.write_event(JsonEvent::ObjectKey("value"))?; - writer.write_event(JsonEvent::String(bnode.as_str()))?; + writer.write_event(JsonEvent::ObjectKey("type".into()))?; + writer.write_event(JsonEvent::String("bnode".into()))?; + writer.write_event(JsonEvent::ObjectKey("value".into()))?; + writer.write_event(JsonEvent::String(bnode.as_str().into()))?; writer.write_event(JsonEvent::EndObject)?; } TermRef::Literal(literal) => { writer.write_event(JsonEvent::StartObject)?; - writer.write_event(JsonEvent::ObjectKey("type"))?; - writer.write_event(JsonEvent::String("literal"))?; - writer.write_event(JsonEvent::ObjectKey("value"))?; - writer.write_event(JsonEvent::String(literal.value()))?; + writer.write_event(JsonEvent::ObjectKey("type".into()))?; + writer.write_event(JsonEvent::String("literal".into()))?; + writer.write_event(JsonEvent::ObjectKey("value".into()))?; + writer.write_event(JsonEvent::String(literal.value().into()))?; if let Some(language) = literal.language() { - writer.write_event(JsonEvent::ObjectKey("xml:lang"))?; - writer.write_event(JsonEvent::String(language))?; + writer.write_event(JsonEvent::ObjectKey("xml:lang".into()))?; + writer.write_event(JsonEvent::String(language.into()))?; } else if !literal.is_plain() { - writer.write_event(JsonEvent::ObjectKey("datatype"))?; - writer.write_event(JsonEvent::String(literal.datatype().as_str()))?; + writer.write_event(JsonEvent::ObjectKey("datatype".into()))?; + writer.write_event(JsonEvent::String(literal.datatype().as_str().into()))?; } writer.write_event(JsonEvent::EndObject)?; } #[cfg(feature = "rdf-star")] TermRef::Triple(triple) => { writer.write_event(JsonEvent::StartObject)?; - writer.write_event(JsonEvent::ObjectKey("type"))?; - writer.write_event(JsonEvent::String("triple"))?; - writer.write_event(JsonEvent::ObjectKey("value"))?; + writer.write_event(JsonEvent::ObjectKey("type".into()))?; + writer.write_event(JsonEvent::String("triple".into()))?; + writer.write_event(JsonEvent::ObjectKey("value".into()))?; writer.write_event(JsonEvent::StartObject)?; - writer.write_event(JsonEvent::ObjectKey("subject"))?; + writer.write_event(JsonEvent::ObjectKey("subject".into()))?; write_json_term(triple.subject.as_ref().into(), writer)?; - writer.write_event(JsonEvent::ObjectKey("predicate"))?; + writer.write_event(JsonEvent::ObjectKey("predicate".into()))?; write_json_term(triple.predicate.as_ref().into(), writer)?; - writer.write_event(JsonEvent::ObjectKey("object"))?; + writer.write_event(JsonEvent::ObjectKey("object".into()))?; write_json_term(triple.object.as_ref(), writer)?; writer.write_event(JsonEvent::EndObject)?; writer.write_event(JsonEvent::EndObject)?; @@ -124,7 +127,7 @@ fn write_json_term(term: TermRef<'_>, writer: &mut JsonWriter) -> io Ok(()) } -pub enum JsonQueryResultsReader { +pub enum JsonQueryResultsReader { Solutions { variables: Vec, solutions: JsonSolutionsReader, @@ -132,24 +135,23 @@ pub enum JsonQueryResultsReader { Boolean(bool), } -impl JsonQueryResultsReader { - pub fn read(source: R) -> Result { - let mut reader = JsonReader::from_reader(source); - let mut buffer = Vec::default(); +impl JsonQueryResultsReader { + pub fn read(read: R) -> Result { + let mut reader = FromReadJsonReader::new(read); let mut variables = None; let mut buffered_bindings: Option> = None; let mut output_iter = None; - if reader.read_event(&mut buffer)? != JsonEvent::StartObject { + if reader.read_next_event()? != JsonEvent::StartObject { return Err(SyntaxError::msg("SPARQL JSON results should be an object").into()); } loop { - let event = reader.read_event(&mut buffer)?; + let event = reader.read_next_event()?; match event { - JsonEvent::ObjectKey(key) => match key { + JsonEvent::ObjectKey(key) => match key.as_ref() { "head" => { - let extracted_variables = read_head(&mut reader, &mut buffer)?; + let extracted_variables = read_head(&mut reader)?; if let Some(buffered_bindings) = buffered_bindings.take() { let mut mapping = BTreeMap::default(); for (i, var) in extracted_variables.iter().enumerate() { @@ -169,13 +171,13 @@ impl JsonQueryResultsReader { } } "results" => { - if reader.read_event(&mut buffer)? != JsonEvent::StartObject { + if reader.read_next_event()? != JsonEvent::StartObject { return Err(SyntaxError::msg("'results' should be an object").into()); } loop { - match reader.read_event(&mut buffer)? { - JsonEvent::ObjectKey("bindings") => break, // Found - JsonEvent::ObjectKey(_) => ignore_value(&mut reader, &mut buffer)?, + match reader.read_next_event()? { + JsonEvent::ObjectKey(k) if k == "bindings" => break, // Found + JsonEvent::ObjectKey(_) => ignore_value(&mut reader)?, _ => { return Err(SyntaxError::msg( "'results' should contain a 'bindings' key", @@ -184,7 +186,7 @@ impl JsonQueryResultsReader { } } } - if reader.read_event(&mut buffer)? != JsonEvent::StartArray { + if reader.read_next_event()? != JsonEvent::StartArray { return Err(SyntaxError::msg("'bindings' should be an object").into()); } if let Some(variables) = variables { @@ -195,7 +197,7 @@ impl JsonQueryResultsReader { return Ok(Self::Solutions { variables, solutions: JsonSolutionsReader { - kind: JsonSolutionsReaderKind::Streaming { reader, buffer }, + kind: JsonSolutionsReaderKind::Streaming { reader }, mapping, }, }); @@ -205,7 +207,7 @@ impl JsonQueryResultsReader { let mut variables = Vec::new(); let mut values = Vec::new(); loop { - match reader.read_event(&mut buffer)? { + match reader.read_next_event()? { JsonEvent::StartObject => (), JsonEvent::EndObject => { bindings.push((take(&mut variables), take(&mut values))); @@ -215,8 +217,8 @@ impl JsonQueryResultsReader { break; } JsonEvent::ObjectKey(key) => { - variables.push(key.to_owned()); - values.push(read_value(&mut reader, &mut buffer, 0)?); + variables.push(key.into_owned()); + values.push(read_value(&mut reader, 0)?); } _ => { return Err( @@ -227,7 +229,7 @@ impl JsonQueryResultsReader { } } "boolean" => { - return if let JsonEvent::Boolean(v) = reader.read_event(&mut buffer)? { + return if let JsonEvent::Boolean(v) = reader.read_next_event()? { Ok(Self::Boolean(v)) } else { Err(SyntaxError::msg("Unexpected boolean value").into()) @@ -257,38 +259,37 @@ impl JsonQueryResultsReader { } } -pub struct JsonSolutionsReader { +pub struct JsonSolutionsReader { mapping: BTreeMap, kind: JsonSolutionsReaderKind, } -enum JsonSolutionsReaderKind { +enum JsonSolutionsReaderKind { Streaming { - reader: JsonReader, - buffer: Vec, + reader: FromReadJsonReader, }, Buffered { bindings: std::vec::IntoIter<(Vec, Vec)>, }, } -impl JsonSolutionsReader { +impl JsonSolutionsReader { pub fn read_next(&mut self) -> Result>>, ParseError> { match &mut self.kind { - JsonSolutionsReaderKind::Streaming { reader, buffer } => { + JsonSolutionsReaderKind::Streaming { reader } => { let mut new_bindings = vec![None; self.mapping.len()]; loop { - match reader.read_event(buffer)? { + match reader.read_next_event()? { JsonEvent::StartObject => (), JsonEvent::EndObject => return Ok(Some(new_bindings)), JsonEvent::EndArray | JsonEvent::Eof => return Ok(None), JsonEvent::ObjectKey(key) => { - let k = *self.mapping.get(key).ok_or_else(|| { + let k = *self.mapping.get(key.as_ref()).ok_or_else(|| { SyntaxError::msg(format!( "The variable {key} has not been defined in the header" )) })?; - new_bindings[k] = Some(read_value(reader, buffer, 0)?) + new_bindings[k] = Some(read_value(reader, 0)?) } _ => return Err(SyntaxError::msg("Invalid result serialization").into()), } @@ -314,9 +315,8 @@ impl JsonSolutionsReader { } } -fn read_value( - reader: &mut JsonReader, - buffer: &mut Vec, +fn read_value( + reader: &mut FromReadJsonReader, number_of_recursive_calls: usize, ) -> Result { enum Type { @@ -351,28 +351,29 @@ fn read_value( let mut predicate = None; #[cfg(feature = "rdf-star")] let mut object = None; - if reader.read_event(buffer)? != JsonEvent::StartObject { + if reader.read_next_event()? != JsonEvent::StartObject { return Err(SyntaxError::msg("Term serializations should be an object").into()); } loop { - match reader.read_event(buffer)? { - JsonEvent::ObjectKey(key) => match key { + #[allow(unsafe_code)] + // SAFETY: Borrow checker workaround https://github.com/rust-lang/rust/issues/70255 + let next_event = unsafe { + let r: *mut FromReadJsonReader = reader; + &mut *r + } + .read_next_event()?; + match next_event { + JsonEvent::ObjectKey(key) => match key.as_ref() { "type" => state = Some(State::Type), "value" => state = Some(State::Value), "xml:lang" => state = Some(State::Lang), "datatype" => state = Some(State::Datatype), #[cfg(feature = "rdf-star")] - "subject" => { - subject = Some(read_value(reader, buffer, number_of_recursive_calls + 1)?) - } + "subject" => subject = Some(read_value(reader, number_of_recursive_calls + 1)?), #[cfg(feature = "rdf-star")] - "predicate" => { - predicate = Some(read_value(reader, buffer, number_of_recursive_calls + 1)?) - } + "predicate" => predicate = Some(read_value(reader, number_of_recursive_calls + 1)?), #[cfg(feature = "rdf-star")] - "object" => { - object = Some(read_value(reader, buffer, number_of_recursive_calls + 1)?) - } + "object" => object = Some(read_value(reader, number_of_recursive_calls + 1)?), _ => { return Err(SyntaxError::msg(format!( "Unexpected key in term serialization: '{key}'" @@ -389,7 +390,7 @@ fn read_value( } JsonEvent::String(s) => match state { Some(State::Type) => { - match s { + match s.as_ref() { "uri" => t = Some(Type::Uri), "bnode" => t = Some(Type::BNode), "literal" | "typed-literal" => t = Some(Type::Literal), @@ -404,11 +405,11 @@ fn read_value( state = None; } Some(State::Value) => { - value = Some(s.to_owned()); + value = Some(s.into_owned()); state = None; } Some(State::Lang) => { - lang = Some(s.to_owned()); + lang = Some(s.into_owned()); state = None; } Some(State::Datatype) => { @@ -458,7 +459,7 @@ fn read_value( )).into()) } } - Literal::new_language_tagged_literal(value, &lang).map_err(|e| { + Literal::new_language_tagged_literal(value, &*lang).map_err(|e| { SyntaxError::msg(format!("Invalid xml:lang value '{lang}': {e}")) })? } @@ -511,25 +512,22 @@ fn read_value( } } -fn read_head( - reader: &mut JsonReader, - buffer: &mut Vec, -) -> Result, ParseError> { - if reader.read_event(buffer)? != JsonEvent::StartObject { +fn read_head(reader: &mut FromReadJsonReader) -> Result, ParseError> { + if reader.read_next_event()? != JsonEvent::StartObject { return Err(SyntaxError::msg("head should be an object").into()); } let mut variables = Vec::new(); loop { - match reader.read_event(buffer)? { - JsonEvent::ObjectKey(key) => match key { + match reader.read_next_event()? { + JsonEvent::ObjectKey(key) => match key.as_ref() { "vars" => { - if reader.read_event(buffer)? != JsonEvent::StartArray { + if reader.read_next_event()? != JsonEvent::StartArray { return Err(SyntaxError::msg("Variable list should be an array").into()); } loop { - match reader.read_event(buffer)? { + match reader.read_next_event()? { JsonEvent::String(s) => { - let new_var = Variable::new(s).map_err(|e| { + let new_var = Variable::new(s.as_ref()).map_err(|e| { SyntaxError::msg(format!( "Invalid variable declaration '{s}': {e}" )) @@ -552,11 +550,11 @@ fn read_head( } } "link" => { - if reader.read_event(buffer)? != JsonEvent::StartArray { + if reader.read_next_event()? != JsonEvent::StartArray { return Err(SyntaxError::msg("Variable list should be an array").into()); } loop { - match reader.read_event(buffer)? { + match reader.read_next_event()? { JsonEvent::String(_) => (), JsonEvent::EndArray => break, _ => { @@ -565,7 +563,7 @@ fn read_head( } } } - _ => ignore_value(reader, buffer)?, + _ => ignore_value(reader)?, }, JsonEvent::EndObject => return Ok(variables), _ => return Err(SyntaxError::msg("Invalid head serialization").into()), @@ -573,13 +571,10 @@ fn read_head( } } -fn ignore_value( - reader: &mut JsonReader, - buffer: &mut Vec, -) -> Result<(), ParseError> { +fn ignore_value(reader: &mut FromReadJsonReader) -> Result<(), ParseError> { let mut nesting = 0; loop { - match reader.read_event(buffer)? { + match reader.read_next_event()? { JsonEvent::Boolean(_) | JsonEvent::Null | JsonEvent::Number(_) diff --git a/lib/src/sparql/eval.rs b/lib/src/sparql/eval.rs index ed9feba7..e21ee891 100644 --- a/lib/src/sparql/eval.rs +++ b/lib/src/sparql/eval.rs @@ -8,7 +8,7 @@ use crate::sparql::service::ServiceHandler; use crate::storage::numeric_encoder::*; use crate::storage::small_string::SmallString; use digest::Digest; -use json_event_parser::{JsonEvent, JsonWriter}; +use json_event_parser::{JsonEvent, ToWriteJsonWriter}; use md5::Md5; use oxilangtag::LanguageTag; use oxiri::Iri; @@ -5676,21 +5676,21 @@ pub struct EvalNodeWithStats { impl EvalNodeWithStats { pub fn json_node( &self, - writer: &mut JsonWriter, + writer: &mut ToWriteJsonWriter, with_stats: bool, ) -> io::Result<()> { writer.write_event(JsonEvent::StartObject)?; - writer.write_event(JsonEvent::ObjectKey("name"))?; - writer.write_event(JsonEvent::String(&self.label))?; + writer.write_event(JsonEvent::ObjectKey("name".into()))?; + writer.write_event(JsonEvent::String((&self.label).into()))?; if with_stats { - writer.write_event(JsonEvent::ObjectKey("number of results"))?; - writer.write_event(JsonEvent::Number(&self.exec_count.get().to_string()))?; + writer.write_event(JsonEvent::ObjectKey("number of results".into()))?; + writer.write_event(JsonEvent::Number(self.exec_count.get().to_string().into()))?; if let Some(duration) = self.exec_duration.get() { - writer.write_event(JsonEvent::ObjectKey("duration in seconds"))?; - writer.write_event(JsonEvent::Number(&duration.as_seconds().to_string()))?; + writer.write_event(JsonEvent::ObjectKey("duration in seconds".into()))?; + writer.write_event(JsonEvent::Number(duration.as_seconds().to_string().into()))?; } } - writer.write_event(JsonEvent::ObjectKey("children"))?; + writer.write_event(JsonEvent::ObjectKey("children".into()))?; writer.write_event(JsonEvent::StartArray)?; for child in &self.children { child.json_node(writer, with_stats)?; diff --git a/lib/src/sparql/mod.rs b/lib/src/sparql/mod.rs index 82944f95..1972a77e 100644 --- a/lib/src/sparql/mod.rs +++ b/lib/src/sparql/mod.rs @@ -22,7 +22,7 @@ pub use crate::sparql::service::ServiceHandler; use crate::sparql::service::{EmptyServiceHandler, ErrorConversionServiceHandler}; pub(crate) use crate::sparql::update::evaluate_update; use crate::storage::StorageReader; -use json_event_parser::{JsonEvent, JsonWriter}; +use json_event_parser::{JsonEvent, ToWriteJsonWriter}; pub use oxrdf::{Variable, VariableNameParseError}; use oxsdatatypes::{DayTimeDuration, Float}; pub use spargebra::ParseError; @@ -279,22 +279,22 @@ pub struct QueryExplanation { impl QueryExplanation { /// Writes the explanation as JSON. - pub fn write_in_json(&self, output: impl io::Write) -> io::Result<()> { - let mut writer = JsonWriter::from_writer(output); + pub fn write_in_json(&self, write: impl io::Write) -> io::Result<()> { + let mut writer = ToWriteJsonWriter::new(write); writer.write_event(JsonEvent::StartObject)?; if let Some(parsing_duration) = self.parsing_duration { - writer.write_event(JsonEvent::ObjectKey("parsing duration in seconds"))?; + writer.write_event(JsonEvent::ObjectKey("parsing duration in seconds".into()))?; writer.write_event(JsonEvent::Number( - &parsing_duration.as_seconds().to_string(), + parsing_duration.as_seconds().to_string().into(), ))?; } if let Some(planning_duration) = self.planning_duration { - writer.write_event(JsonEvent::ObjectKey("planning duration in seconds"))?; + writer.write_event(JsonEvent::ObjectKey("planning duration in seconds".into()))?; writer.write_event(JsonEvent::Number( - &planning_duration.as_seconds().to_string(), + planning_duration.as_seconds().to_string().into(), ))?; } - writer.write_event(JsonEvent::ObjectKey("plan"))?; + writer.write_event(JsonEvent::ObjectKey("plan".into()))?; self.inner.json_node(&mut writer, self.with_stats)?; writer.write_event(JsonEvent::EndObject) } diff --git a/lints/test_debian_compatibility.py b/lints/test_debian_compatibility.py index 10f6596a..d409bb9d 100644 --- a/lints/test_debian_compatibility.py +++ b/lints/test_debian_compatibility.py @@ -5,7 +5,7 @@ from urllib.request import urlopen TARGET_DEBIAN_VERSIONS = ["sid"] IGNORE_PACKAGES = {"oxigraph-js", "oxigraph-testsuite", "pyoxigraph", "sparql-smith"} -ALLOWED_MISSING_PACKAGES = {"escargot", "oxhttp", "quick-xml"} +ALLOWED_MISSING_PACKAGES = {"escargot", "json-event-parser", "oxhttp", "quick-xml"} base_path = Path(__file__).parent.parent From d280f7d2f76ab9cd4f5457f93badc96b8b2f4478 Mon Sep 17 00:00:00 2001 From: Tpt Date: Mon, 25 Sep 2023 22:06:51 +0200 Subject: [PATCH 103/217] Adds basic location support to sparesults SyntaxError --- lib/oxrdfio/src/lib.rs | 2 +- lib/sparesults/src/csv.rs | 148 +++++++++++++++++++++++++----------- lib/sparesults/src/error.rs | 70 +++++++++++++++-- lib/sparesults/src/lib.rs | 2 +- lib/src/io/mod.rs | 2 +- lib/src/sparql/results.rs | 2 +- python/src/io.rs | 8 +- python/src/sparql.rs | 66 ++++++++++++---- python/tests/test_io.py | 27 +++++++ 9 files changed, 252 insertions(+), 75 deletions(-) diff --git a/lib/oxrdfio/src/lib.rs b/lib/oxrdfio/src/lib.rs index d31ec656..4b51dcf8 100644 --- a/lib/oxrdfio/src/lib.rs +++ b/lib/oxrdfio/src/lib.rs @@ -9,7 +9,7 @@ mod format; mod parser; mod serializer; -pub use error::{ParseError, SyntaxError}; +pub use error::{ParseError, SyntaxError, TextPosition}; pub use format::RdfFormat; #[cfg(feature = "async-tokio")] pub use parser::FromTokioAsyncReadQuadReader; diff --git a/lib/sparesults/src/csv.rs b/lib/sparesults/src/csv.rs index 6fbfeb40..e45f521f 100644 --- a/lib/sparesults/src/csv.rs +++ b/lib/sparesults/src/csv.rs @@ -1,10 +1,10 @@ //! Implementation of [SPARQL 1.1 Query Results CSV and TSV Formats](https://www.w3.org/TR/sparql11-results-csv-tsv/) -use crate::error::{ParseError, SyntaxError, SyntaxErrorKind}; +use crate::error::{ParseError, SyntaxError, SyntaxErrorKind, TextPosition}; use memchr::memchr; use oxrdf::Variable; use oxrdf::{vocab::xsd, *}; -use std::io::{self, BufRead, Read, Write}; +use std::io::{self, Read, Write}; use std::str::{self, FromStr}; const MAX_BUFFER_SIZE: usize = 4096 * 4096; @@ -283,12 +283,13 @@ pub enum TsvQueryResultsReader { } impl TsvQueryResultsReader { - pub fn read(read: R) -> Result { - let mut reader = LineReader::new(read); + pub fn read(mut read: R) -> Result { + let mut reader = LineReader::new(); + let mut buffer = Vec::new(); // We read the header let line = reader - .next_line()? + .next_line(&mut buffer, &mut read)? .trim_matches(|c| matches!(c, ' ' | '\r' | '\n')); if line.eq_ignore_ascii_case("true") { return Ok(Self::Boolean(true)); @@ -318,34 +319,65 @@ impl TsvQueryResultsReader { let column_len = variables.len(); Ok(Self::Solutions { variables, - solutions: TsvSolutionsReader { reader, column_len }, + solutions: TsvSolutionsReader { + read, + buffer, + reader, + column_len, + }, }) } } pub struct TsvSolutionsReader { - reader: LineReader, + read: R, + buffer: Vec, + reader: LineReader, column_len: usize, } -impl TsvSolutionsReader { +impl TsvSolutionsReader { + #[allow(clippy::unwrap_in_result)] pub fn read_next(&mut self) -> Result>>, ParseError> { - let line = self.reader.next_line()?; + let line = self.reader.next_line(&mut self.buffer, &mut self.read)?; if line.is_empty() { return Ok(None); // EOF } let elements = line .split('\t') - .map(|v| { + .enumerate() + .map(|(i, v)| { let v = v.trim(); if v.is_empty() { Ok(None) } else { - Ok(Some(Term::from_str(v).map_err(|e| SyntaxError { - inner: SyntaxErrorKind::Term { - error: e, - term: v.into(), - }, + Ok(Some(Term::from_str(v).map_err(|e| { + let start_position_char = line + .split('\t') + .take(i) + .map(|c| c.chars().count() + 1) + .sum::(); + let start_position_bytes = + line.split('\t').take(i).map(|c| c.len() + 1).sum::(); + SyntaxError { + inner: SyntaxErrorKind::Term { + error: e, + term: v.into(), + location: TextPosition { + line: self.reader.line_count - 1, + column: start_position_char.try_into().unwrap(), + offset: self.reader.last_line_start + + u64::try_from(start_position_bytes).unwrap(), + }..TextPosition { + line: self.reader.line_count - 1, + column: (start_position_char + v.chars().count()) + .try_into() + .unwrap(), + offset: self.reader.last_line_start + + u64::try_from(start_position_bytes + v.len()).unwrap(), + }, + }, + } })?)) } }) @@ -355,64 +387,88 @@ impl TsvSolutionsReader { } else if self.column_len == 0 && elements == [None] { Ok(Some(Vec::new())) // Zero columns case } else { - Err(SyntaxError::msg(format!( - "This TSV files has {} columns but we found a row with {} columns: {}", - self.column_len, - elements.len(), - line - )) + Err(SyntaxError::located_message( + format!( + "This TSV files has {} columns but we found a row on line {} with {} columns: {}", + self.column_len, + self.reader.line_count - 1, + elements.len(), + line + ), + TextPosition { + line: self.reader.line_count - 1, + column: 0, + offset: self.reader.last_line_start, + }..TextPosition { + line: self.reader.line_count - 1, + column: line.chars().count().try_into().unwrap(), + offset: self.reader.last_line_end, + }, + ) .into()) } } } -struct LineReader { - read: R, - buffer: Vec, - start: usize, - end: usize, +struct LineReader { + buffer_start: usize, + buffer_end: usize, + line_count: u64, + last_line_start: u64, + last_line_end: u64, } -impl LineReader { - fn new(read: R) -> Self { +impl LineReader { + fn new() -> Self { Self { - read, - buffer: Vec::new(), - start: 0, - end: 0, + buffer_start: 0, + buffer_end: 0, + line_count: 0, + last_line_start: 0, + last_line_end: 0, } } - fn next_line(&mut self) -> io::Result<&str> { - self.buffer.copy_within(self.start..self.end, 0); - self.end -= self.start; - self.start = 0; + #[allow(clippy::unwrap_in_result)] + fn next_line<'a>( + &mut self, + buffer: &'a mut Vec, + read: &mut impl Read, + ) -> io::Result<&'a str> { let line_end = loop { - if let Some(eol) = memchr(b'\n', &self.buffer[self.start..self.end]) { - break self.start + eol + 1; + if let Some(eol) = memchr(b'\n', &buffer[self.buffer_start..self.buffer_end]) { + break self.buffer_start + eol + 1; + } + if self.buffer_start > buffer.len() / 2 { + buffer.copy_within(self.buffer_start..self.buffer_end, 0); + self.buffer_end -= self.buffer_start; + self.buffer_start = 0; } - if self.end + 1024 > self.buffer.len() { - if self.end + 1024 > MAX_BUFFER_SIZE { + if self.buffer_end + 1024 > buffer.len() { + if self.buffer_end + 1024 > MAX_BUFFER_SIZE { return Err(io::Error::new( io::ErrorKind::OutOfMemory, format!("Reached the buffer maximal size of {MAX_BUFFER_SIZE}"), )); } - self.buffer.resize(self.end + 1024, b'\0'); + buffer.resize(self.buffer_end + 1024, b'\0'); } - let read = self.read.read(&mut self.buffer[self.end..])?; + let read = read.read(&mut buffer[self.buffer_end..])?; if read == 0 { - break self.end; + break self.buffer_end; } - self.end += read; + self.buffer_end += read; }; - let result = str::from_utf8(&self.buffer[self.start..line_end]).map_err(|e| { + let result = str::from_utf8(&buffer[self.buffer_start..line_end]).map_err(|e| { io::Error::new( io::ErrorKind::InvalidData, format!("Invalid UTF-8 in the TSV file: {e}"), ) }); - self.start = line_end; + self.line_count += 1; + self.last_line_start = self.last_line_end; + self.last_line_end += u64::try_from(line_end - self.buffer_start).unwrap(); + self.buffer_start = line_end; result } } diff --git a/lib/sparesults/src/error.rs b/lib/sparesults/src/error.rs index 1096b393..b8fe2eff 100644 --- a/lib/sparesults/src/error.rs +++ b/lib/sparesults/src/error.rs @@ -1,5 +1,6 @@ use oxrdf::TermParseError; use std::error::Error; +use std::ops::Range; use std::sync::Arc; use std::{fmt, io}; @@ -90,8 +91,15 @@ pub struct SyntaxError { pub(crate) enum SyntaxErrorKind { Json(json_event_parser::SyntaxError), Xml(quick_xml::Error), - Term { error: TermParseError, term: String }, - Msg { msg: String }, + Term { + error: TermParseError, + term: String, + location: Range, + }, + Msg { + msg: String, + location: Option>, + }, } impl SyntaxError { @@ -99,7 +107,45 @@ impl SyntaxError { #[inline] pub(crate) fn msg(msg: impl Into) -> Self { Self { - inner: SyntaxErrorKind::Msg { msg: msg.into() }, + inner: SyntaxErrorKind::Msg { + msg: msg.into(), + location: None, + }, + } + } + + /// Builds an error from a printable error message and a location + #[inline] + pub(crate) fn located_message(msg: impl Into, location: Range) -> Self { + Self { + inner: SyntaxErrorKind::Msg { + msg: msg.into(), + location: Some(location), + }, + } + } + + /// The location of the error inside of the file. + #[inline] + pub fn location(&self) -> Option> { + match &self.inner { + SyntaxErrorKind::Json(e) => { + let location = e.location(); + Some( + TextPosition { + line: location.start.line, + column: location.start.column, + offset: location.start.offset, + }..TextPosition { + line: location.end.line, + column: location.end.column, + offset: location.end.offset, + }, + ) + } + SyntaxErrorKind::Term { location, .. } => Some(location.clone()), + SyntaxErrorKind::Msg { location, .. } => location.clone(), + SyntaxErrorKind::Xml(_) => None, } } } @@ -110,8 +156,12 @@ impl fmt::Display for SyntaxError { match &self.inner { SyntaxErrorKind::Json(e) => e.fmt(f), SyntaxErrorKind::Xml(e) => e.fmt(f), - SyntaxErrorKind::Term { error, term } => write!(f, "{error}: {term}"), - SyntaxErrorKind::Msg { msg } => f.write_str(msg), + SyntaxErrorKind::Term { + error, + term, + location, + } => write!(f, "{error} on '{term}' in line {}", location.start.line + 1), + SyntaxErrorKind::Msg { msg, .. } => f.write_str(msg), } } } @@ -144,7 +194,7 @@ impl From for io::Error { _ => Self::new(io::ErrorKind::InvalidData, error), }, SyntaxErrorKind::Term { .. } => Self::new(io::ErrorKind::InvalidData, error), - SyntaxErrorKind::Msg { msg } => Self::new(io::ErrorKind::InvalidData, msg), + SyntaxErrorKind::Msg { msg, .. } => Self::new(io::ErrorKind::InvalidData, msg), } } } @@ -156,3 +206,11 @@ impl From for SyntaxError { } } } + +/// A position in a text i.e. a `line` number starting from 0, a `column` number starting from 0 (in number of code points) and a global file `offset` starting from 0 (in number of bytes). +#[derive(Eq, PartialEq, Debug, Clone, Copy)] +pub struct TextPosition { + pub line: u64, + pub column: u64, + pub offset: u64, +} diff --git a/lib/sparesults/src/lib.rs b/lib/sparesults/src/lib.rs index 95bcef93..301dc2c8 100644 --- a/lib/sparesults/src/lib.rs +++ b/lib/sparesults/src/lib.rs @@ -13,7 +13,7 @@ mod serializer; pub mod solution; mod xml; -pub use crate::error::{ParseError, SyntaxError}; +pub use crate::error::{ParseError, SyntaxError, TextPosition}; pub use crate::format::QueryResultsFormat; pub use crate::parser::{FromReadQueryResultsReader, FromReadSolutionsReader, QueryResultsParser}; pub use crate::serializer::{QueryResultsSerializer, ToWriteSolutionsWriter}; diff --git a/lib/src/io/mod.rs b/lib/src/io/mod.rs index 2e4dd2f4..5e1cb271 100644 --- a/lib/src/io/mod.rs +++ b/lib/src/io/mod.rs @@ -35,6 +35,6 @@ pub use self::read::{DatasetParser, GraphParser}; #[allow(deprecated)] pub use self::write::{DatasetSerializer, GraphSerializer}; pub use oxrdfio::{ - FromReadQuadReader, ParseError, RdfFormat, RdfParser, RdfSerializer, SyntaxError, + FromReadQuadReader, ParseError, RdfFormat, RdfParser, RdfSerializer, SyntaxError, TextPosition, ToWriteQuadWriter, }; diff --git a/lib/src/sparql/results.rs b/lib/src/sparql/results.rs index 0716752b..bbafe70d 100644 --- a/lib/src/sparql/results.rs +++ b/lib/src/sparql/results.rs @@ -43,5 +43,5 @@ pub use sparesults::{ FromReadQueryResultsReader, FromReadSolutionsReader, ParseError, QueryResultsFormat, - QueryResultsParser, QueryResultsSerializer, SyntaxError, + QueryResultsParser, QueryResultsSerializer, SyntaxError, TextPosition, }; diff --git a/python/src/io.rs b/python/src/io.rs index e8da9a8c..4e04a4da 100644 --- a/python/src/io.rs +++ b/python/src/io.rs @@ -401,7 +401,7 @@ pub fn map_parse_error(error: ParseError, file_path: Option) -> PyErr { match error { ParseError::Syntax(error) => { // Python 3.9 does not support end line and end column - if python_version() >= (3, 10, 0) { + if python_version() >= (3, 10) { let params = if let Some(location) = error.location() { ( file_path, @@ -458,12 +458,12 @@ pub fn allow_threads_unsafe(_py: Python<'_>, f: impl FnOnce() -> T) -> T { f() } -fn python_version() -> (u8, u8, u8) { - static VERSION: OnceLock<(u8, u8, u8)> = OnceLock::new(); +pub fn python_version() -> (u8, u8) { + static VERSION: OnceLock<(u8, u8)> = OnceLock::new(); *VERSION.get_or_init(|| { Python::with_gil(|py| { let v = py.version_info(); - (v.major, v.minor, v.patch) + (v.major, v.minor) }) }) } diff --git a/python/src/sparql.rs b/python/src/sparql.rs index 1316eecf..b30d27fe 100644 --- a/python/src/sparql.rs +++ b/python/src/sparql.rs @@ -190,7 +190,10 @@ pub struct PyQuerySolutions { } enum PyQuerySolutionsVariant { Query(QuerySolutionIter), - Reader(FromReadSolutionsReader>), + Reader { + iter: FromReadSolutionsReader>, + file_path: Option, + }, } #[pymethods] @@ -207,8 +210,8 @@ impl PyQuerySolutions { PyQuerySolutionsVariant::Query(inner) => { inner.variables().iter().map(|v| v.clone().into()).collect() } - PyQuerySolutionsVariant::Reader(inner) => { - inner.variables().iter().map(|v| v.clone().into()).collect() + PyQuerySolutionsVariant::Reader { iter, .. } => { + iter.variables().iter().map(|v| v.clone().into()).collect() } } } @@ -252,7 +255,9 @@ impl PyQuerySolutions { output, match &self.inner { PyQuerySolutionsVariant::Query(inner) => inner.variables().to_vec(), - PyQuerySolutionsVariant::Reader(inner) => inner.variables().to_vec(), + PyQuerySolutionsVariant::Reader { iter, .. } => { + iter.variables().to_vec() + } }, ) .map_err(map_io_err)?; @@ -264,10 +269,12 @@ impl PyQuerySolutions { .map_err(map_io_err)?; } } - PyQuerySolutionsVariant::Reader(inner) => { - for solution in inner { + PyQuerySolutionsVariant::Reader { iter, file_path } => { + for solution in iter { writer - .write(&solution.map_err(map_query_results_parse_error)?) + .write(&solution.map_err(|e| { + map_query_results_parse_error(e, file_path.clone()) + })?) .map_err(map_io_err)?; } } @@ -290,10 +297,10 @@ impl PyQuerySolutions { PyQuerySolutionsVariant::Query(inner) => allow_threads_unsafe(py, || { inner.next().transpose().map_err(map_evaluation_error) }), - PyQuerySolutionsVariant::Reader(inner) => inner + PyQuerySolutionsVariant::Reader { iter, file_path } => iter .next() .transpose() - .map_err(map_query_results_parse_error), + .map_err(|e| map_query_results_parse_error(e, file_path.clone())), }? .map(move |inner| PyQuerySolution { inner })) } @@ -498,10 +505,10 @@ pub fn parse_query_results( }; let results = QueryResultsParser::from_format(format) .parse_read(BufReader::new(input)) - .map_err(map_query_results_parse_error)?; + .map_err(|e| map_query_results_parse_error(e, file_path.clone()))?; Ok(match results { - FromReadQueryResultsReader::Solutions(inner) => PyQuerySolutions { - inner: PyQuerySolutionsVariant::Reader(inner), + FromReadQueryResultsReader::Solutions(iter) => PyQuerySolutions { + inner: PyQuerySolutionsVariant::Reader { iter, file_path }, } .into_py(py), FromReadQueryResultsReader::Boolean(inner) => PyQueryBoolean { inner }.into_py(py), @@ -513,7 +520,7 @@ pub fn map_evaluation_error(error: EvaluationError) -> PyErr { EvaluationError::Parsing(error) => PySyntaxError::new_err(error.to_string()), EvaluationError::Storage(error) => map_storage_error(error), EvaluationError::GraphParsing(error) => map_parse_error(error, None), - EvaluationError::ResultsParsing(error) => map_query_results_parse_error(error), + EvaluationError::ResultsParsing(error) => map_query_results_parse_error(error, None), EvaluationError::ResultsSerialization(error) => map_io_err(error), EvaluationError::Service(error) => match error.downcast() { Ok(error) => map_io_err(*error), @@ -523,9 +530,38 @@ pub fn map_evaluation_error(error: EvaluationError) -> PyErr { } } -pub fn map_query_results_parse_error(error: ParseError) -> PyErr { +pub fn map_query_results_parse_error(error: ParseError, file_path: Option) -> PyErr { match error { - ParseError::Syntax(error) => PySyntaxError::new_err(error.to_string()), + ParseError::Syntax(error) => { + // Python 3.9 does not support end line and end column + if python_version() >= (3, 10) { + let params = if let Some(location) = error.location() { + ( + file_path, + Some(location.start.line + 1), + Some(location.start.column + 1), + None::>, + Some(location.end.line + 1), + Some(location.end.column + 1), + ) + } else { + (None, None, None, None, None, None) + }; + PySyntaxError::new_err((error.to_string(), params)) + } else { + let params = if let Some(location) = error.location() { + ( + file_path, + Some(location.start.line + 1), + Some(location.start.column + 1), + None::>, + ) + } else { + (None, None, None, None) + }; + PySyntaxError::new_err((error.to_string(), params)) + } + } ParseError::Io(error) => map_io_err(error), } } diff --git a/python/tests/test_io.py b/python/tests/test_io.py index 9d4a1326..c761148f 100644 --- a/python/tests/test_io.py +++ b/python/tests/test_io.py @@ -205,3 +205,30 @@ class TestParseQuerySolutions(unittest.TestCase): def test_parse_io_error(self) -> None: with self.assertRaises(UnsupportedOperation) as _, TemporaryFile("wb") as fp: parse_query_results(fp, "srx") + + def test_parse_syntax_error_json(self) -> None: + with NamedTemporaryFile() as fp: + fp.write(b"{]") + fp.flush() + with self.assertRaises(SyntaxError) as ctx: + list(parse_query_results(fp.name, "srj")) # type: ignore[arg-type] + self.assertEqual(ctx.exception.filename, fp.name) + self.assertEqual(ctx.exception.lineno, 1) + self.assertEqual(ctx.exception.offset, 2) + if sys.version_info >= (3, 10): + self.assertEqual(ctx.exception.end_lineno, 1) + self.assertEqual(ctx.exception.end_offset, 3) + + def test_parse_syntax_error_tsv(self) -> None: + with NamedTemporaryFile() as fp: + fp.write(b"?a\t?test\n") + fp.write(b"1\t\n") + fp.flush() + with self.assertRaises(SyntaxError) as ctx: + list(parse_query_results(fp.name, "tsv")) # type: ignore[arg-type] + self.assertEqual(ctx.exception.filename, fp.name) + self.assertEqual(ctx.exception.lineno, 2) + self.assertEqual(ctx.exception.offset, 3) + if sys.version_info >= (3, 10): + self.assertEqual(ctx.exception.end_lineno, 2) + self.assertEqual(ctx.exception.end_offset, 9) From 38844f6436524e72b69985fcc285fad0a91a4713 Mon Sep 17 00:00:00 2001 From: Tpt Date: Tue, 26 Sep 2023 22:04:57 +0200 Subject: [PATCH 104/217] sparesults: use Read instead of BufRead as input type Allows to avoid an extra intermediate buffer for JSON and TSV --- lib/sparesults/src/parser.rs | 18 ++++++++++-------- lib/sparesults/src/xml.rs | 14 +++++++------- lib/src/sparql/model.rs | 23 ++++++++++------------- lib/src/sparql/service.rs | 3 +-- python/src/sparql.rs | 5 ++--- testsuite/src/sparql_evaluator.rs | 11 ++++------- 6 files changed, 34 insertions(+), 40 deletions(-) diff --git a/lib/sparesults/src/parser.rs b/lib/sparesults/src/parser.rs index fd7a403d..8833f9ac 100644 --- a/lib/sparesults/src/parser.rs +++ b/lib/sparesults/src/parser.rs @@ -5,7 +5,7 @@ use crate::json::{JsonQueryResultsReader, JsonSolutionsReader}; use crate::solution::QuerySolution; use crate::xml::{XmlQueryResultsReader, XmlSolutionsReader}; use oxrdf::Variable; -use std::io::BufRead; +use std::io::Read; use std::rc::Rc; /// Parsers for [SPARQL query](https://www.w3.org/TR/sparql11-query/) results serialization formats. @@ -47,6 +47,8 @@ impl QueryResultsParser { /// Reads a result file. /// + /// Reads are buffered. + /// /// Example in XML (the API is the same for JSON and TSV): /// ``` /// use sparesults::{QueryResultsFormat, QueryResultsParser, FromReadQueryResultsReader}; @@ -68,7 +70,7 @@ impl QueryResultsParser { /// } /// # Result::<(),sparesults::ParseError>::Ok(()) /// ``` - pub fn parse_read( + pub fn parse_read( &self, reader: R, ) -> Result, ParseError> { @@ -108,7 +110,7 @@ impl QueryResultsParser { } #[deprecated(note = "Use parse_read")] - pub fn read_results( + pub fn read_results( &self, reader: R, ) -> Result, ParseError> { @@ -141,7 +143,7 @@ impl QueryResultsParser { /// } /// # Result::<(),sparesults::ParseError>::Ok(()) /// ``` -pub enum FromReadQueryResultsReader { +pub enum FromReadQueryResultsReader { Solutions(FromReadSolutionsReader), Boolean(bool), } @@ -165,18 +167,18 @@ pub enum FromReadQueryResultsReader { /// # Result::<(),sparesults::ParseError>::Ok(()) /// ``` #[allow(clippy::rc_buffer)] -pub struct FromReadSolutionsReader { +pub struct FromReadSolutionsReader { variables: Rc>, solutions: SolutionsReaderKind, } -enum SolutionsReaderKind { +enum SolutionsReaderKind { Xml(XmlSolutionsReader), Json(JsonSolutionsReader), Tsv(TsvSolutionsReader), } -impl FromReadSolutionsReader { +impl FromReadSolutionsReader { /// Ordered list of the declared variables at the beginning of the results. /// /// Example in TSV (the API is the same for JSON and XML): @@ -196,7 +198,7 @@ impl FromReadSolutionsReader { } } -impl Iterator for FromReadSolutionsReader { +impl Iterator for FromReadSolutionsReader { type Item = Result; fn next(&mut self) -> Option> { diff --git a/lib/sparesults/src/xml.rs b/lib/sparesults/src/xml.rs index 84e7e099..73d26fff 100644 --- a/lib/sparesults/src/xml.rs +++ b/lib/sparesults/src/xml.rs @@ -8,7 +8,7 @@ use quick_xml::events::{BytesDecl, BytesEnd, BytesStart, BytesText, Event}; use quick_xml::{Reader, Writer}; use std::borrow::Cow; use std::collections::BTreeMap; -use std::io::{self, BufRead, Write}; +use std::io::{self, BufReader, Read, Write}; use std::str; use std::sync::Arc; @@ -157,7 +157,7 @@ fn write_xml_term( Ok(()) } -pub enum XmlQueryResultsReader { +pub enum XmlQueryResultsReader { Solutions { variables: Vec, solutions: XmlSolutionsReader, @@ -165,7 +165,7 @@ pub enum XmlQueryResultsReader { Boolean(bool), } -impl XmlQueryResultsReader { +impl XmlQueryResultsReader { pub fn read(source: R) -> Result { enum State { Start, @@ -175,7 +175,7 @@ impl XmlQueryResultsReader { Boolean, } - let mut reader = Reader::from_reader(source); + let mut reader = Reader::from_reader(BufReader::new(source)); reader.trim_text(true); reader.expand_empty_elements(true); @@ -293,8 +293,8 @@ enum State { End, } -pub struct XmlSolutionsReader { - reader: Reader, +pub struct XmlSolutionsReader { + reader: Reader>, buffer: Vec, mapping: BTreeMap, stack: Vec, @@ -303,7 +303,7 @@ pub struct XmlSolutionsReader { object_stack: Vec, } -impl XmlSolutionsReader { +impl XmlSolutionsReader { pub fn read_next(&mut self) -> Result>>, ParseError> { let mut state = State::Start; diff --git a/lib/src/sparql/model.rs b/lib/src/sparql/model.rs index 8ef06ecb..0c7becc6 100644 --- a/lib/src/sparql/model.rs +++ b/lib/src/sparql/model.rs @@ -7,7 +7,7 @@ use crate::sparql::results::{ }; use oxrdf::{Variable, VariableRef}; pub use sparesults::QuerySolution; -use std::io::{BufRead, Write}; +use std::io::{Read, Write}; use std::rc::Rc; /// Results of a [SPARQL query](https://www.w3.org/TR/sparql11-query/). @@ -22,12 +22,9 @@ pub enum QueryResults { impl QueryResults { /// Reads a SPARQL query results serialization. - pub fn read( - reader: impl BufRead + 'static, - format: QueryResultsFormat, - ) -> Result { + pub fn read(read: impl Read + 'static, format: QueryResultsFormat) -> Result { Ok(QueryResultsParser::from_format(format) - .parse_read(reader)? + .parse_read(read)? .into()) } @@ -51,19 +48,19 @@ impl QueryResults { /// ``` pub fn write( self, - writer: impl Write, + write: impl Write, format: QueryResultsFormat, ) -> Result<(), EvaluationError> { let serializer = QueryResultsSerializer::from_format(format); match self { Self::Boolean(value) => { serializer - .serialize_boolean_to_write(writer, value) + .serialize_boolean_to_write(write, value) .map_err(EvaluationError::ResultsSerialization)?; } Self::Solutions(solutions) => { let mut writer = serializer - .serialize_solutions_to_write(writer, solutions.variables().to_vec()) + .serialize_solutions_to_write(write, solutions.variables().to_vec()) .map_err(EvaluationError::ResultsSerialization)?; for solution in solutions { writer @@ -80,7 +77,7 @@ impl QueryResults { let o = VariableRef::new_unchecked("object"); let mut writer = serializer .serialize_solutions_to_write( - writer, + write, vec![s.into_owned(), p.into_owned(), o.into_owned()], ) .map_err(EvaluationError::ResultsSerialization)?; @@ -150,7 +147,7 @@ impl From for QueryResults { } } -impl From> for QueryResults { +impl From> for QueryResults { fn from(reader: FromReadQueryResultsReader) -> Self { match reader { FromReadQueryResultsReader::Solutions(s) => Self::Solutions(s.into()), @@ -211,7 +208,7 @@ impl QuerySolutionIter { } } -impl From> for QuerySolutionIter { +impl From> for QuerySolutionIter { fn from(reader: FromReadSolutionsReader) -> Self { Self { variables: Rc::new(reader.variables().to_vec()), @@ -279,10 +276,10 @@ mod tests { #![allow(clippy::panic_in_result_fn)] use super::*; + use std::io::Cursor; #[test] fn test_serialization_roundtrip() -> Result<(), EvaluationError> { - use std::io::Cursor; use std::str; for format in [ diff --git a/lib/src/sparql/service.rs b/lib/src/sparql/service.rs index ae397ee2..dec189ae 100644 --- a/lib/src/sparql/service.rs +++ b/lib/src/sparql/service.rs @@ -5,7 +5,6 @@ use crate::sparql::http::Client; use crate::sparql::model::QueryResults; use crate::sparql::results::QueryResultsFormat; use std::error::Error; -use std::io::BufReader; use std::time::Duration; /// Handler for [SPARQL 1.1 Federated Query](https://www.w3.org/TR/sparql11-federated-query/) SERVICE. @@ -121,6 +120,6 @@ impl ServiceHandler for SimpleServiceHandler { .map_err(|e| EvaluationError::Service(Box::new(e)))?; let format = QueryResultsFormat::from_media_type(&content_type) .ok_or_else(|| EvaluationError::UnsupportedContentType(content_type))?; - Ok(QueryResults::read(BufReader::new(body), format)?) + Ok(QueryResults::read(body, format)?) } } diff --git a/python/src/sparql.rs b/python/src/sparql.rs index b30d27fe..963aa0df 100644 --- a/python/src/sparql.rs +++ b/python/src/sparql.rs @@ -17,7 +17,6 @@ use pyo3::exceptions::{ }; use pyo3::prelude::*; use pyo3::types::PyBytes; -use std::io::BufReader; use std::path::PathBuf; use std::vec::IntoIter; @@ -191,7 +190,7 @@ pub struct PyQuerySolutions { enum PyQuerySolutionsVariant { Query(QuerySolutionIter), Reader { - iter: FromReadSolutionsReader>, + iter: FromReadSolutionsReader, file_path: Option, }, } @@ -504,7 +503,7 @@ pub fn parse_query_results( PyReadable::from_data(input) }; let results = QueryResultsParser::from_format(format) - .parse_read(BufReader::new(input)) + .parse_read(input) .map_err(|e| map_query_results_parse_error(e, file_path.clone()))?; Ok(match results { FromReadQueryResultsReader::Solutions(iter) => PyQuerySolutions { diff --git a/testsuite/src/sparql_evaluator.rs b/testsuite/src/sparql_evaluator.rs index 67081be6..14bdb38d 100644 --- a/testsuite/src/sparql_evaluator.rs +++ b/testsuite/src/sparql_evaluator.rs @@ -12,7 +12,7 @@ use oxigraph::store::Store; use sparopt::Optimizer; use std::collections::HashMap; use std::fmt::Write; -use std::io::{self, BufReader, Cursor}; +use std::io::{self, Cursor}; use std::ops::Deref; use std::str::FromStr; use std::sync::{Arc, Mutex, OnceLock}; @@ -106,12 +106,12 @@ fn evaluate_positive_result_syntax_test(test: &Test, format: QueryResultsFormat) .as_deref() .ok_or_else(|| anyhow!("No action found"))?; let actual_results = StaticQueryResults::from_query_results( - QueryResults::read(Cursor::new(read_file_to_string(action_file)?), format)?, + QueryResults::read(read_file(action_file)?, format)?, true, )?; if let Some(result_file) = test.result.as_deref() { let expected_results = StaticQueryResults::from_query_results( - QueryResults::read(Cursor::new(read_file_to_string(result_file)?), format)?, + QueryResults::read(read_file(result_file)?, format)?, true, )?; ensure!( @@ -277,10 +277,7 @@ fn load_sparql_query_result(url: &str) -> Result { .rsplit_once('.') .and_then(|(_, extension)| QueryResultsFormat::from_extension(extension)) { - StaticQueryResults::from_query_results( - QueryResults::read(BufReader::new(read_file(url)?), format)?, - false, - ) + StaticQueryResults::from_query_results(QueryResults::read(read_file(url)?, format)?, false) } else { StaticQueryResults::from_graph(&load_graph(url, guess_rdf_format(url)?, false)?) } From 70b1c52166b36e80dc5caafd1e0db36038b290aa Mon Sep 17 00:00:00 2001 From: Tpt Date: Fri, 6 Oct 2023 11:03:04 +0200 Subject: [PATCH 105/217] Python: uses 3.12 in CI + builds for it --- .github/workflows/artifacts.yml | 4 ++-- .github/workflows/manylinux_build.sh | 4 ++-- .github/workflows/musllinux_build.sh | 4 ++-- .github/workflows/tests.yml | 4 ++-- python/requirements.dev.txt | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/artifacts.yml b/.github/workflows/artifacts.yml index a9d6eac9..62123141 100644 --- a/.github/workflows/artifacts.yml +++ b/.github/workflows/artifacts.yml @@ -179,7 +179,7 @@ jobs: - uses: Swatinem/rust-cache@v2 - uses: actions/setup-python@v4 with: - python-version: "3.11" + python-version: "3.12" cache: pip cache-dependency-path: '**/requirements.dev.txt' - run: pip install -r python/requirements.dev.txt @@ -217,7 +217,7 @@ jobs: - uses: Swatinem/rust-cache@v2 - uses: actions/setup-python@v4 with: - python-version: "3.11" + python-version: "3.12" cache: pip cache-dependency-path: '**/requirements.dev.txt' - run: Remove-Item -LiteralPath "C:\msys64\" -Force -Recurse diff --git a/.github/workflows/manylinux_build.sh b/.github/workflows/manylinux_build.sh index e7ee8ec1..ccefe8b7 100644 --- a/.github/workflows/manylinux_build.sh +++ b/.github/workflows/manylinux_build.sh @@ -8,14 +8,14 @@ chmod +x rustup-init source "$HOME/.cargo/env" export PATH="${PATH}:/opt/python/cp37-cp37m/bin:/opt/python/cp38-cp38/bin:/opt/python/cp39-cp39/bin:/opt/python/cp310-cp310/bin:/opt/python/cp311-cp311/bin" cd python -python3.11 -m venv venv +python3.12 -m venv venv source venv/bin/activate pip install -r requirements.dev.txt maturin develop --release python generate_stubs.py pyoxigraph pyoxigraph.pyi --black maturin build --release --no-default-features --features abi3 --features rustls --compatibility manylinux2014 if [ %for_each_version% ]; then - for VERSION in 7 8 9 10 11; do + for VERSION in 8 9 10 11 12; do maturin build --release --no-default-features --features rustls --interpreter "python3.$VERSION" --compatibility manylinux2014 done fi diff --git a/.github/workflows/musllinux_build.sh b/.github/workflows/musllinux_build.sh index ec6c54e1..e85faa3f 100644 --- a/.github/workflows/musllinux_build.sh +++ b/.github/workflows/musllinux_build.sh @@ -6,14 +6,14 @@ chmod +x rustup-init source "$HOME/.cargo/env" export PATH="${PATH}:/opt/python/cp37-cp37m/bin:/opt/python/cp38-cp38/bin:/opt/python/cp39-cp39/bin:/opt/python/cp310-cp310/bin:/opt/python/cp311-cp311/bin" cd python -python3.11 -m venv venv +python3.12 -m venv venv source venv/bin/activate pip install -r requirements.dev.txt maturin develop --release python generate_stubs.py pyoxigraph pyoxigraph.pyi --black maturin build --release --no-default-features --features abi3 --features rustls --compatibility musllinux_1_2 if [ %for_each_version% ]; then - for VERSION in 7 8 9 10 11; do + for VERSION in 8 9 10 11 12; do maturin build --release --no-default-features --features rustls --interpreter "python3.$VERSION" --compatibility musllinux_1_2 done fi diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 953b9195..801c60b2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -223,7 +223,7 @@ jobs: - uses: Swatinem/rust-cache@v2 - uses: actions/setup-python@v4 with: - python-version: "3.11" + python-version: "3.12" cache: pip cache-dependency-path: '**/requirements.dev.txt' - run: pip install -r python/requirements.dev.txt @@ -278,7 +278,7 @@ jobs: - uses: Swatinem/rust-cache@v2 - uses: actions/setup-python@v4 with: - python-version: "3.11" + python-version: "3.12" cache: pip cache-dependency-path: '**/requirements.dev.txt' - run: pip install "maturin~=1.0" diff --git a/python/requirements.dev.txt b/python/requirements.dev.txt index fb239ec8..8595505c 100644 --- a/python/requirements.dev.txt +++ b/python/requirements.dev.txt @@ -2,5 +2,5 @@ black~=23.1 furo maturin~=1.0 mypy~=1.0 -ruff~=0.0.255 +ruff~=0.0.292 sphinx~=7.0 From 64f45cd11b4bbe4e64a12cb310d7ac1cbd52a0ca Mon Sep 17 00:00:00 2001 From: Tpt Date: Mon, 9 Oct 2023 12:21:04 +0200 Subject: [PATCH 106/217] Makes recent Clippy happy --- lib/oxrdf/src/interning.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/oxrdf/src/interning.rs b/lib/oxrdf/src/interning.rs index bc00c4a8..e647c26b 100644 --- a/lib/oxrdf/src/interning.rs +++ b/lib/oxrdf/src/interning.rs @@ -14,6 +14,7 @@ pub struct Interner { } impl Interner { + #[allow(clippy::never_loop)] fn get_or_intern(&mut self, value: &str) -> Key { let mut hash = self.hash(value); loop { From 8d348b2a6fed250ee16c2c0cd957aacb2814173c Mon Sep 17 00:00:00 2001 From: Tpt Date: Thu, 12 Oct 2023 17:38:28 +0200 Subject: [PATCH 107/217] Upgrades PyO3 --- Cargo.lock | 297 +++++++++++++++++++------------------------ lib/Cargo.toml | 2 +- python/Cargo.toml | 2 +- python/src/io.rs | 59 +++------ python/src/sparql.rs | 57 ++++----- python/src/store.rs | 8 +- 6 files changed, 174 insertions(+), 251 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d293c2c1..605c7a9e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -19,9 +19,9 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "aho-corasick" -version = "1.0.5" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c378d78423fdad8089616f827526ee33c19f2fddbd5de1629152c9593ba4783" +checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" dependencies = [ "memchr", ] @@ -34,9 +34,9 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "anstream" -version = "0.5.0" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f58811cfac344940f1a400b6e6231ce35171f614f26439e80f8c1465c5cc0c" +checksum = "2ab91ebe16eb252986481c5b62f6098f3b698a45e34b5b98200cf20dd2484a44" dependencies = [ "anstyle", "anstyle-parse", @@ -48,15 +48,15 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.2" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15c4c2c83f81532e5845a733998b6971faca23490340a418e9b72a3ec9de12ea" +checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87" [[package]] name = "anstyle-parse" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "938874ff5980b03a87c5524b3ae5b59cf99b1d6bc836848df7bc5ada9643c333" +checksum = "317b9a89c1868f5ea6ff1d9539a69f45dffc21ce321ac1fd1160dfa48c8e2140" dependencies = [ "utf8parse", ] @@ -72,9 +72,9 @@ dependencies = [ [[package]] name = "anstyle-wincon" -version = "2.1.0" +version = "3.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58f54d10c6dfa51283a066ceab3ec1ab78d13fae00aa49243a45e4571fb79dfd" +checksum = "f0699d10d2f4d628a98ee7b57b289abbc98ff3bad977cb3152709d4bf2330628" dependencies = [ "anstyle", "windows-sys", @@ -88,9 +88,9 @@ checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" [[package]] name = "arbitrary" -version = "1.3.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2d098ff73c1ca148721f37baad5ea6a465a13f9573aba8641fbbbae8164a54e" +checksum = "a2e1373abdaa212b704512ec2bd8b26bd0b7d5c3f70117411a5d9a451383c859" dependencies = [ "derive_arbitrary", ] @@ -171,7 +171,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.31", + "syn", "which", ] @@ -198,9 +198,9 @@ dependencies = [ [[package]] name = "bstr" -version = "1.6.2" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c2f7349907b712260e64b0afe2f84692af14a454be26187d9df565c7f69266a" +checksum = "c79ad7fb2dd38f3dabd76b09c6a5a20c038fc0213ef1e9afd30eb777f120f019" dependencies = [ "memchr", "regex-automata", @@ -209,9 +209,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.13.0" +version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" +checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" [[package]] name = "bytes" @@ -290,9 +290,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.2" +version = "4.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a13b88d2c62ff462f88e4a121f17a82c1af05693a2f192b5c38d14de73c19f6" +checksum = "d04704f56c2cde07f43e8e2c154b43f216dc5c92fc98ada720177362f953b956" dependencies = [ "clap_builder", "clap_derive", @@ -300,9 +300,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.2" +version = "4.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bb9faaa7c2ef94b2743a21f5a29e6f0010dff4caa69ac8e9d6cf8b6fa74da08" +checksum = "0e231faeaca65ebd1ea3c737966bf858971cd38c3849107aa3ea7de90a804e45" dependencies = [ "anstream", "anstyle", @@ -319,7 +319,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.31", + "syn", ] [[package]] @@ -390,7 +390,7 @@ dependencies = [ "clap", "criterion-plot", "is-terminal", - "itertools", + "itertools 0.10.5", "num-traits", "once_cell", "oorandom", @@ -411,17 +411,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" dependencies = [ "cast", - "itertools", -] - -[[package]] -name = "crossbeam-channel" -version = "0.5.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" -dependencies = [ - "cfg-if", - "crossbeam-utils", + "itertools 0.10.5", ] [[package]] @@ -481,7 +471,7 @@ checksum = "53e0efad4403bfc52dc201159c4b842a246a14b98c64b55dfd0f2d89729dfeb8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn", ] [[package]] @@ -514,25 +504,14 @@ checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" [[package]] name = "errno" -version = "0.3.3" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "136526188508e25c6fef639d7927dfb3e0e3084488bf202267829cf7fc23dbdd" +checksum = "ac3e13f66a2f95e32a39eaa81f6b95d42878ca0e1db0c7543723dfe12557e860" dependencies = [ - "errno-dragonfly", "libc", "windows-sys", ] -[[package]] -name = "errno-dragonfly" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" -dependencies = [ - "cc", - "libc", -] - [[package]] name = "escargot" version = "0.5.8" @@ -547,9 +526,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.0.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" +checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" [[package]] name = "flate2" @@ -682,9 +661,9 @@ checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] name = "hermit-abi" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" +checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" [[package]] name = "hex" @@ -736,9 +715,9 @@ dependencies = [ [[package]] name = "indoc" -version = "1.0.9" +version = "2.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa799dd5ed20a7e349f3b4639aa80d74549c81716d9ec4f994c9b5815598306" +checksum = "1e186cfbae8084e513daff4240b4797e342f988cecda4fb6c939150f96315fd8" [[package]] name = "is-terminal" @@ -760,6 +739,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.9" @@ -768,9 +756,9 @@ checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" [[package]] name = "jobserver" -version = "0.1.26" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2" +checksum = "8c37f63953c4c63420ed5fd3d6d398c719489b9f872b9fa683262f8edd363c7d" dependencies = [ "libc", ] @@ -814,9 +802,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.147" +version = "0.2.149" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" +checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b" [[package]] name = "libloading" @@ -830,9 +818,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.4.5" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57bcfdad1b858c2db7c38303a6d2ad4dfaf5eb53dfeb0910128b2c26d6158503" +checksum = "da2479e8c062e40bf0066ffa0bc823de0a9368974af99c9f6df941d2c231e03f" [[package]] name = "lock_api" @@ -852,18 +840,19 @@ checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" [[package]] name = "md-5" -version = "0.10.5" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6365506850d44bff6e2fbcb5176cf63650e48bd45ef2fe2665ae1570e0f4b9ca" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ + "cfg-if", "digest", ] [[package]] name = "memchr" -version = "2.6.3" +version = "2.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c" +checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" [[package]] name = "memoffset" @@ -925,23 +914,13 @@ checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" [[package]] name = "num-traits" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2" +checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" dependencies = [ "autocfg", ] -[[package]] -name = "num_cpus" -version = "1.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" -dependencies = [ - "hermit-abi", - "libc", -] - [[package]] name = "object" version = "0.32.1" @@ -986,7 +965,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn", ] [[package]] @@ -1199,9 +1178,9 @@ checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" [[package]] name = "peg" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a07f2cafdc3babeebc087e499118343442b742cc7c31b4d054682cc598508554" +checksum = "400bcab7d219c38abf8bd7cc2054eb9bbbd4312d66f6a5557d572a203f646f61" dependencies = [ "peg-macros", "peg-runtime", @@ -1209,9 +1188,9 @@ dependencies = [ [[package]] name = "peg-macros" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a90084dc05cf0428428e3d12399f39faad19b0909f64fb9170c9fdd6d9cd49b" +checksum = "46e61cce859b76d19090f62da50a9fe92bab7c2a5f09e183763559a2ac392c90" dependencies = [ "peg-runtime", "proc-macro2", @@ -1220,9 +1199,9 @@ dependencies = [ [[package]] name = "peg-runtime" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fa00462b37ead6d11a82c9d568b26682d78e0477dc02d1966c013af80969739" +checksum = "36bae92c60fa2398ce4678b98b2c4b5a7c61099961ca1fa305aec04a9ad28922" [[package]] name = "percent-encoding" @@ -1278,14 +1257,14 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "predicates" -version = "3.0.3" +version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09963355b9f467184c04017ced4a2ba2d75cbcb4e7462690d388233253d4b1a9" +checksum = "6dfc28575c2e3f19cb3c73b93af36460ae898d426eba6fc15b9bd2a5220758a0" dependencies = [ "anstyle", "difflib", "float-cmp", - "itertools", + "itertools 0.11.0", "normalize-line-endings", "predicates-core", "regex", @@ -1314,23 +1293,23 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d" dependencies = [ "proc-macro2", - "syn 2.0.31", + "syn", ] [[package]] name = "proc-macro2" -version = "1.0.66" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" +checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da" dependencies = [ "unicode-ident", ] [[package]] name = "pyo3" -version = "0.19.2" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e681a6cfdc4adcc93b4d3cf993749a4552018ee0a9b65fc0ccfad74352c72a38" +checksum = "04e8453b658fe480c3e70c8ed4e3d3ec33eb74988bd186561b0cc66b85c3bc4b" dependencies = [ "cfg-if", "indoc", @@ -1345,9 +1324,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.19.2" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "076c73d0bc438f7a4ef6fdd0c3bb4732149136abd952b110ac93e4edb13a6ba5" +checksum = "a96fe70b176a89cff78f2fa7b3c930081e163d5379b4dcdf993e3ae29ca662e5" dependencies = [ "once_cell", "target-lexicon", @@ -1355,9 +1334,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.19.2" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e53cee42e77ebe256066ba8aa77eff722b3bb91f3419177cf4cd0f304d3284d9" +checksum = "214929900fd25e6604661ed9cf349727c8920d47deff196c4e28165a6ef2a96b" dependencies = [ "libc", "pyo3-build-config", @@ -1365,25 +1344,26 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.19.2" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfeb4c99597e136528c6dd7d5e3de5434d1ceaf487436a3f03b2d56b6fc9efd1" +checksum = "dac53072f717aa1bfa4db832b39de8c875b7c7af4f4a6fe93cdbf9264cf8383b" dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 1.0.109", + "syn", ] [[package]] name = "pyo3-macros-backend" -version = "0.19.2" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "947dc12175c254889edc0c02e399476c2f652b4b9ebd123aa655c224de259536" +checksum = "7774b5a8282bd4f25f803b1f0d945120be959a36c72e08e7cd031c792fdfd424" dependencies = [ + "heck", "proc-macro2", "quote", - "syn 1.0.109", + "syn", ] [[package]] @@ -1445,9 +1425,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.7.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b" +checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1" dependencies = [ "either", "rayon-core", @@ -1455,14 +1435,12 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d" +checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed" dependencies = [ - "crossbeam-channel", "crossbeam-deque", "crossbeam-utils", - "num_cpus", ] [[package]] @@ -1476,9 +1454,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.9.5" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "697061221ea1b4a94a624f67d0ae2bfe4e22b8a17b6a192afb11046542cc8c47" +checksum = "d119d7c7ca818f8a53c300863d4f87566aac09943aef5b355bb83969dae75d87" dependencies = [ "aho-corasick", "memchr", @@ -1488,9 +1466,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.3.8" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2f401f4955220693b56f8ec66ee9c78abffd8d1c4f23dc41a23839eb88f0795" +checksum = "465c6fc0621e4abc4187a2bda0937bfd4f722c2730b29562e19689ea796c9a4b" dependencies = [ "aho-corasick", "memchr", @@ -1499,9 +1477,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.7.5" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" +checksum = "56d84fdd47036b038fc80dd333d10b6aab10d5d31f4a366e20014def75328d33" [[package]] name = "ring" @@ -1549,9 +1527,9 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "rustix" -version = "0.38.11" +version = "0.38.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0c3dde1fc030af041adc40e79c0e7fbcf431dd24870053d187d7c66e4b87453" +checksum = "5a74ee2d7c2581cd139b42447d7d9389b889bdaad3a73f1ebb16f2a3237bb19c" dependencies = [ "bitflags 2.4.0", "errno", @@ -1683,14 +1661,14 @@ checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn", ] [[package]] name = "serde_json" -version = "1.0.105" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "693151e1ac27563d6dbcec9dee9fbd5da8539b20fa14ad3752b2e6d363ace360" +checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" dependencies = [ "itoa", "ryu", @@ -1699,9 +1677,9 @@ dependencies = [ [[package]] name = "sha1" -version = "0.10.5" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", "cpufeatures", @@ -1710,9 +1688,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.7" +version = "0.10.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" dependencies = [ "cfg-if", "cpufeatures", @@ -1733,9 +1711,9 @@ checksum = "54ac45299ccbd390721be55b412d41931911f654fa99e2cb8bfb57184b2061fe" [[package]] name = "smallvec" -version = "1.11.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" +checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a" [[package]] name = "sparesults" @@ -1788,20 +1766,9 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "syn" -version = "1.0.109" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "syn" -version = "2.0.31" +version = "2.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "718fa2415bcb8d8bd775917a1bf12a7931b6dfa890753378538118181e0cb398" +checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b" dependencies = [ "proc-macro2", "quote", @@ -1865,9 +1832,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17f6bb557fd245c28e6411aa56b6403c689ad95061f50e4be16c274e70a17e48" +checksum = "426f806f4089c493dcac0d24c29c01e2c38baf8e30f1b716ee37e83d200b18fe" dependencies = [ "deranged", "itoa", @@ -1878,15 +1845,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a942f44339478ef67935ab2bbaec2fb0322496cf3cbe84b261e06ac3814c572" +checksum = "4ad70d68dba9e1f8aceda7aa6711965dfec1cac869f311a51bd08b3a2ccbce20" dependencies = [ "time-core", ] @@ -1918,9 +1885,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.32.0" +version = "1.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" +checksum = "4f38200e3ef7995e5ef13baec2f432a6da0aa9ac495b2c0e8f3b7eec2c92d653" dependencies = [ "backtrace", "bytes", @@ -1936,14 +1903,14 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn", ] [[package]] name = "typenum" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "unicode-bidi" @@ -1953,9 +1920,9 @@ checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" [[package]] name = "unicode-ident" -version = "1.0.11" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] name = "unicode-normalization" @@ -1968,15 +1935,15 @@ dependencies = [ [[package]] name = "unicode-width" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" +checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" [[package]] name = "unindent" -version = "0.1.11" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1766d682d402817b5ac4490b3c3002d91dfa0d22812f341609f97b08757359c" +checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce" [[package]] name = "untrusted" @@ -2059,7 +2026,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.31", + "syn", "wasm-bindgen-shared", ] @@ -2081,7 +2048,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -2150,9 +2117,9 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" dependencies = [ "winapi 0.3.9", ] @@ -2231,30 +2198,28 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "zstd" -version = "0.12.4" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a27595e173641171fc74a1232b7b1c7a7cb6e18222c11e9dfb9888fa424c53c" +checksum = "bffb3309596d527cfcba7dfc6ed6052f1d39dfbd7c867aa2e865e4a449c10110" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "6.0.6" +version = "7.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee98ffd0b48ee95e6c5168188e44a54550b1564d9d530ee21d5f0eaed1069581" +checksum = "43747c7422e2924c11144d5229878b98180ef8b06cca4ab5af37afc8a8d8ea3e" dependencies = [ - "libc", "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.8+zstd.1.5.5" +version = "2.0.9+zstd.1.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5556e6ee25d32df2586c098bbfa278803692a20d0ab9565e049480d52707ec8c" +checksum = "9e16efa8a874a0481a574084d34cc26fdb3b99627480f785888deb6386506656" dependencies = [ "cc", - "libc", "pkg-config", ] diff --git a/lib/Cargo.toml b/lib/Cargo.toml index 1ea01640..a5d65ee1 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -56,7 +56,7 @@ js-sys = { version = "0.3.60", optional = true } [target.'cfg(not(target_family = "wasm"))'.dev-dependencies] criterion = "0.5" oxhttp = "0.2.0-alpha.1" -zstd = "0.12" +zstd = ">=0.12, <0.14" [package.metadata.docs.rs] all-features = true diff --git a/python/Cargo.toml b/python/Cargo.toml index 9ecee7af..58538d3a 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -26,4 +26,4 @@ rustls = ["oxigraph/http-client-rustls-native"] [dependencies] oxigraph = { version = "0.4.0-alpha.1-dev", path="../lib" } -pyo3 = { version = "0.19", features = ["extension-module"] } +pyo3 = { version = "0.20", features = ["extension-module"] } diff --git a/python/src/io.rs b/python/src/io.rs index 4e04a4da..73395d71 100644 --- a/python/src/io.rs +++ b/python/src/io.rs @@ -9,7 +9,6 @@ use pyo3::intern; use pyo3::prelude::*; use pyo3::types::PyBytes; use std::cmp::max; -use std::error::Error; use std::ffi::OsStr; use std::fs::File; use std::io::{self, BufWriter, Cursor, Read, Write}; @@ -63,7 +62,7 @@ pub fn parse( let file_path = input.extract::().ok(); let format = parse_format(format, file_path.as_deref())?; let input = if let Some(file_path) = &file_path { - PyReadable::from_file(file_path, py).map_err(map_io_err)? + PyReadable::from_file(file_path, py)? } else { PyReadable::from_data(input) }; @@ -144,10 +143,9 @@ pub fn serialize<'a>( )); } writer.write_quad(quad) - } - .map_err(map_io_err)?; + }?; } - writer.finish().map_err(map_io_err) + Ok(writer.finish()?) }, output, format, @@ -229,10 +227,7 @@ impl PyWritable { let format = parse_format::(format, file_path.as_deref())?; let output = if let Some(output) = output { if let Some(file_path) = &file_path { - Self::File( - py.allow_threads(|| File::create(file_path)) - .map_err(map_io_err)?, - ) + Self::File(py.allow_threads(|| File::create(file_path))?) } else { Self::Io(PyIo(output.into())) } @@ -240,9 +235,7 @@ impl PyWritable { PyWritable::Bytes(Vec::new()) }; let writer = write(BufWriter::new(output), format)?; - py.allow_threads(|| writer.into_inner()) - .map_err(|e| map_io_err(e.into_error()))? - .close(py) + py.allow_threads(|| writer.into_inner())?.close(py) } fn close(self, py: Python<'_>) -> PyResult> { @@ -252,12 +245,11 @@ impl PyWritable { py.allow_threads(|| { file.flush()?; file.sync_all() - }) - .map_err(map_io_err)?; + })?; Ok(None) } Self::Io(mut io) => { - py.allow_threads(|| io.flush()).map_err(map_io_err)?; + py.allow_threads(|| io.flush())?; Ok(None) } } @@ -294,12 +286,10 @@ impl Read for PyIo { let read = self .0 .as_ref(py) - .call_method1(intern!(py, "read"), (to_read,)) - .map_err(to_io_err)?; + .call_method1(intern!(py, "read"), (to_read,))?; let bytes = read .extract::<&[u8]>() - .or_else(|e| read.extract::<&str>().map(str::as_bytes).map_err(|_| e)) - .map_err(to_io_err)?; + .or_else(|_| read.extract::<&str>().map(str::as_bytes))?; buf[..bytes.len()].copy_from_slice(bytes); Ok(bytes.len()) }) @@ -309,21 +299,17 @@ impl Read for PyIo { impl Write for PyIo { fn write(&mut self, buf: &[u8]) -> io::Result { Python::with_gil(|py| { - self.0 + Ok(self + .0 .as_ref(py) - .call_method1(intern!(py, "write"), (PyBytes::new(py, buf),)) - .map_err(to_io_err)? - .extract::() - .map_err(to_io_err) + .call_method1(intern!(py, "write"), (PyBytes::new(py, buf),))? + .extract::()?) }) } fn flush(&mut self) -> io::Result<()> { Python::with_gil(|py| { - self.0 - .as_ref(py) - .call_method0(intern!(py, "flush")) - .map_err(to_io_err)?; + self.0.as_ref(py).call_method0(intern!(py, "flush"))?; Ok(()) }) } @@ -382,21 +368,6 @@ pub fn parse_format(format: Option<&str>, path: Option<&Path>) -> PyR } } -fn to_io_err(error: PyErr) -> io::Error { - io::Error::new(io::ErrorKind::Other, error) -} - -pub fn map_io_err(error: io::Error) -> PyErr { - if error - .get_ref() - .map_or(false, <(dyn Error + Send + Sync + 'static)>::is::) - { - *error.into_inner().unwrap().downcast().unwrap() - } else { - error.into() - } -} - pub fn map_parse_error(error: ParseError, file_path: Option) -> PyErr { match error { ParseError::Syntax(error) => { @@ -429,7 +400,7 @@ pub fn map_parse_error(error: ParseError, file_path: Option) -> PyErr { PySyntaxError::new_err((error.to_string(), params)) } } - ParseError::Io(error) => map_io_err(error), + ParseError::Io(error) => error.into(), } } diff --git a/python/src/sparql.rs b/python/src/sparql.rs index 963aa0df..98698708 100644 --- a/python/src/sparql.rs +++ b/python/src/sparql.rs @@ -12,11 +12,10 @@ use oxigraph::sparql::{ Variable, }; use pyo3::basic::CompareOp; -use pyo3::exceptions::{ - PyNotImplementedError, PyRuntimeError, PySyntaxError, PyTypeError, PyValueError, -}; +use pyo3::exceptions::{PyRuntimeError, PySyntaxError, PyTypeError, PyValueError}; use pyo3::prelude::*; use pyo3::types::PyBytes; +use std::io; use std::path::PathBuf; use std::vec::IntoIter; @@ -121,14 +120,12 @@ impl PyQuerySolution { buffer } - fn __richcmp__(&self, other: &Self, op: CompareOp) -> PyResult { - match op { - CompareOp::Eq => Ok(self.inner == other.inner), - CompareOp::Ne => Ok(self.inner != other.inner), - _ => Err(PyNotImplementedError::new_err( - "Ordering is not implemented", - )), - } + fn __eq__(&self, other: &Self) -> bool { + self.inner == other.inner + } + + fn __ne__(&self, other: &Self) -> bool { + self.inner != other.inner } fn __len__(&self) -> usize { @@ -258,28 +255,23 @@ impl PyQuerySolutions { iter.variables().to_vec() } }, - ) - .map_err(map_io_err)?; + )?; match &mut self.inner { PyQuerySolutionsVariant::Query(inner) => { for solution in inner { - writer - .write(&solution.map_err(map_evaluation_error)?) - .map_err(map_io_err)?; + writer.write(&solution.map_err(map_evaluation_error)?)?; } } PyQuerySolutionsVariant::Reader { iter, file_path } => { for solution in iter { - writer - .write(&solution.map_err(|e| { - map_query_results_parse_error(e, file_path.clone()) - })?) - .map_err(map_io_err)?; + writer.write(&solution.map_err(|e| { + map_query_results_parse_error(e, file_path.clone()) + })?)?; } } } - writer.finish().map_err(map_io_err) + Ok(writer.finish()?) }, output, format, @@ -355,9 +347,8 @@ impl PyQueryBoolean { PyWritable::do_write( |output, format| { py.allow_threads(|| { - QueryResultsSerializer::from_format(format) - .serialize_boolean_to_write(output, self.inner) - .map_err(map_io_err) + Ok(QueryResultsSerializer::from_format(format) + .serialize_boolean_to_write(output, self.inner)?) }) }, output, @@ -435,11 +426,9 @@ impl PyQueryTriples { |output, format| { let mut writer = RdfSerializer::from_format(format).serialize_to_write(output); for triple in &mut self.inner { - writer - .write_triple(&triple.map_err(map_evaluation_error)?) - .map_err(map_io_err)?; + writer.write_triple(&triple.map_err(map_evaluation_error)?)?; } - writer.finish().map_err(map_io_err) + Ok(writer.finish()?) }, output, format, @@ -498,7 +487,7 @@ pub fn parse_query_results( let file_path = input.extract::().ok(); let format = parse_format(format, file_path.as_deref())?; let input = if let Some(file_path) = &file_path { - PyReadable::from_file(file_path, py).map_err(map_io_err)? + PyReadable::from_file(file_path, py)? } else { PyReadable::from_data(input) }; @@ -520,9 +509,9 @@ pub fn map_evaluation_error(error: EvaluationError) -> PyErr { EvaluationError::Storage(error) => map_storage_error(error), EvaluationError::GraphParsing(error) => map_parse_error(error, None), EvaluationError::ResultsParsing(error) => map_query_results_parse_error(error, None), - EvaluationError::ResultsSerialization(error) => map_io_err(error), - EvaluationError::Service(error) => match error.downcast() { - Ok(error) => map_io_err(*error), + EvaluationError::ResultsSerialization(error) => error.into(), + EvaluationError::Service(error) => match error.downcast::() { + Ok(error) => (*error).into(), Err(error) => PyRuntimeError::new_err(error.to_string()), }, _ => PyRuntimeError::new_err(error.to_string()), @@ -561,6 +550,6 @@ pub fn map_query_results_parse_error(error: ParseError, file_path: Option map_io_err(error), + ParseError::Io(error) => error.into(), } } diff --git a/python/src/store.rs b/python/src/store.rs index 1befc92a..5574e652 100644 --- a/python/src/store.rs +++ b/python/src/store.rs @@ -1,8 +1,6 @@ #![allow(clippy::needless_option_as_deref)] -use crate::io::{ - allow_threads_unsafe, map_io_err, map_parse_error, parse_format, PyReadable, PyWritable, -}; +use crate::io::{allow_threads_unsafe, map_parse_error, parse_format, PyReadable, PyWritable}; use crate::model::*; use crate::sparql::*; use oxigraph::io::RdfFormat; @@ -396,7 +394,7 @@ impl PyStore { let file_path = input.extract::().ok(); let format = parse_format::(format, file_path.as_deref())?; let input = if let Some(file_path) = &file_path { - PyReadable::from_file(file_path, py).map_err(map_io_err)? + PyReadable::from_file(file_path, py)? } else { PyReadable::from_data(input) }; @@ -465,7 +463,7 @@ impl PyStore { let file_path = input.extract::().ok(); let format = parse_format::(format, file_path.as_deref())?; let input = if let Some(file_path) = &file_path { - PyReadable::from_file(file_path, py).map_err(map_io_err)? + PyReadable::from_file(file_path, py)? } else { PyReadable::from_data(input) }; From 832a4ba27d1ca83933d80145d9dac6a078bd6048 Mon Sep 17 00:00:00 2001 From: Tpt Date: Thu, 19 Oct 2023 18:41:06 +0200 Subject: [PATCH 108/217] JS: Upgrades Biome --- js/biome.json | 2 +- js/package.json | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/js/biome.json b/js/biome.json index 72ae2ce8..966a8b1e 100644 --- a/js/biome.json +++ b/js/biome.json @@ -2,7 +2,7 @@ "$schema": "https://biomejs.dev/schemas/1.0.0/schema.json", "formatter": { "indentStyle": "space", - "indentSize": 4, + "indentWidth": 4, "lineWidth": 100 }, "linter": { diff --git a/js/package.json b/js/package.json index 02577037..e638da9f 100644 --- a/js/package.json +++ b/js/package.json @@ -3,9 +3,9 @@ "description": "Oxigraph JS build and tests", "private": true, "devDependencies": { + "@biomejs/biome": "^1.0.0", "@rdfjs/data-model": "^2.0.1", - "mocha": "^10.0.0", - "@biomejs/biome": "^1.0.0" + "mocha": "^10.0.0" }, "scripts": { "fmt": "biome format . --write && biome check . --apply-unsafe", From 517df6d59ef0c133c8312fbd4af46ce99a961693 Mon Sep 17 00:00:00 2001 From: Tpt Date: Wed, 18 Oct 2023 21:43:18 +0200 Subject: [PATCH 109/217] Testsuite: Makes use of mf:assumedTestBase --- testsuite/N3 | 2 +- testsuite/rdf-canon | 2 +- testsuite/rdf-tests | 2 +- testsuite/src/files.rs | 5 ++- testsuite/src/manifest.rs | 80 ++++++++++++++++++++++++--------------- testsuite/src/vocab.rs | 3 ++ 6 files changed, 59 insertions(+), 35 deletions(-) diff --git a/testsuite/N3 b/testsuite/N3 index 5fa35bf6..73f4e00c 160000 --- a/testsuite/N3 +++ b/testsuite/N3 @@ -1 +1 @@ -Subproject commit 5fa35bf602669a467cfd0ab24cc732fe49f2b927 +Subproject commit 73f4e00c15828f030749567f61cea2f5d7d9c59b diff --git a/testsuite/rdf-canon b/testsuite/rdf-canon index 9b3efebe..0503facf 160000 --- a/testsuite/rdf-canon +++ b/testsuite/rdf-canon @@ -1 +1 @@ -Subproject commit 9b3efebebe0e5338debfc4d3ddd02f6adf0a852c +Subproject commit 0503facfaa0825686afc1f533f487816de54d9b7 diff --git a/testsuite/rdf-tests b/testsuite/rdf-tests index aaa24e47..a0b910e0 160000 --- a/testsuite/rdf-tests +++ b/testsuite/rdf-tests @@ -1 +1 @@ -Subproject commit aaa24e4729a89bdee004bc3042814159eb689a19 +Subproject commit a0b910e0f2cff4d3a7e32f15ce0bfbc7db2768fb diff --git a/testsuite/src/files.rs b/testsuite/src/files.rs index fc7f5a20..d153ba74 100644 --- a/testsuite/src/files.rs +++ b/testsuite/src/files.rs @@ -34,9 +34,10 @@ pub fn load_to_graph( url: &str, graph: &mut Graph, format: RdfFormat, + base_iri: Option<&str>, ignore_errors: bool, ) -> Result<()> { - let parser = RdfParser::from_format(format).with_base_iri(url)?; + let parser = RdfParser::from_format(format).with_base_iri(base_iri.unwrap_or(url))?; for t in parser.parse_read(read_file(url)?) { match t { Ok(t) => { @@ -54,7 +55,7 @@ pub fn load_to_graph( pub fn load_graph(url: &str, format: RdfFormat, ignore_errors: bool) -> Result { let mut graph = Graph::new(); - load_to_graph(url, &mut graph, format, ignore_errors)?; + load_to_graph(url, &mut graph, format, None, ignore_errors)?; Ok(graph) } diff --git a/testsuite/src/manifest.rs b/testsuite/src/manifest.rs index 3a70442e..00a0f6a6 100644 --- a/testsuite/src/manifest.rs +++ b/testsuite/src/manifest.rs @@ -1,6 +1,6 @@ use crate::files::{guess_rdf_format, load_to_graph}; use crate::vocab::*; -use anyhow::{bail, Result}; +use anyhow::{bail, Context, Result}; use oxigraph::model::vocab::*; use oxigraph::model::*; use std::collections::VecDeque; @@ -273,46 +273,66 @@ impl TestManifest { return Ok(None); }; self.graph.clear(); - load_to_graph(&url, &mut self.graph, guess_rdf_format(&url)?, false)?; + load_to_graph(&url, &mut self.graph, guess_rdf_format(&url)?, None, false)?; let manifests = self .graph .subjects_for_predicate_object(rdf::TYPE, mf::MANIFEST) .collect::>(); if manifests.len() != 1 { - bail!("The file {url} should contain a single manifest"); + bail!("The file should contain a single manifest"); } - for manifest in manifests { - match self + let mut manifest = manifests[0]; + if let Some(base_iri) = self + .graph + .object_for_subject_predicate(manifest, mf::ASSUMED_TEST_BASE) + { + let Term::NamedNode(base_iri) = base_iri.into_owned() else { + bail!("Invalid base IRI: {base_iri}"); + }; + self.graph.clear(); + load_to_graph( + &url, + &mut self.graph, + guess_rdf_format(&url)?, + Some(base_iri.as_str()), + false, + )?; + manifest = self .graph - .object_for_subject_predicate(manifest, mf::INCLUDE) - { - Some(TermRef::BlankNode(list)) => { - self.manifests_to_do.extend( - RdfListIterator::iter(&self.graph, list.into()).filter_map(|m| match m { - Term::NamedNode(nm) => Some(nm.into_string()), - _ => None, - }), - ); - } - Some(_) => bail!("invalid tests list"), - None => (), + .subject_for_predicate_object(rdf::TYPE, mf::MANIFEST) + .context("no manifest found")?; + } + + match self + .graph + .object_for_subject_predicate(manifest, mf::INCLUDE) + { + Some(TermRef::BlankNode(list)) => { + self.manifests_to_do.extend( + RdfListIterator::iter(&self.graph, list.into()).filter_map(|m| match m { + Term::NamedNode(nm) => Some(nm.into_string()), + _ => None, + }), + ); } + Some(_) => bail!("invalid tests list"), + None => (), + } - // New tests - match self - .graph - .object_for_subject_predicate(manifest, mf::ENTRIES) - { - Some(TermRef::BlankNode(list)) => { - self.tests_to_do - .extend(RdfListIterator::iter(&self.graph, list.into())); - } - Some(term) => { - bail!("Invalid tests list. Got term {term}"); - } - None => (), + // New tests + match self + .graph + .object_for_subject_predicate(manifest, mf::ENTRIES) + { + Some(TermRef::BlankNode(list)) => { + self.tests_to_do + .extend(RdfListIterator::iter(&self.graph, list.into())); + } + Some(term) => { + bail!("Invalid tests list. Got term {term}"); } + None => (), } Ok(Some(())) } diff --git a/testsuite/src/vocab.rs b/testsuite/src/vocab.rs index 64327755..0fdc8f0d 100644 --- a/testsuite/src/vocab.rs +++ b/testsuite/src/vocab.rs @@ -43,6 +43,9 @@ pub mod mf { pub const ACTION: NamedNodeRef<'_> = NamedNodeRef::new_unchecked( "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#action", ); + pub const ASSUMED_TEST_BASE: NamedNodeRef<'_> = NamedNodeRef::new_unchecked( + "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#assumedTestBase", + ); pub const RESULT: NamedNodeRef<'_> = NamedNodeRef::new_unchecked( "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#result", ); From ef429e6d1bc7b7add36992e8fcff4ad1ce789c27 Mon Sep 17 00:00:00 2001 From: Tpt Date: Thu, 19 Oct 2023 18:33:17 +0200 Subject: [PATCH 110/217] Uses anyhow context function more often --- cli/src/main.rs | 20 ++++---- testsuite/src/files.rs | 4 +- testsuite/src/parser_evaluator.rs | 54 ++++++---------------- testsuite/src/sparql_evaluator.rs | 76 +++++++++---------------------- 4 files changed, 49 insertions(+), 105 deletions(-) diff --git a/cli/src/main.rs b/cli/src/main.rs index 16371325..07c84592 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -1,5 +1,5 @@ #![allow(clippy::print_stderr, clippy::cast_precision_loss, clippy::use_debug)] -use anyhow::{anyhow, bail, ensure, Context}; +use anyhow::{bail, ensure, Context}; use clap::{Parser, Subcommand, ValueHint}; use flate2::read::MultiGzDecoder; use oxhttp::model::{Body, HeaderName, HeaderValue, Method, Request, Response, Status}; @@ -387,9 +387,7 @@ pub fn main() -> anyhow::Result<()> { bulk_load( &loader, stdin().lock(), - format.ok_or_else(|| { - anyhow!("The --format option must be set when loading from stdin") - })?, + format.context("The --format option must be set when loading from stdin")?, base.as_deref(), graph, ) @@ -547,8 +545,9 @@ pub fn main() -> anyhow::Result<()> { } } else if let Some(results_file) = &results_file { format_from_path(results_file, |ext| { - QueryResultsFormat::from_extension(ext) - .ok_or_else(|| anyhow!("The file extension '{ext}' is unknown")) + QueryResultsFormat::from_extension(ext).with_context(|| { + format!("The file extension '{ext}' is unknown") + }) })? } else { bail!("The --results-format option must be set when writing to stdout") @@ -587,8 +586,9 @@ pub fn main() -> anyhow::Result<()> { } } else if let Some(results_file) = &results_file { format_from_path(results_file, |ext| { - QueryResultsFormat::from_extension(ext) - .ok_or_else(|| anyhow!("The file extension '{ext}' is unknown")) + QueryResultsFormat::from_extension(ext).with_context(|| { + format!("The file extension '{ext}' is unknown") + }) })? } else { bail!("The --results-format option must be set when writing to stdout") @@ -701,7 +701,7 @@ pub fn main() -> anyhow::Result<()> { if let Some(base) = from_base { parser = parser .with_base_iri(&base) - .with_context(|| anyhow!("Invalid base IRI {base}"))?; + .with_context(|| format!("Invalid base IRI {base}"))?; } let to_format = if let Some(format) = to_format { @@ -864,7 +864,7 @@ fn format_from_path( fn rdf_format_from_path(path: &Path) -> anyhow::Result { format_from_path(path, |ext| { RdfFormat::from_extension(ext) - .ok_or_else(|| anyhow!("The file extension '{ext}' is unknown")) + .with_context(|| format!("The file extension '{ext}' is unknown")) }) } diff --git a/testsuite/src/files.rs b/testsuite/src/files.rs index d153ba74..7ded2d1f 100644 --- a/testsuite/src/files.rs +++ b/testsuite/src/files.rs @@ -1,4 +1,4 @@ -use anyhow::{anyhow, bail, Context, Result}; +use anyhow::{bail, Context, Result}; use oxigraph::io::{RdfFormat, RdfParser}; use oxigraph::model::{Dataset, Graph}; use oxttl::n3::N3Quad; @@ -90,7 +90,7 @@ pub fn load_dataset(url: &str, format: RdfFormat, ignore_errors: bool) -> Result pub fn guess_rdf_format(url: &str) -> Result { url.rsplit_once('.') .and_then(|(_, extension)| RdfFormat::from_extension(extension)) - .ok_or_else(|| anyhow!("Serialization type not found for {url}")) + .with_context(|| format!("Serialization type not found for {url}")) } pub fn load_n3(url: &str, ignore_errors: bool) -> Result> { diff --git a/testsuite/src/parser_evaluator.rs b/testsuite/src/parser_evaluator.rs index c0122640..adcaf595 100644 --- a/testsuite/src/parser_evaluator.rs +++ b/testsuite/src/parser_evaluator.rs @@ -2,7 +2,7 @@ use crate::evaluator::TestEvaluator; use crate::files::{guess_rdf_format, load_dataset, load_n3, read_file_to_string}; use crate::manifest::Test; use crate::report::{dataset_diff, format_diff}; -use anyhow::{anyhow, bail, ensure, Result}; +use anyhow::{bail, ensure, Context, Result}; use oxigraph::io::RdfFormat; use oxigraph::model::{BlankNode, Dataset, Quad}; use oxttl::n3::{N3Quad, N3Term}; @@ -94,28 +94,19 @@ pub fn register_parser_tests(evaluator: &mut TestEvaluator) { } fn evaluate_positive_syntax_test(test: &Test, format: RdfFormat) -> Result<()> { - let action = test - .action - .as_deref() - .ok_or_else(|| anyhow!("No action found"))?; - load_dataset(action, format, false).map_err(|e| anyhow!("Parse error: {e}"))?; + let action = test.action.as_deref().context("No action found")?; + load_dataset(action, format, false).context("Parse error")?; Ok(()) } fn evaluate_positive_n3_syntax_test(test: &Test) -> Result<()> { - let action = test - .action - .as_deref() - .ok_or_else(|| anyhow!("No action found"))?; - load_n3(action, false).map_err(|e| anyhow!("Parse error: {e}"))?; + let action = test.action.as_deref().context("No action found")?; + load_n3(action, false).context("Parse error")?; Ok(()) } fn evaluate_negative_syntax_test(test: &Test, format: RdfFormat) -> Result<()> { - let action = test - .action - .as_deref() - .ok_or_else(|| anyhow!("No action found"))?; + let action = test.action.as_deref().context("No action found")?; let Err(error) = load_dataset(action, format, false) else { bail!("File parsed without errors even if it should not"); }; @@ -131,10 +122,7 @@ fn evaluate_negative_syntax_test(test: &Test, format: RdfFormat) -> Result<()> { } fn evaluate_negative_n3_syntax_test(test: &Test) -> Result<()> { - let action = test - .action - .as_deref() - .ok_or_else(|| anyhow!("No action found"))?; + let action = test.action.as_deref().context("No action found")?; ensure!( load_n3(action, false).is_err(), "File parsed without errors even if it should not" @@ -143,19 +131,13 @@ fn evaluate_negative_n3_syntax_test(test: &Test) -> Result<()> { } fn evaluate_eval_test(test: &Test, format: RdfFormat, ignore_errors: bool) -> Result<()> { - let action = test - .action - .as_deref() - .ok_or_else(|| anyhow!("No action found"))?; + let action = test.action.as_deref().context("No action found")?; let mut actual_dataset = load_dataset(action, format, ignore_errors) - .map_err(|e| anyhow!("Parse error on file {action}: {e}"))?; + .with_context(|| format!("Parse error on file {action}"))?; actual_dataset.canonicalize(); - let results = test - .result - .as_ref() - .ok_or_else(|| anyhow!("No tests result found"))?; + let results = test.result.as_ref().context("No tests result found")?; let mut expected_dataset = load_dataset(results, guess_rdf_format(results)?, false) - .map_err(|e| anyhow!("Parse error on file {results}: {e}"))?; + .with_context(|| format!("Parse error on file {results}"))?; expected_dataset.canonicalize(); ensure!( expected_dataset == actual_dataset, @@ -166,20 +148,14 @@ fn evaluate_eval_test(test: &Test, format: RdfFormat, ignore_errors: bool) -> Re } fn evaluate_n3_eval_test(test: &Test, ignore_errors: bool) -> Result<()> { - let action = test - .action - .as_deref() - .ok_or_else(|| anyhow!("No action found"))?; + let action = test.action.as_deref().context("No action found")?; let mut actual_dataset = n3_to_dataset( - load_n3(action, ignore_errors).map_err(|e| anyhow!("Parse error on file {action}: {e}"))?, + load_n3(action, ignore_errors).with_context(|| format!("Parse error on file {action}"))?, ); actual_dataset.canonicalize(); - let results = test - .result - .as_ref() - .ok_or_else(|| anyhow!("No tests result found"))?; + let results = test.result.as_ref().context("No tests result found")?; let mut expected_dataset = n3_to_dataset( - load_n3(results, false).map_err(|e| anyhow!("Parse error on file {results}: {e}"))?, + load_n3(results, false).with_context(|| format!("Parse error on file {results}"))?, ); expected_dataset.canonicalize(); ensure!( diff --git a/testsuite/src/sparql_evaluator.rs b/testsuite/src/sparql_evaluator.rs index 14bdb38d..383be3e2 100644 --- a/testsuite/src/sparql_evaluator.rs +++ b/testsuite/src/sparql_evaluator.rs @@ -3,7 +3,7 @@ use crate::files::*; use crate::manifest::*; use crate::report::{dataset_diff, format_diff}; use crate::vocab::*; -use anyhow::{anyhow, bail, ensure, Error, Result}; +use anyhow::{bail, ensure, Context, Error, Result}; use oxigraph::model::vocab::*; use oxigraph::model::*; use oxigraph::sparql::results::QueryResultsFormat; @@ -77,22 +77,16 @@ pub fn register_sparql_tests(evaluator: &mut TestEvaluator) { } fn evaluate_positive_syntax_test(test: &Test) -> Result<()> { - let query_file = test - .action - .as_deref() - .ok_or_else(|| anyhow!("No action found"))?; + let query_file = test.action.as_deref().context("No action found")?; let query = Query::parse(&read_file_to_string(query_file)?, Some(query_file)) - .map_err(|e| anyhow!("Not able to parse with error: {e}"))?; + .context("Not able to parse")?; Query::parse(&query.to_string(), None) - .map_err(|e| anyhow!("Failure to deserialize \"{query}\" with error: {e}"))?; + .with_context(|| format!("Failure to deserialize \"{query}\""))?; Ok(()) } fn evaluate_negative_syntax_test(test: &Test) -> Result<()> { - let query_file = test - .action - .as_deref() - .ok_or_else(|| anyhow!("No action found"))?; + let query_file = test.action.as_deref().context("No action found")?; ensure!( Query::parse(&read_file_to_string(query_file)?, Some(query_file)).is_err(), "Oxigraph parses even if it should not." @@ -101,10 +95,7 @@ fn evaluate_negative_syntax_test(test: &Test) -> Result<()> { } fn evaluate_positive_result_syntax_test(test: &Test, format: QueryResultsFormat) -> Result<()> { - let action_file = test - .action - .as_deref() - .ok_or_else(|| anyhow!("No action found"))?; + let action_file = test.action.as_deref().context("No action found")?; let actual_results = StaticQueryResults::from_query_results( QueryResults::read(read_file(action_file)?, format)?, true, @@ -124,10 +115,7 @@ fn evaluate_positive_result_syntax_test(test: &Test, format: QueryResultsFormat) } fn evaluate_negative_result_syntax_test(test: &Test, format: QueryResultsFormat) -> Result<()> { - let action_file = test - .action - .as_deref() - .ok_or_else(|| anyhow!("No action found"))?; + let action_file = test.action.as_deref().context("No action found")?; ensure!( QueryResults::read(Cursor::new(read_file_to_string(action_file)?), format) .map_err(Error::from) @@ -146,18 +134,15 @@ fn evaluate_evaluation_test(test: &Test) -> Result<()> { for (name, value) in &test.graph_data { load_graph_to_store(value, &store, name.clone())?; } - let query_file = test - .query - .as_deref() - .ok_or_else(|| anyhow!("No action found"))?; + let query_file = test.query.as_deref().context("No action found")?; let options = QueryOptions::default() .with_service_handler(StaticServiceHandler::new(&test.service_data)?); let query = Query::parse(&read_file_to_string(query_file)?, Some(query_file)) - .map_err(|e| anyhow!("Failure to parse query with error: {e}"))?; + .context("Failure to parse query")?; // We check parsing roundtrip Query::parse(&query.to_string(), None) - .map_err(|e| anyhow!("Failure to deserialize \"{query}\" with error: {e}"))?; + .with_context(|| format!("Failure to deserialize \"{query}\""))?; // FROM and FROM NAMED support. We make sure the data is in the store if !query.dataset().is_default_dataset() { @@ -176,7 +161,7 @@ fn evaluate_evaluation_test(test: &Test) -> Result<()> { } let expected_results = load_sparql_query_result(test.result.as_ref().unwrap()) - .map_err(|e| anyhow!("Error constructing expected graph: {e}"))?; + .context("Error constructing expected graph")?; let with_order = if let StaticQueryResults::Solutions { ordered, .. } = &expected_results { *ordered } else { @@ -190,7 +175,7 @@ fn evaluate_evaluation_test(test: &Test) -> Result<()> { } let actual_results = store .query_opt(query.clone(), options) - .map_err(|e| anyhow!("Failure to execute query with error: {e}"))?; + .context("Failure to execute query")?; let actual_results = StaticQueryResults::from_query_results(actual_results, with_order)?; ensure!( @@ -205,22 +190,16 @@ fn evaluate_evaluation_test(test: &Test) -> Result<()> { } fn evaluate_positive_update_syntax_test(test: &Test) -> Result<()> { - let update_file = test - .action - .as_deref() - .ok_or_else(|| anyhow!("No action found"))?; + let update_file = test.action.as_deref().context("No action found")?; let update = Update::parse(&read_file_to_string(update_file)?, Some(update_file)) - .map_err(|e| anyhow!("Not able to parse with error: {e}"))?; + .context("Not able to parse")?; Update::parse(&update.to_string(), None) - .map_err(|e| anyhow!("Failure to deserialize \"{update}\" with error: {e}"))?; + .with_context(|| format!("Failure to deserialize \"{update}\""))?; Ok(()) } fn evaluate_negative_update_syntax_test(test: &Test) -> Result<()> { - let update_file = test - .action - .as_deref() - .ok_or_else(|| anyhow!("No action found"))?; + let update_file = test.action.as_deref().context("No action found")?; ensure!( Update::parse(&read_file_to_string(update_file)?, Some(update_file)).is_err(), "Oxigraph parses even if it should not." @@ -245,20 +224,15 @@ fn evaluate_update_evaluation_test(test: &Test) -> Result<()> { load_graph_to_store(value, &result_store, name.clone())?; } - let update_file = test - .update - .as_deref() - .ok_or_else(|| anyhow!("No action found"))?; + let update_file = test.update.as_deref().context("No action found")?; let update = Update::parse(&read_file_to_string(update_file)?, Some(update_file)) - .map_err(|e| anyhow!("Failure to parse update with error: {e}"))?; + .context("Failure to parse update")?; // We check parsing roundtrip Update::parse(&update.to_string(), None) - .map_err(|e| anyhow!("Failure to deserialize \"{update}\" with error: {e}"))?; + .with_context(|| format!("Failure to deserialize \"{update}\""))?; - store - .update(update) - .map_err(|e| anyhow!("Failure to execute update with error: {e}"))?; + store.update(update).context("Failure to execute update")?; let mut store_dataset: Dataset = store.iter().collect::>()?; store_dataset.canonicalize(); let mut result_store_dataset: Dataset = result_store.iter().collect::>()?; @@ -689,10 +663,7 @@ fn load_dataset_to_store(url: &str, store: &Store) -> Result<()> { } fn evaluate_query_optimization_test(test: &Test) -> Result<()> { - let action = test - .action - .as_deref() - .ok_or_else(|| anyhow!("No action found"))?; + let action = test.action.as_deref().context("No action found")?; let actual = (&Optimizer::optimize_graph_pattern( (&if let spargebra::Query::Select { pattern, .. } = spargebra::Query::parse(&read_file_to_string(action)?, Some(action))? @@ -704,10 +675,7 @@ fn evaluate_query_optimization_test(test: &Test) -> Result<()> { .into(), )) .into(); - let result = test - .result - .as_ref() - .ok_or_else(|| anyhow!("No tests result found"))?; + let result = test.result.as_ref().context("No tests result found")?; let spargebra::Query::Select { pattern: expected, .. } = spargebra::Query::parse(&read_file_to_string(result)?, Some(result))? From 564762401216b2843d96c66283411a15dcddd990 Mon Sep 17 00:00:00 2001 From: Tpt Date: Wed, 18 Oct 2023 22:15:44 +0200 Subject: [PATCH 111/217] Testsuite: executes C14N tests --- lib/oxrdf/src/literal.rs | 41 ------------------------------- testsuite/rdf-tests | 2 +- testsuite/src/parser_evaluator.rs | 20 +++++++++++++++ testsuite/tests/parser.rs | 9 +++++++ 4 files changed, 30 insertions(+), 42 deletions(-) diff --git a/lib/oxrdf/src/literal.rs b/lib/oxrdf/src/literal.rs index 0e0b6da8..91f8455f 100644 --- a/lib/oxrdf/src/literal.rs +++ b/lib/oxrdf/src/literal.rs @@ -637,7 +637,6 @@ pub fn print_quoted_str(string: &str, f: &mut impl Write) -> fmt::Result { #[cfg(test)] mod tests { use super::*; - use std::str::FromStr; #[test] fn test_simple_literal_equality() { @@ -668,44 +667,4 @@ mod tests { assert_eq!("NaN", Literal::from(f32::NAN).value()); assert_eq!("NaN", Literal::from(f64::NAN).value()); } - - #[test] - fn test_canoincal_escaping() { - assert_eq!( - Literal::from_str(r#""\u0000\u0001\u0002\u0003\u0004\u0005\u0006\u0007\u0008\u0009\u000a\u000b\u000c\u000d\u000e\u000f""#).unwrap().to_string(), - r#""\u0000\u0001\u0002\u0003\u0004\u0005\u0006\u0007\b\t\n\u000B\f\r\u000E\u000F""# - ); - assert_eq!( - Literal::from_str(r#""\u0010\u0011\u0012\u0013\u0014\u0015\u0016\u0017\u0018\u0019\u001a\u001b\u001c\u001d\u001e\u001f""#).unwrap().to_string(), - r#""\u0010\u0011\u0012\u0013\u0014\u0015\u0016\u0017\u0018\u0019\u001A\u001B\u001C\u001D\u001E\u001F""# - ); - assert_eq!( - Literal::from_str(r#""\u0020\u0021\u0022\u0023\u0024\u0025\u0026\u0027\u0028\u0029\u002a\u002b\u002c\u002d\u002e\u002f""#).unwrap().to_string(), - r##"" !\"#$%&'()*+,-./""## - ); - assert_eq!( - Literal::from_str(r#""\u0030\u0031\u0032\u0033\u0034\u0035\u0036\u0037\u0038\u0039\u003a\u003b\u003c\u003d\u003e\u003f""#).unwrap().to_string(), - r#""0123456789:;<=>?""# - ); - assert_eq!( - Literal::from_str(r#""\u0040\u0041\u0042\u0043\u0044\u0045\u0046\u0047\u0048\u0049\u004a\u004b\u004c\u004d\u004e\u004f""#).unwrap().to_string(), - r#""@ABCDEFGHIJKLMNO""# - ); - assert_eq!( - Literal::from_str(r#""\u0050\u0051\u0052\u0053\u0054\u0055\u0056\u0057\u0058\u0059\u005a\u005b\u005c\u005d\u005e\u005f""#).unwrap().to_string(), - r#""PQRSTUVWXYZ[\\]^_""# - ); - assert_eq!( - Literal::from_str(r#""\u0060\u0061\u0062\u0063\u0064\u0065\u0066\u0067\u0068\u0069\u006a\u006b\u006c\u006d\u006e\u006f""#).unwrap().to_string(), - r#""`abcdefghijklmno""# - ); - assert_eq!( - Literal::from_str(r#""\u0070\u0071\u0072\u0073\u0074\u0075\u0076\u0077\u0078\u0079\u007a\u007b\u007c\u007d\u007e\u007f""#).unwrap().to_string(), - r#""pqrstuvwxyz{|}~\u007F""# - ); - assert_eq!( - Literal::from_str(r#""\u0080\u0081\u0082\u0083\u0084\u0085\u0086\u0087\u0088\u0089\u008a\u008b\u008c\u008d\u008e\u008f""#).unwrap().to_string(), - "\"\u{80}\u{81}\u{82}\u{83}\u{84}\u{85}\u{86}\u{87}\u{88}\u{89}\u{8a}\u{8b}\u{8c}\u{8d}\u{8e}\u{8f}\"" - ); - } } diff --git a/testsuite/rdf-tests b/testsuite/rdf-tests index a0b910e0..59b0496b 160000 --- a/testsuite/rdf-tests +++ b/testsuite/rdf-tests @@ -1 +1 @@ -Subproject commit a0b910e0f2cff4d3a7e32f15ce0bfbc7db2768fb +Subproject commit 59b0496b050e1c2397f22d94af0609d8324d95f7 diff --git a/testsuite/src/parser_evaluator.rs b/testsuite/src/parser_evaluator.rs index adcaf595..0d3a22e6 100644 --- a/testsuite/src/parser_evaluator.rs +++ b/testsuite/src/parser_evaluator.rs @@ -67,6 +67,10 @@ pub fn register_parser_tests(evaluator: &mut TestEvaluator) { evaluator.register("http://www.w3.org/ns/rdftest#TestTrigNegativeEval", |t| { evaluate_negative_syntax_test(t, RdfFormat::TriG) }); + evaluator.register( + "http://www.w3.org/ns/rdftest#TestNTriplesPositiveC14N", + |t| evaluate_positive_c14n_test(t, RdfFormat::NTriples), + ); evaluator.register( "https://w3c.github.io/rdf-canon/tests/vocab#RDFC10EvalTest", |t| evaluate_positive_syntax_test(t, RdfFormat::NQuads), //TODO: not a proper implementation! @@ -166,6 +170,22 @@ fn evaluate_n3_eval_test(test: &Test, ignore_errors: bool) -> Result<()> { Ok(()) } +fn evaluate_positive_c14n_test(test: &Test, format: RdfFormat) -> Result<()> { + let action = test.action.as_deref().context("No action found")?; + let actual = load_dataset(action, format, false) + .with_context(|| format!("Parse error on file {action}"))? + .to_string(); + let results = test.result.as_ref().context("No tests result found")?; + let expected = + read_file_to_string(results).with_context(|| format!("Read error on file {results}"))?; + ensure!( + expected == actual, + "The two files are not equal. Diff:\n{}", + format_diff(&expected, &actual, "c14n") + ); + Ok(()) +} + fn n3_to_dataset(quads: Vec) -> Dataset { quads .into_iter() diff --git a/testsuite/tests/parser.rs b/testsuite/tests/parser.rs index 9e7141b5..0a272fd4 100644 --- a/testsuite/tests/parser.rs +++ b/testsuite/tests/parser.rs @@ -19,6 +19,15 @@ fn rdf12_n_triples_syntax_w3c_testsuite() -> Result<()> { ) } +#[cfg(not(windows))] // Tests don't like git auto "\r\n" on Windows +#[test] +fn rdf12_n_triples_c14n_w3c_testsuite() -> Result<()> { + check_testsuite( + "https://w3c.github.io/rdf-tests/rdf/rdf12/rdf-n-triples/c14n/manifest.ttl", + &[], + ) +} + #[test] fn rdf11_n_quads_w3c_testsuite() -> Result<()> { check_testsuite( From ea80c11d6e7962869e27cdbe292b781e07b0a9bf Mon Sep 17 00:00:00 2001 From: Tpt Date: Fri, 20 Oct 2023 09:40:10 +0200 Subject: [PATCH 112/217] CI: Run clippy on all targets --- .github/workflows/tests.yml | 24 +++++------ cli/src/main.rs | 2 + lib/oxrdf/src/blank_node.rs | 16 ++++---- lib/oxrdf/src/dataset.rs | 67 +++++++++++++++++-------------- lib/oxrdf/src/literal.rs | 2 + lib/oxrdfxml/src/serializer.rs | 29 +++++++------ lib/oxsdatatypes/src/boolean.rs | 2 + lib/oxsdatatypes/src/date_time.rs | 11 ++--- lib/oxsdatatypes/src/decimal.rs | 42 ++++++++++--------- lib/oxsdatatypes/src/double.rs | 2 + lib/oxsdatatypes/src/duration.rs | 2 + lib/oxsdatatypes/src/float.rs | 2 + lib/oxsdatatypes/src/integer.rs | 28 +++++++------ lib/oxttl/src/trig.rs | 2 + lib/oxttl/src/turtle.rs | 2 + lib/sparesults/src/csv.rs | 2 + lib/src/sparql/eval.rs | 3 ++ lib/src/storage/binary_encoder.rs | 2 + lib/tests/store.rs | 10 ++--- 19 files changed, 145 insertions(+), 105 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 801c60b2..2c0dcfbb 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -28,29 +28,29 @@ jobs: submodules: true - run: rustup update && rustup default 1.70.0 && rustup component add clippy - uses: Swatinem/rust-cache@v2 - - run: cargo clippy -- -D warnings -D clippy::all + - run: cargo clippy --all-targets -- -D warnings -D clippy::all working-directory: ./lib/oxsdatatypes - - run: cargo clippy -- -D warnings -D clippy::all + - run: cargo clippy --all-targets -- -D warnings -D clippy::all working-directory: ./lib/oxrdf - - run: cargo clippy -- -D warnings -D clippy::all + - run: cargo clippy --all-targets -- -D warnings -D clippy::all working-directory: ./lib/oxrdfxml - - run: cargo clippy -- -D warnings -D clippy::all + - run: cargo clippy --all-targets -- -D warnings -D clippy::all working-directory: ./lib/oxttl - - run: cargo clippy -- -D warnings -D clippy::all + - run: cargo clippy --all-targets -- -D warnings -D clippy::all working-directory: ./lib/oxrdfio - - run: cargo clippy -- -D warnings -D clippy::all + - run: cargo clippy --all-targets -- -D warnings -D clippy::all working-directory: ./lib/sparesults - - run: cargo clippy -- -D warnings -D clippy::all + - run: cargo clippy --all-targets -- -D warnings -D clippy::all working-directory: ./lib/spargebra - - run: cargo clippy -- -D warnings -D clippy::all + - run: cargo clippy --all-targets -- -D warnings -D clippy::all working-directory: ./lib/sparopt - - run: cargo clippy -- -D warnings -D clippy::all + - run: cargo clippy --all-targets -- -D warnings -D clippy::all working-directory: ./lib - - run: cargo clippy -- -D warnings -D clippy::all + - run: cargo clippy --all-targets -- -D warnings -D clippy::all working-directory: ./python - - run: cargo clippy -- -D warnings -D clippy::all + - run: cargo clippy --all-targets -- -D warnings -D clippy::all working-directory: ./cli - - run: cargo clippy -- -D warnings -D clippy::all + - run: cargo clippy --all-targets -- -D warnings -D clippy::all working-directory: ./testsuite clippy_wasm_js: diff --git a/cli/src/main.rs b/cli/src/main.rs index 07c84592..6b2abba4 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -1818,6 +1818,8 @@ fn systemd_notify_ready() -> io::Result<()> { #[cfg(test)] mod tests { + #![allow(clippy::panic_in_result_fn)] + use super::*; use anyhow::Result; use assert_cmd::Command; diff --git a/lib/oxrdf/src/blank_node.rs b/lib/oxrdf/src/blank_node.rs index bfd27231..937b1efb 100644 --- a/lib/oxrdf/src/blank_node.rs +++ b/lib/oxrdf/src/blank_node.rs @@ -363,6 +363,8 @@ impl Error for BlankNodeIdParseError {} #[cfg(test)] mod tests { + #![allow(clippy::panic_in_result_fn)] + use super::*; #[test] @@ -379,13 +381,13 @@ mod tests { #[test] fn new_validation() { - assert!(BlankNode::new("").is_err()); - assert!(BlankNode::new("a").is_ok()); - assert!(BlankNode::new("-").is_err()); - assert!(BlankNode::new("a-").is_ok()); - assert!(BlankNode::new(".").is_err()); - assert!(BlankNode::new("a.").is_err()); - assert!(BlankNode::new("a.a").is_ok()); + BlankNode::new("").unwrap_err(); + BlankNode::new("a").unwrap(); + BlankNode::new("-").unwrap_err(); + BlankNode::new("a-").unwrap(); + BlankNode::new(".").unwrap_err(); + BlankNode::new("a.").unwrap_err(); + BlankNode::new("a.a").unwrap(); } #[test] diff --git a/lib/oxrdf/src/dataset.rs b/lib/oxrdf/src/dataset.rs index 017825b0..eb655dea 100644 --- a/lib/oxrdf/src/dataset.rs +++ b/lib/oxrdf/src/dataset.rs @@ -1567,35 +1567,40 @@ type QuadsPerBlankNode = HashMap< )>, >; -#[test] -fn test_canon() { - let mut dataset = Dataset::new(); - dataset.insert(QuadRef::new( - BlankNode::default().as_ref(), - NamedNodeRef::new_unchecked("http://ex"), - BlankNode::default().as_ref(), - GraphNameRef::DefaultGraph, - )); - dataset.insert(QuadRef::new( - BlankNode::default().as_ref(), - NamedNodeRef::new_unchecked("http://ex"), - BlankNode::default().as_ref(), - GraphNameRef::DefaultGraph, - )); - dataset.canonicalize(); - let mut dataset2 = Dataset::new(); - dataset2.insert(QuadRef::new( - BlankNode::default().as_ref(), - NamedNodeRef::new_unchecked("http://ex"), - BlankNode::default().as_ref(), - GraphNameRef::DefaultGraph, - )); - dataset2.insert(QuadRef::new( - BlankNode::default().as_ref(), - NamedNodeRef::new_unchecked("http://ex"), - BlankNode::default().as_ref(), - GraphNameRef::DefaultGraph, - )); - dataset2.canonicalize(); - assert_eq!(dataset, dataset2); +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_canon() { + let mut dataset = Dataset::new(); + dataset.insert(QuadRef::new( + BlankNode::default().as_ref(), + NamedNodeRef::new_unchecked("http://ex"), + BlankNode::default().as_ref(), + GraphNameRef::DefaultGraph, + )); + dataset.insert(QuadRef::new( + BlankNode::default().as_ref(), + NamedNodeRef::new_unchecked("http://ex"), + BlankNode::default().as_ref(), + GraphNameRef::DefaultGraph, + )); + dataset.canonicalize(); + let mut dataset2 = Dataset::new(); + dataset2.insert(QuadRef::new( + BlankNode::default().as_ref(), + NamedNodeRef::new_unchecked("http://ex"), + BlankNode::default().as_ref(), + GraphNameRef::DefaultGraph, + )); + dataset2.insert(QuadRef::new( + BlankNode::default().as_ref(), + NamedNodeRef::new_unchecked("http://ex"), + BlankNode::default().as_ref(), + GraphNameRef::DefaultGraph, + )); + dataset2.canonicalize(); + assert_eq!(dataset, dataset2); + } } diff --git a/lib/oxrdf/src/literal.rs b/lib/oxrdf/src/literal.rs index 91f8455f..809f10e4 100644 --- a/lib/oxrdf/src/literal.rs +++ b/lib/oxrdf/src/literal.rs @@ -636,6 +636,8 @@ pub fn print_quoted_str(string: &str, f: &mut impl Write) -> fmt::Result { #[cfg(test)] mod tests { + #![allow(clippy::panic_in_result_fn)] + use super::*; #[test] diff --git a/lib/oxrdfxml/src/serializer.rs b/lib/oxrdfxml/src/serializer.rs index 01e5b4b4..8a025390 100644 --- a/lib/oxrdfxml/src/serializer.rs +++ b/lib/oxrdfxml/src/serializer.rs @@ -338,16 +338,21 @@ fn split_iri(iri: &str) -> (&str, &str) { } } -#[test] -fn test_split_iri() { - assert_eq!( - split_iri("http://schema.org/Person"), - ("http://schema.org/", "Person") - ); - assert_eq!(split_iri("http://schema.org/"), ("http://schema.org/", "")); - assert_eq!( - split_iri("http://schema.org#foo"), - ("http://schema.org#", "foo") - ); - assert_eq!(split_iri("urn:isbn:foo"), ("urn:isbn:", "foo")); +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_split_iri() { + assert_eq!( + split_iri("http://schema.org/Person"), + ("http://schema.org/", "Person") + ); + assert_eq!(split_iri("http://schema.org/"), ("http://schema.org/", "")); + assert_eq!( + split_iri("http://schema.org#foo"), + ("http://schema.org#", "foo") + ); + assert_eq!(split_iri("urn:isbn:foo"), ("urn:isbn:", "foo")); + } } diff --git a/lib/oxsdatatypes/src/boolean.rs b/lib/oxsdatatypes/src/boolean.rs index de29debc..688af076 100644 --- a/lib/oxsdatatypes/src/boolean.rs +++ b/lib/oxsdatatypes/src/boolean.rs @@ -85,6 +85,8 @@ impl fmt::Display for Boolean { #[cfg(test)] mod tests { + #![allow(clippy::panic_in_result_fn)] + use super::*; #[test] diff --git a/lib/oxsdatatypes/src/date_time.rs b/lib/oxsdatatypes/src/date_time.rs index 6c525794..f7c80431 100644 --- a/lib/oxsdatatypes/src/date_time.rs +++ b/lib/oxsdatatypes/src/date_time.rs @@ -2452,6 +2452,8 @@ impl Error for InvalidTimezoneError {} #[cfg(test)] mod tests { + #![allow(clippy::panic_in_result_fn)] + use super::*; #[test] @@ -2613,9 +2615,9 @@ mod tests { assert_eq!(GMonth::from_str("--01+01:00")?.to_string(), "--01+01:00"); assert_eq!(GMonth::from_str("--01")?.to_string(), "--01"); - assert!(GYear::from_str("02020").is_err()); - assert!(GYear::from_str("+2020").is_err()); - assert!(GYear::from_str("33").is_err()); + GYear::from_str("02020").unwrap_err(); + GYear::from_str("+2020").unwrap_err(); + GYear::from_str("33").unwrap_err(); assert_eq!(Time::from_str("00:00:00+14:00")?, Time::MIN); assert_eq!(Time::from_str("24:00:00-14:00")?, Time::MAX); @@ -3155,10 +3157,9 @@ mod tests { } #[test] - fn g_year_month_from_date() -> Result<(), ParseDateTimeError> { + fn g_year_month_from_date() { assert_eq!(GYearMonth::from(Date::MIN), GYearMonth::MIN); assert_eq!(GYearMonth::from(Date::MAX), GYearMonth::MAX); - Ok(()) } #[test] diff --git a/lib/oxsdatatypes/src/decimal.rs b/lib/oxsdatatypes/src/decimal.rs index fca06e5a..bb2090fd 100644 --- a/lib/oxsdatatypes/src/decimal.rs +++ b/lib/oxsdatatypes/src/decimal.rs @@ -674,6 +674,8 @@ impl Error for TooLargeForDecimalError {} #[cfg(test)] mod tests { + #![allow(clippy::panic_in_result_fn)] + use super::*; #[test] @@ -688,14 +690,14 @@ mod tests { #[test] fn from_str() -> Result<(), ParseDecimalError> { - assert!(Decimal::from_str("").is_err()); - assert!(Decimal::from_str("+").is_err()); - assert!(Decimal::from_str("-").is_err()); - assert!(Decimal::from_str(".").is_err()); - assert!(Decimal::from_str("+.").is_err()); - assert!(Decimal::from_str("-.").is_err()); - assert!(Decimal::from_str("a").is_err()); - assert!(Decimal::from_str(".a").is_err()); + Decimal::from_str("").unwrap_err(); + Decimal::from_str("+").unwrap_err(); + Decimal::from_str("-").unwrap_err(); + Decimal::from_str(".").unwrap_err(); + Decimal::from_str("+.").unwrap_err(); + Decimal::from_str("-.").unwrap_err(); + Decimal::from_str("a").unwrap_err(); + Decimal::from_str(".a").unwrap_err(); assert_eq!(Decimal::from_str("210")?.to_string(), "210"); assert_eq!(Decimal::from_str("1000")?.to_string(), "1000"); assert_eq!(Decimal::from_str("-1.23")?.to_string(), "-1.23"); @@ -713,8 +715,8 @@ mod tests { assert_eq!(Decimal::from_str("-0")?.to_string(), "0"); assert_eq!(Decimal::from_str(&Decimal::MAX.to_string())?, Decimal::MAX); assert_eq!(Decimal::from_str(&Decimal::MIN.to_string())?, Decimal::MIN); - assert!(Decimal::from_str("0.0000000000000000001").is_err()); - assert!(Decimal::from_str("1000000000000000000000").is_err()); + Decimal::from_str("0.0000000000000000001").unwrap_err(); + Decimal::from_str("1000000000000000000000").unwrap_err(); assert_eq!( Decimal::from_str("0.100000000000000000000000000").unwrap(), Decimal::from_str("0.1").unwrap() @@ -1019,11 +1021,11 @@ mod tests { Decimal::try_from(Float::from(-123.5)).ok(), Some(Decimal::from_str("-123.5")?) ); - assert!(Decimal::try_from(Float::from(f32::NAN)).is_err()); - assert!(Decimal::try_from(Float::from(f32::INFINITY)).is_err()); - assert!(Decimal::try_from(Float::from(f32::NEG_INFINITY)).is_err()); - assert!(Decimal::try_from(Float::from(f32::MIN)).is_err()); - assert!(Decimal::try_from(Float::from(f32::MAX)).is_err()); + Decimal::try_from(Float::from(f32::NAN)).unwrap_err(); + Decimal::try_from(Float::from(f32::INFINITY)).unwrap_err(); + Decimal::try_from(Float::from(f32::NEG_INFINITY)).unwrap_err(); + Decimal::try_from(Float::from(f32::MIN)).unwrap_err(); + Decimal::try_from(Float::from(f32::MAX)).unwrap_err(); assert!( Decimal::try_from(Float::from(1_672_507_300_000.)) .unwrap() @@ -1059,11 +1061,11 @@ mod tests { .unwrap() < Decimal::from(1) ); - assert!(Decimal::try_from(Double::from(f64::NAN)).is_err()); - assert!(Decimal::try_from(Double::from(f64::INFINITY)).is_err()); - assert!(Decimal::try_from(Double::from(f64::NEG_INFINITY)).is_err()); - assert!(Decimal::try_from(Double::from(f64::MIN)).is_err()); - assert!(Decimal::try_from(Double::from(f64::MAX)).is_err()); + Decimal::try_from(Double::from(f64::NAN)).unwrap_err(); + Decimal::try_from(Double::from(f64::INFINITY)).unwrap_err(); + Decimal::try_from(Double::from(f64::NEG_INFINITY)).unwrap_err(); + Decimal::try_from(Double::from(f64::MIN)).unwrap_err(); + Decimal::try_from(Double::from(f64::MAX)).unwrap_err(); Ok(()) } diff --git a/lib/oxsdatatypes/src/double.rs b/lib/oxsdatatypes/src/double.rs index bc040738..5ce5790d 100644 --- a/lib/oxsdatatypes/src/double.rs +++ b/lib/oxsdatatypes/src/double.rs @@ -261,6 +261,8 @@ impl Div for Double { #[cfg(test)] mod tests { + #![allow(clippy::panic_in_result_fn)] + use super::*; #[test] diff --git a/lib/oxsdatatypes/src/duration.rs b/lib/oxsdatatypes/src/duration.rs index f8162272..1cf33ffb 100644 --- a/lib/oxsdatatypes/src/duration.rs +++ b/lib/oxsdatatypes/src/duration.rs @@ -996,6 +996,8 @@ impl From for ParseDurationError { #[cfg(test)] mod tests { + #![allow(clippy::panic_in_result_fn)] + use super::*; #[test] diff --git a/lib/oxsdatatypes/src/float.rs b/lib/oxsdatatypes/src/float.rs index 996a6401..e8784b84 100644 --- a/lib/oxsdatatypes/src/float.rs +++ b/lib/oxsdatatypes/src/float.rs @@ -251,6 +251,8 @@ impl Div for Float { #[cfg(test)] mod tests { + #![allow(clippy::panic_in_result_fn)] + use super::*; #[test] diff --git a/lib/oxsdatatypes/src/integer.rs b/lib/oxsdatatypes/src/integer.rs index 352e521a..376deea6 100644 --- a/lib/oxsdatatypes/src/integer.rs +++ b/lib/oxsdatatypes/src/integer.rs @@ -278,6 +278,8 @@ impl Error for TooLargeForIntegerError {} #[cfg(test)] mod tests { + #![allow(clippy::panic_in_result_fn)] + use super::*; #[test] @@ -286,7 +288,7 @@ mod tests { assert_eq!(Integer::from_str("-0")?.to_string(), "0"); assert_eq!(Integer::from_str("123")?.to_string(), "123"); assert_eq!(Integer::from_str("-123")?.to_string(), "-123"); - assert!(Integer::from_str("123456789123456789123456789123456789123456789").is_err()); + Integer::from_str("123456789123456789123456789123456789123456789").unwrap_err(); Ok(()) } @@ -304,11 +306,11 @@ mod tests { Integer::try_from(Float::from(-123.1)).ok(), Some(Integer::from_str("-123")?) ); - assert!(Integer::try_from(Float::from(f32::NAN)).is_err()); - assert!(Integer::try_from(Float::from(f32::INFINITY)).is_err()); - assert!(Integer::try_from(Float::from(f32::NEG_INFINITY)).is_err()); - assert!(Integer::try_from(Float::from(f32::MIN)).is_err()); - assert!(Integer::try_from(Float::from(f32::MAX)).is_err()); + Integer::try_from(Float::from(f32::NAN)).unwrap_err(); + Integer::try_from(Float::from(f32::INFINITY)).unwrap_err(); + Integer::try_from(Float::from(f32::NEG_INFINITY)).unwrap_err(); + Integer::try_from(Float::from(f32::MIN)).unwrap_err(); + Integer::try_from(Float::from(f32::MAX)).unwrap_err(); assert!( Integer::try_from(Float::from(1_672_507_300_000.)) .unwrap() @@ -344,11 +346,11 @@ mod tests { .unwrap() < Integer::from(10) ); - assert!(Integer::try_from(Double::from(f64::NAN)).is_err()); - assert!(Integer::try_from(Double::from(f64::INFINITY)).is_err()); - assert!(Integer::try_from(Double::from(f64::NEG_INFINITY)).is_err()); - assert!(Integer::try_from(Double::from(f64::MIN)).is_err()); - assert!(Integer::try_from(Double::from(f64::MAX)).is_err()); + Integer::try_from(Double::from(f64::NAN)).unwrap_err(); + Integer::try_from(Double::from(f64::INFINITY)).unwrap_err(); + Integer::try_from(Double::from(f64::NEG_INFINITY)).unwrap_err(); + Integer::try_from(Double::from(f64::MIN)).unwrap_err(); + Integer::try_from(Double::from(f64::MAX)).unwrap_err(); Ok(()) } @@ -362,8 +364,8 @@ mod tests { Integer::try_from(Decimal::from_str("-123.1").unwrap()).ok(), Some(Integer::from_str("-123")?) ); - assert!(Integer::try_from(Decimal::MIN).is_err()); - assert!(Integer::try_from(Decimal::MAX).is_err()); + Integer::try_from(Decimal::MIN).unwrap_err(); + Integer::try_from(Decimal::MAX).unwrap_err(); Ok(()) } diff --git a/lib/oxttl/src/trig.rs b/lib/oxttl/src/trig.rs index 1b5e04a3..748e3d6a 100644 --- a/lib/oxttl/src/trig.rs +++ b/lib/oxttl/src/trig.rs @@ -860,6 +860,8 @@ fn is_turtle_double(value: &str) -> bool { #[cfg(test)] mod tests { + #![allow(clippy::panic_in_result_fn)] + use super::*; use oxrdf::vocab::xsd; use oxrdf::{BlankNodeRef, GraphNameRef, LiteralRef, NamedNodeRef}; diff --git a/lib/oxttl/src/turtle.rs b/lib/oxttl/src/turtle.rs index 7106758d..2272e9f8 100644 --- a/lib/oxttl/src/turtle.rs +++ b/lib/oxttl/src/turtle.rs @@ -666,6 +666,8 @@ impl LowLevelTurtleWriter { #[cfg(test)] mod tests { + #![allow(clippy::panic_in_result_fn)] + use super::*; use oxrdf::{BlankNodeRef, LiteralRef, NamedNodeRef}; diff --git a/lib/sparesults/src/csv.rs b/lib/sparesults/src/csv.rs index e45f521f..b0dc99f1 100644 --- a/lib/sparesults/src/csv.rs +++ b/lib/sparesults/src/csv.rs @@ -475,6 +475,8 @@ impl LineReader { #[cfg(test)] mod tests { + #![allow(clippy::panic_in_result_fn)] + use super::*; use std::error::Error; use std::rc::Rc; diff --git a/lib/src/sparql/eval.rs b/lib/src/sparql/eval.rs index e21ee891..6f52228d 100644 --- a/lib/src/sparql/eval.rs +++ b/lib/src/sparql/eval.rs @@ -5873,7 +5873,10 @@ impl Timer { #[cfg(test)] mod tests { + #![allow(clippy::panic_in_result_fn)] + use super::*; + #[test] fn uuid() { let mut buffer = String::default(); diff --git a/lib/src/storage/binary_encoder.rs b/lib/src/storage/binary_encoder.rs index cd2272dd..c06a8d7e 100644 --- a/lib/src/storage/binary_encoder.rs +++ b/lib/src/storage/binary_encoder.rs @@ -633,6 +633,8 @@ pub fn write_term(sink: &mut Vec, term: &EncodedTerm) { #[cfg(test)] mod tests { + #![allow(clippy::panic_in_result_fn)] + use super::*; use crate::model::TermRef; use crate::storage::numeric_encoder::*; diff --git a/lib/tests/store.rs b/lib/tests/store.rs index e17fb69c..c9e67bfb 100644 --- a/lib/tests/store.rs +++ b/lib/tests/store.rs @@ -291,15 +291,15 @@ fn test_bad_stt_open() -> Result<(), Box> { let dir = TempDir::default(); let store = Store::open(&dir.0)?; remove_dir_all(&dir.0)?; - assert!(store + store .bulk_loader() .load_quads(once(Quad::new( NamedNode::new_unchecked("http://example.com/s"), NamedNode::new_unchecked("http://example.com/p"), NamedNode::new_unchecked("http://example.com/o"), - GraphName::DefaultGraph + GraphName::DefaultGraph, ))) - .is_err()); + .unwrap_err(); Ok(()) } @@ -350,7 +350,7 @@ fn test_bad_backup() -> Result<(), Box> { let backup_dir = TempDir::default(); create_dir_all(&backup_dir.0)?; - assert!(Store::open(&store_dir)?.backup(&backup_dir.0).is_err()); + Store::open(&store_dir)?.backup(&backup_dir.0).unwrap_err(); Ok(()) } @@ -358,7 +358,7 @@ fn test_bad_backup() -> Result<(), Box> { #[cfg(not(target_family = "wasm"))] fn test_backup_on_in_memory() -> Result<(), Box> { let backup_dir = TempDir::default(); - assert!(Store::new()?.backup(&backup_dir).is_err()); + Store::new()?.backup(&backup_dir).unwrap_err(); Ok(()) } From a2a6c5a41e492cf26f092b4c4249b9ca85f60f57 Mon Sep 17 00:00:00 2001 From: Tpt Date: Thu, 2 Nov 2023 18:18:56 +0100 Subject: [PATCH 113/217] Python: annotate Triple and Quad with sequence behaviors They should behave like tuples --- python/src/model.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/src/model.rs b/python/src/model.rs index 61f327a4..ffd3f0e1 100644 --- a/python/src/model.rs +++ b/python/src/model.rs @@ -614,7 +614,7 @@ impl IntoPy for PyTerm { /// A triple could also be easily destructed into its components: /// /// >>> (s, p, o) = Triple(NamedNode('http://example.com'), NamedNode('http://example.com/p'), Literal('1')) -#[pyclass(frozen, name = "Triple", module = "pyoxigraph")] +#[pyclass(frozen, sequence, name = "Triple", module = "pyoxigraph")] #[derive(Eq, PartialEq, Debug, Clone, Hash)] pub struct PyTriple { inner: Triple, @@ -812,7 +812,7 @@ impl IntoPy for PyGraphName { /// A quad could also be easily destructed into its components: /// /// >>> (s, p, o, g) = Quad(NamedNode('http://example.com'), NamedNode('http://example.com/p'), Literal('1'), NamedNode('http://example.com/g')) -#[pyclass(frozen, name = "Quad", module = "pyoxigraph")] +#[pyclass(frozen, sequence, name = "Quad", module = "pyoxigraph")] #[derive(Eq, PartialEq, Debug, Clone, Hash)] pub struct PyQuad { inner: Quad, From ab5f5c1c6066df8ca528811322947e045f96e925 Mon Sep 17 00:00:00 2001 From: Tpt Date: Thu, 2 Nov 2023 21:25:27 +0100 Subject: [PATCH 114/217] Upgrades quick-xml --- Cargo.lock | 4 ++-- lib/oxrdfxml/Cargo.toml | 2 +- lib/oxrdfxml/src/parser.rs | 2 +- lib/sparesults/Cargo.toml | 2 +- lib/sparesults/src/xml.rs | 16 +++++++--------- testsuite/src/lib.rs | 2 +- 6 files changed, 13 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 605c7a9e..20bf72d6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1376,9 +1376,9 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.30.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eff6510e86862b57b210fd8cbe8ed3f0d7d600b9c2863cd4549a2e033c66e956" +checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33" dependencies = [ "memchr", "tokio", diff --git a/lib/oxrdfxml/Cargo.toml b/lib/oxrdfxml/Cargo.toml index 83739b35..660a631d 100644 --- a/lib/oxrdfxml/Cargo.toml +++ b/lib/oxrdfxml/Cargo.toml @@ -22,7 +22,7 @@ async-tokio = ["dep:tokio", "quick-xml/async-tokio"] oxrdf = { version = "0.2.0-alpha.1-dev", path = "../oxrdf" } oxilangtag = "0.1" oxiri = "0.2" -quick-xml = ">=0.29, <0.31" +quick-xml = ">=0.29, <0.32" tokio = { version = "1.29", optional = true, features = ["io-util"] } [dev-dependencies] diff --git a/lib/oxrdfxml/src/parser.rs b/lib/oxrdfxml/src/parser.rs index 086d094a..f13bbbd4 100644 --- a/lib/oxrdfxml/src/parser.rs +++ b/lib/oxrdfxml/src/parser.rs @@ -1144,7 +1144,7 @@ impl RdfXmlReader { fn convert_attribute(&self, attribute: &Attribute) -> Result { Ok(attribute .decode_and_unescape_value_with(&self.reader, |e| self.resolve_entity(e))? - .to_string()) + .into_owned()) } fn convert_iri_attribute( diff --git a/lib/sparesults/Cargo.toml b/lib/sparesults/Cargo.toml index 2516cdfa..e5ffe16f 100644 --- a/lib/sparesults/Cargo.toml +++ b/lib/sparesults/Cargo.toml @@ -22,7 +22,7 @@ rdf-star = ["oxrdf/rdf-star"] json-event-parser = "0.2.0-alpha.1" memchr = "2.5" oxrdf = { version = "0.2.0-alpha.1-dev", path="../oxrdf" } -quick-xml = ">=0.29, <0.31" +quick-xml = ">=0.29, <0.32" [package.metadata.docs.rs] all-features = true diff --git a/lib/sparesults/src/xml.rs b/lib/sparesults/src/xml.rs index 73d26fff..7df4fa9e 100644 --- a/lib/sparesults/src/xml.rs +++ b/lib/sparesults/src/xml.rs @@ -28,7 +28,7 @@ fn do_write_boolean_xml_result(sink: W, value: bool) -> Result XmlSolutionsWriter { .with_attribute(("name", variable.as_str())) .write_empty()?; } - Ok(()) + quick_xml::Result::Ok(()) })?; writer.write_event(Event::Start(BytesStart::new("results")))?; Ok(Self { writer }) @@ -81,12 +81,9 @@ impl XmlSolutionsWriter { writer .create_element("binding") .with_attribute(("name", variable.as_str())) - .write_inner_content(|writer| { - write_xml_term(value, writer)?; - Ok(()) - })?; + .write_inner_content(|writer| write_xml_term(value, writer))?; } - Ok(()) + quick_xml::Result::Ok(()) })?; Ok(()) } @@ -150,7 +147,7 @@ fn write_xml_term( .write_inner_content(|writer| { write_xml_term(triple.object.as_ref(), writer) })?; - Ok(()) + quick_xml::Result::Ok(()) })?; } } @@ -370,7 +367,8 @@ impl XmlSolutionsReader { } else if event.local_name().as_ref() == b"bnode" { state = State::BNode; } else if event.local_name().as_ref() == b"literal" { - for attr in event.attributes().flatten() { + for attr in event.attributes() { + let attr = attr.map_err(quick_xml::Error::from)?; if attr.key.as_ref() == b"xml:lang" { lang = Some( attr.decode_and_unescape_value(&self.reader)?.to_string(), diff --git a/testsuite/src/lib.rs b/testsuite/src/lib.rs index fc42e12e..e9ce6df0 100644 --- a/testsuite/src/lib.rs +++ b/testsuite/src/lib.rs @@ -27,7 +27,7 @@ pub fn check_testsuite(manifest_url: &str, ignored_tests: &[&str]) -> Result<()> for result in results { if let Err(error) = &result.outcome { if !ignored_tests.contains(&result.test.as_str()) { - errors.push(format!("{}: failed with error {}", result.test, error)) + errors.push(format!("{}: failed with error {error:?}", result.test)) } } } From 8a7c6cf2c1ccc322ae029ffbd04ab5943e0524ff Mon Sep 17 00:00:00 2001 From: Tpt Date: Sat, 4 Nov 2023 20:51:54 +0100 Subject: [PATCH 115/217] Uses Ruff instead of Black --- .github/workflows/artifacts.yml | 4 +- .github/workflows/manylinux_build.sh | 2 +- .github/workflows/musllinux_build.sh | 2 +- .github/workflows/tests.yml | 8 +- python/generate_stubs.py | 119 +++++++-------------------- python/pyproject.toml | 1 - python/requirements.dev.txt | 3 +- python/tests/test_io.py | 8 +- python/tests/test_model.py | 8 +- python/tests/test_store.py | 8 +- 10 files changed, 43 insertions(+), 120 deletions(-) diff --git a/.github/workflows/artifacts.yml b/.github/workflows/artifacts.yml index 62123141..5e170600 100644 --- a/.github/workflows/artifacts.yml +++ b/.github/workflows/artifacts.yml @@ -187,7 +187,7 @@ jobs: working-directory: ./python - run: pip install --no-index --find-links=target/wheels/ pyoxigraph - run: rm -r target/wheels - - run: python generate_stubs.py pyoxigraph pyoxigraph.pyi --black + - run: python generate_stubs.py pyoxigraph pyoxigraph.pyi --ruff working-directory: ./python - run: maturin build --release --target universal2-apple-darwin --features abi3 working-directory: ./python @@ -226,7 +226,7 @@ jobs: working-directory: ./python - run: pip install --no-index --find-links=target/wheels/ pyoxigraph - run: rm -r target/wheels - - run: python generate_stubs.py pyoxigraph pyoxigraph.pyi --black + - run: python generate_stubs.py pyoxigraph pyoxigraph.pyi --ruff working-directory: ./python - run: maturin build --release --features abi3 working-directory: ./python diff --git a/.github/workflows/manylinux_build.sh b/.github/workflows/manylinux_build.sh index ccefe8b7..fe1d7a0f 100644 --- a/.github/workflows/manylinux_build.sh +++ b/.github/workflows/manylinux_build.sh @@ -12,7 +12,7 @@ python3.12 -m venv venv source venv/bin/activate pip install -r requirements.dev.txt maturin develop --release -python generate_stubs.py pyoxigraph pyoxigraph.pyi --black +python generate_stubs.py pyoxigraph pyoxigraph.pyi --ruff maturin build --release --no-default-features --features abi3 --features rustls --compatibility manylinux2014 if [ %for_each_version% ]; then for VERSION in 8 9 10 11 12; do diff --git a/.github/workflows/musllinux_build.sh b/.github/workflows/musllinux_build.sh index e85faa3f..6a519d20 100644 --- a/.github/workflows/musllinux_build.sh +++ b/.github/workflows/musllinux_build.sh @@ -10,7 +10,7 @@ python3.12 -m venv venv source venv/bin/activate pip install -r requirements.dev.txt maturin develop --release -python generate_stubs.py pyoxigraph pyoxigraph.pyi --black +python generate_stubs.py pyoxigraph pyoxigraph.pyi --ruff maturin build --release --no-default-features --features abi3 --features rustls --compatibility musllinux_1_2 if [ %for_each_version% ]; then for VERSION in 8 9 10 11 12; do diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2c0dcfbb..d0e45d20 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -227,8 +227,6 @@ jobs: cache: pip cache-dependency-path: '**/requirements.dev.txt' - run: pip install -r python/requirements.dev.txt - - run: python -m black --check --diff --color . - working-directory: ./python - run: maturin build -m python/Cargo.toml - run: pip install --no-index --find-links=target/wheels/ pyoxigraph - run: rm -r target/wheels @@ -238,13 +236,15 @@ jobs: working-directory: ./python/docs - run: sphinx-build -M html . build working-directory: ./python/docs - - run: python generate_stubs.py pyoxigraph pyoxigraph.pyi --black + - run: python generate_stubs.py pyoxigraph pyoxigraph.pyi --ruff working-directory: ./python - run: python -m mypy.stubtest pyoxigraph --allowlist=mypy_allowlist.txt working-directory: ./python - run: python -m mypy generate_stubs.py tests --strict working-directory: ./python - - run: python -m ruff check . + - run: python -m ruff format --check . + working-directory: ./python + - run: python -m ruff check --output-format=github . working-directory: ./python python_msv: diff --git a/python/generate_stubs.py b/python/generate_stubs.py index f343fd53..5498f2db 100644 --- a/python/generate_stubs.py +++ b/python/generate_stubs.py @@ -77,9 +77,7 @@ def module_stubs(module: Any) -> ast.Module: if member_name.startswith("__"): pass elif inspect.isclass(member_value): - classes.append( - class_stubs(member_name, member_value, element_path, types_to_import) - ) + classes.append(class_stubs(member_name, member_value, element_path, types_to_import)) elif inspect.isbuiltin(member_value): functions.append( function_stub( @@ -93,16 +91,12 @@ def module_stubs(module: Any) -> ast.Module: else: logging.warning(f"Unsupported root construction {member_name}") return ast.Module( - body=[ast.Import(names=[ast.alias(name=t)]) for t in sorted(types_to_import)] - + classes - + functions, + body=[ast.Import(names=[ast.alias(name=t)]) for t in sorted(types_to_import)] + classes + functions, type_ignores=[], ) -def class_stubs( - cls_name: str, cls_def: Any, element_path: List[str], types_to_import: Set[str] -) -> ast.ClassDef: +def class_stubs(cls_name: str, cls_def: Any, element_path: List[str], types_to_import: Set[str]) -> ast.ClassDef: attributes: List[ast.AST] = [] methods: List[ast.AST] = [] magic_methods: List[ast.AST] = [] @@ -124,20 +118,11 @@ def class_stubs( ] except ValueError as e: if "no signature found" not in str(e): - raise ValueError( - f"Error while parsing signature of {cls_name}.__init_" - ) from e - elif ( - member_value == OBJECT_MEMBERS.get(member_name) - or BUILTINS.get(member_name, ()) is None - ): + raise ValueError(f"Error while parsing signature of {cls_name}.__init_") from e + elif member_value == OBJECT_MEMBERS.get(member_name) or BUILTINS.get(member_name, ()) is None: pass elif inspect.isdatadescriptor(member_value): - attributes.extend( - data_descriptor_stub( - member_name, member_value, current_element_path, types_to_import - ) - ) + attributes.extend(data_descriptor_stub(member_name, member_value, current_element_path, types_to_import)) elif inspect.isroutine(member_value): (magic_methods if member_name.startswith("__") else methods).append( function_stub( @@ -154,9 +139,7 @@ def class_stubs( target=ast.Name(id=member_name, ctx=AST_STORE), annotation=ast.Subscript( value=_path_to_type("typing", "Tuple"), - slice=ast.Tuple( - elts=[_path_to_type("str"), ast.Ellipsis()], ctx=AST_LOAD - ), + slice=ast.Tuple(elts=[_path_to_type("str"), ast.Ellipsis()], ctx=AST_LOAD), ctx=AST_LOAD, ), value=ast.Constant(member_value), @@ -164,9 +147,7 @@ def class_stubs( ) ) else: - logging.warning( - f"Unsupported member {member_name} of class {'.'.join(element_path)}" - ) + logging.warning(f"Unsupported member {member_name} of class {'.'.join(element_path)}") doc = inspect.getdoc(cls_def) doc_comment = build_doc_comment(doc) if doc else None @@ -174,13 +155,7 @@ def class_stubs( cls_name, bases=[], keywords=[], - body=( - ([doc_comment] if doc_comment else []) - + attributes - + methods - + magic_methods - + constants - ) + body=(([doc_comment] if doc_comment else []) + attributes + methods + magic_methods + constants) or [AST_ELLIPSIS], decorator_list=[_path_to_type("typing", "final")], ) @@ -239,9 +214,7 @@ def function_stub( arguments_stub(fn_name, fn_def, doc or "", element_path, types_to_import), body or [AST_ELLIPSIS], decorator_list=decorator_list, - returns=returns_stub(fn_name, doc, element_path, types_to_import) - if doc - else None, + returns=returns_stub(fn_name, doc, element_path, types_to_import) if doc else None, lineno=0, ) @@ -253,9 +226,7 @@ def arguments_stub( element_path: List[str], types_to_import: Set[str], ) -> ast.arguments: - real_parameters: Mapping[str, inspect.Parameter] = inspect.signature( - callable_def - ).parameters + real_parameters: Mapping[str, inspect.Parameter] = inspect.signature(callable_def).parameters if callable_name == "__init__": real_parameters = { "self": inspect.Parameter("self", inspect.Parameter.POSITIONAL_ONLY), @@ -285,9 +256,7 @@ def arguments_stub( if type.endswith(", optional"): optional_params.add(match[0]) type = type[:-10] - parsed_param_types[match[0]] = convert_type_from_doc( - type, element_path, types_to_import - ) + parsed_param_types[match[0]] = convert_type_from_doc(type, element_path, types_to_import) # we parse the parameters posonlyargs = [] @@ -303,9 +272,7 @@ def arguments_stub( f"The parameter {param.name} of {'.'.join(element_path)} " "has no type definition in the function documentation" ) - param_ast = ast.arg( - arg=param.name, annotation=parsed_param_types.get(param.name) - ) + param_ast = ast.arg(arg=param.name, annotation=parsed_param_types.get(param.name)) default_ast = None if param.default != param.empty: @@ -346,9 +313,7 @@ def arguments_stub( ) -def returns_stub( - callable_name: str, doc: str, element_path: List[str], types_to_import: Set[str] -) -> Optional[ast.AST]: +def returns_stub(callable_name: str, doc: str, element_path: List[str], types_to_import: Set[str]) -> Optional[ast.AST]: m = re.findall(r"^ *:rtype: *([^\n]*) *$", doc, re.MULTILINE) if len(m) == 0: builtin = BUILTINS.get(callable_name) @@ -359,22 +324,16 @@ def returns_stub( "has no type definition using :rtype: in the function documentation" ) if len(m) > 1: - raise ValueError( - f"Multiple return type annotations found with :rtype: for {'.'.join(element_path)}" - ) + raise ValueError(f"Multiple return type annotations found with :rtype: for {'.'.join(element_path)}") return convert_type_from_doc(m[0], element_path, types_to_import) -def convert_type_from_doc( - type_str: str, element_path: List[str], types_to_import: Set[str] -) -> ast.AST: +def convert_type_from_doc(type_str: str, element_path: List[str], types_to_import: Set[str]) -> ast.AST: type_str = type_str.strip() return parse_type_to_ast(type_str, element_path, types_to_import) -def parse_type_to_ast( - type_str: str, element_path: List[str], types_to_import: Set[str] -) -> ast.AST: +def parse_type_to_ast(type_str: str, element_path: List[str], types_to_import: Set[str]) -> ast.AST: # let's tokenize tokens = [] current_token = "" @@ -412,26 +371,18 @@ def parse_type_to_ast( else: or_groups[-1].append(e) if any(not g for g in or_groups): - raise ValueError( - f"Not able to parse type '{type_str}' used by {'.'.join(element_path)}" - ) + raise ValueError(f"Not able to parse type '{type_str}' used by {'.'.join(element_path)}") new_elements: List[ast.AST] = [] for group in or_groups: if len(group) == 1 and isinstance(group[0], str): parts = group[0].split(".") if any(not p for p in parts): - raise ValueError( - f"Not able to parse type '{type_str}' used by {'.'.join(element_path)}" - ) + raise ValueError(f"Not able to parse type '{type_str}' used by {'.'.join(element_path)}") if len(parts) > 1: types_to_import.add(parts[0]) new_elements.append(_path_to_type(*parts)) - elif ( - len(group) == 2 - and isinstance(group[0], str) - and isinstance(group[1], list) - ): + elif len(group) == 2 and isinstance(group[0], str) and isinstance(group[1], list): if group[0] not in GENERICS: raise ValueError( f"Constructor {group[0]} is not supported in type '{type_str}' used by {'.'.join(element_path)}" @@ -444,9 +395,7 @@ def parse_type_to_ast( ) ) else: - raise ValueError( - f"Not able to parse type '{type_str}' used by {'.'.join(element_path)}" - ) + raise ValueError(f"Not able to parse type '{type_str}' used by {'.'.join(element_path)}") return ( ast.Subscript( value=_path_to_type("typing", "Union"), @@ -471,33 +420,21 @@ def build_doc_comment(doc: str) -> Optional[ast.Expr]: return ast.Expr(value=ast.Constant(text)) if text else None -def format_with_black(code: str) -> str: - result = subprocess.run( - ["python", "-m", "black", "-t", "py38", "--pyi", "-"], - input=code.encode(), - capture_output=True, - ) - result.check_returncode() - return result.stdout.decode() +def format_with_ruff(file: str) -> None: + subprocess.check_call(["python", "-m", "ruff", "format", file]) if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Extract Python type stub from a python module." - ) - parser.add_argument( - "module_name", help="Name of the Python module for which generate stubs" - ) + parser = argparse.ArgumentParser(description="Extract Python type stub from a python module.") + parser.add_argument("module_name", help="Name of the Python module for which generate stubs") parser.add_argument( "out", help="Name of the Python stub file to write to", type=argparse.FileType("wt"), ) - parser.add_argument( - "--black", help="Formats the generated stubs using Black", action="store_true" - ) + parser.add_argument("--ruff", help="Formats the generated stubs using Ruff", action="store_true") args = parser.parse_args() stub_content = ast.unparse(module_stubs(importlib.import_module(args.module_name))) - if args.black: - stub_content = format_with_black(stub_content) args.out.write(stub_content) + if args.ruff: + format_with_ruff(args.out.name) diff --git a/python/pyproject.toml b/python/pyproject.toml index dbdc8cd1..441baffb 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -39,7 +39,6 @@ select = [ "FBT", "I", "ICN", - "ISC", "N", "PIE", "PTH", diff --git a/python/requirements.dev.txt b/python/requirements.dev.txt index 8595505c..67eb7d2b 100644 --- a/python/requirements.dev.txt +++ b/python/requirements.dev.txt @@ -1,6 +1,5 @@ -black~=23.1 furo maturin~=1.0 mypy~=1.0 -ruff~=0.0.292 +ruff~=0.1.0 sphinx~=7.0 diff --git a/python/tests/test_io.py b/python/tests/test_io.py index c761148f..65223049 100644 --- a/python/tests/test_io.py +++ b/python/tests/test_io.py @@ -176,9 +176,7 @@ class TestSerialize(unittest.TestCase): class TestParseQuerySolutions(unittest.TestCase): def test_parse_file(self) -> None: with NamedTemporaryFile(suffix=".tsv") as fp: - fp.write( - b'?s\t?p\t?o\n\t\t"1"\n' - ) + fp.write(b'?s\t?p\t?o\n\t\t"1"\n') fp.flush() r = parse_query_results(fp.name) self.assertIsInstance(r, QuerySolutions) @@ -188,9 +186,7 @@ class TestParseQuerySolutions(unittest.TestCase): def test_parse_not_existing_file(self) -> None: with self.assertRaises(IOError) as _: - parse_query_results( - "/tmp/not-existing-oxigraph-file.ttl", "application/json" - ) + parse_query_results("/tmp/not-existing-oxigraph-file.ttl", "application/json") def test_parse_str_io(self) -> None: result = parse_query_results(StringIO("true"), "tsv") diff --git a/python/tests/test_model.py b/python/tests/test_model.py index 2931d06d..6bed69fd 100644 --- a/python/tests/test_model.py +++ b/python/tests/test_model.py @@ -138,9 +138,7 @@ class TestLiteral(unittest.TestCase): self.assertEqual(copy.deepcopy(number), number) def test_basic_match(self) -> None: - match_works( - self, 'Literal("foo", language="en")', 'Literal("foo", language="en")' - ) + match_works(self, 'Literal("foo", language="en")', 'Literal("foo", language="en")') match_works( self, 'Literal("1", datatype=XSD_INTEGER)', @@ -149,9 +147,7 @@ class TestLiteral(unittest.TestCase): def test_wildcard_match(self) -> None: match_works(self, 'Literal("foo", language="en")', "Literal(v, language=l)") - match_works( - self, 'Literal("1", datatype=XSD_INTEGER)', "Literal(v, datatype=d)" - ) + match_works(self, 'Literal("1", datatype=XSD_INTEGER)', "Literal(v, datatype=d)") class TestTriple(unittest.TestCase): diff --git a/python/tests/test_store.py b/python/tests/test_store.py index 84107d84..b9fc1be8 100644 --- a/python/tests/test_store.py +++ b/python/tests/test_store.py @@ -149,9 +149,7 @@ class TestStore(unittest.TestCase): store.add(Quad(foo, bar, baz, graph)) results: Any = store.query("SELECT ?s WHERE { ?s ?p ?o }") self.assertEqual(len(list(results)), 0) - results = store.query( - "SELECT ?s WHERE { ?s ?p ?o }", use_default_graph_as_union=True - ) + results = store.query("SELECT ?s WHERE { ?s ?p ?o }", use_default_graph_as_union=True) self.assertEqual(len(list(results)), 1) results = store.query( "SELECT ?s WHERE { ?s ?p ?o }", @@ -246,9 +244,7 @@ class TestStore(unittest.TestCase): def test_update_star(self) -> None: store = Store() - store.update( - "PREFIX : INSERT DATA { :alice :claims << :bob :age 23 >> }" - ) + store.update("PREFIX : INSERT DATA { :alice :claims << :bob :age 23 >> }") results: Any = store.query( "PREFIX : SELECT ?p ?a WHERE { ?p :claims << :bob :age ?a >> }" ) From 48db7f872be6a87e5282e4b9472e859839cd4fae Mon Sep 17 00:00:00 2001 From: Tpt Date: Fri, 13 Oct 2023 15:49:39 +0200 Subject: [PATCH 116/217] Python: any os.PathLike path is now supported Improves stub generation --- python/generate_stubs.py | 127 ++++++++++++++++++--------------------- python/src/io.rs | 8 +-- python/src/sparql.rs | 10 +-- python/src/store.rs | 20 +++--- 4 files changed, 76 insertions(+), 89 deletions(-) diff --git a/python/generate_stubs.py b/python/generate_stubs.py index 5498f2db..7d0469ec 100644 --- a/python/generate_stubs.py +++ b/python/generate_stubs.py @@ -5,65 +5,60 @@ import inspect import logging import re import subprocess +from functools import reduce from typing import Any, Dict, List, Mapping, Optional, Set, Tuple, Union -def _path_to_type(*elements: str) -> ast.AST: - base: ast.AST = ast.Name(id=elements[0], ctx=AST_LOAD) +def path_to_type(*elements: str) -> ast.AST: + base: ast.AST = ast.Name(id=elements[0], ctx=ast.Load()) for e in elements[1:]: - base = ast.Attribute(value=base, attr=e, ctx=AST_LOAD) + base = ast.Attribute(value=base, attr=e, ctx=ast.Load()) return base -AST_LOAD = ast.Load() -AST_ELLIPSIS = ast.Ellipsis() -AST_STORE = ast.Store() -AST_TYPING_ANY = _path_to_type("typing", "Any") -GENERICS = { - "iterable": _path_to_type("typing", "Iterable"), - "iterator": _path_to_type("typing", "Iterator"), - "list": _path_to_type("typing", "List"), - "io": _path_to_type("typing", "IO"), -} OBJECT_MEMBERS = dict(inspect.getmembers(object)) - - BUILTINS: Dict[str, Union[None, Tuple[List[ast.AST], ast.AST]]] = { "__annotations__": None, - "__bool__": ([], _path_to_type("bool")), - "__bytes__": ([], _path_to_type("bytes")), + "__bool__": ([], path_to_type("bool")), + "__bytes__": ([], path_to_type("bytes")), "__class__": None, - "__contains__": ([AST_TYPING_ANY], _path_to_type("bool")), + "__contains__": ([path_to_type("typing", "Any")], path_to_type("bool")), "__del__": None, - "__delattr__": ([_path_to_type("str")], _path_to_type("None")), - "__delitem__": ([AST_TYPING_ANY], AST_TYPING_ANY), + "__delattr__": ([path_to_type("str")], path_to_type("None")), + "__delitem__": ([path_to_type("typing", "Any")], path_to_type("typing", "Any")), "__dict__": None, "__dir__": None, "__doc__": None, - "__eq__": ([AST_TYPING_ANY], _path_to_type("bool")), - "__format__": ([_path_to_type("str")], _path_to_type("str")), - "__ge__": ([AST_TYPING_ANY], _path_to_type("bool")), - "__getattribute__": ([_path_to_type("str")], AST_TYPING_ANY), - "__getitem__": ([AST_TYPING_ANY], AST_TYPING_ANY), - "__gt__": ([AST_TYPING_ANY], _path_to_type("bool")), - "__hash__": ([], _path_to_type("int")), - "__init__": ([], _path_to_type("None")), + "__eq__": ([path_to_type("typing", "Any")], path_to_type("bool")), + "__format__": ([path_to_type("str")], path_to_type("str")), + "__ge__": ([path_to_type("typing", "Any")], path_to_type("bool")), + "__getattribute__": ([path_to_type("str")], path_to_type("typing", "Any")), + "__getitem__": ([path_to_type("typing", "Any")], path_to_type("typing", "Any")), + "__gt__": ([path_to_type("typing", "Any")], path_to_type("bool")), + "__hash__": ([], path_to_type("int")), + "__init__": ([], path_to_type("None")), "__init_subclass__": None, - "__iter__": ([], AST_TYPING_ANY), - "__le__": ([AST_TYPING_ANY], _path_to_type("bool")), - "__len__": ([], _path_to_type("int")), - "__lt__": ([AST_TYPING_ANY], _path_to_type("bool")), + "__iter__": ([], path_to_type("typing", "Any")), + "__le__": ([path_to_type("typing", "Any")], path_to_type("bool")), + "__len__": ([], path_to_type("int")), + "__lt__": ([path_to_type("typing", "Any")], path_to_type("bool")), "__module__": None, - "__ne__": ([AST_TYPING_ANY], _path_to_type("bool")), + "__ne__": ([path_to_type("typing", "Any")], path_to_type("bool")), "__new__": None, - "__next__": ([], AST_TYPING_ANY), + "__next__": ([], path_to_type("typing", "Any")), "__reduce__": None, "__reduce_ex__": None, - "__repr__": ([], _path_to_type("str")), - "__setattr__": ([_path_to_type("str"), AST_TYPING_ANY], _path_to_type("None")), - "__setitem__": ([AST_TYPING_ANY, AST_TYPING_ANY], AST_TYPING_ANY), + "__repr__": ([], path_to_type("str")), + "__setattr__": ( + [path_to_type("str"), path_to_type("typing", "Any")], + path_to_type("None"), + ), + "__setitem__": ( + [path_to_type("typing", "Any"), path_to_type("typing", "Any")], + path_to_type("typing", "Any"), + ), "__sizeof__": None, - "__str__": ([], _path_to_type("str")), + "__str__": ([], path_to_type("str")), "__subclasshook__": None, } @@ -136,11 +131,11 @@ def class_stubs(cls_name: str, cls_def: Any, element_path: List[str], types_to_i elif member_name == "__match_args__": constants.append( ast.AnnAssign( - target=ast.Name(id=member_name, ctx=AST_STORE), + target=ast.Name(id=member_name, ctx=ast.Store()), annotation=ast.Subscript( - value=_path_to_type("typing", "Tuple"), - slice=ast.Tuple(elts=[_path_to_type("str"), ast.Ellipsis()], ctx=AST_LOAD), - ctx=AST_LOAD, + value=path_to_type("tuple"), + slice=ast.Tuple(elts=[path_to_type("str"), ast.Ellipsis()], ctx=ast.Load()), + ctx=ast.Load(), ), value=ast.Constant(member_value), simple=1, @@ -156,8 +151,8 @@ def class_stubs(cls_name: str, cls_def: Any, element_path: List[str], types_to_i bases=[], keywords=[], body=(([doc_comment] if doc_comment else []) + attributes + methods + magic_methods + constants) - or [AST_ELLIPSIS], - decorator_list=[_path_to_type("typing", "final")], + or [ast.Ellipsis()], + decorator_list=[path_to_type("typing", "final")], ) @@ -182,8 +177,8 @@ def data_descriptor_stub( ) assign = ast.AnnAssign( - target=ast.Name(id=data_desc_name, ctx=AST_STORE), - annotation=annotation or AST_TYPING_ANY, + target=ast.Name(id=data_desc_name, ctx=ast.Store()), + annotation=annotation or path_to_type("typing", "Any"), simple=1, ) doc_comment = build_doc_comment(doc_comment) if doc_comment else None @@ -212,7 +207,7 @@ def function_stub( return ast.FunctionDef( fn_name, arguments_stub(fn_name, fn_def, doc or "", element_path, types_to_import), - body or [AST_ELLIPSIS], + body or [ast.Ellipsis()], decorator_list=decorator_list, returns=returns_stub(fn_name, doc, element_path, types_to_import) if doc else None, lineno=0, @@ -352,11 +347,11 @@ def parse_type_to_ast(type_str: str, element_path: List[str], types_to_import: S # let's first parse nested parenthesis stack: List[List[Any]] = [[]] for token in tokens: - if token == "(": + if token == "[": children: List[str] = [] stack[-1].append(children) stack.append(children) - elif token == ")": + elif token == "]": stack.pop() else: stack[-1].append(token) @@ -376,39 +371,31 @@ def parse_type_to_ast(type_str: str, element_path: List[str], types_to_import: S new_elements: List[ast.AST] = [] for group in or_groups: if len(group) == 1 and isinstance(group[0], str): - parts = group[0].split(".") - if any(not p for p in parts): - raise ValueError(f"Not able to parse type '{type_str}' used by {'.'.join(element_path)}") - if len(parts) > 1: - types_to_import.add(parts[0]) - new_elements.append(_path_to_type(*parts)) + new_elements.append(concatenated_path_to_type(group[0], element_path, types_to_import)) elif len(group) == 2 and isinstance(group[0], str) and isinstance(group[1], list): - if group[0] not in GENERICS: - raise ValueError( - f"Constructor {group[0]} is not supported in type '{type_str}' used by {'.'.join(element_path)}" - ) new_elements.append( ast.Subscript( - value=GENERICS[group[0]], + value=concatenated_path_to_type(group[0], element_path, types_to_import), slice=parse_sequence(group[1]), - ctx=AST_LOAD, + ctx=ast.Load(), ) ) else: raise ValueError(f"Not able to parse type '{type_str}' used by {'.'.join(element_path)}") - return ( - ast.Subscript( - value=_path_to_type("typing", "Union"), - slice=ast.Tuple(elts=new_elements, ctx=AST_LOAD), - ctx=AST_LOAD, - ) - if len(new_elements) > 1 - else new_elements[0] - ) + return reduce(lambda left, right: ast.BinOp(left=left, op=ast.BitOr(), right=right), new_elements) return parse_sequence(stack[0]) +def concatenated_path_to_type(path: str, element_path: List[str], types_to_import: Set[str]) -> ast.AST: + parts = path.split(".") + if any(not p for p in parts): + raise ValueError(f"Not able to parse type '{path}' used by {'.'.join(element_path)}") + if len(parts) > 1: + types_to_import.add(".".join(parts[:-1])) + return path_to_type(*parts) + + def build_doc_comment(doc: str) -> Optional[ast.Expr]: lines = [line.strip() for line in doc.split("\n")] clean_lines = [] diff --git a/python/src/io.rs b/python/src/io.rs index 73395d71..19c0b631 100644 --- a/python/src/io.rs +++ b/python/src/io.rs @@ -31,7 +31,7 @@ use std::sync::OnceLock; /// and ``application/xml`` or ``xml`` for `RDF/XML `_. /// /// :param input: The I/O object or file path to read from. For example, it could be a file path as a string or a file reader opened in binary mode with ``open('my_file.ttl', 'rb')``. -/// :type input: io(bytes) or io(str) or str or pathlib.Path +/// :type input: typing.IO[bytes] or typing.IO[str] or str or os.PathLike[str] /// :param format: the format of the RDF serialization using a media type like ``text/turtle`` or an extension like `ttl`. If :py:const:`None`, the format is guessed from the file name extension. /// :type format: str or None, optional /// :param base_iri: the base IRI used to resolve the relative IRIs in the file or :py:const:`None` if relative IRI resolution should not be done. @@ -41,7 +41,7 @@ use std::sync::OnceLock; /// :param rename_blank_nodes: Renames the blank nodes identifiers from the ones set in the serialization to random ids. This allows to avoid identifier conflicts when merging graphs together. /// :type rename_blank_nodes: bool, optional /// :return: an iterator of RDF triples or quads depending on the format. -/// :rtype: iterator(Quad) +/// :rtype: collections.abc.Iterator[Quad] /// :raises ValueError: if the format is not supported. /// :raises SyntaxError: if the provided data is invalid. /// :raises OSError: if a system error happens while reading the file. @@ -101,9 +101,9 @@ pub fn parse( /// and ``application/xml`` or ``xml`` for `RDF/XML `_. /// /// :param input: the RDF triples and quads to serialize. -/// :type input: iterable(Triple) or iterable(Quad) +/// :type input: collections.abc.Iterable[Triple] or collections.abc.Iterable[Quad] /// :param output: The binary I/O object or file path to write to. For example, it could be a file path as a string or a file writer opened in binary mode with ``open('my_file.ttl', 'wb')``. If :py:const:`None`, a :py:class:`bytes` buffer is returned with the serialized content. -/// :type output: io(bytes) or str or pathlib.Path or None, optional +/// :type output: typing.IO[bytes] or str or os.PathLike[str] or None, optional /// :param format: the format of the RDF serialization using a media type like ``text/turtle`` or an extension like `ttl`. If :py:const:`None`, the format is guessed from the file name extension. /// :type format: str or None, optional /// :return: py:class:`bytes` with the serialization if the ``output`` parameter is :py:const:`None`, :py:const:`None` if ``output`` is set. diff --git a/python/src/sparql.rs b/python/src/sparql.rs index 98698708..207a4818 100644 --- a/python/src/sparql.rs +++ b/python/src/sparql.rs @@ -195,7 +195,7 @@ enum PyQuerySolutionsVariant { #[pymethods] impl PyQuerySolutions { /// :return: the ordered list of all variables that could appear in the query results - /// :rtype: list(Variable) + /// :rtype: list[Variable] /// /// >>> store = Store() /// >>> store.query('SELECT ?s WHERE { ?s ?p ?o }').variables @@ -225,7 +225,7 @@ impl PyQuerySolutions { /// For example, ``application/json`` could also be used for `JSON `_. /// /// :param output: The binary I/O object or file path to write to. For example, it could be a file path as a string or a file writer opened in binary mode with ``open('my_file.ttl', 'wb')``. If :py:const:`None`, a :py:class:`bytes` buffer is returned with the serialized content. - /// :type output: io(bytes) or str or pathlib.Path or None, optional + /// :type output: typing.IO[bytes] or str or os.PathLike[str] or None, optional /// :param format: the format of the query results serialization using a media type like ``text/csv`` or an extension like `csv`. If :py:const:`None`, the format is guessed from the file name extension. /// :type format: str or None, optional /// :rtype: bytes or None @@ -325,7 +325,7 @@ impl PyQueryBoolean { /// For example, ``application/json`` could also be used for `JSON `_. /// /// :param output: The binary I/O object or file path to write to. For example, it could be a file path as a string or a file writer opened in binary mode with ``open('my_file.ttl', 'wb')``. If :py:const:`None`, a :py:class:`bytes` buffer is returned with the serialized content. - /// :type output: io(bytes) or str or pathlib.Path or None, optional + /// :type output: typing.IO[bytes] or str or os.PathLike[str] or None, optional /// :param format: the format of the query results serialization using a media type like ``text/csv`` or an extension like `csv`. If :py:const:`None`, the format is guessed from the file name extension. /// :type format: str or None, optional /// :rtype: bytes or None @@ -403,7 +403,7 @@ impl PyQueryTriples { /// and ``application/xml`` or ``xml`` for `RDF/XML `_. /// /// :param output: The binary I/O object or file path to write to. For example, it could be a file path as a string or a file writer opened in binary mode with ``open('my_file.ttl', 'wb')``. If :py:const:`None`, a :py:class:`bytes` buffer is returned with the serialized content. - /// :type output: io(bytes) or str or pathlib.Path or None, optional + /// :type output: typing.IO[bytes] or str or os.PathLike[str] or None, optional /// :param format: the format of the RDF serialization using a media type like ``text/turtle`` or an extension like `ttl`. If :py:const:`None`, the format is guessed from the file name extension. /// :type format: str or None, optional /// :rtype: bytes or None @@ -461,7 +461,7 @@ impl PyQueryTriples { /// For example, ``application/json`` could also be used for `JSON `_. /// /// :param input: The I/O object or file path to read from. For example, it could be a file path as a string or a file reader opened in binary mode with ``open('my_file.ttl', 'rb')``. -/// :type input: io(bytes) or io(str) or str or pathlib.Path +/// :type input: typing.IO[bytes] or typing.IO[str] or str or os.PathLike[str] /// :param format: the format of the RDF serialization using a media type like ``text/turtle`` or an extension like `ttl`. If :py:const:`None`, the format is guessed from the file name extension. /// :type format: str or None, optional /// :return: an iterator of :py:class:`QuerySolution` or a :py:class:`bool`. diff --git a/python/src/store.rs b/python/src/store.rs index 5574e652..54fad079 100644 --- a/python/src/store.rs +++ b/python/src/store.rs @@ -28,7 +28,7 @@ use std::path::PathBuf; /// :param path: the path of the directory in which the store should read and write its data. If the directory does not exist, it is created. /// If no directory is provided a temporary one is created and removed when the Python garbage collector removes the store. /// In this case, the store data are kept in memory and never written on disk. -/// :type path: str or pathlib.Path or None, optional +/// :type path: str or os.PathLike[str] or None, optional /// :raises OSError: if the target directory contains invalid data or could not be accessed. /// /// The :py:class:`str` function provides a serialization of the store in NQuads: @@ -137,7 +137,7 @@ impl PyStore { /// The :py:func:`bulk_extend` method is also available for much faster loading of a large number of quads but without transactional guarantees. /// /// :param quads: the quads to add. - /// :type quads: iterable(Quad) + /// :type quads: collections.abc.Iterable[Quad] /// :rtype: None /// :raises OSError: if an error happens during the quad insertion. /// @@ -162,7 +162,7 @@ impl PyStore { /// Only a part of the data might be written to the store. /// /// :param quads: the quads to add. - /// :type quads: iterable(Quad) + /// :type quads: collections.abc.Iterable[Quad] /// :rtype: None /// :raises OSError: if an error happens during the quad insertion. /// @@ -210,7 +210,7 @@ impl PyStore { /// :param graph_name: the quad graph name. To match only the default graph, use :py:class:`DefaultGraph`. To match everything use :py:const:`None`. /// :type graph_name: NamedNode or BlankNode or DefaultGraph or None, optional /// :return: an iterator of the quads matching the pattern. - /// :rtype: iterator(Quad) + /// :rtype: collections.abc.Iterator[Quad] /// :raises OSError: if an error happens during the quads lookup. /// /// >>> store = Store() @@ -246,9 +246,9 @@ impl PyStore { /// :param use_default_graph_as_union: if the SPARQL query should look for triples in all the dataset graphs by default (i.e. without `GRAPH` operations). Disabled by default. /// :type use_default_graph_as_union: bool, optional /// :param default_graph: list of the graphs that should be used as the query default graph. By default, the store default graph is used. - /// :type default_graph: NamedNode or BlankNode or DefaultGraph or list(NamedNode or BlankNode or DefaultGraph) or None, optional + /// :type default_graph: NamedNode or BlankNode or DefaultGraph or list[NamedNode or BlankNode or DefaultGraph] or None, optional /// :param named_graphs: list of the named graphs that could be used in SPARQL `GRAPH` clause. By default, all the store named graphs are available. - /// :type named_graphs: list(NamedNode or BlankNode) or None, optional + /// :type named_graphs: list[NamedNode or BlankNode] or None, optional /// :return: a :py:class:`bool` for ``ASK`` queries, an iterator of :py:class:`Triple` for ``CONSTRUCT`` and ``DESCRIBE`` queries and an iterator of :py:class:`QuerySolution` for ``SELECT`` queries. /// :rtype: QuerySolutions or QueryBoolean or QueryTriples /// :raises SyntaxError: if the provided query is invalid. @@ -361,7 +361,7 @@ impl PyStore { /// and ``application/xml`` or ``xml`` for `RDF/XML `_. /// /// :param input: The I/O object or file path to read from. For example, it could be a file path as a string or a file reader opened in binary mode with ``open('my_file.ttl', 'rb')``. - /// :type input: io(bytes) or io(str) or str or pathlib.Path + /// :type input: typing.IO[bytes] or typing.IO[str] or str or os.PathLike[str] /// :param format: the format of the RDF serialization using a media type like ``text/turtle`` or an extension like `ttl`. If :py:const:`None`, the format is guessed from the file name extension. /// :type format: str or None, optional /// :param base_iri: the base IRI used to resolve the relative IRIs in the file or :py:const:`None` if relative IRI resolution should not be done. @@ -430,7 +430,7 @@ impl PyStore { /// and ``application/xml`` or ``xml`` for `RDF/XML `_. /// /// :param input: The I/O object or file path to read from. For example, it could be a file path as a string or a file reader opened in binary mode with ``open('my_file.ttl', 'rb')``. - /// :type input: io(bytes) or io(str) or str or pathlib.Path + /// :type input: typing.IO[bytes] or typing.IO[str] or str or os.PathLike[str] /// :param format: the format of the RDF serialization using a media type like ``text/turtle`` or an extension like `ttl`. If :py:const:`None`, the format is guessed from the file name extension. /// :type format: str or None, optional /// :param base_iri: the base IRI used to resolve the relative IRIs in the file or :py:const:`None` if relative IRI resolution should not be done. @@ -497,7 +497,7 @@ impl PyStore { /// and ``application/xml`` or ``xml`` for `RDF/XML `_. /// /// :param output: The binary I/O object or file path to write to. For example, it could be a file path as a string or a file writer opened in binary mode with ``open('my_file.ttl', 'wb')``. If :py:const:`None`, a :py:class:`bytes` buffer is returned with the serialized content. - /// :type output: io(bytes) or str or pathlib.Path or None, optional + /// :type output: typing.IO[bytes] or str or os.PathLike[str] or None, optional /// :param format: the format of the RDF serialization using a media type like ``text/turtle`` or an extension like `ttl`. If :py:const:`None`, the format is guessed from the file name extension. /// :type format: str or None, optional /// :param from_graph: the store graph from which dump the triples. Required if the serialization format does not support named graphs. If it does supports named graphs the full dataset is written. @@ -551,7 +551,7 @@ impl PyStore { /// Returns an iterator over all the store named graphs. /// /// :return: an iterator of the store graph names. - /// :rtype: iterator(NamedNode or BlankNode) + /// :rtype: collections.abc.Iterator[NamedNode or BlankNode] /// :raises OSError: if an error happens during the named graphs lookup. /// /// >>> store = Store() From 261f9c64a59a7d547b2d384e8d57d944065ed290 Mon Sep 17 00:00:00 2001 From: Tpt Date: Fri, 13 Oct 2023 17:51:02 +0200 Subject: [PATCH 117/217] Python: I/O adds a `path` parameter to read/write from/to a file --- oxrocksdb-sys/rocksdb | 2 +- python/src/io.rs | 69 +++++++++++++++++++++++++++----------- python/src/store.rs | 56 +++++++++++++++---------------- python/tests/test_io.py | 38 +++++++++++++++++---- python/tests/test_store.py | 12 +++---- 5 files changed, 114 insertions(+), 63 deletions(-) diff --git a/oxrocksdb-sys/rocksdb b/oxrocksdb-sys/rocksdb index 5f2d6f0c..1ce22dd6 160000 --- a/oxrocksdb-sys/rocksdb +++ b/oxrocksdb-sys/rocksdb @@ -1 +1 @@ -Subproject commit 5f2d6f0cba9858130be48ae129dd9c9dcafe0f97 +Subproject commit 1ce22dd6376b124d17eff7d96e0809d2f4b4ae70 diff --git a/python/src/io.rs b/python/src/io.rs index 19c0b631..44cc48e9 100644 --- a/python/src/io.rs +++ b/python/src/io.rs @@ -30,10 +30,12 @@ use std::sync::OnceLock; /// For example, ``application/turtle`` could also be used for `Turtle `_ /// and ``application/xml`` or ``xml`` for `RDF/XML `_. /// -/// :param input: The I/O object or file path to read from. For example, it could be a file path as a string or a file reader opened in binary mode with ``open('my_file.ttl', 'rb')``. -/// :type input: typing.IO[bytes] or typing.IO[str] or str or os.PathLike[str] +/// :param input: The :py:class:`str`, :py:class:`bytes` or I/O object to read from. For example, it could be the file content as a string or a file reader opened in binary mode with ``open('my_file.ttl', 'rb')``. +/// :type input: bytes or str or typing.IO[bytes] or typing.IO[str] or None, optional /// :param format: the format of the RDF serialization using a media type like ``text/turtle`` or an extension like `ttl`. If :py:const:`None`, the format is guessed from the file name extension. /// :type format: str or None, optional +/// :param path: The file path to read from. Replaces the ``input`` parameter. +/// :type path: str or os.PathLike[str] or None, optional /// :param base_iri: the base IRI used to resolve the relative IRIs in the file or :py:const:`None` if relative IRI resolution should not be done. /// :type base_iri: str or None, optional /// :param without_named_graphs: Sets that the parser must fail when parsing a named graph. @@ -46,26 +48,21 @@ use std::sync::OnceLock; /// :raises SyntaxError: if the provided data is invalid. /// :raises OSError: if a system error happens while reading the file. /// -/// >>> input = io.BytesIO(b'

"1" .') -/// >>> list(parse(input, "text/turtle", base_iri="http://example.com/")) +/// >>> list(parse(input=b'

"1" .', format="text/turtle", base_iri="http://example.com/")) /// [ predicate= object=> graph_name=>] #[pyfunction] -#[pyo3(signature = (input, /, format = None, *, base_iri = None, without_named_graphs = false, rename_blank_nodes = false))] +#[pyo3(signature = (input = None, format = None, *, path = None, base_iri = None, without_named_graphs = false, rename_blank_nodes = false))] pub fn parse( - input: &PyAny, + input: Option, format: Option<&str>, + path: Option, base_iri: Option<&str>, without_named_graphs: bool, rename_blank_nodes: bool, py: Python<'_>, ) -> PyResult { - let file_path = input.extract::().ok(); - let format = parse_format(format, file_path.as_deref())?; - let input = if let Some(file_path) = &file_path { - PyReadable::from_file(file_path, py)? - } else { - PyReadable::from_data(input) - }; + let input = PyReadable::from_args(&path, input, py)?; + let format = parse_format(format, path.as_deref())?; let mut parser = RdfParser::from_format(format); if let Some(base_iri) = base_iri { parser = parser @@ -80,7 +77,7 @@ pub fn parse( } Ok(PyQuadReader { inner: parser.parse_read(input), - file_path, + file_path: path, } .into_py(py)) } @@ -120,7 +117,7 @@ pub fn parse( /// >>> output.getvalue() /// b' "1" .\n' #[pyfunction] -#[pyo3(signature = (input, output = None, /, format = None))] +#[pyo3(signature = (input, output = None, format = None))] pub fn serialize<'a>( input: &PyAny, output: Option<&PyAny>, @@ -167,13 +164,12 @@ impl PyQuadReader { fn __next__(&mut self, py: Python<'_>) -> PyResult> { py.allow_threads(|| { - self.inner + Ok(self + .inner .next() - .map(|q| { - Ok(q.map_err(|e| map_parse_error(e, self.file_path.clone()))? - .into()) - }) .transpose() + .map_err(|e| map_parse_error(e, self.file_path.clone()))? + .map(PyQuad::from)) }) } } @@ -185,6 +181,22 @@ pub enum PyReadable { } impl PyReadable { + pub fn from_args( + path: &Option, + input: Option, + py: Python<'_>, + ) -> PyResult { + match (path, input) { + (Some(_), Some(_)) => Err(PyValueError::new_err( + "input and file_path can't be both set at the same time", + )), + (Some(path), None) => Ok(PyReadable::from_file(path, py)?), + (None, Some(input)) => Ok(input.into()), + (None, None) => Err(PyValueError::new_err( + "Either input or file_path must be set", + )), + } + } pub fn from_file(file: &Path, py: Python<'_>) -> io::Result { Ok(Self::File(py.allow_threads(|| File::open(file))?)) } @@ -210,6 +222,23 @@ impl Read for PyReadable { } } +#[derive(FromPyObject)] +pub enum PyReadableInput { + String(String), + Bytes(Vec), + Io(PyObject), +} + +impl From for PyReadable { + fn from(input: PyReadableInput) -> Self { + match input { + PyReadableInput::String(string) => Self::Bytes(Cursor::new(string.into_bytes())), + PyReadableInput::Bytes(bytes) => Self::Bytes(Cursor::new(bytes)), + PyReadableInput::Io(io) => Self::Io(PyIo(io)), + } + } +} + pub enum PyWritable { Bytes(Vec), Io(PyIo), diff --git a/python/src/store.rs b/python/src/store.rs index 54fad079..4340d03e 100644 --- a/python/src/store.rs +++ b/python/src/store.rs @@ -1,6 +1,8 @@ #![allow(clippy::needless_option_as_deref)] -use crate::io::{allow_threads_unsafe, map_parse_error, parse_format, PyReadable, PyWritable}; +use crate::io::{ + allow_threads_unsafe, map_parse_error, parse_format, PyReadable, PyReadableInput, PyWritable, +}; use crate::model::*; use crate::sparql::*; use oxigraph::io::RdfFormat; @@ -360,10 +362,12 @@ impl PyStore { /// For example, ``application/turtle`` could also be used for `Turtle `_ /// and ``application/xml`` or ``xml`` for `RDF/XML `_. /// - /// :param input: The I/O object or file path to read from. For example, it could be a file path as a string or a file reader opened in binary mode with ``open('my_file.ttl', 'rb')``. - /// :type input: typing.IO[bytes] or typing.IO[str] or str or os.PathLike[str] - /// :param format: the format of the RDF serialization using a media type like ``text/turtle`` or an extension like `ttl`. If :py:const:`None`, the format is guessed from the file name extension. + /// :param input: The :py:class:`str`, :py:class:`bytes` or I/O object to read from. For example, it could be the file content as a string or a file reader opened in binary mode with ``open('my_file.ttl', 'rb')``. + /// :type input: bytes or str or typing.IO[bytes] or typing.IO[str] or None, optional + /// :param format: the format of the RDF serialization using a media type like ``text/turtle`` or an extension like `ttl`. If :py:const:`None`, the format is guessed from the file name extension. /// :type format: str or None, optional + /// :param path: The file path to read from. Replaces the ``input`` parameter. + /// :type path: str or os.PathLike[str] or None, optional /// :param base_iri: the base IRI used to resolve the relative IRIs in the file or :py:const:`None` if relative IRI resolution should not be done. /// :type base_iri: str or None, optional /// :param to_graph: if it is a file composed of triples, the graph in which the triples should be stored. By default, the default graph is used. @@ -374,14 +378,15 @@ impl PyStore { /// :raises OSError: if an error happens during a quad insertion or if a system error happens while reading the file. /// /// >>> store = Store() - /// >>> store.load(io.BytesIO(b'

"1" .'), "text/turtle", base_iri="http://example.com/", to_graph=NamedNode("http://example.com/g")) + /// >>> store.load(input='

"1" .', format="text/turtle", base_iri="http://example.com/", to_graph=NamedNode("http://example.com/g")) /// >>> list(store) /// [ predicate= object=> graph_name=>] - #[pyo3(signature = (input, /, format = None, *, base_iri = None, to_graph = None))] + #[pyo3(signature = (input = None, format = None, *, path = None, base_iri = None, to_graph = None))] fn load( &self, - input: &PyAny, + input: Option, format: Option<&str>, + path: Option, base_iri: Option<&str>, to_graph: Option<&PyAny>, py: Python<'_>, @@ -391,13 +396,8 @@ impl PyStore { } else { None }; - let file_path = input.extract::().ok(); - let format = parse_format::(format, file_path.as_deref())?; - let input = if let Some(file_path) = &file_path { - PyReadable::from_file(file_path, py)? - } else { - PyReadable::from_data(input) - }; + let input = PyReadable::from_args(&path, input, py)?; + let format: RdfFormat = parse_format(format, path.as_deref())?; py.allow_threads(|| { if let Some(to_graph_name) = to_graph_name { self.inner @@ -405,7 +405,7 @@ impl PyStore { } else { self.inner.load_dataset(input, format, base_iri) } - .map_err(|e| map_loader_error(e, file_path)) + .map_err(|e| map_loader_error(e, path)) }) } @@ -429,10 +429,12 @@ impl PyStore { /// For example, ``application/turtle`` could also be used for `Turtle `_ /// and ``application/xml`` or ``xml`` for `RDF/XML `_. /// - /// :param input: The I/O object or file path to read from. For example, it could be a file path as a string or a file reader opened in binary mode with ``open('my_file.ttl', 'rb')``. - /// :type input: typing.IO[bytes] or typing.IO[str] or str or os.PathLike[str] - /// :param format: the format of the RDF serialization using a media type like ``text/turtle`` or an extension like `ttl`. If :py:const:`None`, the format is guessed from the file name extension. + /// :param input: The :py:class:`str`, :py:class:`bytes` or I/O object to read from. For example, it could be the file content as a string or a file reader opened in binary mode with ``open('my_file.ttl', 'rb')``. + /// :type input: bytes or str or typing.IO[bytes] or typing.IO[str] or None, optional + /// :param format: the format of the RDF serialization using a media type like ``text/turtle`` or an extension like `ttl`. If :py:const:`None`, the format is guessed from the file name extension. /// :type format: str or None, optional + /// :param path: The file path to read from. Replaces the ``input`` parameter. + /// :type path: str or os.PathLike[str] or None, optional /// :param base_iri: the base IRI used to resolve the relative IRIs in the file or :py:const:`None` if relative IRI resolution should not be done. /// :type base_iri: str or None, optional /// :param to_graph: if it is a file composed of triples, the graph in which the triples should be stored. By default, the default graph is used. @@ -443,14 +445,15 @@ impl PyStore { /// :raises OSError: if an error happens during a quad insertion or if a system error happens while reading the file. /// /// >>> store = Store() - /// >>> store.bulk_load(io.BytesIO(b'

"1" .'), "text/turtle", base_iri="http://example.com/", to_graph=NamedNode("http://example.com/g")) + /// >>> store.bulk_load(input=b'

"1" .', format="text/turtle", base_iri="http://example.com/", to_graph=NamedNode("http://example.com/g")) /// >>> list(store) /// [ predicate= object=> graph_name=>] - #[pyo3(signature = (input, /, format = None, *, base_iri = None, to_graph = None))] + #[pyo3(signature = (input = None, format = None, *, path = None, base_iri = None, to_graph = None))] fn bulk_load( &self, - input: &PyAny, + input: Option, format: Option<&str>, + path: Option, base_iri: Option<&str>, to_graph: Option<&PyAny>, py: Python<'_>, @@ -460,13 +463,8 @@ impl PyStore { } else { None }; - let file_path = input.extract::().ok(); - let format = parse_format::(format, file_path.as_deref())?; - let input = if let Some(file_path) = &file_path { - PyReadable::from_file(file_path, py)? - } else { - PyReadable::from_data(input) - }; + let input = PyReadable::from_args(&path, input, py)?; + let format: RdfFormat = parse_format(format, path.as_deref())?; py.allow_threads(|| { if let Some(to_graph_name) = to_graph_name { self.inner @@ -477,7 +475,7 @@ impl PyStore { .bulk_loader() .load_dataset(input, format, base_iri) } - .map_err(|e| map_loader_error(e, file_path)) + .map_err(|e| map_loader_error(e, path)) }) } diff --git a/python/tests/test_io.py b/python/tests/test_io.py index 65223049..596c6fd3 100644 --- a/python/tests/test_io.py +++ b/python/tests/test_io.py @@ -33,13 +33,37 @@ class TestParse(unittest.TestCase): fp.write('

"éù" .'.encode()) fp.flush() self.assertEqual( - list(parse(fp.name, base_iri="http://example.com/")), + list(parse(path=fp.name, base_iri="http://example.com/")), [EXAMPLE_TRIPLE], ) def test_parse_not_existing_file(self) -> None: with self.assertRaises(IOError) as _: - parse("/tmp/not-existing-oxigraph-file.ttl", "text/turtle") + parse(path="/tmp/not-existing-oxigraph-file.ttl", format="text/turtle") + + def test_parse_str(self) -> None: + self.assertEqual( + list( + parse( + '

"éù" .', + "text/turtle", + base_iri="http://example.com/", + ) + ), + [EXAMPLE_TRIPLE], + ) + + def test_parse_bytes(self) -> None: + self.assertEqual( + list( + parse( + '

"éù" .'.encode(), + "text/turtle", + base_iri="http://example.com/", + ) + ), + [EXAMPLE_TRIPLE], + ) def test_parse_str_io(self) -> None: self.assertEqual( @@ -85,7 +109,7 @@ class TestParse(unittest.TestCase): self.assertEqual( list( parse( - StringIO(' {

"1" }'), + ' {

"1" }', "application/trig", base_iri="http://example.com/", ) @@ -99,7 +123,7 @@ class TestParse(unittest.TestCase): fp.write(b' "p" "1"') fp.flush() with self.assertRaises(SyntaxError) as ctx: - list(parse(fp.name, "text/turtle")) + list(parse(path=fp.name, format="text/turtle")) self.assertEqual(ctx.exception.filename, fp.name) self.assertEqual(ctx.exception.lineno, 2) self.assertEqual(ctx.exception.offset, 7) @@ -111,7 +135,7 @@ class TestParse(unittest.TestCase): with self.assertRaises(SyntaxError) as _: list( parse( - StringIO(' {

"1" }'), + ' {

"1" }', "application/trig", base_iri="http://example.com/", without_named_graphs=True, @@ -122,14 +146,14 @@ class TestParse(unittest.TestCase): self.assertNotEqual( list( parse( - StringIO('_:s "o" .'), + '_:s "o" .', "application/n-triples", rename_blank_nodes=True, ) ), list( parse( - StringIO('_:s "o" .'), + '_:s "o" .', "application/n-triples", rename_blank_nodes=True, ) diff --git a/python/tests/test_store.py b/python/tests/test_store.py index b9fc1be8..001755fd 100644 --- a/python/tests/test_store.py +++ b/python/tests/test_store.py @@ -1,5 +1,5 @@ import unittest -from io import BytesIO, UnsupportedOperation +from io import BytesIO, StringIO, UnsupportedOperation from pathlib import Path from tempfile import NamedTemporaryFile, TemporaryDirectory, TemporaryFile from typing import Any @@ -253,7 +253,7 @@ class TestStore(unittest.TestCase): def test_load_ntriples_to_default_graph(self) -> None: store = Store() store.load( - BytesIO(b" ."), + b" .", "application/n-triples", ) self.assertEqual(set(store), {Quad(foo, bar, baz, DefaultGraph())}) @@ -261,7 +261,7 @@ class TestStore(unittest.TestCase): def test_load_ntriples_to_named_graph(self) -> None: store = Store() store.load( - BytesIO(b" ."), + " .", "application/n-triples", to_graph=graph, ) @@ -279,7 +279,7 @@ class TestStore(unittest.TestCase): def test_load_nquads(self) -> None: store = Store() store.load( - BytesIO(b" ."), + StringIO(" ."), "nq", ) self.assertEqual(set(store), {Quad(foo, bar, baz, graph)}) @@ -287,7 +287,7 @@ class TestStore(unittest.TestCase): def test_load_trig_with_base_iri(self) -> None: store = Store() store.load( - BytesIO(b" { <> . }"), + " { <> . }", "application/trig", base_iri="http://baz", ) @@ -298,7 +298,7 @@ class TestStore(unittest.TestCase): fp.write(b" .") fp.flush() store = Store() - store.load(fp.name) + store.load(path=fp.name) self.assertEqual(set(store), {Quad(foo, bar, baz, graph)}) def test_load_with_io_error(self) -> None: From e6d98445e600bb6ff7b33106d1363987ba3e6eb4 Mon Sep 17 00:00:00 2001 From: Tpt Date: Fri, 17 Nov 2023 08:52:26 +0100 Subject: [PATCH 118/217] Makes newer Clippy happy --- lib/oxrdf/src/dataset.rs | 1 + lib/oxrdfio/src/parser.rs | 2 +- lib/oxsdatatypes/src/date_time.rs | 75 ++++++++++++++++--------------- lib/oxttl/src/lexer.rs | 4 +- lib/src/sparql/eval.rs | 22 ++++----- lib/src/storage/backend/mod.rs | 4 +- 6 files changed, 54 insertions(+), 54 deletions(-) diff --git a/lib/oxrdf/src/dataset.rs b/lib/oxrdf/src/dataset.rs index eb655dea..f240a33a 100644 --- a/lib/oxrdf/src/dataset.rs +++ b/lib/oxrdf/src/dataset.rs @@ -185,6 +185,7 @@ impl Dataset { .map(move |q| self.decode_spog(q)) } + #[allow(clippy::map_identity)] fn interned_quads_for_subject( &self, subject: &InternedSubject, diff --git a/lib/oxrdfio/src/parser.rs b/lib/oxrdfio/src/parser.rs index bc88cfc4..6258ba18 100644 --- a/lib/oxrdfio/src/parser.rs +++ b/lib/oxrdfio/src/parser.rs @@ -1,6 +1,6 @@ //! Utilities to read RDF graphs and datasets. -pub use crate::error::{ParseError, SyntaxError}; +pub use crate::error::ParseError; use crate::format::RdfFormat; use oxrdf::{BlankNode, GraphName, IriParseError, Quad, Subject, Term, Triple}; #[cfg(feature = "async-tokio")] diff --git a/lib/oxsdatatypes/src/date_time.rs b/lib/oxsdatatypes/src/date_time.rs index f7c80431..c7a1c16a 100644 --- a/lib/oxsdatatypes/src/date_time.rs +++ b/lib/oxsdatatypes/src/date_time.rs @@ -420,19 +420,20 @@ impl Time { #[inline] #[must_use] pub fn checked_add_duration(self, rhs: impl Into) -> Option { - DateTime::new( - 1972, - 12, - 31, - self.hour(), - self.minute(), - self.second(), - self.timezone_offset(), + Some( + DateTime::new( + 1972, + 12, + 31, + self.hour(), + self.minute(), + self.second(), + self.timezone_offset(), + ) + .ok()? + .checked_add_duration(rhs)? + .into(), ) - .ok()? - .checked_add_duration(rhs)? - .try_into() - .ok() } /// [op:subtract-dayTimeDuration-from-time](https://www.w3.org/TR/xpath-functions-31/#func-subtract-dayTimeDuration-from-time) @@ -450,38 +451,40 @@ impl Time { #[inline] #[must_use] pub fn checked_sub_duration(self, rhs: impl Into) -> Option { - DateTime::new( - 1972, - 12, - 31, - self.hour(), - self.minute(), - self.second(), - self.timezone_offset(), + Some( + DateTime::new( + 1972, + 12, + 31, + self.hour(), + self.minute(), + self.second(), + self.timezone_offset(), + ) + .ok()? + .checked_sub_duration(rhs)? + .into(), ) - .ok()? - .checked_sub_duration(rhs)? - .try_into() - .ok() } // [fn:adjust-time-to-timezone](https://www.w3.org/TR/xpath-functions-31/#func-adjust-time-to-timezone) #[inline] #[must_use] pub fn adjust(self, timezone_offset: Option) -> Option { - DateTime::new( - 1972, - 12, - 31, - self.hour(), - self.minute(), - self.second(), - self.timezone_offset(), + Some( + DateTime::new( + 1972, + 12, + 31, + self.hour(), + self.minute(), + self.second(), + self.timezone_offset(), + ) + .ok()? + .adjust(timezone_offset)? + .into(), ) - .ok()? - .adjust(timezone_offset)? - .try_into() - .ok() } /// Checks if the two values are [identical](https://www.w3.org/TR/xmlschema11-2/#identity). diff --git a/lib/oxttl/src/lexer.rs b/lib/oxttl/src/lexer.rs index 65dba56e..6dd1d024 100644 --- a/lib/oxttl/src/lexer.rs +++ b/lib/oxttl/src/lexer.rs @@ -350,9 +350,7 @@ impl N3Lexer { let a = char::from(*data.get(i)?); i += 1; let b = char::from(*data.get(i)?); - if !matches!(a, '0'..='9' | 'A'..='F' | 'a'..='f') - || !matches!(b, '0'..='9' | 'A'..='F' | 'a'..='f') - { + if !a.is_ascii_hexdigit() || !b.is_ascii_hexdigit() { return Some((i + 1, Err(( i - 2..=i, format!("escapes in IRIs should be % followed by two hexadecimal characters, found '%{a}{b}'") ).into()))); diff --git a/lib/src/sparql/eval.rs b/lib/src/sparql/eval.rs index 6f52228d..2997b83d 100644 --- a/lib/src/sparql/eval.rs +++ b/lib/src/sparql/eval.rs @@ -1575,7 +1575,7 @@ impl SimpleEvaluator { } }) } - Function::BNode => match parameters.get(0) { + Function::BNode => match parameters.first() { Some(id) => { let id = self.expression_evaluator(id, encoded_variables, stat_children); @@ -2612,7 +2612,7 @@ impl SimpleEvaluator { Some(Decimal::try_from(value).ok()?.into()) } EncodedTerm::IntegerLiteral(value) => { - Some(Decimal::try_from(value).ok()?.into()) + Some(Decimal::from(value).into()) } EncodedTerm::DecimalLiteral(value) => Some(value.into()), EncodedTerm::BooleanLiteral(value) => { @@ -2658,7 +2658,7 @@ impl SimpleEvaluator { Rc::new(move |tuple| match e(tuple)? { EncodedTerm::TimeLiteral(value) => Some(value.into()), EncodedTerm::DateTimeLiteral(value) => { - Some(Time::try_from(value).ok()?.into()) + Some(Time::from(value).into()) } EncodedTerm::SmallStringLiteral(value) => { parse_time_str(&value) @@ -3250,7 +3250,7 @@ fn equals(a: &EncodedTerm, b: &EncodedTerm) -> Option { EncodedTerm::FloatLiteral(b) => Some(a == b), EncodedTerm::DoubleLiteral(b) => Some(Double::from(*a) == *b), EncodedTerm::IntegerLiteral(b) => Some(*a == Float::from(*b)), - EncodedTerm::DecimalLiteral(b) => Some(*a == (*b).try_into().ok()?), + EncodedTerm::DecimalLiteral(b) => Some(*a == (*b).into()), _ if b.is_unknown_typed_literal() => None, _ => Some(false), }, @@ -3258,7 +3258,7 @@ fn equals(a: &EncodedTerm, b: &EncodedTerm) -> Option { EncodedTerm::FloatLiteral(b) => Some(*a == Double::from(*b)), EncodedTerm::DoubleLiteral(b) => Some(a == b), EncodedTerm::IntegerLiteral(b) => Some(*a == Double::from(*b)), - EncodedTerm::DecimalLiteral(b) => Some(*a == (*b).try_into().ok()?), + EncodedTerm::DecimalLiteral(b) => Some(*a == (*b).into()), _ if b.is_unknown_typed_literal() => None, _ => Some(false), }, @@ -3271,8 +3271,8 @@ fn equals(a: &EncodedTerm, b: &EncodedTerm) -> Option { _ => Some(false), }, EncodedTerm::DecimalLiteral(a) => match b { - EncodedTerm::FloatLiteral(b) => Some(Float::try_from(*a).ok()? == *b), - EncodedTerm::DoubleLiteral(b) => Some(Double::try_from(*a).ok()? == *b), + EncodedTerm::FloatLiteral(b) => Some(Float::from(*a) == *b), + EncodedTerm::DoubleLiteral(b) => Some(Double::from(*a) == *b), EncodedTerm::IntegerLiteral(b) => Some(*a == Decimal::from(*b)), EncodedTerm::DecimalLiteral(b) => Some(a == b), _ if b.is_unknown_typed_literal() => None, @@ -3537,14 +3537,14 @@ fn partial_cmp_literals( EncodedTerm::FloatLiteral(b) => a.partial_cmp(b), EncodedTerm::DoubleLiteral(b) => Double::from(*a).partial_cmp(b), EncodedTerm::IntegerLiteral(b) => a.partial_cmp(&Float::from(*b)), - EncodedTerm::DecimalLiteral(b) => a.partial_cmp(&(*b).try_into().ok()?), + EncodedTerm::DecimalLiteral(b) => a.partial_cmp(&(*b).into()), _ => None, }, EncodedTerm::DoubleLiteral(a) => match b { EncodedTerm::FloatLiteral(b) => a.partial_cmp(&(*b).into()), EncodedTerm::DoubleLiteral(b) => a.partial_cmp(b), EncodedTerm::IntegerLiteral(b) => a.partial_cmp(&Double::from(*b)), - EncodedTerm::DecimalLiteral(b) => a.partial_cmp(&(*b).try_into().ok()?), + EncodedTerm::DecimalLiteral(b) => a.partial_cmp(&(*b).into()), _ => None, }, EncodedTerm::IntegerLiteral(a) => match b { @@ -3555,8 +3555,8 @@ fn partial_cmp_literals( _ => None, }, EncodedTerm::DecimalLiteral(a) => match b { - EncodedTerm::FloatLiteral(b) => Float::try_from(*a).ok()?.partial_cmp(b), - EncodedTerm::DoubleLiteral(b) => Double::try_from(*a).ok()?.partial_cmp(b), + EncodedTerm::FloatLiteral(b) => Float::from(*a).partial_cmp(b), + EncodedTerm::DoubleLiteral(b) => Double::from(*a).partial_cmp(b), EncodedTerm::IntegerLiteral(b) => a.partial_cmp(&Decimal::from(*b)), EncodedTerm::DecimalLiteral(b) => a.partial_cmp(b), _ => None, diff --git a/lib/src/storage/backend/mod.rs b/lib/src/storage/backend/mod.rs index 0fea9616..9261fd56 100644 --- a/lib/src/storage/backend/mod.rs +++ b/lib/src/storage/backend/mod.rs @@ -4,9 +4,7 @@ #[cfg(target_family = "wasm")] pub use fallback::{ColumnFamily, ColumnFamilyDefinition, Db, Iter, Reader, Transaction}; #[cfg(not(target_family = "wasm"))] -pub use rocksdb::{ - ColumnFamily, ColumnFamilyDefinition, Db, Iter, Reader, SstFileWriter, Transaction, -}; +pub use rocksdb::{ColumnFamily, ColumnFamilyDefinition, Db, Iter, Reader, Transaction}; #[cfg(target_family = "wasm")] mod fallback; From cc41448b18b658afe5b9466e8d8a2970ba83b2f3 Mon Sep 17 00:00:00 2001 From: Tpt Date: Thu, 16 Nov 2023 19:50:14 +0100 Subject: [PATCH 119/217] Python: harmonizes parse_query_results and parse signatures --- python/src/io.rs | 15 +--------- python/src/sparql.rs | 63 ++++++++++++++++++++--------------------- python/tests/test_io.py | 20 +++++++++---- 3 files changed, 46 insertions(+), 52 deletions(-) diff --git a/python/src/io.rs b/python/src/io.rs index 44cc48e9..8e24a24d 100644 --- a/python/src/io.rs +++ b/python/src/io.rs @@ -190,26 +190,13 @@ impl PyReadable { (Some(_), Some(_)) => Err(PyValueError::new_err( "input and file_path can't be both set at the same time", )), - (Some(path), None) => Ok(PyReadable::from_file(path, py)?), + (Some(path), None) => Ok(Self::File(py.allow_threads(|| File::open(path))?)), (None, Some(input)) => Ok(input.into()), (None, None) => Err(PyValueError::new_err( "Either input or file_path must be set", )), } } - pub fn from_file(file: &Path, py: Python<'_>) -> io::Result { - Ok(Self::File(py.allow_threads(|| File::open(file))?)) - } - - pub fn from_data(data: &PyAny) -> Self { - if let Ok(bytes) = data.extract::>() { - Self::Bytes(Cursor::new(bytes)) - } else if let Ok(string) = data.extract::() { - Self::Bytes(Cursor::new(string.into_bytes())) - } else { - Self::Io(PyIo(data.into())) - } - } } impl Read for PyReadable { diff --git a/python/src/sparql.rs b/python/src/sparql.rs index 207a4818..863bda9b 100644 --- a/python/src/sparql.rs +++ b/python/src/sparql.rs @@ -12,7 +12,7 @@ use oxigraph::sparql::{ Variable, }; use pyo3::basic::CompareOp; -use pyo3::exceptions::{PyRuntimeError, PySyntaxError, PyTypeError, PyValueError}; +use pyo3::exceptions::{PyRuntimeError, PySyntaxError, PyValueError}; use pyo3::prelude::*; use pyo3::types::PyBytes; use std::io; @@ -132,22 +132,13 @@ impl PyQuerySolution { self.inner.len() } - fn __getitem__(&self, input: &PyAny) -> PyResult> { - if let Ok(key) = usize::extract(input) { - Ok(self.inner.get(key).map(|term| PyTerm::from(term.clone()))) - } else if let Ok(key) = <&str>::extract(input) { - Ok(self.inner.get(key).map(|term| PyTerm::from(term.clone()))) - } else if let Ok(key) = input.extract::>() { - Ok(self - .inner - .get(<&Variable>::from(&*key)) - .map(|term| PyTerm::from(term.clone()))) - } else { - Err(PyTypeError::new_err(format!( - "{} is not an integer of a string", - input.get_type().name()?, - ))) + fn __getitem__(&self, key: PySolutionKey<'_>) -> Option { + match key { + PySolutionKey::Usize(key) => self.inner.get(key), + PySolutionKey::Str(key) => self.inner.get(key), + PySolutionKey::Variable(key) => self.inner.get(<&Variable>::from(&*key)), } + .map(|term| PyTerm::from(term.clone())) } #[allow(clippy::unnecessary_to_owned)] @@ -158,6 +149,13 @@ impl PyQuerySolution { } } +#[derive(FromPyObject)] +pub enum PySolutionKey<'a> { + Usize(usize), + Str(&'a str), + Variable(PyRef<'a, PyVariable>), +} + #[pyclass(module = "pyoxigraph")] pub struct SolutionValueIter { inner: IntoIter>, @@ -460,43 +458,42 @@ impl PyQueryTriples { /// It supports also some media type and extension aliases. /// For example, ``application/json`` could also be used for `JSON `_. /// -/// :param input: The I/O object or file path to read from. For example, it could be a file path as a string or a file reader opened in binary mode with ``open('my_file.ttl', 'rb')``. -/// :type input: typing.IO[bytes] or typing.IO[str] or str or os.PathLike[str] +/// :param input: The :py:class:`str`, :py:class:`bytes` or I/O object to read from. For example, it could be the file content as a string or a file reader opened in binary mode with ``open('my_file.ttl', 'rb')``. +/// :type input: bytes or str or typing.IO[bytes] or typing.IO[str] or None, optional /// :param format: the format of the RDF serialization using a media type like ``text/turtle`` or an extension like `ttl`. If :py:const:`None`, the format is guessed from the file name extension. /// :type format: str or None, optional +/// :param path: The file path to read from. Replaces the ``input`` parameter. +/// :type path: str or os.PathLike[str] or None, optional /// :return: an iterator of :py:class:`QuerySolution` or a :py:class:`bool`. /// :rtype: QuerySolutions or QueryBoolean /// :raises ValueError: if the format is not supported. /// :raises SyntaxError: if the provided data is invalid. /// :raises OSError: if a system error happens while reading the file. /// -/// >>> input = io.BytesIO(b'?s\t?p\t?o\n\t\t1\n') -/// >>> list(parse_query_results(input, "text/tsv")) +/// >>> list(parse_query_results('?s\t?p\t?o\n\t\t1\n', "text/tsv")) /// [ p= o=>>] /// -/// >>> input = io.BytesIO(b'{"head":{},"boolean":true}') -/// >>> parse_query_results(input, "application/sparql-results+json") +/// >>> parse_query_results('{"head":{},"boolean":true}', "application/sparql-results+json") /// #[pyfunction] -#[pyo3(signature = (input, /, format = None))] +#[pyo3(signature = (input = None, format = None, *, path = None))] pub fn parse_query_results( - input: &PyAny, + input: Option, format: Option<&str>, + path: Option, py: Python<'_>, ) -> PyResult { - let file_path = input.extract::().ok(); - let format = parse_format(format, file_path.as_deref())?; - let input = if let Some(file_path) = &file_path { - PyReadable::from_file(file_path, py)? - } else { - PyReadable::from_data(input) - }; + let input = PyReadable::from_args(&path, input, py)?; + let format = parse_format(format, path.as_deref())?; let results = QueryResultsParser::from_format(format) .parse_read(input) - .map_err(|e| map_query_results_parse_error(e, file_path.clone()))?; + .map_err(|e| map_query_results_parse_error(e, path.clone()))?; Ok(match results { FromReadQueryResultsReader::Solutions(iter) => PyQuerySolutions { - inner: PyQuerySolutionsVariant::Reader { iter, file_path }, + inner: PyQuerySolutionsVariant::Reader { + iter, + file_path: path, + }, } .into_py(py), FromReadQueryResultsReader::Boolean(inner) => PyQueryBoolean { inner }.into_py(py), diff --git a/python/tests/test_io.py b/python/tests/test_io.py index 596c6fd3..9c8d4047 100644 --- a/python/tests/test_io.py +++ b/python/tests/test_io.py @@ -202,7 +202,7 @@ class TestParseQuerySolutions(unittest.TestCase): with NamedTemporaryFile(suffix=".tsv") as fp: fp.write(b'?s\t?p\t?o\n\t\t"1"\n') fp.flush() - r = parse_query_results(fp.name) + r = parse_query_results(path=fp.name) self.assertIsInstance(r, QuerySolutions) results = list(r) # type: ignore[arg-type] self.assertEqual(results[0]["s"], NamedNode("http://example.com/s")) @@ -210,10 +210,20 @@ class TestParseQuerySolutions(unittest.TestCase): def test_parse_not_existing_file(self) -> None: with self.assertRaises(IOError) as _: - parse_query_results("/tmp/not-existing-oxigraph-file.ttl", "application/json") + parse_query_results(path="/tmp/not-existing-oxigraph-file.ttl", format="application/json") + + def test_parse_str(self) -> None: + result = parse_query_results("true", "tsv") + self.assertIsInstance(result, QueryBoolean) + self.assertTrue(result) + + def test_parse_bytes(self) -> None: + result = parse_query_results(b"false", "tsv") + self.assertIsInstance(result, QueryBoolean) + self.assertFalse(result) def test_parse_str_io(self) -> None: - result = parse_query_results(StringIO("true"), "tsv") + result = parse_query_results("true", "tsv") self.assertIsInstance(result, QueryBoolean) self.assertTrue(result) @@ -231,7 +241,7 @@ class TestParseQuerySolutions(unittest.TestCase): fp.write(b"{]") fp.flush() with self.assertRaises(SyntaxError) as ctx: - list(parse_query_results(fp.name, "srj")) # type: ignore[arg-type] + list(parse_query_results(path=fp.name, format="srj")) # type: ignore[arg-type] self.assertEqual(ctx.exception.filename, fp.name) self.assertEqual(ctx.exception.lineno, 1) self.assertEqual(ctx.exception.offset, 2) @@ -245,7 +255,7 @@ class TestParseQuerySolutions(unittest.TestCase): fp.write(b"1\t\n") fp.flush() with self.assertRaises(SyntaxError) as ctx: - list(parse_query_results(fp.name, "tsv")) # type: ignore[arg-type] + list(parse_query_results(path=fp.name, format="tsv")) # type: ignore[arg-type] self.assertEqual(ctx.exception.filename, fp.name) self.assertEqual(ctx.exception.lineno, 2) self.assertEqual(ctx.exception.offset, 3) From d19947414e4158891e81e1283d5c428505e84fae Mon Sep 17 00:00:00 2001 From: Tpt Date: Thu, 16 Nov 2023 18:49:23 +0100 Subject: [PATCH 120/217] The N-Triples serializer outputs canonical N-Triples --- lib/oxrdfio/src/serializer.rs | 2 +- lib/oxttl/src/ntriples.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/oxrdfio/src/serializer.rs b/lib/oxrdfio/src/serializer.rs index 9bfaceec..347e40f2 100644 --- a/lib/oxrdfio/src/serializer.rs +++ b/lib/oxrdfio/src/serializer.rs @@ -26,7 +26,7 @@ use tokio::io::AsyncWrite; /// It currently supports the following formats: /// * [N3](https://w3c.github.io/N3/spec/) ([`RdfFormat::N3`]) /// * [N-Quads](https://www.w3.org/TR/n-quads/) ([`RdfFormat::NQuads`]) -/// * [N-Triples](https://www.w3.org/TR/n-triples/) ([`RdfFormat::NTriples`]) +/// * [canonical](https://www.w3.org/TR/n-triples/#canonical-ntriples) [N-Triples](https://www.w3.org/TR/n-triples/) ([`RdfFormat::NTriples`]) /// * [RDF/XML](https://www.w3.org/TR/rdf-syntax-grammar/) ([`RdfFormat::RdfXml`]) /// * [TriG](https://www.w3.org/TR/trig/) ([`RdfFormat::TriG`]) /// * [Turtle](https://www.w3.org/TR/turtle/) ([`RdfFormat::Turtle`]) diff --git a/lib/oxttl/src/ntriples.rs b/lib/oxttl/src/ntriples.rs index 4fd06227..674dc27a 100644 --- a/lib/oxttl/src/ntriples.rs +++ b/lib/oxttl/src/ntriples.rs @@ -315,7 +315,7 @@ impl LowLevelNTriplesReader { } } -/// A [N-Triples](https://www.w3.org/TR/n-triples/) serializer. +/// A [canonical](https://www.w3.org/TR/n-triples/#canonical-ntriples) [N-Triples](https://www.w3.org/TR/n-triples/) serializer. /// /// Support for [N-Triples-star](https://w3c.github.io/rdf-star/cg-spec/2021-12-17.html#n-triples-star) is available behind the `rdf-star` feature. /// From ddf589ea14a7a57173ad00d0203a5feb85252878 Mon Sep 17 00:00:00 2001 From: Tpt Date: Fri, 17 Nov 2023 17:55:33 +0100 Subject: [PATCH 121/217] Python: Introduces enums for RDF and SPARQL result formats --- python/docs/io.rst | 11 +- python/docs/model.rst | 13 +- python/docs/sparql.rst | 17 +- python/docs/store.rst | 3 +- python/generate_stubs.py | 11 ++ python/src/io.rs | 336 +++++++++++++++++++++++++++---------- python/src/lib.rs | 2 + python/src/model.rs | 2 +- python/src/sparql.rs | 273 ++++++++++++++++++++++++------ python/src/store.rs | 80 ++++----- python/tests/test_io.py | 50 +++--- python/tests/test_store.py | 40 +++-- 12 files changed, 597 insertions(+), 241 deletions(-) diff --git a/python/docs/io.rst b/python/docs/io.rst index 8be36f43..edf3fba0 100644 --- a/python/docs/io.rst +++ b/python/docs/io.rst @@ -1,14 +1,21 @@ RDF Parsing and Serialization ============================= +.. py:currentmodule:: pyoxigraph Oxigraph provides functions to parse and serialize RDF files: Parsing """"""" -.. autofunction:: pyoxigraph.parse +.. autofunction:: parse Serialization """"""""""""" -.. autofunction:: pyoxigraph.serialize +.. autofunction:: serialize + + +Formats +""""""" +.. autoclass:: RdfFormat + :members: diff --git a/python/docs/model.rst b/python/docs/model.rst index 1950e853..28a7279c 100644 --- a/python/docs/model.rst +++ b/python/docs/model.rst @@ -1,37 +1,38 @@ RDF Model ========= +.. py:currentmodule:: pyoxigraph Oxigraph provides python classes to represents basic RDF concepts: `IRIs `_ """"""""""""""""""""""""""""""""""""""""""""""""""""""" -.. autoclass:: pyoxigraph.NamedNode +.. autoclass:: NamedNode :members: `Blank Nodes `_ """"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" -.. autoclass:: pyoxigraph.BlankNode +.. autoclass:: BlankNode :members: `Literals `_ """"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" -.. autoclass:: pyoxigraph.Literal +.. autoclass:: Literal :members: `Triples `_ """"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" -.. autoclass:: pyoxigraph.Triple +.. autoclass:: Triple :members: Quads (`triples `_ in a `RDF dataset `_) """"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" -.. autoclass:: pyoxigraph.Quad +.. autoclass:: Quad :members: -.. autoclass:: pyoxigraph.DefaultGraph +.. autoclass:: DefaultGraph :members: diff --git a/python/docs/sparql.rst b/python/docs/sparql.rst index 1e87d179..824a42cb 100644 --- a/python/docs/sparql.rst +++ b/python/docs/sparql.rst @@ -1,32 +1,33 @@ SPARQL utility objects -============================= +====================== +.. py:currentmodule:: pyoxigraph Oxigraph provides also some utilities related to SPARQL queries: - Variable """""""" -.. autoclass:: pyoxigraph.Variable +.. autoclass:: Variable :members: ``SELECT`` solutions """""""""""""""""""" -.. autoclass:: pyoxigraph.QuerySolutions +.. autoclass:: QuerySolutions :members: -.. autoclass:: pyoxigraph.QuerySolution +.. autoclass:: QuerySolution :members: ``ASK`` results """"""""""""""" -.. autoclass:: pyoxigraph.QueryBoolean +.. autoclass:: QueryBoolean :members: ``CONSTRUCT`` results """"""""""""""""""""" -.. autoclass:: pyoxigraph.QueryTriples +.. autoclass:: QueryTriples :members: Query results parsing """"""""""""""""""""" -.. autoclass:: pyoxigraph.parse_query_results +.. autofunction:: parse_query_results +.. autoclass:: QueryResultsFormat :members: diff --git a/python/docs/store.rst b/python/docs/store.rst index 2fea9ab3..b6af6ef5 100644 --- a/python/docs/store.rst +++ b/python/docs/store.rst @@ -1,5 +1,6 @@ RDF Store ========= +.. py:currentmodule:: pyoxigraph -.. autoclass:: pyoxigraph.Store +.. autoclass:: Store :members: diff --git a/python/generate_stubs.py b/python/generate_stubs.py index 7d0469ec..5626c8de 100644 --- a/python/generate_stubs.py +++ b/python/generate_stubs.py @@ -141,6 +141,17 @@ def class_stubs(cls_name: str, cls_def: Any, element_path: List[str], types_to_i simple=1, ) ) + elif member_value is not None: + constants.append( + ast.AnnAssign( + target=ast.Name(id=member_name, ctx=ast.Store()), + annotation=concatenated_path_to_type( + member_value.__class__.__name__, element_path, types_to_import + ), + value=ast.Ellipsis(), + simple=1, + ) + ) else: logging.warning(f"Unsupported member {member_name} of class {'.'.join(element_path)}") diff --git a/python/src/io.rs b/python/src/io.rs index 8e24a24d..a87b7982 100644 --- a/python/src/io.rs +++ b/python/src/io.rs @@ -1,9 +1,8 @@ #![allow(clippy::needless_option_as_deref)] -use crate::model::{PyQuad, PyTriple}; +use crate::model::{hash, PyQuad, PyTriple}; use oxigraph::io::{FromReadQuadReader, ParseError, RdfFormat, RdfParser, RdfSerializer}; use oxigraph::model::QuadRef; -use oxigraph::sparql::results::QueryResultsFormat; use pyo3::exceptions::{PySyntaxError, PyValueError}; use pyo3::intern; use pyo3::prelude::*; @@ -19,12 +18,12 @@ use std::sync::OnceLock; /// /// It currently supports the following formats: /// -/// * `N-Triples `_ (``application/n-triples`` or ``nt``) -/// * `N-Quads `_ (``application/n-quads`` or ``nq``) -/// * `Turtle `_ (``text/turtle`` or ``ttl``) -/// * `TriG `_ (``application/trig`` or ``trig``) -/// * `N3 `_ (``text/n3`` or ``n3``) -/// * `RDF/XML `_ (``application/rdf+xml`` or ``rdf``) +/// * `N-Triples `_ (:py:attr:`RdfFormat.N_TRIPLES`) +/// * `N-Quads `_ (:py:attr:`RdfFormat.N_QUADS`) +/// * `Turtle `_ (:py:attr:`RdfFormat.TURTLE`) +/// * `TriG `_ (:py:attr:`RdfFormat.TRIG`) +/// * `N3 `_ (:py:attr:`RdfFormat.N3`) +/// * `RDF/XML `_ (:py:attr:`RdfFormat.RDF_XML`) /// /// It supports also some media type and extension aliases. /// For example, ``application/turtle`` could also be used for `Turtle `_ @@ -32,8 +31,8 @@ use std::sync::OnceLock; /// /// :param input: The :py:class:`str`, :py:class:`bytes` or I/O object to read from. For example, it could be the file content as a string or a file reader opened in binary mode with ``open('my_file.ttl', 'rb')``. /// :type input: bytes or str or typing.IO[bytes] or typing.IO[str] or None, optional -/// :param format: the format of the RDF serialization using a media type like ``text/turtle`` or an extension like `ttl`. If :py:const:`None`, the format is guessed from the file name extension. -/// :type format: str or None, optional +/// :param format: the format of the RDF serialization. If :py:const:`None`, the format is guessed from the file name extension. +/// :type format: RdfFormat or None, optional /// :param path: The file path to read from. Replaces the ``input`` parameter. /// :type path: str or os.PathLike[str] or None, optional /// :param base_iri: the base IRI used to resolve the relative IRIs in the file or :py:const:`None` if relative IRI resolution should not be done. @@ -48,13 +47,13 @@ use std::sync::OnceLock; /// :raises SyntaxError: if the provided data is invalid. /// :raises OSError: if a system error happens while reading the file. /// -/// >>> list(parse(input=b'

"1" .', format="text/turtle", base_iri="http://example.com/")) +/// >>> list(parse(input=b'

"1" .', format=RdfFormat.TURTLE, base_iri="http://example.com/")) /// [ predicate= object=> graph_name=>] #[pyfunction] #[pyo3(signature = (input = None, format = None, *, path = None, base_iri = None, without_named_graphs = false, rename_blank_nodes = false))] pub fn parse( input: Option, - format: Option<&str>, + format: Option, path: Option, base_iri: Option<&str>, without_named_graphs: bool, @@ -62,7 +61,7 @@ pub fn parse( py: Python<'_>, ) -> PyResult { let input = PyReadable::from_args(&path, input, py)?; - let format = parse_format(format, path.as_deref())?; + let format = lookup_rdf_format(format, path.as_deref())?; let mut parser = RdfParser::from_format(format); if let Some(base_iri) = base_iri { parser = parser @@ -86,12 +85,12 @@ pub fn parse( /// /// It currently supports the following formats: /// -/// * `N-Triples `_ (``application/n-triples`` or ``nt``) -/// * `N-Quads `_ (``application/n-quads`` or ``nq``) -/// * `Turtle `_ (``text/turtle`` or ``ttl``) -/// * `TriG `_ (``application/trig`` or ``trig``) -/// * `N3 `_ (``text/n3`` or ``n3``) -/// * `RDF/XML `_ (``application/rdf+xml`` or ``rdf``) +/// * `canonical `_ `N-Triples `_ (:py:attr:`RdfFormat.N_TRIPLES`) +/// * `N-Quads `_ (:py:attr:`RdfFormat.N_QUADS`) +/// * `Turtle `_ (:py:attr:`RdfFormat.TURTLE`) +/// * `TriG `_ (:py:attr:`RdfFormat.TRIG`) +/// * `N3 `_ (:py:attr:`RdfFormat.N3`) +/// * `RDF/XML `_ (:py:attr:`RdfFormat.RDF_XML`) /// /// It supports also some media type and extension aliases. /// For example, ``application/turtle`` could also be used for `Turtle `_ @@ -101,31 +100,32 @@ pub fn parse( /// :type input: collections.abc.Iterable[Triple] or collections.abc.Iterable[Quad] /// :param output: The binary I/O object or file path to write to. For example, it could be a file path as a string or a file writer opened in binary mode with ``open('my_file.ttl', 'wb')``. If :py:const:`None`, a :py:class:`bytes` buffer is returned with the serialized content. /// :type output: typing.IO[bytes] or str or os.PathLike[str] or None, optional -/// :param format: the format of the RDF serialization using a media type like ``text/turtle`` or an extension like `ttl`. If :py:const:`None`, the format is guessed from the file name extension. -/// :type format: str or None, optional -/// :return: py:class:`bytes` with the serialization if the ``output`` parameter is :py:const:`None`, :py:const:`None` if ``output`` is set. +/// :param format: the format of the RDF serialization. If :py:const:`None`, the format is guessed from the file name extension. +/// :type format: RdfFormat or None, optional +/// :return: :py:class:`bytes` with the serialization if the ``output`` parameter is :py:const:`None`, :py:const:`None` if ``output`` is set. /// :rtype: bytes or None /// :raises ValueError: if the format is not supported. /// :raises TypeError: if a triple is given during a quad format serialization or reverse. /// :raises OSError: if a system error happens while writing the file. /// -/// >>> serialize([Triple(NamedNode('http://example.com'), NamedNode('http://example.com/p'), Literal('1'))], format="ttl") +/// >>> serialize([Triple(NamedNode('http://example.com'), NamedNode('http://example.com/p'), Literal('1'))], format=RdfFormat.TURTLE) /// b' "1" .\n' /// /// >>> output = io.BytesIO() -/// >>> serialize([Triple(NamedNode('http://example.com'), NamedNode('http://example.com/p'), Literal('1'))], output, "text/turtle") +/// >>> serialize([Triple(NamedNode('http://example.com'), NamedNode('http://example.com/p'), Literal('1'))], output, RdfFormat.TURTLE) /// >>> output.getvalue() /// b' "1" .\n' #[pyfunction] #[pyo3(signature = (input, output = None, format = None))] pub fn serialize<'a>( input: &PyAny, - output: Option<&PyAny>, - format: Option<&str>, + output: Option, + format: Option, py: Python<'a>, ) -> PyResult> { PyWritable::do_write( - |output, format| { + |output, file_path| { + let format = lookup_rdf_format(format, file_path.as_deref())?; let mut writer = RdfSerializer::from_format(format).serialize_to_write(output); for i in input.iter()? { let i = i?; @@ -145,7 +145,6 @@ pub fn serialize<'a>( Ok(writer.finish()?) }, output, - format, py, ) } @@ -174,6 +173,193 @@ impl PyQuadReader { } } +/// RDF serialization formats. +/// +/// The following formats are supported: +/// * `N-Triples `_ (:py:attr:`RdfFormat.N_TRIPLES`) +/// * `N-Quads `_ (:py:attr:`RdfFormat.N_QUADS`) +/// * `Turtle `_ (:py:attr:`RdfFormat.TURTLE`) +/// * `TriG `_ (:py:attr:`RdfFormat.TRIG`) +/// * `N3 `_ (:py:attr:`RdfFormat.N3`) +/// * `RDF/XML `_ (:py:attr:`RdfFormat.RDF_XML`) +#[pyclass(name = "RdfFormat", module = "pyoxigraph")] +#[derive(Clone)] +pub struct PyRdfFormat { + inner: RdfFormat, +} + +#[pymethods] +impl PyRdfFormat { + /// `N3 `_ + #[classattr] + const N3: Self = Self { + inner: RdfFormat::N3, + }; + + /// `N-Quads `_ + #[classattr] + const N_QUADS: Self = Self { + inner: RdfFormat::NQuads, + }; + + /// `N-Triples `_ + #[classattr] + const N_TRIPLES: Self = Self { + inner: RdfFormat::NTriples, + }; + + /// `RDF/XML `_ + #[classattr] + const RDF_XML: Self = Self { + inner: RdfFormat::RdfXml, + }; + + /// `TriG `_ + #[classattr] + const TRIG: Self = Self { + inner: RdfFormat::TriG, + }; + + /// `Turtle `_ + #[classattr] + const TURTLE: Self = Self { + inner: RdfFormat::Turtle, + }; + + /// :return: the format canonical IRI according to the `Unique URIs for file formats registry `_. + /// :rtype: str + /// + /// >>> RdfFormat.N_TRIPLES.iri + /// 'http://www.w3.org/ns/formats/N-Triples' + #[getter] + fn iri(&self) -> &'static str { + self.inner.iri() + } + + /// :return: the format `IANA media type `_. + /// :rtype: str + /// + /// >>> RdfFormat.N_TRIPLES.media_type + /// 'application/n-triples' + #[getter] + fn media_type(&self) -> &'static str { + self.inner.media_type() + } + + /// :return: the format `IANA-registered `_ file extension. + /// :rtype: str + /// + /// >>> RdfFormat.N_TRIPLES.file_extension + /// 'nt' + #[getter] + pub fn file_extension(&self) -> &'static str { + self.inner.file_extension() + } + + /// :return: the format name. + /// :rtype: str + /// + /// >>> RdfFormat.N_TRIPLES.name + /// 'N-Triples' + #[getter] + pub const fn name(&self) -> &'static str { + self.inner.name() + } + + /// :return: if the formats supports `RDF datasets `_ and not only `RDF graphs `_. + /// :rtype: bool + /// + /// >>> RdfFormat.N_TRIPLES.supports_datasets + /// False + /// >>> RdfFormat.N_QUADS.supports_datasets + /// True + #[getter] + pub fn supports_datasets(&self) -> bool { + self.inner.supports_datasets() + } + + /// :return: if the formats supports `RDF-star quoted triples `_. + /// :rtype: bool + /// + /// >>> RdfFormat.N_TRIPLES.supports_rdf_star + /// True + /// >>> RdfFormat.RDF_XML.supports_rdf_star + /// False + #[getter] + pub const fn supports_rdf_star(&self) -> bool { + self.inner.supports_rdf_star() + } + + /// Looks for a known format from a media type. + /// + /// It supports some media type aliases. + /// For example, "application/xml" is going to return RDF/XML even if it is not its canonical media type. + /// + /// :param media_type: the media type. + /// :type media_type: str + /// :return: :py:class:`RdfFormat` if the media type is known or :py:const:`None` if not. + /// :rtype: RdfFormat or None + /// + /// >>> RdfFormat.from_media_type("text/turtle; charset=utf-8") + /// + #[staticmethod] + pub fn from_media_type(media_type: &str) -> Option { + Some(Self { + inner: RdfFormat::from_media_type(media_type)?, + }) + } + + /// Looks for a known format from an extension. + /// + /// It supports some aliases. + /// + /// :param extension: the extension. + /// :type extension: str + /// :return: :py:class:`RdfFormat` if the extension is known or :py:const:`None` if not. + /// :rtype: RdfFormat or None + /// + /// >>> RdfFormat.from_extension("nt") + /// + #[staticmethod] + pub fn from_extension(extension: &str) -> Option { + Some(Self { + inner: RdfFormat::from_extension(extension)?, + }) + } + + fn __str__(&self) -> &'static str { + self.inner.name() + } + + fn __repr__(&self) -> String { + format!("", self.inner.name()) + } + + fn __hash__(&self) -> u64 { + hash(&self.inner) + } + + fn __eq__(&self, other: &Self) -> bool { + self.inner == other.inner + } + + fn __ne__(&self, other: &Self) -> bool { + self.inner != other.inner + } + + /// :rtype: RdfFormat + fn __copy__(slf: PyRef<'_, Self>) -> PyRef { + slf + } + + /// :type memo: typing.Any + /// :rtype: RdfFormat + #[allow(unused_variables)] + fn __deepcopy__<'a>(slf: PyRef<'a, Self>, memo: &'_ PyAny) -> PyRef<'a, Self> { + slf + } +} + pub enum PyReadable { Bytes(Cursor>), Io(PyIo), @@ -233,24 +419,20 @@ pub enum PyWritable { } impl PyWritable { - pub fn do_write<'a, F: Format>( - write: impl FnOnce(BufWriter, F) -> PyResult>, - output: Option<&PyAny>, - format: Option<&str>, - py: Python<'a>, - ) -> PyResult> { - let file_path = output.and_then(|output| output.extract::().ok()); - let format = parse_format::(format, file_path.as_deref())?; - let output = if let Some(output) = output { - if let Some(file_path) = &file_path { - Self::File(py.allow_threads(|| File::create(file_path))?) - } else { - Self::Io(PyIo(output.into())) - } - } else { - PyWritable::Bytes(Vec::new()) + pub fn do_write( + write: impl FnOnce(BufWriter, Option) -> PyResult>, + output: Option, + py: Python<'_>, + ) -> PyResult> { + let (output, file_path) = match output { + Some(PyWritableOutput::Path(file_path)) => ( + Self::File(py.allow_threads(|| File::create(&file_path))?), + Some(file_path), + ), + Some(PyWritableOutput::Io(object)) => (Self::Io(PyIo(object)), None), + None => (Self::Bytes(Vec::new()), None), }; - let writer = write(BufWriter::new(output), format)?; + let writer = write(BufWriter::new(output), file_path)?; py.allow_threads(|| writer.into_inner())?.close(py) } @@ -290,6 +472,12 @@ impl Write for PyWritable { } } +#[derive(FromPyObject)] +pub enum PyWritableOutput { + Path(PathBuf), + Io(PyObject), +} + pub struct PyIo(PyObject); impl Read for PyIo { @@ -331,57 +519,23 @@ impl Write for PyIo { } } -pub trait Format: Sized { - fn from_media_type(media_type: &str) -> Option; - fn from_extension(extension: &str) -> Option; -} - -impl Format for RdfFormat { - fn from_media_type(media_type: &str) -> Option { - Self::from_media_type(media_type) - } - - fn from_extension(extension: &str) -> Option { - Self::from_extension(extension) - } -} - -impl Format for QueryResultsFormat { - fn from_media_type(media_type: &str) -> Option { - Self::from_media_type(media_type) +pub fn lookup_rdf_format(format: Option, path: Option<&Path>) -> PyResult { + if let Some(format) = format { + return Ok(format.inner); } - - fn from_extension(extension: &str) -> Option { - Self::from_extension(extension) - } -} - -pub fn parse_format(format: Option<&str>, path: Option<&Path>) -> PyResult { - let format = if let Some(format) = format { - format - } else if let Some(path) = path { - if let Some(ext) = path.extension().and_then(OsStr::to_str) { - ext - } else { - return Err(PyValueError::new_err(format!( - "The file name {} has no extension to guess a file format from", - path.display() - ))); - } - } else { + let Some(path) = path else { return Err(PyValueError::new_err( "The format parameter is required when a file path is not given", )); }; - if format.contains('/') { - F::from_media_type(format).ok_or_else(|| { - PyValueError::new_err(format!("Not supported RDF format media type: {format}")) - }) - } else { - F::from_extension(format).ok_or_else(|| { - PyValueError::new_err(format!("Not supported RDF format extension: {format}")) - }) - } + let Some(ext) = path.extension().and_then(OsStr::to_str) else { + return Err(PyValueError::new_err(format!( + "The file name {} has no extension to guess a file format from", + path.display() + ))); + }; + RdfFormat::from_extension(ext) + .ok_or_else(|| PyValueError::new_err(format!("Not supported RDF format extension: {ext}"))) } pub fn map_parse_error(error: ParseError, file_path: Option) -> PyErr { diff --git a/python/src/lib.rs b/python/src/lib.rs index d1b76022..051e2718 100644 --- a/python/src/lib.rs +++ b/python/src/lib.rs @@ -35,6 +35,8 @@ fn pyoxigraph(_py: Python<'_>, module: &PyModule) -> PyResult<()> { module.add_class::()?; module.add_class::()?; module.add_class::()?; + module.add_class::()?; + module.add_class::()?; module.add_wrapped(wrap_pyfunction!(parse))?; module.add_wrapped(wrap_pyfunction!(parse_query_results))?; module.add_wrapped(wrap_pyfunction!(serialize))?; diff --git a/python/src/model.rs b/python/src/model.rs index ffd3f0e1..db3f0258 100644 --- a/python/src/model.rs +++ b/python/src/model.rs @@ -1276,7 +1276,7 @@ fn eq_compare_other_type(op: CompareOp) -> PyResult { } } -fn hash(t: &impl Hash) -> u64 { +pub(crate) fn hash(t: &impl Hash) -> u64 { let mut s = DefaultHasher::new(); t.hash(&mut s); s.finish() diff --git a/python/src/sparql.rs b/python/src/sparql.rs index 863bda9b..36506e02 100644 --- a/python/src/sparql.rs +++ b/python/src/sparql.rs @@ -4,8 +4,8 @@ use crate::store::map_storage_error; use oxigraph::io::RdfSerializer; use oxigraph::model::Term; use oxigraph::sparql::results::{ - FromReadQueryResultsReader, FromReadSolutionsReader, ParseError, QueryResultsParser, - QueryResultsSerializer, + FromReadQueryResultsReader, FromReadSolutionsReader, ParseError, QueryResultsFormat, + QueryResultsParser, QueryResultsSerializer, }; use oxigraph::sparql::{ EvaluationError, Query, QueryResults, QuerySolution, QuerySolutionIter, QueryTripleIter, @@ -15,8 +15,9 @@ use pyo3::basic::CompareOp; use pyo3::exceptions::{PyRuntimeError, PySyntaxError, PyValueError}; use pyo3::prelude::*; use pyo3::types::PyBytes; +use std::ffi::OsStr; use std::io; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::vec::IntoIter; pub fn parse_query( @@ -214,18 +215,18 @@ impl PyQuerySolutions { /// /// It currently supports the following formats: /// - /// * `XML `_ (``application/sparql-results+xml`` or ``srx``) - /// * `JSON `_ (``application/sparql-results+json`` or ``srj``) - /// * `CSV `_ (``text/csv`` or ``csv``) - /// * `TSV `_ (``text/tab-separated-values`` or ``tsv``) + /// * `XML `_ (:py:attr:`QueryResultsFormat.XML`) + /// * `JSON `_ (:py:attr:`QueryResultsFormat.JSON`) + /// * `CSV `_ (:py:attr:`QueryResultsFormat.CSV`) + /// * `TSV `_ (:py:attr:`QueryResultsFormat.TSV`) /// /// It supports also some media type and extension aliases. /// For example, ``application/json`` could also be used for `JSON `_. /// /// :param output: The binary I/O object or file path to write to. For example, it could be a file path as a string or a file writer opened in binary mode with ``open('my_file.ttl', 'wb')``. If :py:const:`None`, a :py:class:`bytes` buffer is returned with the serialized content. /// :type output: typing.IO[bytes] or str or os.PathLike[str] or None, optional - /// :param format: the format of the query results serialization using a media type like ``text/csv`` or an extension like `csv`. If :py:const:`None`, the format is guessed from the file name extension. - /// :type format: str or None, optional + /// :param format: the format of the query results serialization. If :py:const:`None`, the format is guessed from the file name extension. + /// :type format: QueryResultsFormat or None, optional /// :rtype: bytes or None /// :raises ValueError: if the format is not supported. /// :raises OSError: if a system error happens while writing the file. @@ -233,17 +234,18 @@ impl PyQuerySolutions { /// >>> store = Store() /// >>> store.add(Quad(NamedNode('http://example.com'), NamedNode('http://example.com/p'), Literal('1'))) /// >>> results = store.query("SELECT ?s ?p ?o WHERE { ?s ?p ?o }") - /// >>> results.serialize(format="json") + /// >>> results.serialize(format=QueryResultsFormat.JSON) /// b'{"head":{"vars":["s","p","o"]},"results":{"bindings":[{"s":{"type":"uri","value":"http://example.com"},"p":{"type":"uri","value":"http://example.com/p"},"o":{"type":"literal","value":"1"}}]}}' - #[pyo3(signature = (output = None, /, format = None))] + #[pyo3(signature = (output = None, format = None))] fn serialize<'a>( &mut self, - output: Option<&PyAny>, - format: Option<&str>, + output: Option, + format: Option, py: Python<'a>, ) -> PyResult> { PyWritable::do_write( - |output, format| { + |output, file_path| { + let format = lookup_query_results_format(format, file_path.as_deref())?; let mut writer = QueryResultsSerializer::from_format(format) .serialize_solutions_to_write( output, @@ -272,7 +274,6 @@ impl PyQuerySolutions { Ok(writer.finish()?) }, output, - format, py, ) } @@ -314,18 +315,18 @@ impl PyQueryBoolean { /// /// It currently supports the following formats: /// - /// * `XML `_ (``application/sparql-results+xml`` or ``srx``) - /// * `JSON `_ (``application/sparql-results+json`` or ``srj``) - /// * `CSV `_ (``text/csv`` or ``csv``) - /// * `TSV `_ (``text/tab-separated-values`` or ``tsv``) + /// * `XML `_ (:py:attr:`QueryResultsFormat.XML`) + /// * `JSON `_ (:py:attr:`QueryResultsFormat.JSON`) + /// * `CSV `_ (:py:attr:`QueryResultsFormat.CSV`) + /// * `TSV `_ (:py:attr:`QueryResultsFormat.TSV`) /// /// It supports also some media type and extension aliases. /// For example, ``application/json`` could also be used for `JSON `_. /// /// :param output: The binary I/O object or file path to write to. For example, it could be a file path as a string or a file writer opened in binary mode with ``open('my_file.ttl', 'wb')``. If :py:const:`None`, a :py:class:`bytes` buffer is returned with the serialized content. /// :type output: typing.IO[bytes] or str or os.PathLike[str] or None, optional - /// :param format: the format of the query results serialization using a media type like ``text/csv`` or an extension like `csv`. If :py:const:`None`, the format is guessed from the file name extension. - /// :type format: str or None, optional + /// :param format: the format of the query results serialization. If :py:const:`None`, the format is guessed from the file name extension. + /// :type format: QueryResultsFormat or None, optional /// :rtype: bytes or None /// :raises ValueError: if the format is not supported. /// :raises OSError: if a system error happens while writing the file. @@ -333,24 +334,24 @@ impl PyQueryBoolean { /// >>> store = Store() /// >>> store.add(Quad(NamedNode('http://example.com'), NamedNode('http://example.com/p'), Literal('1'))) /// >>> results = store.query("ASK { ?s ?p ?o }") - /// >>> results.serialize(format="json") + /// >>> results.serialize(format=QueryResultsFormat.JSON) /// b'{"head":{},"boolean":true}' - #[pyo3(signature = (output = None, /, format = None))] + #[pyo3(signature = (output = None, format = None))] fn serialize<'a>( &mut self, - output: Option<&PyAny>, - format: Option<&str>, + output: Option, + format: Option, py: Python<'a>, ) -> PyResult> { PyWritable::do_write( - |output, format| { + |output, file_path| { + let format = lookup_query_results_format(format, file_path.as_deref())?; py.allow_threads(|| { Ok(QueryResultsSerializer::from_format(format) .serialize_boolean_to_write(output, self.inner)?) }) }, output, - format, py, ) } @@ -389,12 +390,12 @@ impl PyQueryTriples { /// /// It currently supports the following formats: /// - /// * `N-Triples `_ (``application/n-triples`` or ``nt``) - /// * `N-Quads `_ (``application/n-quads`` or ``nq``) - /// * `Turtle `_ (``text/turtle`` or ``ttl``) - /// * `TriG `_ (``application/trig`` or ``trig``) - /// * `N3 `_ (``text/n3`` or ``n3``) - /// * `RDF/XML `_ (``application/rdf+xml`` or ``rdf``) + /// * `canonical `_ `N-Triples `_ (:py:attr:`RdfFormat.N_TRIPLES`) + /// * `N-Quads `_ (:py:attr:`RdfFormat.N_QUADS`) + /// * `Turtle `_ (:py:attr:`RdfFormat.TURTLE`) + /// * `TriG `_ (:py:attr:`RdfFormat.TRIG`) + /// * `N3 `_ (:py:attr:`RdfFormat.N3`) + /// * `RDF/XML `_ (:py:attr:`RdfFormat.RDF_XML`) /// /// It supports also some media type and extension aliases. /// For example, ``application/turtle`` could also be used for `Turtle `_ @@ -402,8 +403,8 @@ impl PyQueryTriples { /// /// :param output: The binary I/O object or file path to write to. For example, it could be a file path as a string or a file writer opened in binary mode with ``open('my_file.ttl', 'wb')``. If :py:const:`None`, a :py:class:`bytes` buffer is returned with the serialized content. /// :type output: typing.IO[bytes] or str or os.PathLike[str] or None, optional - /// :param format: the format of the RDF serialization using a media type like ``text/turtle`` or an extension like `ttl`. If :py:const:`None`, the format is guessed from the file name extension. - /// :type format: str or None, optional + /// :param format: the format of the RDF serialization. If :py:const:`None`, the format is guessed from the file name extension. + /// :type format: RdfFormat or None, optional /// :rtype: bytes or None /// :raises ValueError: if the format is not supported. /// :raises OSError: if a system error happens while writing the file. @@ -411,17 +412,18 @@ impl PyQueryTriples { /// >>> store = Store() /// >>> store.add(Quad(NamedNode('http://example.com'), NamedNode('http://example.com/p'), Literal('1'))) /// >>> results = store.query("CONSTRUCT { ?s ?p ?o } WHERE { ?s ?p ?o }") - /// >>> results.serialize(format="nt") + /// >>> results.serialize(format=RdfFormat.N_TRIPLES) /// b' "1" .\n' - #[pyo3(signature = (output = None, /, format = None))] + #[pyo3(signature = (output = None, format = None))] fn serialize<'a>( &mut self, - output: Option<&PyAny>, - format: Option<&str>, + output: Option, + format: Option, py: Python<'a>, ) -> PyResult> { PyWritable::do_write( - |output, format| { + |output, file_path| { + let format = lookup_rdf_format(format, file_path.as_deref())?; let mut writer = RdfSerializer::from_format(format).serialize_to_write(output); for triple in &mut self.inner { writer.write_triple(&triple.map_err(map_evaluation_error)?)?; @@ -429,7 +431,6 @@ impl PyQueryTriples { Ok(writer.finish()?) }, output, - format, py, ) } @@ -450,18 +451,17 @@ impl PyQueryTriples { /// /// It currently supports the following formats: /// -/// * `XML `_ (``application/sparql-results+xml`` or ``srx``) -/// * `JSON `_ (``application/sparql-results+json`` or ``srj``) -/// * `CSV `_ (``text/csv`` or ``csv``) -/// * `TSV `_ (``text/tab-separated-values`` or ``tsv``) +/// * `XML `_ (:py:attr:`QueryResultsFormat.XML`) +/// * `JSON `_ (:py:attr:`QueryResultsFormat.JSON`) +/// * `TSV `_ (:py:attr:`QueryResultsFormat.TSV`) /// /// It supports also some media type and extension aliases. /// For example, ``application/json`` could also be used for `JSON `_. /// /// :param input: The :py:class:`str`, :py:class:`bytes` or I/O object to read from. For example, it could be the file content as a string or a file reader opened in binary mode with ``open('my_file.ttl', 'rb')``. /// :type input: bytes or str or typing.IO[bytes] or typing.IO[str] or None, optional -/// :param format: the format of the RDF serialization using a media type like ``text/turtle`` or an extension like `ttl`. If :py:const:`None`, the format is guessed from the file name extension. -/// :type format: str or None, optional +/// :param format: the format of the query results serialization. If :py:const:`None`, the format is guessed from the file name extension. +/// :type format: QueryResultsFormat or None, optional /// :param path: The file path to read from. Replaces the ``input`` parameter. /// :type path: str or os.PathLike[str] or None, optional /// :return: an iterator of :py:class:`QuerySolution` or a :py:class:`bool`. @@ -470,21 +470,21 @@ impl PyQueryTriples { /// :raises SyntaxError: if the provided data is invalid. /// :raises OSError: if a system error happens while reading the file. /// -/// >>> list(parse_query_results('?s\t?p\t?o\n\t\t1\n', "text/tsv")) +/// >>> list(parse_query_results('?s\t?p\t?o\n\t\t1\n', QueryResultsFormat.TSV)) /// [ p= o=>>] /// -/// >>> parse_query_results('{"head":{},"boolean":true}', "application/sparql-results+json") +/// >>> parse_query_results('{"head":{},"boolean":true}', QueryResultsFormat.JSON) /// #[pyfunction] #[pyo3(signature = (input = None, format = None, *, path = None))] pub fn parse_query_results( input: Option, - format: Option<&str>, + format: Option, path: Option, py: Python<'_>, ) -> PyResult { let input = PyReadable::from_args(&path, input, py)?; - let format = parse_format(format, path.as_deref())?; + let format = lookup_query_results_format(format, path.as_deref())?; let results = QueryResultsParser::from_format(format) .parse_read(input) .map_err(|e| map_query_results_parse_error(e, path.clone()))?; @@ -500,6 +500,177 @@ pub fn parse_query_results( }) } +/// `SPARQL query `_ results serialization formats. +/// +/// The following formats are supported: +/// * `XML `_ (:py:attr:`QueryResultsFormat.XML`) +/// * `JSON `_ (:py:attr:`QueryResultsFormat.JSON`) +/// * `CSV `_ (:py:attr:`QueryResultsFormat.CSV`) +/// * `TSV `_ (:py:attr:`QueryResultsFormat.TSV`) +#[pyclass(name = "QueryResultsFormat", module = "pyoxigraph")] +#[derive(Clone)] +pub struct PyQueryResultsFormat { + inner: QueryResultsFormat, +} + +#[pymethods] +impl PyQueryResultsFormat { + /// `SPARQL Query Results XML Format `_ + #[classattr] + const XML: Self = Self { + inner: QueryResultsFormat::Xml, + }; + + /// `SPARQL Query Results JSON Format `_ + #[classattr] + const JSON: Self = Self { + inner: QueryResultsFormat::Json, + }; + + /// `SPARQL Query Results CSV Format `_ + #[classattr] + const CSV: Self = Self { + inner: QueryResultsFormat::Csv, + }; + + /// `SPARQL Query Results TSV Format `_ + #[classattr] + const TSV: Self = Self { + inner: QueryResultsFormat::Tsv, + }; + + /// :return: the format canonical IRI according to the `Unique URIs for file formats registry `_. + /// :rtype: str + /// + /// >>> QueryResultsFormat.JSON.iri + /// 'http://www.w3.org/ns/formats/SPARQL_Results_JSON' + #[getter] + fn iri(&self) -> &'static str { + self.inner.iri() + } + + /// :return: the format `IANA media type `_. + /// :rtype: str + /// + /// >>> QueryResultsFormat.JSON.media_type + /// 'application/sparql-results+json' + #[getter] + fn media_type(&self) -> &'static str { + self.inner.media_type() + } + + /// :return: the format `IANA-registered `_ file extension. + /// :rtype: str + /// + /// >>> QueryResultsFormat.JSON.file_extension + /// 'srj' + #[getter] + fn file_extension(&self) -> &'static str { + self.inner.file_extension() + } + + /// :return: the format name. + /// :rtype: str + /// + /// >>> QueryResultsFormat.JSON.name + /// 'SPARQL Results in JSON' + #[getter] + pub const fn name(&self) -> &'static str { + self.inner.name() + } + + /// Looks for a known format from a media type. + /// + /// It supports some media type aliases. + /// For example, "application/xml" is going to return :py:const:`QueryResultsFormat.XML` even if it is not its canonical media type. + /// + /// :param media_type: the media type. + /// :type media_type: str + /// :return: :py:class:`QueryResultsFormat` if the media type is known or :py:const:`None` if not. + /// :rtype: QueryResultsFormat or None + /// + /// >>> QueryResultsFormat.from_media_type("application/sparql-results+json; charset=utf-8") + /// + #[staticmethod] + fn from_media_type(media_type: &str) -> Option { + Some(Self { + inner: QueryResultsFormat::from_media_type(media_type)?, + }) + } + + /// Looks for a known format from an extension. + /// + /// It supports some aliases. + /// + /// :param extension: the extension. + /// :type extension: str + /// :return: :py:class:`QueryResultsFormat` if the extension is known or :py:const:`None` if not. + /// :rtype: QueryResultsFormat or None + /// + /// >>> QueryResultsFormat.from_extension("json") + /// + #[staticmethod] + fn from_extension(extension: &str) -> Option { + Some(Self { + inner: QueryResultsFormat::from_extension(extension)?, + }) + } + + fn __str__(&self) -> &'static str { + self.inner.name() + } + + fn __repr__(&self) -> String { + format!("", self.inner.name()) + } + + fn __hash__(&self) -> u64 { + hash(&self.inner) + } + + fn __eq__(&self, other: &Self) -> bool { + self.inner == other.inner + } + + fn __ne__(&self, other: &Self) -> bool { + self.inner != other.inner + } + + /// :rtype: QueryResultsFormat + fn __copy__(slf: PyRef<'_, Self>) -> PyRef { + slf + } + + /// :type memo: typing.Any + /// :rtype: QueryResultsFormat + #[allow(unused_variables)] + fn __deepcopy__<'a>(slf: PyRef<'a, Self>, memo: &'_ PyAny) -> PyRef<'a, Self> { + slf + } +} + +pub fn lookup_query_results_format( + format: Option, + path: Option<&Path>, +) -> PyResult { + if let Some(format) = format { + return Ok(format.inner); + } + let Some(path) = path else { + return Err(PyValueError::new_err( + "The format parameter is required when a file path is not given", + )); + }; + let Some(ext) = path.extension().and_then(OsStr::to_str) else { + return Err(PyValueError::new_err(format!( + "The file name {} has no extension to guess a file format from", + path.display() + ))); + }; + QueryResultsFormat::from_extension(ext) + .ok_or_else(|| PyValueError::new_err(format!("Not supported RDF format extension: {ext}"))) +} + pub fn map_evaluation_error(error: EvaluationError) -> PyErr { match error { EvaluationError::Parsing(error) => PySyntaxError::new_err(error.to_string()), diff --git a/python/src/store.rs b/python/src/store.rs index 4340d03e..f9e0b800 100644 --- a/python/src/store.rs +++ b/python/src/store.rs @@ -1,11 +1,11 @@ #![allow(clippy::needless_option_as_deref)] use crate::io::{ - allow_threads_unsafe, map_parse_error, parse_format, PyReadable, PyReadableInput, PyWritable, + allow_threads_unsafe, lookup_rdf_format, map_parse_error, PyRdfFormat, PyReadable, + PyReadableInput, PyWritable, PyWritableOutput, }; use crate::model::*; use crate::sparql::*; -use oxigraph::io::RdfFormat; use oxigraph::model::{GraphName, GraphNameRef}; use oxigraph::sparql::Update; use oxigraph::store::{self, LoaderError, SerializerError, StorageError, Store}; @@ -351,12 +351,12 @@ impl PyStore { /// /// It currently supports the following formats: /// - /// * `N-Triples `_ (``application/n-triples`` or ``nt``) - /// * `N-Quads `_ (``application/n-quads`` or ``nq``) - /// * `Turtle `_ (``text/turtle`` or ``ttl``) - /// * `TriG `_ (``application/trig`` or ``trig``) - /// * `N3 `_ (``text/n3`` or ``n3``) - /// * `RDF/XML `_ (``application/rdf+xml`` or ``rdf``) + /// * `N-Triples `_ (:py:attr:`RdfFormat.N_TRIPLES`) + /// * `N-Quads `_ (:py:attr:`RdfFormat.N_QUADS`) + /// * `Turtle `_ (:py:attr:`RdfFormat.TURTLE`) + /// * `TriG `_ (:py:attr:`RdfFormat.TRIG`) + /// * `N3 `_ (:py:attr:`RdfFormat.N3`) + /// * `RDF/XML `_ (:py:attr:`RdfFormat.RDF_XML`) /// /// It supports also some media type and extension aliases. /// For example, ``application/turtle`` could also be used for `Turtle `_ @@ -364,8 +364,8 @@ impl PyStore { /// /// :param input: The :py:class:`str`, :py:class:`bytes` or I/O object to read from. For example, it could be the file content as a string or a file reader opened in binary mode with ``open('my_file.ttl', 'rb')``. /// :type input: bytes or str or typing.IO[bytes] or typing.IO[str] or None, optional - /// :param format: the format of the RDF serialization using a media type like ``text/turtle`` or an extension like `ttl`. If :py:const:`None`, the format is guessed from the file name extension. - /// :type format: str or None, optional + /// :param format: the format of the RDF serialization. If :py:const:`None`, the format is guessed from the file name extension. + /// :type format: RdfFormat or None, optional /// :param path: The file path to read from. Replaces the ``input`` parameter. /// :type path: str or os.PathLike[str] or None, optional /// :param base_iri: the base IRI used to resolve the relative IRIs in the file or :py:const:`None` if relative IRI resolution should not be done. @@ -378,14 +378,14 @@ impl PyStore { /// :raises OSError: if an error happens during a quad insertion or if a system error happens while reading the file. /// /// >>> store = Store() - /// >>> store.load(input='

"1" .', format="text/turtle", base_iri="http://example.com/", to_graph=NamedNode("http://example.com/g")) + /// >>> store.load(input='

"1" .', format=RdfFormat.TURTLE, base_iri="http://example.com/", to_graph=NamedNode("http://example.com/g")) /// >>> list(store) /// [ predicate= object=> graph_name=>] #[pyo3(signature = (input = None, format = None, *, path = None, base_iri = None, to_graph = None))] fn load( &self, input: Option, - format: Option<&str>, + format: Option, path: Option, base_iri: Option<&str>, to_graph: Option<&PyAny>, @@ -397,7 +397,7 @@ impl PyStore { None }; let input = PyReadable::from_args(&path, input, py)?; - let format: RdfFormat = parse_format(format, path.as_deref())?; + let format = lookup_rdf_format(format, path.as_deref())?; py.allow_threads(|| { if let Some(to_graph_name) = to_graph_name { self.inner @@ -418,12 +418,12 @@ impl PyStore { /// /// It currently supports the following formats: /// - /// * `N-Triples `_ (``application/n-triples`` or ``nt``) - /// * `N-Quads `_ (``application/n-quads`` or ``nq``) - /// * `Turtle `_ (``text/turtle`` or ``ttl``) - /// * `TriG `_ (``application/trig`` or ``trig``) - /// * `N3 `_ (``text/n3`` or ``n3``) - /// * `RDF/XML `_ (``application/rdf+xml`` or ``rdf``) + /// * `N-Triples `_ (:py:attr:`RdfFormat.N_TRIPLES`) + /// * `N-Quads `_ (:py:attr:`RdfFormat.N_QUADS`) + /// * `Turtle `_ (:py:attr:`RdfFormat.TURTLE`) + /// * `TriG `_ (:py:attr:`RdfFormat.TRIG`) + /// * `N3 `_ (:py:attr:`RdfFormat.N3`) + /// * `RDF/XML `_ (:py:attr:`RdfFormat.RDF_XML`) /// /// It supports also some media type and extension aliases. /// For example, ``application/turtle`` could also be used for `Turtle `_ @@ -431,7 +431,7 @@ impl PyStore { /// /// :param input: The :py:class:`str`, :py:class:`bytes` or I/O object to read from. For example, it could be the file content as a string or a file reader opened in binary mode with ``open('my_file.ttl', 'rb')``. /// :type input: bytes or str or typing.IO[bytes] or typing.IO[str] or None, optional - /// :param format: the format of the RDF serialization using a media type like ``text/turtle`` or an extension like `ttl`. If :py:const:`None`, the format is guessed from the file name extension. + /// :param format: the format of the RDF serialization. If :py:const:`None`, the format is guessed from the file name extension. /// :type format: str or None, optional /// :param path: The file path to read from. Replaces the ``input`` parameter. /// :type path: str or os.PathLike[str] or None, optional @@ -445,14 +445,14 @@ impl PyStore { /// :raises OSError: if an error happens during a quad insertion or if a system error happens while reading the file. /// /// >>> store = Store() - /// >>> store.bulk_load(input=b'

"1" .', format="text/turtle", base_iri="http://example.com/", to_graph=NamedNode("http://example.com/g")) + /// >>> store.bulk_load(input=b'

"1" .', format=RdfFormat.TURTLE, base_iri="http://example.com/", to_graph=NamedNode("http://example.com/g")) /// >>> list(store) /// [ predicate= object=> graph_name=>] #[pyo3(signature = (input = None, format = None, *, path = None, base_iri = None, to_graph = None))] fn bulk_load( &self, input: Option, - format: Option<&str>, + format: Option, path: Option, base_iri: Option<&str>, to_graph: Option<&PyAny>, @@ -464,7 +464,7 @@ impl PyStore { None }; let input = PyReadable::from_args(&path, input, py)?; - let format: RdfFormat = parse_format(format, path.as_deref())?; + let format = lookup_rdf_format(format, path.as_deref())?; py.allow_threads(|| { if let Some(to_graph_name) = to_graph_name { self.inner @@ -483,12 +483,12 @@ impl PyStore { /// /// It currently supports the following formats: /// - /// * `N-Triples `_ (``application/n-triples`` or ``nt``) - /// * `N-Quads `_ (``application/n-quads`` or ``nq``) - /// * `Turtle `_ (``text/turtle`` or ``ttl``) - /// * `TriG `_ (``application/trig`` or ``trig``) - /// * `N3 `_ (``text/n3`` or ``n3``) - /// * `RDF/XML `_ (``application/rdf+xml`` or ``rdf``) + /// * `N-Triples `_ (:py:attr:`RdfFormat.N_TRIPLES`) + /// * `N-Quads `_ (:py:attr:`RdfFormat.N_QUADS`) + /// * `Turtle `_ (:py:attr:`RdfFormat.TURTLE`) + /// * `TriG `_ (:py:attr:`RdfFormat.TRIG`) + /// * `N3 `_ (:py:attr:`RdfFormat.N3`) + /// * `RDF/XML `_ (:py:attr:`RdfFormat.RDF_XML`) /// /// It supports also some media type and extension aliases. /// For example, ``application/turtle`` could also be used for `Turtle `_ @@ -496,31 +496,31 @@ impl PyStore { /// /// :param output: The binary I/O object or file path to write to. For example, it could be a file path as a string or a file writer opened in binary mode with ``open('my_file.ttl', 'wb')``. If :py:const:`None`, a :py:class:`bytes` buffer is returned with the serialized content. /// :type output: typing.IO[bytes] or str or os.PathLike[str] or None, optional - /// :param format: the format of the RDF serialization using a media type like ``text/turtle`` or an extension like `ttl`. If :py:const:`None`, the format is guessed from the file name extension. - /// :type format: str or None, optional + /// :param format: the format of the RDF serialization. If :py:const:`None`, the format is guessed from the file name extension. + /// :type format: RdfFormat or None, optional /// :param from_graph: the store graph from which dump the triples. Required if the serialization format does not support named graphs. If it does supports named graphs the full dataset is written. /// :type from_graph: NamedNode or BlankNode or DefaultGraph or None, optional - /// :return: py:class:`bytes` with the serialization if the ``output`` parameter is :py:const:`None`, :py:const:`None` if ``output`` is set. + /// :return: :py:class:`bytes` with the serialization if the ``output`` parameter is :py:const:`None`, :py:const:`None` if ``output`` is set. /// :rtype: bytes or None /// :raises ValueError: if the format is not supported or the `from_graph` parameter is not given with a syntax not supporting named graphs. /// :raises OSError: if an error happens during a quad lookup or file writing. /// /// >>> store = Store() /// >>> store.add(Quad(NamedNode('http://example.com'), NamedNode('http://example.com/p'), Literal('1'))) - /// >>> store.dump(format="trig") + /// >>> store.dump(format=RdfFormat.TRIG) /// b' "1" .\n' /// /// >>> store = Store() /// >>> store.add(Quad(NamedNode('http://example.com'), NamedNode('http://example.com/p'), Literal('1'), NamedNode('http://example.com/g'))) /// >>> output = io.BytesIO() - /// >>> store.dump(output, "text/turtle", from_graph=NamedNode("http://example.com/g")) + /// >>> store.dump(output, RdfFormat.TURTLE, from_graph=NamedNode("http://example.com/g")) /// >>> output.getvalue() /// b' "1" .\n' - #[pyo3(signature = (output = None, /, format = None, *, from_graph = None))] + #[pyo3(signature = (output = None, format = None, *, from_graph = None))] fn dump<'a>( &self, - output: Option<&PyAny>, - format: Option<&str>, + output: Option, + format: Option, from_graph: Option<&PyAny>, py: Python<'a>, ) -> PyResult> { @@ -529,9 +529,10 @@ impl PyStore { } else { None }; - PyWritable::do_write::( - |output, format| { + PyWritable::do_write( + |output, file_path| { py.allow_threads(|| { + let format = lookup_rdf_format(format, file_path.as_deref())?; if let Some(from_graph_name) = &from_graph_name { self.inner.dump_graph(output, format, from_graph_name) } else { @@ -541,7 +542,6 @@ impl PyStore { }) }, output, - format, py, ) } diff --git a/python/tests/test_io.py b/python/tests/test_io.py index 9c8d4047..fe137eff 100644 --- a/python/tests/test_io.py +++ b/python/tests/test_io.py @@ -8,7 +8,9 @@ from pyoxigraph import ( NamedNode, Quad, QueryBoolean, + QueryResultsFormat, QuerySolutions, + RdfFormat, parse, parse_query_results, serialize, @@ -39,14 +41,14 @@ class TestParse(unittest.TestCase): def test_parse_not_existing_file(self) -> None: with self.assertRaises(IOError) as _: - parse(path="/tmp/not-existing-oxigraph-file.ttl", format="text/turtle") + parse(path="/tmp/not-existing-oxigraph-file.ttl", format=RdfFormat.TURTLE) def test_parse_str(self) -> None: self.assertEqual( list( parse( '

"éù" .', - "text/turtle", + RdfFormat.TURTLE, base_iri="http://example.com/", ) ), @@ -58,7 +60,7 @@ class TestParse(unittest.TestCase): list( parse( '

"éù" .'.encode(), - "text/turtle", + RdfFormat.TURTLE, base_iri="http://example.com/", ) ), @@ -70,7 +72,7 @@ class TestParse(unittest.TestCase): list( parse( StringIO('

"éù" .'), - "text/turtle", + RdfFormat.TURTLE, base_iri="http://example.com/", ) ), @@ -82,7 +84,7 @@ class TestParse(unittest.TestCase): list( parse( StringIO('

"éù" .\n' * 1024), - "text/turtle", + RdfFormat.TURTLE, base_iri="http://example.com/", ) ), @@ -94,7 +96,7 @@ class TestParse(unittest.TestCase): list( parse( BytesIO('

"éù" .'.encode()), - "text/turtle", + RdfFormat.TURTLE, base_iri="http://example.com/", ) ), @@ -103,14 +105,14 @@ class TestParse(unittest.TestCase): def test_parse_io_error(self) -> None: with self.assertRaises(UnsupportedOperation) as _, TemporaryFile("wb") as fp: - list(parse(fp, "nt")) + list(parse(fp, RdfFormat.N_TRIPLES)) def test_parse_quad(self) -> None: self.assertEqual( list( parse( ' {

"1" }', - "application/trig", + RdfFormat.TRIG, base_iri="http://example.com/", ) ), @@ -123,7 +125,7 @@ class TestParse(unittest.TestCase): fp.write(b' "p" "1"') fp.flush() with self.assertRaises(SyntaxError) as ctx: - list(parse(path=fp.name, format="text/turtle")) + list(parse(path=fp.name, format=RdfFormat.TURTLE)) self.assertEqual(ctx.exception.filename, fp.name) self.assertEqual(ctx.exception.lineno, 2) self.assertEqual(ctx.exception.offset, 7) @@ -136,7 +138,7 @@ class TestParse(unittest.TestCase): list( parse( ' {

"1" }', - "application/trig", + RdfFormat.TRIG, base_iri="http://example.com/", without_named_graphs=True, ) @@ -147,14 +149,14 @@ class TestParse(unittest.TestCase): list( parse( '_:s "o" .', - "application/n-triples", + RdfFormat.N_TRIPLES, rename_blank_nodes=True, ) ), list( parse( '_:s "o" .', - "application/n-triples", + RdfFormat.N_TRIPLES, rename_blank_nodes=True, ) ), @@ -164,13 +166,13 @@ class TestParse(unittest.TestCase): class TestSerialize(unittest.TestCase): def test_serialize_to_bytes(self) -> None: self.assertEqual( - (serialize([EXAMPLE_TRIPLE.triple], None, "text/turtle") or b"").decode(), + (serialize([EXAMPLE_TRIPLE.triple], None, RdfFormat.TURTLE) or b"").decode(), ' "éù" .\n', ) def test_serialize_to_bytes_io(self) -> None: output = BytesIO() - serialize([EXAMPLE_TRIPLE.triple], output, "text/turtle") + serialize([EXAMPLE_TRIPLE.triple], output, RdfFormat.TURTLE) self.assertEqual( output.getvalue().decode(), ' "éù" .\n', @@ -186,11 +188,11 @@ class TestSerialize(unittest.TestCase): def test_serialize_io_error(self) -> None: with self.assertRaises(UnsupportedOperation) as _, TemporaryFile("rb") as fp: - serialize([EXAMPLE_TRIPLE], fp, "text/turtle") + serialize([EXAMPLE_TRIPLE], fp, RdfFormat.TURTLE) def test_serialize_quad(self) -> None: output = BytesIO() - serialize([EXAMPLE_QUAD], output, "application/trig") + serialize([EXAMPLE_QUAD], output, RdfFormat.TRIG) self.assertEqual( output.getvalue(), b' {\n\t "1" .\n}\n', @@ -210,38 +212,38 @@ class TestParseQuerySolutions(unittest.TestCase): def test_parse_not_existing_file(self) -> None: with self.assertRaises(IOError) as _: - parse_query_results(path="/tmp/not-existing-oxigraph-file.ttl", format="application/json") + parse_query_results(path="/tmp/not-existing-oxigraph-file.ttl", format=QueryResultsFormat.JSON) def test_parse_str(self) -> None: - result = parse_query_results("true", "tsv") + result = parse_query_results("true", QueryResultsFormat.TSV) self.assertIsInstance(result, QueryBoolean) self.assertTrue(result) def test_parse_bytes(self) -> None: - result = parse_query_results(b"false", "tsv") + result = parse_query_results(b"false", QueryResultsFormat.TSV) self.assertIsInstance(result, QueryBoolean) self.assertFalse(result) def test_parse_str_io(self) -> None: - result = parse_query_results("true", "tsv") + result = parse_query_results("true", QueryResultsFormat.TSV) self.assertIsInstance(result, QueryBoolean) self.assertTrue(result) def test_parse_bytes_io(self) -> None: - result = parse_query_results(BytesIO(b"false"), "tsv") + result = parse_query_results(BytesIO(b"false"), QueryResultsFormat.TSV) self.assertIsInstance(result, QueryBoolean) self.assertFalse(result) def test_parse_io_error(self) -> None: with self.assertRaises(UnsupportedOperation) as _, TemporaryFile("wb") as fp: - parse_query_results(fp, "srx") + parse_query_results(fp, QueryResultsFormat.XML) def test_parse_syntax_error_json(self) -> None: with NamedTemporaryFile() as fp: fp.write(b"{]") fp.flush() with self.assertRaises(SyntaxError) as ctx: - list(parse_query_results(path=fp.name, format="srj")) # type: ignore[arg-type] + list(parse_query_results(path=fp.name, format=QueryResultsFormat.JSON)) # type: ignore[arg-type] self.assertEqual(ctx.exception.filename, fp.name) self.assertEqual(ctx.exception.lineno, 1) self.assertEqual(ctx.exception.offset, 2) @@ -255,7 +257,7 @@ class TestParseQuerySolutions(unittest.TestCase): fp.write(b"1\t\n") fp.flush() with self.assertRaises(SyntaxError) as ctx: - list(parse_query_results(path=fp.name, format="tsv")) # type: ignore[arg-type] + list(parse_query_results(path=fp.name, format=QueryResultsFormat.TSV)) # type: ignore[arg-type] self.assertEqual(ctx.exception.filename, fp.name) self.assertEqual(ctx.exception.lineno, 2) self.assertEqual(ctx.exception.offset, 3) diff --git a/python/tests/test_store.py b/python/tests/test_store.py index 001755fd..4f47b5f0 100644 --- a/python/tests/test_store.py +++ b/python/tests/test_store.py @@ -9,9 +9,12 @@ from pyoxigraph import ( DefaultGraph, NamedNode, Quad, + QueryBoolean, + QueryResultsFormat, QuerySolution, QuerySolutions, QueryTriples, + RdfFormat, Store, Triple, Variable, @@ -190,9 +193,10 @@ class TestStore(unittest.TestCase): def test_select_query_dump(self) -> None: store = Store() store.add(Quad(foo, bar, baz)) - results = store.query("SELECT ?s WHERE { ?s ?p ?o }") + results: QuerySolutions = store.query("SELECT ?s WHERE { ?s ?p ?o }") # type: ignore[assignment] + self.assertIsInstance(results, QuerySolutions) output = BytesIO() - results.serialize(output, "csv") + results.serialize(output, QueryResultsFormat.CSV) self.assertEqual( output.getvalue().decode(), "s\r\nhttp://foo\r\n", @@ -201,9 +205,10 @@ class TestStore(unittest.TestCase): def test_ask_query_dump(self) -> None: store = Store() store.add(Quad(foo, bar, baz)) - results = store.query("ASK { ?s ?p ?o }") + results: QueryBoolean = store.query("ASK { ?s ?p ?o }") # type: ignore[assignment] + self.assertIsInstance(results, QueryBoolean) output = BytesIO() - results.serialize(output, "csv") + results.serialize(output, QueryResultsFormat.CSV) self.assertEqual( output.getvalue().decode(), "true", @@ -212,9 +217,10 @@ class TestStore(unittest.TestCase): def test_construct_query_dump(self) -> None: store = Store() store.add(Quad(foo, bar, baz)) - results = store.query("CONSTRUCT WHERE { ?s ?p ?o }") + results: QueryTriples = store.query("CONSTRUCT WHERE { ?s ?p ?o }") # type: ignore[assignment] + self.assertIsInstance(results, QueryTriples) output = BytesIO() - results.serialize(output, "nt") + results.serialize(output, RdfFormat.N_TRIPLES) self.assertEqual( output.getvalue().decode(), " .\n", @@ -254,7 +260,7 @@ class TestStore(unittest.TestCase): store = Store() store.load( b" .", - "application/n-triples", + RdfFormat.N_TRIPLES, ) self.assertEqual(set(store), {Quad(foo, bar, baz, DefaultGraph())}) @@ -262,7 +268,7 @@ class TestStore(unittest.TestCase): store = Store() store.load( " .", - "application/n-triples", + RdfFormat.N_TRIPLES, to_graph=graph, ) self.assertEqual(set(store), {Quad(foo, bar, baz, graph)}) @@ -271,7 +277,7 @@ class TestStore(unittest.TestCase): store = Store() store.load( BytesIO(b" <> ."), - "text/turtle", + RdfFormat.TURTLE, base_iri="http://baz", ) self.assertEqual(set(store), {Quad(foo, bar, baz, DefaultGraph())}) @@ -280,7 +286,7 @@ class TestStore(unittest.TestCase): store = Store() store.load( StringIO(" ."), - "nq", + RdfFormat.N_QUADS, ) self.assertEqual(set(store), {Quad(foo, bar, baz, graph)}) @@ -288,7 +294,7 @@ class TestStore(unittest.TestCase): store = Store() store.load( " { <> . }", - "application/trig", + RdfFormat.TRIG, base_iri="http://baz", ) self.assertEqual(set(store), {Quad(foo, bar, baz, graph)}) @@ -303,13 +309,13 @@ class TestStore(unittest.TestCase): def test_load_with_io_error(self) -> None: with self.assertRaises(UnsupportedOperation) as _, TemporaryFile("wb") as fp: - Store().load(fp, "application/n-triples") + Store().load(fp, RdfFormat.N_TRIPLES) def test_dump_ntriples(self) -> None: store = Store() store.add(Quad(foo, bar, baz, graph)) output = BytesIO() - store.dump(output, "application/n-triples", from_graph=graph) + store.dump(output, RdfFormat.N_TRIPLES, from_graph=graph) self.assertEqual( output.getvalue(), b" .\n", @@ -319,7 +325,7 @@ class TestStore(unittest.TestCase): store = Store() store.add(Quad(foo, bar, baz, graph)) self.assertEqual( - store.dump(format="nq"), + store.dump(format=RdfFormat.N_QUADS), b" .\n", ) @@ -328,7 +334,7 @@ class TestStore(unittest.TestCase): store.add(Quad(foo, bar, baz, graph)) store.add(Quad(foo, bar, baz)) output = BytesIO() - store.dump(output, "application/trig") + store.dump(output, RdfFormat.TRIG) self.assertEqual( output.getvalue(), b" .\n" @@ -340,7 +346,7 @@ class TestStore(unittest.TestCase): store = Store() store.add(Quad(foo, bar, baz, graph)) file_name = Path(fp.name) - store.dump(file_name, "nq") + store.dump(file_name, RdfFormat.N_QUADS) self.assertEqual( file_name.read_text(), " .\n", @@ -350,7 +356,7 @@ class TestStore(unittest.TestCase): store = Store() store.add(Quad(foo, bar, bar)) with self.assertRaises(OSError) as _, TemporaryFile("rb") as fp: - store.dump(fp, "application/trig") + store.dump(fp, RdfFormat.TRIG) def test_write_in_read(self) -> None: store = Store() From 98caee8f9223b8bf93aa779f0eb3834524906afc Mon Sep 17 00:00:00 2001 From: Tpt Date: Fri, 17 Nov 2023 17:54:22 +0100 Subject: [PATCH 122/217] RDF/XML: avoids to serialize special XML namespaces --- lib/oxrdfxml/src/serializer.rs | 39 ++++++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/lib/oxrdfxml/src/serializer.rs b/lib/oxrdfxml/src/serializer.rs index 8a025390..5883c31f 100644 --- a/lib/oxrdfxml/src/serializer.rs +++ b/lib/oxrdfxml/src/serializer.rs @@ -2,6 +2,7 @@ use crate::utils::*; use oxrdf::{Subject, SubjectRef, TermRef, TripleRef}; use quick_xml::events::*; use quick_xml::Writer; +use std::borrow::Cow; use std::io; use std::io::Write; use std::sync::Arc; @@ -21,7 +22,7 @@ use tokio::io::AsyncWrite; /// NamedNodeRef::new("http://schema.org/Person")?, /// ))?; /// assert_eq!( -/// b"\n\n\t\n\t\t\n\t\n", +/// b"\n\n\t\n\t\t\n\t\n", /// writer.finish()?.as_slice() /// ); /// # Result::<_,Box>::Ok(()) @@ -52,7 +53,7 @@ impl RdfXmlSerializer { /// NamedNodeRef::new("http://schema.org/Person")?, /// ))?; /// assert_eq!( - /// b"\n\n\t\n\t\t\n\t\n", + /// b"\n\n\t\n\t\t\n\t\n", /// writer.finish()?.as_slice() /// ); /// # Result::<_,Box>::Ok(()) @@ -84,7 +85,7 @@ impl RdfXmlSerializer { /// NamedNodeRef::new_unchecked("http://schema.org/Person"), /// )).await?; /// assert_eq!( - /// b"\n\n\t\n\t\t\n\t\n", + /// b"\n\n\t\n\t\t\n\t\n", /// writer.finish().await?.as_slice() /// ); /// # Ok(()) @@ -118,7 +119,7 @@ impl RdfXmlSerializer { /// NamedNodeRef::new("http://schema.org/Person")?, /// ))?; /// assert_eq!( -/// b"\n\n\t\n\t\t\n\t\n", +/// b"\n\n\t\n\t\t\n\t\n", /// writer.finish()?.as_slice() /// ); /// # Result::<_,Box>::Ok(()) @@ -253,13 +254,29 @@ impl InnerRdfXmlWriter { self.current_subject = Some(triple.subject.into_owned()); let (prop_prefix, prop_value) = split_iri(triple.predicate.as_str()); - let (prop_qname, prop_xmlns) = if prop_value.is_empty() { - ("prop:", ("xmlns:prop", prop_prefix)) - } else { - (prop_value, ("xmlns", prop_prefix)) - }; - let mut property_open = BytesStart::new(prop_qname); - property_open.push_attribute(prop_xmlns); + let (prop_qname, prop_xmlns) = + if prop_prefix == "http://www.w3.org/1999/02/22-rdf-syntax-ns#" { + (Cow::Owned(format!("rdf:{prop_value}")), None) + } else if prop_prefix == "http://www.w3.org/2000/xmlns/" { + if prop_value.is_empty() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "The http://www.w3.org/2000/xmlns/ predicate IRI is not allowed in XML", + )); + } + ( + Cow::Borrowed("p:"), + Some(("xmlns:p", triple.predicate.as_str())), + ) + } else if prop_value.is_empty() { + (Cow::Borrowed("p:"), Some(("xmlns:p", prop_prefix))) + } else { + (Cow::Borrowed(prop_value), Some(("xmlns", prop_prefix))) + }; + let mut property_open = BytesStart::new(prop_qname.clone()); + if let Some(prop_xmlns) = prop_xmlns { + property_open.push_attribute(prop_xmlns); + } let content = match triple.object { TermRef::NamedNode(node) => { property_open.push_attribute(("rdf:resource", node.as_str())); From f8034c68e9341010d4949f3c5532cdeba83213bc Mon Sep 17 00:00:00 2001 From: Tpt Date: Sat, 18 Nov 2023 21:20:12 +0100 Subject: [PATCH 123/217] SPARQL: refactor AggregateExpression Avoids code duplication --- lib/spargebra/src/algebra.rs | 241 +++++++++++++---------------------- lib/spargebra/src/parser.rs | 40 +++--- lib/sparopt/src/algebra.rs | 122 ++---------------- lib/sparopt/src/optimizer.rs | 13 +- lib/src/sparql/eval.rs | 99 ++++++-------- 5 files changed, 163 insertions(+), 352 deletions(-) diff --git a/lib/spargebra/src/algebra.rs b/lib/spargebra/src/algebra.rs index d694ba53..6c629246 100644 --- a/lib/spargebra/src/algebra.rs +++ b/lib/spargebra/src/algebra.rs @@ -1114,46 +1114,11 @@ impl<'a> fmt::Display for SparqlGraphRootPattern<'a> { /// A set function used in aggregates (c.f. [`GraphPattern::Group`]). #[derive(Eq, PartialEq, Debug, Clone, Hash)] pub enum AggregateExpression { - /// [Count](https://www.w3.org/TR/sparql11-query/#defn_aggCount). - Count { - expr: Option>, - distinct: bool, - }, - /// [Sum](https://www.w3.org/TR/sparql11-query/#defn_aggSum). - Sum { - expr: Box, - distinct: bool, - }, - /// [Avg](https://www.w3.org/TR/sparql11-query/#defn_aggAvg). - Avg { - expr: Box, - distinct: bool, - }, - /// [Min](https://www.w3.org/TR/sparql11-query/#defn_aggMin). - Min { - expr: Box, - distinct: bool, - }, - /// [Max](https://www.w3.org/TR/sparql11-query/#defn_aggMax). - Max { - expr: Box, - distinct: bool, - }, - /// [GroupConcat](https://www.w3.org/TR/sparql11-query/#defn_aggGroupConcat). - GroupConcat { - expr: Box, - distinct: bool, - separator: Option, - }, - /// [Sample](https://www.w3.org/TR/sparql11-query/#defn_aggSample). - Sample { - expr: Box, - distinct: bool, - }, - /// Custom function. - Custom { - name: NamedNode, - expr: Box, + /// [Count](https://www.w3.org/TR/sparql11-query/#defn_aggCount) with *. + CountSolutions { distinct: bool }, + FunctionCall { + name: AggregateFunction, + expr: Expression, distinct: bool, }, } @@ -1162,82 +1127,39 @@ impl AggregateExpression { /// Formats using the [SPARQL S-Expression syntax](https://jena.apache.org/documentation/notes/sse.html). pub(crate) fn fmt_sse(&self, f: &mut impl fmt::Write) -> fmt::Result { match self { - Self::Count { expr, distinct } => { - write!(f, "(sum")?; + Self::CountSolutions { distinct } => { + write!(f, "(count")?; if *distinct { write!(f, " distinct")?; } - if let Some(expr) = expr { - write!(f, " ")?; - expr.fmt_sse(f)?; - } - write!(f, ")") - } - Self::Sum { expr, distinct } => { - write!(f, "(sum ")?; - if *distinct { - write!(f, "distinct ")?; - } - expr.fmt_sse(f)?; write!(f, ")") } - Self::Avg { expr, distinct } => { - write!(f, "(avg ")?; - if *distinct { - write!(f, "distinct ")?; - } - expr.fmt_sse(f)?; - write!(f, ")") - } - Self::Min { expr, distinct } => { - write!(f, "(min ")?; - if *distinct { - write!(f, "distinct ")?; - } - expr.fmt_sse(f)?; - write!(f, ")") - } - Self::Max { expr, distinct } => { - write!(f, "(max ")?; - if *distinct { - write!(f, "distinct ")?; - } - expr.fmt_sse(f)?; - write!(f, ")") - } - Self::Sample { expr, distinct } => { - write!(f, "(sample ")?; - if *distinct { - write!(f, "distinct ")?; - } - expr.fmt_sse(f)?; - write!(f, ")") - } - Self::GroupConcat { + Self::FunctionCall { + name: + AggregateFunction::GroupConcat { + separator: Some(separator), + }, expr, distinct, - separator, } => { write!(f, "(group_concat ")?; if *distinct { write!(f, "distinct ")?; } expr.fmt_sse(f)?; - if let Some(separator) = separator { - write!(f, " {}", LiteralRef::new_simple_literal(separator))?; - } - write!(f, ")") + write!(f, " {})", LiteralRef::new_simple_literal(separator)) } - Self::Custom { + Self::FunctionCall { name, expr, distinct, } => { - write!(f, "({name}")?; + write!(f, "(")?; + name.fmt_sse(f)?; + write!(f, " ")?; if *distinct { - write!(f, " distinct")?; + write!(f, "distinct ")?; } - write!(f, " ")?; expr.fmt_sse(f)?; write!(f, ")") } @@ -1248,82 +1170,38 @@ impl AggregateExpression { impl fmt::Display for AggregateExpression { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - Self::Count { expr, distinct } => { + Self::CountSolutions { distinct } => { if *distinct { - if let Some(expr) = expr { - write!(f, "COUNT(DISTINCT {expr})") - } else { - write!(f, "COUNT(DISTINCT *)") - } - } else if let Some(expr) = expr { - write!(f, "COUNT({expr})") + write!(f, "COUNT(DISTINCT *)") } else { write!(f, "COUNT(*)") } } - Self::Sum { expr, distinct } => { - if *distinct { - write!(f, "SUM(DISTINCT {expr})") - } else { - write!(f, "SUM({expr})") - } - } - Self::Min { expr, distinct } => { - if *distinct { - write!(f, "MIN(DISTINCT {expr})") - } else { - write!(f, "MIN({expr})") - } - } - Self::Max { expr, distinct } => { - if *distinct { - write!(f, "MAX(DISTINCT {expr})") - } else { - write!(f, "MAX({expr})") - } - } - Self::Avg { expr, distinct } => { - if *distinct { - write!(f, "AVG(DISTINCT {expr})") - } else { - write!(f, "AVG({expr})") - } - } - Self::Sample { expr, distinct } => { - if *distinct { - write!(f, "SAMPLE(DISTINCT {expr})") - } else { - write!(f, "SAMPLE({expr})") - } - } - Self::GroupConcat { + Self::FunctionCall { + name: + AggregateFunction::GroupConcat { + separator: Some(separator), + }, expr, distinct, - separator, } => { if *distinct { - if let Some(separator) = separator { - write!( - f, - "GROUP_CONCAT(DISTINCT {}; SEPARATOR = {})", - expr, - LiteralRef::new_simple_literal(separator) - ) - } else { - write!(f, "GROUP_CONCAT(DISTINCT {expr})") - } - } else if let Some(separator) = separator { write!( f, - "GROUP_CONCAT({}; SEPARATOR = {})", + "GROUP_CONCAT(DISTINCT {}; SEPARATOR = {})", expr, LiteralRef::new_simple_literal(separator) ) } else { - write!(f, "GROUP_CONCAT({expr})") + write!( + f, + "GROUP_CONCAT({}; SEPARATOR = {})", + expr, + LiteralRef::new_simple_literal(separator) + ) } } - Self::Custom { + Self::FunctionCall { name, expr, distinct, @@ -1338,6 +1216,59 @@ impl fmt::Display for AggregateExpression { } } +/// An aggregate function name. +#[derive(Eq, PartialEq, Debug, Clone, Hash)] +pub enum AggregateFunction { + /// [Count](https://www.w3.org/TR/sparql11-query/#defn_aggCount) with *. + Count, + /// [Sum](https://www.w3.org/TR/sparql11-query/#defn_aggSum). + Sum, + /// [Avg](https://www.w3.org/TR/sparql11-query/#defn_aggAvg). + Avg, + /// [Min](https://www.w3.org/TR/sparql11-query/#defn_aggMin). + Min, + /// [Max](https://www.w3.org/TR/sparql11-query/#defn_aggMax). + Max, + /// [GroupConcat](https://www.w3.org/TR/sparql11-query/#defn_aggGroupConcat). + GroupConcat { + separator: Option, + }, + /// [Sample](https://www.w3.org/TR/sparql11-query/#defn_aggSample). + Sample, + Custom(NamedNode), +} + +impl AggregateFunction { + /// Formats using the [SPARQL S-Expression syntax](https://jena.apache.org/documentation/notes/sse.html). + pub(crate) fn fmt_sse(&self, f: &mut impl fmt::Write) -> fmt::Result { + match self { + Self::Count => write!(f, "count"), + Self::Sum => write!(f, "sum"), + Self::Avg => write!(f, "avg"), + Self::Min => write!(f, "min"), + Self::Max => write!(f, "max"), + Self::GroupConcat { .. } => write!(f, "group_concat"), + Self::Sample => write!(f, "sample"), + Self::Custom(iri) => write!(f, "{iri}"), + } + } +} + +impl fmt::Display for AggregateFunction { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Count => write!(f, "COUNT"), + Self::Sum => write!(f, "SUM"), + Self::Avg => write!(f, "AVG"), + Self::Min => write!(f, "MIN"), + Self::Max => write!(f, "MAX"), + Self::GroupConcat { .. } => write!(f, "GROUP_CONCAT"), + Self::Sample => write!(f, "SAMPLE"), + Self::Custom(iri) => iri.fmt(f), + } + } +} + /// An ordering comparator used by [`GraphPattern::OrderBy`]. #[derive(Eq, PartialEq, Debug, Clone, Hash)] pub enum OrderExpression { diff --git a/lib/spargebra/src/parser.rs b/lib/spargebra/src/parser.rs index 60f8038c..52bfbb54 100644 --- a/lib/spargebra/src/parser.rs +++ b/lib/spargebra/src/parser.rs @@ -1918,26 +1918,26 @@ parser! { rule NotExistsFunc() -> Expression = i("NOT") _ i("EXISTS") _ p:GroupGraphPattern() { Expression::Not(Box::new(Expression::Exists(Box::new(p)))) } rule Aggregate() -> AggregateExpression = - i("COUNT") _ "(" _ i("DISTINCT") _ "*" _ ")" { AggregateExpression::Count { expr: None, distinct: true } } / - i("COUNT") _ "(" _ i("DISTINCT") _ e:Expression() _ ")" { AggregateExpression::Count { expr: Some(Box::new(e)), distinct: true } } / - i("COUNT") _ "(" _ "*" _ ")" { AggregateExpression::Count { expr: None, distinct: false } } / - i("COUNT") _ "(" _ e:Expression() _ ")" { AggregateExpression::Count { expr: Some(Box::new(e)), distinct: false } } / - i("SUM") _ "(" _ i("DISTINCT") _ e:Expression() _ ")" { AggregateExpression::Sum { expr: Box::new(e), distinct: true } } / - i("SUM") _ "(" _ e:Expression() _ ")" { AggregateExpression::Sum { expr: Box::new(e), distinct: false } } / - i("MIN") _ "(" _ i("DISTINCT") _ e:Expression() _ ")" { AggregateExpression::Min { expr: Box::new(e), distinct: true } } / - i("MIN") _ "(" _ e:Expression() _ ")" { AggregateExpression::Min { expr: Box::new(e), distinct: false } } / - i("MAX") _ "(" _ i("DISTINCT") _ e:Expression() _ ")" { AggregateExpression::Max { expr: Box::new(e), distinct: true } } / - i("MAX") _ "(" _ e:Expression() _ ")" { AggregateExpression::Max { expr: Box::new(e), distinct: false } } / - i("AVG") _ "(" _ i("DISTINCT") _ e:Expression() _ ")" { AggregateExpression::Avg { expr: Box::new(e), distinct: true } } / - i("AVG") _ "(" _ e:Expression() _ ")" { AggregateExpression::Avg { expr: Box::new(e), distinct: false } } / - i("SAMPLE") _ "(" _ i("DISTINCT") _ e:Expression() _ ")" { AggregateExpression::Sample { expr: Box::new(e), distinct: true } } / - i("SAMPLE") _ "(" _ e:Expression() _ ")" { AggregateExpression::Sample { expr: Box::new(e), distinct: false } } / - i("GROUP_CONCAT") _ "(" _ i("DISTINCT") _ e:Expression() _ ";" _ i("SEPARATOR") _ "=" _ s:String() _ ")" { AggregateExpression::GroupConcat { expr: Box::new(e), distinct: true, separator: Some(s) } } / - i("GROUP_CONCAT") _ "(" _ i("DISTINCT") _ e:Expression() _ ")" { AggregateExpression::GroupConcat { expr: Box::new(e), distinct: true, separator: None } } / - i("GROUP_CONCAT") _ "(" _ e:Expression() _ ";" _ i("SEPARATOR") _ "=" _ s:String() _ ")" { AggregateExpression::GroupConcat { expr: Box::new(e), distinct: true, separator: Some(s) } } / - i("GROUP_CONCAT") _ "(" _ e:Expression() _ ")" { AggregateExpression::GroupConcat { expr: Box::new(e), distinct: false, separator: None } } / - name:iri() _ "(" _ i("DISTINCT") _ e:Expression() _ ")" { AggregateExpression::Custom { name, expr: Box::new(e), distinct: true } } / - name:iri() _ "(" _ e:Expression() _ ")" { AggregateExpression::Custom { name, expr: Box::new(e), distinct: false } } + i("COUNT") _ "(" _ i("DISTINCT") _ "*" _ ")" { AggregateExpression::CountSolutions { distinct: true } } / + i("COUNT") _ "(" _ i("DISTINCT") _ expr:Expression() _ ")" { AggregateExpression::FunctionCall { name: AggregateFunction::Count, expr, distinct: true } } / + i("COUNT") _ "(" _ "*" _ ")" { AggregateExpression::CountSolutions { distinct: false } } / + i("COUNT") _ "(" _ expr:Expression() _ ")" { AggregateExpression::FunctionCall { name: AggregateFunction::Count, expr, distinct: false } } / + i("SUM") _ "(" _ i("DISTINCT") _ expr:Expression() _ ")" { AggregateExpression::FunctionCall { name: AggregateFunction::Sum, expr, distinct: true } } / + i("SUM") _ "(" _ expr:Expression() _ ")" { AggregateExpression::FunctionCall { name: AggregateFunction::Sum, expr, distinct: false } } / + i("MIN") _ "(" _ i("DISTINCT") _ expr:Expression() _ ")" { AggregateExpression::FunctionCall { name: AggregateFunction::Min, expr, distinct: true } } / + i("MIN") _ "(" _ expr:Expression() _ ")" { AggregateExpression::FunctionCall { name: AggregateFunction::Min, expr, distinct: false } } / + i("MAX") _ "(" _ i("DISTINCT") _ expr:Expression() _ ")" { AggregateExpression::FunctionCall { name: AggregateFunction::Max, expr, distinct: true } } / + i("MAX") _ "(" _ expr:Expression() _ ")" { AggregateExpression::FunctionCall { name: AggregateFunction::Max, expr, distinct: false } } / + i("AVG") _ "(" _ i("DISTINCT") _ expr:Expression() _ ")" { AggregateExpression::FunctionCall { name: AggregateFunction::Avg, expr, distinct: true } } / + i("AVG") _ "(" _ expr:Expression() _ ")" { AggregateExpression::FunctionCall { name: AggregateFunction::Avg, expr, distinct: false } } / + i("SAMPLE") _ "(" _ i("DISTINCT") _ expr:Expression() _ ")" { AggregateExpression::FunctionCall { name: AggregateFunction::Sample, expr, distinct: true } } / + i("SAMPLE") _ "(" _ expr:Expression() _ ")" { AggregateExpression::FunctionCall { name: AggregateFunction::Sample, expr, distinct: false } } / + i("GROUP_CONCAT") _ "(" _ i("DISTINCT") _ expr:Expression() _ ";" _ i("SEPARATOR") _ "=" _ s:String() _ ")" { AggregateExpression::FunctionCall { name: AggregateFunction::GroupConcat { separator: Some(s) }, expr, distinct: true } } / + i("GROUP_CONCAT") _ "(" _ i("DISTINCT") _ expr:Expression() _ ")" { AggregateExpression::FunctionCall { name: AggregateFunction::GroupConcat { separator: None }, expr, distinct: true } } / + i("GROUP_CONCAT") _ "(" _ expr:Expression() _ ";" _ i("SEPARATOR") _ "=" _ s:String() _ ")" { AggregateExpression::FunctionCall { name: AggregateFunction::GroupConcat { separator: Some(s) }, expr, distinct: true } } / + i("GROUP_CONCAT") _ "(" _ expr:Expression() _ ")" { AggregateExpression::FunctionCall { name: AggregateFunction::GroupConcat { separator: None }, expr, distinct: false } } / + name:iri() _ "(" _ i("DISTINCT") _ expr:Expression() _ ")" { AggregateExpression::FunctionCall { name: AggregateFunction::Custom(name), expr, distinct: true } } / + name:iri() _ "(" _ expr:Expression() _ ")" { AggregateExpression::FunctionCall { name: AggregateFunction::Custom(name), expr, distinct: false } } rule iriOrFunction() -> Expression = i: iri() _ a: ArgList()? { match a { diff --git a/lib/sparopt/src/algebra.rs b/lib/sparopt/src/algebra.rs index fd7942d5..e5cb0952 100644 --- a/lib/sparopt/src/algebra.rs +++ b/lib/sparopt/src/algebra.rs @@ -3,7 +3,7 @@ use oxrdf::vocab::xsd; use rand::random; use spargebra::algebra::{ - AggregateExpression as AlAggregateExpression, Expression as AlExpression, + AggregateExpression as AlAggregateExpression, AggregateFunction, Expression as AlExpression, GraphPattern as AlGraphPattern, OrderExpression as AlOrderExpression, }; pub use spargebra::algebra::{Function, PropertyPathExpression}; @@ -1538,46 +1538,12 @@ impl Default for MinusAlgorithm { /// A set function used in aggregates (c.f. [`GraphPattern::Group`]). #[derive(Eq, PartialEq, Debug, Clone, Hash)] pub enum AggregateExpression { - /// [Count](https://www.w3.org/TR/sparql11-query/#defn_aggCount). - Count { - expr: Option>, + CountSolutions { distinct: bool, }, - /// [Sum](https://www.w3.org/TR/sparql11-query/#defn_aggSum). - Sum { - expr: Box, - distinct: bool, - }, - /// [Avg](https://www.w3.org/TR/sparql11-query/#defn_aggAvg). - Avg { - expr: Box, - distinct: bool, - }, - /// [Min](https://www.w3.org/TR/sparql11-query/#defn_aggMin). - Min { - expr: Box, - distinct: bool, - }, - /// [Max](https://www.w3.org/TR/sparql11-query/#defn_aggMax). - Max { - expr: Box, - distinct: bool, - }, - /// [GroupConcat](https://www.w3.org/TR/sparql11-query/#defn_aggGroupConcat). - GroupConcat { - expr: Box, - distinct: bool, - separator: Option, - }, - /// [Sample](https://www.w3.org/TR/sparql11-query/#defn_aggSample). - Sample { - expr: Box, - distinct: bool, - }, - /// Custom function. - Custom { - name: NamedNode, - expr: Box, + FunctionCall { + name: AggregateFunction, + expr: Expression, distinct: bool, }, } @@ -1588,48 +1554,16 @@ impl AggregateExpression { graph_name: Option<&NamedNodePattern>, ) -> Self { match expression { - AlAggregateExpression::Count { expr, distinct } => Self::Count { - expr: expr - .as_ref() - .map(|e| Box::new(Expression::from_sparql_algebra(e, graph_name))), - distinct: *distinct, - }, - AlAggregateExpression::Sum { expr, distinct } => Self::Sum { - expr: Box::new(Expression::from_sparql_algebra(expr, graph_name)), - distinct: *distinct, - }, - AlAggregateExpression::Avg { expr, distinct } => Self::Avg { - expr: Box::new(Expression::from_sparql_algebra(expr, graph_name)), - distinct: *distinct, - }, - AlAggregateExpression::Min { expr, distinct } => Self::Min { - expr: Box::new(Expression::from_sparql_algebra(expr, graph_name)), - distinct: *distinct, - }, - AlAggregateExpression::Max { expr, distinct } => Self::Max { - expr: Box::new(Expression::from_sparql_algebra(expr, graph_name)), + AlAggregateExpression::CountSolutions { distinct } => Self::CountSolutions { distinct: *distinct, }, - AlAggregateExpression::GroupConcat { - expr, - distinct, - separator, - } => Self::GroupConcat { - expr: Box::new(Expression::from_sparql_algebra(expr, graph_name)), - distinct: *distinct, - separator: separator.clone(), - }, - AlAggregateExpression::Sample { expr, distinct } => Self::Sample { - expr: Box::new(Expression::from_sparql_algebra(expr, graph_name)), - distinct: *distinct, - }, - AlAggregateExpression::Custom { + AlAggregateExpression::FunctionCall { name, expr, distinct, - } => Self::Custom { + } => Self::FunctionCall { name: name.clone(), - expr: Box::new(Expression::from_sparql_algebra(expr, graph_name)), + expr: Expression::from_sparql_algebra(expr, graph_name), distinct: *distinct, }, } @@ -1639,46 +1573,16 @@ impl AggregateExpression { impl From<&AggregateExpression> for AlAggregateExpression { fn from(expression: &AggregateExpression) -> Self { match expression { - AggregateExpression::Count { expr, distinct } => Self::Count { - expr: expr.as_ref().map(|e| Box::new(e.as_ref().into())), - distinct: *distinct, - }, - AggregateExpression::Sum { expr, distinct } => Self::Sum { - expr: Box::new(expr.as_ref().into()), - distinct: *distinct, - }, - AggregateExpression::Avg { expr, distinct } => Self::Avg { - expr: Box::new(expr.as_ref().into()), - distinct: *distinct, - }, - AggregateExpression::Min { expr, distinct } => Self::Min { - expr: Box::new(expr.as_ref().into()), - distinct: *distinct, - }, - AggregateExpression::Max { expr, distinct } => Self::Max { - expr: Box::new(expr.as_ref().into()), - distinct: *distinct, - }, - AggregateExpression::GroupConcat { - expr, - distinct, - separator, - } => Self::GroupConcat { - expr: Box::new(expr.as_ref().into()), - distinct: *distinct, - separator: separator.clone(), - }, - AggregateExpression::Sample { expr, distinct } => Self::Sample { - expr: Box::new(expr.as_ref().into()), + AggregateExpression::CountSolutions { distinct } => Self::CountSolutions { distinct: *distinct, }, - AggregateExpression::Custom { + AggregateExpression::FunctionCall { name, expr, distinct, - } => Self::Custom { + } => Self::FunctionCall { name: name.clone(), - expr: Box::new(expr.as_ref().into()), + expr: expr.into(), distinct: *distinct, }, } diff --git a/lib/sparopt/src/optimizer.rs b/lib/sparopt/src/optimizer.rs index 87902b59..5dc9d404 100644 --- a/lib/sparopt/src/optimizer.rs +++ b/lib/sparopt/src/optimizer.rs @@ -157,11 +157,14 @@ impl Optimizer { inner, variables, aggregates, - } => GraphPattern::group( - Self::normalize_pattern(*inner, input_types), - variables, - aggregates, - ), + } => { + // TODO: min, max and sample don't care about DISTINCT + GraphPattern::group( + Self::normalize_pattern(*inner, input_types), + variables, + aggregates, + ) + } GraphPattern::Service { name, inner, diff --git a/lib/src/sparql/eval.rs b/lib/src/sparql/eval.rs index 2997b83d..413fcd20 100644 --- a/lib/src/sparql/eval.rs +++ b/lib/src/sparql/eval.rs @@ -18,7 +18,7 @@ use rand::random; use regex::{Regex, RegexBuilder}; use sha1::Sha1; use sha2::{Sha256, Sha384, Sha512}; -use spargebra::algebra::{Function, PropertyPathExpression}; +use spargebra::algebra::{AggregateFunction, Function, PropertyPathExpression}; use spargebra::term::{ GroundSubject, GroundTerm, GroundTermPattern, GroundTriple, NamedNodePattern, TermPattern, TriplePattern, @@ -974,16 +974,8 @@ impl SimpleEvaluator { let aggregate_input_expressions = aggregates .iter() .map(|(_, expression)| match expression { - AggregateExpression::Count { expr, .. } => expr.as_ref().map(|e| { - self.expression_evaluator(e, encoded_variables, stat_children) - }), - AggregateExpression::Sum { expr, .. } - | AggregateExpression::Avg { expr, .. } - | AggregateExpression::Min { expr, .. } - | AggregateExpression::Max { expr, .. } - | AggregateExpression::GroupConcat { expr, .. } - | AggregateExpression::Sample { expr, .. } - | AggregateExpression::Custom { expr, .. } => { + AggregateExpression::CountSolutions { .. } => None, + AggregateExpression::FunctionCall { expr, .. } => { Some(self.expression_evaluator(expr, encoded_variables, stat_children)) } }) @@ -1101,52 +1093,26 @@ impl SimpleEvaluator { dataset: &Rc, expression: &AggregateExpression, ) -> Box Box> { - match expression { - AggregateExpression::Count { distinct, .. } => { - if *distinct { - Box::new(|| Box::new(DistinctAccumulator::new(CountAccumulator::default()))) - } else { - Box::new(|| Box::::default()) + let mut accumulator: Box Box> = match expression { + AggregateExpression::CountSolutions { .. } => { + Box::new(|| Box::::default()) + } + AggregateExpression::FunctionCall { name, .. } => match name { + AggregateFunction::Count => Box::new(|| Box::::default()), + AggregateFunction::Sum => Box::new(|| Box::::default()), + AggregateFunction::Min => { + let dataset = Rc::clone(dataset); + Box::new(move || Box::new(MinAccumulator::new(Rc::clone(&dataset)))) } - } - AggregateExpression::Sum { distinct, .. } => { - if *distinct { - Box::new(|| Box::new(DistinctAccumulator::new(SumAccumulator::default()))) - } else { - Box::new(|| Box::::default()) + AggregateFunction::Max => { + let dataset = Rc::clone(dataset); + Box::new(move || Box::new(MaxAccumulator::new(Rc::clone(&dataset)))) } - } - AggregateExpression::Min { .. } => { - let dataset = Rc::clone(dataset); - Box::new(move || Box::new(MinAccumulator::new(Rc::clone(&dataset)))) - } // DISTINCT does not make sense with min - AggregateExpression::Max { .. } => { - let dataset = Rc::clone(dataset); - Box::new(move || Box::new(MaxAccumulator::new(Rc::clone(&dataset)))) - } // DISTINCT does not make sense with max - AggregateExpression::Avg { distinct, .. } => { - if *distinct { - Box::new(|| Box::new(DistinctAccumulator::new(AvgAccumulator::default()))) - } else { - Box::new(|| Box::::default()) - } - } - AggregateExpression::Sample { .. } => Box::new(|| Box::::default()), // DISTINCT does not make sense with sample - AggregateExpression::GroupConcat { - distinct, - separator, - .. - } => { - let dataset = Rc::clone(dataset); - let separator = Rc::from(separator.as_deref().unwrap_or(" ")); - if *distinct { - Box::new(move || { - Box::new(DistinctAccumulator::new(GroupConcatAccumulator::new( - Rc::clone(&dataset), - Rc::clone(&separator), - ))) - }) - } else { + AggregateFunction::Avg => Box::new(|| Box::::default()), + AggregateFunction::Sample => Box::new(|| Box::::default()), + AggregateFunction::GroupConcat { separator } => { + let dataset = Rc::clone(dataset); + let separator = Rc::from(separator.as_deref().unwrap_or(" ")); Box::new(move || { Box::new(GroupConcatAccumulator::new( Rc::clone(&dataset), @@ -1154,9 +1120,17 @@ impl SimpleEvaluator { )) }) } - } - AggregateExpression::Custom { .. } => Box::new(|| Box::new(FailingAccumulator)), + AggregateFunction::Custom(_) => Box::new(|| Box::new(FailingAccumulator)), + }, + }; + if matches!( + expression, + AggregateExpression::CountSolutions { distinct: true } + | AggregateExpression::FunctionCall { distinct: true, .. } + ) { + accumulator = Box::new(move || Box::new(Deduplicate::new(accumulator()))); } + accumulator } fn expression_evaluator( @@ -5262,14 +5236,13 @@ trait Accumulator { fn state(&self) -> Option; } -#[derive(Default, Debug)] -struct DistinctAccumulator { +struct Deduplicate { seen: HashSet>, - inner: T, + inner: Box, } -impl DistinctAccumulator { - fn new(inner: T) -> Self { +impl Deduplicate { + fn new(inner: Box) -> Self { Self { seen: HashSet::default(), inner, @@ -5277,7 +5250,7 @@ impl DistinctAccumulator { } } -impl Accumulator for DistinctAccumulator { +impl Accumulator for Deduplicate { fn add(&mut self, element: Option) { if self.seen.insert(element.clone()) { self.inner.add(element) From a9fee4f6b85a344fd54825d985b6587b69494211 Mon Sep 17 00:00:00 2001 From: Tpt Date: Tue, 21 Nov 2023 21:43:32 +0100 Subject: [PATCH 124/217] Upgrades dependencies --- Cargo.lock | 229 ++++++++++++++++++++------------------ cli/Cargo.toml | 2 +- cli/src/main.rs | 19 ++-- js/src/lib.rs | 2 +- lib/Cargo.toml | 6 +- lib/sparesults/Cargo.toml | 2 +- oxrocksdb-sys/Cargo.toml | 2 +- oxrocksdb-sys/rocksdb | 2 +- 8 files changed, 139 insertions(+), 125 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 20bf72d6..85ad265f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -88,9 +88,9 @@ checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" [[package]] name = "arbitrary" -version = "1.3.1" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2e1373abdaa212b704512ec2bd8b26bd0b7d5c3f70117411a5d9a451383c859" +checksum = "7d5a26814d8dcb93b0e5a0ff3c6d80a8843bafb21b39e8e18a6f05471870e110" dependencies = [ "derive_arbitrary", ] @@ -148,17 +148,17 @@ dependencies = [ [[package]] name = "base64" -version = "0.21.4" +version = "0.21.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ba43ea6f343b788c8764558649e08df62f86c6ef251fdaeb1ffd010a9ae50a2" +checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" [[package]] name = "bindgen" -version = "0.68.1" +version = "0.69.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "726e4313eb6ec35d2730258ad4e15b547ee75d6afaa1361a922e78e59b7d8078" +checksum = "9ffcebc3849946a7170a05992aac39da343a90676ab392c51a4280981d6379c2" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.4.1", "cexpr", "clang-sys", "lazy_static", @@ -183,9 +183,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" +checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" [[package]] name = "block-buffer" @@ -198,9 +198,9 @@ dependencies = [ [[package]] name = "bstr" -version = "1.7.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c79ad7fb2dd38f3dabd76b09c6a5a20c038fc0213ef1e9afd30eb777f120f019" +checksum = "542f33a8835a0884b006a0c3df3dadd99c0c3f296ed26c2fdc8028e01ad6230c" dependencies = [ "memchr", "regex-automata", @@ -290,9 +290,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.6" +version = "4.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d04704f56c2cde07f43e8e2c154b43f216dc5c92fc98ada720177362f953b956" +checksum = "2275f18819641850fa26c89acc84d465c1bf91ce57bc2748b28c420473352f64" dependencies = [ "clap_builder", "clap_derive", @@ -300,9 +300,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.6" +version = "4.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e231faeaca65ebd1ea3c737966bf858971cd38c3849107aa3ea7de90a804e45" +checksum = "07cdf1b148b25c1e1f7a42225e30a0d99a615cd4637eae7365548dd4529b95bc" dependencies = [ "anstream", "anstyle", @@ -312,9 +312,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.4.2" +version = "4.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0862016ff20d69b84ef8247369fabf5c008a7417002411897d40ee1f4532b873" +checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442" dependencies = [ "heck", "proc-macro2", @@ -324,9 +324,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.5.1" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd7cc57abe963c6d3b9d8be5b06ba7c8957a930305ca90304f24ef040aa6f961" +checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" [[package]] name = "colorchoice" @@ -362,9 +362,9 @@ checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" [[package]] name = "cpufeatures" -version = "0.2.9" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1" +checksum = "ce420fe07aecd3e67c5f910618fe65e94158f6dcc0adf44e00d69ce2bdfe0fd0" dependencies = [ "libc", ] @@ -459,15 +459,18 @@ dependencies = [ [[package]] name = "deranged" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2696e8a945f658fd14dc3b87242e6b80cd0f36ff04ea560fa39082368847946" +checksum = "0f32d04922c60427da6f9fef14d042d9edddef64cb9d4ce0d64d0685fbeb1fd3" +dependencies = [ + "powerfmt", +] [[package]] name = "derive_arbitrary" -version = "1.3.1" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53e0efad4403bfc52dc201159c4b842a246a14b98c64b55dfd0f2d89729dfeb8" +checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611" dependencies = [ "proc-macro2", "quote", @@ -504,9 +507,9 @@ checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" [[package]] name = "errno" -version = "0.3.5" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3e13f66a2f95e32a39eaa81f6b95d42878ca0e1db0c7543723dfe12557e860" +checksum = "f258a7194e7f7c2a7837a8913aeab7fd8c383457034fa20ce4dd3dcb813e8eb8" dependencies = [ "libc", "windows-sys", @@ -532,9 +535,9 @@ checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" [[package]] name = "flate2" -version = "1.0.27" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6c98ee8095e9d1dcbf2fcc6d95acccb90d1c81db1e44725c6a984b1dbdfb010" +checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e" dependencies = [ "crc32fast", "miniz_oxide", @@ -600,9 +603,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.10" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" +checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f" dependencies = [ "cfg-if", "js-sys", @@ -765,18 +768,18 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.64" +version = "0.3.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" +checksum = "54c0c35952f67de54bb584e9fd912b3023117cbafc0a77d8f3dee1fb5f572fe8" dependencies = [ "wasm-bindgen", ] [[package]] name = "json-event-parser" -version = "0.2.0-alpha.1" +version = "0.2.0-alpha.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20a2ad11b373ee8f1d5f9b0632b148a6dc65cf1faa8b2a99c89cbe70411e31a2" +checksum = "5b5ddd02379e99769e117ab30d21ad42dcec8ad3c12be77f9a34779e62d46346" [[package]] name = "kernel32-sys" @@ -802,9 +805,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.149" +version = "0.2.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b" +checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" [[package]] name = "libloading" @@ -818,15 +821,15 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.4.10" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da2479e8c062e40bf0066ffa0bc823de0a9368974af99c9f6df941d2c231e03f" +checksum = "969488b55f8ac402214f3f5fd243ebb7206cf82de60d3172994707a4bcc2b829" [[package]] name = "lock_api" -version = "0.4.10" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16" +checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" dependencies = [ "autocfg", "scopeguard", @@ -944,11 +947,11 @@ checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" [[package]] name = "openssl" -version = "0.10.57" +version = "0.10.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bac25ee399abb46215765b1cb35bc0212377e58a061560d8b29b024fd0430e7c" +checksum = "7a257ad03cd8fb16ad4172fedf8094451e1af1c4b70097636ef2eac9a5f0cc33" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.4.1", "cfg-if", "foreign-types", "libc", @@ -976,9 +979,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.93" +version = "0.9.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db4d56a4c0478783083cfafcc42493dd4a981d41669da64b4572a2a089b51b1d" +checksum = "40a4130519a360279579c2053038317e40eff64d13fd3f004f9e1b72b8a6aaf9" dependencies = [ "cc", "libc", @@ -988,9 +991,9 @@ dependencies = [ [[package]] name = "oxhttp" -version = "0.2.0-alpha.1" +version = "0.2.0-alpha.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "525e4922eaf3a36d3d8534226910b6f5b878446d2f7f20b31da7889eb969f984" +checksum = "9546c028c11488f4661926fe18ced7dd11b573b92c3a73a2c33cd1fd859883e0" dependencies = [ "httparse", "native-tls", @@ -1159,9 +1162,9 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.8" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" +checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" dependencies = [ "cfg-if", "libc", @@ -1249,6 +1252,12 @@ dependencies = [ "plotters-backend", ] +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -1445,18 +1454,18 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.3.5" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" dependencies = [ "bitflags 1.3.2", ] [[package]] name = "regex" -version = "1.10.0" +version = "1.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d119d7c7ca818f8a53c300863d4f87566aac09943aef5b355bb83969dae75d87" +checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" dependencies = [ "aho-corasick", "memchr", @@ -1466,9 +1475,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.1" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "465c6fc0621e4abc4187a2bda0937bfd4f722c2730b29562e19689ea796c9a4b" +checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" dependencies = [ "aho-corasick", "memchr", @@ -1477,23 +1486,22 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56d84fdd47036b038fc80dd333d10b6aab10d5d31f4a366e20014def75328d33" +checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" [[package]] name = "ring" -version = "0.16.20" +version = "0.17.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +checksum = "fb0205304757e5d899b9c2e448b867ffd03ae7f988002e47cd24954391394d0b" dependencies = [ "cc", + "getrandom", "libc", - "once_cell", "spin", "untrusted", - "web-sys", - "winapi 0.3.9", + "windows-sys", ] [[package]] @@ -1527,11 +1535,11 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "rustix" -version = "0.38.18" +version = "0.38.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a74ee2d7c2581cd139b42447d7d9389b889bdaad3a73f1ebb16f2a3237bb19c" +checksum = "dc99bc2d4f1fed22595588a013687477aedf3cdcfb26558c559edb67b4d9b22e" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.4.1", "errno", "libc", "linux-raw-sys", @@ -1540,9 +1548,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.7" +version = "0.21.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd8d6c9f025a446bc4d18ad9632e69aec8f287aa84499ee335599fabd20c3fd8" +checksum = "629648aced5775d558af50b2b4c7b02983a04b312126d45eeead26e7caa498b9" dependencies = [ "log", "ring", @@ -1564,18 +1572,18 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" dependencies = [ "base64", ] [[package]] name = "rustls-webpki" -version = "0.101.6" +version = "0.101.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c7d5dece342910d9ba34d259310cae3e0154b873b35408b787b59bce53d34fe" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" dependencies = [ "ring", "untrusted", @@ -1613,9 +1621,9 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "sct" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" dependencies = [ "ring", "untrusted", @@ -1646,18 +1654,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.188" +version = "1.0.193" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" +checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.188" +version = "1.0.193" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" +checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" dependencies = [ "proc-macro2", "quote", @@ -1666,9 +1674,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.107" +version = "1.0.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" +checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" dependencies = [ "itoa", "ryu", @@ -1711,9 +1719,9 @@ checksum = "54ac45299ccbd390721be55b412d41931911f654fa99e2cb8bfb57184b2061fe" [[package]] name = "smallvec" -version = "1.11.1" +version = "1.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a" +checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970" [[package]] name = "sparesults" @@ -1754,9 +1762,9 @@ dependencies = [ [[package]] name = "spin" -version = "0.5.2" +version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" [[package]] name = "strsim" @@ -1766,9 +1774,9 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "syn" -version = "2.0.38" +version = "2.0.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b" +checksum = "23e78b90f2fcf45d3e842032ce32e3f2d1545ba6636271dcbf24fa306d87be7a" dependencies = [ "proc-macro2", "quote", @@ -1777,15 +1785,15 @@ dependencies = [ [[package]] name = "target-lexicon" -version = "0.12.11" +version = "0.12.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d0e916b1148c8e263850e1ebcbd046f333e0683c724876bb0da63ea4373dc8a" +checksum = "14c39fd04924ca3a864207c66fc2cd7d22d7c016007f9ce846cbb9326331930a" [[package]] name = "tempfile" -version = "3.8.0" +version = "3.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" +checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5" dependencies = [ "cfg-if", "fastrand", @@ -1832,12 +1840,13 @@ dependencies = [ [[package]] name = "time" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "426f806f4089c493dcac0d24c29c01e2c38baf8e30f1b716ee37e83d200b18fe" +checksum = "c4a34ab300f2dee6e562c10a046fc05e358b29f9bf92277f30c3c8d82275f6f5" dependencies = [ "deranged", "itoa", + "powerfmt", "serde", "time-core", "time-macros", @@ -1885,9 +1894,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.33.0" +version = "1.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f38200e3ef7995e5ef13baec2f432a6da0aa9ac495b2c0e8f3b7eec2c92d653" +checksum = "d0c014766411e834f7af5b8f4cf46257aab4036ca95e9d2c144a10f59ad6f5b9" dependencies = [ "backtrace", "bytes", @@ -1897,9 +1906,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" +checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", @@ -1947,9 +1956,9 @@ checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce" [[package]] name = "untrusted" -version = "0.7.1" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" @@ -2007,9 +2016,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.87" +version = "0.2.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" +checksum = "7daec296f25a1bae309c0cd5c29c4b260e510e6d813c286b19eaadf409d40fce" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -2017,9 +2026,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.87" +version = "0.2.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" +checksum = "e397f4664c0e4e428e8313a469aaa58310d302159845980fd23b0f22a847f217" dependencies = [ "bumpalo", "log", @@ -2032,9 +2041,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.87" +version = "0.2.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" +checksum = "5961017b3b08ad5f3fe39f1e79877f8ee7c23c5e5fd5eb80de95abc41f1f16b2" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2042,9 +2051,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.87" +version = "0.2.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" +checksum = "c5353b8dab669f5e10f5bd76df26a9360c748f054f862ff5f3f8aae0c7fb3907" dependencies = [ "proc-macro2", "quote", @@ -2055,15 +2064,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.87" +version = "0.2.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" +checksum = "0d046c5d029ba91a1ed14da14dca44b68bf2f124cfbaf741c54151fdb3e0750b" [[package]] name = "web-sys" -version = "0.3.64" +version = "0.3.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" +checksum = "5db499c5f66323272151db0e666cd34f78617522fb0c1604d31a27c50c206a85" dependencies = [ "js-sys", "wasm-bindgen", diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 6b84f16b..252ae5ff 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -27,7 +27,7 @@ rustls-webpki = ["oxigraph/http-client-rustls-webpki"] [dependencies] anyhow = "1.0.72" -oxhttp = { version = "0.2.0-alpha.1" } +oxhttp = "0.2.0-alpha.2" clap = { version = "4.0", features = ["derive"] } oxigraph = { version = "0.4.0-alpha.1-dev", path = "../lib" } rand = "0.8" diff --git a/cli/src/main.rs b/cli/src/main.rs index 6b2abba4..6dc7cc73 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -22,6 +22,7 @@ use std::env; use std::ffi::OsStr; use std::fs::File; use std::io::{self, stdin, stdout, BufWriter, Read, Write}; +use std::net::ToSocketAddrs; #[cfg(target_os = "linux")] use std::os::unix::net::UnixDatagram; use std::path::{Path, PathBuf}; @@ -312,7 +313,7 @@ pub fn main() -> anyhow::Result<()> { } else { Store::new() }?, - bind, + &bind, false, cors, ), @@ -320,7 +321,7 @@ pub fn main() -> anyhow::Result<()> { location, bind, cors, - } => serve(Store::open_read_only(location)?, bind, true, cors), + } => serve(Store::open_read_only(location)?, &bind, true, cors), Command::ServeSecondary { primary_location, secondary_location, @@ -332,7 +333,7 @@ pub fn main() -> anyhow::Result<()> { } else { Store::open_secondary(primary_location) }?, - bind, + &bind, true, cors, ), @@ -878,8 +879,8 @@ fn rdf_format_from_name(name: &str) -> anyhow::Result { bail!("The file format '{name}' is unknown") } -fn serve(store: Store, bind: String, read_only: bool, cors: bool) -> anyhow::Result<()> { - let server = if cors { +fn serve(store: Store, bind: &str, read_only: bool, cors: bool) -> anyhow::Result<()> { + let mut server = if cors { Server::new(cors_middleware(move |request| { handle_request(request, store.clone(), read_only) .unwrap_or_else(|(status, message)| error(status, message)) @@ -892,11 +893,15 @@ fn serve(store: Store, bind: String, read_only: bool, cors: bool) -> anyhow::Res } .with_global_timeout(HTTP_TIMEOUT) .with_server_name(concat!("Oxigraph/", env!("CARGO_PKG_VERSION")))? - .with_max_num_threads(available_parallelism()?.get() * 128); + .with_max_concurrent_connections(available_parallelism()?.get() * 128); + for socket in bind.to_socket_addrs()? { + server = server.bind(socket); + } + let server = server.spawn()?; #[cfg(target_os = "linux")] systemd_notify_ready()?; eprintln!("Listening for requests at http://{}", &bind); - server.listen(bind)?; + server.join()?; Ok(()) } diff --git a/js/src/lib.rs b/js/src/lib.rs index 769319e8..b985c5c0 100644 --- a/js/src/lib.rs +++ b/js/src/lib.rs @@ -1,4 +1,4 @@ -#![allow(clippy::unused_unit)] +#![allow(clippy::mem_forget)] use wasm_bindgen::prelude::*; mod model; diff --git a/lib/Cargo.toml b/lib/Cargo.toml index a5d65ee1..f2adcc8c 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -28,7 +28,7 @@ rocksdb-debug = [] [dependencies] digest = "0.10" hex = "0.4" -json-event-parser = "0.2.0-alpha.1" +json-event-parser = "0.2.0-alpha.2" md-5 = "0.10" oxilangtag = "0.1" oxiri = "0.2" @@ -47,7 +47,7 @@ sparopt = { version = "0.1.0-alpha.1-dev", path="sparopt", features = ["rdf-star [target.'cfg(not(target_family = "wasm"))'.dependencies] libc = "0.2.147" oxrocksdb-sys = { version = "0.4.0-alpha.1-dev", path="../oxrocksdb-sys" } -oxhttp = { version = "0.2.0-alpha.1", optional = true } +oxhttp = { version = "0.2.0-alpha.2", optional = true } [target.'cfg(all(target_family = "wasm", target_os = "unknown"))'.dependencies] getrandom = "0.2" @@ -55,7 +55,7 @@ js-sys = { version = "0.3.60", optional = true } [target.'cfg(not(target_family = "wasm"))'.dev-dependencies] criterion = "0.5" -oxhttp = "0.2.0-alpha.1" +oxhttp = "0.2.0-alpha.2" zstd = ">=0.12, <0.14" [package.metadata.docs.rs] diff --git a/lib/sparesults/Cargo.toml b/lib/sparesults/Cargo.toml index e5ffe16f..67d75cf4 100644 --- a/lib/sparesults/Cargo.toml +++ b/lib/sparesults/Cargo.toml @@ -19,7 +19,7 @@ default = [] rdf-star = ["oxrdf/rdf-star"] [dependencies] -json-event-parser = "0.2.0-alpha.1" +json-event-parser = "0.2.0-alpha.2" memchr = "2.5" oxrdf = { version = "0.2.0-alpha.1-dev", path="../oxrdf" } quick-xml = ">=0.29, <0.32" diff --git a/oxrocksdb-sys/Cargo.toml b/oxrocksdb-sys/Cargo.toml index 849a8ab7..e77bb4cf 100644 --- a/oxrocksdb-sys/Cargo.toml +++ b/oxrocksdb-sys/Cargo.toml @@ -22,5 +22,5 @@ libc = "0.2.147" [build-dependencies] pkg-config = { version = "0.3.25", optional = true } -bindgen = ">=0.60, <0.69" +bindgen = ">=0.60, <0.70" cc = { version = "1.0.73", features = ["parallel"] } diff --git a/oxrocksdb-sys/rocksdb b/oxrocksdb-sys/rocksdb index 1ce22dd6..8a494fc2 160000 --- a/oxrocksdb-sys/rocksdb +++ b/oxrocksdb-sys/rocksdb @@ -1 +1 @@ -Subproject commit 1ce22dd6376b124d17eff7d96e0809d2f4b4ae70 +Subproject commit 8a494fc2f48c23b948c22ebd23ec92e569dc4802 From 9af2717502cb9c087342d13ec8e32e89d97b0ef8 Mon Sep 17 00:00:00 2001 From: Tpt Date: Wed, 22 Nov 2023 16:28:07 +0100 Subject: [PATCH 125/217] Bulk loader: do not fail on empty files --- lib/src/storage/backend/rocksdb.rs | 3 +++ lib/tests/store.rs | 12 ++++++++++++ 2 files changed, 15 insertions(+) diff --git a/lib/src/storage/backend/rocksdb.rs b/lib/src/storage/backend/rocksdb.rs index fc8f4da2..67766007 100644 --- a/lib/src/storage/backend/rocksdb.rs +++ b/lib/src/storage/backend/rocksdb.rs @@ -814,6 +814,9 @@ impl Db { &self, ssts_for_cf: &[(&ColumnFamily, PathBuf)], ) -> Result<(), StorageError> { + if ssts_for_cf.is_empty() { + return Ok(()); // Rocksdb does not support empty lists + } if let DbKind::ReadWrite(db) = &self.inner { let mut paths_by_cf = HashMap::<_, Vec<_>>::new(); for (cf, path) in ssts_for_cf { diff --git a/lib/tests/store.rs b/lib/tests/store.rs index 5f8a6809..32cb1516 100644 --- a/lib/tests/store.rs +++ b/lib/tests/store.rs @@ -12,6 +12,8 @@ use std::fs::{create_dir, remove_dir_all, File}; use std::io::Cursor; #[cfg(not(target_family = "wasm"))] use std::io::Write; +#[cfg(not(target_family = "wasm"))] +use std::iter::empty; #[cfg(target_os = "linux")] use std::iter::once; #[cfg(not(target_family = "wasm"))] @@ -159,6 +161,16 @@ fn test_bulk_load_graph_lenient() -> Result<(), Box> { Ok(()) } +#[test] +#[cfg(not(target_family = "wasm"))] +fn test_bulk_load_empty() -> Result<(), Box> { + let store = Store::new()?; + store.bulk_loader().load_quads(empty::())?; + assert!(store.is_empty()?); + store.validate()?; + Ok(()) +} + #[test] fn test_load_dataset() -> Result<(), Box> { let store = Store::new()?; From 48174cac1233ca3264051396b23476d8313d75b9 Mon Sep 17 00:00:00 2001 From: Tpt Date: Mon, 20 Nov 2023 21:35:21 +0100 Subject: [PATCH 126/217] Distributes Pypy wheels for linux --- .github/workflows/manylinux_build.sh | 5 ++++- .github/workflows/tests.yml | 20 ++++++++++++++++++++ python/tests/test_store.py | 2 ++ 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/.github/workflows/manylinux_build.sh b/.github/workflows/manylinux_build.sh index fe1d7a0f..366e2867 100644 --- a/.github/workflows/manylinux_build.sh +++ b/.github/workflows/manylinux_build.sh @@ -18,4 +18,7 @@ if [ %for_each_version% ]; then for VERSION in 8 9 10 11 12; do maturin build --release --no-default-features --features rustls --interpreter "python3.$VERSION" --compatibility manylinux2014 done -fi + for VERSION in 9 10; do + maturin build --release --no-default-features --features rustls --interpreter "pypy3.$VERSION" --compatibility manylinux2014 + done +fi \ No newline at end of file diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d0e45d20..236764bf 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -268,6 +268,26 @@ jobs: - run: python -m unittest working-directory: ./python/tests + python_pypy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + submodules: true + - run: rustup update && rustup toolchain install nightly && rustup default 1.70.0 + - uses: Swatinem/rust-cache@v2 + - uses: actions/setup-python@v4 + with: + python-version: "pypy3.10" + cache: pip + cache-dependency-path: '**/requirements.dev.txt' + - run: pip install -r python/requirements.dev.txt + - run: maturin build -m python/Cargo.toml + - run: pip install --no-index --find-links=target/wheels/ pyoxigraph + - run: rm -r target/wheels + - run: python -m unittest + working-directory: ./python/tests + python_windows: runs-on: windows-latest steps: diff --git a/python/tests/test_store.py b/python/tests/test_store.py index 4f47b5f0..9b662ad6 100644 --- a/python/tests/test_store.py +++ b/python/tests/test_store.py @@ -1,3 +1,4 @@ +import gc import unittest from io import BytesIO, StringIO, UnsupportedOperation from pathlib import Path @@ -386,6 +387,7 @@ class TestStore(unittest.TestCase): store = Store(dir) store.add(quad) del store + gc.collect() store = Store.read_only(dir) self.assertEqual(list(store), [quad]) From 99c3a4cce40c9bb4c256cbd4e68a6bcaed4e82df Mon Sep 17 00:00:00 2001 From: Tpt Date: Tue, 21 Nov 2023 21:20:13 +0100 Subject: [PATCH 127/217] CI: Adds a test with latest dependency versions --- .github/workflows/tests.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 236764bf..139e3172 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -128,6 +128,17 @@ jobs: - run: rm Cargo.lock && cargo +nightly update -Z direct-minimal-versions - run: cargo test + test_linux_latest: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + submodules: true + - run: rustup update + - uses: Swatinem/rust-cache@v2 + - run: rm Cargo.lock && cargo update + - run: cargo test + test_linux_address_sanitizer: runs-on: ubuntu-latest steps: From d1cb4cecbd44a0e23e608462a168ebe50f707c53 Mon Sep 17 00:00:00 2001 From: Tpt Date: Thu, 23 Nov 2023 09:03:53 +0100 Subject: [PATCH 128/217] OxRDF: makes more function const --- lib/oxrdf/src/blank_node.rs | 4 ++-- lib/oxrdf/src/literal.rs | 14 +++++++------- lib/oxrdf/src/variable.rs | 4 ++-- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/lib/oxrdf/src/blank_node.rs b/lib/oxrdf/src/blank_node.rs index 937b1efb..e813dd24 100644 --- a/lib/oxrdf/src/blank_node.rs +++ b/lib/oxrdf/src/blank_node.rs @@ -180,7 +180,7 @@ impl<'a> BlankNodeRef<'a> { /// Returns the underlying ID of this blank node. #[inline] - pub fn as_str(self) -> &'a str { + pub const fn as_str(self) -> &'a str { match self.0 { BlankNodeRefContent::Named(id) => id, BlankNodeRefContent::Anonymous { str, .. } => str, @@ -197,7 +197,7 @@ impl<'a> BlankNodeRef<'a> { /// # Result::<_,oxrdf::BlankNodeIdParseError>::Ok(()) /// ``` #[inline] - pub fn unique_id(&self) -> Option { + pub const fn unique_id(&self) -> Option { match self.0 { BlankNodeRefContent::Named(_) => None, BlankNodeRefContent::Anonymous { id, .. } => Some(id), diff --git a/lib/oxrdf/src/literal.rs b/lib/oxrdf/src/literal.rs index 809f10e4..976b863d 100644 --- a/lib/oxrdf/src/literal.rs +++ b/lib/oxrdf/src/literal.rs @@ -459,7 +459,7 @@ enum LiteralRefContent<'a> { impl<'a> LiteralRef<'a> { /// Builds an RDF [simple literal](https://www.w3.org/TR/rdf11-concepts/#dfn-simple-literal). #[inline] - pub fn new_simple_literal(value: &'a str) -> Self { + pub const fn new_simple_literal(value: &'a str) -> Self { LiteralRef(LiteralRefContent::String(value)) } @@ -482,13 +482,13 @@ impl<'a> LiteralRef<'a> { /// /// [`Literal::new_language_tagged_literal()`] is a safe version of this constructor and should be used for untrusted data. #[inline] - pub fn new_language_tagged_literal_unchecked(value: &'a str, language: &'a str) -> Self { + pub const fn new_language_tagged_literal_unchecked(value: &'a str, language: &'a str) -> Self { LiteralRef(LiteralRefContent::LanguageTaggedString { value, language }) } /// The literal [lexical form](https://www.w3.org/TR/rdf11-concepts/#dfn-lexical-form) #[inline] - pub fn value(self) -> &'a str { + pub const fn value(self) -> &'a str { match self.0 { LiteralRefContent::String(value) | LiteralRefContent::LanguageTaggedString { value, .. } @@ -501,7 +501,7 @@ impl<'a> LiteralRef<'a> { /// Language tags are defined by the [BCP47](https://tools.ietf.org/html/bcp47). /// They are normalized to lowercase by this implementation. #[inline] - pub fn language(self) -> Option<&'a str> { + pub const fn language(self) -> Option<&'a str> { match self.0 { LiteralRefContent::LanguageTaggedString { language, .. } => Some(language), _ => None, @@ -513,7 +513,7 @@ impl<'a> LiteralRef<'a> { /// The datatype of [language-tagged string](https://www.w3.org/TR/rdf11-concepts/#dfn-language-tagged-string) is always [rdf:langString](https://www.w3.org/TR/rdf11-concepts/#dfn-language-tagged-string). /// The datatype of [simple literals](https://www.w3.org/TR/rdf11-concepts/#dfn-simple-literal) is [xsd:string](https://www.w3.org/TR/xmlschema11-2/#string). #[inline] - pub fn datatype(self) -> NamedNodeRef<'a> { + pub const fn datatype(self) -> NamedNodeRef<'a> { match self.0 { LiteralRefContent::String(_) => xsd::STRING, LiteralRefContent::LanguageTaggedString { .. } => rdf::LANG_STRING, @@ -526,7 +526,7 @@ impl<'a> LiteralRef<'a> { /// It returns true if the literal is a [language-tagged string](https://www.w3.org/TR/rdf11-concepts/#dfn-language-tagged-string) /// or has the datatype [xsd:string](https://www.w3.org/TR/xmlschema11-2/#string). #[inline] - pub fn is_plain(self) -> bool { + pub const fn is_plain(self) -> bool { matches!( self.0, LiteralRefContent::String(_) | LiteralRefContent::LanguageTaggedString { .. } @@ -552,7 +552,7 @@ impl<'a> LiteralRef<'a> { /// Extract components from this literal #[inline] - pub fn destruct(self) -> (&'a str, Option>, Option<&'a str>) { + pub const fn destruct(self) -> (&'a str, Option>, Option<&'a str>) { match self.0 { LiteralRefContent::String(s) => (s, None, None), LiteralRefContent::LanguageTaggedString { value, language } => { diff --git a/lib/oxrdf/src/variable.rs b/lib/oxrdf/src/variable.rs index 7b9cd732..8bde4d6e 100644 --- a/lib/oxrdf/src/variable.rs +++ b/lib/oxrdf/src/variable.rs @@ -96,12 +96,12 @@ impl<'a> VariableRef<'a> { /// /// [`Variable::new()`] is a safe version of this constructor and should be used for untrusted data. #[inline] - pub fn new_unchecked(name: &'a str) -> Self { + pub const fn new_unchecked(name: &'a str) -> Self { Self { name } } #[inline] - pub fn as_str(&self) -> &str { + pub const fn as_str(self) -> &'a str { self.name } From 756c5394d053cf71d46c33d0fa4b84b2ef53fc95 Mon Sep 17 00:00:00 2001 From: Tpt Date: Thu, 23 Nov 2023 08:56:42 +0100 Subject: [PATCH 129/217] Adds Tokio support to SPARQL results serializers --- Cargo.lock | 4 + lib/sparesults/Cargo.toml | 5 + lib/sparesults/src/csv.rs | 403 +++++++++++++++++++++---------- lib/sparesults/src/json.rs | 246 +++++++++++++------ lib/sparesults/src/serializer.rs | 239 +++++++++++++++--- lib/sparesults/src/xml.rs | 269 +++++++++++++-------- 6 files changed, 828 insertions(+), 338 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 85ad265f..fc64ec63 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -780,6 +780,9 @@ name = "json-event-parser" version = "0.2.0-alpha.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b5ddd02379e99769e117ab30d21ad42dcec8ad3c12be77f9a34779e62d46346" +dependencies = [ + "tokio", +] [[package]] name = "kernel32-sys" @@ -1731,6 +1734,7 @@ dependencies = [ "memchr", "oxrdf", "quick-xml", + "tokio", ] [[package]] diff --git a/lib/sparesults/Cargo.toml b/lib/sparesults/Cargo.toml index 67d75cf4..fcc95b39 100644 --- a/lib/sparesults/Cargo.toml +++ b/lib/sparesults/Cargo.toml @@ -17,12 +17,17 @@ rust-version.workspace = true [features] default = [] rdf-star = ["oxrdf/rdf-star"] +async-tokio = ["dep:tokio", "quick-xml/async-tokio", "json-event-parser/async-tokio"] [dependencies] json-event-parser = "0.2.0-alpha.2" memchr = "2.5" oxrdf = { version = "0.2.0-alpha.1-dev", path="../oxrdf" } quick-xml = ">=0.29, <0.32" +tokio = { version = "1.29", optional = true, features = ["io-util"] } + +[dev-dependencies] +tokio = { version = "1.29", features = ["rt", "macros"] } [package.metadata.docs.rs] all-features = true diff --git a/lib/sparesults/src/csv.rs b/lib/sparesults/src/csv.rs index b0dc99f1..ac02c99e 100644 --- a/lib/sparesults/src/csv.rs +++ b/lib/sparesults/src/csv.rs @@ -6,38 +6,121 @@ use oxrdf::Variable; use oxrdf::{vocab::xsd, *}; use std::io::{self, Read, Write}; use std::str::{self, FromStr}; +#[cfg(feature = "async-tokio")] +use tokio::io::{AsyncWrite, AsyncWriteExt}; const MAX_BUFFER_SIZE: usize = 4096 * 4096; -pub fn write_boolean_csv_result(mut sink: W, value: bool) -> io::Result { - sink.write_all(if value { b"true" } else { b"false" })?; - Ok(sink) +pub fn write_boolean_csv_result(mut write: W, value: bool) -> io::Result { + write.write_all(if value { b"true" } else { b"false" })?; + Ok(write) } -pub struct CsvSolutionsWriter { - sink: W, +#[cfg(feature = "async-tokio")] +pub async fn tokio_async_write_boolean_csv_result( + mut write: W, + value: bool, +) -> io::Result { + write + .write_all(if value { b"true" } else { b"false" }) + .await?; + Ok(write) +} + +pub struct ToWriteCsvSolutionsWriter { + inner: InnerCsvSolutionsWriter, + write: W, + buffer: String, +} + +impl ToWriteCsvSolutionsWriter { + pub fn start(mut write: W, variables: Vec) -> io::Result { + let mut buffer = String::new(); + let inner = InnerCsvSolutionsWriter::start(&mut buffer, variables); + write.write_all(buffer.as_bytes())?; + buffer.clear(); + Ok(Self { + inner, + write, + buffer, + }) + } + + pub fn write<'a>( + &mut self, + solution: impl IntoIterator, TermRef<'a>)>, + ) -> io::Result<()> { + self.inner.write(&mut self.buffer, solution); + self.write.write_all(self.buffer.as_bytes())?; + self.buffer.clear(); + Ok(()) + } + + pub fn finish(self) -> W { + self.write + } +} + +#[cfg(feature = "async-tokio")] +pub struct ToTokioAsyncWriteCsvSolutionsWriter { + inner: InnerCsvSolutionsWriter, + write: W, + buffer: String, +} + +#[cfg(feature = "async-tokio")] +impl ToTokioAsyncWriteCsvSolutionsWriter { + pub async fn start(mut write: W, variables: Vec) -> io::Result { + let mut buffer = String::new(); + let inner = InnerCsvSolutionsWriter::start(&mut buffer, variables); + write.write_all(buffer.as_bytes()).await?; + buffer.clear(); + Ok(Self { + inner, + write, + buffer, + }) + } + + pub async fn write<'a>( + &mut self, + solution: impl IntoIterator, TermRef<'a>)>, + ) -> io::Result<()> { + self.inner.write(&mut self.buffer, solution); + self.write.write_all(self.buffer.as_bytes()).await?; + self.buffer.clear(); + Ok(()) + } + + pub fn finish(self) -> W { + self.write + } +} + +struct InnerCsvSolutionsWriter { variables: Vec, } -impl CsvSolutionsWriter { - pub fn start(mut sink: W, variables: Vec) -> io::Result { +impl InnerCsvSolutionsWriter { + fn start(output: &mut String, variables: Vec) -> Self { let mut start_vars = true; for variable in &variables { if start_vars { start_vars = false; } else { - sink.write_all(b",")?; + output.push(','); } - sink.write_all(variable.as_str().as_bytes())?; + output.push_str(variable.as_str()); } - sink.write_all(b"\r\n")?; - Ok(Self { sink, variables }) + output.push_str("\r\n"); + Self { variables } } - pub fn write<'a>( - &mut self, + fn write<'a>( + &self, + output: &mut String, solution: impl IntoIterator, TermRef<'a>)>, - ) -> io::Result<()> { + ) { let mut values = vec![None; self.variables.len()]; for (variable, value) in solution { if let Some(position) = self.variables.iter().position(|v| *v == variable) { @@ -49,85 +132,147 @@ impl CsvSolutionsWriter { if start_binding { start_binding = false; } else { - self.sink.write_all(b",")?; + output.push(','); } if let Some(value) = value { - write_csv_term(value, &mut self.sink)?; + write_csv_term(output, value); } } - self.sink.write_all(b"\r\n") - } - - pub fn finish(self) -> W { - self.sink + output.push_str("\r\n"); } } -fn write_csv_term<'a>(term: impl Into>, sink: &mut impl Write) -> io::Result<()> { +fn write_csv_term<'a>(output: &mut String, term: impl Into>) { match term.into() { - TermRef::NamedNode(uri) => sink.write_all(uri.as_str().as_bytes()), + TermRef::NamedNode(uri) => output.push_str(uri.as_str()), TermRef::BlankNode(bnode) => { - sink.write_all(b"_:")?; - sink.write_all(bnode.as_str().as_bytes()) + output.push_str("_:"); + output.push_str(bnode.as_str()) } - TermRef::Literal(literal) => write_escaped_csv_string(literal.value(), sink), + TermRef::Literal(literal) => write_escaped_csv_string(output, literal.value()), #[cfg(feature = "rdf-star")] TermRef::Triple(triple) => { - write_csv_term(&triple.subject, sink)?; - sink.write_all(b" ")?; - write_csv_term(&triple.predicate, sink)?; - sink.write_all(b" ")?; - write_csv_term(&triple.object, sink) + write_csv_term(output, &triple.subject); + output.push(' '); + write_csv_term(output, &triple.predicate); + output.push(' '); + write_csv_term(output, &triple.object) } } } -fn write_escaped_csv_string(s: &str, sink: &mut impl Write) -> io::Result<()> { +fn write_escaped_csv_string(output: &mut String, s: &str) { if s.bytes().any(|c| matches!(c, b'"' | b',' | b'\n' | b'\r')) { - sink.write_all(b"\"")?; - for c in s.bytes() { - if c == b'\"' { - sink.write_all(b"\"\"") + output.push('"'); + for c in s.chars() { + if c == '"' { + output.push('"'); + output.push('"'); } else { - sink.write_all(&[c]) - }?; + output.push(c) + }; } - sink.write_all(b"\"") + output.push('"'); } else { - sink.write_all(s.as_bytes()) + output.push_str(s) } } -pub fn write_boolean_tsv_result(mut sink: W, value: bool) -> io::Result { - sink.write_all(if value { b"true" } else { b"false" })?; - Ok(sink) +pub struct ToWriteTsvSolutionsWriter { + inner: InnerTsvSolutionsWriter, + write: W, + buffer: String, } -pub struct TsvSolutionsWriter { - sink: W, +impl ToWriteTsvSolutionsWriter { + pub fn start(mut write: W, variables: Vec) -> io::Result { + let mut buffer = String::new(); + let inner = InnerTsvSolutionsWriter::start(&mut buffer, variables); + write.write_all(buffer.as_bytes())?; + buffer.clear(); + Ok(Self { + inner, + write, + buffer, + }) + } + + pub fn write<'a>( + &mut self, + solution: impl IntoIterator, TermRef<'a>)>, + ) -> io::Result<()> { + self.inner.write(&mut self.buffer, solution); + self.write.write_all(self.buffer.as_bytes())?; + self.buffer.clear(); + Ok(()) + } + + pub fn finish(self) -> W { + self.write + } +} + +#[cfg(feature = "async-tokio")] +pub struct ToTokioAsyncWriteTsvSolutionsWriter { + inner: InnerTsvSolutionsWriter, + write: W, + buffer: String, +} + +#[cfg(feature = "async-tokio")] +impl ToTokioAsyncWriteTsvSolutionsWriter { + pub async fn start(mut write: W, variables: Vec) -> io::Result { + let mut buffer = String::new(); + let inner = InnerTsvSolutionsWriter::start(&mut buffer, variables); + write.write_all(buffer.as_bytes()).await?; + buffer.clear(); + Ok(Self { + inner, + write, + buffer, + }) + } + + pub async fn write<'a>( + &mut self, + solution: impl IntoIterator, TermRef<'a>)>, + ) -> io::Result<()> { + self.inner.write(&mut self.buffer, solution); + self.write.write_all(self.buffer.as_bytes()).await?; + self.buffer.clear(); + Ok(()) + } + + pub fn finish(self) -> W { + self.write + } +} + +struct InnerTsvSolutionsWriter { variables: Vec, } -impl TsvSolutionsWriter { - pub fn start(mut sink: W, variables: Vec) -> io::Result { +impl InnerTsvSolutionsWriter { + fn start(output: &mut String, variables: Vec) -> Self { let mut start_vars = true; for variable in &variables { if start_vars { start_vars = false; } else { - sink.write_all(b"\t")?; + output.push('\t'); } - sink.write_all(b"?")?; - sink.write_all(variable.as_str().as_bytes())?; + output.push('?'); + output.push_str(variable.as_str()); } - sink.write_all(b"\n")?; - Ok(Self { sink, variables }) + output.push('\n'); + Self { variables } } - pub fn write<'a>( - &mut self, + fn write<'a>( + &self, + output: &mut String, solution: impl IntoIterator, TermRef<'a>)>, - ) -> io::Result<()> { + ) { let mut values = vec![None; self.variables.len()]; for (variable, value) in solution { if let Some(position) = self.variables.iter().position(|v| *v == variable) { @@ -139,70 +284,74 @@ impl TsvSolutionsWriter { if start_binding { start_binding = false; } else { - self.sink.write_all(b"\t")?; + output.push('\t'); } if let Some(value) = value { - write_tsv_term(value, &mut self.sink)?; + write_tsv_term(output, value); } } - self.sink.write_all(b"\n") - } - - pub fn finish(self) -> W { - self.sink + output.push('\n'); } } -fn write_tsv_term<'a>(term: impl Into>, sink: &mut impl Write) -> io::Result<()> { +fn write_tsv_term<'a>(output: &mut String, term: impl Into>) { match term.into() { - TermRef::NamedNode(node) => write!(sink, "<{}>", node.as_str()), - TermRef::BlankNode(node) => write!(sink, "_:{}", node.as_str()), + TermRef::NamedNode(node) => { + output.push('<'); + output.push_str(node.as_str()); + output.push('>'); + } + TermRef::BlankNode(node) => { + output.push_str("_:"); + output.push_str(node.as_str()); + } TermRef::Literal(literal) => { let value = literal.value(); if let Some(language) = literal.language() { - write_tsv_quoted_str(value, sink)?; - write!(sink, "@{language}") + write_tsv_quoted_str(output, value); + output.push('@'); + output.push_str(language); } else { match literal.datatype() { - xsd::BOOLEAN if is_turtle_boolean(value) => sink.write_all(value.as_bytes()), - xsd::INTEGER if is_turtle_integer(value) => sink.write_all(value.as_bytes()), - xsd::DECIMAL if is_turtle_decimal(value) => sink.write_all(value.as_bytes()), - xsd::DOUBLE if is_turtle_double(value) => sink.write_all(value.as_bytes()), - xsd::STRING => write_tsv_quoted_str(value, sink), + xsd::BOOLEAN if is_turtle_boolean(value) => output.push_str(value), + xsd::INTEGER if is_turtle_integer(value) => output.push_str(value), + xsd::DECIMAL if is_turtle_decimal(value) => output.push_str(value), + xsd::DOUBLE if is_turtle_double(value) => output.push_str(value), + xsd::STRING => write_tsv_quoted_str(output, value), datatype => { - write_tsv_quoted_str(value, sink)?; - write!(sink, "^^<{}>", datatype.as_str()) + write_tsv_quoted_str(output, value); + output.push_str("^^"); + write_tsv_term(output, datatype); } } } } #[cfg(feature = "rdf-star")] TermRef::Triple(triple) => { - sink.write_all(b"<< ")?; - write_tsv_term(&triple.subject, sink)?; - sink.write_all(b" ")?; - write_tsv_term(&triple.predicate, sink)?; - sink.write_all(b" ")?; - write_tsv_term(&triple.object, sink)?; - sink.write_all(b" >>")?; - Ok(()) + output.push_str("<< "); + write_tsv_term(output, &triple.subject); + output.push(' '); + write_tsv_term(output, &triple.predicate); + output.push(' '); + write_tsv_term(output, &triple.object); + output.push_str(" >>"); } } } -fn write_tsv_quoted_str(string: &str, f: &mut impl Write) -> io::Result<()> { - f.write_all(b"\"")?; - for c in string.bytes() { +fn write_tsv_quoted_str(output: &mut String, string: &str) { + output.push('"'); + for c in string.chars() { match c { - b'\t' => f.write_all(b"\\t"), - b'\n' => f.write_all(b"\\n"), - b'\r' => f.write_all(b"\\r"), - b'"' => f.write_all(b"\\\""), - b'\\' => f.write_all(b"\\\\"), - _ => f.write_all(&[c]), - }?; - } - f.write_all(b"\"") + '\t' => output.push_str("\\t"), + '\n' => output.push_str("\\n"), + '\r' => output.push_str("\\r"), + '"' => output.push_str("\\\""), + '\\' => output.push_str("\\\\"), + _ => output.push(c), + }; + } + output.push('"'); } fn is_turtle_boolean(value: &str) -> bool { @@ -439,7 +588,7 @@ impl LineReader { if let Some(eol) = memchr(b'\n', &buffer[self.buffer_start..self.buffer_end]) { break self.buffer_start + eol + 1; } - if self.buffer_start > buffer.len() / 2 { + if self.buffer_start > 0 { buffer.copy_within(self.buffer_start..self.buffer_end, 0); self.buffer_end -= self.buffer_start; self.buffer_start = 0; @@ -480,7 +629,6 @@ mod tests { use super::*; use std::error::Error; use std::rc::Rc; - use std::str; fn build_example() -> (Vec, Vec>>) { ( @@ -530,21 +678,21 @@ mod tests { } #[test] - fn test_csv_serialization() -> io::Result<()> { + fn test_csv_serialization() { let (variables, solutions) = build_example(); - let mut writer = CsvSolutionsWriter::start(Vec::new(), variables.clone())?; + let mut buffer = String::new(); + let writer = InnerCsvSolutionsWriter::start(&mut buffer, variables.clone()); let variables = Rc::new(variables); for solution in solutions { writer.write( + &mut buffer, variables .iter() .zip(&solution) .filter_map(|(v, s)| s.as_ref().map(|s| (v.as_ref(), s.as_ref()))), - )?; + ); } - let result = writer.finish(); - assert_eq!(str::from_utf8(&result).unwrap(), "x,literal\r\nhttp://example/x,String\r\nhttp://example/x,\"String-with-dquote\"\"\"\r\n_:b0,Blank node\r\n,Missing 'x'\r\n,\r\nhttp://example/x,\r\n_:b1,String-with-lang\r\n_:b1,123\r\n,\"escape,\t\r\n\"\r\n"); - Ok(()) + assert_eq!(buffer, "x,literal\r\nhttp://example/x,String\r\nhttp://example/x,\"String-with-dquote\"\"\"\r\n_:b0,Blank node\r\n,Missing 'x'\r\n,\r\nhttp://example/x,\r\n_:b1,String-with-lang\r\n_:b1,123\r\n,\"escape,\t\r\n\"\r\n"); } #[test] @@ -552,24 +700,25 @@ mod tests { let (variables, solutions) = build_example(); // Write - let mut writer = TsvSolutionsWriter::start(Vec::new(), variables.clone())?; + let mut buffer = String::new(); + let writer = InnerTsvSolutionsWriter::start(&mut buffer, variables.clone()); let variables = Rc::new(variables); for solution in &solutions { writer.write( + &mut buffer, variables .iter() .zip(solution) .filter_map(|(v, s)| s.as_ref().map(|s| (v.as_ref(), s.as_ref()))), - )?; + ); } - let result = writer.finish(); - assert_eq!(str::from_utf8(&result).unwrap(), "?x\t?literal\n\t\"String\"\n\t\"String-with-dquote\\\"\"\n_:b0\t\"Blank node\"\n\t\"Missing 'x'\"\n\t\n\t\n_:b1\t\"String-with-lang\"@en\n_:b1\t123\n\t\"escape,\\t\\r\\n\"\n"); + assert_eq!(buffer, "?x\t?literal\n\t\"String\"\n\t\"String-with-dquote\\\"\"\n_:b0\t\"Blank node\"\n\t\"Missing 'x'\"\n\t\n\t\n_:b1\t\"String-with-lang\"@en\n_:b1\t123\n\t\"escape,\\t\\r\\n\"\n"); // Read if let TsvQueryResultsReader::Solutions { solutions: mut solutions_iter, variables: actual_variables, - } = TsvQueryResultsReader::read(result.as_slice())? + } = TsvQueryResultsReader::read(buffer.as_bytes())? { assert_eq!(actual_variables.as_slice(), variables.as_slice()); let mut rows = Vec::new(); @@ -610,21 +759,19 @@ mod tests { } #[test] - fn test_no_columns_csv_serialization() -> io::Result<()> { - let mut writer = CsvSolutionsWriter::start(Vec::new(), Vec::new())?; - writer.write([])?; - let result = writer.finish(); - assert_eq!(str::from_utf8(&result).unwrap(), "\r\n\r\n"); - Ok(()) + fn test_no_columns_csv_serialization() { + let mut buffer = String::new(); + let writer = InnerCsvSolutionsWriter::start(&mut buffer, Vec::new()); + writer.write(&mut buffer, []); + assert_eq!(buffer, "\r\n\r\n"); } #[test] - fn test_no_columns_tsv_serialization() -> io::Result<()> { - let mut writer = TsvSolutionsWriter::start(Vec::new(), Vec::new())?; - writer.write([])?; - let result = writer.finish(); - assert_eq!(str::from_utf8(&result).unwrap(), "\n\n"); - Ok(()) + fn test_no_columns_tsv_serialization() { + let mut buffer = String::new(); + let writer = InnerTsvSolutionsWriter::start(&mut buffer, Vec::new()); + writer.write(&mut buffer, []); + assert_eq!(buffer, "\n\n"); } #[test] @@ -644,19 +791,17 @@ mod tests { } #[test] - fn test_no_results_csv_serialization() -> io::Result<()> { - let result = - CsvSolutionsWriter::start(Vec::new(), vec![Variable::new_unchecked("a")])?.finish(); - assert_eq!(str::from_utf8(&result).unwrap(), "a\r\n"); - Ok(()) + fn test_no_results_csv_serialization() { + let mut buffer = String::new(); + InnerCsvSolutionsWriter::start(&mut buffer, vec![Variable::new_unchecked("a")]); + assert_eq!(buffer, "a\r\n"); } #[test] - fn test_no_results_tsv_serialization() -> io::Result<()> { - let result = - TsvSolutionsWriter::start(Vec::new(), vec![Variable::new_unchecked("a")])?.finish(); - assert_eq!(str::from_utf8(&result).unwrap(), "?a\n"); - Ok(()) + fn test_no_results_tsv_serialization() { + let mut buffer = String::new(); + InnerTsvSolutionsWriter::start(&mut buffer, vec![Variable::new_unchecked("a")]); + assert_eq!(buffer, "?a\n"); } #[test] diff --git a/lib/sparesults/src/json.rs b/lib/sparesults/src/json.rs index 3dc9be11..cbaa873c 100644 --- a/lib/sparesults/src/json.rs +++ b/lib/sparesults/src/json.rs @@ -1,6 +1,8 @@ //! Implementation of [SPARQL Query Results JSON Format](https://www.w3.org/TR/sparql11-results-json/) use crate::error::{ParseError, SyntaxError}; +#[cfg(feature = "async-tokio")] +use json_event_parser::ToTokioAsyncWriteJsonWriter; use json_event_parser::{FromReadJsonReader, JsonEvent, ToWriteJsonWriter}; use oxrdf::vocab::rdf; use oxrdf::Variable; @@ -8,6 +10,8 @@ use oxrdf::*; use std::collections::BTreeMap; use std::io::{self, Read, Write}; use std::mem::take; +#[cfg(feature = "async-tokio")] +use tokio::io::AsyncWrite; /// This limit is set in order to avoid stack overflow error when parsing nested triples due to too many recursive calls. /// The actual limit value is a wet finger compromise between not failing to parse valid files and avoiding to trigger stack overflow errors. @@ -15,116 +19,210 @@ const MAX_NUMBER_OF_NESTED_TRIPLES: usize = 128; pub fn write_boolean_json_result(write: W, value: bool) -> io::Result { let mut writer = ToWriteJsonWriter::new(write); - writer.write_event(JsonEvent::StartObject)?; - writer.write_event(JsonEvent::ObjectKey("head".into()))?; - writer.write_event(JsonEvent::StartObject)?; - writer.write_event(JsonEvent::EndObject)?; - writer.write_event(JsonEvent::ObjectKey("boolean".into()))?; - writer.write_event(JsonEvent::Boolean(value))?; - writer.write_event(JsonEvent::EndObject)?; + for event in inner_write_boolean_json_result(value) { + writer.write_event(event)?; + } + writer.finish() +} + +#[cfg(feature = "async-tokio")] +pub async fn tokio_async_write_boolean_json_result( + write: W, + value: bool, +) -> io::Result { + let mut writer = ToTokioAsyncWriteJsonWriter::new(write); + for event in inner_write_boolean_json_result(value) { + writer.write_event(event).await?; + } writer.finish() } -pub struct JsonSolutionsWriter { +fn inner_write_boolean_json_result(value: bool) -> [JsonEvent<'static>; 7] { + [ + JsonEvent::StartObject, + JsonEvent::ObjectKey("head".into()), + JsonEvent::StartObject, + JsonEvent::EndObject, + JsonEvent::ObjectKey("boolean".into()), + JsonEvent::Boolean(value), + JsonEvent::EndObject, + ] +} + +pub struct ToWriteJsonSolutionsWriter { + inner: InnerJsonSolutionsWriter, writer: ToWriteJsonWriter, } -impl JsonSolutionsWriter { +impl ToWriteJsonSolutionsWriter { pub fn start(write: W, variables: &[Variable]) -> io::Result { let mut writer = ToWriteJsonWriter::new(write); - writer.write_event(JsonEvent::StartObject)?; - writer.write_event(JsonEvent::ObjectKey("head".into()))?; - writer.write_event(JsonEvent::StartObject)?; - writer.write_event(JsonEvent::ObjectKey("vars".into()))?; - writer.write_event(JsonEvent::StartArray)?; - for variable in variables { - writer.write_event(JsonEvent::String(variable.as_str().into()))?; - } - writer.write_event(JsonEvent::EndArray)?; - writer.write_event(JsonEvent::EndObject)?; - writer.write_event(JsonEvent::ObjectKey("results".into()))?; - writer.write_event(JsonEvent::StartObject)?; - writer.write_event(JsonEvent::ObjectKey("bindings".into()))?; - writer.write_event(JsonEvent::StartArray)?; - Ok(Self { writer }) + let mut buffer = Vec::with_capacity(48); + let inner = InnerJsonSolutionsWriter::start(&mut buffer, variables); + Self::do_write(&mut writer, buffer)?; + Ok(Self { inner, writer }) } pub fn write<'a>( &mut self, solution: impl IntoIterator, TermRef<'a>)>, ) -> io::Result<()> { - self.writer.write_event(JsonEvent::StartObject)?; - for (variable, value) in solution { - self.writer - .write_event(JsonEvent::ObjectKey(variable.as_str().into()))?; - write_json_term(value, &mut self.writer)?; + let mut buffer = Vec::with_capacity(48); + self.inner.write(&mut buffer, solution); + Self::do_write(&mut self.writer, buffer) + } + + pub fn finish(mut self) -> io::Result { + let mut buffer = Vec::with_capacity(4); + self.inner.finish(&mut buffer); + Self::do_write(&mut self.writer, buffer)?; + self.writer.finish() + } + + fn do_write(writer: &mut ToWriteJsonWriter, output: Vec>) -> io::Result<()> { + for event in output { + writer.write_event(event)?; } - self.writer.write_event(JsonEvent::EndObject)?; Ok(()) } +} - pub fn finish(mut self) -> io::Result { - self.writer.write_event(JsonEvent::EndArray)?; - self.writer.write_event(JsonEvent::EndObject)?; - self.writer.write_event(JsonEvent::EndObject)?; +#[cfg(feature = "async-tokio")] +pub struct ToTokioAsyncWriteJsonSolutionsWriter { + inner: InnerJsonSolutionsWriter, + writer: ToTokioAsyncWriteJsonWriter, +} + +#[cfg(feature = "async-tokio")] +impl ToTokioAsyncWriteJsonSolutionsWriter { + pub async fn start(write: W, variables: &[Variable]) -> io::Result { + let mut writer = ToTokioAsyncWriteJsonWriter::new(write); + let mut buffer = Vec::with_capacity(48); + let inner = InnerJsonSolutionsWriter::start(&mut buffer, variables); + Self::do_write(&mut writer, buffer).await?; + Ok(Self { writer, inner }) + } + + pub async fn write<'a>( + &mut self, + solution: impl IntoIterator, TermRef<'a>)>, + ) -> io::Result<()> { + let mut buffer = Vec::with_capacity(48); + self.inner.write(&mut buffer, solution); + Self::do_write(&mut self.writer, buffer).await + } + + pub async fn finish(mut self) -> io::Result { + let mut buffer = Vec::with_capacity(4); + self.inner.finish(&mut buffer); + Self::do_write(&mut self.writer, buffer).await?; self.writer.finish() } + + async fn do_write( + writer: &mut ToTokioAsyncWriteJsonWriter, + output: Vec>, + ) -> io::Result<()> { + for event in output { + writer.write_event(event).await?; + } + Ok(()) + } +} + +struct InnerJsonSolutionsWriter; + +impl InnerJsonSolutionsWriter { + fn start<'a>(output: &mut Vec>, variables: &'a [Variable]) -> Self { + output.push(JsonEvent::StartObject); + output.push(JsonEvent::ObjectKey("head".into())); + output.push(JsonEvent::StartObject); + output.push(JsonEvent::ObjectKey("vars".into())); + output.push(JsonEvent::StartArray); + for variable in variables { + output.push(JsonEvent::String(variable.as_str().into())); + } + output.push(JsonEvent::EndArray); + output.push(JsonEvent::EndObject); + output.push(JsonEvent::ObjectKey("results".into())); + output.push(JsonEvent::StartObject); + output.push(JsonEvent::ObjectKey("bindings".into())); + output.push(JsonEvent::StartArray); + Self {} + } + + #[allow(clippy::unused_self)] + fn write<'a>( + &self, + output: &mut Vec>, + solution: impl IntoIterator, TermRef<'a>)>, + ) { + output.push(JsonEvent::StartObject); + for (variable, value) in solution { + output.push(JsonEvent::ObjectKey(variable.as_str().into())); + write_json_term(output, value); + } + output.push(JsonEvent::EndObject); + } + + #[allow(clippy::unused_self)] + fn finish(self, output: &mut Vec>) { + output.push(JsonEvent::EndArray); + output.push(JsonEvent::EndObject); + output.push(JsonEvent::EndObject); + } } -fn write_json_term( - term: TermRef<'_>, - writer: &mut ToWriteJsonWriter, -) -> io::Result<()> { +fn write_json_term<'a>(output: &mut Vec>, term: TermRef<'a>) { match term { TermRef::NamedNode(uri) => { - writer.write_event(JsonEvent::StartObject)?; - writer.write_event(JsonEvent::ObjectKey("type".into()))?; - writer.write_event(JsonEvent::String("uri".into()))?; - writer.write_event(JsonEvent::ObjectKey("value".into()))?; - writer.write_event(JsonEvent::String(uri.as_str().into()))?; - writer.write_event(JsonEvent::EndObject)?; + output.push(JsonEvent::StartObject); + output.push(JsonEvent::ObjectKey("type".into())); + output.push(JsonEvent::String("uri".into())); + output.push(JsonEvent::ObjectKey("value".into())); + output.push(JsonEvent::String(uri.as_str().into())); + output.push(JsonEvent::EndObject); } TermRef::BlankNode(bnode) => { - writer.write_event(JsonEvent::StartObject)?; - writer.write_event(JsonEvent::ObjectKey("type".into()))?; - writer.write_event(JsonEvent::String("bnode".into()))?; - writer.write_event(JsonEvent::ObjectKey("value".into()))?; - writer.write_event(JsonEvent::String(bnode.as_str().into()))?; - writer.write_event(JsonEvent::EndObject)?; + output.push(JsonEvent::StartObject); + output.push(JsonEvent::ObjectKey("type".into())); + output.push(JsonEvent::String("bnode".into())); + output.push(JsonEvent::ObjectKey("value".into())); + output.push(JsonEvent::String(bnode.as_str().into())); + output.push(JsonEvent::EndObject); } TermRef::Literal(literal) => { - writer.write_event(JsonEvent::StartObject)?; - writer.write_event(JsonEvent::ObjectKey("type".into()))?; - writer.write_event(JsonEvent::String("literal".into()))?; - writer.write_event(JsonEvent::ObjectKey("value".into()))?; - writer.write_event(JsonEvent::String(literal.value().into()))?; + output.push(JsonEvent::StartObject); + output.push(JsonEvent::ObjectKey("type".into())); + output.push(JsonEvent::String("literal".into())); + output.push(JsonEvent::ObjectKey("value".into())); + output.push(JsonEvent::String(literal.value().into())); if let Some(language) = literal.language() { - writer.write_event(JsonEvent::ObjectKey("xml:lang".into()))?; - writer.write_event(JsonEvent::String(language.into()))?; + output.push(JsonEvent::ObjectKey("xml:lang".into())); + output.push(JsonEvent::String(language.into())); } else if !literal.is_plain() { - writer.write_event(JsonEvent::ObjectKey("datatype".into()))?; - writer.write_event(JsonEvent::String(literal.datatype().as_str().into()))?; + output.push(JsonEvent::ObjectKey("datatype".into())); + output.push(JsonEvent::String(literal.datatype().as_str().into())); } - writer.write_event(JsonEvent::EndObject)?; + output.push(JsonEvent::EndObject); } #[cfg(feature = "rdf-star")] TermRef::Triple(triple) => { - writer.write_event(JsonEvent::StartObject)?; - writer.write_event(JsonEvent::ObjectKey("type".into()))?; - writer.write_event(JsonEvent::String("triple".into()))?; - writer.write_event(JsonEvent::ObjectKey("value".into()))?; - writer.write_event(JsonEvent::StartObject)?; - writer.write_event(JsonEvent::ObjectKey("subject".into()))?; - write_json_term(triple.subject.as_ref().into(), writer)?; - writer.write_event(JsonEvent::ObjectKey("predicate".into()))?; - write_json_term(triple.predicate.as_ref().into(), writer)?; - writer.write_event(JsonEvent::ObjectKey("object".into()))?; - write_json_term(triple.object.as_ref(), writer)?; - writer.write_event(JsonEvent::EndObject)?; - writer.write_event(JsonEvent::EndObject)?; + output.push(JsonEvent::StartObject); + output.push(JsonEvent::ObjectKey("type".into())); + output.push(JsonEvent::String("triple".into())); + output.push(JsonEvent::ObjectKey("value".into())); + output.push(JsonEvent::StartObject); + output.push(JsonEvent::ObjectKey("subject".into())); + write_json_term(output, triple.subject.as_ref().into()); + output.push(JsonEvent::ObjectKey("predicate".into())); + write_json_term(output, triple.predicate.as_ref().into()); + output.push(JsonEvent::ObjectKey("object".into())); + write_json_term(output, triple.object.as_ref()); + output.push(JsonEvent::EndObject); + output.push(JsonEvent::EndObject); } } - Ok(()) } pub enum JsonQueryResultsReader { diff --git a/lib/sparesults/src/serializer.rs b/lib/sparesults/src/serializer.rs index 6f5bc7dc..95356fb8 100644 --- a/lib/sparesults/src/serializer.rs +++ b/lib/sparesults/src/serializer.rs @@ -1,11 +1,20 @@ +#[cfg(feature = "async-tokio")] use crate::csv::{ - write_boolean_csv_result, write_boolean_tsv_result, CsvSolutionsWriter, TsvSolutionsWriter, + tokio_async_write_boolean_csv_result, ToTokioAsyncWriteCsvSolutionsWriter, + ToTokioAsyncWriteTsvSolutionsWriter, }; +use crate::csv::{write_boolean_csv_result, ToWriteCsvSolutionsWriter, ToWriteTsvSolutionsWriter}; use crate::format::QueryResultsFormat; -use crate::json::{write_boolean_json_result, JsonSolutionsWriter}; -use crate::xml::{write_boolean_xml_result, XmlSolutionsWriter}; +#[cfg(feature = "async-tokio")] +use crate::json::{tokio_async_write_boolean_json_result, ToTokioAsyncWriteJsonSolutionsWriter}; +use crate::json::{write_boolean_json_result, ToWriteJsonSolutionsWriter}; +#[cfg(feature = "async-tokio")] +use crate::xml::{tokio_async_write_boolean_xml_result, ToTokioAsyncWriteXmlSolutionsWriter}; +use crate::xml::{write_boolean_xml_result, ToWriteXmlSolutionsWriter}; use oxrdf::{TermRef, Variable, VariableRef}; use std::io::{self, Write}; +#[cfg(feature = "async-tokio")] +use tokio::io::AsyncWrite; /// A serializer for [SPARQL query](https://www.w3.org/TR/sparql11-query/) results serialization formats. /// @@ -15,7 +24,7 @@ use std::io::{self, Write}; /// * [SPARQL Query Results CSV Format](https://www.w3.org/TR/sparql11-results-csv-tsv/) ([`QueryResultsFormat::Csv`](QueryResultsFormat::Csv)) /// * [SPARQL Query Results TSV Format](https://www.w3.org/TR/sparql11-results-csv-tsv/) ([`QueryResultsFormat::Tsv`](QueryResultsFormat::Tsv)) /// -/// Example in JSON (the API is the same for XML and TSV): +/// Example in JSON (the API is the same for XML, CSV and TSV): /// ``` /// use sparesults::{QueryResultsFormat, QueryResultsSerializer}; /// use oxrdf::{LiteralRef, Variable, VariableRef}; @@ -49,13 +58,13 @@ impl QueryResultsSerializer { /// Write a boolean query result (from an `ASK` query) into the given [`Write`] implementation. /// - /// Example in XML (the API is the same for JSON and TSV): + /// Example in XML (the API is the same for JSON, CSV and TSV): /// ``` /// use sparesults::{QueryResultsFormat, QueryResultsSerializer}; /// - /// let json_serializer = QueryResultsSerializer::from_format(QueryResultsFormat::Xml); + /// let xml_serializer = QueryResultsSerializer::from_format(QueryResultsFormat::Xml); /// let mut buffer = Vec::new(); - /// json_serializer.serialize_boolean_to_write(&mut buffer, true)?; + /// xml_serializer.serialize_boolean_to_write(&mut buffer, true)?; /// assert_eq!(buffer, b"true"); /// # std::io::Result::Ok(()) /// ``` @@ -63,8 +72,39 @@ impl QueryResultsSerializer { match self.format { QueryResultsFormat::Xml => write_boolean_xml_result(write, value), QueryResultsFormat::Json => write_boolean_json_result(write, value), - QueryResultsFormat::Csv => write_boolean_csv_result(write, value), - QueryResultsFormat::Tsv => write_boolean_tsv_result(write, value), + QueryResultsFormat::Csv | QueryResultsFormat::Tsv => { + write_boolean_csv_result(write, value) + } + } + } + + /// Write a boolean query result (from an `ASK` query) into the given [`AsyncWrite`] implementation. + /// + /// Example in JSON (the API is the same for XML, CSV and TSV): + /// ``` + /// use sparesults::{QueryResultsFormat, QueryResultsSerializer}; + /// + /// # #[tokio::main(flavor = "current_thread")] + /// # async fn main() -> std::io::Result<()> { + /// let json_serializer = QueryResultsSerializer::from_format(QueryResultsFormat::Json); + /// let mut buffer = Vec::new(); + /// json_serializer.serialize_boolean_to_tokio_async_write(&mut buffer, false).await?; + /// assert_eq!(buffer, b"{\"head\":{},\"boolean\":false}"); + /// # Ok(()) + /// # } + /// ``` + #[cfg(feature = "async-tokio")] + pub async fn serialize_boolean_to_tokio_async_write( + &self, + write: W, + value: bool, + ) -> io::Result { + match self.format { + QueryResultsFormat::Xml => tokio_async_write_boolean_xml_result(write, value).await, + QueryResultsFormat::Json => tokio_async_write_boolean_json_result(write, value).await, + QueryResultsFormat::Csv | QueryResultsFormat::Tsv => { + tokio_async_write_boolean_csv_result(write, value).await + } } } @@ -79,15 +119,15 @@ impl QueryResultsSerializer { /// ///

This writer does unbuffered writes. You might want to use [`BufWriter`](io::BufWriter) to avoid that.
/// - /// Example in XML (the API is the same for JSON and TSV): + /// Example in XML (the API is the same for JSON, CSV and TSV): /// ``` /// use sparesults::{QueryResultsFormat, QueryResultsSerializer}; /// use oxrdf::{LiteralRef, Variable, VariableRef}; /// use std::iter::once; /// - /// let json_serializer = QueryResultsSerializer::from_format(QueryResultsFormat::Xml); + /// let xml_serializer = QueryResultsSerializer::from_format(QueryResultsFormat::Xml); /// let mut buffer = Vec::new(); - /// let mut writer = json_serializer.serialize_solutions_to_write(&mut buffer, vec![Variable::new_unchecked("foo"), Variable::new_unchecked("bar")])?; + /// let mut writer = xml_serializer.serialize_solutions_to_write(&mut buffer, vec![Variable::new_unchecked("foo"), Variable::new_unchecked("bar")])?; /// writer.write(once((VariableRef::new_unchecked("foo"), LiteralRef::from("test"))))?; /// writer.finish()?; /// assert_eq!(buffer, b"test"); @@ -100,18 +140,65 @@ impl QueryResultsSerializer { ) -> io::Result> { Ok(ToWriteSolutionsWriter { formatter: match self.format { - QueryResultsFormat::Xml => { - ToWriteSolutionsWriterKind::Xml(XmlSolutionsWriter::start(write, &variables)?) - } - QueryResultsFormat::Json => { - ToWriteSolutionsWriterKind::Json(JsonSolutionsWriter::start(write, &variables)?) - } - QueryResultsFormat::Csv => { - ToWriteSolutionsWriterKind::Csv(CsvSolutionsWriter::start(write, variables)?) - } - QueryResultsFormat::Tsv => { - ToWriteSolutionsWriterKind::Tsv(TsvSolutionsWriter::start(write, variables)?) - } + QueryResultsFormat::Xml => ToWriteSolutionsWriterKind::Xml( + ToWriteXmlSolutionsWriter::start(write, &variables)?, + ), + QueryResultsFormat::Json => ToWriteSolutionsWriterKind::Json( + ToWriteJsonSolutionsWriter::start(write, &variables)?, + ), + QueryResultsFormat::Csv => ToWriteSolutionsWriterKind::Csv( + ToWriteCsvSolutionsWriter::start(write, variables)?, + ), + QueryResultsFormat::Tsv => ToWriteSolutionsWriterKind::Tsv( + ToWriteTsvSolutionsWriter::start(write, variables)?, + ), + }, + }) + } + + /// Returns a `SolutionsWriter` allowing writing query solutions into the given [`Write`] implementation. + /// + ///
Do not forget to run the [`finish`](ToWriteSolutionsWriter::finish()) method to properly write the last bytes of the file.
+ /// + ///
This writer does unbuffered writes. You might want to use [`BufWriter`](io::BufWriter) to avoid that.
+ /// + /// Example in XML (the API is the same for JSON, CSV and TSV): + /// ``` + /// use sparesults::{QueryResultsFormat, QueryResultsSerializer}; + /// use oxrdf::{LiteralRef, Variable, VariableRef}; + /// use std::iter::once; + /// + /// # #[tokio::main(flavor = "current_thread")] + /// # async fn main() -> std::io::Result<()> { + /// let json_serializer = QueryResultsSerializer::from_format(QueryResultsFormat::Json); + /// let mut buffer = Vec::new(); + /// let mut writer = json_serializer.serialize_solutions_to_tokio_async_write(&mut buffer, vec![Variable::new_unchecked("foo"), Variable::new_unchecked("bar")]).await?; + /// writer.write(once((VariableRef::new_unchecked("foo"), LiteralRef::from("test")))).await?; + /// writer.finish().await?; + /// assert_eq!(buffer, b"{\"head\":{\"vars\":[\"foo\",\"bar\"]},\"results\":{\"bindings\":[{\"foo\":{\"type\":\"literal\",\"value\":\"test\"}}]}}"); + /// # Ok(()) + /// # } + /// ``` + #[cfg(feature = "async-tokio")] + pub async fn serialize_solutions_to_tokio_async_write( + &self, + write: W, + variables: Vec, + ) -> io::Result> { + Ok(ToTokioAsyncWriteSolutionsWriter { + formatter: match self.format { + QueryResultsFormat::Xml => ToTokioAsyncWriteSolutionsWriterKind::Xml( + ToTokioAsyncWriteXmlSolutionsWriter::start(write, &variables).await?, + ), + QueryResultsFormat::Json => ToTokioAsyncWriteSolutionsWriterKind::Json( + ToTokioAsyncWriteJsonSolutionsWriter::start(write, &variables).await?, + ), + QueryResultsFormat::Csv => ToTokioAsyncWriteSolutionsWriterKind::Csv( + ToTokioAsyncWriteCsvSolutionsWriter::start(write, variables).await?, + ), + QueryResultsFormat::Tsv => ToTokioAsyncWriteSolutionsWriterKind::Tsv( + ToTokioAsyncWriteTsvSolutionsWriter::start(write, variables).await?, + ), }, }) } @@ -126,22 +213,23 @@ impl QueryResultsSerializer { } } -/// Allows writing query results. +/// Allows writing query results into a [`Write`] implementation. +/// /// Could be built using a [`QueryResultsSerializer`]. /// ///
Do not forget to run the [`finish`](ToWriteSolutionsWriter::finish()) method to properly write the last bytes of the file.
/// ///
This writer does unbuffered writes. You might want to use [`BufWriter`](io::BufWriter) to avoid that.
/// -/// Example in TSV (the API is the same for JSON and XML): +/// Example in TSV (the API is the same for JSON, XML and CSV): /// ``` /// use sparesults::{QueryResultsFormat, QueryResultsSerializer}; /// use oxrdf::{LiteralRef, Variable, VariableRef}; /// use std::iter::once; /// -/// let json_serializer = QueryResultsSerializer::from_format(QueryResultsFormat::Tsv); +/// let tsv_serializer = QueryResultsSerializer::from_format(QueryResultsFormat::Tsv); /// let mut buffer = Vec::new(); -/// let mut writer = json_serializer.serialize_solutions_to_write(&mut buffer, vec![Variable::new_unchecked("foo"), Variable::new_unchecked("bar")])?; +/// let mut writer = tsv_serializer.serialize_solutions_to_write(&mut buffer, vec![Variable::new_unchecked("foo"), Variable::new_unchecked("bar")])?; /// writer.write(once((VariableRef::new_unchecked("foo"), LiteralRef::from("test"))))?; /// writer.finish()?; /// assert_eq!(buffer, b"?foo\t?bar\n\"test\"\t\n"); @@ -153,16 +241,16 @@ pub struct ToWriteSolutionsWriter { } enum ToWriteSolutionsWriterKind { - Xml(XmlSolutionsWriter), - Json(JsonSolutionsWriter), - Csv(CsvSolutionsWriter), - Tsv(TsvSolutionsWriter), + Xml(ToWriteXmlSolutionsWriter), + Json(ToWriteJsonSolutionsWriter), + Csv(ToWriteCsvSolutionsWriter), + Tsv(ToWriteTsvSolutionsWriter), } impl ToWriteSolutionsWriter { /// Writes a solution. /// - /// Example in JSON (the API is the same for XML and TSV): + /// Example in JSON (the API is the same for XML, CSV and TSV): /// ``` /// use sparesults::{QueryResultsFormat, QueryResultsSerializer, QuerySolution}; /// use oxrdf::{Literal, LiteralRef, Variable, VariableRef}; @@ -200,3 +288,88 @@ impl ToWriteSolutionsWriter { } } } + +/// Allows writing query results into an [`AsyncWrite`] implementation. + +/// Could be built using a [`QueryResultsSerializer`]. +/// +///
Do not forget to run the [`finish`](ToTokioAsyncWriteSolutionsWriter::finish()) method to properly write the last bytes of the file.
+/// +///
This writer does unbuffered writes. You might want to use [`BufWriter`](tokio::io::BufWriter) to avoid that.
+/// +/// Example in TSV (the API is the same for JSON, CSV and XML): +/// ``` +/// use sparesults::{QueryResultsFormat, QueryResultsSerializer}; +/// use oxrdf::{LiteralRef, Variable, VariableRef}; +/// use std::iter::once; +/// +/// # #[tokio::main(flavor = "current_thread")] +/// # async fn main() -> std::io::Result<()> { +/// let tsv_serializer = QueryResultsSerializer::from_format(QueryResultsFormat::Tsv); +/// let mut buffer = Vec::new(); +/// let mut writer = tsv_serializer.serialize_solutions_to_tokio_async_write(&mut buffer, vec![Variable::new_unchecked("foo"), Variable::new_unchecked("bar")]).await?; +/// writer.write(once((VariableRef::new_unchecked("foo"), LiteralRef::from("test")))).await?; +/// writer.finish().await?; +/// assert_eq!(buffer, b"?foo\t?bar\n\"test\"\t\n"); +/// # Ok(()) +/// # } +/// ``` +#[cfg(feature = "async-tokio")] +#[must_use] +pub struct ToTokioAsyncWriteSolutionsWriter { + formatter: ToTokioAsyncWriteSolutionsWriterKind, +} + +#[cfg(feature = "async-tokio")] +enum ToTokioAsyncWriteSolutionsWriterKind { + Xml(ToTokioAsyncWriteXmlSolutionsWriter), + Json(ToTokioAsyncWriteJsonSolutionsWriter), + Csv(ToTokioAsyncWriteCsvSolutionsWriter), + Tsv(ToTokioAsyncWriteTsvSolutionsWriter), +} + +#[cfg(feature = "async-tokio")] +impl ToTokioAsyncWriteSolutionsWriter { + /// Writes a solution. + /// + /// Example in JSON (the API is the same for XML, CSV and TSV): + /// ``` + /// use sparesults::{QueryResultsFormat, QueryResultsSerializer, QuerySolution}; + /// use oxrdf::{Literal, LiteralRef, Variable, VariableRef}; + /// use std::iter::once; + /// + /// # #[tokio::main(flavor = "current_thread")] + /// # async fn main() -> std::io::Result<()> { + /// let json_serializer = QueryResultsSerializer::from_format(QueryResultsFormat::Json); + /// let mut buffer = Vec::new(); + /// let mut writer = json_serializer.serialize_solutions_to_tokio_async_write(&mut buffer, vec![Variable::new_unchecked("foo"), Variable::new_unchecked("bar")]).await?; + /// writer.write(once((VariableRef::new_unchecked("foo"), LiteralRef::from("test")))).await?; + /// writer.write(&QuerySolution::from((vec![Variable::new_unchecked("bar")], vec![Some(Literal::from("test").into())]))).await?; + /// writer.finish().await?; + /// assert_eq!(buffer, b"{\"head\":{\"vars\":[\"foo\",\"bar\"]},\"results\":{\"bindings\":[{\"foo\":{\"type\":\"literal\",\"value\":\"test\"}},{\"bar\":{\"type\":\"literal\",\"value\":\"test\"}}]}}"); + /// # Ok(()) + /// # } + /// ``` + pub async fn write<'a>( + &mut self, + solution: impl IntoIterator>, impl Into>)>, + ) -> io::Result<()> { + let solution = solution.into_iter().map(|(v, s)| (v.into(), s.into())); + match &mut self.formatter { + ToTokioAsyncWriteSolutionsWriterKind::Xml(writer) => writer.write(solution).await, + ToTokioAsyncWriteSolutionsWriterKind::Json(writer) => writer.write(solution).await, + ToTokioAsyncWriteSolutionsWriterKind::Csv(writer) => writer.write(solution).await, + ToTokioAsyncWriteSolutionsWriterKind::Tsv(writer) => writer.write(solution).await, + } + } + + /// Writes the last bytes of the file. + pub async fn finish(self) -> io::Result { + match self.formatter { + ToTokioAsyncWriteSolutionsWriterKind::Xml(write) => write.finish().await, + ToTokioAsyncWriteSolutionsWriterKind::Json(write) => write.finish().await, + ToTokioAsyncWriteSolutionsWriterKind::Csv(write) => Ok(write.finish()), + ToTokioAsyncWriteSolutionsWriterKind::Tsv(write) => Ok(write.finish()), + } + } +} diff --git a/lib/sparesults/src/xml.rs b/lib/sparesults/src/xml.rs index 7df4fa9e..808c0f8c 100644 --- a/lib/sparesults/src/xml.rs +++ b/lib/sparesults/src/xml.rs @@ -11,147 +11,212 @@ use std::collections::BTreeMap; use std::io::{self, BufReader, Read, Write}; use std::str; use std::sync::Arc; +#[cfg(feature = "async-tokio")] +use tokio::io::AsyncWrite; -pub fn write_boolean_xml_result(sink: W, value: bool) -> io::Result { - do_write_boolean_xml_result(sink, value).map_err(map_xml_error) +pub fn write_boolean_xml_result(write: W, value: bool) -> io::Result { + let mut writer = Writer::new(write); + for event in inner_write_boolean_xml_result(value) { + writer.write_event(event).map_err(map_xml_error)?; + } + Ok(writer.into_inner()) } -fn do_write_boolean_xml_result(sink: W, value: bool) -> Result { - let mut writer = Writer::new(sink); - writer.write_event(Event::Decl(BytesDecl::new("1.0", None, None)))?; - writer - .create_element("sparql") - .with_attribute(("xmlns", "http://www.w3.org/2005/sparql-results#")) - .write_inner_content(|writer| { - writer - .create_element("head") - .write_text_content(BytesText::new(""))? - .create_element("boolean") - .write_text_content(BytesText::new(if value { "true" } else { "false" }))?; - quick_xml::Result::Ok(()) - })?; +#[cfg(feature = "async-tokio")] +pub async fn tokio_async_write_boolean_xml_result( + write: W, + value: bool, +) -> io::Result { + let mut writer = Writer::new(write); + for event in inner_write_boolean_xml_result(value) { + writer + .write_event_async(event) + .await + .map_err(map_xml_error)?; + } Ok(writer.into_inner()) } -pub struct XmlSolutionsWriter { - writer: Writer, +fn inner_write_boolean_xml_result(value: bool) -> [Event<'static>; 8] { + [ + Event::Decl(BytesDecl::new("1.0", None, None)), + Event::Start( + BytesStart::new("sparql") + .with_attributes([("xmlns", "http://www.w3.org/2005/sparql-results#")]), + ), + Event::Start(BytesStart::new("head")), + Event::End(BytesEnd::new("head")), + Event::Start(BytesStart::new("boolean")), + Event::Text(BytesText::new(if value { "true" } else { "false" })), + Event::End(BytesEnd::new("boolean")), + Event::End(BytesEnd::new("sparql")), + ] } -impl XmlSolutionsWriter { - pub fn start(sink: W, variables: &[Variable]) -> io::Result { - Self::do_start(sink, variables).map_err(map_xml_error) - } +pub struct ToWriteXmlSolutionsWriter { + inner: InnerXmlSolutionsWriter, + writer: Writer, +} - fn do_start(sink: W, variables: &[Variable]) -> Result { - let mut writer = Writer::new(sink); - writer.write_event(Event::Decl(BytesDecl::new("1.0", None, None)))?; - let mut sparql_open = BytesStart::new("sparql"); - sparql_open.push_attribute(("xmlns", "http://www.w3.org/2005/sparql-results#")); - writer.write_event(Event::Start(sparql_open))?; - writer - .create_element("head") - .write_inner_content(|writer| { - for variable in variables { - writer - .create_element("variable") - .with_attribute(("name", variable.as_str())) - .write_empty()?; - } - quick_xml::Result::Ok(()) - })?; - writer.write_event(Event::Start(BytesStart::new("results")))?; - Ok(Self { writer }) +impl ToWriteXmlSolutionsWriter { + pub fn start(write: W, variables: &[Variable]) -> io::Result { + let mut writer = Writer::new(write); + let mut buffer = Vec::with_capacity(48); + let inner = InnerXmlSolutionsWriter::start(&mut buffer, variables); + Self::do_write(&mut writer, buffer)?; + Ok(Self { inner, writer }) } pub fn write<'a>( &mut self, solution: impl IntoIterator, TermRef<'a>)>, ) -> io::Result<()> { - self.do_write(solution).map_err(map_xml_error) + let mut buffer = Vec::with_capacity(48); + self.inner.write(&mut buffer, solution); + Self::do_write(&mut self.writer, buffer) + } + + pub fn finish(mut self) -> io::Result { + let mut buffer = Vec::with_capacity(4); + self.inner.finish(&mut buffer); + Self::do_write(&mut self.writer, buffer)?; + Ok(self.writer.into_inner()) + } + + fn do_write(writer: &mut Writer, output: Vec>) -> io::Result<()> { + for event in output { + writer.write_event(event).map_err(map_xml_error)?; + } + Ok(()) + } +} + +#[cfg(feature = "async-tokio")] +pub struct ToTokioAsyncWriteXmlSolutionsWriter { + inner: InnerXmlSolutionsWriter, + writer: Writer, +} + +#[cfg(feature = "async-tokio")] +impl ToTokioAsyncWriteXmlSolutionsWriter { + pub async fn start(write: W, variables: &[Variable]) -> io::Result { + let mut writer = Writer::new(write); + let mut buffer = Vec::with_capacity(48); + let inner = InnerXmlSolutionsWriter::start(&mut buffer, variables); + Self::do_write(&mut writer, buffer).await?; + Ok(Self { writer, inner }) } - fn do_write<'a>( + pub async fn write<'a>( &mut self, solution: impl IntoIterator, TermRef<'a>)>, - ) -> Result<(), quick_xml::Error> { - self.writer - .create_element("result") - .write_inner_content(|writer| { - for (variable, value) in solution { - writer - .create_element("binding") - .with_attribute(("name", variable.as_str())) - .write_inner_content(|writer| write_xml_term(value, writer))?; - } - quick_xml::Result::Ok(()) - })?; + ) -> io::Result<()> { + let mut buffer = Vec::with_capacity(48); + self.inner.write(&mut buffer, solution); + Self::do_write(&mut self.writer, buffer).await + } + + pub async fn finish(mut self) -> io::Result { + let mut buffer = Vec::with_capacity(4); + self.inner.finish(&mut buffer); + Self::do_write(&mut self.writer, buffer).await?; + Ok(self.writer.into_inner()) + } + + async fn do_write(writer: &mut Writer, output: Vec>) -> io::Result<()> { + for event in output { + writer + .write_event_async(event) + .await + .map_err(map_xml_error)?; + } Ok(()) } +} + +struct InnerXmlSolutionsWriter; - pub fn finish(self) -> io::Result { - self.do_finish().map_err(map_xml_error) +impl InnerXmlSolutionsWriter { + fn start<'a>(output: &mut Vec>, variables: &'a [Variable]) -> Self { + output.push(Event::Decl(BytesDecl::new("1.0", None, None))); + output.push(Event::Start(BytesStart::new("sparql").with_attributes([( + "xmlns", + "http://www.w3.org/2005/sparql-results#", + )]))); + output.push(Event::Start(BytesStart::new("head"))); + for variable in variables { + output.push(Event::Empty( + BytesStart::new("variable").with_attributes([("name", variable.as_str())]), + )); + } + output.push(Event::End(BytesEnd::new("head"))); + output.push(Event::Start(BytesStart::new("results"))); + Self {} } - fn do_finish(mut self) -> Result { - self.writer - .write_event(Event::End(BytesEnd::new("results")))?; - self.writer - .write_event(Event::End(BytesEnd::new("sparql")))?; - Ok(self.writer.into_inner()) + #[allow(clippy::unused_self)] + + fn write<'a>( + &self, + output: &mut Vec>, + solution: impl IntoIterator, TermRef<'a>)>, + ) { + output.push(Event::Start(BytesStart::new("result"))); + for (variable, value) in solution { + output.push(Event::Start( + BytesStart::new("binding").with_attributes([("name", variable.as_str())]), + )); + write_xml_term(output, value); + output.push(Event::End(BytesEnd::new("binding"))); + } + output.push(Event::End(BytesEnd::new("result"))); + } + + #[allow(clippy::unused_self)] + fn finish(self, output: &mut Vec>) { + output.push(Event::End(BytesEnd::new("results"))); + output.push(Event::End(BytesEnd::new("sparql"))); } } -fn write_xml_term( - term: TermRef<'_>, - writer: &mut Writer, -) -> Result<(), quick_xml::Error> { +fn write_xml_term<'a>(output: &mut Vec>, term: TermRef<'a>) { match term { TermRef::NamedNode(uri) => { - writer - .create_element("uri") - .write_text_content(BytesText::new(uri.as_str()))?; + output.push(Event::Start(BytesStart::new("uri"))); + output.push(Event::Text(BytesText::new(uri.as_str()))); + output.push(Event::End(BytesEnd::new("uri"))); } TermRef::BlankNode(bnode) => { - writer - .create_element("bnode") - .write_text_content(BytesText::new(bnode.as_str()))?; + output.push(Event::Start(BytesStart::new("bnode"))); + output.push(Event::Text(BytesText::new(bnode.as_str()))); + output.push(Event::End(BytesEnd::new("bnode"))); } TermRef::Literal(literal) => { - let element = writer.create_element("literal"); - let element = if let Some(language) = literal.language() { - element.with_attribute(("xml:lang", language)) + let mut start = BytesStart::new("literal"); + if let Some(language) = literal.language() { + start.push_attribute(("xml:lang", language)); } else if !literal.is_plain() { - element.with_attribute(("datatype", literal.datatype().as_str())) - } else { - element - }; - element.write_text_content(BytesText::new(literal.value()))?; + start.push_attribute(("datatype", literal.datatype().as_str())) + } + output.push(Event::Start(start)); + output.push(Event::Text(BytesText::new(literal.value()))); + output.push(Event::End(BytesEnd::new("literal"))); } #[cfg(feature = "rdf-star")] TermRef::Triple(triple) => { - writer - .create_element("triple") - .write_inner_content(|writer| { - writer - .create_element("subject") - .write_inner_content(|writer| { - write_xml_term(triple.subject.as_ref().into(), writer) - })?; - writer - .create_element("predicate") - .write_inner_content(|writer| { - write_xml_term(triple.predicate.as_ref().into(), writer) - })?; - writer - .create_element("object") - .write_inner_content(|writer| { - write_xml_term(triple.object.as_ref(), writer) - })?; - quick_xml::Result::Ok(()) - })?; + output.push(Event::Start(BytesStart::new("triple"))); + output.push(Event::Start(BytesStart::new("subject"))); + write_xml_term(output, triple.subject.as_ref().into()); + output.push(Event::End(BytesEnd::new("subject"))); + output.push(Event::Start(BytesStart::new("predicate"))); + write_xml_term(output, triple.predicate.as_ref().into()); + output.push(Event::End(BytesEnd::new("predicate"))); + output.push(Event::Start(BytesStart::new("object"))); + write_xml_term(output, triple.object.as_ref()); + output.push(Event::End(BytesEnd::new("object"))); + output.push(Event::End(BytesEnd::new("triple"))); } } - Ok(()) } pub enum XmlQueryResultsReader { From 6d1d752e0155994e0c92943e1ff4f099ef187743 Mon Sep 17 00:00:00 2001 From: Tpt Date: Wed, 29 Nov 2023 17:49:35 +0100 Subject: [PATCH 130/217] Upgrades RocksDB to 8.8.1 --- oxrocksdb-sys/rocksdb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oxrocksdb-sys/rocksdb b/oxrocksdb-sys/rocksdb index 1ce22dd6..d9f9b9a7 160000 --- a/oxrocksdb-sys/rocksdb +++ b/oxrocksdb-sys/rocksdb @@ -1 +1 @@ -Subproject commit 1ce22dd6376b124d17eff7d96e0809d2f4b4ae70 +Subproject commit d9f9b9a759821252261151c3ca371947734da720 From aa82fc8157c53f90745d92378787545ffa80d9b6 Mon Sep 17 00:00:00 2001 From: Tpt Date: Wed, 29 Nov 2023 17:37:26 +0100 Subject: [PATCH 131/217] Python: builds for Python 12 and pypy on Linux --- .github/workflows/manylinux_build.sh | 5 ++++- .github/workflows/musllinux_build.sh | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/manylinux_build.sh b/.github/workflows/manylinux_build.sh index 1ed3f3e2..669e961a 100644 --- a/.github/workflows/manylinux_build.sh +++ b/.github/workflows/manylinux_build.sh @@ -15,7 +15,10 @@ maturin develop --release -m Cargo.toml python generate_stubs.py pyoxigraph pyoxigraph.pyi --black maturin build --release -m Cargo.toml --features abi3 --compatibility manylinux2014 if [ %for_each_version% ]; then - for VERSION in 7 8 9 10 11; do + for VERSION in 7 8 9 10 11 12; do maturin build --release -m Cargo.toml --interpreter "python3.$VERSION" --compatibility manylinux2014 done + for VERSION in 9 10; do + maturin build --release -m Cargo.toml --interpreter "pypy3.$VERSION" --compatibility manylinux2014 + done fi diff --git a/.github/workflows/musllinux_build.sh b/.github/workflows/musllinux_build.sh index 7dd2fb4c..5f96b322 100644 --- a/.github/workflows/musllinux_build.sh +++ b/.github/workflows/musllinux_build.sh @@ -13,7 +13,7 @@ maturin develop --release -m Cargo.toml python generate_stubs.py pyoxigraph pyoxigraph.pyi --black maturin build --release -m Cargo.toml --features abi3 --compatibility musllinux_1_1 if [ %for_each_version% ]; then - for VERSION in 7 8 9 10 11; do + for VERSION in 7 8 9 10 11 12; do maturin build --release -m Cargo.toml --interpreter "python3.$VERSION" --compatibility musllinux_1_1 done fi From 31733beda87052a7edcd069e7489d083d1c77b86 Mon Sep 17 00:00:00 2001 From: Tpt Date: Wed, 29 Nov 2023 17:40:27 +0100 Subject: [PATCH 132/217] Python: generate sdist on Linux --- .github/workflows/artifacts.yml | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/.github/workflows/artifacts.yml b/.github/workflows/artifacts.yml index e71b1df8..1096a4c8 100644 --- a/.github/workflows/artifacts.yml +++ b/.github/workflows/artifacts.yml @@ -106,6 +106,36 @@ jobs: files: oxigraph_server_${{ github.event.release.tag_name }}_x86_64_windows_msvc.exe if: github.event_name == 'release' + python_sdist: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + submodules: true + - run: rustup update + - uses: Swatinem/rust-cache@v2 + - uses: actions/setup-python@v4 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: '**/requirements.dev.txt' + - run: pip install -r python/requirements.dev.txt + - run: maturin build -m python/Cargo.toml + - run: pip install --no-index --find-links=target/wheels/ pyoxigraph + - run: rm -r target/wheels + - run: python generate_stubs.py pyoxigraph pyoxigraph.pyi --black + working-directory: ./python + - run: maturin sdist -m python/Cargo.toml + - uses: actions/upload-artifact@v3 + with: + name: pyoxigraph_source + path: target/wheels/*.tar.gz + - run: pip install twine && twine upload target/wheels/* + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + if: github.event_name == 'release' + wheel_linux: runs-on: ubuntu-latest strategy: @@ -221,7 +251,6 @@ jobs: - run: python generate_stubs.py pyoxigraph pyoxigraph.pyi --black working-directory: ./python - run: maturin build --release -m python/Cargo.toml --features abi3 - - run: maturin sdist -m python/Cargo.toml - uses: actions/upload-artifact@v3 with: name: pyoxigraph_wheel_x86_64_windows From e1ff1d919c096ae24231698f5028f3b1dba5e240 Mon Sep 17 00:00:00 2001 From: Tpt Date: Wed, 29 Nov 2023 17:52:20 +0100 Subject: [PATCH 133/217] Releases v0.3.21 --- CHANGELOG.md | 8 ++++++++ Cargo.lock | 10 +++++----- js/Cargo.toml | 4 ++-- lib/Cargo.toml | 4 ++-- oxrocksdb-sys/Cargo.toml | 2 +- python/Cargo.toml | 4 ++-- server/Cargo.toml | 4 ++-- 7 files changed, 22 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 789913ca..b55cab3c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +## [0.3.21] - 2023-11-29 + +### Changed +- Bulk loader: do not fail when loading empty files. +- Python: fixes source distribution. +- Upgrades RocksDB to 7.8.1. + + ## [0.3.20] - 2023-10-23 ### Changed diff --git a/Cargo.lock b/Cargo.lock index e3171497..23086aeb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -926,7 +926,7 @@ dependencies = [ [[package]] name = "oxigraph" -version = "0.3.20" +version = "0.3.21" dependencies = [ "criterion", "digest", @@ -958,7 +958,7 @@ dependencies = [ [[package]] name = "oxigraph_js" -version = "0.3.20" +version = "0.3.21" dependencies = [ "console_error_panic_hook", "js-sys", @@ -968,7 +968,7 @@ dependencies = [ [[package]] name = "oxigraph_server" -version = "0.3.20" +version = "0.3.21" dependencies = [ "anyhow", "assert_cmd", @@ -1028,7 +1028,7 @@ dependencies = [ [[package]] name = "oxrocksdb-sys" -version = "0.3.20" +version = "0.3.21" dependencies = [ "bindgen", "cc", @@ -1279,7 +1279,7 @@ dependencies = [ [[package]] name = "pyoxigraph" -version = "0.3.20" +version = "0.3.21" dependencies = [ "oxigraph", "pyo3", diff --git a/js/Cargo.toml b/js/Cargo.toml index 208a23c3..0bd7dee4 100644 --- a/js/Cargo.toml +++ b/js/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "oxigraph_js" -version = "0.3.20" +version = "0.3.21" authors = ["Tpt "] license = "MIT OR Apache-2.0" readme = "README.md" @@ -14,7 +14,7 @@ crate-type = ["cdylib"] name = "oxigraph" [dependencies] -oxigraph = { version = "0.3.20", path="../lib" } +oxigraph = { version = "0.3.21", path="../lib" } wasm-bindgen = "0.2" js-sys = "0.3" console_error_panic_hook = "0.1" diff --git a/lib/Cargo.toml b/lib/Cargo.toml index 371e60c5..e7b88816 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "oxigraph" -version = "0.3.20" +version = "0.3.21" authors = ["Tpt "] license = "MIT OR Apache-2.0" readme = "README.md" @@ -46,7 +46,7 @@ sparesults = { version = "0.1.8", path="sparesults", features = ["rdf-star"] } [target.'cfg(not(target_family = "wasm"))'.dependencies] libc = "0.2" -oxrocksdb-sys = { version = "0.3.20", path="../oxrocksdb-sys" } +oxrocksdb-sys = { version = "0.3.21", path="../oxrocksdb-sys" } oxhttp = { version = "0.1", optional = true } [target.'cfg(all(target_family = "wasm", target_os = "unknown"))'.dependencies] diff --git a/oxrocksdb-sys/Cargo.toml b/oxrocksdb-sys/Cargo.toml index 5aa666bf..ada54b93 100644 --- a/oxrocksdb-sys/Cargo.toml +++ b/oxrocksdb-sys/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "oxrocksdb-sys" -version = "0.3.20" +version = "0.3.21" authors = ["Tpt "] license = "GPL-2.0 OR Apache-2.0" repository = "https://github.com/oxigraph/oxigraph/tree/main/oxrocksdb-sys" diff --git a/python/Cargo.toml b/python/Cargo.toml index c9abd2fc..3f7e9fc5 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyoxigraph" -version = "0.3.20" +version = "0.3.21" authors = ["Tpt"] license = "MIT OR Apache-2.0" readme = "README.md" @@ -19,5 +19,5 @@ doctest = false abi3 = ["pyo3/abi3-py37"] [dependencies] -oxigraph = { version = "0.3.20", path="../lib", features = ["http_client"] } +oxigraph = { version = "0.3.21", path="../lib", features = ["http_client"] } pyo3 = { version = "0.19", features = ["extension-module"] } diff --git a/server/Cargo.toml b/server/Cargo.toml index e46ddaba..3444a8e2 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "oxigraph_server" -version = "0.3.20" +version = "0.3.21" authors = ["Tpt "] license = "MIT OR Apache-2.0" readme = "README.md" @@ -17,7 +17,7 @@ anyhow = "1" oxhttp = { version = "0.1", features = ["rayon"] } clap = { version = "=4.0", features = ["derive"] } clap_lex = "=0.3.0" -oxigraph = { version = "0.3.20", path = "../lib", features = ["http_client"] } +oxigraph = { version = "0.3.21", path = "../lib", features = ["http_client"] } sparesults = { version = "0.1.8", path = "../lib/sparesults", features = ["rdf-star"] } rand = "0.8" url = "2" From f445166942214612eb48d84d20d777653653b0ec Mon Sep 17 00:00:00 2001 From: Tpt Date: Wed, 29 Nov 2023 21:23:40 +0100 Subject: [PATCH 134/217] JS: Applies new biome lints --- js/package.json | 2 +- js/test/model.mjs | 16 ++++++------ js/test/store.mjs | 64 +++++++++++++++++++++++------------------------ 3 files changed, 41 insertions(+), 41 deletions(-) diff --git a/js/package.json b/js/package.json index e638da9f..cac217cb 100644 --- a/js/package.json +++ b/js/package.json @@ -8,7 +8,7 @@ "mocha": "^10.0.0" }, "scripts": { - "fmt": "biome format . --write && biome check . --apply-unsafe", + "fmt": "biome format . --write && biome check . --apply-unsafe && biome format . --write", "test": "biome ci . && wasm-pack build --debug --target nodejs && mocha", "build": "rm -rf pkg && wasm-pack build --release --target web --out-name web && mv pkg pkg-web && wasm-pack build --release --target nodejs --out-name node && mv pkg pkg-node && node build_package.js && rm -r pkg-web && rm -r pkg-node", "release": "npm run build && npm publish ./pkg", diff --git a/js/test/model.mjs b/js/test/model.mjs index 9e816cfe..b9de0164 100644 --- a/js/test/model.mjs +++ b/js/test/model.mjs @@ -6,32 +6,32 @@ import oxigraph from "../pkg/oxigraph.js"; runTests({ factory: oxigraph }); -describe("DataModel", function () { - describe("#toString()", function () { - it("namedNode().toString() should return SPARQL compatible syntax", function () { +describe("DataModel", () => { + describe("#toString()", () => { + it("namedNode().toString() should return SPARQL compatible syntax", () => { assert.strictEqual( "", oxigraph.namedNode("http://example.com").toString(), ); }); - it("blankNode().toString() should return SPARQL compatible syntax", function () { + it("blankNode().toString() should return SPARQL compatible syntax", () => { assert.strictEqual("_:a", oxigraph.blankNode("a").toString()); }); - it("literal().toString() should return SPARQL compatible syntax", function () { + it("literal().toString() should return SPARQL compatible syntax", () => { assert.strictEqual('"a\\"b"@en', oxigraph.literal('a"b', "en").toString()); }); - it("defaultGraph().toString() should return SPARQL compatible syntax", function () { + it("defaultGraph().toString() should return SPARQL compatible syntax", () => { assert.strictEqual("DEFAULT", oxigraph.defaultGraph().toString()); }); - it("variable().toString() should return SPARQL compatible syntax", function () { + it("variable().toString() should return SPARQL compatible syntax", () => { assert.strictEqual("?a", oxigraph.variable("a").toString()); }); - it("quad().toString() should return SPARQL compatible syntax", function () { + it("quad().toString() should return SPARQL compatible syntax", () => { assert.strictEqual( " << >> ", oxigraph diff --git a/js/test/store.mjs b/js/test/store.mjs index c73560a0..0693ac47 100644 --- a/js/test/store.mjs +++ b/js/test/store.mjs @@ -11,17 +11,17 @@ const triple = dataModel.quad( dataModel.literal("o"), ); -describe("Store", function () { - describe("#add()", function () { - it("an added quad should be in the store", function () { +describe("Store", () => { + describe("#add()", () => { + it("an added quad should be in the store", () => { const store = new Store(); store.add(dataModel.quad(ex, ex, triple)); assert(store.has(dataModel.quad(ex, ex, triple))); }); }); - describe("#delete()", function () { - it("an removed quad should not be in the store anymore", function () { + describe("#delete()", () => { + it("an removed quad should not be in the store anymore", () => { const store = new Store([dataModel.quad(triple, ex, ex)]); assert(store.has(dataModel.quad(triple, ex, ex))); store.delete(dataModel.quad(triple, ex, ex)); @@ -29,22 +29,22 @@ describe("Store", function () { }); }); - describe("#has()", function () { - it("an added quad should be in the store", function () { + describe("#has()", () => { + it("an added quad should be in the store", () => { const store = new Store([dataModel.quad(ex, ex, ex)]); assert(store.has(dataModel.quad(ex, ex, ex))); }); }); - describe("#size()", function () { - it("A store with one quad should have 1 for size", function () { + describe("#size()", () => { + it("A store with one quad should have 1 for size", () => { const store = new Store([dataModel.quad(ex, ex, ex)]); assert.strictEqual(1, store.size); }); }); - describe("#match_quads()", function () { - it("blank pattern should return all quads", function () { + describe("#match_quads()", () => { + it("blank pattern should return all quads", () => { const store = new Store([dataModel.quad(ex, ex, ex)]); const results = store.match(); assert.strictEqual(1, results.length); @@ -52,32 +52,32 @@ describe("Store", function () { }); }); - describe("#query()", function () { - it("ASK true", function () { + describe("#query()", () => { + it("ASK true", () => { const store = new Store([dataModel.quad(ex, ex, ex)]); assert.strictEqual(true, store.query("ASK { ?s ?s ?s }")); }); - it("ASK false", function () { + it("ASK false", () => { const store = new Store(); assert.strictEqual(false, store.query("ASK { FILTER(false)}")); }); - it("CONSTRUCT", function () { + it("CONSTRUCT", () => { const store = new Store([dataModel.quad(ex, ex, ex)]); const results = store.query("CONSTRUCT { ?s ?p ?o } WHERE { ?s ?p ?o }"); assert.strictEqual(1, results.length); assert(dataModel.quad(ex, ex, ex).equals(results[0])); }); - it("SELECT", function () { + it("SELECT", () => { const store = new Store([dataModel.quad(ex, ex, ex)]); const results = store.query("SELECT ?s WHERE { ?s ?p ?o }"); assert.strictEqual(1, results.length); assert(ex.equals(results[0].get("s"))); }); - it("SELECT with NOW()", function () { + it("SELECT with NOW()", () => { const store = new Store([dataModel.quad(ex, ex, ex)]); const results = store.query( "SELECT * WHERE { FILTER(2022 <= YEAR(NOW()) && YEAR(NOW()) <= 2100) }", @@ -85,15 +85,15 @@ describe("Store", function () { assert.strictEqual(1, results.length); }); - it("SELECT with RAND()", function () { + it("SELECT with RAND()", () => { const store = new Store([dataModel.quad(ex, ex, ex)]); const results = store.query("SELECT (RAND() AS ?y) WHERE {}"); assert.strictEqual(1, results.length); }); }); - describe("#update()", function () { - it("INSERT DATA", function () { + describe("#update()", () => { + it("INSERT DATA", () => { const store = new Store(); store.update( "INSERT DATA { }", @@ -101,7 +101,7 @@ describe("Store", function () { assert.strictEqual(1, store.size); }); - it("DELETE DATA", function () { + it("DELETE DATA", () => { const store = new Store([dataModel.quad(ex, ex, ex)]); store.update( "DELETE DATA { }", @@ -109,15 +109,15 @@ describe("Store", function () { assert.strictEqual(0, store.size); }); - it("DELETE WHERE", function () { + it("DELETE WHERE", () => { const store = new Store([dataModel.quad(ex, ex, ex)]); store.update("DELETE WHERE { ?v ?v ?v }"); assert.strictEqual(0, store.size); }); }); - describe("#load()", function () { - it("load NTriples in the default graph", function () { + describe("#load()", () => { + it("load NTriples in the default graph", () => { const store = new Store(); store.load( " .", @@ -126,7 +126,7 @@ describe("Store", function () { assert(store.has(dataModel.quad(ex, ex, ex))); }); - it("load NTriples in an other graph", function () { + it("load NTriples in an other graph", () => { const store = new Store(); store.load( " .", @@ -137,7 +137,7 @@ describe("Store", function () { assert(store.has(dataModel.quad(ex, ex, ex, ex))); }); - it("load Turtle with a base IRI", function () { + it("load Turtle with a base IRI", () => { const store = new Store(); store.load( " <> .", @@ -147,7 +147,7 @@ describe("Store", function () { assert(store.has(dataModel.quad(ex, ex, ex))); }); - it("load NQuads", function () { + it("load NQuads", () => { const store = new Store(); store.load( " .", @@ -156,7 +156,7 @@ describe("Store", function () { assert(store.has(dataModel.quad(ex, ex, ex, ex))); }); - it("load TriG with a base IRI", function () { + it("load TriG with a base IRI", () => { const store = new Store(); store.load( "GRAPH <> { <> }", @@ -167,8 +167,8 @@ describe("Store", function () { }); }); - describe("#dump()", function () { - it("dump dataset content", function () { + describe("#dump()", () => { + it("dump dataset content", () => { const store = new Store([dataModel.quad(ex, ex, ex, ex)]); assert.strictEqual( " .\n", @@ -176,7 +176,7 @@ describe("Store", function () { ); }); - it("dump named graph content", function () { + it("dump named graph content", () => { const store = new Store([dataModel.quad(ex, ex, ex, ex)]); assert.strictEqual( " .\n", @@ -184,7 +184,7 @@ describe("Store", function () { ); }); - it("dump default graph content", function () { + it("dump default graph content", () => { const store = new Store([dataModel.quad(ex, ex, ex, ex)]); assert.strictEqual("", store.dump("application/n-triples", dataModel.defaultGraph())); }); From 4f404ab6500618a6fd880d11aef430a6ab2a6411 Mon Sep 17 00:00:00 2001 From: Tpt Date: Wed, 29 Nov 2023 21:19:16 +0100 Subject: [PATCH 135/217] Python: allows again to use string for types (but with a deprecation warning) --- python/src/io.rs | 33 ++++++++++++++++++++++++++++----- python/src/sparql.rs | 28 ++++++++++++++++++++++------ python/src/store.rs | 8 ++++---- 3 files changed, 54 insertions(+), 15 deletions(-) diff --git a/python/src/io.rs b/python/src/io.rs index a87b7982..86896853 100644 --- a/python/src/io.rs +++ b/python/src/io.rs @@ -3,7 +3,7 @@ use crate::model::{hash, PyQuad, PyTriple}; use oxigraph::io::{FromReadQuadReader, ParseError, RdfFormat, RdfParser, RdfSerializer}; use oxigraph::model::QuadRef; -use pyo3::exceptions::{PySyntaxError, PyValueError}; +use pyo3::exceptions::{PyDeprecationWarning, PySyntaxError, PyValueError}; use pyo3::intern; use pyo3::prelude::*; use pyo3::types::PyBytes; @@ -53,7 +53,7 @@ use std::sync::OnceLock; #[pyo3(signature = (input = None, format = None, *, path = None, base_iri = None, without_named_graphs = false, rename_blank_nodes = false))] pub fn parse( input: Option, - format: Option, + format: Option, path: Option, base_iri: Option<&str>, without_named_graphs: bool, @@ -120,7 +120,7 @@ pub fn parse( pub fn serialize<'a>( input: &PyAny, output: Option, - format: Option, + format: Option, py: Python<'a>, ) -> PyResult> { PyWritable::do_write( @@ -519,9 +519,22 @@ impl Write for PyIo { } } -pub fn lookup_rdf_format(format: Option, path: Option<&Path>) -> PyResult { +pub fn lookup_rdf_format( + format: Option, + path: Option<&Path>, +) -> PyResult { if let Some(format) = format { - return Ok(format.inner); + return match format { + PyRdfFormatInput::Object(format) => Ok(format.inner), + PyRdfFormatInput::MediaType(media_type) => { + deprecation_warning("Using string to specify a RDF format is deprecated, please use a RdfFormat object instead.")?; + RdfFormat::from_media_type(&media_type).ok_or_else(|| { + PyValueError::new_err(format!( + "The media type {media_type} is not supported by pyoxigraph" + )) + }) + } + }; } let Some(path) = path else { return Err(PyValueError::new_err( @@ -538,6 +551,12 @@ pub fn lookup_rdf_format(format: Option, path: Option<&Path>) -> Py .ok_or_else(|| PyValueError::new_err(format!("Not supported RDF format extension: {ext}"))) } +#[derive(FromPyObject)] +pub enum PyRdfFormatInput { + Object(PyRdfFormat), + MediaType(String), +} + pub fn map_parse_error(error: ParseError, file_path: Option) -> PyErr { match error { ParseError::Syntax(error) => { @@ -608,3 +627,7 @@ pub fn python_version() -> (u8, u8) { }) }) } + +pub fn deprecation_warning(message: &str) -> PyResult<()> { + Python::with_gil(|py| PyErr::warn(py, py.get_type::(), message, 0)) +} diff --git a/python/src/sparql.rs b/python/src/sparql.rs index 36506e02..8f6c7498 100644 --- a/python/src/sparql.rs +++ b/python/src/sparql.rs @@ -240,7 +240,7 @@ impl PyQuerySolutions { fn serialize<'a>( &mut self, output: Option, - format: Option, + format: Option, py: Python<'a>, ) -> PyResult> { PyWritable::do_write( @@ -340,7 +340,7 @@ impl PyQueryBoolean { fn serialize<'a>( &mut self, output: Option, - format: Option, + format: Option, py: Python<'a>, ) -> PyResult> { PyWritable::do_write( @@ -418,7 +418,7 @@ impl PyQueryTriples { fn serialize<'a>( &mut self, output: Option, - format: Option, + format: Option, py: Python<'a>, ) -> PyResult> { PyWritable::do_write( @@ -479,7 +479,7 @@ impl PyQueryTriples { #[pyo3(signature = (input = None, format = None, *, path = None))] pub fn parse_query_results( input: Option, - format: Option, + format: Option, path: Option, py: Python<'_>, ) -> PyResult { @@ -650,11 +650,21 @@ impl PyQueryResultsFormat { } pub fn lookup_query_results_format( - format: Option, + format: Option, path: Option<&Path>, ) -> PyResult { if let Some(format) = format { - return Ok(format.inner); + return match format { + PyQueryResultsFormatInput::Object(format) => Ok(format.inner), + PyQueryResultsFormatInput::MediaType(media_type) => { + deprecation_warning("Using a string to specify a query results format is deprecated, please use a QueryResultsFormat object instead.")?; + QueryResultsFormat::from_media_type(&media_type).ok_or_else(|| { + PyValueError::new_err(format!( + "The media type {media_type} is not supported by pyoxigraph" + )) + }) + } + }; } let Some(path) = path else { return Err(PyValueError::new_err( @@ -671,6 +681,12 @@ pub fn lookup_query_results_format( .ok_or_else(|| PyValueError::new_err(format!("Not supported RDF format extension: {ext}"))) } +#[derive(FromPyObject)] +pub enum PyQueryResultsFormatInput { + Object(PyQueryResultsFormat), + MediaType(String), +} + pub fn map_evaluation_error(error: EvaluationError) -> PyErr { match error { EvaluationError::Parsing(error) => PySyntaxError::new_err(error.to_string()), diff --git a/python/src/store.rs b/python/src/store.rs index f9e0b800..910247df 100644 --- a/python/src/store.rs +++ b/python/src/store.rs @@ -1,7 +1,7 @@ #![allow(clippy::needless_option_as_deref)] use crate::io::{ - allow_threads_unsafe, lookup_rdf_format, map_parse_error, PyRdfFormat, PyReadable, + allow_threads_unsafe, lookup_rdf_format, map_parse_error, PyRdfFormatInput, PyReadable, PyReadableInput, PyWritable, PyWritableOutput, }; use crate::model::*; @@ -385,7 +385,7 @@ impl PyStore { fn load( &self, input: Option, - format: Option, + format: Option, path: Option, base_iri: Option<&str>, to_graph: Option<&PyAny>, @@ -452,7 +452,7 @@ impl PyStore { fn bulk_load( &self, input: Option, - format: Option, + format: Option, path: Option, base_iri: Option<&str>, to_graph: Option<&PyAny>, @@ -520,7 +520,7 @@ impl PyStore { fn dump<'a>( &self, output: Option, - format: Option, + format: Option, from_graph: Option<&PyAny>, py: Python<'a>, ) -> PyResult> { From 389d993dc405be08be6f22bce7ad34ed1310be01 Mon Sep 17 00:00:00 2001 From: Dan Brickley Date: Wed, 29 Nov 2023 21:02:38 +0000 Subject: [PATCH 136/217] Update README.md typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5313fd0f..c67072f4 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,7 @@ Unless you explicitly state otherwise, any contribution intentionally submitted ## Sponsors * [RelationLabs](https://relationlabs.ai/) that is building [Relation-Graph](https://github.com/relationlabs/Relation-Graph), a SPARQL database module for the [Substrate blockchain platform](https://substrate.io/) based on Oxigraph. -* [Field 33](https://field33.com) that is building [an ontology management plateform](https://plow.pm/). +* [Field 33](https://field33.com) that is building [an ontology management platform](https://plow.pm/). * [Magnus Bakken](https://github.com/magbak) who is building [chrontext](https://github.com/magbak/chrontext), providing a SPARQL query endpoint on top of joint RDF and time series databases. * [ACE IoT Solutions](https://aceiotsolutions.com/), a building IOT platform. * [Albin Larsson](https://byabbe.se/) who is building [GovDirectory](https://www.govdirectory.org/), a directory of public agencies based on Wikidata. From 9979a3d5038fbdab2948022c154630cea6b78a45 Mon Sep 17 00:00:00 2001 From: Tpt Date: Fri, 1 Dec 2023 13:37:17 +0100 Subject: [PATCH 137/217] Allows newer dependency versions --- .github/workflows/tests.yml | 11 +++++++++++ lib/Cargo.toml | 2 +- oxrocksdb-sys/Cargo.toml | 4 ++-- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 363c26ef..e84c2d7d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -132,6 +132,17 @@ jobs: env: RUST_BACKTRACE: 1 + test_linux_latest: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + submodules: true + - run: rustup update + - uses: Swatinem/rust-cache@v2 + - run: rm Cargo.lock && cargo update + - run: cargo test + address_sanitizer: runs-on: ubuntu-latest steps: diff --git a/lib/Cargo.toml b/lib/Cargo.toml index e7b88816..47441a83 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -36,7 +36,7 @@ rio_api = "0.8" rio_turtle = "0.8" rio_xml = "0.8" hex = "0.4" -siphasher = "0.3" +siphasher = ">=0.3,<2.0" lazy_static = "1" json-event-parser = "0.1" oxrdf = { version = "0.1.7", path="oxrdf", features = ["rdf-star", "oxsdatatypes"] } diff --git a/oxrocksdb-sys/Cargo.toml b/oxrocksdb-sys/Cargo.toml index ada54b93..8d448518 100644 --- a/oxrocksdb-sys/Cargo.toml +++ b/oxrocksdb-sys/Cargo.toml @@ -18,5 +18,5 @@ links = "rocksdb" libc = "0.2" [build-dependencies] -bindgen = "0.66" -cc = { version = "=1.0.79", features = ["parallel"] } +bindgen = ">=0.66,<0.70" +cc = { version = "1.0.79", features = ["parallel"] } From d88c2e0a8a513b2591e2269e3598aa495a463320 Mon Sep 17 00:00:00 2001 From: Tpt Date: Fri, 1 Dec 2023 13:42:14 +0100 Subject: [PATCH 138/217] Adds compatibility with lazy_static with spin_no_std feature --- lib/src/storage/backend/rocksdb.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/src/storage/backend/rocksdb.rs b/lib/src/storage/backend/rocksdb.rs index 67766007..3c21c007 100644 --- a/lib/src/storage/backend/rocksdb.rs +++ b/lib/src/storage/backend/rocksdb.rs @@ -1398,6 +1398,7 @@ impl From for StorageError { struct UnsafeEnv(*mut rocksdb_env_t); // Hack for lazy_static. OK because only written in lazy static and used in a thread-safe way by RocksDB +unsafe impl Send for UnsafeEnv {} unsafe impl Sync for UnsafeEnv {} fn path_to_cstring(path: &Path) -> Result { From 03afe5c6c64e1f872b2933ee863ee0085e837a00 Mon Sep 17 00:00:00 2001 From: Tpt Date: Fri, 1 Dec 2023 15:23:01 +0100 Subject: [PATCH 139/217] Releases v0.3.22 --- CHANGELOG.md | 7 +++++++ Cargo.lock | 10 +++++----- js/Cargo.toml | 4 ++-- lib/Cargo.toml | 4 ++-- oxrocksdb-sys/Cargo.toml | 2 +- python/Cargo.toml | 4 ++-- server/Cargo.toml | 4 ++-- 7 files changed, 21 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b55cab3c..9e0007e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## [0.3.22] - 2023-11-29 + +### Changed +- Allows to compile with more recent `bindgen` and `cc` +- Fixes compatibility with `spin_no_std` feature of `lazy_static` + + ## [0.3.21] - 2023-11-29 ### Changed diff --git a/Cargo.lock b/Cargo.lock index 23086aeb..bb2f629c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -926,7 +926,7 @@ dependencies = [ [[package]] name = "oxigraph" -version = "0.3.21" +version = "0.3.22" dependencies = [ "criterion", "digest", @@ -958,7 +958,7 @@ dependencies = [ [[package]] name = "oxigraph_js" -version = "0.3.21" +version = "0.3.22" dependencies = [ "console_error_panic_hook", "js-sys", @@ -968,7 +968,7 @@ dependencies = [ [[package]] name = "oxigraph_server" -version = "0.3.21" +version = "0.3.22" dependencies = [ "anyhow", "assert_cmd", @@ -1028,7 +1028,7 @@ dependencies = [ [[package]] name = "oxrocksdb-sys" -version = "0.3.21" +version = "0.3.22" dependencies = [ "bindgen", "cc", @@ -1279,7 +1279,7 @@ dependencies = [ [[package]] name = "pyoxigraph" -version = "0.3.21" +version = "0.3.22" dependencies = [ "oxigraph", "pyo3", diff --git a/js/Cargo.toml b/js/Cargo.toml index 0bd7dee4..bbe60120 100644 --- a/js/Cargo.toml +++ b/js/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "oxigraph_js" -version = "0.3.21" +version = "0.3.22" authors = ["Tpt "] license = "MIT OR Apache-2.0" readme = "README.md" @@ -14,7 +14,7 @@ crate-type = ["cdylib"] name = "oxigraph" [dependencies] -oxigraph = { version = "0.3.21", path="../lib" } +oxigraph = { version = "0.3.22", path="../lib" } wasm-bindgen = "0.2" js-sys = "0.3" console_error_panic_hook = "0.1" diff --git a/lib/Cargo.toml b/lib/Cargo.toml index 47441a83..fa335dff 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "oxigraph" -version = "0.3.21" +version = "0.3.22" authors = ["Tpt "] license = "MIT OR Apache-2.0" readme = "README.md" @@ -46,7 +46,7 @@ sparesults = { version = "0.1.8", path="sparesults", features = ["rdf-star"] } [target.'cfg(not(target_family = "wasm"))'.dependencies] libc = "0.2" -oxrocksdb-sys = { version = "0.3.21", path="../oxrocksdb-sys" } +oxrocksdb-sys = { version = "0.3.22", path="../oxrocksdb-sys" } oxhttp = { version = "0.1", optional = true } [target.'cfg(all(target_family = "wasm", target_os = "unknown"))'.dependencies] diff --git a/oxrocksdb-sys/Cargo.toml b/oxrocksdb-sys/Cargo.toml index 8d448518..a0e54f5f 100644 --- a/oxrocksdb-sys/Cargo.toml +++ b/oxrocksdb-sys/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "oxrocksdb-sys" -version = "0.3.21" +version = "0.3.22" authors = ["Tpt "] license = "GPL-2.0 OR Apache-2.0" repository = "https://github.com/oxigraph/oxigraph/tree/main/oxrocksdb-sys" diff --git a/python/Cargo.toml b/python/Cargo.toml index 3f7e9fc5..68efa4dd 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyoxigraph" -version = "0.3.21" +version = "0.3.22" authors = ["Tpt"] license = "MIT OR Apache-2.0" readme = "README.md" @@ -19,5 +19,5 @@ doctest = false abi3 = ["pyo3/abi3-py37"] [dependencies] -oxigraph = { version = "0.3.21", path="../lib", features = ["http_client"] } +oxigraph = { version = "0.3.22", path="../lib", features = ["http_client"] } pyo3 = { version = "0.19", features = ["extension-module"] } diff --git a/server/Cargo.toml b/server/Cargo.toml index 3444a8e2..11c48d49 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "oxigraph_server" -version = "0.3.21" +version = "0.3.22" authors = ["Tpt "] license = "MIT OR Apache-2.0" readme = "README.md" @@ -17,7 +17,7 @@ anyhow = "1" oxhttp = { version = "0.1", features = ["rayon"] } clap = { version = "=4.0", features = ["derive"] } clap_lex = "=0.3.0" -oxigraph = { version = "0.3.21", path = "../lib", features = ["http_client"] } +oxigraph = { version = "0.3.22", path = "../lib", features = ["http_client"] } sparesults = { version = "0.1.8", path = "../lib/sparesults", features = ["rdf-star"] } rand = "0.8" url = "2" From 899e55324956a133940b7fcd8a5bf79cba5dc614 Mon Sep 17 00:00:00 2001 From: Tpt Date: Sun, 3 Dec 2023 21:12:13 +0100 Subject: [PATCH 140/217] Makes SPARQL query result Send and Sync --- lib/sparesults/src/csv.rs | 3 --- lib/sparesults/src/parser.rs | 13 ++++++------- lib/sparesults/src/solution.rs | 7 +++---- lib/src/sparql/eval.rs | 27 +++++++++++++-------------- lib/src/sparql/mod.rs | 30 +++++++++++++++++------------- lib/src/sparql/model.rs | 20 ++++++++++---------- lib/src/sparql/service.rs | 2 +- lib/src/sparql/update.rs | 3 ++- lib/src/storage/numeric_encoder.rs | 8 ++++---- 9 files changed, 56 insertions(+), 57 deletions(-) diff --git a/lib/sparesults/src/csv.rs b/lib/sparesults/src/csv.rs index ac02c99e..985092b4 100644 --- a/lib/sparesults/src/csv.rs +++ b/lib/sparesults/src/csv.rs @@ -628,7 +628,6 @@ mod tests { use super::*; use std::error::Error; - use std::rc::Rc; fn build_example() -> (Vec, Vec>>) { ( @@ -682,7 +681,6 @@ mod tests { let (variables, solutions) = build_example(); let mut buffer = String::new(); let writer = InnerCsvSolutionsWriter::start(&mut buffer, variables.clone()); - let variables = Rc::new(variables); for solution in solutions { writer.write( &mut buffer, @@ -702,7 +700,6 @@ mod tests { // Write let mut buffer = String::new(); let writer = InnerTsvSolutionsWriter::start(&mut buffer, variables.clone()); - let variables = Rc::new(variables); for solution in &solutions { writer.write( &mut buffer, diff --git a/lib/sparesults/src/parser.rs b/lib/sparesults/src/parser.rs index 8833f9ac..b0aad24c 100644 --- a/lib/sparesults/src/parser.rs +++ b/lib/sparesults/src/parser.rs @@ -6,7 +6,7 @@ use crate::solution::QuerySolution; use crate::xml::{XmlQueryResultsReader, XmlSolutionsReader}; use oxrdf::Variable; use std::io::Read; -use std::rc::Rc; +use std::sync::Arc; /// Parsers for [SPARQL query](https://www.w3.org/TR/sparql11-query/) results serialization formats. /// @@ -81,7 +81,7 @@ impl QueryResultsParser { solutions, variables, } => FromReadQueryResultsReader::Solutions(FromReadSolutionsReader { - variables: Rc::new(variables), + variables: variables.into(), solutions: SolutionsReaderKind::Xml(solutions), }), }, @@ -91,7 +91,7 @@ impl QueryResultsParser { solutions, variables, } => FromReadQueryResultsReader::Solutions(FromReadSolutionsReader { - variables: Rc::new(variables), + variables: variables.into(), solutions: SolutionsReaderKind::Json(solutions), }), }, @@ -102,7 +102,7 @@ impl QueryResultsParser { solutions, variables, } => FromReadQueryResultsReader::Solutions(FromReadSolutionsReader { - variables: Rc::new(variables), + variables: variables.into(), solutions: SolutionsReaderKind::Tsv(solutions), }), }, @@ -166,9 +166,8 @@ pub enum FromReadQueryResultsReader { /// } /// # Result::<(),sparesults::ParseError>::Ok(()) /// ``` -#[allow(clippy::rc_buffer)] pub struct FromReadSolutionsReader { - variables: Rc>, + variables: Arc<[Variable]>, solutions: SolutionsReaderKind, } @@ -209,7 +208,7 @@ impl Iterator for FromReadSolutionsReader { SolutionsReaderKind::Tsv(reader) => reader.read_next(), } .transpose()? - .map(|values| (Rc::clone(&self.variables), values).into()), + .map(|values| (Arc::clone(&self.variables), values).into()), ) } } diff --git a/lib/sparesults/src/solution.rs b/lib/sparesults/src/solution.rs index 842bc7d3..3990ab79 100644 --- a/lib/sparesults/src/solution.rs +++ b/lib/sparesults/src/solution.rs @@ -4,7 +4,7 @@ use oxrdf::{Term, Variable, VariableRef}; use std::fmt; use std::iter::Zip; use std::ops::Index; -use std::rc::Rc; +use std::sync::Arc; /// Tuple associating variables and terms that are the result of a SPARQL query. /// @@ -18,9 +18,8 @@ use std::rc::Rc; /// assert_eq!(solution.get("foo"), Some(&Literal::from(1).into())); // Get the value of the variable ?foo if it exists (here yes). /// assert_eq!(solution.get(1), None); // Get the value of the second column if it exists (here no). /// ``` -#[allow(clippy::rc_buffer)] pub struct QuerySolution { - variables: Rc>, + variables: Arc<[Variable]>, values: Vec>, } @@ -116,7 +115,7 @@ impl QuerySolution { } } -impl>>, S: Into>>> From<(V, S)> for QuerySolution { +impl>, S: Into>>> From<(V, S)> for QuerySolution { #[inline] fn from((v, s): (V, S)) -> Self { Self { diff --git a/lib/src/sparql/eval.rs b/lib/src/sparql/eval.rs index 413fcd20..c5b35a95 100644 --- a/lib/src/sparql/eval.rs +++ b/lib/src/sparql/eval.rs @@ -1,10 +1,11 @@ use crate::model::vocab::{rdf, xsd}; -use crate::model::{BlankNode, LiteralRef, NamedNode, NamedNodeRef, Term, Triple}; +use crate::model::{BlankNode, LiteralRef, NamedNodeRef, Term, Triple}; use crate::sparql::algebra::{Query, QueryDataset}; use crate::sparql::dataset::DatasetView; use crate::sparql::error::EvaluationError; use crate::sparql::model::*; use crate::sparql::service::ServiceHandler; +use crate::sparql::CustomFunctionRegistry; use crate::storage::numeric_encoder::*; use crate::storage::small_string::SmallString; use digest::Digest; @@ -35,6 +36,7 @@ use std::hash::{Hash, Hasher}; use std::iter::Iterator; use std::iter::{empty, once}; use std::rc::Rc; +use std::sync::Arc; use std::{fmt, io, str}; const REGEX_SIZE_LIMIT: usize = 1_000_000; @@ -119,15 +121,14 @@ impl IntoIterator for EncodedTuple { } type EncodedTuplesIterator = Box>>; -type CustomFunctionRegistry = HashMap Option>>; #[derive(Clone)] pub struct SimpleEvaluator { dataset: Rc, base_iri: Option>>, now: DateTime, - service_handler: Rc>, - custom_functions: Rc, + service_handler: Arc>, + custom_functions: Arc, run_stats: bool, } @@ -135,8 +136,8 @@ impl SimpleEvaluator { pub fn new( dataset: Rc, base_iri: Option>>, - service_handler: Rc>, - custom_functions: Rc, + service_handler: Arc>, + custom_functions: Arc, run_stats: bool, ) -> Self { Self { @@ -149,7 +150,6 @@ impl SimpleEvaluator { } } - #[allow(clippy::rc_buffer)] pub fn evaluate_select(&self, pattern: &GraphPattern) -> (QueryResults, Rc) { let mut variables = Vec::new(); let (eval, stats) = self.graph_pattern_evaluator(pattern, &mut variables); @@ -158,7 +158,7 @@ impl SimpleEvaluator { QueryResults::Solutions(decode_bindings( Rc::clone(&self.dataset), eval(from), - Rc::from(variables), + Arc::from(variables), )), stats, ) @@ -3127,11 +3127,10 @@ pub(super) fn compile_pattern(pattern: &str, flags: Option<&str>) -> Option, iter: EncodedTuplesIterator, - variables: Rc>, + variables: Arc<[Variable]>, ) -> QuerySolutionIter { let tuple_size = variables.len(); QuerySolutionIter::new( @@ -3223,15 +3222,15 @@ fn equals(a: &EncodedTerm, b: &EncodedTerm) -> Option { EncodedTerm::FloatLiteral(a) => match b { EncodedTerm::FloatLiteral(b) => Some(a == b), EncodedTerm::DoubleLiteral(b) => Some(Double::from(*a) == *b), - EncodedTerm::IntegerLiteral(b) => Some(*a == Float::from(*b)), + EncodedTerm::IntegerLiteral(b) => Some(*a == (*b).into()), EncodedTerm::DecimalLiteral(b) => Some(*a == (*b).into()), _ if b.is_unknown_typed_literal() => None, _ => Some(false), }, EncodedTerm::DoubleLiteral(a) => match b { - EncodedTerm::FloatLiteral(b) => Some(*a == Double::from(*b)), + EncodedTerm::FloatLiteral(b) => Some(*a == (*b).into()), EncodedTerm::DoubleLiteral(b) => Some(a == b), - EncodedTerm::IntegerLiteral(b) => Some(*a == Double::from(*b)), + EncodedTerm::IntegerLiteral(b) => Some(*a == (*b).into()), EncodedTerm::DecimalLiteral(b) => Some(*a == (*b).into()), _ if b.is_unknown_typed_literal() => None, _ => Some(false), @@ -3247,7 +3246,7 @@ fn equals(a: &EncodedTerm, b: &EncodedTerm) -> Option { EncodedTerm::DecimalLiteral(a) => match b { EncodedTerm::FloatLiteral(b) => Some(Float::from(*a) == *b), EncodedTerm::DoubleLiteral(b) => Some(Double::from(*a) == *b), - EncodedTerm::IntegerLiteral(b) => Some(*a == Decimal::from(*b)), + EncodedTerm::IntegerLiteral(b) => Some(*a == (*b).into()), EncodedTerm::DecimalLiteral(b) => Some(a == b), _ if b.is_unknown_typed_literal() => None, _ => Some(false), diff --git a/lib/src/sparql/mod.rs b/lib/src/sparql/mod.rs index 1972a77e..262ab868 100644 --- a/lib/src/sparql/mod.rs +++ b/lib/src/sparql/mod.rs @@ -30,6 +30,7 @@ use sparopt::algebra::GraphPattern; use sparopt::Optimizer; use std::collections::HashMap; use std::rc::Rc; +use std::sync::Arc; use std::time::Duration; use std::{fmt, io}; @@ -56,7 +57,7 @@ pub(crate) fn evaluate_query( Rc::new(dataset), base_iri.map(Rc::new), options.service_handler(), - Rc::new(options.custom_functions), + Arc::new(options.custom_functions), run_stats, ) .evaluate_select(&pattern); @@ -76,7 +77,7 @@ pub(crate) fn evaluate_query( Rc::new(dataset), base_iri.map(Rc::new), options.service_handler(), - Rc::new(options.custom_functions), + Arc::new(options.custom_functions), run_stats, ) .evaluate_ask(&pattern); @@ -99,7 +100,7 @@ pub(crate) fn evaluate_query( Rc::new(dataset), base_iri.map(Rc::new), options.service_handler(), - Rc::new(options.custom_functions), + Arc::new(options.custom_functions), run_stats, ) .evaluate_construct(&pattern, &template); @@ -119,7 +120,7 @@ pub(crate) fn evaluate_query( Rc::new(dataset), base_iri.map(Rc::new), options.service_handler(), - Rc::new(options.custom_functions), + Arc::new(options.custom_functions), run_stats, ) .evaluate_describe(&pattern); @@ -155,19 +156,22 @@ pub(crate) fn evaluate_query( /// ``` #[derive(Clone, Default)] pub struct QueryOptions { - service_handler: Option>>, - custom_functions: HashMap Option>>, + service_handler: Option>>, + custom_functions: CustomFunctionRegistry, http_timeout: Option, http_redirection_limit: usize, without_optimizations: bool, } +pub(crate) type CustomFunctionRegistry = + HashMap Option) + Send + Sync>>; + impl QueryOptions { /// Use a given [`ServiceHandler`] to execute [SPARQL 1.1 Federated Query](https://www.w3.org/TR/sparql11-federated-query/) SERVICE calls. #[inline] #[must_use] pub fn with_service_handler(mut self, service_handler: impl ServiceHandler + 'static) -> Self { - self.service_handler = Some(Rc::new(ErrorConversionServiceHandler::wrap( + self.service_handler = Some(Arc::new(ErrorConversionServiceHandler::wrap( service_handler, ))); self @@ -177,7 +181,7 @@ impl QueryOptions { #[inline] #[must_use] pub fn without_service_handler(mut self) -> Self { - self.service_handler = Some(Rc::new(EmptyServiceHandler)); + self.service_handler = Some(Arc::new(EmptyServiceHandler)); self } @@ -227,21 +231,21 @@ impl QueryOptions { pub fn with_custom_function( mut self, name: NamedNode, - evaluator: impl Fn(&[Term]) -> Option + 'static, + evaluator: impl Fn(&[Term]) -> Option + Send + Sync + 'static, ) -> Self { - self.custom_functions.insert(name, Rc::new(evaluator)); + self.custom_functions.insert(name, Arc::new(evaluator)); self } - fn service_handler(&self) -> Rc> { + fn service_handler(&self) -> Arc> { self.service_handler.clone().unwrap_or_else(|| { if cfg!(feature = "http-client") { - Rc::new(service::SimpleServiceHandler::new( + Arc::new(service::SimpleServiceHandler::new( self.http_timeout, self.http_redirection_limit, )) } else { - Rc::new(EmptyServiceHandler) + Arc::new(EmptyServiceHandler) } }) } diff --git a/lib/src/sparql/model.rs b/lib/src/sparql/model.rs index 0c7becc6..81712877 100644 --- a/lib/src/sparql/model.rs +++ b/lib/src/sparql/model.rs @@ -8,7 +8,7 @@ use crate::sparql::results::{ use oxrdf::{Variable, VariableRef}; pub use sparesults::QuerySolution; use std::io::{Read, Write}; -use std::rc::Rc; +use std::sync::Arc; /// Results of a [SPARQL query](https://www.w3.org/TR/sparql11-query/). pub enum QueryResults { @@ -170,22 +170,20 @@ impl From> for QueryResults { /// } /// # Result::<_,Box>::Ok(()) /// ``` -#[allow(clippy::rc_buffer)] pub struct QuerySolutionIter { - variables: Rc>, + variables: Arc<[Variable]>, iter: Box>>, } impl QuerySolutionIter { - #[allow(clippy::rc_buffer)] pub fn new( - variables: Rc>, + variables: Arc<[Variable]>, iter: impl Iterator>, EvaluationError>> + 'static, ) -> Self { Self { - variables: Rc::clone(&variables), + variables: Arc::clone(&variables), iter: Box::new( - iter.map(move |t| t.map(|values| (Rc::clone(&variables), values).into())), + iter.map(move |t| t.map(|values| (Arc::clone(&variables), values).into())), ), } } @@ -211,7 +209,7 @@ impl QuerySolutionIter { impl From> for QuerySolutionIter { fn from(reader: FromReadSolutionsReader) -> Self { Self { - variables: Rc::new(reader.variables().to_vec()), + variables: reader.variables().into(), iter: Box::new(reader.map(|t| t.map_err(EvaluationError::from))), } } @@ -291,10 +289,12 @@ mod tests { QueryResults::Boolean(true), QueryResults::Boolean(false), QueryResults::Solutions(QuerySolutionIter::new( - Rc::new(vec![ + [ Variable::new_unchecked("foo"), Variable::new_unchecked("bar"), - ]), + ] + .as_ref() + .into(), Box::new( vec![ Ok(vec![None, None]), diff --git a/lib/src/sparql/service.rs b/lib/src/sparql/service.rs index dec189ae..2ec0b7a3 100644 --- a/lib/src/sparql/service.rs +++ b/lib/src/sparql/service.rs @@ -48,7 +48,7 @@ use std::time::Duration; /// } /// # Result::<_,Box>::Ok(()) /// ``` -pub trait ServiceHandler { +pub trait ServiceHandler: Send + Sync { type Error: Error + Send + Sync + 'static; /// Evaluates a [`Query`] against a given service identified by a [`NamedNode`]. diff --git a/lib/src/sparql/update.rs b/lib/src/sparql/update.rs index 1744c464..2e318c71 100644 --- a/lib/src/sparql/update.rs +++ b/lib/src/sparql/update.rs @@ -19,6 +19,7 @@ use sparopt::Optimizer; use std::collections::HashMap; use std::io; use std::rc::Rc; +use std::sync::Arc; pub fn evaluate_update<'a, 'b: 'a>( transaction: &'a mut StorageWriter<'b>, @@ -131,7 +132,7 @@ impl<'a, 'b: 'a> SimpleUpdateEvaluator<'a, 'b> { Rc::clone(&dataset), self.base_iri.clone(), self.options.query_options.service_handler(), - Rc::new(self.options.query_options.custom_functions.clone()), + Arc::new(self.options.query_options.custom_functions.clone()), false, ); let mut variables = Vec::new(); diff --git a/lib/src/storage/numeric_encoder.rs b/lib/src/storage/numeric_encoder.rs index 59f7532f..fd0d7544 100644 --- a/lib/src/storage/numeric_encoder.rs +++ b/lib/src/storage/numeric_encoder.rs @@ -8,8 +8,8 @@ use siphasher::sip128::{Hasher128, SipHasher24}; use std::fmt::Debug; use std::hash::Hash; use std::hash::Hasher; -use std::rc::Rc; use std::str; +use std::sync::Arc; #[derive(Eq, PartialEq, Debug, Clone, Copy, Hash)] #[repr(transparent)] @@ -96,7 +96,7 @@ pub enum EncodedTerm { DurationLiteral(Duration), YearMonthDurationLiteral(YearMonthDuration), DayTimeDurationLiteral(DayTimeDuration), - Triple(Rc), + Triple(Arc), } impl PartialEq for EncodedTerm { @@ -471,7 +471,7 @@ impl From for EncodedTerm { impl From for EncodedTerm { fn from(value: EncodedTriple) -> Self { - Self::Triple(Rc::new(value)) + Self::Triple(Arc::new(value)) } } @@ -634,7 +634,7 @@ impl From> for EncodedTerm { impl From> for EncodedTerm { fn from(triple: TripleRef<'_>) -> Self { - Self::Triple(Rc::new(triple.into())) + Self::Triple(Arc::new(triple.into())) } } From efd5eec65dca680a7c7dfc18b25ce85f500b2f9e Mon Sep 17 00:00:00 2001 From: Tpt Date: Thu, 7 Dec 2023 18:42:27 +0100 Subject: [PATCH 141/217] Introduces load_from_read instead of load_graph and load_dataset --- cli/src/main.rs | 34 +++-- js/src/store.rs | 20 ++- lib/benches/store.rs | 7 +- lib/oxrdfio/src/parser.rs | 6 + lib/oxrdfio/src/serializer.rs | 6 + lib/sparesults/src/parser.rs | 6 + lib/sparesults/src/serializer.rs | 6 + lib/src/store.rs | 207 ++++++++++++++++++++++++++---- lib/tests/store.rs | 32 ++--- python/src/store.rs | 34 +++-- testsuite/src/sparql_evaluator.rs | 38 +++--- 11 files changed, 280 insertions(+), 116 deletions(-) diff --git a/cli/src/main.rs b/cli/src/main.rs index 6dc7cc73..2a2f6562 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -780,16 +780,21 @@ pub fn main() -> anyhow::Result<()> { fn bulk_load( loader: &BulkLoader, - reader: impl Read, + read: impl Read, format: RdfFormat, base_iri: Option<&str>, to_graph_name: Option, ) -> anyhow::Result<()> { + let mut parser = RdfParser::from_format(format); if let Some(to_graph_name) = to_graph_name { - loader.load_graph(reader, format, to_graph_name, base_iri) - } else { - loader.load_dataset(reader, format, base_iri) - }?; + parser = parser.with_default_graph(to_graph_name); + } + if let Some(base_iri) = base_iri { + parser = parser + .with_base_iri(base_iri) + .with_context(|| format!("Invalid base IRI {base_iri}"))?; + } + loader.load_from_read(parser, read)?; Ok(()) } @@ -1646,15 +1651,16 @@ fn web_load_graph( } else { None }; + let mut parser = RdfParser::from_format(format) + .without_named_graphs() + .with_default_graph(to_graph_name.clone()); + if let Some(base_iri) = base_iri { + parser = parser.with_base_iri(base_iri).map_err(bad_request)?; + } if url_query_parameter(request, "no_transaction").is_some() { - web_bulk_loader(store, request).load_graph( - request.body_mut(), - format, - to_graph_name.clone(), - base_iri, - ) + web_bulk_loader(store, request).load_from_read(parser, request.body_mut()) } else { - store.load_graph(request.body_mut(), format, to_graph_name.clone(), base_iri) + store.load_from_read(parser, request.body_mut()) } .map_err(loader_to_http_error) } @@ -1665,9 +1671,9 @@ fn web_load_dataset( format: RdfFormat, ) -> Result<(), HttpError> { if url_query_parameter(request, "no_transaction").is_some() { - web_bulk_loader(store, request).load_dataset(request.body_mut(), format, None) + web_bulk_loader(store, request).load_from_read(format, request.body_mut()) } else { - store.load_dataset(request.body_mut(), format, None) + store.load_from_read(format, request.body_mut()) } .map_err(loader_to_http_error) } diff --git a/js/src/store.rs b/js/src/store.rs index 008f24f0..f76786a5 100644 --- a/js/src/store.rs +++ b/js/src/store.rs @@ -4,7 +4,7 @@ use crate::format_err; use crate::model::*; use crate::utils::to_err; use js_sys::{Array, Map}; -use oxigraph::io::RdfFormat; +use oxigraph::io::{RdfFormat, RdfParser}; use oxigraph::model::*; use oxigraph::sparql::QueryResults; use oxigraph::store::Store; @@ -161,18 +161,16 @@ impl JsStore { )); }; + let mut parser = RdfParser::from_format(format); if let Some(to_graph_name) = FROM_JS.with(|c| c.to_optional_term(to_graph_name))? { - self.store.load_graph( - data.as_bytes(), - format, - GraphName::try_from(to_graph_name)?, - base_iri.as_deref(), - ) - } else { - self.store - .load_dataset(data.as_bytes(), format, base_iri.as_deref()) + parser = parser.with_default_graph(GraphName::try_from(to_graph_name)?); + } + if let Some(base_iri) = base_iri { + parser = parser.with_base_iri(base_iri).map_err(to_err)?; } - .map_err(to_err) + self.store + .load_from_read(parser, data.as_bytes()) + .map_err(to_err) } pub fn dump(&self, format: &str, from_graph_name: &JsValue) -> Result { diff --git a/lib/benches/store.rs b/lib/benches/store.rs index 20826041..ae56266b 100644 --- a/lib/benches/store.rs +++ b/lib/benches/store.rs @@ -3,7 +3,6 @@ use criterion::{criterion_group, criterion_main, Criterion, Throughput}; use oxhttp::model::{Method, Request, Status}; use oxigraph::io::RdfFormat; -use oxigraph::model::{GraphName, GraphNameRef}; use oxigraph::sparql::{Query, QueryResults, Update}; use oxigraph::store::Store; use rand::random; @@ -64,16 +63,14 @@ fn store_load(c: &mut Criterion) { } fn do_load(store: &Store, data: &[u8]) { - store - .load_graph(data, RdfFormat::NTriples, GraphName::DefaultGraph, None) - .unwrap(); + store.load_from_read(RdfFormat::NTriples, data).unwrap(); store.optimize().unwrap(); } fn do_bulk_load(store: &Store, data: &[u8]) { store .bulk_loader() - .load_graph(data, RdfFormat::NTriples, GraphNameRef::DefaultGraph, None) + .load_from_read(RdfFormat::NTriples, data) .unwrap(); store.optimize().unwrap(); } diff --git a/lib/oxrdfio/src/parser.rs b/lib/oxrdfio/src/parser.rs index 6258ba18..b9aaf646 100644 --- a/lib/oxrdfio/src/parser.rs +++ b/lib/oxrdfio/src/parser.rs @@ -321,6 +321,12 @@ impl RdfParser { } } +impl From for RdfParser { + fn from(format: RdfFormat) -> Self { + Self::from_format(format) + } +} + /// Parses a RDF file from a [`Read`] implementation. Can be built using [`RdfParser::parse_read`]. /// /// Reads are buffered. diff --git a/lib/oxrdfio/src/serializer.rs b/lib/oxrdfio/src/serializer.rs index 347e40f2..487b4539 100644 --- a/lib/oxrdfio/src/serializer.rs +++ b/lib/oxrdfio/src/serializer.rs @@ -158,6 +158,12 @@ impl RdfSerializer { } } +impl From for RdfSerializer { + fn from(format: RdfFormat) -> Self { + Self::from_format(format) + } +} + /// Writes quads or triples to a [`Write`] implementation. /// /// Can be built using [`RdfSerializer::serialize_to_write`]. diff --git a/lib/sparesults/src/parser.rs b/lib/sparesults/src/parser.rs index b0aad24c..f54f85d3 100644 --- a/lib/sparesults/src/parser.rs +++ b/lib/sparesults/src/parser.rs @@ -118,6 +118,12 @@ impl QueryResultsParser { } } +impl From for QueryResultsParser { + fn from(format: QueryResultsFormat) -> Self { + Self::from_format(format) + } +} + /// The reader for a given read of a results file. /// /// It is either a read boolean ([`bool`]) or a streaming reader of a set of solutions ([`FromReadSolutionsReader`]). diff --git a/lib/sparesults/src/serializer.rs b/lib/sparesults/src/serializer.rs index 95356fb8..dc0baeb9 100644 --- a/lib/sparesults/src/serializer.rs +++ b/lib/sparesults/src/serializer.rs @@ -213,6 +213,12 @@ impl QueryResultsSerializer { } } +impl From for QueryResultsSerializer { + fn from(format: QueryResultsFormat) -> Self { + Self::from_format(format) + } +} + /// Allows writing query results into a [`Write`] implementation. /// /// Could be built using a [`QueryResultsSerializer`]. diff --git a/lib/src/store.rs b/lib/src/store.rs index fa1a243f..e78a137b 100644 --- a/lib/src/store.rs +++ b/lib/src/store.rs @@ -445,6 +445,57 @@ impl Store { .transaction(|mut t| evaluate_update(&mut t, &update, &options)) } + /// Loads a RDF file under into the store. + /// + /// This function is atomic, quite slow and memory hungry. To get much better performances you might want to use the [`bulk_loader`](Store::bulk_loader). + /// + /// Usage example: + /// ``` + /// use oxigraph::store::Store; + /// use oxigraph::io::RdfFormat; + /// use oxigraph::model::*; + /// use oxrdfio::RdfParser; + /// + /// let store = Store::new()?; + /// + /// // insert a dataset file (former load_dataset method) + /// let file = b" ."; + /// store.load_from_read(RdfFormat::NQuads, file.as_ref())?; + /// + /// // insert a graph file (former load_graph method) + /// let file = b"<> <> <> ."; + /// store.load_from_read( + /// RdfParser::from_format(RdfFormat::Turtle) + /// .with_base_iri("http://example.com")? + /// .without_named_graphs() // No named graphs allowed in the input + /// .with_default_graph(NamedNodeRef::new("http://example.com/g2")?), // we put the file default graph inside of a named graph + /// file.as_ref() + /// )?; + /// + /// // we inspect the store contents + /// let ex = NamedNodeRef::new("http://example.com")?; + /// assert!(store.contains(QuadRef::new(ex, ex, ex, NamedNodeRef::new("http://example.com/g")?))?); + /// assert!(store.contains(QuadRef::new(ex, ex, ex, NamedNodeRef::new("http://example.com/g2")?))?); + /// # Result::<_, Box>::Ok(()) + /// ``` + pub fn load_from_read( + &self, + parser: impl Into, + read: impl Read, + ) -> Result<(), LoaderError> { + let quads = parser + .into() + .rename_blank_nodes() + .parse_read(read) + .collect::, _>>()?; + self.storage.transaction(move |mut t| { + for quad in &quads { + t.insert(quad.as_ref())?; + } + Ok(()) + }) + } + /// Loads a graph file (i.e. triples) into the store. /// /// This function is atomic, quite slow and memory hungry. To get much better performances you might want to use the [`bulk_loader`](Store::bulk_loader). @@ -466,6 +517,7 @@ impl Store { /// assert!(store.contains(QuadRef::new(ex, ex, ex, GraphNameRef::DefaultGraph))?); /// # Result::<_, Box>::Ok(()) /// ``` + #[deprecated(note = "Use Store.load_from_read instead")] pub fn load_graph( &self, read: impl Read, @@ -475,8 +527,7 @@ impl Store { ) -> Result<(), LoaderError> { let mut parser = RdfParser::from_format(format.into()) .without_named_graphs() - .with_default_graph(to_graph_name) - .rename_blank_nodes(); + .with_default_graph(to_graph_name); if let Some(base_iri) = base_iri { parser = parser .with_base_iri(base_iri) @@ -485,13 +536,7 @@ impl Store { error: e, })?; } - let quads = parser.parse_read(read).collect::, _>>()?; - self.storage.transaction(move |mut t| { - for quad in &quads { - t.insert(quad.as_ref())?; - } - Ok(()) - }) + self.load_from_read(parser, read) } /// Loads a dataset file (i.e. quads) into the store. @@ -515,13 +560,14 @@ impl Store { /// assert!(store.contains(QuadRef::new(ex, ex, ex, ex))?); /// # Result::<_, Box>::Ok(()) /// ``` + #[deprecated(note = "Use Store.load_from_read instead")] pub fn load_dataset( &self, read: impl Read, format: impl Into, base_iri: Option<&str>, ) -> Result<(), LoaderError> { - let mut parser = RdfParser::from_format(format.into()).rename_blank_nodes(); + let mut parser = RdfParser::from_format(format.into()); if let Some(base_iri) = base_iri { parser = parser .with_base_iri(base_iri) @@ -530,13 +576,7 @@ impl Store { error: e, })?; } - let quads = parser.parse_read(read).collect::, _>>()?; - self.storage.transaction(move |mut t| { - for quad in &quads { - t.insert(quad.as_ref())?; - } - Ok(()) - }) + self.load_from_read(parser, read) } /// Adds a quad to this store. @@ -1062,6 +1102,53 @@ impl<'a> Transaction<'a> { ) } + /// Loads a RDF file into the store. + /// + /// This function is atomic, quite slow and memory hungry. To get much better performances you might want to use the [`bulk_loader`](Store::bulk_loader). + /// + /// Usage example: + /// ``` + /// use oxigraph::store::Store; + /// use oxigraph::io::RdfFormat; + /// use oxigraph::model::*; + /// use oxrdfio::RdfParser; + /// + /// let store = Store::new()?; + /// + /// // insert a dataset file (former load_dataset method) + /// let file = b" ."; + /// store.transaction(|mut t| t.load_from_read(RdfFormat::NQuads, file.as_ref()))?; + /// + /// // insert a graph file (former load_graph method) + /// let file = b"<> <> <> ."; + /// store.transaction(|mut t| + /// t.load_from_read( + /// RdfParser::from_format(RdfFormat::Turtle) + /// .with_base_iri("http://example.com") + /// .unwrap() + /// .without_named_graphs() // No named graphs allowed in the input + /// .with_default_graph(NamedNodeRef::new("http://example.com/g2").unwrap()), // we put the file default graph inside of a named graph + /// file.as_ref() + /// ) + /// )?; + /// + /// // we inspect the store contents + /// let ex = NamedNodeRef::new("http://example.com")?; + /// assert!(store.contains(QuadRef::new(ex, ex, ex, NamedNodeRef::new("http://example.com/g")?))?); + /// assert!(store.contains(QuadRef::new(ex, ex, ex, NamedNodeRef::new("http://example.com/g2")?))?); + /// # Result::<_, Box>::Ok(()) + /// ``` + pub fn load_from_read( + &mut self, + parser: impl Into, + read: impl Read, + ) -> Result<(), LoaderError> { + for quad in parser.into().rename_blank_nodes().parse_read(read) { + self.insert(quad?.as_ref())?; + } + Ok(()) + } + /// Loads a graph file (i.e. triples) into the store. /// /// Usage example: @@ -1083,6 +1170,7 @@ impl<'a> Transaction<'a> { /// assert!(store.contains(QuadRef::new(ex, ex, ex, GraphNameRef::DefaultGraph))?); /// # Result::<_,oxigraph::store::LoaderError>::Ok(()) /// ``` + #[deprecated(note = "Use Transaction.load_from_read instead")] pub fn load_graph( &mut self, read: impl Read, @@ -1092,8 +1180,7 @@ impl<'a> Transaction<'a> { ) -> Result<(), LoaderError> { let mut parser = RdfParser::from_format(format.into()) .without_named_graphs() - .with_default_graph(to_graph_name) - .rename_blank_nodes(); + .with_default_graph(to_graph_name); if let Some(base_iri) = base_iri { parser = parser .with_base_iri(base_iri) @@ -1102,10 +1189,7 @@ impl<'a> Transaction<'a> { error: e, })?; } - for quad in parser.parse_read(read) { - self.writer.insert(quad?.as_ref())?; - } - Ok(()) + self.load_from_read(parser, read) } /// Loads a dataset file (i.e. quads) into the store. @@ -1129,13 +1213,14 @@ impl<'a> Transaction<'a> { /// assert!(store.contains(QuadRef::new(ex, ex, ex, ex))?); /// # Result::<_,oxigraph::store::LoaderError>::Ok(()) /// ``` + #[deprecated(note = "Use Transaction.load_from_read instead")] pub fn load_dataset( &mut self, read: impl Read, format: impl Into, base_iri: Option<&str>, ) -> Result<(), LoaderError> { - let mut parser = RdfParser::from_format(format.into()).rename_blank_nodes(); + let mut parser = RdfParser::from_format(format.into()); if let Some(base_iri) = base_iri { parser = parser .with_base_iri(base_iri) @@ -1144,10 +1229,7 @@ impl<'a> Transaction<'a> { error: e, })?; } - for quad in parser.parse_read(read) { - self.writer.insert(quad?.as_ref())?; - } - Ok(()) + self.load_from_read(parser, read) } /// Adds a quad to this store. @@ -1448,6 +1530,73 @@ impl BulkLoader { self } + /// Loads a file using the bulk loader. + /// + /// This function is optimized for large dataset loading speed. For small files, [`Store::load_dataset`] might be more convenient. + /// + ///
This method is not atomic. + /// If the parsing fails in the middle of the file, only a part of it may be written to the store. + /// Results might get weird if you delete data during the loading process.
+ /// + ///
This method is optimized for speed. See [the struct](BulkLoader) documentation for more details.
+ /// + /// Usage example: + /// Usage example: + /// ``` + /// use oxigraph::store::Store; + /// use oxigraph::io::RdfFormat; + /// use oxigraph::model::*; + /// use oxrdfio::RdfParser; + /// + /// let store = Store::new()?; + /// + /// // insert a dataset file (former load_dataset method) + /// let file = b" ."; + /// store.bulk_loader().load_from_read(RdfFormat::NQuads, file.as_ref())?; + /// + /// // insert a graph file (former load_graph method) + /// let file = b"<> <> <> ."; + /// store.bulk_loader().load_from_read( + /// RdfParser::from_format(RdfFormat::Turtle) + /// .with_base_iri("http://example.com")? + /// .without_named_graphs() // No named graphs allowed in the input + /// .with_default_graph(NamedNodeRef::new("http://example.com/g2")?), // we put the file default graph inside of a named graph + /// file.as_ref() + /// )?; + /// + /// // we inspect the store contents + /// let ex = NamedNodeRef::new("http://example.com")?; + /// assert!(store.contains(QuadRef::new(ex, ex, ex, NamedNodeRef::new("http://example.com/g")?))?); + /// assert!(store.contains(QuadRef::new(ex, ex, ex, NamedNodeRef::new("http://example.com/g2")?))?); + /// # Result::<_, Box>::Ok(()) + /// ``` + pub fn load_from_read( + &self, + parser: impl Into, + read: impl Read, + ) -> Result<(), LoaderError> { + self.load_ok_quads( + parser + .into() + .rename_blank_nodes() + .parse_read(read) + .filter_map(|r| match r { + Ok(q) => Some(Ok(q)), + Err(e) => { + if let Some(callback) = &self.on_parse_error { + if let Err(e) = callback(e) { + Some(Err(e)) + } else { + None + } + } else { + Some(Err(e)) + } + } + }), + ) + } + /// Loads a dataset file using the bulk loader. /// /// This function is optimized for large dataset loading speed. For small files, [`Store::load_dataset`] might be more convenient. @@ -1475,6 +1624,7 @@ impl BulkLoader { /// assert!(store.contains(QuadRef::new(ex, ex, ex, ex))?); /// # Result::<_, Box>::Ok(()) /// ``` + #[deprecated(note = "Use BulkLoader.load_from_read instead")] pub fn load_dataset( &self, read: impl Read, @@ -1533,6 +1683,7 @@ impl BulkLoader { /// assert!(store.contains(QuadRef::new(ex, ex, ex, GraphNameRef::DefaultGraph))?); /// # Result::<_, Box>::Ok(()) /// ``` + #[deprecated(note = "Use BulkLoader.load_from_read instead")] pub fn load_graph( &self, read: impl Read, diff --git a/lib/tests/store.rs b/lib/tests/store.rs index c9e67bfb..070a5351 100644 --- a/lib/tests/store.rs +++ b/lib/tests/store.rs @@ -110,12 +110,7 @@ fn quads(graph_name: impl Into>) -> Vec> #[test] fn test_load_graph() -> Result<(), Box> { let store = Store::new()?; - store.load_graph( - DATA.as_bytes(), - RdfFormat::Turtle, - GraphNameRef::DefaultGraph, - None, - )?; + store.load_from_read(RdfFormat::Turtle, DATA.as_bytes())?; for q in quads(GraphNameRef::DefaultGraph) { assert!(store.contains(q)?); } @@ -127,12 +122,9 @@ fn test_load_graph() -> Result<(), Box> { #[cfg(not(target_family = "wasm"))] fn test_bulk_load_graph() -> Result<(), Box> { let store = Store::new()?; - store.bulk_loader().load_graph( - DATA.as_bytes(), - RdfFormat::Turtle, - GraphName::DefaultGraph, - None, - )?; + store + .bulk_loader() + .load_from_read(RdfFormat::Turtle, DATA.as_bytes())?; for q in quads(GraphNameRef::DefaultGraph) { assert!(store.contains(q)?); } @@ -144,11 +136,9 @@ fn test_bulk_load_graph() -> Result<(), Box> { #[cfg(not(target_family = "wasm"))] fn test_bulk_load_graph_lenient() -> Result<(), Box> { let store = Store::new()?; - store.bulk_loader().on_parse_error(|_| Ok(())).load_graph( - b" .\n .".as_slice(), + store.bulk_loader().on_parse_error(|_| Ok(())).load_from_read( RdfFormat::NTriples, - GraphName::DefaultGraph, - None, + b" .\n .".as_slice(), )?; assert_eq!(store.len()?, 1); assert!(store.contains(QuadRef::new( @@ -164,7 +154,7 @@ fn test_bulk_load_graph_lenient() -> Result<(), Box> { #[test] fn test_load_dataset() -> Result<(), Box> { let store = Store::new()?; - store.load_dataset(GRAPH_DATA.as_bytes(), RdfFormat::TriG, None)?; + store.load_from_read(RdfFormat::TriG, GRAPH_DATA.as_bytes())?; for q in quads(NamedNodeRef::new_unchecked( "http://www.wikidata.org/wiki/Special:EntityData/Q90", )) { @@ -180,7 +170,7 @@ fn test_bulk_load_dataset() -> Result<(), Box> { let store = Store::new()?; store .bulk_loader() - .load_dataset(GRAPH_DATA.as_bytes(), RdfFormat::TriG, None)?; + .load_from_read(RdfFormat::TriG, GRAPH_DATA.as_bytes())?; let graph_name = NamedNodeRef::new_unchecked("http://www.wikidata.org/wiki/Special:EntityData/Q90"); for q in quads(graph_name) { @@ -195,11 +185,9 @@ fn test_bulk_load_dataset() -> Result<(), Box> { fn test_load_graph_generates_new_blank_nodes() -> Result<(), Box> { let store = Store::new()?; for _ in 0..2 { - store.load_graph( - "_:a .".as_bytes(), + store.load_from_read( RdfFormat::NTriples, - GraphName::DefaultGraph, - None, + "_:a .".as_bytes(), )?; } assert_eq!(store.len()?, 2); diff --git a/python/src/store.rs b/python/src/store.rs index 910247df..6068296d 100644 --- a/python/src/store.rs +++ b/python/src/store.rs @@ -6,6 +6,7 @@ use crate::io::{ }; use crate::model::*; use crate::sparql::*; +use oxigraph::io::RdfParser; use oxigraph::model::{GraphName, GraphNameRef}; use oxigraph::sparql::Update; use oxigraph::store::{self, LoaderError, SerializerError, StorageError, Store}; @@ -399,13 +400,18 @@ impl PyStore { let input = PyReadable::from_args(&path, input, py)?; let format = lookup_rdf_format(format, path.as_deref())?; py.allow_threads(|| { + let mut parser = RdfParser::from_format(format); + if let Some(base_iri) = base_iri { + parser = parser + .with_base_iri(base_iri) + .map_err(|e| PyValueError::new_err(e.to_string()))?; + } if let Some(to_graph_name) = to_graph_name { - self.inner - .load_graph(input, format, to_graph_name, base_iri) - } else { - self.inner.load_dataset(input, format, base_iri) + parser = parser.with_default_graph(to_graph_name); } - .map_err(|e| map_loader_error(e, path)) + self.inner + .load_from_read(parser, input) + .map_err(|e| map_loader_error(e, path)) }) } @@ -466,16 +472,18 @@ impl PyStore { let input = PyReadable::from_args(&path, input, py)?; let format = lookup_rdf_format(format, path.as_deref())?; py.allow_threads(|| { + let mut parser = RdfParser::from_format(format); + if let Some(base_iri) = base_iri { + parser = parser + .with_base_iri(base_iri) + .map_err(|e| PyValueError::new_err(e.to_string()))?; + } if let Some(to_graph_name) = to_graph_name { - self.inner - .bulk_loader() - .load_graph(input, format, to_graph_name, base_iri) - } else { - self.inner - .bulk_loader() - .load_dataset(input, format, base_iri) + parser = parser.with_default_graph(to_graph_name); } - .map_err(|e| map_loader_error(e, path)) + self.inner + .load_from_read(parser, input) + .map_err(|e| map_loader_error(e, path)) }) } diff --git a/testsuite/src/sparql_evaluator.rs b/testsuite/src/sparql_evaluator.rs index 383be3e2..df9d471e 100644 --- a/testsuite/src/sparql_evaluator.rs +++ b/testsuite/src/sparql_evaluator.rs @@ -4,6 +4,7 @@ use crate::manifest::*; use crate::report::{dataset_diff, format_diff}; use crate::vocab::*; use anyhow::{bail, ensure, Context, Error, Result}; +use oxigraph::io::RdfParser; use oxigraph::model::vocab::*; use oxigraph::model::*; use oxigraph::sparql::results::QueryResultsFormat; @@ -129,10 +130,10 @@ fn evaluate_negative_result_syntax_test(test: &Test, format: QueryResultsFormat) fn evaluate_evaluation_test(test: &Test) -> Result<()> { let store = get_store()?; if let Some(data) = &test.data { - load_dataset_to_store(data, &store)?; + load_to_store(data, &store, GraphName::DefaultGraph)?; } for (name, value) in &test.graph_data { - load_graph_to_store(value, &store, name.clone())?; + load_to_store(value, &store, name.clone())?; } let query_file = test.query.as_deref().context("No action found")?; let options = QueryOptions::default() @@ -150,13 +151,13 @@ fn evaluate_evaluation_test(test: &Test) -> Result<()> { let GraphName::NamedNode(graph_name) = graph_name else { bail!("Invalid FROM in query {query}"); }; - load_graph_to_store(graph_name.as_str(), &store, graph_name.as_ref())?; + load_to_store(graph_name.as_str(), &store, graph_name.as_ref())?; } for graph_name in query.dataset().available_named_graphs().unwrap_or(&[]) { let NamedOrBlankNode::NamedNode(graph_name) = graph_name else { bail!("Invalid FROM NAMED in query {query}"); }; - load_graph_to_store(graph_name.as_str(), &store, graph_name.as_ref())?; + load_to_store(graph_name.as_str(), &store, graph_name.as_ref())?; } } @@ -210,18 +211,18 @@ fn evaluate_negative_update_syntax_test(test: &Test) -> Result<()> { fn evaluate_update_evaluation_test(test: &Test) -> Result<()> { let store = get_store()?; if let Some(data) = &test.data { - load_dataset_to_store(data, &store)?; + load_to_store(data, &store, GraphName::DefaultGraph)?; } for (name, value) in &test.graph_data { - load_graph_to_store(value, &store, name.clone())?; + load_to_store(value, &store, name.clone())?; } let result_store = get_store()?; if let Some(data) = &test.result { - load_dataset_to_store(data, &result_store)?; + load_to_store(data, &result_store, GraphName::DefaultGraph)?; } for (name, value) in &test.result_graph_data { - load_graph_to_store(value, &result_store, name.clone())?; + load_to_store(value, &result_store, name.clone())?; } let update_file = test.update.as_deref().context("No action found")?; @@ -271,7 +272,7 @@ impl StaticServiceHandler { .map(|(name, data)| { let name = NamedNode::new(name)?; let store = get_store()?; - load_dataset_to_store(data, &store)?; + load_to_store(data, &store, GraphName::DefaultGraph)?; Ok((name, store)) }) .collect::>()?, @@ -643,25 +644,16 @@ fn solutions_to_string(solutions: Vec>, ordered: bool) -> lines.join("\n") } -fn load_graph_to_store( - url: &str, - store: &Store, - to_graph_name: impl Into, -) -> Result<()> { - store.load_graph( +fn load_to_store(url: &str, store: &Store, to_graph_name: impl Into) -> Result<()> { + store.load_from_read( + RdfParser::from_format(guess_rdf_format(url)?) + .with_base_iri(url)? + .with_default_graph(to_graph_name), read_file(url)?, - guess_rdf_format(url)?, - to_graph_name, - Some(url), )?; Ok(()) } -fn load_dataset_to_store(url: &str, store: &Store) -> Result<()> { - store.load_dataset(read_file(url)?, guess_rdf_format(url)?, Some(url))?; - Ok(()) -} - fn evaluate_query_optimization_test(test: &Test) -> Result<()> { let action = test.action.as_deref().context("No action found")?; let actual = (&Optimizer::optimize_graph_pattern( From 4841f890725d72e5da7d7f25f2cf0754c63cc572 Mon Sep 17 00:00:00 2001 From: Tpt Date: Thu, 7 Dec 2023 18:49:20 +0100 Subject: [PATCH 142/217] Server: allows Content-Encoding: gzip requests --- Cargo.lock | 64 ++++++++++++++++++++++++++++++++------------------ cli/Cargo.toml | 2 +- lib/Cargo.toml | 6 ++--- 3 files changed, 45 insertions(+), 27 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fc64ec63..04eb776b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -994,14 +994,16 @@ dependencies = [ [[package]] name = "oxhttp" -version = "0.2.0-alpha.2" +version = "0.2.0-alpha.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9546c028c11488f4661926fe18ced7dd11b573b92c3a73a2c33cd1fd859883e0" +checksum = "7c066f9fac8c5b7cad775f89a53a039a74107b384e170886e748a3d6a9ffe1fd" dependencies = [ + "flate2", "httparse", "native-tls", "rustls", "rustls-native-certs", + "rustls-pki-types", "url", "webpki-roots", ] @@ -1551,44 +1553,55 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.9" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "629648aced5775d558af50b2b4c7b02983a04b312126d45eeead26e7caa498b9" +checksum = "5bc238b76c51bbc449c55ffbc39d03772a057cc8cf783c49d4af4c2537b74a8b" dependencies = [ "log", "ring", + "rustls-pki-types", "rustls-webpki", - "sct", + "subtle", + "zeroize", ] [[package]] name = "rustls-native-certs" -version = "0.6.3" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" +checksum = "8f1fb85efa936c42c6d5fc28d2629bb51e4b2f4b8a5211e297d599cc5a093792" dependencies = [ "openssl-probe", "rustls-pemfile", + "rustls-pki-types", "schannel", "security-framework", ] [[package]] name = "rustls-pemfile" -version = "1.0.4" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +checksum = "35e4980fa29e4c4b212ffb3db068a564cbf560e51d3944b7c88bd8bf5bec64f4" dependencies = [ "base64", + "rustls-pki-types", ] +[[package]] +name = "rustls-pki-types" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7673e0aa20ee4937c6aacfc12bb8341cfbf054cdd21df6bec5fd0629fe9339b" + [[package]] name = "rustls-webpki" -version = "0.101.7" +version = "0.102.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +checksum = "de2635c8bc2b88d367767c5de8ea1d8db9af3f6219eba28442242d9ab81d1b89" dependencies = [ "ring", + "rustls-pki-types", "untrusted", ] @@ -1622,16 +1635,6 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" -[[package]] -name = "sct" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "security-framework" version = "2.9.2" @@ -1776,6 +1779,12 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +[[package]] +name = "subtle" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" + [[package]] name = "syn" version = "2.0.39" @@ -2084,9 +2093,12 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.25.2" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14247bb57be4f377dfb94c72830b8ce8fc6beac03cf4bf7b9732eadd414123fc" +checksum = "0de2cfda980f21be5a7ed2eadb3e6fe074d56022bea2cdeb1a62eb220fc04188" +dependencies = [ + "rustls-pki-types", +] [[package]] name = "which" @@ -2209,6 +2221,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" +[[package]] +name = "zeroize" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" + [[package]] name = "zstd" version = "0.13.0" diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 252ae5ff..0e3e4471 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -27,7 +27,7 @@ rustls-webpki = ["oxigraph/http-client-rustls-webpki"] [dependencies] anyhow = "1.0.72" -oxhttp = "0.2.0-alpha.2" +oxhttp = { version = "0.2.0-alpha.3", features = ["flate2"] } clap = { version = "4.0", features = ["derive"] } oxigraph = { version = "0.4.0-alpha.1-dev", path = "../lib" } rand = "0.8" diff --git a/lib/Cargo.toml b/lib/Cargo.toml index f2adcc8c..39a0661d 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -47,15 +47,15 @@ sparopt = { version = "0.1.0-alpha.1-dev", path="sparopt", features = ["rdf-star [target.'cfg(not(target_family = "wasm"))'.dependencies] libc = "0.2.147" oxrocksdb-sys = { version = "0.4.0-alpha.1-dev", path="../oxrocksdb-sys" } -oxhttp = { version = "0.2.0-alpha.2", optional = true } +oxhttp = { version = "0.2.0-alpha.3", optional = true } [target.'cfg(all(target_family = "wasm", target_os = "unknown"))'.dependencies] -getrandom = "0.2" +getrandom = "0.2.8" js-sys = { version = "0.3.60", optional = true } [target.'cfg(not(target_family = "wasm"))'.dev-dependencies] criterion = "0.5" -oxhttp = "0.2.0-alpha.2" +oxhttp = "0.2.0-alpha.3" zstd = ">=0.12, <0.14" [package.metadata.docs.rs] From 3241f47059f37148fb9c9b02b2f20324b28e93c0 Mon Sep 17 00:00:00 2001 From: Tpt Date: Thu, 7 Dec 2023 18:54:04 +0100 Subject: [PATCH 143/217] Docker: use bookworm --- cli/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/Dockerfile b/cli/Dockerfile index 9454410f..82f4bfe8 100644 --- a/cli/Dockerfile +++ b/cli/Dockerfile @@ -1,4 +1,4 @@ -FROM --platform=$BUILDPLATFORM rust:1-bullseye as builder +FROM --platform=$BUILDPLATFORM rust:1-bookworm as builder ARG BUILDARCH TARGETARCH RUN apt-get update && \ apt-get install -y libclang-dev clang && \ From 4b3f3f327803df0e00406f394ce740e5c4857ef6 Mon Sep 17 00:00:00 2001 From: Tpt Date: Thu, 7 Dec 2023 21:21:37 +0100 Subject: [PATCH 144/217] RDF/XML: properly serialize predicates with xmlns: prefix --- lib/oxrdfxml/src/serializer.rs | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/lib/oxrdfxml/src/serializer.rs b/lib/oxrdfxml/src/serializer.rs index 5883c31f..a19856f0 100644 --- a/lib/oxrdfxml/src/serializer.rs +++ b/lib/oxrdfxml/src/serializer.rs @@ -258,16 +258,7 @@ impl InnerRdfXmlWriter { if prop_prefix == "http://www.w3.org/1999/02/22-rdf-syntax-ns#" { (Cow::Owned(format!("rdf:{prop_value}")), None) } else if prop_prefix == "http://www.w3.org/2000/xmlns/" { - if prop_value.is_empty() { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "The http://www.w3.org/2000/xmlns/ predicate IRI is not allowed in XML", - )); - } - ( - Cow::Borrowed("p:"), - Some(("xmlns:p", triple.predicate.as_str())), - ) + (Cow::Owned(format!("xmlns:{prop_value}")), None) } else if prop_value.is_empty() { (Cow::Borrowed("p:"), Some(("xmlns:p", prop_prefix))) } else { From 735db897ff71fa634ed9220321f614299f3064da Mon Sep 17 00:00:00 2001 From: etiennept <47983879+etiennept@users.noreply.github.com> Date: Fri, 29 Dec 2023 17:39:34 +0100 Subject: [PATCH 145/217] Update package.json replace mv pkg with out-dir --- js/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/js/package.json b/js/package.json index e8744994..c1405e4b 100644 --- a/js/package.json +++ b/js/package.json @@ -10,7 +10,7 @@ "scripts": { "fmt": "rome format . --write && rome check . --apply-unsafe", "test": "rome ci . && wasm-pack build --debug --target nodejs && mocha", - "build": "rm -rf pkg && wasm-pack build --release --target web --out-name web && mv pkg pkg-web && wasm-pack build --release --target nodejs --out-name node && mv pkg pkg-node && node build_package.js && rm -r pkg-web && rm -r pkg-node", + "build": "rm -rf pkg && wasm-pack build --release --target web --out-name web --out-dir pkg-web && wasm-pack build --release --target nodejs --out-name node --out-dir pkg-node && node build_package.js && rm -r pkg-web && rm -r pkg-node", "release": "npm run build && npm publish ./pkg", "pack": "npm run build && npm pack ./pkg" }, From 2a135283d50a35a07944e6e10bd15ac523d97ed6 Mon Sep 17 00:00:00 2001 From: Tpt Date: Fri, 29 Dec 2023 18:25:56 +0100 Subject: [PATCH 146/217] Lints against 1.74 --- .cargo/config.toml | 12 ++++++++++++ .github/workflows/tests.yml | 16 ++++++++-------- lib/oxsdatatypes/src/double.rs | 2 +- lib/oxsdatatypes/src/float.rs | 2 +- lib/oxttl/src/line_formats.rs | 2 +- lib/spargebra/src/algebra.rs | 2 +- lib/spargebra/src/parser.rs | 17 +++++++++-------- lib/src/sparql/eval.rs | 2 +- lib/src/sparql/mod.rs | 2 +- {.cargo => lints}/build_config.py | 12 +++++++++--- 10 files changed, 44 insertions(+), 25 deletions(-) rename {.cargo => lints}/build_config.py (89%) diff --git a/.cargo/config.toml b/.cargo/config.toml index 3f1a93ac..dad85ce3 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -29,6 +29,7 @@ rustflags = [ "-Wclippy::empty-enum", "-Wclippy::empty-structs-with-brackets", "-Wclippy::enum-glob-use", + "-Wclippy::error-impl-error", "-Wclippy::exit", "-Wclippy::expect-used", "-Wclippy::expl-impl-clone-on-copy", @@ -43,8 +44,10 @@ rustflags = [ "-Wclippy::format-push-string", "-Wclippy::from-iter-instead-of-collect", "-Wclippy::get-unwrap", + "-Wclippy::host-endian-bytes", "-Wclippy::if-not-else", "-Wclippy::if-then-some-else-none", + "-Wclippy::ignored-unit-patterns", "-Wclippy::implicit-clone", "-Wclippy::implicit-hasher", "-Wclippy::inconsistent-struct-constructor", @@ -82,7 +85,9 @@ rustflags = [ "-Wclippy::mem-forget", "-Wclippy::mismatching-type-param-order", "-Wclippy::missing-assert-message", + "-Wclippy::missing-asserts-for-indexing", "-Wclippy::missing-enforced-import-renames", + "-Wclippy::missing-fields-in-debug", "-Wclippy::multiple-inherent-impl", "-Wclippy::mut-mut", "-Wclippy::mutex-atomic", @@ -91,6 +96,7 @@ rustflags = [ "-Wclippy::needless-continue", "-Wclippy::needless-for-each", "-Wclippy::needless-pass-by-value", + "-Wclippy::needless-raw-strings", "-Wclippy::negative-feature-names", "-Wclippy::no-effect-underscore-binding", "-Wclippy::no-mangle-with-rust-abi", @@ -101,6 +107,8 @@ rustflags = [ "-Wclippy::print-stderr", "-Wclippy::print-stdout", "-Wclippy::ptr-as-ptr", + "-Wclippy::ptr-cast-constness", + "-Wclippy::pub-without-shorthand", "-Wclippy::range-minus-one", "-Wclippy::range-plus-one", "-Wclippy::rc-buffer", @@ -108,19 +116,23 @@ rustflags = [ "-Wclippy::redundant-closure-for-method-calls", "-Wclippy::redundant-else", "-Wclippy::redundant-feature-names", + "-Wclippy::redundant-type-annotations", "-Wclippy::ref-binding-to-reference", "-Wclippy::ref-option-ref", + "-Wclippy::ref-patterns", "-Wclippy::rest-pat-in-fully-bound-structs", "-Wclippy::return-self-not-must-use", "-Wclippy::same-functions-in-if-condition", "-Wclippy::same-name-method", "-Wclippy::semicolon-inside-block", "-Wclippy::shadow-same", + "-Wclippy::should-panic-without-expect", "-Wclippy::single-match-else", "-Wclippy::stable-sort-primitive", "-Wclippy::str-to-string", "-Wclippy::string-add", "-Wclippy::string-add-assign", + "-Wclippy::string-lit-chars-any", "-Wclippy::string-to-string", "-Wclippy::struct-excessive-bools", "-Wclippy::suspicious-xor-used-as-pow", diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 139e3172..7ce2c3b0 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -26,7 +26,7 @@ jobs: - uses: actions/checkout@v3 with: submodules: true - - run: rustup update && rustup default 1.70.0 && rustup component add clippy + - run: rustup update && rustup default 1.74.1 && rustup component add clippy - uses: Swatinem/rust-cache@v2 - run: cargo clippy --all-targets -- -D warnings -D clippy::all working-directory: ./lib/oxsdatatypes @@ -59,7 +59,7 @@ jobs: - uses: actions/checkout@v3 with: submodules: true - - run: rustup update && rustup default 1.70.0 && rustup target add wasm32-unknown-unknown && rustup component add clippy + - run: rustup update && rustup default 1.74.1 && rustup target add wasm32-unknown-unknown && rustup component add clippy - uses: Swatinem/rust-cache@v2 - run: cargo clippy --lib --tests --target wasm32-unknown-unknown -- -D warnings -D clippy::all working-directory: ./js @@ -70,7 +70,7 @@ jobs: - uses: actions/checkout@v3 with: submodules: true - - run: rustup update && rustup default 1.70.0 && rustup target add wasm32-wasi && rustup component add clippy + - run: rustup update && rustup default 1.74.1 && rustup target add wasm32-wasi && rustup component add clippy - uses: Swatinem/rust-cache@v2 - run: cargo clippy --lib --tests --target wasm32-wasi -- -D warnings -D clippy::all working-directory: ./lib @@ -81,7 +81,7 @@ jobs: - uses: actions/checkout@v3 with: submodules: true - - run: rustup update && rustup default 1.70.0 && rustup target add wasm32-unknown-unknown && rustup component add clippy + - run: rustup update && rustup default 1.74.1 && rustup target add wasm32-unknown-unknown && rustup component add clippy - uses: Swatinem/rust-cache@v2 - run: cargo clippy --lib --tests --target wasm32-unknown-unknown --features getrandom/custom --features oxsdatatypes/custom-now -- -D warnings -D clippy::all working-directory: ./lib @@ -123,7 +123,7 @@ jobs: - uses: actions/checkout@v3 with: submodules: true - - run: rustup update && rustup toolchain install nightly && rustup default 1.70.0 + - run: rustup update && rustup toolchain install nightly && rustup default 1.74.1 - uses: Swatinem/rust-cache@v2 - run: rm Cargo.lock && cargo +nightly update -Z direct-minimal-versions - run: cargo test @@ -200,7 +200,7 @@ jobs: - uses: actions/checkout@v3 with: submodules: true - - run: rustup update && rustup default 1.70.0 + - run: rustup update && rustup default 1.74.1 - uses: Swatinem/rust-cache@v2 - run: cargo doc working-directory: ./lib @@ -264,7 +264,7 @@ jobs: - uses: actions/checkout@v3 with: submodules: true - - run: rustup update && rustup toolchain install nightly && rustup default 1.70.0 + - run: rustup update && rustup toolchain install nightly && rustup default 1.74.1 - uses: Swatinem/rust-cache@v2 - uses: actions/setup-python@v4 with: @@ -285,7 +285,7 @@ jobs: - uses: actions/checkout@v3 with: submodules: true - - run: rustup update && rustup toolchain install nightly && rustup default 1.70.0 + - run: rustup update && rustup toolchain install nightly && rustup default 1.74.1 - uses: Swatinem/rust-cache@v2 - uses: actions/setup-python@v4 with: diff --git a/lib/oxsdatatypes/src/double.rs b/lib/oxsdatatypes/src/double.rs index 5ce5790d..a9d19ac5 100644 --- a/lib/oxsdatatypes/src/double.rs +++ b/lib/oxsdatatypes/src/double.rs @@ -75,7 +75,7 @@ impl Double { #[inline] #[must_use] pub fn is_identical_with(self, other: Self) -> bool { - self.value.to_ne_bytes() == other.value.to_ne_bytes() + self.value.to_bits() == other.value.to_bits() } pub const MIN: Self = Self { value: f64::MIN }; diff --git a/lib/oxsdatatypes/src/float.rs b/lib/oxsdatatypes/src/float.rs index e8784b84..af4c66f7 100644 --- a/lib/oxsdatatypes/src/float.rs +++ b/lib/oxsdatatypes/src/float.rs @@ -75,7 +75,7 @@ impl Float { #[inline] #[must_use] pub fn is_identical_with(self, other: Self) -> bool { - self.value.to_ne_bytes() == other.value.to_ne_bytes() + self.value.to_bits() == other.value.to_bits() } pub const MIN: Self = Self { value: f32::MIN }; diff --git a/lib/oxttl/src/line_formats.rs b/lib/oxttl/src/line_formats.rs index 4d6919a5..8c22c1f8 100644 --- a/lib/oxttl/src/line_formats.rs +++ b/lib/oxttl/src/line_formats.rs @@ -246,7 +246,7 @@ impl RuleRecognizer for NQuadsRecognizer { self.emit_quad(results, GraphName::DefaultGraph); errors.push("Triples should be followed by a dot".into()) } - [NQuadsState::ExpectLiteralAnnotationOrGraphNameOrDot { ref value }] => { + [NQuadsState::ExpectLiteralAnnotationOrGraphNameOrDot { value }] => { self.objects.push(Literal::new_simple_literal(value).into()); self.emit_quad(results, GraphName::DefaultGraph); errors.push("Triples should be followed by a dot".into()) diff --git a/lib/spargebra/src/algebra.rs b/lib/spargebra/src/algebra.rs index 6c629246..37d1da8a 100644 --- a/lib/spargebra/src/algebra.rs +++ b/lib/spargebra/src/algebra.rs @@ -969,7 +969,7 @@ impl GraphPattern { right.lookup_in_scope_variables(callback); } Self::Graph { name, inner } => { - if let NamedNodePattern::Variable(ref g) = name { + if let NamedNodePattern::Variable(g) = &name { callback(g); } inner.lookup_in_scope_variables(callback); diff --git a/lib/spargebra/src/parser.rs b/lib/spargebra/src/parser.rs index 52bfbb54..2fb22910 100644 --- a/lib/spargebra/src/parser.rs +++ b/lib/spargebra/src/parser.rs @@ -1,3 +1,4 @@ +#![allow(clippy::ignored_unit_patterns)] use crate::algebra::*; use crate::query::*; use crate::term::*; @@ -87,9 +88,9 @@ impl fmt::Display for ParseError { impl Error for ParseError { #[inline] fn source(&self) -> Option<&(dyn Error + 'static)> { - match self.inner { - ParseErrorKind::InvalidBaseIri(ref e) => Some(e), - ParseErrorKind::Parser(ref e) => Some(e), + match &self.inner { + ParseErrorKind::InvalidBaseIri(e) => Some(e), + ParseErrorKind::Parser(e) => Some(e), } } } @@ -1072,7 +1073,7 @@ parser! { GraphNamePattern::Variable(graph_name) => GraphPattern::Graph { name: graph_name.clone().into(), inner: Box::new(bgp) }, } }).reduce(new_join).unwrap_or_default(); - let delete = d.into_iter().map(GroundQuadPattern::try_from).collect::,_>>().map_err(|_| "Blank nodes are not allowed in DELETE WHERE")?; + let delete = d.into_iter().map(GroundQuadPattern::try_from).collect::,_>>().map_err(|()| "Blank nodes are not allowed in DELETE WHERE")?; Ok(vec![GraphUpdateOperation::DeleteInsert { delete, insert: Vec::new(), @@ -1150,7 +1151,7 @@ parser! { } rule DeleteClause() -> Vec = i("DELETE") _ q:QuadPattern() {? - q.into_iter().map(GroundQuadPattern::try_from).collect::,_>>().map_err(|_| "Blank nodes are not allowed in DELETE WHERE") + q.into_iter().map(GroundQuadPattern::try_from).collect::,_>>().map_err(|()| "Blank nodes are not allowed in DELETE WHERE") } rule InsertClause() -> Vec = i("INSERT") _ q:QuadPattern() { q } @@ -1179,10 +1180,10 @@ parser! { rule QuadPattern() -> Vec = "{" _ q:Quads() _ "}" { q } rule QuadData() -> Vec = "{" _ q:Quads() _ "}" {? - q.into_iter().map(Quad::try_from).collect::, ()>>().map_err(|_| "Variables are not allowed in INSERT DATA") + q.into_iter().map(Quad::try_from).collect::, ()>>().map_err(|()| "Variables are not allowed in INSERT DATA") } rule GroundQuadData() -> Vec = "{" _ q:Quads() _ "}" {? - q.into_iter().map(|q| GroundQuad::try_from(Quad::try_from(q)?)).collect::, ()>>().map_err(|_| "Variables and blank nodes are not allowed in DELETE DATA") + q.into_iter().map(|q| GroundQuad::try_from(Quad::try_from(q)?)).collect::, ()>>().map_err(|()| "Variables and blank nodes are not allowed in DELETE DATA") } rule Quads() -> Vec = q:(Quads_TriplesTemplate() / Quads_QuadsNotTriples()) ** (_) { @@ -1703,7 +1704,7 @@ parser! { rule QuotedTripleData() -> GroundTriple = "<<" _ s:DataValueTerm() _ p:QuotedTripleData_p() _ o:DataValueTerm() _ ">>" {? Ok(GroundTriple { - subject: s.try_into().map_err(|_| "Literals are not allowed in subject position of nested patterns")?, + subject: s.try_into().map_err(|()| "Literals are not allowed in subject position of nested patterns")?, predicate: p, object: o }) diff --git a/lib/src/sparql/eval.rs b/lib/src/sparql/eval.rs index c5b35a95..4adb260b 100644 --- a/lib/src/sparql/eval.rs +++ b/lib/src/sparql/eval.rs @@ -5508,7 +5508,7 @@ fn slice_key(slice: &[T], element: &T) -> Option { } fn generate_uuid(buffer: &mut String) { - let mut uuid = random::().to_ne_bytes(); + let mut uuid = random::().to_le_bytes(); uuid[6] = (uuid[6] & 0x0F) | 0x40; uuid[8] = (uuid[8] & 0x3F) | 0x80; diff --git a/lib/src/sparql/mod.rs b/lib/src/sparql/mod.rs index 262ab868..f48ea908 100644 --- a/lib/src/sparql/mod.rs +++ b/lib/src/sparql/mod.rs @@ -320,6 +320,6 @@ impl fmt::Debug for QueryExplanation { ); } obj.field("tree", &self.inner); - obj.finish() + obj.finish_non_exhaustive() } } diff --git a/.cargo/build_config.py b/lints/build_config.py similarity index 89% rename from .cargo/build_config.py rename to lints/build_config.py index 44079d9c..749c0622 100644 --- a/.cargo/build_config.py +++ b/lints/build_config.py @@ -1,7 +1,8 @@ import json +from pathlib import Path from urllib.request import urlopen -MSRV = "1.70.0" +MSRV = "1.74.0" DEFAULT_BUILD_FLAGS = { "-Wtrivial-casts", "-Wtrivial-numeric-casts", @@ -10,9 +11,11 @@ DEFAULT_BUILD_FLAGS = { "-Wunused-qualifications", } FLAGS_BLACKLIST = { + "-Wclippy::absolute-paths", # TODO: might be nice "-Wclippy::alloc-instead-of-core", "-Wclippy::arithmetic-side-effects", # TODO: might be nice "-Wclippy::as-conversions", + "-Wclippy::big-endian-bytes", "-Wclippy::cargo-common-metadata", # TODO: might be nice "-Wclippy::doc-markdown", # Too many false positives "-Wclippy::default-numeric-fallback", @@ -25,9 +28,10 @@ FLAGS_BLACKLIST = { "-Wclippy::impl-trait-in-params", "-Wclippy::implicit-return", "-Wclippy::indexing-slicing", - "-Wclippy::integer-arithmetic", "-Wclippy::integer-division", + "-Wclippy::little-endian-bytes", "-Wclippy::map-err-ignore", + "-Wclippy::min-ident-chars", "-Wclippy::missing-docs-in-private-items", "-Wclippy::missing-errors-doc", "-Wclippy::missing-inline-in-public-items", @@ -43,11 +47,13 @@ FLAGS_BLACKLIST = { "-Wclippy::option-option", "-Wclippy::pattern-type-mismatch", "-Wclippy::pub-use", + "-Wclippy::pub-with-shorthand", "-Wclippy::question-mark-used", "-Wclippy::self-named-module-files", # TODO: might be nice "-Wclippy::semicolon-if-nothing-returned", # TODO: might be nice "-Wclippy::semicolon-outside-block", "-Wclippy::similar-names", + "-Wclippy::single-call-fn", "-Wclippy::single-char-lifetime-names", "-Wclippy::std-instead-of-alloc", "-Wclippy::std-instead-of-core", @@ -76,7 +82,7 @@ for flag in FLAGS_BLACKLIST: else: print(f"Unused blacklisted flag: {flag}") -with open("./config.toml", "wt") as fp: +with (Path(__file__).parent.parent / ".cargo" / "config.toml").open("wt") as fp: fp.write("[build]\n") fp.write("rustflags = [\n") for flag in sorted(build_flags): From 4084acb9b871d7bf2588b9543c9518edc0c96985 Mon Sep 17 00:00:00 2001 From: Tpt Date: Fri, 29 Dec 2023 16:50:31 +0100 Subject: [PATCH 147/217] Renames dump_dataset and dump_graph to dump_to_write and dump_graph_to_write --- cli/src/main.rs | 4 +- js/src/store.rs | 9 ++-- lib/oxrdfio/src/parser.rs | 18 +++++++ lib/oxrdfio/src/serializer.rs | 11 +++++ lib/src/io/format.rs | 32 ++++++++++++- lib/src/store.rs | 90 +++++++++++++++++++++++++++-------- lib/tests/store.rs | 4 +- python/src/store.rs | 5 +- 8 files changed, 144 insertions(+), 29 deletions(-) diff --git a/cli/src/main.rs b/cli/src/main.rs index 2a2f6562..4ab05892 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -806,9 +806,9 @@ fn dump( ) -> anyhow::Result { ensure!(format.supports_datasets() || from_graph_name.is_some(), "The --graph option is required when writing a format not supporting datasets like NTriples, Turtle or RDF/XML"); Ok(if let Some(from_graph_name) = from_graph_name { - store.dump_graph(write, format, from_graph_name) + store.dump_graph_to_write(from_graph_name, format, write) } else { - store.dump_dataset(write, format) + store.dump_to_write(format, write) }?) } diff --git a/js/src/store.rs b/js/src/store.rs index f76786a5..bb0af9e7 100644 --- a/js/src/store.rs +++ b/js/src/store.rs @@ -177,10 +177,13 @@ impl JsStore { let format = rdf_format(format)?; let buffer = if let Some(from_graph_name) = FROM_JS.with(|c| c.to_optional_term(from_graph_name))? { - self.store - .dump_graph(Vec::new(), format, &GraphName::try_from(from_graph_name)?) + self.store.dump_graph_to_write( + &GraphName::try_from(from_graph_name)?, + format, + Vec::new(), + ) } else { - self.store.dump_dataset(Vec::new(), format) + self.store.dump_to_write(format, Vec::new()) } .map_err(to_err)?; String::from_utf8(buffer).map_err(to_err) diff --git a/lib/oxrdfio/src/parser.rs b/lib/oxrdfio/src/parser.rs index b9aaf646..1b69f955 100644 --- a/lib/oxrdfio/src/parser.rs +++ b/lib/oxrdfio/src/parser.rs @@ -125,6 +125,24 @@ impl RdfParser { } } + /// The format the parser uses. + /// + /// ``` + /// use oxrdfio::{RdfParser, RdfFormat}; + /// + /// assert_eq!(RdfParser::from_format(RdfFormat::Turtle).format(), RdfFormat::Turtle); + /// ``` + pub fn format(&self) -> RdfFormat { + match &self.inner { + RdfParserKind::N3(_) => RdfFormat::N3, + RdfParserKind::NQuads(_) => RdfFormat::NQuads, + RdfParserKind::NTriples(_) => RdfFormat::NTriples, + RdfParserKind::RdfXml(_) => RdfFormat::RdfXml, + RdfParserKind::TriG(_) => RdfFormat::TriG, + RdfParserKind::Turtle(_) => RdfFormat::Turtle, + } + } + /// Provides an IRI that could be used to resolve the file relative IRIs. /// /// ``` diff --git a/lib/oxrdfio/src/serializer.rs b/lib/oxrdfio/src/serializer.rs index 487b4539..797bdf90 100644 --- a/lib/oxrdfio/src/serializer.rs +++ b/lib/oxrdfio/src/serializer.rs @@ -60,6 +60,17 @@ impl RdfSerializer { Self { format } } + /// The format the serializer serializes to. + /// + /// ``` + /// use oxrdfio::{RdfSerializer, RdfFormat}; + /// + /// assert_eq!(RdfSerializer::from_format(RdfFormat::Turtle).format(), RdfFormat::Turtle); + /// ``` + pub fn format(&self) -> RdfFormat { + self.format + } + /// Writes to a [`Write`] implementation. /// ///
Do not forget to run the [`finish`](ToWriteQuadWriter::finish()) method to properly write the last bytes of the file.
diff --git a/lib/src/io/format.rs b/lib/src/io/format.rs index b100d392..e6ff7b3c 100644 --- a/lib/src/io/format.rs +++ b/lib/src/io/format.rs @@ -1,6 +1,6 @@ #![allow(deprecated)] -use oxrdfio::RdfFormat; +use oxrdfio::{RdfFormat, RdfParser, RdfSerializer}; /// [RDF graph](https://www.w3.org/TR/rdf11-concepts/#dfn-rdf-graph) serialization formats. /// @@ -108,6 +108,7 @@ impl GraphFormat { } impl From for RdfFormat { + #[inline] fn from(format: GraphFormat) -> Self { match format { GraphFormat::NTriples => Self::NTriples, @@ -117,6 +118,20 @@ impl From for RdfFormat { } } +impl From for RdfParser { + #[inline] + fn from(format: GraphFormat) -> Self { + RdfFormat::from(format).into() + } +} + +impl From for RdfSerializer { + #[inline] + fn from(format: GraphFormat) -> Self { + RdfFormat::from(format).into() + } +} + /// [RDF dataset](https://www.w3.org/TR/rdf11-concepts/#dfn-rdf-dataset) serialization formats. /// /// This enumeration is non exhaustive. New formats like JSON-LD will be added in the future. @@ -215,6 +230,7 @@ impl DatasetFormat { } impl From for RdfFormat { + #[inline] fn from(format: DatasetFormat) -> Self { match format { DatasetFormat::NQuads => Self::NQuads, @@ -223,6 +239,20 @@ impl From for RdfFormat { } } +impl From for RdfParser { + #[inline] + fn from(format: DatasetFormat) -> Self { + RdfFormat::from(format).into() + } +} + +impl From for RdfSerializer { + #[inline] + fn from(format: DatasetFormat) -> Self { + RdfFormat::from(format).into() + } +} + impl TryFrom for GraphFormat { type Error = (); diff --git a/lib/src/store.rs b/lib/src/store.rs index e78a137b..4ff084f6 100644 --- a/lib/src/store.rs +++ b/lib/src/store.rs @@ -639,6 +639,37 @@ impl Store { self.transaction(move |mut t| t.remove(quad)) } + /// Dumps the store into a file. + /// + /// ``` + /// use oxigraph::store::Store; + /// use oxigraph::io::RdfFormat; + /// + /// let file = " .\n".as_bytes(); + /// + /// let store = Store::new()?; + /// store.load_from_read(RdfFormat::NQuads, file)?; + /// + /// let buffer = store.dump_to_write(RdfFormat::NQuads, Vec::new())?; + /// assert_eq!(file, buffer.as_slice()); + /// # std::io::Result::Ok(()) + /// ``` + pub fn dump_to_write( + &self, + serializer: impl Into, + write: W, + ) -> Result { + let serializer = serializer.into(); + if !serializer.format().supports_datasets() { + return Err(SerializerError::DatasetFormatExpected(serializer.format())); + } + let mut writer = serializer.serialize_to_write(write); + for quad in self.iter() { + writer.write_quad(&quad?)?; + } + Ok(writer.finish()?) + } + /// Dumps a store graph into a file. /// /// Usage example: @@ -653,23 +684,51 @@ impl Store { /// store.load_graph(file, RdfFormat::NTriples, GraphName::DefaultGraph, None)?; /// /// let mut buffer = Vec::new(); - /// store.dump_graph(&mut buffer, RdfFormat::NTriples, GraphNameRef::DefaultGraph)?; + /// store.dump_graph_to_write(GraphNameRef::DefaultGraph, RdfFormat::NTriples, &mut buffer)?; /// assert_eq!(file, buffer.as_slice()); /// # std::io::Result::Ok(()) /// ``` - pub fn dump_graph<'a, W: Write>( + pub fn dump_graph_to_write<'a, W: Write>( &self, - write: W, - format: impl Into, from_graph_name: impl Into>, + serializer: impl Into, + write: W, ) -> Result { - let mut writer = RdfSerializer::from_format(format.into()).serialize_to_write(write); + let mut writer = serializer.into().serialize_to_write(write); for quad in self.quads_for_pattern(None, None, None, Some(from_graph_name.into())) { writer.write_triple(quad?.as_ref())?; } Ok(writer.finish()?) } + /// Dumps a store graph into a file. + /// + /// Usage example: + /// ``` + /// use oxigraph::store::Store; + /// use oxigraph::io::RdfFormat; + /// use oxigraph::model::*; + /// + /// let file = " .\n".as_bytes(); + /// + /// let store = Store::new()?; + /// store.load_graph(file, RdfFormat::NTriples, GraphName::DefaultGraph, None)?; + /// + /// let mut buffer = Vec::new(); + /// store.dump_graph(&mut buffer, RdfFormat::NTriples, GraphNameRef::DefaultGraph)?; + /// assert_eq!(file, buffer.as_slice()); + /// # std::io::Result::Ok(()) + /// ``` + #[deprecated(note = "use Store.dump_graph_to_write instead", since = "0.4.0")] + pub fn dump_graph<'a, W: Write>( + &self, + write: W, + format: impl Into, + from_graph_name: impl Into>, + ) -> Result { + self.dump_graph_to_write(from_graph_name, format.into(), write) + } + /// Dumps the store into a file. /// /// ``` @@ -679,26 +738,19 @@ impl Store { /// let file = " .\n".as_bytes(); /// /// let store = Store::new()?; - /// store.load_dataset(file, RdfFormat::NQuads, None)?; + /// store.load_from_read(RdfFormat::NQuads, file)?; /// /// let buffer = store.dump_dataset(Vec::new(), RdfFormat::NQuads)?; /// assert_eq!(file, buffer.as_slice()); /// # std::io::Result::Ok(()) /// ``` + #[deprecated(note = "use Store.dump_to_write instead", since = "0.4.0")] pub fn dump_dataset( &self, write: W, format: impl Into, ) -> Result { - let format = format.into(); - if !format.supports_datasets() { - return Err(SerializerError::DatasetFormatExpected(format)); - } - let mut writer = RdfSerializer::from_format(format).serialize_to_write(write); - for quad in self.iter() { - writer.write_quad(&quad?)?; - } - Ok(writer.finish()?) + self.dump_to_write(format.into(), write) } /// Returns all the store named graphs. @@ -876,7 +928,7 @@ impl Store { /// but hard links will be used to point to the original database immutable snapshots. /// This allows cheap regular backups. /// - /// If you want to move your data to another RDF storage system, you should have a look at the [`Store::dump_dataset`] function instead. + /// If you want to move your data to another RDF storage system, you should have a look at the [`Store::dump_to_write`] function instead. #[cfg(not(target_family = "wasm"))] pub fn backup(&self, target_directory: impl AsRef) -> Result<(), StorageError> { self.storage.backup(target_directory.as_ref()) @@ -894,7 +946,7 @@ impl Store { /// /// // quads file insertion /// let file = b" ."; - /// store.bulk_loader().load_dataset(file.as_ref(), RdfFormat::NQuads, None)?; + /// store.bulk_loader().load_from_read(RdfFormat::NQuads, file.as_ref())?; /// /// // we inspect the store contents /// let ex = NamedNodeRef::new("http://example.com")?; @@ -1467,7 +1519,7 @@ impl Iterator for GraphNameIter { /// /// // quads file insertion /// let file = b" ."; -/// store.bulk_loader().load_dataset(file.as_ref(), RdfFormat::NQuads, None)?; +/// store.bulk_loader().load_from_read(RdfFormat::NQuads, file.as_ref())?; /// /// // we inspect the store contents /// let ex = NamedNodeRef::new("http://example.com")?; @@ -1532,7 +1584,7 @@ impl BulkLoader { /// Loads a file using the bulk loader. /// - /// This function is optimized for large dataset loading speed. For small files, [`Store::load_dataset`] might be more convenient. + /// This function is optimized for large dataset loading speed. For small files, [`Store::load_from_read`] might be more convenient. /// ///
This method is not atomic. /// If the parsing fails in the middle of the file, only a part of it may be written to the store. diff --git a/lib/tests/store.rs b/lib/tests/store.rs index 070a5351..e7e074d9 100644 --- a/lib/tests/store.rs +++ b/lib/tests/store.rs @@ -202,7 +202,7 @@ fn test_dump_graph() -> Result<(), Box> { } let mut buffer = Vec::new(); - store.dump_graph(&mut buffer, RdfFormat::NTriples, GraphNameRef::DefaultGraph)?; + store.dump_graph_to_write(GraphNameRef::DefaultGraph, RdfFormat::NTriples, &mut buffer)?; assert_eq!( buffer.into_iter().filter(|c| *c == b'\n').count(), NUMBER_OF_TRIPLES @@ -217,7 +217,7 @@ fn test_dump_dataset() -> Result<(), Box> { store.insert(q)?; } - let buffer = store.dump_dataset(Vec::new(), RdfFormat::NQuads)?; + let buffer = store.dump_to_write(RdfFormat::NQuads, Vec::new())?; assert_eq!( buffer.into_iter().filter(|c| *c == b'\n').count(), NUMBER_OF_TRIPLES diff --git a/python/src/store.rs b/python/src/store.rs index 6068296d..8923b1f6 100644 --- a/python/src/store.rs +++ b/python/src/store.rs @@ -542,9 +542,10 @@ impl PyStore { py.allow_threads(|| { let format = lookup_rdf_format(format, file_path.as_deref())?; if let Some(from_graph_name) = &from_graph_name { - self.inner.dump_graph(output, format, from_graph_name) + self.inner + .dump_graph_to_write(from_graph_name, format, output) } else { - self.inner.dump_dataset(output, format) + self.inner.dump_to_write(format, output) } .map_err(map_serializer_error) }) From 604d1bbe2ed7d5c5a555679720c1477b4df1aeff Mon Sep 17 00:00:00 2001 From: Tpt Date: Fri, 29 Dec 2023 15:45:11 +0100 Subject: [PATCH 148/217] BulkLoader: rename set_* methods to with_* methods --- lib/src/storage/mod.rs | 8 +++----- lib/src/store.rs | 29 +++++++++++++++++++---------- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/lib/src/storage/mod.rs b/lib/src/storage/mod.rs index 126f5fcc..01419239 100644 --- a/lib/src/storage/mod.rs +++ b/lib/src/storage/mod.rs @@ -1193,6 +1193,7 @@ impl<'a> StorageWriter<'a> { } #[cfg(not(target_family = "wasm"))] +#[must_use] pub struct StorageBulkLoader { storage: Storage, hooks: Vec>, @@ -1211,19 +1212,16 @@ impl StorageBulkLoader { } } - #[must_use] - pub fn set_num_threads(mut self, num_threads: usize) -> Self { + pub fn with_num_threads(mut self, num_threads: usize) -> Self { self.num_threads = Some(num_threads); self } - #[must_use] - pub fn set_max_memory_size_in_megabytes(mut self, max_memory_size: usize) -> Self { + pub fn with_max_memory_size_in_megabytes(mut self, max_memory_size: usize) -> Self { self.max_memory_size = Some(max_memory_size); self } - #[must_use] pub fn on_progress(mut self, callback: impl Fn(u64) + 'static) -> Self { self.hooks.push(Box::new(callback)); self diff --git a/lib/src/store.rs b/lib/src/store.rs index 4ff084f6..2b077b1f 100644 --- a/lib/src/store.rs +++ b/lib/src/store.rs @@ -1503,8 +1503,8 @@ impl Iterator for GraphNameIter { /// Results might get weird if you delete data during the loading process. /// ///
It is optimized for speed.
-/// Memory usage is configurable using [`BulkLoader::set_max_memory_size_in_megabytes`] -/// and the number of used threads with [`BulkLoader::set_num_threads`]. +/// Memory usage is configurable using [`BulkLoader::with_max_memory_size_in_megabytes`] +/// and the number of used threads with [`BulkLoader::with_num_threads`]. /// By default the memory consumption target (excluding the system and RocksDB internal consumption) /// is around 2GB per thread and 2 threads. /// These targets are considered per loaded file. @@ -1527,6 +1527,7 @@ impl Iterator for GraphNameIter { /// # Result::<_, Box>::Ok(()) /// ``` #[cfg(not(target_family = "wasm"))] +#[must_use] pub struct BulkLoader { storage: StorageBulkLoader, on_parse_error: Option Result<(), ParseError>>>, @@ -1539,12 +1540,17 @@ impl BulkLoader { /// This number must be at last 2 (one for parsing and one for loading). /// /// The default value is 2. - #[must_use] - pub fn set_num_threads(mut self, num_threads: usize) -> Self { - self.storage = self.storage.set_num_threads(num_threads); + pub fn with_num_threads(mut self, num_threads: usize) -> Self { + self.storage = self.storage.with_num_threads(num_threads); self } + #[doc(hidden)] + #[deprecated(note = "Use with_num_threads", since = "0.4.0")] + pub fn set_num_threads(self, num_threads: usize) -> Self { + self.with_num_threads(num_threads) + } + /// Sets a rough idea of the maximal amount of memory to be used by this operation. /// /// This number must be at last a few megabytes per thread. @@ -1554,16 +1560,20 @@ impl BulkLoader { /// (for example if the data contains very long IRIs or literals). /// /// By default, a target 2GB per used thread is used. - #[must_use] - pub fn set_max_memory_size_in_megabytes(mut self, max_memory_size: usize) -> Self { + pub fn with_max_memory_size_in_megabytes(mut self, max_memory_size: usize) -> Self { self.storage = self .storage - .set_max_memory_size_in_megabytes(max_memory_size); + .with_max_memory_size_in_megabytes(max_memory_size); self } + #[doc(hidden)] + #[deprecated(note = "Use with_max_memory_size_in_megabytes", since = "0.4.0")] + pub fn set_max_memory_size_in_megabytes(self, max_memory_size: usize) -> Self { + self.with_max_memory_size_in_megabytes(max_memory_size) + } + /// Adds a `callback` evaluated from time to time with the number of loaded triples. - #[must_use] pub fn on_progress(mut self, callback: impl Fn(u64) + 'static) -> Self { self.storage = self.storage.on_progress(callback); self @@ -1573,7 +1583,6 @@ impl BulkLoader { /// by returning `Ok` or fail by returning `Err`. /// /// By default the parsing fails. - #[must_use] pub fn on_parse_error( mut self, callback: impl Fn(ParseError) -> Result<(), ParseError> + 'static, From 2e9ac3cc1a37f98ec48e05bfafa74b05073c7d38 Mon Sep 17 00:00:00 2001 From: Tpt Date: Fri, 29 Dec 2023 15:06:27 +0100 Subject: [PATCH 149/217] Python Store.backup: allows pathlib Issue #699 --- python/src/store.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/src/store.rs b/python/src/store.rs index 8923b1f6..81abb840 100644 --- a/python/src/store.rs +++ b/python/src/store.rs @@ -726,10 +726,10 @@ impl PyStore { /// If you want to move your data to another RDF storage system, you should have a look at the :py:func:`dump_dataset` function instead. /// /// :param target_directory: the directory name to save the database to. - /// :type target_directory: str + /// :type target_directory: str or os.PathLike[str] /// :rtype: None /// :raises OSError: if an error happens during the backup. - fn backup(&self, target_directory: &str, py: Python<'_>) -> PyResult<()> { + fn backup(&self, target_directory: PathBuf, py: Python<'_>) -> PyResult<()> { py.allow_threads(|| { self.inner .backup(target_directory) From 4756217787b8cc857dc0185f3a822b5cb389a1e3 Mon Sep 17 00:00:00 2001 From: Tpt Date: Sat, 16 Dec 2023 21:20:29 +0100 Subject: [PATCH 150/217] Adds "since" to the #[deprecated] annotations --- lib/sparesults/src/parser.rs | 2 +- lib/sparesults/src/serializer.rs | 4 ++-- lib/src/io/format.rs | 4 ++-- lib/src/io/read.rs | 4 ++-- lib/src/io/write.rs | 4 ++-- lib/src/store.rs | 12 ++++++------ 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/lib/sparesults/src/parser.rs b/lib/sparesults/src/parser.rs index f54f85d3..0a826e96 100644 --- a/lib/sparesults/src/parser.rs +++ b/lib/sparesults/src/parser.rs @@ -109,7 +109,7 @@ impl QueryResultsParser { }) } - #[deprecated(note = "Use parse_read")] + #[deprecated(note = "use parse_read", since = "0.4.0")] pub fn read_results( &self, reader: R, diff --git a/lib/sparesults/src/serializer.rs b/lib/sparesults/src/serializer.rs index dc0baeb9..f9879206 100644 --- a/lib/sparesults/src/serializer.rs +++ b/lib/sparesults/src/serializer.rs @@ -108,7 +108,7 @@ impl QueryResultsSerializer { } } - #[deprecated(note = "Use serialize_boolean_to_write")] + #[deprecated(note = "use serialize_boolean_to_write", since = "0.4.0")] pub fn write_boolean_result(&self, writer: W, value: bool) -> io::Result { self.serialize_boolean_to_write(writer, value) } @@ -203,7 +203,7 @@ impl QueryResultsSerializer { }) } - #[deprecated(note = "Use serialize_solutions_to_write")] + #[deprecated(note = "use serialize_solutions_to_write", since = "0.4.0")] pub fn solutions_writer( &self, writer: W, diff --git a/lib/src/io/format.rs b/lib/src/io/format.rs index e6ff7b3c..89ba37f9 100644 --- a/lib/src/io/format.rs +++ b/lib/src/io/format.rs @@ -7,7 +7,7 @@ use oxrdfio::{RdfFormat, RdfParser, RdfSerializer}; /// This enumeration is non exhaustive. New formats like JSON-LD will be added in the future. #[derive(Eq, PartialEq, Debug, Clone, Copy, Hash)] #[non_exhaustive] -#[deprecated(note = "Use RdfFormat instead")] +#[deprecated(note = "use RdfFormat instead", since = "0.4.0")] pub enum GraphFormat { /// [N-Triples](https://www.w3.org/TR/n-triples/) NTriples, @@ -137,7 +137,7 @@ impl From for RdfSerializer { /// This enumeration is non exhaustive. New formats like JSON-LD will be added in the future. #[derive(Eq, PartialEq, Debug, Clone, Copy, Hash)] #[non_exhaustive] -#[deprecated(note = "Use RdfFormat instead")] +#[deprecated(note = "use RdfFormat instead", since = "0.4.0")] pub enum DatasetFormat { /// [N-Quads](https://www.w3.org/TR/n-quads/) NQuads, diff --git a/lib/src/io/read.rs b/lib/src/io/read.rs index 6960112a..33065615 100644 --- a/lib/src/io/read.rs +++ b/lib/src/io/read.rs @@ -27,7 +27,7 @@ use std::io::Read; /// assert_eq!(triples[0].subject.to_string(), ""); /// # std::io::Result::Ok(()) /// ``` -#[deprecated(note = "Use RdfParser instead")] +#[deprecated(note = "use RdfParser instead", since = "0.4.0")] pub struct GraphParser { inner: RdfParser, } @@ -118,7 +118,7 @@ impl Iterator for TripleReader { /// assert_eq!(quads[0].subject.to_string(), ""); /// # std::io::Result::Ok(()) /// ``` -#[deprecated(note = "Use RdfParser instead")] +#[deprecated(note = "use RdfParser instead", since = "0.4.0")] pub struct DatasetParser { inner: RdfParser, } diff --git a/lib/src/io/write.rs b/lib/src/io/write.rs index df1cab61..3c787ebe 100644 --- a/lib/src/io/write.rs +++ b/lib/src/io/write.rs @@ -30,7 +30,7 @@ use std::io::{self, Write}; /// assert_eq!(buffer.as_slice(), " .\n".as_bytes()); /// # Result::<_,Box>::Ok(()) /// ``` -#[deprecated(note = "Use RdfSerializer instead")] +#[deprecated(note = "use RdfSerializer instead", since = "0.4.0")] pub struct GraphSerializer { inner: RdfSerializer, } @@ -113,7 +113,7 @@ impl TripleWriter { /// assert_eq!(buffer.as_slice(), " .\n".as_bytes()); /// # Result::<_,Box>::Ok(()) /// ``` -#[deprecated(note = "Use RdfSerializer instead")] +#[deprecated(note = "use RdfSerializer instead", since = "0.4.0")] pub struct DatasetSerializer { inner: RdfSerializer, } diff --git a/lib/src/store.rs b/lib/src/store.rs index 2b077b1f..988befed 100644 --- a/lib/src/store.rs +++ b/lib/src/store.rs @@ -517,7 +517,7 @@ impl Store { /// assert!(store.contains(QuadRef::new(ex, ex, ex, GraphNameRef::DefaultGraph))?); /// # Result::<_, Box>::Ok(()) /// ``` - #[deprecated(note = "Use Store.load_from_read instead")] + #[deprecated(note = "use Store.load_from_read instead", since = "0.4.0")] pub fn load_graph( &self, read: impl Read, @@ -560,7 +560,7 @@ impl Store { /// assert!(store.contains(QuadRef::new(ex, ex, ex, ex))?); /// # Result::<_, Box>::Ok(()) /// ``` - #[deprecated(note = "Use Store.load_from_read instead")] + #[deprecated(note = "use Store.load_from_read instead", since = "0.4.0")] pub fn load_dataset( &self, read: impl Read, @@ -1222,7 +1222,7 @@ impl<'a> Transaction<'a> { /// assert!(store.contains(QuadRef::new(ex, ex, ex, GraphNameRef::DefaultGraph))?); /// # Result::<_,oxigraph::store::LoaderError>::Ok(()) /// ``` - #[deprecated(note = "Use Transaction.load_from_read instead")] + #[deprecated(note = "use Transaction.load_from_read instead", since = "0.4.0")] pub fn load_graph( &mut self, read: impl Read, @@ -1265,7 +1265,7 @@ impl<'a> Transaction<'a> { /// assert!(store.contains(QuadRef::new(ex, ex, ex, ex))?); /// # Result::<_,oxigraph::store::LoaderError>::Ok(()) /// ``` - #[deprecated(note = "Use Transaction.load_from_read instead")] + #[deprecated(note = "use Transaction.load_from_read instead", since = "0.4.0")] pub fn load_dataset( &mut self, read: impl Read, @@ -1685,7 +1685,7 @@ impl BulkLoader { /// assert!(store.contains(QuadRef::new(ex, ex, ex, ex))?); /// # Result::<_, Box>::Ok(()) /// ``` - #[deprecated(note = "Use BulkLoader.load_from_read instead")] + #[deprecated(note = "use BulkLoader.load_from_read instead", since = "0.4.0")] pub fn load_dataset( &self, read: impl Read, @@ -1744,7 +1744,7 @@ impl BulkLoader { /// assert!(store.contains(QuadRef::new(ex, ex, ex, GraphNameRef::DefaultGraph))?); /// # Result::<_, Box>::Ok(()) /// ``` - #[deprecated(note = "Use BulkLoader.load_from_read instead")] + #[deprecated(note = "use BulkLoader.load_from_read instead", since = "0.4.0")] pub fn load_graph( &self, read: impl Read, From 025bd2afd2fdd0ba37be0d2e93f7a0c530561955 Mon Sep 17 00:00:00 2001 From: Tpt Date: Fri, 29 Dec 2023 21:22:19 +0100 Subject: [PATCH 151/217] Uses new cargo lint configuration system --- .cargo/config.toml | 174 --------------------------- .github/workflows/tests.yml | 6 +- Cargo.toml | 184 +++++++++++++++++++++++++++++ cli/Cargo.toml | 3 + js/Cargo.toml | 3 + lib/Cargo.toml | 3 + lib/oxrdf/Cargo.toml | 3 + lib/oxrdf/src/interning.rs | 4 +- lib/oxrdfio/Cargo.toml | 3 + lib/oxrdfxml/Cargo.toml | 3 + lib/oxrdfxml/src/parser.rs | 10 +- lib/oxsdatatypes/Cargo.toml | 3 + lib/oxsdatatypes/src/date_time.rs | 3 +- lib/oxttl/Cargo.toml | 3 + lib/oxttl/src/line_formats.rs | 2 +- lib/oxttl/src/n3.rs | 2 +- lib/oxttl/src/terse.rs | 2 +- lib/sparesults/Cargo.toml | 3 + lib/sparesults/src/json.rs | 2 +- lib/sparesults/src/xml.rs | 2 +- lib/spargebra/Cargo.toml | 3 + lib/sparopt/Cargo.toml | 3 + lib/sparql-smith/Cargo.toml | 3 + lib/src/sparql/algebra.rs | 8 +- lib/src/sparql/model.rs | 2 + lib/src/sparql/service.rs | 1 + lib/src/storage/backend/rocksdb.rs | 26 ++-- lib/src/storage/error.rs | 7 +- lints/build_clippy_config.py | 83 +++++++++++++ lints/build_config.py | 90 -------------- python/Cargo.toml | 3 + python/src/io.rs | 8 +- python/src/model.rs | 52 ++++---- python/src/sparql.rs | 8 +- python/src/store.rs | 4 +- testsuite/Cargo.toml | 3 + 36 files changed, 384 insertions(+), 338 deletions(-) delete mode 100644 .cargo/config.toml create mode 100644 lints/build_clippy_config.py delete mode 100644 lints/build_config.py diff --git a/.cargo/config.toml b/.cargo/config.toml deleted file mode 100644 index dad85ce3..00000000 --- a/.cargo/config.toml +++ /dev/null @@ -1,174 +0,0 @@ -[build] -rustflags = [ - "-Wclippy::allow-attributes", - "-Wclippy::allow-attributes-without-reason", - "-Wclippy::as-underscore", - "-Wclippy::assertions-on-result-states", - "-Wclippy::bool-to-int-with-if", - "-Wclippy::borrow-as-ptr", - "-Wclippy::case-sensitive-file-extension-comparisons", - "-Wclippy::cast-lossless", - "-Wclippy::cast-possible-truncation", - "-Wclippy::cast-possible-wrap", - "-Wclippy::cast-precision-loss", - "-Wclippy::cast-ptr-alignment", - "-Wclippy::cast-sign-loss", - "-Wclippy::checked-conversions", - "-Wclippy::clone-on-ref-ptr", - "-Wclippy::cloned-instead-of-copied", - "-Wclippy::copy-iterator", - "-Wclippy::create-dir", - "-Wclippy::dbg-macro", - "-Wclippy::decimal-literal-representation", - "-Wclippy::default-trait-access", - "-Wclippy::default-union-representation", - "-Wclippy::deref-by-slicing", - "-Wclippy::disallowed-script-idents", - "-Wclippy::doc-link-with-quotes", - "-Wclippy::empty-drop", - "-Wclippy::empty-enum", - "-Wclippy::empty-structs-with-brackets", - "-Wclippy::enum-glob-use", - "-Wclippy::error-impl-error", - "-Wclippy::exit", - "-Wclippy::expect-used", - "-Wclippy::expl-impl-clone-on-copy", - "-Wclippy::explicit-deref-methods", - "-Wclippy::explicit-into-iter-loop", - "-Wclippy::explicit-iter-loop", - "-Wclippy::filetype-is-file", - "-Wclippy::filter-map-next", - "-Wclippy::flat-map-option", - "-Wclippy::fn-params-excessive-bools", - "-Wclippy::fn-to-numeric-cast-any", - "-Wclippy::format-push-string", - "-Wclippy::from-iter-instead-of-collect", - "-Wclippy::get-unwrap", - "-Wclippy::host-endian-bytes", - "-Wclippy::if-not-else", - "-Wclippy::if-then-some-else-none", - "-Wclippy::ignored-unit-patterns", - "-Wclippy::implicit-clone", - "-Wclippy::implicit-hasher", - "-Wclippy::inconsistent-struct-constructor", - "-Wclippy::index-refutable-slice", - "-Wclippy::inefficient-to-string", - "-Wclippy::inline-always", - "-Wclippy::inline-asm-x86-att-syntax", - "-Wclippy::inline-asm-x86-intel-syntax", - "-Wclippy::invalid-upcast-comparisons", - "-Wclippy::items-after-statements", - "-Wclippy::iter-not-returning-iterator", - "-Wclippy::large-digit-groups", - "-Wclippy::large-futures", - "-Wclippy::large-include-file", - "-Wclippy::large-stack-arrays", - "-Wclippy::large-types-passed-by-value", - "-Wclippy::let-underscore-must-use", - "-Wclippy::let-underscore-untyped", - "-Wclippy::linkedlist", - "-Wclippy::lossy-float-literal", - "-Wclippy::macro-use-imports", - "-Wclippy::manual-assert", - "-Wclippy::manual-instant-elapsed", - "-Wclippy::manual-let-else", - "-Wclippy::manual-ok-or", - "-Wclippy::manual-string-new", - "-Wclippy::many-single-char-names", - "-Wclippy::map-unwrap-or", - "-Wclippy::match-bool", - "-Wclippy::match-on-vec-items", - "-Wclippy::match-same-arms", - "-Wclippy::match-wild-err-arm", - "-Wclippy::match-wildcard-for-single-variants", - "-Wclippy::maybe-infinite-iter", - "-Wclippy::mem-forget", - "-Wclippy::mismatching-type-param-order", - "-Wclippy::missing-assert-message", - "-Wclippy::missing-asserts-for-indexing", - "-Wclippy::missing-enforced-import-renames", - "-Wclippy::missing-fields-in-debug", - "-Wclippy::multiple-inherent-impl", - "-Wclippy::mut-mut", - "-Wclippy::mutex-atomic", - "-Wclippy::naive-bytecount", - "-Wclippy::needless-bitwise-bool", - "-Wclippy::needless-continue", - "-Wclippy::needless-for-each", - "-Wclippy::needless-pass-by-value", - "-Wclippy::needless-raw-strings", - "-Wclippy::negative-feature-names", - "-Wclippy::no-effect-underscore-binding", - "-Wclippy::no-mangle-with-rust-abi", - "-Wclippy::non-ascii-literal", - "-Wclippy::panic", - "-Wclippy::panic-in-result-fn", - "-Wclippy::partial-pub-fields", - "-Wclippy::print-stderr", - "-Wclippy::print-stdout", - "-Wclippy::ptr-as-ptr", - "-Wclippy::ptr-cast-constness", - "-Wclippy::pub-without-shorthand", - "-Wclippy::range-minus-one", - "-Wclippy::range-plus-one", - "-Wclippy::rc-buffer", - "-Wclippy::rc-mutex", - "-Wclippy::redundant-closure-for-method-calls", - "-Wclippy::redundant-else", - "-Wclippy::redundant-feature-names", - "-Wclippy::redundant-type-annotations", - "-Wclippy::ref-binding-to-reference", - "-Wclippy::ref-option-ref", - "-Wclippy::ref-patterns", - "-Wclippy::rest-pat-in-fully-bound-structs", - "-Wclippy::return-self-not-must-use", - "-Wclippy::same-functions-in-if-condition", - "-Wclippy::same-name-method", - "-Wclippy::semicolon-inside-block", - "-Wclippy::shadow-same", - "-Wclippy::should-panic-without-expect", - "-Wclippy::single-match-else", - "-Wclippy::stable-sort-primitive", - "-Wclippy::str-to-string", - "-Wclippy::string-add", - "-Wclippy::string-add-assign", - "-Wclippy::string-lit-chars-any", - "-Wclippy::string-to-string", - "-Wclippy::struct-excessive-bools", - "-Wclippy::suspicious-xor-used-as-pow", - "-Wclippy::tests-outside-test-module", - "-Wclippy::todo", - "-Wclippy::transmute-ptr-to-ptr", - "-Wclippy::trivially-copy-pass-by-ref", - "-Wclippy::try-err", - "-Wclippy::unchecked-duration-subtraction", - "-Wclippy::undocumented-unsafe-blocks", - "-Wclippy::unicode-not-nfc", - "-Wclippy::unimplemented", - "-Wclippy::uninlined-format-args", - "-Wclippy::unnecessary-box-returns", - "-Wclippy::unnecessary-join", - "-Wclippy::unnecessary-safety-comment", - "-Wclippy::unnecessary-safety-doc", - "-Wclippy::unnecessary-self-imports", - "-Wclippy::unnecessary-wraps", - "-Wclippy::unneeded-field-pattern", - "-Wclippy::unnested-or-patterns", - "-Wclippy::unreadable-literal", - "-Wclippy::unsafe-derive-deserialize", - "-Wclippy::unseparated-literal-suffix", - "-Wclippy::unused-async", - "-Wclippy::unused-self", - "-Wclippy::unwrap-in-result", - "-Wclippy::use-debug", - "-Wclippy::used-underscore-binding", - "-Wclippy::verbose-bit-mask", - "-Wclippy::verbose-file-reads", - "-Wclippy::wildcard-dependencies", - "-Wclippy::zero-sized-map-values", - "-Wtrivial-casts", - "-Wtrivial-numeric-casts", - "-Wunsafe-code", - "-Wunused-lifetimes", - "-Wunused-qualifications", -] diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7ce2c3b0..ee5f4f9b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -123,7 +123,7 @@ jobs: - uses: actions/checkout@v3 with: submodules: true - - run: rustup update && rustup toolchain install nightly && rustup default 1.74.1 + - run: rustup update && rustup toolchain install nightly && rustup default 1.70.0 - uses: Swatinem/rust-cache@v2 - run: rm Cargo.lock && cargo +nightly update -Z direct-minimal-versions - run: cargo test @@ -264,7 +264,7 @@ jobs: - uses: actions/checkout@v3 with: submodules: true - - run: rustup update && rustup toolchain install nightly && rustup default 1.74.1 + - run: rustup update && rustup toolchain install nightly && rustup default 1.70.0 - uses: Swatinem/rust-cache@v2 - uses: actions/setup-python@v4 with: @@ -285,7 +285,7 @@ jobs: - uses: actions/checkout@v3 with: submodules: true - - run: rustup update && rustup toolchain install nightly && rustup default 1.74.1 + - run: rustup update - uses: Swatinem/rust-cache@v2 - uses: actions/setup-python@v4 with: diff --git a/Cargo.toml b/Cargo.toml index 8264afa9..be87aaac 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,9 +26,193 @@ homepage = "https://oxigraph.org/" edition = "2021" rust-version = "1.70" +[workspace.lints.rust] +absolute_paths_not_starting_with_crate = "warn" +elided_lifetimes_in_paths = "warn" +explicit_outlives_requirements = "warn" +let_underscore_drop = "warn" +macro_use_extern_crate = "warn" +# TODO missing_docs = "warn" +trivial_casts = "warn" +trivial_numeric_casts = "warn" +unsafe_code = "warn" +unused_import_braces = "warn" +unused_lifetimes = "warn" +unused_macro_rules = "warn" +unused_qualifications = "warn" + +[workspace.lints.clippy] +allow_attributes = "warn" +allow_attributes_without_reason = "warn" +as_underscore = "warn" +assertions_on_result_states = "warn" +bool_to_int_with_if = "warn" +borrow_as_ptr = "warn" +case_sensitive_file_extension_comparisons = "warn" +cast_lossless = "warn" +cast_possible_truncation = "warn" +cast_possible_wrap = "warn" +cast_precision_loss = "warn" +cast_ptr_alignment = "warn" +cast_sign_loss = "warn" +checked_conversions = "warn" +clone_on_ref_ptr = "warn" +cloned_instead_of_copied = "warn" +copy_iterator = "warn" +create_dir = "warn" +dbg_macro = "warn" +decimal_literal_representation = "warn" +default_trait_access = "warn" +default_union_representation = "warn" +deref_by_slicing = "warn" +disallowed_script_idents = "warn" +doc_link_with_quotes = "warn" +empty_drop = "warn" +empty_enum = "warn" +empty_structs_with_brackets = "warn" +enum_glob_use = "warn" +error_impl_error = "warn" +exit = "warn" +expect_used = "warn" +expl_impl_clone_on_copy = "warn" +explicit_deref_methods = "warn" +explicit_into_iter_loop = "warn" +explicit_iter_loop = "warn" +filetype_is_file = "warn" +filter_map_next = "warn" +flat_map_option = "warn" +fn_params_excessive_bools = "warn" +fn_to_numeric_cast_any = "warn" +format_push_string = "warn" +from_iter_instead_of_collect = "warn" +get_unwrap = "warn" +host_endian_bytes = "warn" +if_not_else = "warn" +if_then_some_else_none = "warn" +ignored_unit_patterns = "warn" +implicit_clone = "warn" +implicit_hasher = "warn" +inconsistent_struct_constructor = "warn" +index_refutable_slice = "warn" +inefficient_to_string = "warn" +inline_always = "warn" +inline_asm_x86_att_syntax = "warn" +inline_asm_x86_intel_syntax = "warn" +invalid_upcast_comparisons = "warn" +items_after_statements = "warn" +iter_not_returning_iterator = "warn" +large_digit_groups = "warn" +large_futures = "warn" +large_include_file = "warn" +large_stack_arrays = "warn" +large_types_passed_by_value = "warn" +let_underscore_must_use = "warn" +let_underscore_untyped = "warn" +linkedlist = "warn" +lossy_float_literal = "warn" +macro_use_imports = "warn" +manual_assert = "warn" +manual_instant_elapsed = "warn" +manual_let_else = "warn" +manual_ok_or = "warn" +manual_string_new = "warn" +many_single_char_names = "warn" +map_unwrap_or = "warn" +match_bool = "warn" +match_on_vec_items = "warn" +match_same_arms = "warn" +match_wild_err_arm = "warn" +match_wildcard_for_single_variants = "warn" +maybe_infinite_iter = "warn" +mem_forget = "warn" +mismatching_type_param_order = "warn" +missing_assert_message = "warn" +missing_asserts_for_indexing = "warn" +missing_enforced_import_renames = "warn" +missing_fields_in_debug = "warn" +multiple_inherent_impl = "warn" +mut_mut = "warn" +mutex_atomic = "warn" +naive_bytecount = "warn" +needless_bitwise_bool = "warn" +needless_continue = "warn" +needless_for_each = "warn" +needless_pass_by_value = "warn" +needless_raw_strings = "warn" +negative_feature_names = "warn" +no_effect_underscore_binding = "warn" +no_mangle_with_rust_abi = "warn" +non_ascii_literal = "warn" +panic = "warn" +panic_in_result_fn = "warn" +partial_pub_fields = "warn" +print_stderr = "warn" +print_stdout = "warn" +ptr_as_ptr = "warn" +ptr_cast_constness = "warn" +pub_without_shorthand = "warn" +range_minus_one = "warn" +range_plus_one = "warn" +rc_buffer = "warn" +rc_mutex = "warn" +redundant_closure_for_method_calls = "warn" +redundant_else = "warn" +redundant_feature_names = "warn" +redundant_type_annotations = "warn" +ref_binding_to_reference = "warn" +ref_option_ref = "warn" +ref_patterns = "warn" +rest_pat_in_fully_bound_structs = "warn" +return_self_not_must_use = "warn" +same_functions_in_if_condition = "warn" +same_name_method = "warn" +semicolon_inside_block = "warn" +shadow_same = "warn" +should_panic_without_expect = "warn" +single_match_else = "warn" +stable_sort_primitive = "warn" +str_to_string = "warn" +string_add = "warn" +string_add_assign = "warn" +string_lit_chars_any = "warn" +string_to_string = "warn" +struct_excessive_bools = "warn" +suspicious_xor_used_as_pow = "warn" +tests_outside_test_module = "warn" +todo = "warn" +transmute_ptr_to_ptr = "warn" +trivially_copy_pass_by_ref = "warn" +try_err = "warn" +unchecked_duration_subtraction = "warn" +undocumented_unsafe_blocks = "warn" +unicode_not_nfc = "warn" +unimplemented = "warn" +uninlined_format_args = "warn" +unnecessary_box_returns = "warn" +unnecessary_join = "warn" +unnecessary_safety_comment = "warn" +unnecessary_safety_doc = "warn" +unnecessary_self_imports = "warn" +unnecessary_wraps = "warn" +unneeded_field_pattern = "warn" +unnested_or_patterns = "warn" +unreadable_literal = "warn" +unsafe_derive_deserialize = "warn" +unseparated_literal_suffix = "warn" +unused_async = "warn" +unused_self = "warn" +unwrap_in_result = "warn" +use_debug = "warn" +used_underscore_binding = "warn" +verbose_bit_mask = "warn" +verbose_file_reads = "warn" +wildcard_dependencies = "warn" +zero_sized_map_values = "warn" + [profile.release] lto = true codegen-units = 1 [profile.release.package.oxigraph-js] opt-level = "z" + diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 0e3e4471..f60ce1ca 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -41,3 +41,6 @@ assert_cmd = "2.0" assert_fs = "1.0" escargot = "0.5" predicates = ">=2.0, <4.0" + +[lints] +workspace = true diff --git a/js/Cargo.toml b/js/Cargo.toml index bfc2ffd5..41f1d0ef 100644 --- a/js/Cargo.toml +++ b/js/Cargo.toml @@ -20,3 +20,6 @@ oxigraph = { version = "0.4.0-alpha.1-dev", path="../lib", features = ["js"] } wasm-bindgen = "0.2.83" js-sys = "0.3.60" console_error_panic_hook = "0.1.7" + +[lints] +workspace = true diff --git a/lib/Cargo.toml b/lib/Cargo.toml index 39a0661d..dbb7f599 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -58,6 +58,9 @@ criterion = "0.5" oxhttp = "0.2.0-alpha.3" zstd = ">=0.12, <0.14" +[lints] +workspace = true + [package.metadata.docs.rs] all-features = true rustdoc-args = ["--cfg", "docsrs"] diff --git a/lib/oxrdf/Cargo.toml b/lib/oxrdf/Cargo.toml index a61523a3..2196bc9a 100644 --- a/lib/oxrdf/Cargo.toml +++ b/lib/oxrdf/Cargo.toml @@ -24,6 +24,9 @@ oxilangtag = "0.1" oxiri = "0.2" oxsdatatypes = { version = "0.2.0-alpha.1-dev", path="../oxsdatatypes", optional = true } +[lints] +workspace = true + [package.metadata.docs.rs] all-features = true rustdoc-args = ["--cfg", "docsrs"] \ No newline at end of file diff --git a/lib/oxrdf/src/interning.rs b/lib/oxrdf/src/interning.rs index e647c26b..ef436149 100644 --- a/lib/oxrdf/src/interning.rs +++ b/lib/oxrdf/src/interning.rs @@ -101,7 +101,7 @@ impl InternedNamedNode { }) } - pub fn decode_from(self, interner: &Interner) -> NamedNodeRef { + pub fn decode_from(self, interner: &Interner) -> NamedNodeRef<'_> { NamedNodeRef::new_unchecked(interner.resolve(self.id)) } @@ -154,7 +154,7 @@ impl InternedBlankNode { } } - pub fn decode_from(self, interner: &Interner) -> BlankNodeRef { + pub fn decode_from(self, interner: &Interner) -> BlankNodeRef<'_> { BlankNodeRef::new_unchecked(match self { Self::Number { id } => &interner.string_for_blank_node_id[&id], Self::Other { id } => interner.resolve(id), diff --git a/lib/oxrdfio/Cargo.toml b/lib/oxrdfio/Cargo.toml index 45683a5e..0c14f9fa 100644 --- a/lib/oxrdfio/Cargo.toml +++ b/lib/oxrdfio/Cargo.toml @@ -28,6 +28,9 @@ tokio = { version = "1.29", optional = true, features = ["io-util"] } [dev-dependencies] tokio = { version = "1.29", features = ["rt", "macros"] } +[lints] +workspace = true + [package.metadata.docs.rs] all-features = true rustdoc-args = ["--cfg", "docsrs"] diff --git a/lib/oxrdfxml/Cargo.toml b/lib/oxrdfxml/Cargo.toml index 660a631d..bc5e8993 100644 --- a/lib/oxrdfxml/Cargo.toml +++ b/lib/oxrdfxml/Cargo.toml @@ -28,6 +28,9 @@ tokio = { version = "1.29", optional = true, features = ["io-util"] } [dev-dependencies] tokio = { version = "1.29", features = ["rt", "macros"] } +[lints] +workspace = true + [package.metadata.docs.rs] all-features = true rustdoc-args = ["--cfg", "docsrs"] diff --git a/lib/oxrdfxml/src/parser.rs b/lib/oxrdfxml/src/parser.rs index f13bbbd4..ebbe80d1 100644 --- a/lib/oxrdfxml/src/parser.rs +++ b/lib/oxrdfxml/src/parser.rs @@ -417,7 +417,11 @@ struct RdfXmlReader { } impl RdfXmlReader { - fn parse_event(&mut self, event: Event, results: &mut Vec) -> Result<(), ParseError> { + fn parse_event( + &mut self, + event: Event<'_>, + results: &mut Vec, + ) -> Result<(), ParseError> { match event { Event::Start(event) => self.parse_start_event(&event, results), Event::End(event) => self.parse_end_event(&event, results), @@ -902,7 +906,7 @@ impl RdfXmlReader { fn resolve_ns_name( &self, - namespace: ResolveResult, + namespace: ResolveResult<'_>, local_name: LocalName<'_>, ) -> Result { match namespace { @@ -1141,7 +1145,7 @@ impl RdfXmlReader { } } - fn convert_attribute(&self, attribute: &Attribute) -> Result { + fn convert_attribute(&self, attribute: &Attribute<'_>) -> Result { Ok(attribute .decode_and_unescape_value_with(&self.reader, |e| self.resolve_entity(e))? .into_owned()) diff --git a/lib/oxsdatatypes/Cargo.toml b/lib/oxsdatatypes/Cargo.toml index 10f9f4f2..46eeaa20 100644 --- a/lib/oxsdatatypes/Cargo.toml +++ b/lib/oxsdatatypes/Cargo.toml @@ -21,6 +21,9 @@ custom-now = [] [target.'cfg(all(target_family = "wasm", target_os = "unknown"))'.dependencies] js-sys = { version = "0.3.60", optional = true } +[lints] +workspace = true + [package.metadata.docs.rs] all-features = true rustdoc-args = ["--cfg", "docsrs"] diff --git a/lib/oxsdatatypes/src/date_time.rs b/lib/oxsdatatypes/src/date_time.rs index c7a1c16a..3c21cf43 100644 --- a/lib/oxsdatatypes/src/date_time.rs +++ b/lib/oxsdatatypes/src/date_time.rs @@ -3178,8 +3178,9 @@ mod tests { #[cfg(feature = "custom-now")] #[test] fn custom_now() { + #[allow(unsafe_code)] #[no_mangle] - fn custom_ox_now() -> Duration { + extern "Rust" fn custom_ox_now() -> Duration { Duration::default() } DateTime::now(); diff --git a/lib/oxttl/Cargo.toml b/lib/oxttl/Cargo.toml index 98b583bf..eeeb10a3 100644 --- a/lib/oxttl/Cargo.toml +++ b/lib/oxttl/Cargo.toml @@ -29,6 +29,9 @@ tokio = { version = "1.29", optional = true, features = ["io-util"] } [dev-dependencies] tokio = { version = "1.29", features = ["rt", "macros"] } +[lints] +workspace = true + [package.metadata.docs.rs] all-features = true rustdoc-args = ["--cfg", "docsrs"] diff --git a/lib/oxttl/src/line_formats.rs b/lib/oxttl/src/line_formats.rs index 8c22c1f8..ec439a57 100644 --- a/lib/oxttl/src/line_formats.rs +++ b/lib/oxttl/src/line_formats.rs @@ -53,7 +53,7 @@ impl RuleRecognizer for NQuadsRecognizer { fn recognize_next( mut self, - token: N3Token, + token: N3Token<'_>, context: &mut NQuadsRecognizerContext, results: &mut Vec, errors: &mut Vec, diff --git a/lib/oxttl/src/n3.rs b/lib/oxttl/src/n3.rs index c4f02613..ac6d0388 100644 --- a/lib/oxttl/src/n3.rs +++ b/lib/oxttl/src/n3.rs @@ -615,7 +615,7 @@ impl RuleRecognizer for N3Recognizer { fn recognize_next( mut self, - token: N3Token, + token: N3Token<'_>, context: &mut N3RecognizerContext, results: &mut Vec, errors: &mut Vec, diff --git a/lib/oxttl/src/terse.rs b/lib/oxttl/src/terse.rs index ecce10b6..46dcd740 100644 --- a/lib/oxttl/src/terse.rs +++ b/lib/oxttl/src/terse.rs @@ -45,7 +45,7 @@ impl RuleRecognizer for TriGRecognizer { fn recognize_next( mut self, - token: N3Token, + token: N3Token<'_>, context: &mut TriGRecognizerContext, results: &mut Vec, errors: &mut Vec, diff --git a/lib/sparesults/Cargo.toml b/lib/sparesults/Cargo.toml index fcc95b39..e13b22f5 100644 --- a/lib/sparesults/Cargo.toml +++ b/lib/sparesults/Cargo.toml @@ -29,6 +29,9 @@ tokio = { version = "1.29", optional = true, features = ["io-util"] } [dev-dependencies] tokio = { version = "1.29", features = ["rt", "macros"] } +[lints] +workspace = true + [package.metadata.docs.rs] all-features = true rustdoc-args = ["--cfg", "docsrs"] diff --git a/lib/sparesults/src/json.rs b/lib/sparesults/src/json.rs index cbaa873c..85b03fcd 100644 --- a/lib/sparesults/src/json.rs +++ b/lib/sparesults/src/json.rs @@ -100,7 +100,7 @@ impl ToTokioAsyncWriteJsonSolutionsWriter { let mut buffer = Vec::with_capacity(48); let inner = InnerJsonSolutionsWriter::start(&mut buffer, variables); Self::do_write(&mut writer, buffer).await?; - Ok(Self { writer, inner }) + Ok(Self { inner, writer }) } pub async fn write<'a>( diff --git a/lib/sparesults/src/xml.rs b/lib/sparesults/src/xml.rs index 808c0f8c..3c2d91a8 100644 --- a/lib/sparesults/src/xml.rs +++ b/lib/sparesults/src/xml.rs @@ -104,7 +104,7 @@ impl ToTokioAsyncWriteXmlSolutionsWriter { let mut buffer = Vec::with_capacity(48); let inner = InnerXmlSolutionsWriter::start(&mut buffer, variables); Self::do_write(&mut writer, buffer).await?; - Ok(Self { writer, inner }) + Ok(Self { inner, writer }) } pub async fn write<'a>( diff --git a/lib/spargebra/Cargo.toml b/lib/spargebra/Cargo.toml index cfc2e8cb..42a14b20 100644 --- a/lib/spargebra/Cargo.toml +++ b/lib/spargebra/Cargo.toml @@ -27,6 +27,9 @@ oxiri = "0.2" oxilangtag = "0.1" oxrdf = { version = "0.2.0-alpha.1-dev", path="../oxrdf" } +[lints] +workspace = true + [package.metadata.docs.rs] all-features = true rustdoc-args = ["--cfg", "docsrs"] diff --git a/lib/sparopt/Cargo.toml b/lib/sparopt/Cargo.toml index 586388bd..845209ba 100644 --- a/lib/sparopt/Cargo.toml +++ b/lib/sparopt/Cargo.toml @@ -25,6 +25,9 @@ oxrdf = { version = "0.2.0-alpha.1-dev", path="../oxrdf" } rand = "0.8" spargebra = { version = "0.3.0-alpha.1-dev", path="../spargebra" } +[lints] +workspace = true + [package.metadata.docs.rs] all-features = true rustdoc-args = ["--cfg", "docsrs"] diff --git a/lib/sparql-smith/Cargo.toml b/lib/sparql-smith/Cargo.toml index 22d361d6..4217afdf 100644 --- a/lib/sparql-smith/Cargo.toml +++ b/lib/sparql-smith/Cargo.toml @@ -20,5 +20,8 @@ limit-offset = ["order"] order = [] sep-0006 = [] +[lints] +workspace = true + [dependencies] arbitrary = { version = "1.3", features = ["derive"] } diff --git a/lib/src/sparql/algebra.rs b/lib/src/sparql/algebra.rs index 17b6cc05..d83241c6 100644 --- a/lib/src/sparql/algebra.rs +++ b/lib/src/sparql/algebra.rs @@ -70,7 +70,7 @@ impl FromStr for Query { } } -impl<'a> TryFrom<&'a str> for Query { +impl TryFrom<&str> for Query { type Error = spargebra::ParseError; fn try_from(query: &str) -> Result { @@ -78,7 +78,7 @@ impl<'a> TryFrom<&'a str> for Query { } } -impl<'a> TryFrom<&'a String> for Query { +impl TryFrom<&String> for Query { type Error = spargebra::ParseError; fn try_from(query: &String) -> Result { @@ -163,7 +163,7 @@ impl FromStr for Update { } } -impl<'a> TryFrom<&'a str> for Update { +impl TryFrom<&str> for Update { type Error = spargebra::ParseError; fn try_from(update: &str) -> Result { @@ -171,7 +171,7 @@ impl<'a> TryFrom<&'a str> for Update { } } -impl<'a> TryFrom<&'a String> for Update { +impl TryFrom<&String> for Update { type Error = spargebra::ParseError; fn try_from(update: &String) -> Result { diff --git a/lib/src/sparql/model.rs b/lib/src/sparql/model.rs index 81712877..326e7603 100644 --- a/lib/src/sparql/model.rs +++ b/lib/src/sparql/model.rs @@ -176,6 +176,8 @@ pub struct QuerySolutionIter { } impl QuerySolutionIter { + /// Construct a new iterator of solution from an ordered list of solution variables and an iterator of solution tuples + /// (each tuple using the same ordering as the variable list such that tuple element 0 is the value for the variable 0...) pub fn new( variables: Arc<[Variable]>, iter: impl Iterator>, EvaluationError>> + 'static, diff --git a/lib/src/sparql/service.rs b/lib/src/sparql/service.rs index 2ec0b7a3..4db172e4 100644 --- a/lib/src/sparql/service.rs +++ b/lib/src/sparql/service.rs @@ -49,6 +49,7 @@ use std::time::Duration; /// # Result::<_,Box>::Ok(()) /// ``` pub trait ServiceHandler: Send + Sync { + /// The service evaluation error. type Error: Error + Send + Sync + 'static; /// Evaluates a [`Query`] against a given service identified by a [`NamedNode`]. diff --git a/lib/src/storage/backend/rocksdb.rs b/lib/src/storage/backend/rocksdb.rs index 5e6f4103..ba364cd6 100644 --- a/lib/src/storage/backend/rocksdb.rs +++ b/lib/src/storage/backend/rocksdb.rs @@ -30,24 +30,14 @@ use std::thread::{available_parallelism, yield_now}; use std::{ptr, slice}; macro_rules! ffi_result { - ( $($function:ident)::*() ) => { - ffi_result_impl!($($function)::*()) - }; - - ( $($function:ident)::*( $arg1:expr $(, $arg:expr)* $(,)? ) ) => { - ffi_result_impl!($($function)::*($arg1 $(, $arg)* ,)) - }; -} - -macro_rules! ffi_result_impl { - ( $($function:ident)::*( $($arg:expr,)*) ) => {{ + ( $($function:ident)::*( $arg1:expr $(, $arg:expr)* $(,)? ) ) => {{ let mut status = rocksdb_status_t { code: rocksdb_status_code_t_rocksdb_status_code_ok, subcode: rocksdb_status_subcode_t_rocksdb_status_subcode_none, severity: rocksdb_status_severity_t_rocksdb_status_severity_none, string: ptr::null() }; - let result = $($function)::*($($arg,)* &mut status); + let result = $($function)::*($arg1 $(, $arg)* , &mut status); if status.code == rocksdb_status_code_t_rocksdb_status_code_ok { Ok(result) } else { @@ -119,8 +109,7 @@ impl Drop for RwDbHandler { rocksdb_block_based_options_destroy(self.block_based_table_options); } if self.in_memory { - #[allow(clippy::let_underscore_must_use)] - let _: io::Result<()> = remove_dir_all(&self.path); + drop(remove_dir_all(&self.path)); } } } @@ -154,8 +143,7 @@ impl Drop for RoDbHandler { rocksdb_options_destroy(self.options); } if let Some(path) = &self.path_to_remove { - #[allow(clippy::let_underscore_must_use)] - let _: io::Result<()> = remove_dir_all(path); + drop(remove_dir_all(path)); } } } @@ -571,9 +559,9 @@ impl Db { DbKind::ReadOnly(db) => { if db.is_secondary { // We try to refresh (and ignore the errors) - #[allow(clippy::let_underscore_must_use)] - let _: Result<(), ErrorStatus> = - ffi_result!(rocksdb_try_catch_up_with_primary_with_status(db.db)); + drop(ffi_result!(rocksdb_try_catch_up_with_primary_with_status( + db.db + ))); } let options = rocksdb_readoptions_create_copy(db.read_options); Reader { diff --git a/lib/src/storage/error.rs b/lib/src/storage/error.rs index 690b6bfd..8c874d77 100644 --- a/lib/src/storage/error.rs +++ b/lib/src/storage/error.rs @@ -128,7 +128,12 @@ pub enum LoaderError { /// An error raised during the insertion in the store. Storage(StorageError), /// The base IRI is invalid. - InvalidBaseIri { iri: String, error: IriParseError }, + InvalidBaseIri { + /// The IRI itself. + iri: String, + /// The parsing error. + error: IriParseError, + }, } impl fmt::Display for LoaderError { diff --git a/lints/build_clippy_config.py b/lints/build_clippy_config.py new file mode 100644 index 00000000..f56bcb00 --- /dev/null +++ b/lints/build_clippy_config.py @@ -0,0 +1,83 @@ +import json +from pathlib import Path +from urllib.request import urlopen + +import tomlkit + +MSRV = "1.74.0" +LINT_BLACKLIST = { + "absolute_paths", # TODO: might be nice + "alloc_instead_of_core", + "arithmetic_side_effects", # TODO: might be nice + "as_conversions", + "big_endian_bytes", + "cargo_common_metadata", # TODO: might be nice + "doc_markdown", # Too many false positives + "default_numeric_fallback", + "else_if_without_else", + "exhaustive_enums", + "exhaustive_structs", + "float_arithmetic", + "float_cmp", + "float_cmp_const", + "impl_trait_in_params", + "implicit_return", + "indexing_slicing", + "integer_division", + "little_endian_bytes", + "map_err_ignore", + "min_ident_chars", + "missing_docs_in_private_items", + "missing_errors_doc", + "missing_inline_in_public_items", + "missing_panics_doc", + "missing_trait_methods", + "mixed_read_write_in_expression", + "mod_module_files", + "module_name_repetitions", + "modulo_arithmetic", + "multiple_crate_versions", + "multiple_unsafe_ops_per_block", + "must_use_candidate", # TODO: might be nice + "option_option", + "pattern_type_mismatch", + "pub_use", + "pub_with_shorthand", + "question_mark_used", + "self_named_module_files", # TODO: might be nice + "semicolon_if_nothing_returned", # TODO: might be nice + "semicolon_outside_block", + "similar_names", + "single_call_fn", + "single_char_lifetime_names", + "std_instead_of_alloc", + "std_instead_of_core", + "shadow_reuse", + "shadow_unrelated", + "string_slice", # TODO: might be nice + "too_many_lines", + "separated_literal_suffix", + "unreachable", # TODO: might be nice + "unwrap_used", # TODO: might be nice to use expect instead + "wildcard_enum_match_arm", # TODO: might be nice + "wildcard_imports", # TODO: might be nice +} + +lints = set() +with urlopen( + f"https://rust-lang.github.io/rust-clippy/rust-{MSRV}/lints.json" +) as response: + for lint in json.load(response): + if lint["level"] == "allow" and lint["group"] != "nursery": + lints.add(lint["id"]) + +for flag in LINT_BLACKLIST: + if flag in lints: + lints.remove(flag) + else: + print(f"Unused blacklisted flag: {flag}") + +cargo_path = Path(__file__).parent.parent / "Cargo.toml" +cargo_toml = tomlkit.parse(cargo_path.read_text()) +cargo_toml["workspace"]["lints"]["clippy"] = {lint: "warn" for lint in sorted(lints)} +cargo_path.write_text(tomlkit.dumps(cargo_toml)) diff --git a/lints/build_config.py b/lints/build_config.py deleted file mode 100644 index 749c0622..00000000 --- a/lints/build_config.py +++ /dev/null @@ -1,90 +0,0 @@ -import json -from pathlib import Path -from urllib.request import urlopen - -MSRV = "1.74.0" -DEFAULT_BUILD_FLAGS = { - "-Wtrivial-casts", - "-Wtrivial-numeric-casts", - "-Wunsafe-code", - "-Wunused-lifetimes", - "-Wunused-qualifications", -} -FLAGS_BLACKLIST = { - "-Wclippy::absolute-paths", # TODO: might be nice - "-Wclippy::alloc-instead-of-core", - "-Wclippy::arithmetic-side-effects", # TODO: might be nice - "-Wclippy::as-conversions", - "-Wclippy::big-endian-bytes", - "-Wclippy::cargo-common-metadata", # TODO: might be nice - "-Wclippy::doc-markdown", # Too many false positives - "-Wclippy::default-numeric-fallback", - "-Wclippy::else-if-without-else", - "-Wclippy::exhaustive-enums", - "-Wclippy::exhaustive-structs", - "-Wclippy::float-arithmetic", - "-Wclippy::float-cmp", - "-Wclippy::float-cmp-const", - "-Wclippy::impl-trait-in-params", - "-Wclippy::implicit-return", - "-Wclippy::indexing-slicing", - "-Wclippy::integer-division", - "-Wclippy::little-endian-bytes", - "-Wclippy::map-err-ignore", - "-Wclippy::min-ident-chars", - "-Wclippy::missing-docs-in-private-items", - "-Wclippy::missing-errors-doc", - "-Wclippy::missing-inline-in-public-items", - "-Wclippy::missing-panics-doc", - "-Wclippy::missing-trait-methods", - "-Wclippy::mixed-read-write-in-expression", - "-Wclippy::mod-module-files", - "-Wclippy::module-name-repetitions", - "-Wclippy::modulo-arithmetic", - "-Wclippy::multiple-crate-versions", - "-Wclippy::multiple-unsafe-ops-per-block", - "-Wclippy::must-use-candidate", # TODO: might be nice - "-Wclippy::option-option", - "-Wclippy::pattern-type-mismatch", - "-Wclippy::pub-use", - "-Wclippy::pub-with-shorthand", - "-Wclippy::question-mark-used", - "-Wclippy::self-named-module-files", # TODO: might be nice - "-Wclippy::semicolon-if-nothing-returned", # TODO: might be nice - "-Wclippy::semicolon-outside-block", - "-Wclippy::similar-names", - "-Wclippy::single-call-fn", - "-Wclippy::single-char-lifetime-names", - "-Wclippy::std-instead-of-alloc", - "-Wclippy::std-instead-of-core", - "-Wclippy::shadow-reuse", - "-Wclippy::shadow-unrelated", - "-Wclippy::string-slice", # TODO: might be nice - "-Wclippy::too-many-lines", - "-Wclippy::separated-literal-suffix", - "-Wclippy::unreachable", # TODO: might be nice - "-Wclippy::unwrap-used", # TODO: might be nice to use expect instead - "-Wclippy::wildcard-enum-match-arm", # TODO: might be nice - "-Wclippy::wildcard-imports", # TODO: might be nice -} - -build_flags = set(DEFAULT_BUILD_FLAGS) -with urlopen( - f"https://rust-lang.github.io/rust-clippy/rust-{MSRV}/lints.json" -) as response: - for lint in json.load(response): - if lint["level"] == "allow" and lint["group"] != "nursery": - build_flags.add(f"-Wclippy::{lint['id'].replace('_', '-')}") - -for flag in FLAGS_BLACKLIST: - if flag in build_flags: - build_flags.remove(flag) - else: - print(f"Unused blacklisted flag: {flag}") - -with (Path(__file__).parent.parent / ".cargo" / "config.toml").open("wt") as fp: - fp.write("[build]\n") - fp.write("rustflags = [\n") - for flag in sorted(build_flags): - fp.write(f' "{flag}",\n') - fp.write("]\n") diff --git a/python/Cargo.toml b/python/Cargo.toml index 58538d3a..531c01df 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -27,3 +27,6 @@ rustls = ["oxigraph/http-client-rustls-native"] [dependencies] oxigraph = { version = "0.4.0-alpha.1-dev", path="../lib" } pyo3 = { version = "0.20", features = ["extension-module"] } + +[lints] +workspace = true diff --git a/python/src/io.rs b/python/src/io.rs index 86896853..b8c42443 100644 --- a/python/src/io.rs +++ b/python/src/io.rs @@ -129,10 +129,10 @@ pub fn serialize<'a>( let mut writer = RdfSerializer::from_format(format).serialize_to_write(output); for i in input.iter()? { let i = i?; - if let Ok(triple) = i.extract::>() { + if let Ok(triple) = i.extract::>() { writer.write_triple(&*triple) } else { - let quad = i.extract::>()?; + let quad = i.extract::>()?; let quad = QuadRef::from(&*quad); if !quad.graph_name.is_default_graph() && !format.supports_datasets() { return Err(PyValueError::new_err( @@ -157,7 +157,7 @@ pub struct PyQuadReader { #[pymethods] impl PyQuadReader { - fn __iter__(slf: PyRef<'_, Self>) -> PyRef { + fn __iter__(slf: PyRef<'_, Self>) -> PyRef<'_, Self> { slf } @@ -348,7 +348,7 @@ impl PyRdfFormat { } /// :rtype: RdfFormat - fn __copy__(slf: PyRef<'_, Self>) -> PyRef { + fn __copy__(slf: PyRef<'_, Self>) -> PyRef<'_, Self> { slf } diff --git a/python/src/model.rs b/python/src/model.rs index db3f0258..cf673d65 100644 --- a/python/src/model.rs +++ b/python/src/model.rs @@ -96,7 +96,7 @@ impl PyNamedNode { } fn __richcmp__(&self, other: &PyAny, op: CompareOp) -> PyResult { - if let Ok(other) = other.extract::>() { + if let Ok(other) = other.extract::>() { Ok(op.matches(self.cmp(&other))) } else if PyBlankNode::is_type_of(other) || PyLiteral::is_type_of(other) @@ -116,7 +116,7 @@ impl PyNamedNode { } /// :rtype: NamedNode - fn __copy__(slf: PyRef<'_, Self>) -> PyRef { + fn __copy__(slf: PyRef<'_, Self>) -> PyRef<'_, Self> { slf } @@ -223,7 +223,7 @@ impl PyBlankNode { } fn __richcmp__(&self, other: &PyAny, op: CompareOp) -> PyResult { - if let Ok(other) = other.extract::>() { + if let Ok(other) = other.extract::>() { eq_compare(self, &other, op) } else if PyNamedNode::is_type_of(other) || PyLiteral::is_type_of(other) @@ -243,7 +243,7 @@ impl PyBlankNode { } /// :rtype: BlankNode - fn __copy__(slf: PyRef<'_, Self>) -> PyRef { + fn __copy__(slf: PyRef<'_, Self>) -> PyRef<'_, Self> { slf } @@ -380,7 +380,7 @@ impl PyLiteral { } fn __richcmp__(&self, other: &PyAny, op: CompareOp) -> PyResult { - if let Ok(other) = other.extract::>() { + if let Ok(other) = other.extract::>() { eq_compare(self, &other, op) } else if PyNamedNode::is_type_of(other) || PyBlankNode::is_type_of(other) @@ -406,7 +406,7 @@ impl PyLiteral { } /// :rtype: Literal - fn __copy__(slf: PyRef<'_, Self>) -> PyRef { + fn __copy__(slf: PyRef<'_, Self>) -> PyRef<'_, Self> { slf } @@ -454,7 +454,7 @@ impl PyDefaultGraph { } fn __richcmp__(&self, other: &PyAny, op: CompareOp) -> PyResult { - if let Ok(other) = other.extract::>() { + if let Ok(other) = other.extract::>() { eq_compare(self, &other, op) } else if PyNamedNode::is_type_of(other) || PyBlankNode::is_type_of(other) @@ -474,7 +474,7 @@ impl PyDefaultGraph { } /// :rtype: DefaultGraph - fn __copy__(slf: PyRef<'_, Self>) -> PyRef { + fn __copy__(slf: PyRef<'_, Self>) -> PyRef<'_, Self> { slf } @@ -735,7 +735,7 @@ impl PyTriple { } /// :rtype: Triple - fn __copy__(slf: PyRef<'_, Self>) -> PyRef { + fn __copy__(slf: PyRef<'_, Self>) -> PyRef<'_, Self> { slf } @@ -972,7 +972,7 @@ impl PyQuad { } /// :rtype: Quad - fn __copy__(slf: PyRef<'_, Self>) -> PyRef { + fn __copy__(slf: PyRef<'_, Self>) -> PyRef<'_, Self> { slf } @@ -1064,7 +1064,7 @@ impl PyVariable { } /// :rtype: Variable - fn __copy__(slf: PyRef<'_, Self>) -> PyRef { + fn __copy__(slf: PyRef<'_, Self>) -> PyRef<'_, Self> { slf } @@ -1093,7 +1093,7 @@ impl<'a> TryFrom<&'a PyAny> for PyNamedNodeRef<'a> { type Error = PyErr; fn try_from(value: &'a PyAny) -> PyResult { - if let Ok(node) = value.extract::>() { + if let Ok(node) = value.extract::>() { Ok(Self(node)) } else { Err(PyTypeError::new_err(format!( @@ -1122,9 +1122,9 @@ impl<'a> TryFrom<&'a PyAny> for PyNamedOrBlankNodeRef<'a> { type Error = PyErr; fn try_from(value: &'a PyAny) -> PyResult { - if let Ok(node) = value.extract::>() { + if let Ok(node) = value.extract::>() { Ok(Self::NamedNode(node)) - } else if let Ok(node) = value.extract::>() { + } else if let Ok(node) = value.extract::>() { Ok(Self::BlankNode(node)) } else { Err(PyTypeError::new_err(format!( @@ -1155,11 +1155,11 @@ impl<'a> TryFrom<&'a PyAny> for PySubjectRef<'a> { type Error = PyErr; fn try_from(value: &'a PyAny) -> PyResult { - if let Ok(node) = value.extract::>() { + if let Ok(node) = value.extract::>() { Ok(Self::NamedNode(node)) - } else if let Ok(node) = value.extract::>() { + } else if let Ok(node) = value.extract::>() { Ok(Self::BlankNode(node)) - } else if let Ok(node) = value.extract::>() { + } else if let Ok(node) = value.extract::>() { Ok(Self::Triple(node)) } else { Err(PyTypeError::new_err(format!( @@ -1198,13 +1198,13 @@ impl<'a> TryFrom<&'a PyAny> for PyTermRef<'a> { type Error = PyErr; fn try_from(value: &'a PyAny) -> PyResult { - if let Ok(node) = value.extract::>() { + if let Ok(node) = value.extract::>() { Ok(Self::NamedNode(node)) - } else if let Ok(node) = value.extract::>() { + } else if let Ok(node) = value.extract::>() { Ok(Self::BlankNode(node)) - } else if let Ok(node) = value.extract::>() { + } else if let Ok(node) = value.extract::>() { Ok(Self::Literal(node)) - } else if let Ok(node) = value.extract::>() { + } else if let Ok(node) = value.extract::>() { Ok(Self::Triple(node)) } else { Err(PyTypeError::new_err(format!( @@ -1241,11 +1241,11 @@ impl<'a> TryFrom<&'a PyAny> for PyGraphNameRef<'a> { type Error = PyErr; fn try_from(value: &'a PyAny) -> PyResult { - if let Ok(node) = value.extract::>() { + if let Ok(node) = value.extract::>() { Ok(Self::NamedNode(node)) - } else if let Ok(node) = value.extract::>() { + } else if let Ok(node) = value.extract::>() { Ok(Self::BlankNode(node)) - } else if value.extract::>().is_ok() { + } else if value.extract::>().is_ok() { Ok(Self::DefaultGraph) } else { Err(PyTypeError::new_err(format!( @@ -1341,7 +1341,7 @@ pub struct TripleComponentsIter { #[pymethods] impl TripleComponentsIter { - fn __iter__(slf: PyRef<'_, Self>) -> PyRef { + fn __iter__(slf: PyRef<'_, Self>) -> PyRef<'_, Self> { slf } @@ -1357,7 +1357,7 @@ pub struct QuadComponentsIter { #[pymethods] impl QuadComponentsIter { - fn __iter__(slf: PyRef<'_, Self>) -> PyRef { + fn __iter__(slf: PyRef<'_, Self>) -> PyRef<'_, Self> { slf } diff --git a/python/src/sparql.rs b/python/src/sparql.rs index 8f6c7498..9bce5ab5 100644 --- a/python/src/sparql.rs +++ b/python/src/sparql.rs @@ -164,7 +164,7 @@ pub struct SolutionValueIter { #[pymethods] impl SolutionValueIter { - fn __iter__(slf: PyRef<'_, Self>) -> PyRef { + fn __iter__(slf: PyRef<'_, Self>) -> PyRef<'_, Self> { slf } @@ -278,7 +278,7 @@ impl PyQuerySolutions { ) } - fn __iter__(slf: PyRef<'_, Self>) -> PyRef { + fn __iter__(slf: PyRef<'_, Self>) -> PyRef<'_, Self> { slf } @@ -435,7 +435,7 @@ impl PyQueryTriples { ) } - fn __iter__(slf: PyRef<'_, Self>) -> PyRef { + fn __iter__(slf: PyRef<'_, Self>) -> PyRef<'_, Self> { slf } @@ -637,7 +637,7 @@ impl PyQueryResultsFormat { } /// :rtype: QueryResultsFormat - fn __copy__(slf: PyRef<'_, Self>) -> PyRef { + fn __copy__(slf: PyRef<'_, Self>) -> PyRef<'_, Self> { slf } diff --git a/python/src/store.rs b/python/src/store.rs index 81abb840..90b3858c 100644 --- a/python/src/store.rs +++ b/python/src/store.rs @@ -767,7 +767,7 @@ pub struct QuadIter { #[pymethods] impl QuadIter { - fn __iter__(slf: PyRef<'_, Self>) -> PyRef { + fn __iter__(slf: PyRef<'_, Self>) -> PyRef<'_, Self> { slf } @@ -786,7 +786,7 @@ pub struct GraphNameIter { #[pymethods] impl GraphNameIter { - fn __iter__(slf: PyRef<'_, Self>) -> PyRef { + fn __iter__(slf: PyRef<'_, Self>) -> PyRef<'_, Self> { slf } diff --git a/testsuite/Cargo.toml b/testsuite/Cargo.toml index 59b751c2..6e24e309 100644 --- a/testsuite/Cargo.toml +++ b/testsuite/Cargo.toml @@ -25,6 +25,9 @@ criterion = "0.5" rio_api = "0.8" rio_turtle = "0.8" +[lints] +workspace = true + [[bench]] name = "parser" harness = false \ No newline at end of file From ed94f56ab4f461f201c1e584ef9f59e3a6ffac9f Mon Sep 17 00:00:00 2001 From: Tpt Date: Sat, 30 Dec 2023 10:49:28 +0100 Subject: [PATCH 152/217] Fixes linux aarch64 build --- .github/workflows/artifacts.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/artifacts.yml b/.github/workflows/artifacts.yml index 5e170600..afbf06e9 100644 --- a/.github/workflows/artifacts.yml +++ b/.github/workflows/artifacts.yml @@ -23,7 +23,8 @@ jobs: - run: rustup update && rustup target add aarch64-unknown-linux-gnu - run: | sudo apt update && sudo apt-get install -y g++-aarch64-linux-gnu - echo -e "\n\n[target.aarch64-unknown-linux-gnu]\nlinker = \"aarch64-linux-gnu-gcc\"" >> .cargo/config.toml + mkdir .cargo + echo -e "[target.aarch64-unknown-linux-gnu]\nlinker = \"aarch64-linux-gnu-gcc\"" >> .cargo/config.toml - uses: Swatinem/rust-cache@v2 - run: cargo build --release --no-default-features --features rustls-native working-directory: ./cli From 5cc3e37876f7d362eb47854a5f1991dbf922f849 Mon Sep 17 00:00:00 2001 From: Tpt Date: Sat, 30 Dec 2023 16:58:37 +0100 Subject: [PATCH 153/217] Upgrades Cargo lock file --- Cargo.lock | 345 ++++++++++++++++++++++++++++++----------------------- 1 file changed, 196 insertions(+), 149 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 04eb776b..29e2795f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -34,9 +34,9 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "anstream" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ab91ebe16eb252986481c5b62f6098f3b698a45e34b5b98200cf20dd2484a44" +checksum = "d664a92ecae85fd0a7392615844904654d1d5f5514837f471ddef4a057aba1b6" dependencies = [ "anstyle", "anstyle-parse", @@ -54,37 +54,37 @@ checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87" [[package]] name = "anstyle-parse" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "317b9a89c1868f5ea6ff1d9539a69f45dffc21ce321ac1fd1160dfa48c8e2140" +checksum = "c75ac65da39e5fe5ab759307499ddad880d724eed2f6ce5b5e8a26f4f387928c" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.0.0" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" +checksum = "e28923312444cdd728e4738b3f9c9cac739500909bb3d3c94b43551b16517648" dependencies = [ - "windows-sys", + "windows-sys 0.52.0", ] [[package]] name = "anstyle-wincon" -version = "3.0.1" +version = "3.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0699d10d2f4d628a98ee7b57b289abbc98ff3bad977cb3152709d4bf2330628" +checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7" dependencies = [ "anstyle", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] name = "anyhow" -version = "1.0.75" +version = "1.0.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" +checksum = "c9d19de80eff169429ac1e9f48fffb163916b448a44e8e046186232046d9e1f9" [[package]] name = "arbitrary" @@ -112,9 +112,9 @@ dependencies = [ [[package]] name = "assert_fs" -version = "1.0.13" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f070617a68e5c2ed5d06ee8dd620ee18fb72b99f6c094bed34cf8ab07c875b48" +checksum = "adc5d78e9048d836d12a0c0040ca5f45b18a94d204b4ba4f677a8a7de162426b" dependencies = [ "anstyle", "doc-comment", @@ -198,9 +198,9 @@ dependencies = [ [[package]] name = "bstr" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "542f33a8835a0884b006a0c3df3dadd99c0c3f296ed26c2fdc8028e01ad6230c" +checksum = "c48f0051a4b4c5e0b6d365cd04af53aeaa209e3cc15ec2cdb69e73cc87fbd0dc" dependencies = [ "memchr", "regex-automata", @@ -290,9 +290,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.8" +version = "4.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2275f18819641850fa26c89acc84d465c1bf91ce57bc2748b28c420473352f64" +checksum = "dcfab8ba68f3668e89f6ff60f5b205cea56aa7b769451a59f34b8682f51c056d" dependencies = [ "clap_builder", "clap_derive", @@ -300,9 +300,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.8" +version = "4.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07cdf1b148b25c1e1f7a42225e30a0d99a615cd4637eae7365548dd4529b95bc" +checksum = "fb7fb5e4e979aec3be7791562fcba452f94ad85e954da024396433e0e25a79e9" dependencies = [ "anstream", "anstyle", @@ -346,9 +346,9 @@ dependencies = [ [[package]] name = "core-foundation" -version = "0.9.3" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" dependencies = [ "core-foundation-sys", "libc", @@ -356,9 +356,9 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.8.4" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" +checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" [[package]] name = "cpufeatures" @@ -416,9 +416,9 @@ dependencies = [ [[package]] name = "crossbeam-deque" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" +checksum = "fca89a0e215bab21874660c67903c5f143333cab1da83d041c7ded6053774751" dependencies = [ "cfg-if", "crossbeam-epoch", @@ -427,22 +427,20 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.15" +version = "0.9.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" +checksum = "0e3681d554572a651dda4186cd47240627c3d0114d45a95f6ad27f2f22e7548d" dependencies = [ "autocfg", "cfg-if", "crossbeam-utils", - "memoffset", - "scopeguard", ] [[package]] name = "crossbeam-utils" -version = "0.8.16" +version = "0.8.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" +checksum = "c3a430a770ebd84726f584a90ee7f020d28db52c6d02138900f22341f866d39c" dependencies = [ "cfg-if", ] @@ -459,9 +457,9 @@ dependencies = [ [[package]] name = "deranged" -version = "0.3.9" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f32d04922c60427da6f9fef14d042d9edddef64cb9d4ce0d64d0685fbeb1fd3" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" dependencies = [ "powerfmt", ] @@ -507,12 +505,12 @@ checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" [[package]] name = "errno" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f258a7194e7f7c2a7837a8913aeab7fd8c383457034fa20ce4dd3dcb813e8eb8" +checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -552,12 +550,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "fnv" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" - [[package]] name = "foreign-types" version = "0.3.2" @@ -575,9 +567,9 @@ checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" [[package]] name = "form_urlencoded" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" dependencies = [ "percent-encoding", ] @@ -616,9 +608,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.28.0" +version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" +checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" [[package]] name = "glob" @@ -628,15 +620,15 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] name = "globset" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "759c97c1e17c55525b57192c06a267cda0ac5210b222d6b82189a2338fa1c13d" +checksum = "57da3b9b5b85bd66f31093f8c408b90a74431672542466497dcbdfdc02034be1" dependencies = [ "aho-corasick", "bstr", - "fnv", "log", - "regex", + "regex-automata", + "regex-syntax", ] [[package]] @@ -676,11 +668,11 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "home" -version = "0.5.5" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" +checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" dependencies = [ - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -691,9 +683,9 @@ checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" [[package]] name = "idna" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" dependencies = [ "unicode-bidi", "unicode-normalization", @@ -701,17 +693,16 @@ dependencies = [ [[package]] name = "ignore" -version = "0.4.20" +version = "0.4.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbe7873dab538a9a44ad79ede1faf5f30d49f9a5c883ddbab48bce81b64b7492" +checksum = "747ad1b4ae841a78e8aba0d63adbfbeaea26b517b63705d47856b73015d27060" dependencies = [ + "crossbeam-deque", "globset", - "lazy_static", "log", "memchr", - "regex", + "regex-automata", "same-file", - "thread_local", "walkdir", "winapi-util", ] @@ -724,13 +715,13 @@ checksum = "1e186cfbae8084e513daff4240b4797e342f988cecda4fb6c939150f96315fd8" [[package]] name = "is-terminal" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" +checksum = "0bad00257d07be169d870ab665980b06cdb366d792ad690bf2e76876dc503455" dependencies = [ "hermit-abi", "rustix", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -753,9 +744,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" +checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" [[package]] name = "jobserver" @@ -768,9 +759,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.65" +version = "0.3.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54c0c35952f67de54bb584e9fd912b3023117cbafc0a77d8f3dee1fb5f572fe8" +checksum = "cee9c64da59eae3b50095c18d3e74f8b73c0b86d2792824ff01bbce68ba229ca" dependencies = [ "wasm-bindgen", ] @@ -808,9 +799,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.150" +version = "0.2.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" +checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4" [[package]] name = "libloading" @@ -824,9 +815,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.4.11" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "969488b55f8ac402214f3f5fd243ebb7206cf82de60d3172994707a4bcc2b829" +checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456" [[package]] name = "lock_api" @@ -856,9 +847,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.6.4" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" +checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" [[package]] name = "memoffset" @@ -929,18 +920,18 @@ dependencies = [ [[package]] name = "object" -version = "0.32.1" +version = "0.32.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" +checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" dependencies = [ "memchr", ] [[package]] name = "once_cell" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "oorandom" @@ -950,9 +941,9 @@ checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" [[package]] name = "openssl" -version = "0.10.59" +version = "0.10.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a257ad03cd8fb16ad4172fedf8094451e1af1c4b70097636ef2eac9a5f0cc33" +checksum = "8cde4d2d9200ad5909f8dac647e29482e07c3a35de8a13fce7c9c7747ad9f671" dependencies = [ "bitflags 2.4.1", "cfg-if", @@ -982,9 +973,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.95" +version = "0.9.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40a4130519a360279579c2053038317e40eff64d13fd3f004f9e1b72b8a6aaf9" +checksum = "c1665caf8ab2dc9aef43d1c0023bd904633a6a05cb30b0ad59bec2ae986e57a7" dependencies = [ "cc", "libc", @@ -1175,7 +1166,7 @@ dependencies = [ "libc", "redox_syscall", "smallvec", - "windows-targets", + "windows-targets 0.48.5", ] [[package]] @@ -1213,9 +1204,9 @@ checksum = "36bae92c60fa2398ce4678b98b2c4b5a7c61099961ca1fa305aec04a9ad28922" [[package]] name = "percent-encoding" -version = "2.3.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pin-project-lite" @@ -1225,9 +1216,9 @@ checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" [[package]] name = "pkg-config" -version = "0.3.27" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" +checksum = "69d3587f8a9e599cc7ec2c00e331f71c4e69a5f9a4b8a6efd5b07466b9736f9a" [[package]] name = "plotters" @@ -1312,9 +1303,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.69" +version = "1.0.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da" +checksum = "75cb1540fadbd5b8fbccc4dddad2734eba435053f725621c070711a14bb5f4b8" dependencies = [ "unicode-ident", ] @@ -1497,16 +1488,16 @@ checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" [[package]] name = "ring" -version = "0.17.5" +version = "0.17.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb0205304757e5d899b9c2e448b867ffd03ae7f988002e47cd24954391394d0b" +checksum = "688c63d65483050968b2a8937f7995f443e27041a0f7700aa59b0822aedebb74" dependencies = [ "cc", "getrandom", "libc", "spin", "untrusted", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -1540,22 +1531,22 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "rustix" -version = "0.38.25" +version = "0.38.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc99bc2d4f1fed22595588a013687477aedf3cdcfb26558c559edb67b4d9b22e" +checksum = "72e572a5e8ca657d7366229cdde4bd14c4eb5499a9573d4d366fe1b599daa316" dependencies = [ "bitflags 2.4.1", "errno", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] name = "rustls" -version = "0.22.0" +version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bc238b76c51bbc449c55ffbc39d03772a057cc8cf783c49d4af4c2537b74a8b" +checksum = "fe6b63262c9fcac8659abfaa96cac103d28166d3ff3eaf8f412e19f3ae9e5a48" dependencies = [ "log", "ring", @@ -1590,9 +1581,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7673e0aa20ee4937c6aacfc12bb8341cfbf054cdd21df6bec5fd0629fe9339b" +checksum = "9e9d979b3ce68192e42760c7810125eb6cf2ea10efae545a156063e61f314e2a" [[package]] name = "rustls-webpki" @@ -1607,9 +1598,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.15" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" +checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" [[package]] name = "same-file" @@ -1622,11 +1613,11 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" +checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" dependencies = [ - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -1787,9 +1778,9 @@ checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" [[package]] name = "syn" -version = "2.0.39" +version = "2.0.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23e78b90f2fcf45d3e842032ce32e3f2d1545ba6636271dcbf24fa306d87be7a" +checksum = "ee659fb5f3d355364e1f3e5bc10fb82068efbf824a1e9d1c9504244a6469ad53" dependencies = [ "proc-macro2", "quote", @@ -1804,15 +1795,15 @@ checksum = "14c39fd04924ca3a864207c66fc2cd7d22d7c016007f9ce846cbb9326331930a" [[package]] name = "tempfile" -version = "3.8.1" +version = "3.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5" +checksum = "01ce4141aa927a6d1bd34a041795abd0db1cccba5d5f24b009f694bdf3a1f3fa" dependencies = [ "cfg-if", "fastrand", "redox_syscall", "rustix", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -1841,21 +1832,11 @@ dependencies = [ "term", ] -[[package]] -name = "thread_local" -version = "1.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" -dependencies = [ - "cfg-if", - "once_cell", -] - [[package]] name = "time" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4a34ab300f2dee6e562c10a046fc05e358b29f9bf92277f30c3c8d82275f6f5" +checksum = "f657ba42c3f86e7680e53c8cd3af8abbe56b5491790b46e22e19c0d57463583e" dependencies = [ "deranged", "itoa", @@ -1873,9 +1854,9 @@ checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ad70d68dba9e1f8aceda7aa6711965dfec1cac869f311a51bd08b3a2ccbce20" +checksum = "26197e33420244aeb70c3e8c78376ca46571bc4e701e4791c2cd9f57dcb3a43f" dependencies = [ "time-core", ] @@ -1907,9 +1888,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.34.0" +version = "1.35.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0c014766411e834f7af5b8f4cf46257aab4036ca95e9d2c144a10f59ad6f5b9" +checksum = "c89b4efa943be685f629b149f53829423f8f5531ea21249408e8e2f8671ec104" dependencies = [ "backtrace", "bytes", @@ -1936,9 +1917,9 @@ checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "unicode-bidi" -version = "0.3.13" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" +checksum = "6f2528f27a9eb2b21e69c95319b30bd0efd85d09c379741b0f78ea1d86be2416" [[package]] name = "unicode-ident" @@ -1975,9 +1956,9 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" +checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" dependencies = [ "form_urlencoded", "idna", @@ -2029,9 +2010,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.88" +version = "0.2.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7daec296f25a1bae309c0cd5c29c4b260e510e6d813c286b19eaadf409d40fce" +checksum = "0ed0d4f68a3015cc185aff4db9506a015f4b96f95303897bfa23f846db54064e" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -2039,9 +2020,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.88" +version = "0.2.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e397f4664c0e4e428e8313a469aaa58310d302159845980fd23b0f22a847f217" +checksum = "1b56f625e64f3a1084ded111c4d5f477df9f8c92df113852fa5a374dbda78826" dependencies = [ "bumpalo", "log", @@ -2054,9 +2035,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.88" +version = "0.2.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5961017b3b08ad5f3fe39f1e79877f8ee7c23c5e5fd5eb80de95abc41f1f16b2" +checksum = "0162dbf37223cd2afce98f3d0785506dcb8d266223983e4b5b525859e6e182b2" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2064,9 +2045,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.88" +version = "0.2.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5353b8dab669f5e10f5bd76df26a9360c748f054f862ff5f3f8aae0c7fb3907" +checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283" dependencies = [ "proc-macro2", "quote", @@ -2077,15 +2058,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.88" +version = "0.2.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d046c5d029ba91a1ed14da14dca44b68bf2f124cfbaf741c54151fdb3e0750b" +checksum = "7ab9b36309365056cd639da3134bf87fa8f3d86008abf99e612384a6eecd459f" [[package]] name = "web-sys" -version = "0.3.65" +version = "0.3.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5db499c5f66323272151db0e666cd34f78617522fb0c1604d31a27c50c206a85" +checksum = "50c24a44ec86bb68fbecd1b3efed7e85ea5621b39b35ef2766b66cd984f8010f" dependencies = [ "js-sys", "wasm-bindgen", @@ -2161,7 +2142,16 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets", + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.0", ] [[package]] @@ -2170,13 +2160,28 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" +dependencies = [ + "windows_aarch64_gnullvm 0.52.0", + "windows_aarch64_msvc 0.52.0", + "windows_i686_gnu 0.52.0", + "windows_i686_msvc 0.52.0", + "windows_x86_64_gnu 0.52.0", + "windows_x86_64_gnullvm 0.52.0", + "windows_x86_64_msvc 0.52.0", ] [[package]] @@ -2185,42 +2190,84 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" + [[package]] name = "windows_aarch64_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" + [[package]] name = "windows_i686_gnu" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" +[[package]] +name = "windows_i686_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" + [[package]] name = "windows_i686_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" +[[package]] +name = "windows_i686_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" + [[package]] name = "windows_x86_64_gnu" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" + [[package]] name = "windows_x86_64_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" + [[package]] name = "zeroize" version = "1.7.0" From 4c79e7ee7883bb2f2c68938c9c9d358a2d37a8cb Mon Sep 17 00:00:00 2001 From: Tpt Date: Sat, 30 Dec 2023 17:00:46 +0100 Subject: [PATCH 154/217] Upgrades testsuite --- testsuite/N3 | 2 +- testsuite/rdf-canon | 2 +- testsuite/rdf-tests | 2 +- testsuite/tests/parser.rs | 19 +++++++++++++++---- 4 files changed, 18 insertions(+), 7 deletions(-) diff --git a/testsuite/N3 b/testsuite/N3 index 73f4e00c..1425e335 160000 --- a/testsuite/N3 +++ b/testsuite/N3 @@ -1 +1 @@ -Subproject commit 73f4e00c15828f030749567f61cea2f5d7d9c59b +Subproject commit 1425e33527f725262103a6640eadb242d80a9b22 diff --git a/testsuite/rdf-canon b/testsuite/rdf-canon index 0503facf..eaf67e39 160000 --- a/testsuite/rdf-canon +++ b/testsuite/rdf-canon @@ -1 +1 @@ -Subproject commit 0503facfaa0825686afc1f533f487816de54d9b7 +Subproject commit eaf67e398b2358f9987c9550443c109d43ce89b1 diff --git a/testsuite/rdf-tests b/testsuite/rdf-tests index 59b0496b..35fb5dd6 160000 --- a/testsuite/rdf-tests +++ b/testsuite/rdf-tests @@ -1 +1 @@ -Subproject commit 59b0496b050e1c2397f22d94af0609d8324d95f7 +Subproject commit 35fb5dd65b9d504770008a85b4ff577cc581fcee diff --git a/testsuite/tests/parser.rs b/testsuite/tests/parser.rs index 0a272fd4..af39b857 100644 --- a/testsuite/tests/parser.rs +++ b/testsuite/tests/parser.rs @@ -3,6 +3,8 @@ use anyhow::Result; use oxigraph_testsuite::check_testsuite; +// TODO: add support of language directions + #[test] fn rdf11_n_triples_w3c_testsuite() -> Result<()> { check_testsuite( @@ -15,7 +17,10 @@ fn rdf11_n_triples_w3c_testsuite() -> Result<()> { fn rdf12_n_triples_syntax_w3c_testsuite() -> Result<()> { check_testsuite( "https://w3c.github.io/rdf-tests/rdf/rdf12/rdf-n-triples/syntax/manifest.ttl", - &[], + &[ + "https://w3c.github.io/rdf-tests/rdf/rdf12/rdf-n-triples/syntax#ntriples-base-1", + "https://w3c.github.io/rdf-tests/rdf/rdf12/rdf-n-triples/syntax#ntriples-base-2", + ], ) } @@ -24,7 +29,7 @@ fn rdf12_n_triples_syntax_w3c_testsuite() -> Result<()> { fn rdf12_n_triples_c14n_w3c_testsuite() -> Result<()> { check_testsuite( "https://w3c.github.io/rdf-tests/rdf/rdf12/rdf-n-triples/c14n/manifest.ttl", - &[], + &["https://w3c.github.io/rdf-tests/rdf/rdf12/rdf-n-triples/c14n#dirlangtagged_string"], ) } @@ -49,7 +54,10 @@ fn rdf11_turtle_w3c_testsuite() -> Result<()> { fn rdf12_turtle_syntax_w3c_testsuite() -> Result<()> { check_testsuite( "https://w3c.github.io/rdf-tests/rdf/rdf12/rdf-turtle/syntax/manifest.ttl", - &[], + &[ + "https://w3c.github.io/rdf-tests/rdf/rdf12/rdf-turtle/syntax#nt-ttl-base-1", + "https://w3c.github.io/rdf-tests/rdf/rdf12/rdf-turtle/syntax#nt-ttl-base-2", + ], ) } @@ -74,7 +82,10 @@ fn rdf11_trig_w3c_testsuite() -> Result<()> { fn rdf12_trig_syntax_w3c_testsuite() -> Result<()> { check_testsuite( "https://w3c.github.io/rdf-tests/rdf/rdf12/rdf-trig/syntax/manifest.ttl", - &[], + &[ + "https://w3c.github.io/rdf-tests/rdf/rdf12/rdf-trig/syntax#trig-base-1", + "https://w3c.github.io/rdf-tests/rdf/rdf12/rdf-trig/syntax#trig-base-2", + ], ) } From bde73e5d72db43d3ffe0b91bb0f92660a03b909e Mon Sep 17 00:00:00 2001 From: Tpt Date: Sat, 30 Dec 2023 21:45:38 +0100 Subject: [PATCH 155/217] Updates README --- README.md | 8 ++++---- cli/README.md | 5 ++++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 57745826..6147e9fc 100644 --- a/README.md +++ b/README.md @@ -19,8 +19,6 @@ Oxigraph is in heavy development and SPARQL query evaluation has not been optimi The development roadmap is using [GitHub milestones](https://github.com/oxigraph/oxigraph/milestones?direction=desc&sort=completeness&state=open). Oxigraph internal design [is described on the wiki](https://github.com/oxigraph/oxigraph/wiki/Architecture). -The future 0.4 release is currently in development in the [next branch](https://github.com/oxigraph/oxigraph/tree/next). - It is split into multiple parts: - [The database written as a Rust library](https://crates.io/crates/oxigraph). Its source code is in the `lib` directory. @@ -30,6 +28,7 @@ It is split into multiple parts: - [JavaScript bindings for Oxigraph](https://www.npmjs.com/package/oxigraph). WebAssembly is used to package Oxigraph into a NodeJS compatible NPM package. Its source code is in the `js` directory. [![npm](https://img.shields.io/npm/v/oxigraph)](https://www.npmjs.com/package/oxigraph) - [Oxigraph binary](https://crates.io/crates/oxigraph-cli) that provides a standalone command line tool allowing to manipulate RDF data and spawn a a web server implementing the [SPARQL 1.1 Protocol](https://www.w3.org/TR/sparql11-protocol/) and the [SPARQL 1.1 Graph Store Protocol](https://www.w3.org/TR/sparql11-http-rdf-update/). Its source code is in the `cli` directory. + Note that it was previously named [Oxigraph server](https://crates.io/crates/oxigraph-server). [![Latest Version](https://img.shields.io/crates/v/oxigraph-cli.svg)](https://crates.io/crates/oxigraph-cli) Oxigraph implements the following specifications: @@ -72,8 +71,9 @@ Unless you explicitly state otherwise, any contribution intentionally submitted ## Sponsors * [RelationLabs](https://relationlabs.ai/) that is building [Relation-Graph](https://github.com/relationlabs/Relation-Graph), a SPARQL database module for the [Substrate blockchain platform](https://substrate.io/) based on Oxigraph. -* [Field 33](https://field33.com) that is building [an ontology management platform](https://plow.pm/). -* [Magnus Bakken](https://github.com/magbak) who is building [chrontext](https://github.com/magbak/chrontext), providing a SPARQL query endpoint on top of joint RDF and time series databases. +* [Field 33](https://field33.com) that was building [an ontology management platform](https://plow.pm/). +* [Magnus Bakken](https://github.com/magbak) who is building [Data Treehouse](https://www.data-treehouse.com/), a time-series + RDF datalake platform, and [chrontext](https://github.com/magbak/chrontext), a SPARQL query endpoint on top of joint RDF and time series databases. +* [DeciSym.AI](https://www.decisym.ai/) a cyber security consulting company providing RDF-based software. * [ACE IoT Solutions](https://aceiotsolutions.com/), a building IOT platform. * [Albin Larsson](https://byabbe.se/) who is building [GovDirectory](https://www.govdirectory.org/), a directory of public agencies based on Wikidata. diff --git a/cli/README.md b/cli/README.md index aaa1f3e4..8d4b14d2 100644 --- a/cli/README.md +++ b/cli/README.md @@ -13,7 +13,7 @@ It also allows to spawn a HTTP server on top of the database. Oxigraph is in heavy development and SPARQL query evaluation has not been optimized yet. -Oxigraph provides different installation methods for Oxigraph server: +Oxigraph provides different installation methods for Oxigraph CLI: * [`cargo install`](#installation) (multiplatform) * [A Docker image](#using-a-docker-image) * [A Homebrew formula](#homebrew) @@ -30,6 +30,9 @@ Oxigraph implements the following specifications: A preliminary benchmark [is provided](../bench/README.md). +Note that Oxigraph CLI was previously named Oxigraph Server before version 0.4. Older versions are available under [this name](https://crates.io/crates/oxigraph_server). + + ## Installation You need to have [a recent stable version of Rust and Cargo installed](https://www.rust-lang.org/tools/install). From 1761672b413e839a38c5d0e079ef88581b1536d6 Mon Sep 17 00:00:00 2001 From: Tpt Date: Sun, 31 Dec 2023 13:45:05 +0100 Subject: [PATCH 156/217] Upgrades Pyo3 --- Cargo.lock | 28 ++++++++++++++-------------- python/Cargo.toml | 2 +- python/src/lib.rs | 1 - 3 files changed, 15 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 29e2795f..870fd48b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -82,9 +82,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.77" +version = "1.0.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9d19de80eff169429ac1e9f48fffb163916b448a44e8e046186232046d9e1f9" +checksum = "ca87830a3e3fb156dc96cfbd31cb620265dd053be734723f22b760d6cc3c3051" [[package]] name = "arbitrary" @@ -1303,18 +1303,18 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.71" +version = "1.0.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75cb1540fadbd5b8fbccc4dddad2734eba435053f725621c070711a14bb5f4b8" +checksum = "a293318316cf6478ec1ad2a21c49390a8d5b5eae9fab736467d93fbc0edc29c5" dependencies = [ "unicode-ident", ] [[package]] name = "pyo3" -version = "0.20.0" +version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04e8453b658fe480c3e70c8ed4e3d3ec33eb74988bd186561b0cc66b85c3bc4b" +checksum = "e82ad98ce1991c9c70c3464ba4187337b9c45fcbbb060d46dca15f0c075e14e2" dependencies = [ "cfg-if", "indoc", @@ -1329,9 +1329,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.20.0" +version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a96fe70b176a89cff78f2fa7b3c930081e163d5379b4dcdf993e3ae29ca662e5" +checksum = "5503d0b3aee2c7a8dbb389cd87cd9649f675d4c7f60ca33699a3e3859d81a891" dependencies = [ "once_cell", "target-lexicon", @@ -1339,9 +1339,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.20.0" +version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "214929900fd25e6604661ed9cf349727c8920d47deff196c4e28165a6ef2a96b" +checksum = "18a79e8d80486a00d11c0dcb27cd2aa17c022cc95c677b461f01797226ba8f41" dependencies = [ "libc", "pyo3-build-config", @@ -1349,9 +1349,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.20.0" +version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dac53072f717aa1bfa4db832b39de8c875b7c7af4f4a6fe93cdbf9264cf8383b" +checksum = "1f4b0dc7eaa578604fab11c8c7ff8934c71249c61d4def8e272c76ed879f03d4" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -1361,9 +1361,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.20.0" +version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7774b5a8282bd4f25f803b1f0d945120be959a36c72e08e7cd031c792fdfd424" +checksum = "816a4f709e29ddab2e3cdfe94600d554c5556cad0ddfeea95c47b580c3247fa4" dependencies = [ "heck", "proc-macro2", diff --git a/python/Cargo.toml b/python/Cargo.toml index 531c01df..059c5b23 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -26,7 +26,7 @@ rustls = ["oxigraph/http-client-rustls-native"] [dependencies] oxigraph = { version = "0.4.0-alpha.1-dev", path="../lib" } -pyo3 = { version = "0.20", features = ["extension-module"] } +pyo3 = { version = "0.20.1", features = ["extension-module"] } [lints] workspace = true diff --git a/python/src/lib.rs b/python/src/lib.rs index 051e2718..cabb8ee2 100644 --- a/python/src/lib.rs +++ b/python/src/lib.rs @@ -1,5 +1,4 @@ #![allow( - clippy::used_underscore_binding, clippy::unused_self, clippy::trivially_copy_pass_by_ref, unused_qualifications From bdf5d593ee882abf4c51d2ae8a8df1f3dc9ab1c9 Mon Sep 17 00:00:00 2001 From: Tpt Date: Sun, 31 Dec 2023 16:00:20 +0100 Subject: [PATCH 157/217] CI: Share code to setup Rust --- .github/actions/setup-rust/action.yml | 27 ++++++++ .github/workflows/artifacts.yml | 31 +++++---- .github/workflows/tests.yml | 97 ++++++++++++++------------- 3 files changed, 93 insertions(+), 62 deletions(-) create mode 100644 .github/actions/setup-rust/action.yml diff --git a/.github/actions/setup-rust/action.yml b/.github/actions/setup-rust/action.yml new file mode 100644 index 00000000..09349135 --- /dev/null +++ b/.github/actions/setup-rust/action.yml @@ -0,0 +1,27 @@ +name: 'Setup Rust' +description: 'Setup Rust using Rustup' +inputs: + version: + description: 'Rust version to use. By default latest stable version' + required: false + default: 'stable' + component: + description: 'Rust extra component to install like clippy' + required: false + target: + description: 'Rust extra target to install like wasm32-unknown-unknown' + required: false +runs: + using: "composite" + steps: + - run: rustup update + shell: bash + - run: rustup default ${{ inputs.version }} + shell: bash + - run: rustup component add ${{ inputs.component }} + shell: bash + if: ${{ inputs.component }} + - run: rustup target add ${{ inputs.target }} + shell: bash + if: ${{ inputs.target }} + - uses: Swatinem/rust-cache@v2 diff --git a/.github/workflows/artifacts.yml b/.github/workflows/artifacts.yml index 0a5c7adc..d984b0c5 100644 --- a/.github/workflows/artifacts.yml +++ b/.github/workflows/artifacts.yml @@ -20,12 +20,13 @@ jobs: - uses: actions/checkout@v3 with: submodules: true - - run: rustup update && rustup target add aarch64-unknown-linux-gnu + - uses: ./.github/actions/setup-rust + with: + target: aarch64-unknown-linux-gnu - run: | sudo apt update && sudo apt-get install -y g++-aarch64-linux-gnu mkdir .cargo echo -e "[target.aarch64-unknown-linux-gnu]\nlinker = \"aarch64-linux-gnu-gcc\"" >> .cargo/config.toml - - uses: Swatinem/rust-cache@v2 - run: cargo build --release --no-default-features --features rustls-native working-directory: ./cli - run: cargo build --release --target aarch64-unknown-linux-gnu --no-default-features --features rustls-native @@ -61,8 +62,9 @@ jobs: - uses: actions/checkout@v3 with: submodules: true - - run: rustup update && rustup target add aarch64-apple-darwin - - uses: Swatinem/rust-cache@v2 + - uses: ./.github/actions/setup-rust + with: + target: aarch64-apple-darwin - run: cargo build --release working-directory: ./cli - run: cargo build --release --target aarch64-apple-darwin @@ -92,8 +94,7 @@ jobs: - uses: actions/checkout@v3 with: submodules: true - - run: rustup update - - uses: Swatinem/rust-cache@v2 + - uses: ./.github/actions/setup-rust - run: Remove-Item -LiteralPath "C:\msys64\" -Force -Recurse - run: cargo build --release working-directory: ./cli @@ -114,8 +115,7 @@ jobs: - uses: actions/checkout@v3 with: submodules: true - - run: rustup update - - uses: Swatinem/rust-cache@v2 + - uses: ./.github/actions/setup-rust - uses: actions/setup-python@v4 with: python-version: "3.12" @@ -152,7 +152,7 @@ jobs: with: platforms: linux/${{ matrix.architecture }} if: github.event_name == 'release' && matrix.architecture != 'x86_64' - - uses: Swatinem/rust-cache@v2 + - uses: ./.github/actions/setup-rust - run: sed 's/%arch%/${{ matrix.architecture }}/g' .github/workflows/manylinux_build.sh | sed 's/%for_each_version%/${{ github.event_name == 'release' || '' }}/g' > .github/workflows/manylinux_build_script.sh - run: docker run -v "$(pwd)":/workdir --platform linux/${{ matrix.architecture }} quay.io/pypa/manylinux2014_${{ matrix.architecture }} /bin/bash /workdir/.github/workflows/manylinux_build_script.sh if: github.event_name == 'release' || matrix.architecture == 'x86_64' @@ -181,7 +181,7 @@ jobs: with: platforms: linux/${{ matrix.architecture }} if: github.event_name == 'release' && matrix.architecture != 'x86_64' - - uses: Swatinem/rust-cache@v2 + - uses: ./.github/actions/setup-rust - run: sed 's/%arch%/${{ matrix.architecture }}/g' .github/workflows/musllinux_build.sh | sed 's/%for_each_version%/${{ github.event_name == 'release' || '' }}/g' > .github/workflows/musllinux_build_script.sh - run: docker run -v "$(pwd)":/workdir --platform linux/${{ matrix.architecture }} quay.io/pypa/musllinux_1_2_${{ matrix.architecture }} /bin/bash /workdir/.github/workflows/musllinux_build_script.sh if: github.event_name == 'release' || matrix.architecture == 'x86_64' @@ -206,8 +206,9 @@ jobs: - uses: actions/checkout@v3 with: submodules: true - - run: rustup update && rustup target add aarch64-apple-darwin - - uses: Swatinem/rust-cache@v2 + - uses: ./.github/actions/setup-rust + with: + target: aarch64-apple-darwin - uses: actions/setup-python@v4 with: python-version: "3.12" @@ -244,8 +245,7 @@ jobs: - uses: actions/checkout@v3 with: submodules: true - - run: rustup update - - uses: Swatinem/rust-cache@v2 + - uses: ./.github/actions/setup-rust - uses: actions/setup-python@v4 with: python-version: "3.12" @@ -276,8 +276,7 @@ jobs: - uses: actions/checkout@v3 with: submodules: true - - run: rustup update - - uses: Swatinem/rust-cache@v2 + - uses: ./.github/actions/setup-rust - run: cargo install wasm-pack || true - uses: actions/setup-node@v3 with: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ee5f4f9b..d721ca4f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -17,7 +17,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - run: rustup update && rustup component add rustfmt + - uses: ./.github/actions/setup-rust + with: + component: rustfmt - run: cargo fmt -- --check clippy: @@ -26,8 +28,10 @@ jobs: - uses: actions/checkout@v3 with: submodules: true - - run: rustup update && rustup default 1.74.1 && rustup component add clippy - - uses: Swatinem/rust-cache@v2 + - uses: ./.github/actions/setup-rust + with: + version: 1.74.1 + component: clippy - run: cargo clippy --all-targets -- -D warnings -D clippy::all working-directory: ./lib/oxsdatatypes - run: cargo clippy --all-targets -- -D warnings -D clippy::all @@ -59,8 +63,11 @@ jobs: - uses: actions/checkout@v3 with: submodules: true - - run: rustup update && rustup default 1.74.1 && rustup target add wasm32-unknown-unknown && rustup component add clippy - - uses: Swatinem/rust-cache@v2 + - uses: ./.github/actions/setup-rust + with: + version: 1.74.1 + target: wasm32-unknown-unknown + component: clippy - run: cargo clippy --lib --tests --target wasm32-unknown-unknown -- -D warnings -D clippy::all working-directory: ./js @@ -70,8 +77,11 @@ jobs: - uses: actions/checkout@v3 with: submodules: true - - run: rustup update && rustup default 1.74.1 && rustup target add wasm32-wasi && rustup component add clippy - - uses: Swatinem/rust-cache@v2 + - uses: ./.github/actions/setup-rust + with: + version: 1.74.1 + target: wasm32-wasi + component: clippy - run: cargo clippy --lib --tests --target wasm32-wasi -- -D warnings -D clippy::all working-directory: ./lib @@ -81,8 +91,11 @@ jobs: - uses: actions/checkout@v3 with: submodules: true - - run: rustup update && rustup default 1.74.1 && rustup target add wasm32-unknown-unknown && rustup component add clippy - - uses: Swatinem/rust-cache@v2 + - uses: ./.github/actions/setup-rust + with: + version: 1.74.1 + target: wasm32-unknown-unknown + component: clippy - run: cargo clippy --lib --tests --target wasm32-unknown-unknown --features getrandom/custom --features oxsdatatypes/custom-now -- -D warnings -D clippy::all working-directory: ./lib @@ -90,8 +103,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - run: rustup update - - uses: Swatinem/rust-cache@v2 + - uses: ./.github/actions/setup-rust - run: cargo install cargo-deny || true - run: cargo deny check @@ -102,8 +114,7 @@ jobs: - uses: actions/checkout@v3 with: submodules: true - - run: rustup update - - uses: Swatinem/rust-cache@v2 + - uses: ./.github/actions/setup-rust - run: cargo install cargo-semver-checks || true - run: cargo semver-checks check-release --exclude oxrocksdb-sys --exclude oxigraph-js --exclude pyoxigraph --exclude oxigraph-testsuite --exclude oxigraph-cli @@ -113,8 +124,7 @@ jobs: - uses: actions/checkout@v3 with: submodules: true - - run: rustup update - - uses: Swatinem/rust-cache@v2 + - uses: ./.github/actions/setup-rust - run: cargo test test_linux_msv: @@ -123,8 +133,10 @@ jobs: - uses: actions/checkout@v3 with: submodules: true - - run: rustup update && rustup toolchain install nightly && rustup default 1.70.0 - - uses: Swatinem/rust-cache@v2 + - uses: ./.github/actions/setup-rust + with: + version: 1.70.0 + - run: rustup toolchain install nightly - run: rm Cargo.lock && cargo +nightly update -Z direct-minimal-versions - run: cargo test @@ -134,8 +146,7 @@ jobs: - uses: actions/checkout@v3 with: submodules: true - - run: rustup update - - uses: Swatinem/rust-cache@v2 + - uses: ./.github/actions/setup-rust - run: rm Cargo.lock && cargo update - run: cargo test @@ -145,9 +156,10 @@ jobs: - uses: actions/checkout@v3 with: submodules: true - - run: rustup update && rustup default nightly + - uses: ./.github/actions/setup-rust + with: + version: nightly - run: sudo apt-get install -y llvm - - uses: Swatinem/rust-cache@v2 - run: cargo test --tests --target x86_64-unknown-linux-gnu --workspace --exclude pyoxigraph --exclude oxigraph-testsuite --exclude oxigraph-cli env: RUSTFLAGS: -Z sanitizer=address @@ -158,8 +170,7 @@ jobs: - uses: actions/checkout@v3 with: submodules: true - - run: rustup update - - uses: Swatinem/rust-cache@v2 + - uses: ./.github/actions/setup-rust - uses: actions/cache@v3 with: path: rocksdb @@ -173,12 +184,9 @@ jobs: - uses: actions/checkout@v3 with: submodules: true - - run: rustup update - - uses: Swatinem/rust-cache@v2 + - uses: ./.github/actions/setup-rust - run: Remove-Item -LiteralPath "C:\msys64\" -Force -Recurse - run: cargo test - env: - RUST_BACKTRACE: 1 test_wasi: runs-on: ubuntu-latest @@ -186,13 +194,12 @@ jobs: - uses: actions/checkout@v3 with: submodules: true - - run: rustup update && rustup target add wasm32-wasi - - uses: Swatinem/rust-cache@v2 + - uses: ./.github/actions/setup-rust + with: + target: wasm32-wasi - uses: taiki-e/install-action@wasmtime - run: cargo install cargo-wasi || true - run: cargo wasi test --workspace --exclude oxigraph-js --exclude oxigraph-cli --exclude oxigraph-testsuite --exclude oxrocksdb-sys --exclude pyoxigraph - env: - RUST_BACKTRACE: 1 rustdoc: runs-on: ubuntu-latest @@ -200,8 +207,9 @@ jobs: - uses: actions/checkout@v3 with: submodules: true - - run: rustup update && rustup default 1.74.1 - - uses: Swatinem/rust-cache@v2 + - uses: ./.github/actions/setup-rust + with: + version: 1.74.1 - run: cargo doc working-directory: ./lib env: @@ -211,8 +219,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - run: rustup update - - uses: Swatinem/rust-cache@v2 + - uses: ./.github/actions/setup-rust - run: cargo install wasm-pack || true - uses: actions/setup-node@v3 with: @@ -230,8 +237,7 @@ jobs: - uses: actions/checkout@v3 with: submodules: true - - run: rustup update - - uses: Swatinem/rust-cache@v2 + - uses: ./.github/actions/setup-rust - uses: actions/setup-python@v4 with: python-version: "3.12" @@ -264,8 +270,10 @@ jobs: - uses: actions/checkout@v3 with: submodules: true - - run: rustup update && rustup toolchain install nightly && rustup default 1.70.0 - - uses: Swatinem/rust-cache@v2 + - uses: ./.github/actions/setup-rust + with: + version: 1.70.0 + - run: rustup toolchain install nightly - uses: actions/setup-python@v4 with: python-version: "3.8" @@ -285,8 +293,7 @@ jobs: - uses: actions/checkout@v3 with: submodules: true - - run: rustup update - - uses: Swatinem/rust-cache@v2 + - uses: ./.github/actions/setup-rust - uses: actions/setup-python@v4 with: python-version: "pypy3.10" @@ -305,8 +312,7 @@ jobs: - uses: actions/checkout@v3 with: submodules: true - - run: rustup update - - uses: Swatinem/rust-cache@v2 + - uses: ./.github/actions/setup-rust - uses: actions/setup-python@v4 with: python-version: "3.12" @@ -323,7 +329,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - uses: Swatinem/rust-cache@v2 + - uses: ./.github/actions/setup-rust - run: cargo install typos-cli || true - run: typos @@ -421,6 +427,5 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - run: rustup update - - uses: Swatinem/rust-cache@v2 + - uses: ./.github/actions/setup-rust - run: python lints/test_debian_compatibility.py From 790501e1b389e6bed93d4c946ead16a63da8bdc0 Mon Sep 17 00:00:00 2001 From: Tpt Date: Sun, 31 Dec 2023 17:38:04 +0100 Subject: [PATCH 158/217] Setup codspeed and improves benchmarks --- .github/workflows/tests.yml | 15 +++ Cargo.lock | 55 ++++++---- lib/Cargo.toml | 2 +- lib/benches/store.rs | 168 +++++++++++++---------------- lints/test_debian_compatibility.py | 2 +- testsuite/Cargo.toml | 4 +- testsuite/benches/parser.rs | 97 +++-------------- 7 files changed, 143 insertions(+), 200 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d721ca4f..227a70cb 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -429,3 +429,18 @@ jobs: - uses: actions/checkout@v3 - uses: ./.github/actions/setup-rust - run: python lints/test_debian_compatibility.py + + codspeed: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - uses: ./.github/actions/setup-rust + - run: cargo install cargo-codspeed || true + - run: cargo codspeed build -p oxigraph --features http-client-native-tls + - run: cargo codspeed build -p oxigraph-testsuite + - uses: CodSpeedHQ/action@v2 + with: + run: cargo codspeed run + token: ${{ secrets.CODSPEED_TOKEN }} diff --git a/Cargo.lock b/Cargo.lock index 870fd48b..809ca10e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -328,12 +328,44 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" +[[package]] +name = "codspeed" +version = "2.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eb4ab4dcb6554eb4f590fb16f99d3b102ab76f5f56554c9a5340518b32c499b" +dependencies = [ + "colored", + "libc", + "serde_json", +] + +[[package]] +name = "codspeed-criterion-compat" +version = "2.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc07a3d3f7e0c8961d0ffdee149d39b231bafdcdc3d978dc5ad790c615f55f3f" +dependencies = [ + "codspeed", + "colored", + "criterion", +] + [[package]] name = "colorchoice" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" +[[package]] +name = "colored" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbf2150cce219b664a8a70df7a1f933836724b503f8a413af9365b4dcc4d90b8" +dependencies = [ + "lazy_static", + "windows-sys 0.48.0", +] + [[package]] name = "console_error_panic_hook" version = "0.1.7" @@ -1003,7 +1035,7 @@ dependencies = [ name = "oxigraph" version = "0.4.0-alpha.1-dev" dependencies = [ - "criterion", + "codspeed-criterion-compat", "digest", "getrandom", "hex", @@ -1064,11 +1096,9 @@ version = "0.4.0-alpha.1-dev" dependencies = [ "anyhow", "clap", - "criterion", + "codspeed-criterion-compat", "oxigraph", "oxttl", - "rio_api", - "rio_turtle", "spargebra", "sparopt", "text-diff", @@ -1500,23 +1530,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "rio_api" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1924fa1f0e6d851f9b73b3c569e607c368a0d92995d99d563ad7bf1414696603" - -[[package]] -name = "rio_turtle" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cec59971eafd99b9c7e3544bfcabafea81a7072ac51c9f46985ca0bd7ba6016" -dependencies = [ - "oxilangtag", - "oxiri", - "rio_api", -] - [[package]] name = "rustc-demangle" version = "0.1.23" diff --git a/lib/Cargo.toml b/lib/Cargo.toml index dbb7f599..3df87dfe 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -54,7 +54,7 @@ getrandom = "0.2.8" js-sys = { version = "0.3.60", optional = true } [target.'cfg(not(target_family = "wasm"))'.dev-dependencies] -criterion = "0.5" +codspeed-criterion-compat = "2.3.3" oxhttp = "0.2.0-alpha.3" zstd = ">=0.12, <0.14" diff --git a/lib/benches/store.rs b/lib/benches/store.rs index ae56266b..22e03e26 100644 --- a/lib/benches/store.rs +++ b/lib/benches/store.rs @@ -1,65 +1,56 @@ #![allow(clippy::panic)] -use criterion::{criterion_group, criterion_main, Criterion, Throughput}; +use codspeed_criterion_compat::{criterion_group, criterion_main, Criterion, Throughput}; use oxhttp::model::{Method, Request, Status}; -use oxigraph::io::RdfFormat; +use oxigraph::io::{RdfFormat, RdfParser}; use oxigraph::sparql::{Query, QueryResults, Update}; use oxigraph::store::Store; use rand::random; use std::env::temp_dir; use std::fs::{remove_dir_all, File}; -use std::io::{BufRead, BufReader, Read}; +use std::io::Read; use std::path::{Path, PathBuf}; +use std::str; + +fn parse_nt(c: &mut Criterion) { + let data = read_data("explore-1000.nt.zst"); + let mut group = c.benchmark_group("parse"); + group.throughput(Throughput::Bytes(data.len() as u64)); + group.sample_size(50); + group.bench_function("parse BSBM explore 1000", |b| { + b.iter(|| { + for r in RdfParser::from_format(RdfFormat::NTriples).parse_read(data.as_slice()) { + r.unwrap(); + } + }) + }); +} fn store_load(c: &mut Criterion) { - { - let mut data = Vec::new(); - read_data("explore-1000.nt.zst") - .read_to_end(&mut data) - .unwrap(); - - let mut group = c.benchmark_group("store load"); - group.throughput(Throughput::Bytes(data.len() as u64)); - group.sample_size(10); - group.bench_function("load BSBM explore 1000 in memory", |b| { - b.iter(|| { - let store = Store::new().unwrap(); - do_load(&store, &data); - }) - }); - group.bench_function("load BSBM explore 1000 in on disk", |b| { - b.iter(|| { - let path = TempDir::default(); - let store = Store::open(&path).unwrap(); - do_load(&store, &data); - }) - }); - group.bench_function("load BSBM explore 1000 in on disk with bulk load", |b| { - b.iter(|| { - let path = TempDir::default(); - let store = Store::open(&path).unwrap(); - do_bulk_load(&store, &data); - }) - }); - } - - { - let mut data = Vec::new(); - read_data("explore-10000.nt.zst") - .read_to_end(&mut data) - .unwrap(); - - let mut group = c.benchmark_group("store load large"); - group.throughput(Throughput::Bytes(data.len() as u64)); - group.sample_size(10); - group.bench_function("load BSBM explore 10000 in on disk with bulk load", |b| { - b.iter(|| { - let path = TempDir::default(); - let store = Store::open(&path).unwrap(); - do_bulk_load(&store, &data); - }) - }); - } + let data = read_data("explore-1000.nt.zst"); + let mut group = c.benchmark_group("store load"); + group.throughput(Throughput::Bytes(data.len() as u64)); + group.sample_size(10); + group.bench_function("load BSBM explore 1000 in memory", |b| { + b.iter(|| { + let store = Store::new().unwrap(); + do_load(&store, &data); + }) + }); + group.bench_function("load BSBM explore 1000 in on disk", |b| { + b.iter(|| { + let path = TempDir::default(); + let store = Store::open(&path).unwrap(); + do_load(&store, &data); + }) + }); + group.bench_function("load BSBM explore 1000 in on disk with bulk load", |b| { + b.iter(|| { + let path = TempDir::default(); + let store = Store::open(&path).unwrap(); + do_bulk_load(&store, &data); + }) + }); } fn do_load(store: &Store, data: &[u8]) { @@ -76,23 +67,12 @@ fn do_bulk_load(store: &Store, data: &[u8]) { } fn store_query_and_update(c: &mut Criterion) { - let mut data = Vec::new(); - read_data("explore-1000.nt.zst") - .read_to_end(&mut data) - .unwrap(); - - let operations = BufReader::new(read_data("mix-exploreAndUpdate-1000.tsv.zst")) - .lines() - .map(|l| { - let l = l.unwrap(); - let mut parts = l.trim().split('\t'); - let kind = parts.next().unwrap(); - let operation = parts.next().unwrap(); - match kind { - "query" => Operation::Query(Query::parse(operation, None).unwrap()), - "update" => Operation::Update(Update::parse(operation, None).unwrap()), - _ => panic!("Unexpected operation kind {kind}"), - } + let data = read_data("explore-1000.nt.zst"); + let operations = bsbm_sparql_operation() + .into_iter() + .map(|op| match op { + RawOperation::Query(q) => Operation::Query(Query::parse(&q, None).unwrap()), + RawOperation::Update(q) => Operation::Update(Update::parse(&q, None).unwrap()), }) .collect::>(); let query_operations = operations @@ -151,26 +131,7 @@ fn run_operation(store: &Store, operations: &[Operation]) { } fn sparql_parsing(c: &mut Criterion) { - let mut data = Vec::new(); - read_data("explore-1000.nt.zst") - .read_to_end(&mut data) - .unwrap(); - - let operations = BufReader::new(read_data("mix-exploreAndUpdate-1000.tsv.zst")) - .lines() - .map(|l| { - let l = l.unwrap(); - let mut parts = l.trim().split('\t'); - let kind = parts.next().unwrap(); - let operation = parts.next().unwrap(); - match kind { - "query" => RawOperation::Query(operation.to_owned()), - "update" => RawOperation::Update(operation.to_owned()), - _ => panic!("Unexpected operation kind {kind}"), - } - }) - .collect::>(); - + let operations = bsbm_sparql_operation(); let mut group = c.benchmark_group("sparql parsing"); group.sample_size(10); group.throughput(Throughput::Bytes( @@ -198,11 +159,12 @@ fn sparql_parsing(c: &mut Criterion) { }); } +criterion_group!(parse, parse_nt); criterion_group!(store, sparql_parsing, store_query_and_update, store_load); -criterion_main!(store); +criterion_main!(parse, store); -fn read_data(file: &str) -> impl Read { +fn read_data(file: &str) -> Vec { if !Path::new(file).exists() { let client = oxhttp::Client::new().with_redirection_limit(5); let url = format!("https://github.com/Tpt/bsbm-tools/releases/download/v0.2/{file}"); @@ -216,7 +178,31 @@ fn read_data(file: &str) -> impl Read { ); std::io::copy(&mut response.into_body(), &mut File::create(file).unwrap()).unwrap(); } - zstd::Decoder::new(File::open(file).unwrap()).unwrap() + let mut buf = Vec::new(); + zstd::Decoder::new(File::open(file).unwrap()) + .unwrap() + .read_to_end(&mut buf) + .unwrap(); + buf +} + +fn bsbm_sparql_operation() -> Vec { + String::from_utf8(read_data("mix-exploreAndUpdate-1000.tsv.zst")) + .unwrap() + .lines() + .rev() + .take(300) // We take only 10 groups + .map(|l| { + let mut parts = l.trim().split('\t'); + let kind = parts.next().unwrap(); + let operation = parts.next().unwrap(); + match kind { + "query" => RawOperation::Query(operation.into()), + "update" => RawOperation::Update(operation.into()), + _ => panic!("Unexpected operation kind {kind}"), + } + }) + .collect() } #[derive(Clone)] diff --git a/lints/test_debian_compatibility.py b/lints/test_debian_compatibility.py index d409bb9d..27530f96 100644 --- a/lints/test_debian_compatibility.py +++ b/lints/test_debian_compatibility.py @@ -5,7 +5,7 @@ from urllib.request import urlopen TARGET_DEBIAN_VERSIONS = ["sid"] IGNORE_PACKAGES = {"oxigraph-js", "oxigraph-testsuite", "pyoxigraph", "sparql-smith"} -ALLOWED_MISSING_PACKAGES = {"escargot", "json-event-parser", "oxhttp", "quick-xml"} +ALLOWED_MISSING_PACKAGES = {"codspeed-criterion-compat", "escargot", "json-event-parser", "oxhttp", "quick-xml"} base_path = Path(__file__).parent.parent diff --git a/testsuite/Cargo.toml b/testsuite/Cargo.toml index 6e24e309..be1b1ac0 100644 --- a/testsuite/Cargo.toml +++ b/testsuite/Cargo.toml @@ -21,9 +21,7 @@ text-diff = "0.4" time = { version = "0.3", features = ["formatting"] } [dev-dependencies] -criterion = "0.5" -rio_api = "0.8" -rio_turtle = "0.8" +codspeed-criterion-compat = "2.3.3" [lints] workspace = true diff --git a/testsuite/benches/parser.rs b/testsuite/benches/parser.rs index c14fb921..364b4009 100644 --- a/testsuite/benches/parser.rs +++ b/testsuite/benches/parser.rs @@ -1,11 +1,11 @@ #![allow(clippy::print_stderr)] use anyhow::Result; -use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; +use codspeed_criterion_compat::{ + criterion_group, criterion_main, BenchmarkId, Criterion, Throughput, +}; use oxigraph_testsuite::files::read_file; use oxigraph_testsuite::manifest::TestManifest; -use rio_api::parser::*; -use rio_turtle::*; use std::io::Read; fn test_data_from_testsuite(manifest_uri: String, include_tests_types: &[&str]) -> Result> { @@ -53,7 +53,7 @@ fn parse_bench( group.finish(); } -fn parse_oxttl_ntriples(c: &mut Criterion, name: &str, data: &[u8]) { +fn parse_ntriples(c: &mut Criterion, name: &str, data: &[u8]) { parse_bench(c, "oxttl ntriples", name, data, |data| { let mut parser = oxttl::NTriplesParser::new().parse(); parser.extend_from_slice(data); @@ -64,7 +64,7 @@ fn parse_oxttl_ntriples(c: &mut Criterion, name: &str, data: &[u8]) { }); } -fn parse_oxttl_turtle(c: &mut Criterion, name: &str, data: &[u8]) { +fn parse_turtle(c: &mut Criterion, name: &str, data: &[u8]) { parse_bench(c, "oxttl turtle", name, data, |data| { let mut parser = oxttl::TurtleParser::new().parse(); parser.extend_from_slice(data); @@ -75,74 +75,8 @@ fn parse_oxttl_turtle(c: &mut Criterion, name: &str, data: &[u8]) { }); } -fn parse_rio_ntriples(c: &mut Criterion, name: &str, data: &[u8]) { - parse_bench(c, "rio ntriples", name, data, |data| { - let mut count: u64 = 0; - NTriplesParser::new(data) - .parse_all::(&mut |_| { - count += 1; - Ok(()) - }) - .unwrap(); - }); -} - -fn parse_rio_turtle(c: &mut Criterion, name: &str, data: &[u8]) { - parse_bench(c, "rio turtle", name, data, |data| { - let mut count: u64 = 0; - TurtleParser::new(data, None) - .parse_all::(&mut |_| { - count += 1; - Ok(()) - }) - .unwrap(); - }); -} - -fn bench_parse_oxttl_ntriples_with_ntriples(c: &mut Criterion) { - parse_oxttl_ntriples( - c, - "ntriples", - &match ntriples_test_data() { - Ok(d) => d, - Err(e) => { - eprintln!("{e}"); - return; - } - }, - ) -} - -fn bench_parse_oxttl_ntriples_with_turtle(c: &mut Criterion) { - parse_oxttl_turtle( - c, - "ntriples", - &match ntriples_test_data() { - Ok(d) => d, - Err(e) => { - eprintln!("{e}"); - return; - } - }, - ) -} - -fn bench_parse_oxttl_turtle_with_turtle(c: &mut Criterion) { - parse_oxttl_turtle( - c, - "turtle", - &match turtle_test_data() { - Ok(d) => d, - Err(e) => { - eprintln!("{e}"); - return; - } - }, - ) -} - -fn bench_parse_rio_ntriples_with_ntriples(c: &mut Criterion) { - parse_rio_ntriples( +fn bench_parse_ntriples_with_ntriples(c: &mut Criterion) { + parse_ntriples( c, "ntriples", &match ntriples_test_data() { @@ -155,8 +89,8 @@ fn bench_parse_rio_ntriples_with_ntriples(c: &mut Criterion) { ) } -fn bench_parse_rio_ntriples_with_turtle(c: &mut Criterion) { - parse_rio_turtle( +fn bench_parse_ntriples_with_turtle(c: &mut Criterion) { + parse_turtle( c, "ntriples", &match ntriples_test_data() { @@ -169,8 +103,8 @@ fn bench_parse_rio_ntriples_with_turtle(c: &mut Criterion) { ) } -fn bench_parse_rio_turtle_with_turtle(c: &mut Criterion) { - parse_rio_turtle( +fn bench_parse_turtle_with_turtle(c: &mut Criterion) { + parse_turtle( c, "turtle", &match turtle_test_data() { @@ -185,12 +119,9 @@ fn bench_parse_rio_turtle_with_turtle(c: &mut Criterion) { criterion_group!( w3c_testsuite, - bench_parse_rio_ntriples_with_ntriples, - bench_parse_rio_ntriples_with_turtle, - bench_parse_rio_turtle_with_turtle, - bench_parse_oxttl_ntriples_with_ntriples, - bench_parse_oxttl_ntriples_with_turtle, - bench_parse_oxttl_turtle_with_turtle + bench_parse_ntriples_with_ntriples, + bench_parse_ntriples_with_turtle, + bench_parse_turtle_with_turtle ); criterion_main!(w3c_testsuite); From d170b536091a67132c01aaf0ff0a53536a17261d Mon Sep 17 00:00:00 2001 From: Tpt Date: Tue, 2 Jan 2024 21:13:31 +0100 Subject: [PATCH 159/217] N3: Fixes stack overflow errors --- lib/oxttl/src/n3.rs | 211 ++++++++++++++++++++------------------------ 1 file changed, 95 insertions(+), 116 deletions(-) diff --git a/lib/oxttl/src/n3.rs b/lib/oxttl/src/n3.rs index ac6d0388..a81318d3 100644 --- a/lib/oxttl/src/n3.rs +++ b/lib/oxttl/src/n3.rs @@ -620,7 +620,7 @@ impl RuleRecognizer for N3Recognizer { results: &mut Vec, errors: &mut Vec, ) -> Self { - if let Some(rule) = self.stack.pop() { + while let Some(rule) = self.stack.pop() { match rule { // [1] n3Doc ::= ( ( n3Statement ".") | sparqlDirective) * // [2] n3Statement ::= n3Directive | triples @@ -635,45 +635,42 @@ impl RuleRecognizer for N3Recognizer { match token { N3Token::PlainKeyword(k) if k.eq_ignore_ascii_case("base") => { self.stack.push(N3State::BaseExpectIri); - self + return self; } N3Token::PlainKeyword(k) if k.eq_ignore_ascii_case("prefix") => { self.stack.push(N3State::PrefixExpectPrefix); - self + return self; } N3Token::LangTag("prefix") => { self.stack.push(N3State::N3DocExpectDot); self.stack.push(N3State::PrefixExpectPrefix); - self + return self; } N3Token::LangTag("base") => { self.stack.push(N3State::N3DocExpectDot); self.stack.push(N3State::BaseExpectIri); - self + return self; } _ => { self.stack.push(N3State::N3DocExpectDot); self.stack.push(N3State::Triples); - self.recognize_next(token, context, results, errors) } } - }, + } N3State::N3DocExpectDot => { if token == N3Token::Punctuation(".") { - self - } else { - errors.push("A dot is expected at the end of N3 statements".into()); - self.recognize_next(token, context, results, errors) + return self; } - }, - N3State::BaseExpectIri => match token { + errors.push("A dot is expected at the end of N3 statements".into()); + } + N3State::BaseExpectIri => return match token { N3Token::IriRef(iri) => { context.lexer_options.base_iri = Some(iri); self } _ => self.error(errors, "The BASE keyword should be followed by an IRI"), }, - N3State::PrefixExpectPrefix => match token { + N3State::PrefixExpectPrefix => return match token { N3Token::PrefixedName { prefix, local, .. } if local.is_empty() => { self.stack.push(N3State::PrefixExpectIri { name: prefix.to_owned() }); self @@ -682,7 +679,7 @@ impl RuleRecognizer for N3Recognizer { self.error(errors, "The PREFIX keyword should be followed by a prefix like 'ex:'") } }, - N3State::PrefixExpectIri { name } => match token { + N3State::PrefixExpectIri { name } => return match token { N3Token::IriRef(iri) => { context.prefixes.insert(name, iri); self @@ -693,51 +690,39 @@ impl RuleRecognizer for N3Recognizer { N3State::Triples => { self.stack.push(N3State::TriplesMiddle); self.stack.push(N3State::Path); - self.recognize_next(token, context, results, errors) - }, - N3State::TriplesMiddle => if matches!(token, N3Token::Punctuation("." | "]" | "}" | ")")) { - self.recognize_next(token, context, results, errors) - } else { + } + N3State::TriplesMiddle => if matches!(token, N3Token::Punctuation("." | "]" | "}" | ")")) {} else { self.stack.push(N3State::TriplesEnd); self.stack.push(N3State::PredicateObjectList); - self.recognize_next(token, context, results, errors) }, N3State::TriplesEnd => { self.terms.pop(); - self.recognize_next(token, context, results, errors) - }, + } // [10] predicateObjectList ::= verb objectList ( ";" ( verb objectList) ? ) * N3State::PredicateObjectList => { self.stack.push(N3State::PredicateObjectListEnd); self.stack.push(N3State::ObjectsList); self.stack.push(N3State::Verb); - self.recognize_next(token, context, results, errors) - }, + } N3State::PredicateObjectListEnd => { self.predicates.pop(); if token == N3Token::Punctuation(";") { self.stack.push(N3State::PredicateObjectListPossibleContinuation); - self - } else { - self.recognize_next(token, context, results, errors) + return self; } - }, + } N3State::PredicateObjectListPossibleContinuation => if token == N3Token::Punctuation(";") { self.stack.push(N3State::PredicateObjectListPossibleContinuation); - self - } else if matches!(token, N3Token::Punctuation(";" | "." | "}" | "]" | ")")) { - self.recognize_next(token, context, results, errors) - } else { + return self; + } else if matches!(token, N3Token::Punctuation(";" | "." | "}" | "]" | ")")) {} else { self.stack.push(N3State::PredicateObjectListEnd); self.stack.push(N3State::ObjectsList); self.stack.push(N3State::Verb); - self.recognize_next(token, context, results, errors) }, // [11] objectList ::= object ( "," object) * N3State::ObjectsList => { self.stack.push(N3State::ObjectsListEnd); self.stack.push(N3State::Path); - self.recognize_next(token, context, results, errors) } N3State::ObjectsListEnd => { let object = self.terms.pop().unwrap(); @@ -757,68 +742,63 @@ impl RuleRecognizer for N3Recognizer { if token == N3Token::Punctuation(",") { self.stack.push(N3State::ObjectsListEnd); self.stack.push(N3State::Path); - self - } else { - self.recognize_next(token, context, results, errors) + return self; } - }, + } // [12] verb ::= predicate | "a" | ( "has" expression) | ( "is" expression "of") | "=" | "<=" | "=>" // [14] predicate ::= expression | ( "<-" expression) N3State::Verb => match token { N3Token::PlainKeyword("a") => { self.predicates.push(Predicate::Regular(rdf::TYPE.into())); - self + return self; } N3Token::PlainKeyword("has") => { self.stack.push(N3State::AfterRegularVerb); self.stack.push(N3State::Path); - self + return self; } N3Token::PlainKeyword("is") => { self.stack.push(N3State::AfterVerbIs); self.stack.push(N3State::Path); - self + return self; } N3Token::Punctuation("=") => { self.predicates.push(Predicate::Regular(NamedNode::new_unchecked("http://www.w3.org/2002/07/owl#sameAs").into())); - self + return self; } N3Token::Punctuation("=>") => { self.predicates.push(Predicate::Regular(NamedNode::new_unchecked("http://www.w3.org/2000/10/swap/log#implies").into())); - self + return self; } N3Token::Punctuation("<=") => { self.predicates.push(Predicate::Inverted(NamedNode::new_unchecked("http://www.w3.org/2000/10/swap/log#implies").into())); - self + return self; } N3Token::Punctuation("<-") => { self.stack.push(N3State::AfterInvertedVerb); self.stack.push(N3State::Path); - self + return self; } - _ => { + _ => { self.stack.push(N3State::AfterRegularVerb); self.stack.push(N3State::Path); - self.recognize_next(token, context, results, errors) } } N3State::AfterRegularVerb => { self.predicates.push(Predicate::Regular(self.terms.pop().unwrap())); - self.recognize_next(token, context, results, errors) } N3State::AfterInvertedVerb => { self.predicates.push(Predicate::Inverted(self.terms.pop().unwrap())); - self.recognize_next(token, context, results, errors) } - N3State::AfterVerbIs => match token { + N3State::AfterVerbIs => return match token { N3Token::PlainKeyword("of") => { self.predicates.push(Predicate::Inverted(self.terms.pop().unwrap())); self - }, + } _ => { self.error(errors, "The keyword 'is' should be followed by a predicate then by the keyword 'of'") } - } + }, // [13] subject ::= expression // [15] object ::= expression // [16] expression ::= path @@ -826,30 +806,28 @@ impl RuleRecognizer for N3Recognizer { N3State::Path => { self.stack.push(N3State::PathFollowUp); self.stack.push(N3State::PathItem); - self.recognize_next(token, context, results, errors) } N3State::PathFollowUp => match token { N3Token::Punctuation("!") => { self.stack.push(N3State::PathAfterIndicator { is_inverse: false }); self.stack.push(N3State::PathItem); - self + return self; } N3Token::Punctuation("^") => { self.stack.push(N3State::PathAfterIndicator { is_inverse: true }); self.stack.push(N3State::PathItem); - self + return self; } - _ => self.recognize_next(token, context, results, errors) + _ => () }, N3State::PathAfterIndicator { is_inverse } => { let predicate = self.terms.pop().unwrap(); let previous = self.terms.pop().unwrap(); let current = BlankNode::default(); - results.push(if is_inverse { self.quad(current.clone(), predicate, previous) } else { self.quad(previous, predicate, current.clone())}); + results.push(if is_inverse { self.quad(current.clone(), predicate, previous) } else { self.quad(previous, predicate, current.clone()) }); self.terms.push(current.into()); self.stack.push(N3State::PathFollowUp); - self.recognize_next(token, context, results, errors) - }, + } // [18] pathItem ::= iri | blankNode | quickVar | collection | blankNodePropertyList | iriPropertyList | literal | formula // [19] literal ::= rdfLiteral | numericLiteral | BOOLEAN_LITERAL // [20] blankNodePropertyList ::= "[" predicateObjectList "]" @@ -863,7 +841,7 @@ impl RuleRecognizer for N3Recognizer { // [29] blankNode ::= BLANK_NODE_LABEL | ANON // [30] quickVar ::= QUICK_VAR_NAME N3State::PathItem => { - match token { + return match token { N3Token::IriRef(iri) => { self.terms.push(NamedNode::from(iri).into()); self @@ -872,8 +850,8 @@ impl RuleRecognizer for N3Recognizer { Ok(t) => { self.terms.push(t.into()); self - }, - Err(e) => self.error(errors, e) + } + Err(e) => self.error(errors, e) } N3Token::BlankNodeLabel(bnode) => { self.terms.push(BlankNode::new_unchecked(bnode).into()); @@ -920,32 +898,32 @@ impl RuleRecognizer for N3Recognizer { self.stack.push(N3State::FormulaContent); self } - _ => self.error(errors, "TOKEN is not a valid RDF value") + _ => + self.error(errors, "TOKEN is not a valid RDF value") + } } N3State::PropertyListMiddle => match token { N3Token::Punctuation("]") => { self.terms.push(BlankNode::default().into()); - self - }, + return self; + } N3Token::PlainKeyword("id") => { self.stack.push(N3State::IriPropertyList); - self - }, - _ => { + return self; + } + _ => { self.terms.push(BlankNode::default().into()); self.stack.push(N3State::PropertyListEnd); self.stack.push(N3State::PredicateObjectList); - self.recognize_next(token, context, results, errors) } } N3State::PropertyListEnd => if token == N3Token::Punctuation("]") { - self + return self; } else { errors.push("blank node property lists should end with a ']'".into()); - self.recognize_next(token, context, results, errors) } - N3State::IriPropertyList => match token { + N3State::IriPropertyList => return match token { N3Token::IriRef(id) => { self.terms.push(NamedNode::from(id).into()); self.stack.push(N3State::PropertyListEnd); @@ -958,23 +936,24 @@ impl RuleRecognizer for N3Recognizer { self.stack.push(N3State::PropertyListEnd); self.stack.push(N3State::PredicateObjectList); self - }, - Err(e) => self.error(errors, e) + } + Err(e) => { + self.error(errors, e) + } } _ => { self.error(errors, "The '[ id' construction should be followed by an IRI") } - } + }, N3State::CollectionBeginning => if let N3Token::Punctuation(")") = token { self.terms.push(rdf::NIL.into()); - self + return self; } else { let root = BlankNode::default(); self.terms.push(root.clone().into()); self.terms.push(root.into()); self.stack.push(N3State::CollectionPossibleEnd); self.stack.push(N3State::Path); - self.recognize_next(token, context, results, errors) }, N3State::CollectionPossibleEnd => { let value = self.terms.pop().unwrap(); @@ -988,35 +967,32 @@ impl RuleRecognizer for N3Recognizer { results.push(self.quad( old, rdf::REST, - rdf::NIL + rdf::NIL, )); - self - } else { - let new = BlankNode::default(); - results.push(self.quad( - old, - rdf::REST, - new.clone() - )); - self.terms.push(new.into()); - self.stack.push(N3State::CollectionPossibleEnd); - self.stack.push(N3State::Path); - self.recognize_next(token, context, results, errors) + return self; } + let new = BlankNode::default(); + results.push(self.quad( + old, + rdf::REST, + new.clone(), + )); + self.terms.push(new.into()); + self.stack.push(N3State::CollectionPossibleEnd); + self.stack.push(N3State::Path); } N3State::LiteralPossibleSuffix { value } => { match token { N3Token::LangTag(lang) => { self.terms.push(Literal::new_language_tagged_literal_unchecked(value, lang.to_ascii_lowercase()).into()); - self - }, + return self; + } N3Token::Punctuation("^^") => { self.stack.push(N3State::LiteralExpectDatatype { value }); - self + return self; } - _ => { + _ => { self.terms.push(Literal::new_simple_literal(value).into()); - self.recognize_next(token, context, results, errors) } } } @@ -1024,17 +1000,20 @@ impl RuleRecognizer for N3Recognizer { match token { N3Token::IriRef(datatype) => { self.terms.push(Literal::new_typed_literal(value, datatype).into()); - self - }, + return self; + } N3Token::PrefixedName { prefix, local, might_be_invalid_iri } => match resolve_local_name(prefix, &local, might_be_invalid_iri, &context.prefixes) { - Ok(datatype) =>{ + Ok(datatype) => { self.terms.push(Literal::new_typed_literal(value, datatype).into()); - self - }, - Err(e) => self.error(errors, e) + return self; + } + Err(e) => { + return self.error(errors, e); + } } - _ => { - self.error(errors, "Expecting a datatype IRI after '^^, found TOKEN").recognize_next(token, context, results, errors) + _ => { + errors.push("Expecting a datatype IRI after '^^, found TOKEN".into()); + self.stack.clear(); } } } @@ -1043,32 +1022,31 @@ impl RuleRecognizer for N3Recognizer { match token { N3Token::Punctuation("}") => { self.terms.push(self.contexts.pop().unwrap().into()); - self + return self; } N3Token::PlainKeyword(k)if k.eq_ignore_ascii_case("base") => { self.stack.push(N3State::FormulaContent); self.stack.push(N3State::BaseExpectIri); - self + return self; } N3Token::PlainKeyword(k)if k.eq_ignore_ascii_case("prefix") => { self.stack.push(N3State::FormulaContent); self.stack.push(N3State::PrefixExpectPrefix); - self + return self; } N3Token::LangTag("prefix") => { self.stack.push(N3State::FormulaContentExpectDot); self.stack.push(N3State::PrefixExpectPrefix); - self + return self; } N3Token::LangTag("base") => { self.stack.push(N3State::FormulaContentExpectDot); self.stack.push(N3State::BaseExpectIri); - self + return self; } - _ => { + _ => { self.stack.push(N3State::FormulaContentExpectDot); self.stack.push(N3State::Triples); - self.recognize_next(token, context, results, errors) } } } @@ -1076,21 +1054,22 @@ impl RuleRecognizer for N3Recognizer { match token { N3Token::Punctuation("}") => { self.terms.push(self.contexts.pop().unwrap().into()); - self + return self; } N3Token::Punctuation(".") => { self.stack.push(N3State::FormulaContent); - self + return self; } - _ => { + _ => { errors.push("A dot is expected at the end of N3 statements".into()); self.stack.push(N3State::FormulaContent); - self.recognize_next(token, context, results, errors) } } } } - } else if token == N3Token::Punctuation(".") { + } + // Empty stack + if token == N3Token::Punctuation(".") { self.stack.push(N3State::N3Doc); self } else { From a84b898fdaef37ce624ba45c2eedae5ca32653b8 Mon Sep 17 00:00:00 2001 From: Tpt Date: Sun, 31 Dec 2023 16:19:31 +0100 Subject: [PATCH 160/217] Parsers: adds "unchecked" option for faster parsing Does not validate IRIs and language tags --- Cargo.lock | 4 +- cli/Cargo.toml | 2 +- cli/src/main.rs | 11 ++ fuzz/fuzz_targets/nquads.rs | 45 +++++--- fuzz/fuzz_targets/trig.rs | 66 ++++++++--- lib/Cargo.toml | 2 +- lib/benches/store.rs | 10 ++ lib/oxrdf/Cargo.toml | 2 +- lib/oxrdfio/src/parser.rs | 72 ++++++------ lib/oxrdfxml/Cargo.toml | 2 +- lib/oxrdfxml/src/parser.rs | 130 ++++++++++++--------- lib/oxttl/Cargo.toml | 2 +- lib/oxttl/src/lexer.rs | 175 +++++++++++++++++------------ lib/oxttl/src/line_formats.rs | 13 ++- lib/oxttl/src/n3.rs | 41 +++++-- lib/oxttl/src/nquads.rs | 13 +++ lib/oxttl/src/ntriples.rs | 36 ++++++ lib/oxttl/src/terse.rs | 38 ++++--- lib/oxttl/src/trig.rs | 13 +++ lib/oxttl/src/turtle.rs | 13 +++ lib/spargebra/Cargo.toml | 2 +- lib/src/store.rs | 12 +- lints/test_debian_compatibility.py | 2 +- 23 files changed, 476 insertions(+), 230 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 809ca10e..e49add4e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1113,9 +1113,9 @@ checksum = "8d91edf4fbb970279443471345a4e8c491bf05bb283b3e6c88e4e606fd8c181b" [[package]] name = "oxiri" -version = "0.2.2" +version = "0.2.3-alpha.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb175ec8981211357b7b379869c2f8d555881c55ea62311428ec0de46d89bd5c" +checksum = "b225dad32cfaa43a960b93f01fa7f87528ac07e794b80f6d9a0153e0222557e2" [[package]] name = "oxrdf" diff --git a/cli/Cargo.toml b/cli/Cargo.toml index f60ce1ca..ff4dac7a 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -32,7 +32,7 @@ clap = { version = "4.0", features = ["derive"] } oxigraph = { version = "0.4.0-alpha.1-dev", path = "../lib" } rand = "0.8" url = "2.4" -oxiri = "0.2" +oxiri = "0.2.3-alpha.1" flate2 = "1.0" rayon-core = "1.11" diff --git a/cli/src/main.rs b/cli/src/main.rs index 4ab05892..b0f7a6fa 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -121,6 +121,8 @@ enum Command { destination: PathBuf, }, /// Load file(s) into the store. + /// + /// Feel free to enable the --lenient option if you know your input is valid to get better performances. Load { /// Directory in which Oxigraph data are persisted. #[arg(short, long, value_hint = ValueHint::DirPath)] @@ -143,6 +145,8 @@ enum Command { #[arg(long, value_hint = ValueHint::Url)] base: Option, /// Attempt to keep loading even if the data file is invalid. + /// + /// This disables most of validation on RDF content. #[arg(long)] lenient: bool, /// Name of the graph to load the data to. @@ -391,6 +395,7 @@ pub fn main() -> anyhow::Result<()> { format.context("The --format option must be set when loading from stdin")?, base.as_deref(), graph, + lenient, ) } else { ThreadPoolBuilder::new() @@ -444,6 +449,7 @@ pub fn main() -> anyhow::Result<()> { }), base.as_deref(), graph, + lenient, ) } else { bulk_load( @@ -454,6 +460,7 @@ pub fn main() -> anyhow::Result<()> { }), base.as_deref(), graph, + lenient, ) } } { @@ -784,6 +791,7 @@ fn bulk_load( format: RdfFormat, base_iri: Option<&str>, to_graph_name: Option, + lenient: bool, ) -> anyhow::Result<()> { let mut parser = RdfParser::from_format(format); if let Some(to_graph_name) = to_graph_name { @@ -794,6 +802,9 @@ fn bulk_load( .with_base_iri(base_iri) .with_context(|| format!("Invalid base IRI {base_iri}"))?; } + if lenient { + parser = parser.unchecked(); + } loader.load_from_read(parser, read)?; Ok(()) } diff --git a/fuzz/fuzz_targets/nquads.rs b/fuzz/fuzz_targets/nquads.rs index c964e229..8852343c 100644 --- a/fuzz/fuzz_targets/nquads.rs +++ b/fuzz/fuzz_targets/nquads.rs @@ -4,43 +4,60 @@ use libfuzzer_sys::fuzz_target; use oxrdf::Quad; use oxttl::{NQuadsParser, NQuadsSerializer}; -fn parse<'a>(chunks: impl IntoIterator) -> (Vec, Vec) { +fn parse<'a>( + chunks: impl IntoIterator, + unchecked: bool, +) -> (Vec, Vec) { let mut quads = Vec::new(); let mut errors = Vec::new(); - let mut parser = NQuadsParser::new().with_quoted_triples().parse(); + let mut parser = NQuadsParser::new().with_quoted_triples(); + if unchecked { + parser = parser.unchecked(); + } + let mut reader = parser.parse(); for chunk in chunks { - parser.extend_from_slice(chunk); - while let Some(result) = parser.read_next() { + reader.extend_from_slice(chunk); + while let Some(result) = reader.read_next() { match result { Ok(quad) => quads.push(quad), Err(error) => errors.push(error.to_string()), } } } - parser.end(); - while let Some(result) = parser.read_next() { + reader.end(); + while let Some(result) = reader.read_next() { match result { Ok(quad) => quads.push(quad), Err(error) => errors.push(error.to_string()), } } - assert!(parser.is_end()); + assert!(reader.is_end()); (quads, errors) } fuzz_target!(|data: &[u8]| { // We parse with splitting - let (quads, errors) = parse(data.split(|c| *c == 0xFF)); + let (quads, errors) = parse(data.split(|c| *c == 0xFF), false); // We parse without splitting - let (quads_without_split, errors_without_split) = parse([data - .iter() - .copied() - .filter(|c| *c != 0xFF) - .collect::>() - .as_slice()]); + let (quads_without_split, errors_without_split) = parse( + [data + .iter() + .copied() + .filter(|c| *c != 0xFF) + .collect::>() + .as_slice()], + false, + ); assert_eq!(quads, quads_without_split); assert_eq!(errors, errors_without_split); + // We test also unchecked if valid + if errors.is_empty() { + let (quads_unchecked, errors_unchecked) = parse(data.split(|c| *c == 0xFF), true); + assert!(errors_unchecked.is_empty()); + assert_eq!(quads, quads_unchecked); + } + // We serialize let mut writer = NQuadsSerializer::new().serialize_to_write(Vec::new()); for quad in &quads { diff --git a/fuzz/fuzz_targets/trig.rs b/fuzz/fuzz_targets/trig.rs index c0713e69..1ce03d1b 100644 --- a/fuzz/fuzz_targets/trig.rs +++ b/fuzz/fuzz_targets/trig.rs @@ -4,31 +4,37 @@ use libfuzzer_sys::fuzz_target; use oxrdf::{Dataset, GraphName, Quad, Subject, Term, Triple}; use oxttl::{TriGParser, TriGSerializer}; -fn parse<'a>(chunks: impl IntoIterator) -> (Vec, Vec) { +fn parse<'a>( + chunks: impl IntoIterator, + unchecked: bool, +) -> (Vec, Vec) { let mut quads = Vec::new(); let mut errors = Vec::new(); let mut parser = TriGParser::new() .with_quoted_triples() .with_base_iri("http://example.com/") - .unwrap() - .parse(); + .unwrap(); + if unchecked { + parser = parser.unchecked(); + } + let mut reader = parser.parse(); for chunk in chunks { - parser.extend_from_slice(chunk); - while let Some(result) = parser.read_next() { + reader.extend_from_slice(chunk); + while let Some(result) = reader.read_next() { match result { Ok(quad) => quads.push(quad), Err(error) => errors.push(error.to_string()), } } } - parser.end(); - while let Some(result) = parser.read_next() { + reader.end(); + while let Some(result) = reader.read_next() { match result { Ok(quad) => quads.push(quad), Err(error) => errors.push(error.to_string()), } } - assert!(parser.is_end()); + assert!(reader.is_end()); (quads, errors) } @@ -66,14 +72,22 @@ fn serialize_quads(quads: &[Quad]) -> Vec { fuzz_target!(|data: &[u8]| { // We parse with splitting - let (quads, errors) = parse(data.split(|c| *c == 0xFF)); + let (quads, errors) = parse(data.split(|c| *c == 0xFF), false); // We parse without splitting - let (quads_without_split, errors_without_split) = parse([data - .iter() - .copied() - .filter(|c| *c != 0xFF) - .collect::>() - .as_slice()]); + let (quads_without_split, errors_without_split) = parse( + [data + .iter() + .copied() + .filter(|c| *c != 0xFF) + .collect::>() + .as_slice()], + false, + ); + let (quads_unchecked, errors_unchecked) = parse(data.split(|c| *c == 0xFF), true); + if errors.is_empty() { + assert!(errors_unchecked.is_empty()); + } + let bnodes_count = quads.iter().map(count_quad_blank_nodes).sum::(); if bnodes_count == 0 { assert_eq!( @@ -83,6 +97,15 @@ fuzz_target!(|data: &[u8]| { String::from_utf8_lossy(&serialize_quads(&quads)), String::from_utf8_lossy(&serialize_quads(&quads_without_split)) ); + if errors.is_empty() { + assert_eq!( + quads, + quads_unchecked, + "Validating:\n{}\nUnchecked:\n{}", + String::from_utf8_lossy(&serialize_quads(&quads)), + String::from_utf8_lossy(&serialize_quads(&quads_unchecked)) + ); + } } else if bnodes_count <= 4 { let mut dataset_with_split = quads.iter().collect::(); let mut dataset_without_split = quads_without_split.iter().collect::(); @@ -95,6 +118,19 @@ fuzz_target!(|data: &[u8]| { String::from_utf8_lossy(&serialize_quads(&quads)), String::from_utf8_lossy(&serialize_quads(&quads_without_split)) ); + if errors.is_empty() { + if errors.is_empty() { + let mut dataset_unchecked = quads_unchecked.iter().collect::(); + dataset_unchecked.canonicalize(); + assert_eq!( + dataset_with_split, + dataset_unchecked, + "Validating:\n{}\nUnchecked:\n{}", + String::from_utf8_lossy(&serialize_quads(&quads)), + String::from_utf8_lossy(&serialize_quads(&quads_unchecked)) + ); + } + } } assert_eq!(errors, errors_without_split); diff --git a/lib/Cargo.toml b/lib/Cargo.toml index 3df87dfe..fbb25452 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -31,7 +31,7 @@ hex = "0.4" json-event-parser = "0.2.0-alpha.2" md-5 = "0.10" oxilangtag = "0.1" -oxiri = "0.2" +oxiri = "0.2.3-alpha.1" oxrdf = { version = "0.2.0-alpha.1-dev", path = "oxrdf", features = ["rdf-star", "oxsdatatypes"] } oxrdfio = { version = "0.1.0-alpha.1-dev", path = "oxrdfio", features = ["rdf-star"] } oxsdatatypes = { version = "0.2.0-alpha.1-dev", path="oxsdatatypes" } diff --git a/lib/benches/store.rs b/lib/benches/store.rs index 22e03e26..0a606135 100644 --- a/lib/benches/store.rs +++ b/lib/benches/store.rs @@ -24,6 +24,16 @@ fn parse_nt(c: &mut Criterion) { } }) }); + group.bench_function("parse BSBM explore 1000 unchecked", |b| { + b.iter(|| { + for r in RdfParser::from_format(RdfFormat::NTriples) + .unchecked() + .parse_read(data.as_slice()) + { + r.unwrap(); + } + }) + }); } fn store_load(c: &mut Criterion) { diff --git a/lib/oxrdf/Cargo.toml b/lib/oxrdf/Cargo.toml index 2196bc9a..1eb030b8 100644 --- a/lib/oxrdf/Cargo.toml +++ b/lib/oxrdf/Cargo.toml @@ -21,7 +21,7 @@ rdf-star = [] [dependencies] rand = "0.8" oxilangtag = "0.1" -oxiri = "0.2" +oxiri = "0.2.3-alpha.1" oxsdatatypes = { version = "0.2.0-alpha.1-dev", path="../oxsdatatypes", optional = true } [lints] diff --git a/lib/oxrdfio/src/parser.rs b/lib/oxrdfio/src/parser.rs index 1b69f955..2046a784 100644 --- a/lib/oxrdfio/src/parser.rs +++ b/lib/oxrdfio/src/parser.rs @@ -158,20 +158,16 @@ impl RdfParser { /// # Result::<_,Box>::Ok(()) /// ``` #[inline] - pub fn with_base_iri(self, base_iri: impl Into) -> Result { - Ok(Self { - inner: match self.inner { - RdfParserKind::N3(p) => RdfParserKind::N3(p), - RdfParserKind::NTriples(p) => RdfParserKind::NTriples(p), - RdfParserKind::NQuads(p) => RdfParserKind::NQuads(p), - RdfParserKind::RdfXml(p) => RdfParserKind::RdfXml(p.with_base_iri(base_iri)?), - RdfParserKind::TriG(p) => RdfParserKind::TriG(p.with_base_iri(base_iri)?), - RdfParserKind::Turtle(p) => RdfParserKind::Turtle(p.with_base_iri(base_iri)?), - }, - default_graph: self.default_graph, - without_named_graphs: self.without_named_graphs, - rename_blank_nodes: self.rename_blank_nodes, - }) + pub fn with_base_iri(mut self, base_iri: impl Into) -> Result { + self.inner = match self.inner { + RdfParserKind::N3(p) => RdfParserKind::N3(p), + RdfParserKind::NTriples(p) => RdfParserKind::NTriples(p), + RdfParserKind::NQuads(p) => RdfParserKind::NQuads(p), + RdfParserKind::RdfXml(p) => RdfParserKind::RdfXml(p.with_base_iri(base_iri)?), + RdfParserKind::TriG(p) => RdfParserKind::TriG(p.with_base_iri(base_iri)?), + RdfParserKind::Turtle(p) => RdfParserKind::Turtle(p.with_base_iri(base_iri)?), + }; + Ok(self) } /// Provides the name graph name that should replace the default graph in the returned quads. @@ -190,13 +186,9 @@ impl RdfParser { /// # Result::<_,Box>::Ok(()) /// ``` #[inline] - pub fn with_default_graph(self, default_graph: impl Into) -> Self { - Self { - inner: self.inner, - default_graph: default_graph.into(), - without_named_graphs: self.without_named_graphs, - rename_blank_nodes: self.rename_blank_nodes, - } + pub fn with_default_graph(mut self, default_graph: impl Into) -> Self { + self.default_graph = default_graph.into(); + self } /// Sets that the parser must fail if parsing a named graph. @@ -212,13 +204,9 @@ impl RdfParser { /// assert!(parser.parse_read(file.as_bytes()).next().unwrap().is_err()); /// ``` #[inline] - pub fn without_named_graphs(self) -> Self { - Self { - inner: self.inner, - default_graph: self.default_graph, - without_named_graphs: true, - rename_blank_nodes: self.rename_blank_nodes, - } + pub fn without_named_graphs(mut self) -> Self { + self.without_named_graphs = true; + self } /// Renames the blank nodes ids from the ones set in the serialization to random ids. @@ -240,13 +228,27 @@ impl RdfParser { /// # Result::<_,Box>::Ok(()) /// ``` #[inline] - pub fn rename_blank_nodes(self) -> Self { - Self { - inner: self.inner, - default_graph: self.default_graph, - without_named_graphs: self.without_named_graphs, - rename_blank_nodes: true, - } + pub fn rename_blank_nodes(mut self) -> Self { + self.rename_blank_nodes = true; + self + } + + /// Assumes the file is valid to make parsing faster. + /// + /// It will skip some validations. + /// + /// Note that if the file is actually not valid, then broken RDF might be emitted by the parser. + #[inline] + pub fn unchecked(mut self) -> Self { + self.inner = match self.inner { + RdfParserKind::N3(p) => RdfParserKind::N3(p.unchecked()), + RdfParserKind::NTriples(p) => RdfParserKind::NTriples(p.unchecked()), + RdfParserKind::NQuads(p) => RdfParserKind::NQuads(p.unchecked()), + RdfParserKind::RdfXml(p) => RdfParserKind::RdfXml(p.unchecked()), + RdfParserKind::TriG(p) => RdfParserKind::TriG(p.unchecked()), + RdfParserKind::Turtle(p) => RdfParserKind::Turtle(p.unchecked()), + }; + self } /// Parses from a [`Read`] implementation and returns an iterator of quads. diff --git a/lib/oxrdfxml/Cargo.toml b/lib/oxrdfxml/Cargo.toml index bc5e8993..1b309626 100644 --- a/lib/oxrdfxml/Cargo.toml +++ b/lib/oxrdfxml/Cargo.toml @@ -21,7 +21,7 @@ async-tokio = ["dep:tokio", "quick-xml/async-tokio"] [dependencies] oxrdf = { version = "0.2.0-alpha.1-dev", path = "../oxrdf" } oxilangtag = "0.1" -oxiri = "0.2" +oxiri = "0.2.3-alpha.1" quick-xml = ">=0.29, <0.32" tokio = { version = "1.29", optional = true, features = ["io-util"] } diff --git a/lib/oxrdfxml/src/parser.rs b/lib/oxrdfxml/src/parser.rs index ebbe80d1..ba0b27a1 100644 --- a/lib/oxrdfxml/src/parser.rs +++ b/lib/oxrdfxml/src/parser.rs @@ -52,6 +52,7 @@ use tokio::io::{AsyncRead, BufReader as AsyncBufReader}; #[derive(Default)] #[must_use] pub struct RdfXmlParser { + unchecked: bool, base: Option>, } @@ -62,6 +63,17 @@ impl RdfXmlParser { Self::default() } + /// Assumes the file is valid to make parsing faster. + /// + /// It will skip some validations. + /// + /// Note that if the file is actually not valid, then broken RDF might be emitted by the parser. + #[inline] + pub fn unchecked(mut self) -> Self { + self.unchecked = true; + self + } + #[inline] pub fn with_base_iri(mut self, base_iri: impl Into) -> Result { self.base = Some(Iri::parse(base_iri.into())?); @@ -158,6 +170,7 @@ impl RdfXmlParser { in_literal_depth: 0, known_rdf_id: HashSet::default(), is_end: false, + unchecked: self.unchecked, } } } @@ -414,6 +427,7 @@ struct RdfXmlReader { in_literal_depth: usize, known_rdf_id: HashSet, is_end: bool, + unchecked: bool, } impl RdfXmlReader { @@ -551,19 +565,28 @@ impl RdfXmlReader { let attribute = attribute.map_err(Error::InvalidAttr)?; if attribute.key.as_ref().starts_with(b"xml") { if attribute.key.as_ref() == b"xml:lang" { - let tag = self.convert_attribute(&attribute)?; - language = Some( + let tag = self.convert_attribute(&attribute)?.to_ascii_lowercase(); + language = Some(if self.unchecked { + tag + } else { LanguageTag::parse(tag.to_ascii_lowercase()) .map_err(|error| SyntaxError { inner: SyntaxErrorKind::InvalidLanguageTag { tag, error }, })? - .into_inner(), - ); + .into_inner() + }); } else if attribute.key.as_ref() == b"xml:base" { let iri = self.convert_attribute(&attribute)?; - base_iri = Some(Iri::parse(iri.clone()).map_err(|error| SyntaxError { - inner: SyntaxErrorKind::InvalidIri { iri, error }, - })?) + base_iri = Some( + if self.unchecked { + Iri::parse_unchecked(iri.clone()) + } else { + Iri::parse(iri.clone()) + } + .map_err(|error| SyntaxError { + inner: SyntaxErrorKind::InvalidIri { iri, error }, + })?, + ) } else { // We ignore other xml attributes } @@ -622,12 +645,7 @@ impl RdfXmlReader { .into()); } else { property_attrs.push(( - NamedNode::new(attribute_url.clone()).map_err(|error| SyntaxError { - inner: SyntaxErrorKind::InvalidIri { - iri: attribute_url, - error, - }, - })?, + self.parse_iri(attribute_url)?, self.convert_attribute(&attribute)?, )); } @@ -637,7 +655,7 @@ impl RdfXmlReader { //Parsing with the base URI let id_attr = match id_attr { Some(iri) => { - let iri = resolve(&base_iri, iri)?; + let iri = self.resolve_iri(&base_iri, iri)?; if self.known_rdf_id.contains(iri.as_str()) { return Err(SyntaxError::msg(format!( "{} has already been used as rdf:ID value", @@ -701,12 +719,7 @@ impl RdfXmlReader { .into()); } else { Self::build_node_elt( - NamedNode::new(tag_name.clone()).map_err(|error| SyntaxError { - inner: SyntaxErrorKind::InvalidIri { - iri: tag_name, - error, - }, - })?, + self.parse_iri(tag_name)?, base_iri, language, id_attr, @@ -727,12 +740,7 @@ impl RdfXmlReader { .into()); } Self::build_node_elt( - NamedNode::new(tag_name.clone()).map_err(|error| SyntaxError { - inner: SyntaxErrorKind::InvalidIri { - iri: tag_name, - error, - }, - })?, + self.parse_iri(tag_name)?, base_iri, language, id_attr, @@ -766,12 +774,7 @@ impl RdfXmlReader { )) .into()); } else { - NamedNode::new(tag_name.clone()).map_err(|error| SyntaxError { - inner: SyntaxErrorKind::InvalidIri { - iri: tag_name, - error, - }, - })? + self.parse_iri(tag_name)? }; match parse_type { RdfXmlParseType::Default => { @@ -1156,32 +1159,51 @@ impl RdfXmlReader { base_iri: &Option>, attribute: &Attribute<'_>, ) -> Result { - Ok(resolve(base_iri, self.convert_attribute(attribute)?)?) + Ok(self.resolve_iri(base_iri, self.convert_attribute(attribute)?)?) } - fn resolve_entity(&self, e: &str) -> Option<&str> { - self.custom_entities.get(e).map(String::as_str) + fn resolve_iri( + &self, + base_iri: &Option>, + relative_iri: String, + ) -> Result { + if let Some(base_iri) = base_iri { + Ok(NamedNode::new_unchecked( + if self.unchecked { + base_iri.resolve_unchecked(&relative_iri) + } else { + base_iri.resolve(&relative_iri) + } + .map_err(|error| SyntaxError { + inner: SyntaxErrorKind::InvalidIri { + iri: relative_iri, + error, + }, + })? + .into_inner(), + )) + } else { + self.parse_iri(relative_iri) + } } -} -fn resolve(base_iri: &Option>, relative_iri: String) -> Result { - if let Some(base_iri) = base_iri { - Ok(base_iri - .resolve(&relative_iri) - .map_err(|error| SyntaxError { - inner: SyntaxErrorKind::InvalidIri { - iri: relative_iri, - error, - }, - })? - .into()) - } else { - NamedNode::new(relative_iri.clone()).map_err(|error| SyntaxError { - inner: SyntaxErrorKind::InvalidIri { - iri: relative_iri, - error, - }, - }) + fn parse_iri(&self, relative_iri: String) -> Result { + Ok(NamedNode::new_unchecked(if self.unchecked { + relative_iri + } else { + Iri::parse(relative_iri.clone()) + .map_err(|error| SyntaxError { + inner: SyntaxErrorKind::InvalidIri { + iri: relative_iri, + error, + }, + })? + .into_inner() + })) + } + + fn resolve_entity(&self, e: &str) -> Option<&str> { + self.custom_entities.get(e).map(String::as_str) } } diff --git a/lib/oxttl/Cargo.toml b/lib/oxttl/Cargo.toml index eeeb10a3..3301fe0e 100644 --- a/lib/oxttl/Cargo.toml +++ b/lib/oxttl/Cargo.toml @@ -22,7 +22,7 @@ async-tokio = ["dep:tokio"] [dependencies] memchr = "2.5" oxrdf = { version = "0.2.0-alpha.1-dev", path = "../oxrdf" } -oxiri = "0.2" +oxiri = "0.2.3-alpha.1" oxilangtag = "0.1" tokio = { version = "1.29", optional = true, features = ["io-util"] } diff --git a/lib/oxttl/src/lexer.rs b/lib/oxttl/src/lexer.rs index 6dd1d024..ef95938e 100644 --- a/lib/oxttl/src/lexer.rs +++ b/lib/oxttl/src/lexer.rs @@ -6,12 +6,12 @@ use oxrdf::NamedNode; use std::borrow::Cow; use std::cmp::min; use std::collections::HashMap; -use std::ops::{Range, RangeInclusive}; +use std::ops::Range; use std::str; #[derive(Debug, PartialEq, Eq)] pub enum N3Token<'a> { - IriRef(Iri), + IriRef(String), PrefixedName { prefix: &'a str, local: Cow<'a, str>, @@ -42,6 +42,7 @@ pub struct N3LexerOptions { pub struct N3Lexer { mode: N3LexerMode, + unchecked: bool, } // TODO: there are a lot of 'None' (missing data) returned even if the stream is ending!!! @@ -61,7 +62,7 @@ impl TokenRecognizer for N3Lexer { b'<' => match *data.get(1)? { b'<' => Some((2, Ok(N3Token::Punctuation("<<")))), b'=' if self.mode == N3LexerMode::N3 => { - if let Some((consumed, result)) = Self::recognize_iri(data, options) { + if let Some((consumed, result)) = self.recognize_iri(data, options) { Some(if let Ok(result) = result { (consumed, Ok(result)) } else { @@ -74,7 +75,7 @@ impl TokenRecognizer for N3Lexer { } } b'-' if self.mode == N3LexerMode::N3 => { - if let Some((consumed, result)) = Self::recognize_iri(data, options) { + if let Some((consumed, result)) = self.recognize_iri(data, options) { Some(if let Ok(result) = result { (consumed, Ok(result)) } else { @@ -86,7 +87,7 @@ impl TokenRecognizer for N3Lexer { None } } - _ => Self::recognize_iri(data, options), + _ => self.recognize_iri(data, options), }, b'>' => { if *data.get(1)? == b'>' { @@ -119,7 +120,7 @@ impl TokenRecognizer for N3Lexer { Self::recognize_string(data, b'\'') } } - b'@' => Self::recognize_lang_tag(data), + b'@' => self.recognize_lang_tag(data), b'.' => match data.get(1) { Some(b'0'..=b'9') => Self::recognize_number(data), Some(_) => Some((1, Ok(N3Token::Punctuation(".")))), @@ -162,18 +163,19 @@ impl TokenRecognizer for N3Lexer { } } b'0'..=b'9' | b'+' | b'-' => Self::recognize_number(data), - b'?' => Self::recognize_variable(data, is_ending), - _ => Self::recognize_pname_or_keyword(data, is_ending), + b'?' => self.recognize_variable(data, is_ending), + _ => self.recognize_pname_or_keyword(data, is_ending), } } } impl N3Lexer { - pub fn new(mode: N3LexerMode) -> Self { - Self { mode } + pub fn new(mode: N3LexerMode, unchecked: bool) -> Self { + Self { mode, unchecked } } fn recognize_iri( + &self, data: &[u8], options: &N3LexerOptions, ) -> Option<(usize, Result, TokenRecognizerError>)> { @@ -186,7 +188,8 @@ impl N3Lexer { i += end; match data[i] { b'>' => { - return Some((i + 1, Self::parse_iri(string, 0..=i, options))); + #[allow(clippy::range_plus_one)] + return Some((i + 1, self.parse_iri(string, 0..i + 1, options))); } b'\\' => { let (additional, c) = Self::recognize_escape(&data[i..], i, false)?; @@ -205,29 +208,36 @@ impl N3Lexer { } fn parse_iri( + &self, iri: Vec, - position: RangeInclusive, + position: Range, options: &N3LexerOptions, ) -> Result, TokenRecognizerError> { - let iri = String::from_utf8(iri).map_err(|e| { - ( - position.clone(), - format!("The IRI contains invalid UTF-8 characters: {e}"), - ) - })?; - let iri = if let Some(base_iri) = options.base_iri.as_ref() { - base_iri.resolve(&iri) - } else { - Iri::parse(iri) - } - .map_err(|e| (position, e.to_string()))?; - Ok(N3Token::IriRef(iri)) + let iri = string_from_utf8(iri, position.clone())?; + Ok(N3Token::IriRef( + if let Some(base_iri) = options.base_iri.as_ref() { + if self.unchecked { + base_iri.resolve_unchecked(&iri) + } else { + base_iri.resolve(&iri) + } + .map_err(|e| (position, e.to_string()))? + .into_inner() + } else if self.unchecked { + iri + } else { + Iri::parse(iri) + .map_err(|e| (position, e.to_string()))? + .into_inner() + }, + )) } - fn recognize_pname_or_keyword( - data: &[u8], + fn recognize_pname_or_keyword<'a>( + &self, + data: &'a [u8], is_ending: bool, - ) -> Option<(usize, Result, TokenRecognizerError>)> { + ) -> Option<(usize, Result, TokenRecognizerError>)> { // [139s] PNAME_NS ::= PN_PREFIX? ':' // [140s] PNAME_LN ::= PNAME_NS PN_LOCAL @@ -303,7 +313,8 @@ impl N3Lexer { )); } - let (consumed, pn_local_result) = Self::recognize_optional_pn_local(&data[i..], is_ending)?; + let (consumed, pn_local_result) = + self.recognize_optional_pn_local(&data[i..], is_ending)?; Some(( consumed + i, pn_local_result.map(|(local, might_be_invalid_iri)| N3Token::PrefixedName { @@ -314,12 +325,13 @@ impl N3Lexer { )) } - fn recognize_variable( - data: &[u8], + fn recognize_variable<'a>( + &self, + data: &'a [u8], is_ending: bool, - ) -> Option<(usize, Result, TokenRecognizerError>)> { + ) -> Option<(usize, Result, TokenRecognizerError>)> { // [36] QUICK_VAR_NAME ::= "?" PN_LOCAL - let (consumed, result) = Self::recognize_optional_pn_local(&data[1..], is_ending)?; + let (consumed, result) = self.recognize_optional_pn_local(&data[1..], is_ending)?; Some(( consumed + 1, result.and_then(|(name, _)| { @@ -332,10 +344,11 @@ impl N3Lexer { )) } - fn recognize_optional_pn_local( - data: &[u8], + fn recognize_optional_pn_local<'a>( + &self, + data: &'a [u8], is_ending: bool, - ) -> Option<(usize, Result<(Cow<'_, str>, bool), TokenRecognizerError>)> { + ) -> Option<(usize, Result<(Cow<'a, str>, bool), TokenRecognizerError>)> { // [168s] PN_LOCAL ::= (PN_CHARS_U | ':' | [0-9] | PLX) ((PN_CHARS | '.' | ':' | PLX)* (PN_CHARS | ':' | PLX))? let mut i = 0; let mut buffer = None; // Buffer if there are some escaped characters @@ -359,23 +372,25 @@ impl N3Lexer { } else if c == '\\' { i += 1; let a = char::from(*data.get(i)?); - if matches!( - a, - '_' | '~' - | '.' - | '-' - | '!' - | '$' - | '&' - | '\'' - | '(' - | ')' - | '*' - | '+' - | ',' - | ';' - | '=' - ) { + if self.unchecked + || matches!( + a, + '_' | '~' + | '.' + | '-' + | '!' + | '$' + | '&' + | '\'' + | '(' + | ')' + | '*' + | '+' + | ',' + | ';' + | '=' + ) + { // ok to escape } else if matches!(a, '/' | '?' | '#' | '@' | '%') { // ok to escape but requires IRI validation @@ -406,12 +421,18 @@ impl N3Lexer { { return Some((0, Ok((Cow::Borrowed(""), false)))); } - might_be_invalid_iri |= - Self::is_possible_pn_chars_base_but_not_valid_iri(c) || c == ':'; + if !self.unchecked { + might_be_invalid_iri |= + Self::is_possible_pn_chars_base_but_not_valid_iri(c) + || c == ':'; + } i += consumed; } else if Self::is_possible_pn_chars(c) || c == ':' || c == '.' { - might_be_invalid_iri |= - Self::is_possible_pn_chars_base_but_not_valid_iri(c) || c == ':'; + if !self.unchecked { + might_be_invalid_iri |= + Self::is_possible_pn_chars_base_but_not_valid_iri(c) + || c == ':'; + } i += consumed; } else { let buffer = if let Some(mut buffer) = buffer { @@ -518,9 +539,10 @@ impl N3Lexer { } } - fn recognize_lang_tag( - data: &[u8], - ) -> Option<(usize, Result, TokenRecognizerError>)> { + fn recognize_lang_tag<'a>( + &self, + data: &'a [u8], + ) -> Option<(usize, Result, TokenRecognizerError>)> { // [144s] LANGTAG ::= '@' [a-zA-Z]+ ('-' [a-zA-Z0-9]+)* let mut is_last_block_empty = true; for (i, c) in data[1..].iter().enumerate() { @@ -532,25 +554,29 @@ impl N3Lexer { Err((1..2, "A language code should always start with a letter").into()), )); } else if is_last_block_empty { - return Some((i, Self::parse_lang_tag(&data[1..i], 1..i - 1))); + return Some((i, self.parse_lang_tag(&data[1..i], 1..i - 1))); } else if *c == b'-' { is_last_block_empty = true; } else { - return Some((i + 1, Self::parse_lang_tag(&data[1..=i], 1..i))); + return Some((i + 1, self.parse_lang_tag(&data[1..=i], 1..i))); } } None } - fn parse_lang_tag( - lang_tag: &[u8], + fn parse_lang_tag<'a>( + &self, + lang_tag: &'a [u8], position: Range, - ) -> Result, TokenRecognizerError> { - Ok(N3Token::LangTag( - LanguageTag::parse(str_from_utf8(lang_tag, position.clone())?) + ) -> Result, TokenRecognizerError> { + let lang_tag = str_from_utf8(lang_tag, position.clone())?; + Ok(N3Token::LangTag(if self.unchecked { + lang_tag + } else { + LanguageTag::parse(lang_tag) .map_err(|e| (position.clone(), e.to_string()))? - .into_inner(), - )) + .into_inner() + })) } fn recognize_string( @@ -933,3 +959,14 @@ fn str_from_utf8(data: &[u8], range: Range) -> Result<&str, TokenRecogniz .into() }) } + +fn string_from_utf8(data: Vec, range: Range) -> Result { + String::from_utf8(data).map_err(|e| { + ( + range.start + e.utf8_error().valid_up_to() + ..min(range.end, range.start + e.utf8_error().valid_up_to() + 4), + format!("Invalid UTF-8: {e}"), + ) + .into() + }) +} diff --git a/lib/oxttl/src/line_formats.rs b/lib/oxttl/src/line_formats.rs index ec439a57..fc48cd53 100644 --- a/lib/oxttl/src/line_formats.rs +++ b/lib/oxttl/src/line_formats.rs @@ -63,7 +63,7 @@ impl RuleRecognizer for NQuadsRecognizer { NQuadsState::ExpectSubject => match token { N3Token::IriRef(s) => { self.subjects - .push(NamedNode::from(s).into()); + .push(NamedNode::new_unchecked(s).into()); self.stack.push(NQuadsState::ExpectPredicate); self } @@ -86,7 +86,7 @@ impl RuleRecognizer for NQuadsRecognizer { NQuadsState::ExpectPredicate => match token { N3Token::IriRef(p) => { self.predicates - .push(p.into()); + .push(NamedNode::new_unchecked(p)); self.stack.push(NQuadsState::ExpectedObject); self } @@ -98,7 +98,7 @@ impl RuleRecognizer for NQuadsRecognizer { NQuadsState::ExpectedObject => match token { N3Token::IriRef(o) => { self.objects - .push(NamedNode::from(o).into()); + .push(NamedNode::new_unchecked(o).into()); self.stack .push(NQuadsState::ExpectPossibleGraphOrEndOfQuotedTriple); self @@ -155,7 +155,7 @@ impl RuleRecognizer for NQuadsRecognizer { self.objects.push( Literal::new_typed_literal( value, - d + NamedNode::new_unchecked(d) ) .into(), ); @@ -171,7 +171,7 @@ impl RuleRecognizer for NQuadsRecognizer { N3Token::IriRef(g) if context.with_graph_name => { self.emit_quad( results, - NamedNode::from(g).into(), + NamedNode::new_unchecked(g).into(), ); self.stack.push(NQuadsState::ExpectDot); self @@ -264,10 +264,11 @@ impl NQuadsRecognizer { pub fn new_parser( with_graph_name: bool, #[cfg(feature = "rdf-star")] with_quoted_triples: bool, + unchecked: bool, ) -> Parser { Parser::new( Lexer::new( - N3Lexer::new(N3LexerMode::NTriples), + N3Lexer::new(N3LexerMode::NTriples, unchecked), MIN_BUFFER_SIZE, MAX_BUFFER_SIZE, true, diff --git a/lib/oxttl/src/n3.rs b/lib/oxttl/src/n3.rs index a81318d3..72db1611 100644 --- a/lib/oxttl/src/n3.rs +++ b/lib/oxttl/src/n3.rs @@ -206,6 +206,7 @@ impl From for N3Quad { #[derive(Default)] #[must_use] pub struct N3Parser { + unchecked: bool, base: Option>, prefixes: HashMap>, } @@ -217,6 +218,17 @@ impl N3Parser { Self::default() } + /// Assumes the file is valid to make parsing faster. + /// + /// It will skip some validations. + /// + /// Note that if the file is actually not valid, then broken RDF might be emitted by the parser. + #[inline] + pub fn unchecked(mut self) -> Self { + self.unchecked = true; + self + } + #[inline] pub fn with_base_iri(mut self, base_iri: impl Into) -> Result { self.base = Some(Iri::parse(base_iri.into())?); @@ -345,7 +357,7 @@ impl N3Parser { /// ``` pub fn parse(self) -> LowLevelN3Reader { LowLevelN3Reader { - parser: N3Recognizer::new_parser(self.base, self.prefixes), + parser: N3Recognizer::new_parser(self.unchecked, self.base, self.prefixes), } } } @@ -665,8 +677,13 @@ impl RuleRecognizer for N3Recognizer { } N3State::BaseExpectIri => return match token { N3Token::IriRef(iri) => { - context.lexer_options.base_iri = Some(iri); - self + match Iri::parse_unchecked(iri) { + Ok(iri) => { + context.lexer_options.base_iri = Some(iri); + self + } + Err(e) => self.error(errors, format!("Invalid base IRI: {e}")) + } } _ => self.error(errors, "The BASE keyword should be followed by an IRI"), }, @@ -681,8 +698,13 @@ impl RuleRecognizer for N3Recognizer { }, N3State::PrefixExpectIri { name } => return match token { N3Token::IriRef(iri) => { - context.prefixes.insert(name, iri); - self + match Iri::parse_unchecked(iri) { + Ok(iri) => { + context.prefixes.insert(name, iri); + self + } + Err(e) => self.error(errors, format!("Invalid prefix IRI: {e}")) + } } _ => self.error(errors, "The PREFIX declaration should be followed by a prefix and its value as an IRI"), }, @@ -843,7 +865,7 @@ impl RuleRecognizer for N3Recognizer { N3State::PathItem => { return match token { N3Token::IriRef(iri) => { - self.terms.push(NamedNode::from(iri).into()); + self.terms.push(NamedNode::new_unchecked(iri).into()); self } N3Token::PrefixedName { prefix, local, might_be_invalid_iri } => match resolve_local_name(prefix, &local, might_be_invalid_iri, &context.prefixes) { @@ -925,7 +947,7 @@ impl RuleRecognizer for N3Recognizer { } N3State::IriPropertyList => return match token { N3Token::IriRef(id) => { - self.terms.push(NamedNode::from(id).into()); + self.terms.push(NamedNode::new_unchecked(id).into()); self.stack.push(N3State::PropertyListEnd); self.stack.push(N3State::PredicateObjectList); self @@ -999,7 +1021,7 @@ impl RuleRecognizer for N3Recognizer { N3State::LiteralExpectDatatype { value } => { match token { N3Token::IriRef(datatype) => { - self.terms.push(Literal::new_typed_literal(value, datatype).into()); + self.terms.push(Literal::new_typed_literal(value, NamedNode::new_unchecked(datatype)).into()); return self; } N3Token::PrefixedName { prefix, local, might_be_invalid_iri } => match resolve_local_name(prefix, &local, might_be_invalid_iri, &context.prefixes) { @@ -1096,12 +1118,13 @@ impl RuleRecognizer for N3Recognizer { impl N3Recognizer { pub fn new_parser( + unchecked: bool, base_iri: Option>, prefixes: HashMap>, ) -> Parser { Parser::new( Lexer::new( - N3Lexer::new(N3LexerMode::N3), + N3Lexer::new(N3LexerMode::N3, unchecked), MIN_BUFFER_SIZE, MAX_BUFFER_SIZE, true, diff --git a/lib/oxttl/src/nquads.rs b/lib/oxttl/src/nquads.rs index c2bf35cd..f150b2d2 100644 --- a/lib/oxttl/src/nquads.rs +++ b/lib/oxttl/src/nquads.rs @@ -37,6 +37,7 @@ use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; #[derive(Default)] #[must_use] pub struct NQuadsParser { + unchecked: bool, #[cfg(feature = "rdf-star")] with_quoted_triples: bool, } @@ -48,6 +49,17 @@ impl NQuadsParser { Self::default() } + /// Assumes the file is valid to make parsing faster. + /// + /// It will skip some validations. + /// + /// Note that if the file is actually not valid, then broken RDF might be emitted by the parser. + #[inline] + pub fn unchecked(mut self) -> Self { + self.unchecked = true; + self + } + /// Enables [N-Quads-star](https://w3c.github.io/rdf-star/cg-spec/2021-12-17.html#n-quads-star). #[cfg(feature = "rdf-star")] #[inline] @@ -165,6 +177,7 @@ impl NQuadsParser { true, #[cfg(feature = "rdf-star")] self.with_quoted_triples, + self.unchecked, ), } } diff --git a/lib/oxttl/src/ntriples.rs b/lib/oxttl/src/ntriples.rs index 674dc27a..995643bc 100644 --- a/lib/oxttl/src/ntriples.rs +++ b/lib/oxttl/src/ntriples.rs @@ -38,6 +38,7 @@ use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; #[derive(Default)] #[must_use] pub struct NTriplesParser { + unchecked: bool, #[cfg(feature = "rdf-star")] with_quoted_triples: bool, } @@ -49,6 +50,17 @@ impl NTriplesParser { Self::default() } + /// Assumes the file is valid to make parsing faster. + /// + /// It will skip some validations. + /// + /// Note that if the file is actually not valid, then broken RDF might be emitted by the parser. /// + #[inline] + pub fn unchecked(mut self) -> Self { + self.unchecked = true; + self + } + /// Enables [N-Triples-star](https://w3c.github.io/rdf-star/cg-spec/2021-12-17.html#n-triples-star). #[cfg(feature = "rdf-star")] #[inline] @@ -166,6 +178,7 @@ impl NTriplesParser { false, #[cfg(feature = "rdf-star")] self.with_quoted_triples, + self.unchecked, ), } } @@ -542,3 +555,26 @@ impl LowLevelNTriplesWriter { writeln!(write, "{} .", t.into()) } } + +#[cfg(test)] +mod tests { + use super::*; + use oxrdf::{Literal, NamedNode}; + + #[test] + fn unchecked_parsing() { + let triples = NTriplesParser::new() + .unchecked() + .parse_read(" \"baz\"@toolonglangtag .".as_bytes()) + .collect::, _>>() + .unwrap(); + assert_eq!( + triples, + [Triple::new( + NamedNode::new_unchecked("foo"), + NamedNode::new_unchecked("bar"), + Literal::new_language_tagged_literal_unchecked("baz", "toolonglangtag"), + )] + ) + } +} diff --git a/lib/oxttl/src/terse.rs b/lib/oxttl/src/terse.rs index 46dcd740..6c83fb71 100644 --- a/lib/oxttl/src/terse.rs +++ b/lib/oxttl/src/terse.rs @@ -107,8 +107,13 @@ impl RuleRecognizer for TriGRecognizer { }, TriGState::BaseExpectIri => match token { N3Token::IriRef(iri) => { - context.lexer_options.base_iri = Some(iri); - self + match Iri::parse_unchecked(iri) { + Ok(iri) => { + context.lexer_options.base_iri = Some(iri); + self + } + Err(e) => self.error(errors, format!("Invalid base IRI: {e}")) + } } _ => self.error(errors, "The BASE keyword should be followed by an IRI"), }, @@ -123,9 +128,13 @@ impl RuleRecognizer for TriGRecognizer { }, TriGState::PrefixExpectIri { name } => match token { N3Token::IriRef(iri) => { - context.prefixes.insert(name, iri); - self - } + match Iri::parse_unchecked(iri) { + Ok(iri) => { + context.prefixes.insert(name, iri); + self + } + Err(e) => self.error(errors, format!("Invalid prefix IRI: {e}")) + } } _ => self.error(errors, "The PREFIX declaration should be followed by a prefix and its value as an IRI"), }, // [3g] triplesOrGraph ::= labelOrSubject ( wrappedGraph | predicateObjectList '.' ) | quotedTriple predicateObjectList '.' @@ -133,7 +142,7 @@ impl RuleRecognizer for TriGRecognizer { TriGState::TriplesOrGraph => match token { N3Token::IriRef(iri) => { self.stack.push(TriGState::WrappedGraphOrPredicateObjectList { - term: NamedNode::from(iri).into() + term: NamedNode::new_unchecked(iri).into() }); self } @@ -291,7 +300,7 @@ impl RuleRecognizer for TriGRecognizer { self } N3Token::IriRef(iri) => { - self.cur_subject.push(NamedNode::from(iri).into()); + self.cur_subject.push(NamedNode::new_unchecked(iri).into()); self.stack.push(TriGState::PredicateObjectList); self } @@ -337,7 +346,7 @@ impl RuleRecognizer for TriGRecognizer { // [7g] labelOrSubject ::= iri | BlankNode TriGState::GraphName => match token { N3Token::IriRef(iri) => { - self.cur_graph = NamedNode::from(iri).into(); + self.cur_graph = NamedNode::new_unchecked(iri).into(); self } N3Token::PrefixedName { prefix, local, might_be_invalid_iri } => match resolve_local_name(prefix, &local, might_be_invalid_iri, &context.prefixes) { @@ -451,7 +460,7 @@ impl RuleRecognizer for TriGRecognizer { self } N3Token::IriRef(iri) => { - self.cur_predicate.push(NamedNode::from(iri)); + self.cur_predicate.push(NamedNode::new_unchecked(iri)); self } N3Token::PrefixedName { prefix, local, might_be_invalid_iri } => match resolve_local_name(prefix, &local, might_be_invalid_iri, &context.prefixes) { @@ -479,7 +488,7 @@ impl RuleRecognizer for TriGRecognizer { // [137s] BlankNode ::= BLANK_NODE_LABEL | ANON TriGState::Object => match token { N3Token::IriRef(iri) => { - self.cur_object.push(NamedNode::from(iri).into()); + self.cur_object.push(NamedNode::new_unchecked(iri).into()); self.emit_quad(results); self } @@ -626,7 +635,7 @@ impl RuleRecognizer for TriGRecognizer { TriGState::LiteralExpectDatatype { value, emit } => { match token { N3Token::IriRef(datatype) => { - self.cur_object.push(Literal::new_typed_literal(value, datatype).into()); + self.cur_object.push(Literal::new_typed_literal(value, NamedNode::new_unchecked(datatype)).into()); if emit { self.emit_quad(results); } @@ -688,7 +697,7 @@ impl RuleRecognizer for TriGRecognizer { self } N3Token::IriRef(iri) => { - self.cur_subject.push(NamedNode::from(iri).into()); + self.cur_subject.push(NamedNode::new_unchecked(iri).into()); self } N3Token::PrefixedName { prefix, local, might_be_invalid_iri } => match resolve_local_name(prefix, &local, might_be_invalid_iri, &context.prefixes) { @@ -720,7 +729,7 @@ impl RuleRecognizer for TriGRecognizer { self } N3Token::IriRef(iri) => { - self.cur_object.push(NamedNode::from(iri).into()); + self.cur_object.push(NamedNode::new_unchecked(iri).into()); self } N3Token::PrefixedName { prefix, local, might_be_invalid_iri } => match resolve_local_name(prefix, &local, might_be_invalid_iri, &context.prefixes) { @@ -823,12 +832,13 @@ impl TriGRecognizer { pub fn new_parser( with_graph_name: bool, #[cfg(feature = "rdf-star")] with_quoted_triples: bool, + unchecked: bool, base_iri: Option>, prefixes: HashMap>, ) -> Parser { Parser::new( Lexer::new( - N3Lexer::new(N3LexerMode::Turtle), + N3Lexer::new(N3LexerMode::Turtle, unchecked), MIN_BUFFER_SIZE, MAX_BUFFER_SIZE, true, diff --git a/lib/oxttl/src/trig.rs b/lib/oxttl/src/trig.rs index 748e3d6a..61cff3af 100644 --- a/lib/oxttl/src/trig.rs +++ b/lib/oxttl/src/trig.rs @@ -42,6 +42,7 @@ use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; #[derive(Default)] #[must_use] pub struct TriGParser { + unchecked: bool, base: Option>, prefixes: HashMap>, #[cfg(feature = "rdf-star")] @@ -55,6 +56,17 @@ impl TriGParser { Self::default() } + /// Assumes the file is valid to make parsing faster. + /// + /// It will skip some validations. + /// + /// Note that if the file is actually not valid, then broken RDF might be emitted by the parser. + #[inline] + pub fn unchecked(mut self) -> Self { + self.unchecked = true; + self + } + #[inline] pub fn with_base_iri(mut self, base_iri: impl Into) -> Result { self.base = Some(Iri::parse(base_iri.into())?); @@ -192,6 +204,7 @@ impl TriGParser { true, #[cfg(feature = "rdf-star")] self.with_quoted_triples, + self.unchecked, self.base, self.prefixes, ), diff --git a/lib/oxttl/src/turtle.rs b/lib/oxttl/src/turtle.rs index 2272e9f8..f9d31232 100644 --- a/lib/oxttl/src/turtle.rs +++ b/lib/oxttl/src/turtle.rs @@ -44,6 +44,7 @@ use tokio::io::{AsyncRead, AsyncWrite}; #[derive(Default)] #[must_use] pub struct TurtleParser { + unchecked: bool, base: Option>, prefixes: HashMap>, #[cfg(feature = "rdf-star")] @@ -57,6 +58,17 @@ impl TurtleParser { Self::default() } + /// Assumes the file is valid to make parsing faster. + /// + /// It will skip some validations. + /// + /// Note that if the file is actually not valid, then broken RDF might be emitted by the parser. + #[inline] + pub fn unchecked(mut self) -> Self { + self.unchecked = true; + self + } + #[inline] pub fn with_base_iri(mut self, base_iri: impl Into) -> Result { self.base = Some(Iri::parse(base_iri.into())?); @@ -194,6 +206,7 @@ impl TurtleParser { false, #[cfg(feature = "rdf-star")] self.with_quoted_triples, + self.unchecked, self.base, self.prefixes, ), diff --git a/lib/spargebra/Cargo.toml b/lib/spargebra/Cargo.toml index 42a14b20..e75cfbea 100644 --- a/lib/spargebra/Cargo.toml +++ b/lib/spargebra/Cargo.toml @@ -23,7 +23,7 @@ sep-0006 = [] [dependencies] peg = "0.8" rand = "0.8" -oxiri = "0.2" +oxiri = "0.2.3-alpha.1" oxilangtag = "0.1" oxrdf = { version = "0.2.0-alpha.1-dev", path="../oxrdf" } diff --git a/lib/src/store.rs b/lib/src/store.rs index 988befed..cec9baae 100644 --- a/lib/src/store.rs +++ b/lib/src/store.rs @@ -1053,7 +1053,6 @@ impl<'a> Transaction<'a> { /// Retrieves quads with a filter on each quad component. /// /// Usage example: - /// Usage example: /// ``` /// use oxigraph::store::{StorageError, Store}; /// use oxigraph::model::*; @@ -1601,19 +1600,22 @@ impl BulkLoader { /// ///
This method is optimized for speed. See [the struct](BulkLoader) documentation for more details.
/// - /// Usage example: + /// To get better speed on valid datasets, consider enabling [`RdfParser::unchecked`] option to skip some validations. + /// /// Usage example: /// ``` /// use oxigraph::store::Store; - /// use oxigraph::io::RdfFormat; + /// use oxigraph::io::{RdfParser, RdfFormat}; /// use oxigraph::model::*; - /// use oxrdfio::RdfParser; /// /// let store = Store::new()?; /// /// // insert a dataset file (former load_dataset method) /// let file = b" ."; - /// store.bulk_loader().load_from_read(RdfFormat::NQuads, file.as_ref())?; + /// store.bulk_loader().load_from_read( + /// RdfParser::from_format(RdfFormat::NQuads).unchecked(), // we inject a custom parser with options + /// file.as_ref() + /// )?; /// /// // insert a graph file (former load_graph method) /// let file = b"<> <> <> ."; diff --git a/lints/test_debian_compatibility.py b/lints/test_debian_compatibility.py index 27530f96..ef00126f 100644 --- a/lints/test_debian_compatibility.py +++ b/lints/test_debian_compatibility.py @@ -5,7 +5,7 @@ from urllib.request import urlopen TARGET_DEBIAN_VERSIONS = ["sid"] IGNORE_PACKAGES = {"oxigraph-js", "oxigraph-testsuite", "pyoxigraph", "sparql-smith"} -ALLOWED_MISSING_PACKAGES = {"codspeed-criterion-compat", "escargot", "json-event-parser", "oxhttp", "quick-xml"} +ALLOWED_MISSING_PACKAGES = {"codspeed-criterion-compat", "escargot", "json-event-parser", "oxhttp", "oxiri", "quick-xml"} base_path = Path(__file__).parent.parent From a5781d1187935ce84a87980881261f8953be7d29 Mon Sep 17 00:00:00 2001 From: Tpt Date: Sat, 30 Dec 2023 10:49:28 +0100 Subject: [PATCH 161/217] Releases v0.4.0-alpha.1 --- .github/workflows/artifacts.yml | 12 +++++ CHANGELOG.md | 55 +++++++++++++++++++++++ Cargo.lock | 76 ++++++++++++++++---------------- Cargo.toml | 2 +- cli/Cargo.toml | 2 +- js/Cargo.toml | 2 +- lib/Cargo.toml | 14 +++--- lib/README.md | 8 ++-- lib/oxrdf/Cargo.toml | 4 +- lib/oxrdf/README.md | 2 + lib/oxrdfio/Cargo.toml | 10 ++--- lib/oxrdfio/README.md | 2 + lib/oxrdfio/src/parser.rs | 7 +-- lib/oxrdfio/src/serializer.rs | 32 ++++++++++---- lib/oxrdfxml/Cargo.toml | 6 +-- lib/oxrdfxml/README.md | 2 + lib/oxsdatatypes/Cargo.toml | 2 +- lib/oxttl/Cargo.toml | 6 +-- lib/oxttl/src/nquads.rs | 3 +- lib/oxttl/src/trig.rs | 3 +- lib/oxttl/src/turtle.rs | 3 +- lib/sparesults/Cargo.toml | 4 +- lib/sparesults/README.md | 3 +- lib/sparesults/src/serializer.rs | 32 ++++++++++---- lib/spargebra/Cargo.toml | 4 +- lib/sparopt/Cargo.toml | 6 +-- lib/sparql-smith/Cargo.toml | 2 +- lib/src/io/mod.rs | 2 + lib/src/io/write.rs | 8 +++- lib/src/store.rs | 42 +++++++++++++----- python/Cargo.toml | 2 +- python/docs/migration.rst | 13 ++++++ 32 files changed, 259 insertions(+), 112 deletions(-) diff --git a/.github/workflows/artifacts.yml b/.github/workflows/artifacts.yml index d984b0c5..77cafa38 100644 --- a/.github/workflows/artifacts.yml +++ b/.github/workflows/artifacts.yml @@ -351,12 +351,24 @@ jobs: - run: cargo publish working-directory: ./lib/oxrdf continue-on-error: true + - run: cargo publish + working-directory: ./lib/oxrdfxml + continue-on-error: true + - run: cargo publish + working-directory: ./lib/oxttl + continue-on-error: true + - run: cargo publish + working-directory: ./lib/oxrdfio + continue-on-error: true - run: cargo publish working-directory: ./lib/sparesults continue-on-error: true - run: cargo publish working-directory: ./lib/spargebra continue-on-error: true + - run: cargo publish + working-directory: ./lib/sparopt + continue-on-error: true - run: cargo publish working-directory: ./lib/sparql-smith continue-on-error: true diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e0007e5..5dba40d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,58 @@ +## [0.4.0-alpha.1] - 2024-01-03 + +### Added +- `sparopt` crate: A new still quite naive query optimizer. +- `oxttl` crate: A N-Triples/N-Quads/Turtle/TriG/N3 parser and serializer compatible with Tokio. +- `oxrdfxml` crate: A RDF/XML parser and serializer compatible with Tokio. +- `oxrdfio` crate: A stand-alone crate with oxigraph I/O related APIs. +- Rust: SPARQL results I/O is now exposed in the `oxigraph` crate (`oxigraph::sparql::results` module). +- Rust: It is now possible to dynamically link rocksdb with the `rocksdb-pkg-config` feature. +- Python: error location is now included in some `SyntaxError` exceptions. +- Python: the file type can be guessed from the file path extension during parsing and serialization. +- Python: the serialization method returns a `bytes` value if no output-related argument is given. +- Python: SPARQL query results I/O is now exposed (`parse_query_results` function and `.serialize` method). +- Python: `RdfFormat` and `QueryResultsFormat` enum to encode supported formats. +- CLI: a `convert` command to convert RDF file between different formats. + +### Removed +- Rust: automated flush at the end of serialization. This should be done explicitly now. +- oxsdatatypes: Deprecated methods. +- Python: 3.7 and Musl linux 1.1 support. +- Python: `GraphName.value`. + +### Changed +- SPARQL: a digit after `.` is now required for `xsd:decimal`. +- SPARQL: calendar subtraction returns `xsd:dayTimeDuration` and not `xsd:duration`. +- SPARQL: Unicode escapes (`\u` and `\U`) are now only supported in IRIs and strings and not everywhere. +- Literal serialization now produces canonical N-Triples according to the RDF 1.2 and RDF Dataset Canonicalization drafts +- Rust: MSRV is now 1.70. +- Rust Makes `GraphName` implement `Default`. +- Rust: `wasm32-unknown-unknown` does not assumes JS platform by default. Enable the `js` feature for that. +- Rust: Parsers take `Read` and not `BufRead` for input. +- Rust: `GraphFormat` and `DatasetFormat` have been merged into `RdfFormat`. +- Rust: `GraphParser` and `DatasetParser` have been merged into `RdfParser`. +- Rust: `GraphSerializer` and `DatasetSerializer` have been merged into `RdfSerializer`. +- Rust: query results are now `Send` and `Sync`. +- Rust: `Store.load_graph` and `Store.load_dataset` have been merged into a `load_from_read` method. +- Rust: `Store.dump_graph` and `Store.dump_dataset` have been renamed to `dump_graph_to_write` and `dump_to_write`. +- Rust: `BulkLoader.set_*` methods have been renamed to `BulkLoader.with_*`. +- oxsdatatypes: pass by-values instead of by-reference parameters when relevant. +- oxsdatatypes: error types have been redesigned. +- oxsdatatypes: return an error when building not serializable duration (year-month and day-time of opposite signs). +- sparesults: renames some methods to move closer to the new oxrdfio crate. +- Python: raise `OSError` instead of `IOError` on OS errors. +- Python: the `mime_type` parameter have been renamed to `format`. +- Python: boolean SPARQL results are now encoded with a `QueryBoolean` class and not a simple `bool`. +- Python: a `path` parameter has been added to all I/O method to read from a file. + The existing `input` parameter now consider `str` values to be a serialization to parse. +- JS: the `mime_type` parameter have been renamed to `format`. +- CLI: the `oxigraph_server` binary has been renamed to `oxigraph`. +- CLI: the `--location` argument is now part of sub-commands where it is relevant. + `oxigraph_server --location foo serve` is not possible anymore. + One need to write `oxigraph serve --location foo`. +- CLI: is is now possible to upload gzip encoded files to the HTTP API with the `Content-Encoding: gzip` header. + + ## [0.3.22] - 2023-11-29 ### Changed diff --git a/Cargo.lock b/Cargo.lock index e49add4e..4a860157 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -82,9 +82,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.78" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca87830a3e3fb156dc96cfbd31cb620265dd053be734723f22b760d6cc3c3051" +checksum = "080e9890a082662b09c1ad45f567faeeb47f22b5fb23895fbe1e651e718e25ca" [[package]] name = "arbitrary" @@ -279,9 +279,9 @@ dependencies = [ [[package]] name = "clang-sys" -version = "1.6.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c688fc74432808e3eb684cae8830a86be1d66a2bd58e1f248ed0960a590baf6f" +checksum = "67523a3b4be3ce1989d607a828d036249522dd9c1c8de7f4dd2dae43a37369d1" dependencies = [ "glob", "libc", @@ -837,12 +837,12 @@ checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4" [[package]] name = "libloading" -version = "0.7.4" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" +checksum = "c571b676ddfc9a8c12f1f3d3085a7b163966a8fd8098a90640953ce5f6170161" dependencies = [ "cfg-if", - "winapi 0.3.9", + "windows-sys 0.48.0", ] [[package]] @@ -1033,7 +1033,7 @@ dependencies = [ [[package]] name = "oxigraph" -version = "0.4.0-alpha.1-dev" +version = "0.4.0-alpha.1" dependencies = [ "codspeed-criterion-compat", "digest", @@ -1063,7 +1063,7 @@ dependencies = [ [[package]] name = "oxigraph-cli" -version = "0.4.0-alpha.1-dev" +version = "0.4.0-alpha.1" dependencies = [ "anyhow", "assert_cmd", @@ -1082,7 +1082,7 @@ dependencies = [ [[package]] name = "oxigraph-js" -version = "0.4.0-alpha.1-dev" +version = "0.4.0-alpha.1" dependencies = [ "console_error_panic_hook", "js-sys", @@ -1092,7 +1092,7 @@ dependencies = [ [[package]] name = "oxigraph-testsuite" -version = "0.4.0-alpha.1-dev" +version = "0.4.0-alpha.1" dependencies = [ "anyhow", "clap", @@ -1119,7 +1119,7 @@ checksum = "b225dad32cfaa43a960b93f01fa7f87528ac07e794b80f6d9a0153e0222557e2" [[package]] name = "oxrdf" -version = "0.2.0-alpha.1-dev" +version = "0.2.0-alpha.1" dependencies = [ "oxilangtag", "oxiri", @@ -1129,7 +1129,7 @@ dependencies = [ [[package]] name = "oxrdfio" -version = "0.1.0-alpha.1-dev" +version = "0.1.0-alpha.1" dependencies = [ "oxrdf", "oxrdfxml", @@ -1139,7 +1139,7 @@ dependencies = [ [[package]] name = "oxrdfxml" -version = "0.1.0-alpha.1-dev" +version = "0.1.0-alpha.1" dependencies = [ "oxilangtag", "oxiri", @@ -1150,7 +1150,7 @@ dependencies = [ [[package]] name = "oxrocksdb-sys" -version = "0.4.0-alpha.1-dev" +version = "0.4.0-alpha.1" dependencies = [ "bindgen", "cc", @@ -1160,14 +1160,14 @@ dependencies = [ [[package]] name = "oxsdatatypes" -version = "0.2.0-alpha.1-dev" +version = "0.2.0-alpha.1" dependencies = [ "js-sys", ] [[package]] name = "oxttl" -version = "0.1.0-alpha.1-dev" +version = "0.1.0-alpha.1" dependencies = [ "memchr", "oxilangtag", @@ -1323,9 +1323,9 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d" +checksum = "a41cf62165e97c7f814d2221421dbb9afcbcdb0a88068e5ea206e19951c2cbb5" dependencies = [ "proc-macro2", "syn", @@ -1333,9 +1333,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.72" +version = "1.0.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a293318316cf6478ec1ad2a21c49390a8d5b5eae9fab736467d93fbc0edc29c5" +checksum = "2de98502f212cfcea8d0bb305bd0f49d7ebdd75b64ba0a68f937d888f4e0d6db" dependencies = [ "unicode-ident", ] @@ -1403,7 +1403,7 @@ dependencies = [ [[package]] name = "pyoxigraph" -version = "0.4.0-alpha.1-dev" +version = "0.4.0-alpha.1" dependencies = [ "oxigraph", "pyo3", @@ -1421,9 +1421,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.33" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" dependencies = [ "proc-macro2", ] @@ -1664,18 +1664,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.193" +version = "1.0.194" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89" +checksum = "0b114498256798c94a0689e1a15fec6005dee8ac1f41de56404b67afc2a4b773" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.193" +version = "1.0.194" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" +checksum = "a3385e45322e8f9931410f01b3031ec534c3947d0e94c18049af4d9f9907d4e0" dependencies = [ "proc-macro2", "quote", @@ -1684,9 +1684,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.108" +version = "1.0.110" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" +checksum = "6fbd975230bada99c8bb618e0c365c2eefa219158d5c6c29610fd09ff1833257" dependencies = [ "itoa", "ryu", @@ -1735,7 +1735,7 @@ checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970" [[package]] name = "sparesults" -version = "0.2.0-alpha.1-dev" +version = "0.2.0-alpha.1" dependencies = [ "json-event-parser", "memchr", @@ -1746,7 +1746,7 @@ dependencies = [ [[package]] name = "spargebra" -version = "0.3.0-alpha.1-dev" +version = "0.3.0-alpha.1" dependencies = [ "oxilangtag", "oxiri", @@ -1757,7 +1757,7 @@ dependencies = [ [[package]] name = "sparopt" -version = "0.1.0-alpha.1-dev" +version = "0.1.0-alpha.1" dependencies = [ "oxrdf", "rand", @@ -1766,7 +1766,7 @@ dependencies = [ [[package]] name = "sparql-smith" -version = "0.1.0-alpha.5-dev" +version = "0.1.0-alpha.5" dependencies = [ "arbitrary", ] @@ -1791,9 +1791,9 @@ checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" [[package]] name = "syn" -version = "2.0.43" +version = "2.0.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee659fb5f3d355364e1f3e5bc10fb82068efbf824a1e9d1c9504244a6469ad53" +checksum = "89456b690ff72fddcecf231caedbe615c59480c93358a93dfae7fc29e3ebbf0e" dependencies = [ "proc-macro2", "quote", @@ -1802,9 +1802,9 @@ dependencies = [ [[package]] name = "target-lexicon" -version = "0.12.12" +version = "0.12.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c39fd04924ca3a864207c66fc2cd7d22d7c016007f9ce846cbb9326331930a" +checksum = "69758bda2e78f098e4ccb393021a0963bb3442eac05f135c30f61b7370bbafae" [[package]] name = "tempfile" diff --git a/Cargo.toml b/Cargo.toml index be87aaac..1e3a4c68 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,7 @@ members = [ resolver = "2" [workspace.package] -version = "0.4.0-alpha.1-dev" +version = "0.4.0-alpha.1" authors = ["Tpt "] license = "MIT OR Apache-2.0" homepage = "https://oxigraph.org/" diff --git a/cli/Cargo.toml b/cli/Cargo.toml index ff4dac7a..4faf1d55 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -29,7 +29,7 @@ rustls-webpki = ["oxigraph/http-client-rustls-webpki"] anyhow = "1.0.72" oxhttp = { version = "0.2.0-alpha.3", features = ["flate2"] } clap = { version = "4.0", features = ["derive"] } -oxigraph = { version = "0.4.0-alpha.1-dev", path = "../lib" } +oxigraph = { version = "0.4.0-alpha.1", path = "../lib" } rand = "0.8" url = "2.4" oxiri = "0.2.3-alpha.1" diff --git a/js/Cargo.toml b/js/Cargo.toml index 41f1d0ef..db53f10f 100644 --- a/js/Cargo.toml +++ b/js/Cargo.toml @@ -16,7 +16,7 @@ crate-type = ["cdylib"] name = "oxigraph" [dependencies] -oxigraph = { version = "0.4.0-alpha.1-dev", path="../lib", features = ["js"] } +oxigraph = { version = "0.4.0-alpha.1", path="../lib", features = ["js"] } wasm-bindgen = "0.2.83" js-sys = "0.3.60" console_error_panic_hook = "0.1.7" diff --git a/lib/Cargo.toml b/lib/Cargo.toml index fbb25452..8d120326 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -32,21 +32,21 @@ json-event-parser = "0.2.0-alpha.2" md-5 = "0.10" oxilangtag = "0.1" oxiri = "0.2.3-alpha.1" -oxrdf = { version = "0.2.0-alpha.1-dev", path = "oxrdf", features = ["rdf-star", "oxsdatatypes"] } -oxrdfio = { version = "0.1.0-alpha.1-dev", path = "oxrdfio", features = ["rdf-star"] } -oxsdatatypes = { version = "0.2.0-alpha.1-dev", path="oxsdatatypes" } +oxrdf = { version = "0.2.0-alpha.1", path = "oxrdf", features = ["rdf-star", "oxsdatatypes"] } +oxrdfio = { version = "0.1.0-alpha.1", path = "oxrdfio", features = ["rdf-star"] } +oxsdatatypes = { version = "0.2.0-alpha.1", path="oxsdatatypes" } rand = "0.8" regex = "1.7" sha1 = "0.10" sha2 = "0.10" siphasher = ">=0.3, <2.0" -sparesults = { version = "0.2.0-alpha.1-dev", path = "sparesults", features = ["rdf-star"] } -spargebra = { version = "0.3.0-alpha.1-dev", path = "spargebra", features = ["rdf-star", "sep-0002", "sep-0006"] } -sparopt = { version = "0.1.0-alpha.1-dev", path="sparopt", features = ["rdf-star", "sep-0002", "sep-0006"] } +sparesults = { version = "0.2.0-alpha.1", path = "sparesults", features = ["rdf-star"] } +spargebra = { version = "0.3.0-alpha.1", path = "spargebra", features = ["rdf-star", "sep-0002", "sep-0006"] } +sparopt = { version = "0.1.0-alpha.1", path="sparopt", features = ["rdf-star", "sep-0002", "sep-0006"] } [target.'cfg(not(target_family = "wasm"))'.dependencies] libc = "0.2.147" -oxrocksdb-sys = { version = "0.4.0-alpha.1-dev", path="../oxrocksdb-sys" } +oxrocksdb-sys = { version = "0.4.0-alpha.1", path="../oxrocksdb-sys" } oxhttp = { version = "0.2.0-alpha.3", optional = true } [target.'cfg(all(target_family = "wasm", target_os = "unknown"))'.dependencies] diff --git a/lib/README.md b/lib/README.md index 31a3e510..4e4f72df 100644 --- a/lib/README.md +++ b/lib/README.md @@ -14,7 +14,7 @@ It also provides a set of utility functions for reading, writing, and processing Oxigraph is in heavy development and SPARQL query evaluation has not been optimized yet. -Oxigraph also provides [a standalone HTTP server](https://crates.io/crates/oxigraph-cli) and [a Python library](https://pyoxigraph.readthedocs.io/) based on this library. +Oxigraph also provides [a CLI tool](https://crates.io/crates/oxigraph-cli) and [a Python library](https://pyoxigraph.readthedocs.io/) based on this library. Oxigraph implements the following specifications: @@ -48,8 +48,8 @@ if let QueryResults::Solutions(mut solutions) = store.query("SELECT ?s WHERE { ``` Some parts of this library are available as standalone crates: -* [`oxrdf`](https://crates.io/crates/oxrdf), datastructures encoding RDF basic concepts (the `oxigraph::model` module). -* [`oxrdfio`](https://crates.io/crates/oxrdfio), a unified parser and serializer API for RDF formats. It itself relies on: +* [`oxrdf`](https://crates.io/crates/oxrdf), datastructures encoding RDF basic concepts (the [`oxigraph::model`](crate::model) module). +* [`oxrdfio`](https://crates.io/crates/oxrdfio), a unified parser and serializer API for RDF formats (the [`oxigraph::io`](crate::io) module). It itself relies on: * [`oxttl`](https://crates.io/crates/oxttl), N-Triple, N-Quad, Turtle, TriG and N3 parsing and serialization. * [`oxrdfxml`](https://crates.io/crates/oxrdfxml), RDF/XML parsing and serialization. * [`spargebra`](https://crates.io/crates/spargebra), a SPARQL parser. @@ -57,7 +57,7 @@ Some parts of this library are available as standalone crates: * [`sparopt`](https://crates.io/crates/sparesults), a SPARQL optimizer. * [`oxsdatatypes`](https://crates.io/crates/oxsdatatypes), an implementation of some XML Schema datatypes. -To build the library, don't forget to clone the submodules using `git clone --recursive https://github.com/oxigraph/oxigraph.git` to clone the repository including submodules or `git submodule update --init` to add submodules to the already cloned repository. +To build the library locally, don't forget to clone the submodules using `git clone --recursive https://github.com/oxigraph/oxigraph.git` to clone the repository including submodules or `git submodule update --init` to add submodules to the already cloned repository. ## License diff --git a/lib/oxrdf/Cargo.toml b/lib/oxrdf/Cargo.toml index 1eb030b8..d843f5d5 100644 --- a/lib/oxrdf/Cargo.toml +++ b/lib/oxrdf/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "oxrdf" -version = "0.2.0-alpha.1-dev" +version = "0.2.0-alpha.1" authors.workspace = true license.workspace = true readme = "README.md" @@ -22,7 +22,7 @@ rdf-star = [] rand = "0.8" oxilangtag = "0.1" oxiri = "0.2.3-alpha.1" -oxsdatatypes = { version = "0.2.0-alpha.1-dev", path="../oxsdatatypes", optional = true } +oxsdatatypes = { version = "0.2.0-alpha.1", path="../oxsdatatypes", optional = true } [lints] workspace = true diff --git a/lib/oxrdf/README.md b/lib/oxrdf/README.md index 8511d95f..88ffa621 100644 --- a/lib/oxrdf/README.md +++ b/lib/oxrdf/README.md @@ -15,6 +15,8 @@ Support for [RDF-star](https://w3c.github.io/rdf-star/cg-spec/2021-12-17.html) i OxRDF is inspired by [RDF/JS](https://rdf.js.org/data-model-spec/) and [Apache Commons RDF](http://commons.apache.org/proper/commons-rdf/). +Use [`oxrdfio`](https://crates.io/crates/oxrdfio) if you need to read or write RDF files. + Usage example: ```rust diff --git a/lib/oxrdfio/Cargo.toml b/lib/oxrdfio/Cargo.toml index 0c14f9fa..5a8da469 100644 --- a/lib/oxrdfio/Cargo.toml +++ b/lib/oxrdfio/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "oxrdfio" -version = "0.1.0-alpha.1-dev" +version = "0.1.0-alpha.1" authors.workspace = true license.workspace = true readme = "README.md" @@ -9,7 +9,7 @@ repository = "https://github.com/oxigraph/oxigraph/tree/master/lib/oxrdfxml" homepage.workspace = true documentation = "https://docs.rs/oxrdfio" description = """ -Parser for various RDF serializations +Parser and serializer for various RDF formats """ edition.workspace = true rust-version.workspace = true @@ -20,9 +20,9 @@ async-tokio = ["dep:tokio", "oxrdfxml/async-tokio", "oxttl/async-tokio"] rdf-star = ["oxrdf/rdf-star", "oxttl/rdf-star"] [dependencies] -oxrdf = { version = "0.2.0-alpha.1-dev", path = "../oxrdf" } -oxrdfxml = { version = "0.1.0-alpha.1-dev", path = "../oxrdfxml" } -oxttl = { version = "0.1.0-alpha.1-dev", path = "../oxttl" } +oxrdf = { version = "0.2.0-alpha.1", path = "../oxrdf" } +oxrdfxml = { version = "0.1.0-alpha.1", path = "../oxrdfxml" } +oxttl = { version = "0.1.0-alpha.1", path = "../oxttl" } tokio = { version = "1.29", optional = true, features = ["io-util"] } [dev-dependencies] diff --git a/lib/oxrdfio/README.md b/lib/oxrdfio/README.md index 318c718c..921c8365 100644 --- a/lib/oxrdfio/README.md +++ b/lib/oxrdfio/README.md @@ -21,6 +21,8 @@ Support for [SPARQL-star](https://w3c.github.io/rdf-star/cg-spec/2021-12-17.html It is designed as a low level parser compatible with both synchronous and asynchronous I/O (behind the `async-tokio` feature). +The entry points of this library are the two [`RdfParser`] and [`RdfSerializer`] structs. + Usage example converting a Turtle file to a N-Triples file: ```rust use oxrdfio::{RdfFormat, RdfParser, RdfSerializer}; diff --git a/lib/oxrdfio/src/parser.rs b/lib/oxrdfio/src/parser.rs index 2046a784..eb60e05e 100644 --- a/lib/oxrdfio/src/parser.rs +++ b/lib/oxrdfio/src/parser.rs @@ -37,9 +37,10 @@ use tokio::io::AsyncRead; /// * [Turtle](https://www.w3.org/TR/turtle/) ([`RdfFormat::Turtle`]) /// /// Note the useful options: -/// - [`with_base_iri`](RdfParser::with_base_iri) to resolve the relative IRIs. -/// - [`rename_blank_nodes`](RdfParser::rename_blank_nodes) to rename the blank nodes to auto-generated numbers to avoid conflicts when merging RDF graphs together. -/// - [`without_named_graphs`](RdfParser::without_named_graphs) to parse a single graph. +/// - [`with_base_iri`](Self::with_base_iri) to resolve the relative IRIs. +/// - [`rename_blank_nodes`](Self::rename_blank_nodes) to rename the blank nodes to auto-generated numbers to avoid conflicts when merging RDF graphs together. +/// - [`without_named_graphs`](Self::without_named_graphs) to parse a single graph. +/// - [`unchecked`](Self::unchecked) to skip some validations if the file is already known to be valid. /// /// ``` /// use oxrdfio::{RdfFormat, RdfParser}; diff --git a/lib/oxrdfio/src/serializer.rs b/lib/oxrdfio/src/serializer.rs index 797bdf90..cd132cd5 100644 --- a/lib/oxrdfio/src/serializer.rs +++ b/lib/oxrdfio/src/serializer.rs @@ -73,9 +73,13 @@ impl RdfSerializer { /// Writes to a [`Write`] implementation. /// - ///
Do not forget to run the [`finish`](ToWriteQuadWriter::finish()) method to properly write the last bytes of the file.
+ ///
/// - ///
This writer does unbuffered writes. You might want to use [`BufWriter`](io::BufWriter) to avoid that.
+ /// Do not forget to run the [`finish`](ToWriteQuadWriter::finish()) method to properly write the last bytes of the file.
+ /// + ///
+ /// + /// This writer does unbuffered writes. You might want to use [`BufWriter`](io::BufWriter) to avoid that.
/// /// ``` /// use oxrdfio::{RdfFormat, RdfSerializer}; @@ -118,9 +122,13 @@ impl RdfSerializer { /// Writes to a Tokio [`AsyncWrite`] implementation. /// - ///
Do not forget to run the [`finish`](ToTokioAsyncWriteQuadWriter::finish()) method to properly write the last bytes of the file.
+ ///
+ /// + /// Do not forget to run the [`finish`](ToTokioAsyncWriteQuadWriter::finish()) method to properly write the last bytes of the file.
+ /// + ///
/// - ///
This writer does unbuffered writes. You might want to use [`BufWriter`](tokio::io::BufWriter) to avoid that.
+ /// This writer does unbuffered writes. You might want to use [`BufWriter`](tokio::io::BufWriter) to avoid that.
/// /// ``` /// use oxrdfio::{RdfFormat, RdfSerializer}; @@ -179,9 +187,13 @@ impl From for RdfSerializer { /// /// Can be built using [`RdfSerializer::serialize_to_write`]. /// -///
Do not forget to run the [`finish`](ToWriteQuadWriter::finish()) method to properly write the last bytes of the file.
+///
/// -///
This writer does unbuffered writes. You might want to use [`BufWriter`](io::BufWriter) to avoid that.
+/// Do not forget to run the [`finish`](ToWriteQuadWriter::finish()) method to properly write the last bytes of the file.
+/// +///
+/// +/// This writer does unbuffered writes. You might want to use [`BufWriter`](io::BufWriter) to avoid that.
/// /// ``` /// use oxrdfio::{RdfFormat, RdfSerializer}; @@ -248,9 +260,13 @@ impl ToWriteQuadWriter { /// /// Can be built using [`RdfSerializer::serialize_to_write`]. /// -///
Do not forget to run the [`finish`](ToWriteQuadWriter::finish()) method to properly write the last bytes of the file.
+///
+/// +/// Do not forget to run the [`finish`](ToWriteQuadWriter::finish()) method to properly write the last bytes of the file.
+/// +///
/// -///
This writer does unbuffered writes. You might want to use [`BufWriter`](io::BufWriter) to avoid that.
+/// This writer does unbuffered writes. You might want to use [`BufWriter`](io::BufWriter) to avoid that.
/// /// ``` /// use oxrdfio::{RdfFormat, RdfSerializer}; diff --git a/lib/oxrdfxml/Cargo.toml b/lib/oxrdfxml/Cargo.toml index 1b309626..adc1ab01 100644 --- a/lib/oxrdfxml/Cargo.toml +++ b/lib/oxrdfxml/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "oxrdfxml" -version = "0.1.0-alpha.1-dev" +version = "0.1.0-alpha.1" authors.workspace = true license.workspace = true readme = "README.md" @@ -8,7 +8,7 @@ keywords = ["RDFXML", "XML", "RDF"] repository = "https://github.com/oxigraph/oxigraph/tree/master/lib/oxrdfxml" homepage.workspace = true description = """ -Parser for the RDF/XML language +Parser and serializer for the RDF/XML format """ documentation = "https://docs.rs/oxrdfxml" edition.workspace = true @@ -19,7 +19,7 @@ default = [] async-tokio = ["dep:tokio", "quick-xml/async-tokio"] [dependencies] -oxrdf = { version = "0.2.0-alpha.1-dev", path = "../oxrdf" } +oxrdf = { version = "0.2.0-alpha.1", path = "../oxrdf" } oxilangtag = "0.1" oxiri = "0.2.3-alpha.1" quick-xml = ">=0.29, <0.32" diff --git a/lib/oxrdfxml/README.md b/lib/oxrdfxml/README.md index 54c1e117..8575d19b 100644 --- a/lib/oxrdfxml/README.md +++ b/lib/oxrdfxml/README.md @@ -9,6 +9,8 @@ OxRDF/XML OxRdfXml is a parser and serializer for [RDF/XML](https://www.w3.org/TR/rdf-syntax-grammar/). +The entry points of this library are the two [`RdfXmlParser`] and [`RdfXmlSerializer`] structs. + Usage example counting the number of people in a RDF/XML file: ```rust use oxrdf::{NamedNodeRef, vocab::rdf}; diff --git a/lib/oxsdatatypes/Cargo.toml b/lib/oxsdatatypes/Cargo.toml index 46eeaa20..8a0ca7be 100644 --- a/lib/oxsdatatypes/Cargo.toml +++ b/lib/oxsdatatypes/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "oxsdatatypes" -version = "0.2.0-alpha.1-dev" +version = "0.2.0-alpha.1" authors.workspace = true license.workspace = true readme = "README.md" diff --git a/lib/oxttl/Cargo.toml b/lib/oxttl/Cargo.toml index 3301fe0e..de0ee35d 100644 --- a/lib/oxttl/Cargo.toml +++ b/lib/oxttl/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "oxttl" -version = "0.1.0-alpha.1-dev" +version = "0.1.0-alpha.1" authors.workspace = true license.workspace = true readme = "README.md" @@ -8,7 +8,7 @@ keywords = ["N-Triples", "N-Quads", "Turtle", "TriG", "N3"] repository = "https://github.com/oxigraph/oxigraph/tree/master/lib/oxttl" homepage.workspace = true description = """ -Parser for languages related to RDF Turtle (N-Triples, N-Quads, Turtle, TriG and N3) +Parser and serializer for languages related to RDF Turtle (N-Triples, N-Quads, Turtle, TriG and N3) """ documentation = "https://docs.rs/oxttl" edition.workspace = true @@ -21,7 +21,7 @@ async-tokio = ["dep:tokio"] [dependencies] memchr = "2.5" -oxrdf = { version = "0.2.0-alpha.1-dev", path = "../oxrdf" } +oxrdf = { version = "0.2.0-alpha.1", path = "../oxrdf" } oxiri = "0.2.3-alpha.1" oxilangtag = "0.1" tokio = { version = "1.29", optional = true, features = ["io-util"] } diff --git a/lib/oxttl/src/nquads.rs b/lib/oxttl/src/nquads.rs index f150b2d2..9be1843b 100644 --- a/lib/oxttl/src/nquads.rs +++ b/lib/oxttl/src/nquads.rs @@ -1,4 +1,5 @@ -//! A [N-Quads](https://www.w3.org/TR/n-quads/) streaming parser implemented by [`NQuadsParser`]. +//! A [N-Quads](https://www.w3.org/TR/n-quads/) streaming parser implemented by [`NQuadsParser`] +//! and a serializer implemented by [`NQuadsSerializer`]. use crate::line_formats::NQuadsRecognizer; #[cfg(feature = "async-tokio")] diff --git a/lib/oxttl/src/trig.rs b/lib/oxttl/src/trig.rs index 61cff3af..e97cdce3 100644 --- a/lib/oxttl/src/trig.rs +++ b/lib/oxttl/src/trig.rs @@ -1,4 +1,5 @@ -//! A [TriG](https://www.w3.org/TR/trig/) streaming parser implemented by [`TriGParser`]. +//! A [TriG](https://www.w3.org/TR/trig/) streaming parser implemented by [`TriGParser`] +//! and a serializer implemented by [`TriGSerializer`]. use crate::terse::TriGRecognizer; #[cfg(feature = "async-tokio")] diff --git a/lib/oxttl/src/turtle.rs b/lib/oxttl/src/turtle.rs index f9d31232..ca0eedb1 100644 --- a/lib/oxttl/src/turtle.rs +++ b/lib/oxttl/src/turtle.rs @@ -1,4 +1,5 @@ -//! A [Turtle](https://www.w3.org/TR/turtle/) streaming parser implemented by [`TurtleParser`]. +//! A [Turtle](https://www.w3.org/TR/turtle/) streaming parser implemented by [`TurtleParser`] +//! and a serializer implemented by [`TurtleSerializer`]. use crate::terse::TriGRecognizer; #[cfg(feature = "async-tokio")] diff --git a/lib/sparesults/Cargo.toml b/lib/sparesults/Cargo.toml index e13b22f5..7042102a 100644 --- a/lib/sparesults/Cargo.toml +++ b/lib/sparesults/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sparesults" -version = "0.2.0-alpha.1-dev" +version = "0.2.0-alpha.1" authors.workspace = true license.workspace = true readme = "README.md" @@ -22,7 +22,7 @@ async-tokio = ["dep:tokio", "quick-xml/async-tokio", "json-event-parser/async-to [dependencies] json-event-parser = "0.2.0-alpha.2" memchr = "2.5" -oxrdf = { version = "0.2.0-alpha.1-dev", path="../oxrdf" } +oxrdf = { version = "0.2.0-alpha.1", path="../oxrdf" } quick-xml = ">=0.29, <0.32" tokio = { version = "1.29", optional = true, features = ["io-util"] } diff --git a/lib/sparesults/README.md b/lib/sparesults/README.md index 0c61ded1..df5a0fb3 100644 --- a/lib/sparesults/README.md +++ b/lib/sparesults/README.md @@ -15,8 +15,9 @@ Support for [SPARQL-star](https://w3c.github.io/rdf-star/cg-spec/2021-12-17.html This crate is intended to be a building piece for SPARQL client and server implementations in Rust like [Oxigraph](https://oxigraph.org). -Usage example converting a JSON result file into a TSV result file: +The entry points of this library are the two [`QueryResultsParser`] and [`QueryResultsSerializer`] structs. +Usage example converting a JSON result file into a TSV result file: ```rust use sparesults::{QueryResultsFormat, QueryResultsParser, FromReadQueryResultsReader, QueryResultsSerializer}; use std::io::Result; diff --git a/lib/sparesults/src/serializer.rs b/lib/sparesults/src/serializer.rs index f9879206..9a4ba143 100644 --- a/lib/sparesults/src/serializer.rs +++ b/lib/sparesults/src/serializer.rs @@ -115,9 +115,13 @@ impl QueryResultsSerializer { /// Returns a `SolutionsWriter` allowing writing query solutions into the given [`Write`] implementation. /// - ///
Do not forget to run the [`finish`](ToWriteSolutionsWriter::finish()) method to properly write the last bytes of the file.
+ ///
/// - ///
This writer does unbuffered writes. You might want to use [`BufWriter`](io::BufWriter) to avoid that.
+ /// Do not forget to run the [`finish`](ToWriteSolutionsWriter::finish()) method to properly write the last bytes of the file.
+ /// + ///
+ /// + /// This writer does unbuffered writes. You might want to use [`BufWriter`](io::BufWriter) to avoid that.
/// /// Example in XML (the API is the same for JSON, CSV and TSV): /// ``` @@ -158,9 +162,13 @@ impl QueryResultsSerializer { /// Returns a `SolutionsWriter` allowing writing query solutions into the given [`Write`] implementation. /// - ///
Do not forget to run the [`finish`](ToWriteSolutionsWriter::finish()) method to properly write the last bytes of the file.
+ ///
+ /// + /// Do not forget to run the [`finish`](ToWriteSolutionsWriter::finish()) method to properly write the last bytes of the file.
+ /// + ///
/// - ///
This writer does unbuffered writes. You might want to use [`BufWriter`](io::BufWriter) to avoid that.
+ /// This writer does unbuffered writes. You might want to use [`BufWriter`](io::BufWriter) to avoid that.
/// /// Example in XML (the API is the same for JSON, CSV and TSV): /// ``` @@ -223,9 +231,13 @@ impl From for QueryResultsSerializer { /// /// Could be built using a [`QueryResultsSerializer`]. /// -///
Do not forget to run the [`finish`](ToWriteSolutionsWriter::finish()) method to properly write the last bytes of the file.
+///
/// -///
This writer does unbuffered writes. You might want to use [`BufWriter`](io::BufWriter) to avoid that.
+/// Do not forget to run the [`finish`](ToWriteSolutionsWriter::finish()) method to properly write the last bytes of the file.
+/// +///
+/// +/// This writer does unbuffered writes. You might want to use [`BufWriter`](io::BufWriter) to avoid that.
/// /// Example in TSV (the API is the same for JSON, XML and CSV): /// ``` @@ -299,9 +311,13 @@ impl ToWriteSolutionsWriter { /// Could be built using a [`QueryResultsSerializer`]. /// -///
Do not forget to run the [`finish`](ToTokioAsyncWriteSolutionsWriter::finish()) method to properly write the last bytes of the file.
+///
+/// +/// Do not forget to run the [`finish`](ToTokioAsyncWriteSolutionsWriter::finish()) method to properly write the last bytes of the file.
+/// +///
/// -///
This writer does unbuffered writes. You might want to use [`BufWriter`](tokio::io::BufWriter) to avoid that.
+/// This writer does unbuffered writes. You might want to use [`BufWriter`](tokio::io::BufWriter) to avoid that.
/// /// Example in TSV (the API is the same for JSON, CSV and XML): /// ``` diff --git a/lib/spargebra/Cargo.toml b/lib/spargebra/Cargo.toml index e75cfbea..bee08a35 100644 --- a/lib/spargebra/Cargo.toml +++ b/lib/spargebra/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "spargebra" -version = "0.3.0-alpha.1-dev" +version = "0.3.0-alpha.1" authors.workspace = true license.workspace = true readme = "README.md" @@ -25,7 +25,7 @@ peg = "0.8" rand = "0.8" oxiri = "0.2.3-alpha.1" oxilangtag = "0.1" -oxrdf = { version = "0.2.0-alpha.1-dev", path="../oxrdf" } +oxrdf = { version = "0.2.0-alpha.1", path="../oxrdf" } [lints] workspace = true diff --git a/lib/sparopt/Cargo.toml b/lib/sparopt/Cargo.toml index 845209ba..8f07afb7 100644 --- a/lib/sparopt/Cargo.toml +++ b/lib/sparopt/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sparopt" -version = "0.1.0-alpha.1-dev" +version = "0.1.0-alpha.1" authors.workspace = true license.workspace = true readme = "README.md" @@ -21,9 +21,9 @@ sep-0002 = ["spargebra/sep-0002"] sep-0006 = ["spargebra/sep-0006"] [dependencies] -oxrdf = { version = "0.2.0-alpha.1-dev", path="../oxrdf" } +oxrdf = { version = "0.2.0-alpha.1", path="../oxrdf" } rand = "0.8" -spargebra = { version = "0.3.0-alpha.1-dev", path="../spargebra" } +spargebra = { version = "0.3.0-alpha.1", path="../spargebra" } [lints] workspace = true diff --git a/lib/sparql-smith/Cargo.toml b/lib/sparql-smith/Cargo.toml index 4217afdf..9682bd79 100644 --- a/lib/sparql-smith/Cargo.toml +++ b/lib/sparql-smith/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sparql-smith" -version = "0.1.0-alpha.5-dev" +version = "0.1.0-alpha.5" authors.workspace = true license.workspace = true readme = "README.md" diff --git a/lib/src/io/mod.rs b/lib/src/io/mod.rs index 5e1cb271..883abb2a 100644 --- a/lib/src/io/mod.rs +++ b/lib/src/io/mod.rs @@ -1,5 +1,7 @@ //! Utilities to read and write RDF graphs and datasets using [OxRDF I/O](https://crates.io/crates/oxrdfio). //! +//! The entry points of this module are the two [`RdfParser`] and [`RdfSerializer`] structs. +//! //! Usage example converting a Turtle file to a N-Triples file: //! ``` //! use oxigraph::io::{RdfFormat, RdfParser, RdfSerializer}; diff --git a/lib/src/io/write.rs b/lib/src/io/write.rs index 3c787ebe..7a9007c0 100644 --- a/lib/src/io/write.rs +++ b/lib/src/io/write.rs @@ -55,7 +55,9 @@ impl GraphSerializer { /// Allows writing triples. /// Could be built using a [`GraphSerializer`]. /// -///
Do not forget to run the [`finish`](TripleWriter::finish()) method to properly write the last bytes of the file.
+///
+/// +/// Do not forget to run the [`finish`](TripleWriter::finish()) method to properly write the last bytes of the file.
/// /// ``` /// use oxigraph::io::{GraphFormat, GraphSerializer}; @@ -138,7 +140,9 @@ impl DatasetSerializer { /// Allows writing triples. /// Could be built using a [`DatasetSerializer`]. /// -///
Do not forget to run the [`finish`](QuadWriter::finish()) method to properly write the last bytes of the file.
+///
+/// +/// Do not forget to run the [`finish`](QuadWriter::finish()) method to properly write the last bytes of the file.
/// /// ``` /// use oxigraph::io::{DatasetFormat, DatasetSerializer}; diff --git a/lib/src/store.rs b/lib/src/store.rs index cec9baae..9b448141 100644 --- a/lib/src/store.rs +++ b/lib/src/store.rs @@ -1,5 +1,7 @@ //! API to access an on-disk [RDF dataset](https://www.w3.org/TR/rdf11-concepts/#dfn-rdf-dataset). //! +//! The entry point of the module is the [`Store`] struct. +//! //! Usage example: //! ``` //! use oxigraph::store::Store; @@ -605,7 +607,9 @@ impl Store { /// Adds atomically a set of quads to this store. /// - ///
This operation uses a memory heavy transaction internally, use the [`bulk_loader`](Store::bulk_loader) if you plan to add ten of millions of triples.
+ ///
+ /// + /// This operation uses a memory heavy transaction internally, use the [`bulk_loader`](Store::bulk_loader) if you plan to add ten of millions of triples.
pub fn extend( &self, quads: impl IntoIterator>, @@ -918,7 +922,9 @@ impl Store { /// After its creation, the backup is usable using [`Store::open`] /// like a regular Oxigraph database and operates independently from the original database. /// - ///
Backups are only possible for on-disk databases created using [`Store::open`].
+ ///
+ /// + /// Backups are only possible for on-disk databases created using [`Store::open`].
/// Temporary in-memory databases created using [`Store::new`] are not compatible with RocksDB backup system. /// ///
An error is raised if the `target_directory` already exists.
@@ -1497,13 +1503,15 @@ impl Iterator for GraphNameIter { /// A bulk loader allowing to load at lot of data quickly into the store. /// -///
The operations provided here are not atomic.
+///
The operations provided here are not atomic. /// If the operation fails in the middle, only a part of the data may be written to the store. -/// Results might get weird if you delete data during the loading process. +/// Results might get weird if you delete data during the loading process.
/// -///
It is optimized for speed.
-/// Memory usage is configurable using [`BulkLoader::with_max_memory_size_in_megabytes`] -/// and the number of used threads with [`BulkLoader::with_num_threads`]. +///
+/// +/// It is optimized for speed.
+/// Memory usage is configurable using [`with_max_memory_size_in_megabytes`](Self::with_max_memory_size_in_megabytes) +/// and the number of used threads with [`with_num_threads`](Self::with_num_threads). /// By default the memory consumption target (excluding the system and RocksDB internal consumption) /// is around 2GB per thread and 2 threads. /// These targets are considered per loaded file. @@ -1598,7 +1606,9 @@ impl BulkLoader { /// If the parsing fails in the middle of the file, only a part of it may be written to the store. /// Results might get weird if you delete data during the loading process.
/// - ///
This method is optimized for speed. See [the struct](BulkLoader) documentation for more details.
+ ///
+ /// + /// This method is optimized for speed. See [the struct](Self) documentation for more details.
/// /// To get better speed on valid datasets, consider enabling [`RdfParser::unchecked`] option to skip some validations. /// @@ -1668,7 +1678,9 @@ impl BulkLoader { /// If the parsing fails in the middle of the file, only a part of it may be written to the store. /// Results might get weird if you delete data during the loading process.
/// - ///
This method is optimized for speed. See [the struct](BulkLoader) documentation for more details.
+ ///
+ /// + /// This method is optimized for speed. See [the struct](Self) documentation for more details.
/// /// Usage example: /// ``` @@ -1727,7 +1739,9 @@ impl BulkLoader { /// If the parsing fails in the middle of the file, only a part of it may be written to the store. /// Results might get weird if you delete data during the loading process. /// - ///
This method is optimized for speed. See [the struct](BulkLoader) documentation for more details.
+ ///
+ /// + /// This method is optimized for speed. See [the struct](Self) documentation for more details.
/// /// Usage example: /// ``` @@ -1788,7 +1802,9 @@ impl BulkLoader { /// If the process fails in the middle of the file, only a part of the data may be written to the store. /// Results might get weird if you delete data during the loading process. /// - ///
This method is optimized for speed. See [the struct](BulkLoader) documentation for more details.
+ ///
+ /// + /// This method is optimized for speed. See [the struct](Self) documentation for more details.
pub fn load_quads( &self, quads: impl IntoIterator>, @@ -1802,7 +1818,9 @@ impl BulkLoader { /// If the process fails in the middle of the file, only a part of the data may be written to the store. /// Results might get weird if you delete data during the loading process. /// - ///
This method is optimized for speed. See [the struct](BulkLoader) documentation for more details.
+ ///
+ /// + /// This method is optimized for speed. See [the struct](Self) documentation for more details.
pub fn load_ok_quads + From>( &self, quads: impl IntoIterator, EI>>, diff --git a/python/Cargo.toml b/python/Cargo.toml index 059c5b23..ef9b5933 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -25,7 +25,7 @@ rocksdb-pkg-config = ["oxigraph/rocksdb-pkg-config"] rustls = ["oxigraph/http-client-rustls-native"] [dependencies] -oxigraph = { version = "0.4.0-alpha.1-dev", path="../lib" } +oxigraph = { version = "0.4.0-alpha.1", path="../lib" } pyo3 = { version = "0.20.1", features = ["extension-module"] } [lints] diff --git a/python/docs/migration.rst b/python/docs/migration.rst index 0911835a..3fe44d80 100644 --- a/python/docs/migration.rst +++ b/python/docs/migration.rst @@ -1,6 +1,19 @@ Migration Guide =============== +From 0.3 to 0.4 +""""""""""""""" + +* Python 3.7 and ``musllinux_1_1`` support have been removed. +* :py:class:`OSError` is now raised instead of :py:class:`IOError` on OS errors. +* The ``mime_type`` parameter have been renamed to ``format`` in I/O functions. + Using :py:class:`RdfFormat` is recommended to describe formats. +* Boolean SPARQL results are now encoded with the :py:class:`QueryBoolean` class and not a simple :py:class:`bool`. +* A `path` parameter has been added to all I/O method to read from a file. + The existing ``input`` parameter now consider :py:class:`str` values to be a serialization to parse. + For example, ``parse(path="foo.ttl")`` will parse the file ``foo.ttl`` whereas ``parse("foo", format=RdfFormat.N_TRIPLES)`` will parse a N-Triples file which content is ``foo``. + + From 0.2 to 0.3 """"""""""""""" From 391e8d7662301490ef8980ff6dd545ffbeb3ec87 Mon Sep 17 00:00:00 2001 From: Tpt Date: Wed, 3 Jan 2024 17:21:08 +0100 Subject: [PATCH 162/217] Fixes and ensures 32bits x86 support --- .github/workflows/artifacts.yml | 2 +- .github/workflows/tests.yml | 15 +++++++++++++- lib/src/storage/backend/rocksdb.rs | 7 ++++--- lib/src/storage/mod.rs | 32 ++++++++++++++++++++---------- oxrocksdb-sys/build.rs | 10 +++++----- 5 files changed, 45 insertions(+), 21 deletions(-) diff --git a/.github/workflows/artifacts.yml b/.github/workflows/artifacts.yml index 77cafa38..371ad3db 100644 --- a/.github/workflows/artifacts.yml +++ b/.github/workflows/artifacts.yml @@ -24,7 +24,7 @@ jobs: with: target: aarch64-unknown-linux-gnu - run: | - sudo apt update && sudo apt-get install -y g++-aarch64-linux-gnu + sudo apt-get install -y g++-aarch64-linux-gnu mkdir .cargo echo -e "[target.aarch64-unknown-linux-gnu]\nlinker = \"aarch64-linux-gnu-gcc\"" >> .cargo/config.toml - run: cargo build --release --no-default-features --features rustls-native diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 227a70cb..e7401054 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -118,7 +118,7 @@ jobs: - run: cargo install cargo-semver-checks || true - run: cargo semver-checks check-release --exclude oxrocksdb-sys --exclude oxigraph-js --exclude pyoxigraph --exclude oxigraph-testsuite --exclude oxigraph-cli - test_linux: + test_linux_x86_64: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -127,6 +127,19 @@ jobs: - uses: ./.github/actions/setup-rust - run: cargo test + test_linux_i686: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + submodules: true + - uses: ./.github/actions/setup-rust + with: + target: i686-unknown-linux-gnu + - run: sudo apt-get install -y g++-multilib + - run: cargo test --target i686-unknown-linux-gnu --no-default-features --features http-client-rustls-native + working-directory: ./lib + test_linux_msv: runs-on: ubuntu-latest steps: diff --git a/lib/src/storage/backend/rocksdb.rs b/lib/src/storage/backend/rocksdb.rs index afa934e1..acac6585 100644 --- a/lib/src/storage/backend/rocksdb.rs +++ b/lib/src/storage/backend/rocksdb.rs @@ -13,6 +13,7 @@ use libc::{self, c_void, free}; use oxrocksdb_sys::*; use rand::random; use std::borrow::Borrow; +#[cfg(unix)] use std::cmp::min; use std::collections::HashMap; use std::env::temp_dir; @@ -1405,7 +1406,7 @@ fn path_to_cstring(path: &Path) -> Result { } #[cfg(unix)] -fn available_file_descriptors() -> io::Result> { +fn available_file_descriptors() -> io::Result> { let mut rlimit = libc::rlimit { rlim_cur: 0, rlim_max: 0, @@ -1418,12 +1419,12 @@ fn available_file_descriptors() -> io::Result> { } #[cfg(windows)] -fn available_file_descriptors() -> io::Result> { +fn available_file_descriptors() -> io::Result> { Ok(Some(512)) // https://docs.microsoft.com/en-us/cpp/c-runtime-library/file-handling } #[cfg(not(any(unix, windows)))] -fn available_file_descriptors() -> io::Result> { +fn available_file_descriptors() -> io::Result> { Ok(None) } diff --git a/lib/src/storage/mod.rs b/lib/src/storage/mod.rs index 01419239..f592aeb9 100644 --- a/lib/src/storage/mod.rs +++ b/lib/src/storage/mod.rs @@ -26,7 +26,7 @@ use std::mem::{swap, take}; #[cfg(not(target_family = "wasm"))] use std::path::{Path, PathBuf}; #[cfg(not(target_family = "wasm"))] -use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Mutex; #[cfg(not(target_family = "wasm"))] use std::{io, thread}; @@ -1249,7 +1249,7 @@ impl StorageBulkLoader { ) .into()); } - let done_counter = AtomicU64::new(0); + let done_counter = Mutex::new(0); let mut done_and_displayed_counter = 0; thread::scope(|thread_scope| { let mut threads = VecDeque::with_capacity(num_threads - 1); @@ -1280,7 +1280,7 @@ impl StorageBulkLoader { )?; for thread in threads { map_thread_result(thread.join()).map_err(StorageError::Io)??; - self.on_possible_progress(&done_counter, &mut done_and_displayed_counter); + self.on_possible_progress(&done_counter, &mut done_and_displayed_counter)?; } Ok(()) }) @@ -1291,17 +1291,17 @@ impl StorageBulkLoader { buffer: &mut Vec, threads: &mut VecDeque>>, thread_scope: &'scope thread::Scope<'scope, '_>, - done_counter: &'scope AtomicU64, + done_counter: &'scope Mutex, done_and_displayed_counter: &mut u64, num_threads: usize, batch_size: usize, ) -> Result<(), StorageError> { - self.on_possible_progress(done_counter, done_and_displayed_counter); + self.on_possible_progress(done_counter, done_and_displayed_counter)?; // We avoid to have too many threads if threads.len() >= num_threads { if let Some(thread) = threads.pop_front() { map_thread_result(thread.join()).map_err(StorageError::Io)??; - self.on_possible_progress(done_counter, done_and_displayed_counter); + self.on_possible_progress(done_counter, done_and_displayed_counter)?; } } let mut buffer_to_load = Vec::with_capacity(batch_size); @@ -1313,15 +1313,22 @@ impl StorageBulkLoader { Ok(()) } - fn on_possible_progress(&self, done: &AtomicU64, done_and_displayed: &mut u64) { - let new_counter = done.load(Ordering::Relaxed); - let display_step = u64::try_from(DEFAULT_BULK_LOAD_BATCH_SIZE).unwrap(); + fn on_possible_progress( + &self, + done: &Mutex, + done_and_displayed: &mut u64, + ) -> Result<(), StorageError> { + let new_counter = *done + .lock() + .map_err(|_| io::Error::new(io::ErrorKind::Other, "Mutex poisoned"))?; + let display_step = DEFAULT_BULK_LOAD_BATCH_SIZE as u64; if new_counter / display_step > *done_and_displayed / display_step { for hook in &self.hooks { hook(new_counter); } } *done_and_displayed = new_counter; + Ok(()) } } @@ -1346,11 +1353,14 @@ impl<'a> FileBulkLoader<'a> { } } - fn load(&mut self, quads: Vec, counter: &AtomicU64) -> Result<(), StorageError> { + fn load(&mut self, quads: Vec, counter: &Mutex) -> Result<(), StorageError> { self.encode(quads)?; let size = self.triples.len() + self.quads.len(); self.save()?; - counter.fetch_add(size.try_into().unwrap_or(u64::MAX), Ordering::Relaxed); + *counter + .lock() + .map_err(|_| io::Error::new(io::ErrorKind::Other, "Mutex poisoned"))? += + size.try_into().unwrap_or(u64::MAX); Ok(()) } diff --git a/oxrocksdb-sys/build.rs b/oxrocksdb-sys/build.rs index abae57b5..8771573e 100644 --- a/oxrocksdb-sys/build.rs +++ b/oxrocksdb-sys/build.rs @@ -153,7 +153,7 @@ fn build_rocksdb() { config.define("NOMINMAX", None); config.define("ROCKSDB_WINDOWS_UTF8_FILENAMES", None); - if target == "x86_64-pc-windows-gnu" { + if target.contains("pc-windows-gnu") { // Tell MinGW to create localtime_r wrapper of localtime_s function. config.define("_POSIX_C_SOURCE", Some("1")); // Tell MinGW to use at least Windows Vista headers instead of the ones of Windows XP. @@ -193,10 +193,10 @@ fn build_rocksdb() { if target.contains("msvc") { config.flag("-EHsc").flag("-std:c++17"); } else { - config - .flag("-std=c++17") - .flag("-Wno-invalid-offsetof") - .define("HAVE_UINT128_EXTENSION", Some("1")); + config.flag("-std=c++17").flag("-Wno-invalid-offsetof"); + if target.contains("x86_64") || target.contains("aarch64") { + config.define("HAVE_UINT128_EXTENSION", Some("1")); + } } for file in lib_sources { From f2a2bd5b5dd8f849aa9b031818bd37d4cd126dc0 Mon Sep 17 00:00:00 2001 From: Tpt Date: Wed, 3 Jan 2024 15:19:09 +0100 Subject: [PATCH 163/217] CI: Improves Python build and upload --- .github/workflows/artifacts.yml | 58 ++++++++++++++++++---------- .github/workflows/manylinux_build.sh | 2 +- .github/workflows/musllinux_build.sh | 2 +- 3 files changed, 40 insertions(+), 22 deletions(-) diff --git a/.github/workflows/artifacts.yml b/.github/workflows/artifacts.yml index 371ad3db..387b67be 100644 --- a/.github/workflows/artifacts.yml +++ b/.github/workflows/artifacts.yml @@ -111,6 +111,11 @@ jobs: python_sdist: runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/p/pyoxigraph + permissions: + id-token: write steps: - uses: actions/checkout@v3 with: @@ -132,14 +137,18 @@ jobs: with: name: pyoxigraph_source path: target/wheels/*.tar.gz - - run: pip install twine && twine upload target/wheels/* - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + - uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: target/wheels if: github.event_name == 'release' wheel_linux: runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/p/pyoxigraph + permissions: + id-token: write strategy: matrix: architecture: [ "x86_64", "aarch64" ] @@ -158,17 +167,20 @@ jobs: if: github.event_name == 'release' || matrix.architecture == 'x86_64' - uses: actions/upload-artifact@v3 with: - name: pyoxigraph_wheel_x86_64_linux_gnu + name: pyoxigraph_wheel_linux_gnu path: target/wheels/*.whl - uses: pypa/gh-action-pypi-publish@release/v1 with: - user: __token__ - password: ${{ secrets.PYPI_PASSWORD }} packages-dir: target/wheels if: github.event_name == 'release' wheel_linux_musl: runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/p/pyoxigraph + permissions: + id-token: write strategy: matrix: architecture: [ "x86_64", "aarch64" ] @@ -187,17 +199,20 @@ jobs: if: github.event_name == 'release' || matrix.architecture == 'x86_64' - uses: actions/upload-artifact@v3 with: - name: pyoxigraph_wheel_x86_64_linux_musl + name: pyoxigraph_wheel_linux_musl path: target/wheels/*.whl - uses: pypa/gh-action-pypi-publish@release/v1 with: - user: __token__ - password: ${{ secrets.PYPI_PASSWORD }} packages-dir: target/wheels if: github.event_name == 'release' wheel_mac: runs-on: macos-latest + environment: + name: pypi + url: https://pypi.org/p/pyoxigraph + permissions: + id-token: write env: DEVELOPER_DIR: '/Applications/Xcode.app/Contents/Developer' SDKROOT: '/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk' @@ -231,16 +246,20 @@ jobs: if: github.event_name == 'release' - uses: actions/upload-artifact@v3 with: - name: pyoxigraph_wheel_universal2_mac + name: pyoxigraph_wheel_mac path: target/wheels/*.whl - - run: pip install twine && twine upload target/wheels/* - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + - uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: target/wheels if: github.event_name == 'release' wheel_windows: runs-on: windows-latest + environment: + name: pypi + url: https://pypi.org/p/pyoxigraph + permissions: + id-token: write steps: - uses: actions/checkout@v3 with: @@ -262,12 +281,11 @@ jobs: - run: maturin build --release -m python/Cargo.toml --features abi3 - uses: actions/upload-artifact@v3 with: - name: pyoxigraph_wheel_x86_64_windows + name: pyoxigraph_wheel_windows path: target/wheels/*.whl - - run: pip install twine && twine upload target/wheels/* - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + - uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: target/wheels if: github.event_name == 'release' npm_tarball: diff --git a/.github/workflows/manylinux_build.sh b/.github/workflows/manylinux_build.sh index 366e2867..48c69a2a 100644 --- a/.github/workflows/manylinux_build.sh +++ b/.github/workflows/manylinux_build.sh @@ -11,7 +11,7 @@ cd python python3.12 -m venv venv source venv/bin/activate pip install -r requirements.dev.txt -maturin develop --release +maturin develop --release --no-default-features --features rustls python generate_stubs.py pyoxigraph pyoxigraph.pyi --ruff maturin build --release --no-default-features --features abi3 --features rustls --compatibility manylinux2014 if [ %for_each_version% ]; then diff --git a/.github/workflows/musllinux_build.sh b/.github/workflows/musllinux_build.sh index 6a519d20..17d1233d 100644 --- a/.github/workflows/musllinux_build.sh +++ b/.github/workflows/musllinux_build.sh @@ -9,7 +9,7 @@ cd python python3.12 -m venv venv source venv/bin/activate pip install -r requirements.dev.txt -maturin develop --release +maturin develop --release --no-default-features --features rustls python generate_stubs.py pyoxigraph pyoxigraph.pyi --ruff maturin build --release --no-default-features --features abi3 --features rustls --compatibility musllinux_1_2 if [ %for_each_version% ]; then From 42a66f62b90957405cb2a4027c378ef1c2f8dbf1 Mon Sep 17 00:00:00 2001 From: Tpt Date: Thu, 4 Jan 2024 15:31:28 +0100 Subject: [PATCH 164/217] Enables cargo semver-checks again --- .github/workflows/tests.yml | 6 +++++- Cargo.lock | 12 ++++++------ Cargo.toml | 8 +------- cli/Cargo.toml | 12 ++++++------ js/Cargo.toml | 12 ++++++------ lib/Cargo.toml | 18 +++++++++--------- lib/oxrdf/Cargo.toml | 11 +++++------ lib/oxrdfio/Cargo.toml | 9 ++++----- lib/oxrdfxml/Cargo.toml | 9 ++++----- lib/oxsdatatypes/Cargo.toml | 9 ++++----- lib/oxttl/Cargo.toml | 9 ++++----- lib/sparesults/Cargo.toml | 11 +++++------ lib/spargebra/Cargo.toml | 11 +++++------ lib/sparopt/Cargo.toml | 13 ++++++------- lib/sparql-smith/Cargo.toml | 9 ++++----- oxrocksdb-sys/Cargo.toml | 8 ++++---- python/Cargo.toml | 12 ++++++------ testsuite/Cargo.toml | 18 +++++++++--------- 18 files changed, 93 insertions(+), 104 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e7401054..67153706 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -109,13 +109,17 @@ jobs: semver_checks: runs-on: ubuntu-latest - if: false steps: - uses: actions/checkout@v3 with: submodules: true - uses: ./.github/actions/setup-rust - run: cargo install cargo-semver-checks || true + - uses: actions/cache@v3 + with: + path: rocksdb + key: ${{ runner.os }}-rocksdb-8.0.0 + - run: bash .github/workflows/install_rocksdb.sh - run: cargo semver-checks check-release --exclude oxrocksdb-sys --exclude oxigraph-js --exclude pyoxigraph --exclude oxigraph-testsuite --exclude oxigraph-cli test_linux_x86_64: diff --git a/Cargo.lock b/Cargo.lock index 4a860157..96f7d6fa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1033,7 +1033,7 @@ dependencies = [ [[package]] name = "oxigraph" -version = "0.4.0-alpha.1" +version = "0.4.0-alpha.2-dev" dependencies = [ "codspeed-criterion-compat", "digest", @@ -1063,7 +1063,7 @@ dependencies = [ [[package]] name = "oxigraph-cli" -version = "0.4.0-alpha.1" +version = "0.4.0-alpha.2-dev" dependencies = [ "anyhow", "assert_cmd", @@ -1082,7 +1082,7 @@ dependencies = [ [[package]] name = "oxigraph-js" -version = "0.4.0-alpha.1" +version = "0.4.0-alpha.2-dev" dependencies = [ "console_error_panic_hook", "js-sys", @@ -1092,7 +1092,7 @@ dependencies = [ [[package]] name = "oxigraph-testsuite" -version = "0.4.0-alpha.1" +version = "0.4.0-alpha.2-dev" dependencies = [ "anyhow", "clap", @@ -1150,7 +1150,7 @@ dependencies = [ [[package]] name = "oxrocksdb-sys" -version = "0.4.0-alpha.1" +version = "0.4.0-alpha.2-dev" dependencies = [ "bindgen", "cc", @@ -1403,7 +1403,7 @@ dependencies = [ [[package]] name = "pyoxigraph" -version = "0.4.0-alpha.1" +version = "0.4.0-alpha.2-dev" dependencies = [ "oxigraph", "pyo3", diff --git a/Cargo.toml b/Cargo.toml index 1e3a4c68..3ec94c5e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,13 +18,7 @@ members = [ ] resolver = "2" -[workspace.package] -version = "0.4.0-alpha.1" -authors = ["Tpt "] -license = "MIT OR Apache-2.0" -homepage = "https://oxigraph.org/" -edition = "2021" -rust-version = "1.70" +# TODO: use workspace.package shared metadata when https://github.com/obi1kenobi/cargo-semver-checks/issues/462 will be fixed [workspace.lints.rust] absolute_paths_not_starting_with_crate = "warn" diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 4faf1d55..6953a318 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -1,8 +1,8 @@ [package] name = "oxigraph-cli" -version.workspace = true -authors.workspace = true -license.workspace = true +version = "0.4.0-alpha.2-dev" +authors = ["Tpt "] +license = "MIT OR Apache-2.0" readme = "README.md" keywords = ["RDF", "SPARQL", "graph-database", "database"] categories = ["command-line-utilities", "database"] @@ -11,8 +11,8 @@ homepage = "https://oxigraph.org/cli/" description = """ Oxigraph command line toolkit and SPARQL HTTP server """ -edition.workspace = true -rust-version.workspace = true +edition = "2021" +rust-version = "1.70" [[bin]] name = "oxigraph" @@ -29,7 +29,7 @@ rustls-webpki = ["oxigraph/http-client-rustls-webpki"] anyhow = "1.0.72" oxhttp = { version = "0.2.0-alpha.3", features = ["flate2"] } clap = { version = "4.0", features = ["derive"] } -oxigraph = { version = "0.4.0-alpha.1", path = "../lib" } +oxigraph = { version = "0.4.0-alpha.2-dev", path = "../lib" } rand = "0.8" url = "2.4" oxiri = "0.2.3-alpha.1" diff --git a/js/Cargo.toml b/js/Cargo.toml index db53f10f..727b15f7 100644 --- a/js/Cargo.toml +++ b/js/Cargo.toml @@ -1,14 +1,14 @@ [package] name = "oxigraph-js" -version.workspace = true -authors.workspace = true -license.workspace = true +version = "0.4.0-alpha.2-dev" +authors = ["Tpt "] +license = "MIT OR Apache-2.0" readme = "README.md" keywords = ["RDF", "N-Triples", "Turtle", "RDF/XML", "SPARQL"] repository = "https://github.com/oxigraph/oxigraph/tree/main/js" description = "JavaScript bindings of Oxigraph" -edition.workspace = true -rust-version.workspace = true +edition = "2021" +rust-version = "1.70" publish = false [lib] @@ -16,7 +16,7 @@ crate-type = ["cdylib"] name = "oxigraph" [dependencies] -oxigraph = { version = "0.4.0-alpha.1", path="../lib", features = ["js"] } +oxigraph = { path = "../lib", features = ["js"] } wasm-bindgen = "0.2.83" js-sys = "0.3.60" console_error_panic_hook = "0.1.7" diff --git a/lib/Cargo.toml b/lib/Cargo.toml index 8d120326..9ea03887 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -1,19 +1,19 @@ [package] name = "oxigraph" -version.workspace = true -authors.workspace = true -license.workspace = true +version = "0.4.0-alpha.2-dev" +authors = ["Tpt "] +license = "MIT OR Apache-2.0" readme = "README.md" keywords = ["RDF", "SPARQL", "graph-database", "database"] categories = ["database-implementations"] repository = "https://github.com/oxigraph/oxigraph/tree/main/lib" -homepage.workspace = true +homepage = "https://oxigraph.org/" documentation = "https://docs.rs/oxigraph" description = """ a SPARQL database and RDF toolkit """ -edition.workspace = true -rust-version.workspace = true +edition = "2021" +rust-version = "1.70" [features] default = [] @@ -34,7 +34,7 @@ oxilangtag = "0.1" oxiri = "0.2.3-alpha.1" oxrdf = { version = "0.2.0-alpha.1", path = "oxrdf", features = ["rdf-star", "oxsdatatypes"] } oxrdfio = { version = "0.1.0-alpha.1", path = "oxrdfio", features = ["rdf-star"] } -oxsdatatypes = { version = "0.2.0-alpha.1", path="oxsdatatypes" } +oxsdatatypes = { version = "0.2.0-alpha.1", path = "oxsdatatypes" } rand = "0.8" regex = "1.7" sha1 = "0.10" @@ -42,11 +42,11 @@ sha2 = "0.10" siphasher = ">=0.3, <2.0" sparesults = { version = "0.2.0-alpha.1", path = "sparesults", features = ["rdf-star"] } spargebra = { version = "0.3.0-alpha.1", path = "spargebra", features = ["rdf-star", "sep-0002", "sep-0006"] } -sparopt = { version = "0.1.0-alpha.1", path="sparopt", features = ["rdf-star", "sep-0002", "sep-0006"] } +sparopt = { version = "0.1.0-alpha.1", path = "sparopt", features = ["rdf-star", "sep-0002", "sep-0006"] } [target.'cfg(not(target_family = "wasm"))'.dependencies] libc = "0.2.147" -oxrocksdb-sys = { version = "0.4.0-alpha.1", path="../oxrocksdb-sys" } +oxrocksdb-sys = { version = "0.4.0-alpha.2-dev", path = "../oxrocksdb-sys" } oxhttp = { version = "0.2.0-alpha.3", optional = true } [target.'cfg(all(target_family = "wasm", target_os = "unknown"))'.dependencies] diff --git a/lib/oxrdf/Cargo.toml b/lib/oxrdf/Cargo.toml index d843f5d5..fdf94a51 100644 --- a/lib/oxrdf/Cargo.toml +++ b/lib/oxrdf/Cargo.toml @@ -1,18 +1,17 @@ [package] name = "oxrdf" version = "0.2.0-alpha.1" -authors.workspace = true -license.workspace = true +authors = ["Tpt "] +license = "MIT OR Apache-2.0" readme = "README.md" keywords = ["RDF"] repository = "https://github.com/oxigraph/oxigraph/tree/main/lib/oxrdf" -homepage.workspace = true description = """ A library providing basic data structures related to RDF """ documentation = "https://docs.rs/oxrdf" -edition.workspace = true -rust-version.workspace = true +edition = "2021" +rust-version = "1.70" [features] default = [] @@ -22,7 +21,7 @@ rdf-star = [] rand = "0.8" oxilangtag = "0.1" oxiri = "0.2.3-alpha.1" -oxsdatatypes = { version = "0.2.0-alpha.1", path="../oxsdatatypes", optional = true } +oxsdatatypes = { version = "0.2.0-alpha.1", path = "../oxsdatatypes", optional = true } [lints] workspace = true diff --git a/lib/oxrdfio/Cargo.toml b/lib/oxrdfio/Cargo.toml index 5a8da469..6ded48d6 100644 --- a/lib/oxrdfio/Cargo.toml +++ b/lib/oxrdfio/Cargo.toml @@ -1,18 +1,17 @@ [package] name = "oxrdfio" version = "0.1.0-alpha.1" -authors.workspace = true -license.workspace = true +authors = ["Tpt "] +license = "MIT OR Apache-2.0" readme = "README.md" keywords = ["RDF"] repository = "https://github.com/oxigraph/oxigraph/tree/master/lib/oxrdfxml" -homepage.workspace = true documentation = "https://docs.rs/oxrdfio" description = """ Parser and serializer for various RDF formats """ -edition.workspace = true -rust-version.workspace = true +edition = "2021" +rust-version = "1.70" [features] default = [] diff --git a/lib/oxrdfxml/Cargo.toml b/lib/oxrdfxml/Cargo.toml index adc1ab01..2906ac0e 100644 --- a/lib/oxrdfxml/Cargo.toml +++ b/lib/oxrdfxml/Cargo.toml @@ -1,18 +1,17 @@ [package] name = "oxrdfxml" version = "0.1.0-alpha.1" -authors.workspace = true -license.workspace = true +authors = ["Tpt "] +license = "MIT OR Apache-2.0" readme = "README.md" keywords = ["RDFXML", "XML", "RDF"] repository = "https://github.com/oxigraph/oxigraph/tree/master/lib/oxrdfxml" -homepage.workspace = true description = """ Parser and serializer for the RDF/XML format """ documentation = "https://docs.rs/oxrdfxml" -edition.workspace = true -rust-version.workspace = true +edition = "2021" +rust-version = "1.70" [features] default = [] diff --git a/lib/oxsdatatypes/Cargo.toml b/lib/oxsdatatypes/Cargo.toml index 8a0ca7be..f7bc1f31 100644 --- a/lib/oxsdatatypes/Cargo.toml +++ b/lib/oxsdatatypes/Cargo.toml @@ -1,18 +1,17 @@ [package] name = "oxsdatatypes" version = "0.2.0-alpha.1" -authors.workspace = true -license.workspace = true +authors = ["Tpt "] +license = "MIT OR Apache-2.0" readme = "README.md" keywords = ["XSD"] repository = "https://github.com/oxigraph/oxigraph/tree/main/lib/oxsdatatypes" -homepage.workspace = true description = """ An implementation of some XSD datatypes for SPARQL implementations """ documentation = "https://docs.rs/oxsdatatypes" -edition.workspace = true -rust-version.workspace = true +edition = "2021" +rust-version = "1.70" [features] js = ["js-sys"] diff --git a/lib/oxttl/Cargo.toml b/lib/oxttl/Cargo.toml index de0ee35d..c19cdffe 100644 --- a/lib/oxttl/Cargo.toml +++ b/lib/oxttl/Cargo.toml @@ -1,18 +1,17 @@ [package] name = "oxttl" version = "0.1.0-alpha.1" -authors.workspace = true -license.workspace = true +authors = ["Tpt "] +license = "MIT OR Apache-2.0" readme = "README.md" keywords = ["N-Triples", "N-Quads", "Turtle", "TriG", "N3"] repository = "https://github.com/oxigraph/oxigraph/tree/master/lib/oxttl" -homepage.workspace = true description = """ Parser and serializer for languages related to RDF Turtle (N-Triples, N-Quads, Turtle, TriG and N3) """ documentation = "https://docs.rs/oxttl" -edition.workspace = true -rust-version.workspace = true +edition = "2021" +rust-version = "1.70" [features] default = [] diff --git a/lib/sparesults/Cargo.toml b/lib/sparesults/Cargo.toml index 7042102a..0ef5eb23 100644 --- a/lib/sparesults/Cargo.toml +++ b/lib/sparesults/Cargo.toml @@ -1,18 +1,17 @@ [package] name = "sparesults" version = "0.2.0-alpha.1" -authors.workspace = true -license.workspace = true +authors = ["Tpt "] +license = "MIT OR Apache-2.0" readme = "README.md" keywords = ["SPARQL"] repository = "https://github.com/oxigraph/oxigraph/tree/main/lib/sparesults" -homepage.workspace = true description = """ SPARQL query results formats parsers and serializers """ documentation = "https://docs.rs/sparesults" -edition.workspace = true -rust-version.workspace = true +edition = "2021" +rust-version = "1.70" [features] default = [] @@ -22,7 +21,7 @@ async-tokio = ["dep:tokio", "quick-xml/async-tokio", "json-event-parser/async-to [dependencies] json-event-parser = "0.2.0-alpha.2" memchr = "2.5" -oxrdf = { version = "0.2.0-alpha.1", path="../oxrdf" } +oxrdf = { version = "0.2.0-alpha.1", path = "../oxrdf" } quick-xml = ">=0.29, <0.32" tokio = { version = "1.29", optional = true, features = ["io-util"] } diff --git a/lib/spargebra/Cargo.toml b/lib/spargebra/Cargo.toml index bee08a35..620bb041 100644 --- a/lib/spargebra/Cargo.toml +++ b/lib/spargebra/Cargo.toml @@ -1,18 +1,17 @@ [package] name = "spargebra" version = "0.3.0-alpha.1" -authors.workspace = true -license.workspace = true +authors = ["Tpt "] +license = "MIT OR Apache-2.0" readme = "README.md" keywords = ["SPARQL"] repository = "https://github.com/oxigraph/oxigraph/tree/main/lib/spargebra" -homepage.workspace = true documentation = "https://docs.rs/spargebra" description = """ A SPARQL parser """ -edition.workspace = true -rust-version.workspace = true +edition = "2021" +rust-version = "1.70" [features] default = [] @@ -25,7 +24,7 @@ peg = "0.8" rand = "0.8" oxiri = "0.2.3-alpha.1" oxilangtag = "0.1" -oxrdf = { version = "0.2.0-alpha.1", path="../oxrdf" } +oxrdf = { version = "0.2.0-alpha.1", path = "../oxrdf" } [lints] workspace = true diff --git a/lib/sparopt/Cargo.toml b/lib/sparopt/Cargo.toml index 8f07afb7..27698646 100644 --- a/lib/sparopt/Cargo.toml +++ b/lib/sparopt/Cargo.toml @@ -1,18 +1,17 @@ [package] name = "sparopt" version = "0.1.0-alpha.1" -authors.workspace = true -license.workspace = true +authors = ["Tpt "] +license = "MIT OR Apache-2.0" readme = "README.md" keywords = ["SPARQL"] repository = "https://github.com/oxigraph/oxigraph/tree/main/lib/sparopt" -homepage.workspace = true documentation = "https://docs.rs/sparopt" description = """ A SPARQL optimizer """ -edition.workspace = true -rust-version.workspace = true +edition = "2021" +rust-version = "1.70" [features] default = [] @@ -21,9 +20,9 @@ sep-0002 = ["spargebra/sep-0002"] sep-0006 = ["spargebra/sep-0006"] [dependencies] -oxrdf = { version = "0.2.0-alpha.1", path="../oxrdf" } +oxrdf = { version = "0.2.0-alpha.1", path = "../oxrdf" } rand = "0.8" -spargebra = { version = "0.3.0-alpha.1", path="../spargebra" } +spargebra = { version = "0.3.0-alpha.1", path = "../spargebra" } [lints] workspace = true diff --git a/lib/sparql-smith/Cargo.toml b/lib/sparql-smith/Cargo.toml index 9682bd79..b18b94df 100644 --- a/lib/sparql-smith/Cargo.toml +++ b/lib/sparql-smith/Cargo.toml @@ -1,18 +1,17 @@ [package] name = "sparql-smith" version = "0.1.0-alpha.5" -authors.workspace = true -license.workspace = true +authors = ["Tpt "] +license = "MIT OR Apache-2.0" readme = "README.md" keywords = ["SPARQL"] repository = "https://github.com/oxigraph/oxigraph/tree/main/lib/sparql-smith" -homepage.workspace = true documentation = "https://docs.rs/sparql-smith" description = """ A SPARQL test cases generator """ -edition.workspace = true -rust-version.workspace = true +edition = "2021" +rust-version = "1.70" [features] default = [] diff --git a/oxrocksdb-sys/Cargo.toml b/oxrocksdb-sys/Cargo.toml index e77bb4cf..c767fb74 100644 --- a/oxrocksdb-sys/Cargo.toml +++ b/oxrocksdb-sys/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "oxrocksdb-sys" -version.workspace = true -authors.workspace = true +version = "0.4.0-alpha.2-dev" +authors = ["Tpt "] license = "GPL-2.0 OR Apache-2.0" repository = "https://github.com/oxigraph/oxigraph/tree/main/oxrocksdb-sys" readme = "README.md" @@ -9,8 +9,8 @@ description = """ Rust bindings for RocksDB for Oxigraph usage. """ documentation = "https://docs.rs/oxrocksdb-sys" -edition.workspace = true -rust-version.workspace = true +edition = "2021" +rust-version = "1.70" build = "build.rs" links = "rocksdb" diff --git a/python/Cargo.toml b/python/Cargo.toml index ef9b5933..4c5607b1 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -1,15 +1,15 @@ [package] name = "pyoxigraph" -version.workspace = true -authors.workspace = true -license.workspace = true +version = "0.4.0-alpha.2-dev" +authors = ["Tpt "] +license = "MIT OR Apache-2.0" readme = "README.md" keywords = ["RDF", "SPARQL", "graph-database", "database"] repository = "https://github.com/oxigraph/oxigraph/tree/main/python" homepage = "https://pyoxigraph.readthedocs.io/" description = "Python bindings of Oxigraph, a SPARQL database and RDF toolkit" -edition.workspace = true -rust-version.workspace = true +edition = "2021" +rust-version = "1.70" publish = false [lib] @@ -25,7 +25,7 @@ rocksdb-pkg-config = ["oxigraph/rocksdb-pkg-config"] rustls = ["oxigraph/http-client-rustls-native"] [dependencies] -oxigraph = { version = "0.4.0-alpha.1", path="../lib" } +oxigraph.path = "../lib" pyo3 = { version = "0.20.1", features = ["extension-module"] } [lints] diff --git a/testsuite/Cargo.toml b/testsuite/Cargo.toml index be1b1ac0..79f6b3da 100644 --- a/testsuite/Cargo.toml +++ b/testsuite/Cargo.toml @@ -1,22 +1,22 @@ [package] name = "oxigraph-testsuite" -version.workspace = true -authors.workspace = true -license.workspace = true +version = "0.4.0-alpha.2-dev" +authors = ["Tpt "] +license = "MIT OR Apache-2.0" description = """ Implementation of W3C testsuites for Oxigraph """ -edition.workspace = true -rust-version.workspace = true +edition = "2021" +rust-version = "1.70" publish = false [dependencies] anyhow = "1.0.72" clap = { version = "4.0", features = ["derive"] } -oxigraph = { path = "../lib" } -oxttl = { path= "../lib/oxttl" } -sparopt = { path = "../lib/sparopt" } -spargebra = { path = "../lib/spargebra" } +oxigraph.path = "../lib" +oxttl.path = "../lib/oxttl" +sparopt.path = "../lib/sparopt" +spargebra.path = "../lib/spargebra" text-diff = "0.4" time = { version = "0.3", features = ["formatting"] } From 93eab63868fc4991e521493d25504d20da298ef4 Mon Sep 17 00:00:00 2001 From: Tpt Date: Thu, 4 Jan 2024 20:25:37 +0100 Subject: [PATCH 165/217] Python: QuerySolution is thread safe --- python/src/sparql.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/src/sparql.rs b/python/src/sparql.rs index 9bce5ab5..c79a6b7e 100644 --- a/python/src/sparql.rs +++ b/python/src/sparql.rs @@ -101,7 +101,7 @@ pub fn query_results_to_python(py: Python<'_>, results: QueryResults) -> PyObjec /// >>> s, p, o = solution /// >>> s /// -#[pyclass(frozen, unsendable, name = "QuerySolution", module = "pyoxigraph")] +#[pyclass(frozen, name = "QuerySolution", module = "pyoxigraph")] pub struct PyQuerySolution { inner: QuerySolution, } From f01796b1a498e53775a717b783dd6f1b2e930491 Mon Sep 17 00:00:00 2001 From: Tpt Date: Fri, 5 Jan 2024 10:28:17 +0100 Subject: [PATCH 166/217] Python: Runs doctests as part of the unittests --- .github/workflows/tests.yml | 2 -- python/src/io.rs | 4 ++++ python/src/store.rs | 1 + python/tests/test_doc.py | 37 +++++++++++++++++++++++++++++++++++++ 4 files changed, 42 insertions(+), 2 deletions(-) create mode 100644 python/tests/test_doc.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 67153706..bff292d5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -266,8 +266,6 @@ jobs: - run: rm -r target/wheels - run: python -m unittest working-directory: ./python/tests - - run: sphinx-build -M doctest . build - working-directory: ./python/docs - run: sphinx-build -M html . build working-directory: ./python/docs - run: python generate_stubs.py pyoxigraph pyoxigraph.pyi --ruff diff --git a/python/src/io.rs b/python/src/io.rs index b8c42443..8258fda2 100644 --- a/python/src/io.rs +++ b/python/src/io.rs @@ -111,6 +111,7 @@ pub fn parse( /// >>> serialize([Triple(NamedNode('http://example.com'), NamedNode('http://example.com/p'), Literal('1'))], format=RdfFormat.TURTLE) /// b' "1" .\n' /// +/// >>> import io /// >>> output = io.BytesIO() /// >>> serialize([Triple(NamedNode('http://example.com'), NamedNode('http://example.com/p'), Literal('1'))], output, RdfFormat.TURTLE) /// >>> output.getvalue() @@ -182,6 +183,9 @@ impl PyQuadReader { /// * `TriG `_ (:py:attr:`RdfFormat.TRIG`) /// * `N3 `_ (:py:attr:`RdfFormat.N3`) /// * `RDF/XML `_ (:py:attr:`RdfFormat.RDF_XML`) +/// +/// >>> RdfFormat.N3.media_type +/// 'text/n3' #[pyclass(name = "RdfFormat", module = "pyoxigraph")] #[derive(Clone)] pub struct PyRdfFormat { diff --git a/python/src/store.rs b/python/src/store.rs index 90b3858c..ccc56e35 100644 --- a/python/src/store.rs +++ b/python/src/store.rs @@ -518,6 +518,7 @@ impl PyStore { /// >>> store.dump(format=RdfFormat.TRIG) /// b' "1" .\n' /// + /// >>> import io /// >>> store = Store() /// >>> store.add(Quad(NamedNode('http://example.com'), NamedNode('http://example.com/p'), Literal('1'), NamedNode('http://example.com/g'))) /// >>> output = io.BytesIO() diff --git a/python/tests/test_doc.py b/python/tests/test_doc.py new file mode 100644 index 00000000..c0ba505b --- /dev/null +++ b/python/tests/test_doc.py @@ -0,0 +1,37 @@ +# type: ignore +import inspect +from doctest import DocTestFinder, DocTestSuite + +import pyoxigraph + + +class ExtendedDocTestFinder(DocTestFinder): + """ + More aggressive doctest lookup + """ + + def _find(self, tests, obj, name, module, source_lines, globs, seen): + # If we've already processed this object, then ignore it. + if id(obj) in seen: + return + seen[id(obj)] = 1 + + # Find a test for this object, and add it to the list of tests. + test = self._get_test(obj, name, module, globs, source_lines) + if test is not None: + tests.append(test) + + # Look for tests in a module's contained objects. + if inspect.ismodule(obj) or inspect.isclass(obj): + for valname, val in obj.__dict__.items(): + if valname == "__doc__": + continue + # Special handling for staticmethod/classmethod. + if isinstance(val, (staticmethod, classmethod)): + val = val.__func__ + self._find(tests, val, f"{name}.{valname}", module, source_lines, globs, seen) + + +def load_tests(_loader, tests, _ignore): + tests.addTests(DocTestSuite(pyoxigraph, test_finder=ExtendedDocTestFinder())) + return tests From d1da94b08b422535f69d5622a016b1216b68899d Mon Sep 17 00:00:00 2001 From: Tpt Date: Fri, 5 Jan 2024 11:32:51 +0100 Subject: [PATCH 167/217] Runs sphinx-lint in the CI --- .github/workflows/tests.yml | 2 ++ python/requirements.dev.txt | 1 + 2 files changed, 3 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index bff292d5..b5156bb7 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -278,6 +278,8 @@ jobs: working-directory: ./python - run: python -m ruff check --output-format=github . working-directory: ./python + - run: sphinx-lint docs + working-directory: ./python python_msv: runs-on: ubuntu-latest diff --git a/python/requirements.dev.txt b/python/requirements.dev.txt index 67eb7d2b..f260a9e9 100644 --- a/python/requirements.dev.txt +++ b/python/requirements.dev.txt @@ -3,3 +3,4 @@ maturin~=1.0 mypy~=1.0 ruff~=0.1.0 sphinx~=7.0 +sphinx-lint~=0.9.1 From 0d23f4ae4807ab531ea11eba509f416e3dc93a84 Mon Sep 17 00:00:00 2001 From: Tpt Date: Fri, 5 Jan 2024 18:01:02 +0100 Subject: [PATCH 168/217] Architecture diagram --- README.md | 23 ++++++-- docs/arch-diagram.svg | 120 ++++++++++++++++++++++++++++++++++++++++++ docs/arch-diagram.txt | 35 ++++++++++++ 3 files changed, 173 insertions(+), 5 deletions(-) create mode 100644 docs/arch-diagram.svg create mode 100644 docs/arch-diagram.txt diff --git a/README.md b/README.md index 6147e9fc..e035c11b 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,12 @@ Oxigraph is in heavy development and SPARQL query evaluation has not been optimi The development roadmap is using [GitHub milestones](https://github.com/oxigraph/oxigraph/milestones?direction=desc&sort=completeness&state=open). Oxigraph internal design [is described on the wiki](https://github.com/oxigraph/oxigraph/wiki/Architecture). +Oxigraph implements the following specifications: + +- [SPARQL 1.1 Query](https://www.w3.org/TR/sparql11-query/), [SPARQL 1.1 Update](https://www.w3.org/TR/sparql11-update/), and [SPARQL 1.1 Federated Query](https://www.w3.org/TR/sparql11-federated-query/). +- [Turtle](https://www.w3.org/TR/turtle/), [TriG](https://www.w3.org/TR/trig/), [N-Triples](https://www.w3.org/TR/n-triples/), [N-Quads](https://www.w3.org/TR/n-quads/), and [RDF XML](https://www.w3.org/TR/rdf-syntax-grammar/) RDF serialization formats for both data ingestion and retrieval using the [Rio library](https://github.com/oxigraph/rio). +- [SPARQL Query Results XML Format](https://www.w3.org/TR/rdf-sparql-XMLres/), [SPARQL 1.1 Query Results JSON Format](https://www.w3.org/TR/sparql11-results-json/) and [SPARQL 1.1 Query Results CSV and TSV Formats](https://www.w3.org/TR/sparql11-results-csv-tsv/). + It is split into multiple parts: - [The database written as a Rust library](https://crates.io/crates/oxigraph). Its source code is in the `lib` directory. @@ -31,11 +37,18 @@ It is split into multiple parts: Note that it was previously named [Oxigraph server](https://crates.io/crates/oxigraph-server). [![Latest Version](https://img.shields.io/crates/v/oxigraph-cli.svg)](https://crates.io/crates/oxigraph-cli) -Oxigraph implements the following specifications: - -- [SPARQL 1.1 Query](https://www.w3.org/TR/sparql11-query/), [SPARQL 1.1 Update](https://www.w3.org/TR/sparql11-update/), and [SPARQL 1.1 Federated Query](https://www.w3.org/TR/sparql11-federated-query/). -- [Turtle](https://www.w3.org/TR/turtle/), [TriG](https://www.w3.org/TR/trig/), [N-Triples](https://www.w3.org/TR/n-triples/), [N-Quads](https://www.w3.org/TR/n-quads/), and [RDF XML](https://www.w3.org/TR/rdf-syntax-grammar/) RDF serialization formats for both data ingestion and retrieval using the [Rio library](https://github.com/oxigraph/rio). -- [SPARQL Query Results XML Format](https://www.w3.org/TR/rdf-sparql-XMLres/), [SPARQL 1.1 Query Results JSON Format](https://www.w3.org/TR/sparql11-results-json/) and [SPARQL 1.1 Query Results CSV and TSV Formats](https://www.w3.org/TR/sparql11-results-csv-tsv/). +Also, some parts of Oxigraph are available as standalone Rust crates: +* [`oxrdf`](https://crates.io/crates/oxrdf), datastructures encoding RDF basic concepts (the [`oxigraph::model`](crate::model) module). +* [`oxrdfio`](https://crates.io/crates/oxrdfio), a unified parser and serializer API for RDF formats (the [`oxigraph::io`](crate::io) module). It itself relies on: + * [`oxttl`](https://crates.io/crates/oxttl), N-Triple, N-Quad, Turtle, TriG and N3 parsing and serialization. + * [`oxrdfxml`](https://crates.io/crates/oxrdfxml), RDF/XML parsing and serialization. +* [`spargebra`](https://crates.io/crates/spargebra), a SPARQL parser. +* [`sparesults`](https://crates.io/crates/sparesults), parsers and serializers for SPARQL result formats. +* [`sparopt`](https://crates.io/crates/sparesults), a SPARQL optimizer. +* [`oxsdatatypes`](https://crates.io/crates/oxsdatatypes), an implementation of some XML Schema datatypes. + +The library layers in Oxigraph. The elements above depend on the elements below: +![Oxigraph libraries architecture diagram](./docs/arch-diagram.svg) A preliminary benchmark [is provided](bench/README.md). There is also [a document describing Oxigraph technical architecture](https://github.com/oxigraph/oxigraph/wiki/Architecture). diff --git a/docs/arch-diagram.svg b/docs/arch-diagram.svg new file mode 100644 index 00000000..749c466b --- /dev/null +++ b/docs/arch-diagram.svg @@ -0,0 +1,120 @@ + + + + + + + + + + + + + + + + + + + + + + oxigraph CLI + + pyoxigraph + + oxigraph + JS + + oxigraph + + oxrdfio + + sparopt + + oxttl + + oxrdfxml + + spargebra + + sparesults + + oxrdf + + oxsdatatypes + \ No newline at end of file diff --git a/docs/arch-diagram.txt b/docs/arch-diagram.txt new file mode 100644 index 00000000..ff3a5be0 --- /dev/null +++ b/docs/arch-diagram.txt @@ -0,0 +1,35 @@ ++------------------+ +----------------+ +-----------------+ ++ oxigraph CLI {r} + + pyoxigraph {p} + + oxigraph JS {j} + ++------------------+ +----------------+ +-----------------+ + ++---------------------------------------------------------------------------+ ++ oxigraph (Rust) {r} + ++---------------------------------------------------------------------------+ + + +----------------------------+ +-------------+ + + oxrdfio {r} + + sparopt {r} + + +----------------------------+ +-------------+ + + +-----------+ +--------------+ +-----------------+ +----------------+ + + oxttl {r} + + oxrdfxml {r} + + spargebra {r} + + sparesults {r} + + +-----------+ +--------------+ +-----------------+ +----------------+ + + +-----------------------------------------------------------------------+ + + oxrdf {r} + + +-----------------------------------------------------------------------+ + ++------------------+ ++ oxsdatatypes {r} + ++------------------+ + + +# Legend: +r = { + fill: papayawhip; +} +p = { + fill: lightyellow; +} +j = { + fill: lightgreen; +} From dcabf50ab62c6c91bad45cbe5e7686d6f6c8c44f Mon Sep 17 00:00:00 2001 From: Tpt Date: Mon, 8 Jan 2024 09:18:47 +0100 Subject: [PATCH 169/217] Fixes Docker image Issue #723 --- cli/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/Dockerfile b/cli/Dockerfile index 82f4bfe8..004f7525 100644 --- a/cli/Dockerfile +++ b/cli/Dockerfile @@ -19,7 +19,7 @@ RUN if [ "$BUILDARCH" != "$TARGETARCH" ] && [ "$TARGETARCH" = "arm64" ] ; \ cargo build --release --no-default-features --features rustls-webpki ; \ fi -FROM --platform=$TARGETPLATFORM gcr.io/distroless/cc-debian11 +FROM --platform=$TARGETPLATFORM gcr.io/distroless/cc-debian12 COPY --from=builder /oxigraph/target/release/oxigraph /usr/local/bin/oxigraph ENTRYPOINT [ "/usr/local/bin/oxigraph" ] CMD [ "serve", "--location", "/data", "--bind", "0.0.0.0:7878" ] From d4eaa3c5eff4b0cfd77fb6a43fb3a4b70834cc46 Mon Sep 17 00:00:00 2001 From: Tpt Date: Mon, 8 Jan 2024 09:41:12 +0100 Subject: [PATCH 170/217] Docker: use semver tagging --- .github/workflows/artifacts.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/artifacts.yml b/.github/workflows/artifacts.yml index 387b67be..c23dfc35 100644 --- a/.github/workflows/artifacts.yml +++ b/.github/workflows/artifacts.yml @@ -337,6 +337,11 @@ jobs: images: | ${{ github.repository }},enable=${{ github.event_name == 'release' }} ghcr.io/${{ github.repository }} + tags: | + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} - uses: docker/build-push-action@v4 with: context: . From 5cf8025aecd70222068174540c6045297c252c42 Mon Sep 17 00:00:00 2001 From: Tpt Date: Mon, 8 Jan 2024 10:12:48 +0100 Subject: [PATCH 171/217] CI: Fuzzer: Removes unused clone --- .github/workflows/tests.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b5156bb7..fe8fa119 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -361,9 +361,6 @@ jobs: if: github.event_name == 'pull_request' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - submodules: true - uses: google/clusterfuzzlite/actions/build_fuzzers@v1 with: language: rust @@ -384,9 +381,6 @@ jobs: if: github.event_name != 'pull_request' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - submodules: true - uses: google/clusterfuzzlite/actions/build_fuzzers@v1 with: language: rust From cffc536eb9df85f4ff474923f4498b25df07c3be Mon Sep 17 00:00:00 2001 From: Tpt Date: Mon, 8 Jan 2024 09:57:52 +0100 Subject: [PATCH 172/217] Releases v0.4.0-alpha.2 --- CHANGELOG.md | 11 ++++++ Cargo.lock | 72 ++++++++++++++++++++-------------------- cli/Cargo.toml | 4 +-- js/Cargo.toml | 2 +- lib/Cargo.toml | 4 +-- oxrocksdb-sys/Cargo.toml | 2 +- python/Cargo.toml | 2 +- testsuite/Cargo.toml | 2 +- 8 files changed, 55 insertions(+), 44 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5dba40d1..2da6d30a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,14 @@ +## [0.4.0-alpha.2] - 2024-01-08 + +### Added +- i686 linux support + +### Changed +- Docker: fixes Docker image Glib version error. +- Docker: tags now use semver e.g. `0.3.22` and not `v0.3.22`. Preversions are also not tagged `latest` anymore. +- Python: `QuerySolution` is now thread safe. + + ## [0.4.0-alpha.1] - 2024-01-03 ### Added diff --git a/Cargo.lock b/Cargo.lock index 96f7d6fa..652ddf86 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -290,9 +290,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.12" +version = "4.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcfab8ba68f3668e89f6ff60f5b205cea56aa7b769451a59f34b8682f51c056d" +checksum = "52bdc885e4cacc7f7c9eedc1ef6da641603180c783c41a15c264944deeaab642" dependencies = [ "clap_builder", "clap_derive", @@ -394,9 +394,9 @@ checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" [[package]] name = "cpufeatures" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce420fe07aecd3e67c5f910618fe65e94158f6dcc0adf44e00d69ce2bdfe0fd0" +checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" dependencies = [ "libc", ] @@ -725,9 +725,9 @@ dependencies = [ [[package]] name = "ignore" -version = "0.4.21" +version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "747ad1b4ae841a78e8aba0d63adbfbeaea26b517b63705d47856b73015d27060" +checksum = "b46810df39e66e925525d6e38ce1e7f6e1d208f72dc39757880fcb66e2c58af1" dependencies = [ "crossbeam-deque", "globset", @@ -831,9 +831,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.151" +version = "0.2.152" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4" +checksum = "13e3bf6590cbc649f4d1a3eefc9d5d6eb746f5200ffb04e5e142700b8faa56e7" [[package]] name = "libloading" @@ -1033,7 +1033,7 @@ dependencies = [ [[package]] name = "oxigraph" -version = "0.4.0-alpha.2-dev" +version = "0.4.0-alpha.2" dependencies = [ "codspeed-criterion-compat", "digest", @@ -1063,7 +1063,7 @@ dependencies = [ [[package]] name = "oxigraph-cli" -version = "0.4.0-alpha.2-dev" +version = "0.4.0-alpha.2" dependencies = [ "anyhow", "assert_cmd", @@ -1082,7 +1082,7 @@ dependencies = [ [[package]] name = "oxigraph-js" -version = "0.4.0-alpha.2-dev" +version = "0.4.0-alpha.2" dependencies = [ "console_error_panic_hook", "js-sys", @@ -1092,7 +1092,7 @@ dependencies = [ [[package]] name = "oxigraph-testsuite" -version = "0.4.0-alpha.2-dev" +version = "0.0.0" dependencies = [ "anyhow", "clap", @@ -1150,7 +1150,7 @@ dependencies = [ [[package]] name = "oxrocksdb-sys" -version = "0.4.0-alpha.2-dev" +version = "0.4.0-alpha.2" dependencies = [ "bindgen", "cc", @@ -1333,18 +1333,18 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.74" +version = "1.0.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2de98502f212cfcea8d0bb305bd0f49d7ebdd75b64ba0a68f937d888f4e0d6db" +checksum = "95fc56cda0b5c3325f5fbbd7ff9fda9e02bb00bb3dac51252d2f1bfa1cb8cc8c" dependencies = [ "unicode-ident", ] [[package]] name = "pyo3" -version = "0.20.1" +version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e82ad98ce1991c9c70c3464ba4187337b9c45fcbbb060d46dca15f0c075e14e2" +checksum = "9a89dc7a5850d0e983be1ec2a463a171d20990487c3cfcd68b5363f1ee3d6fe0" dependencies = [ "cfg-if", "indoc", @@ -1359,9 +1359,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.20.1" +version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5503d0b3aee2c7a8dbb389cd87cd9649f675d4c7f60ca33699a3e3859d81a891" +checksum = "07426f0d8fe5a601f26293f300afd1a7b1ed5e78b2a705870c5f30893c5163be" dependencies = [ "once_cell", "target-lexicon", @@ -1369,9 +1369,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.20.1" +version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18a79e8d80486a00d11c0dcb27cd2aa17c022cc95c677b461f01797226ba8f41" +checksum = "dbb7dec17e17766b46bca4f1a4215a85006b4c2ecde122076c562dd058da6cf1" dependencies = [ "libc", "pyo3-build-config", @@ -1379,9 +1379,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.20.1" +version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f4b0dc7eaa578604fab11c8c7ff8934c71249c61d4def8e272c76ed879f03d4" +checksum = "05f738b4e40d50b5711957f142878cfa0f28e054aa0ebdfc3fd137a843f74ed3" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -1391,9 +1391,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.20.1" +version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "816a4f709e29ddab2e3cdfe94600d554c5556cad0ddfeea95c47b580c3247fa4" +checksum = "0fc910d4851847827daf9d6cdd4a823fbdaab5b8818325c5e97a86da79e8881f" dependencies = [ "heck", "proc-macro2", @@ -1403,7 +1403,7 @@ dependencies = [ [[package]] name = "pyoxigraph" -version = "0.4.0-alpha.2-dev" +version = "0.4.0-alpha.2" dependencies = [ "oxigraph", "pyo3", @@ -1600,9 +1600,9 @@ checksum = "9e9d979b3ce68192e42760c7810125eb6cf2ea10efae545a156063e61f314e2a" [[package]] name = "rustls-webpki" -version = "0.102.0" +version = "0.102.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de2635c8bc2b88d367767c5de8ea1d8db9af3f6219eba28442242d9ab81d1b89" +checksum = "ef4ca26037c909dedb327b48c3327d0ba91d3dd3c4e05dad328f210ffb68e95b" dependencies = [ "ring", "rustls-pki-types", @@ -1664,18 +1664,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.194" +version = "1.0.195" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b114498256798c94a0689e1a15fec6005dee8ac1f41de56404b67afc2a4b773" +checksum = "63261df402c67811e9ac6def069e4786148c4563f4b50fd4bf30aa370d626b02" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.194" +version = "1.0.195" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3385e45322e8f9931410f01b3031ec534c3947d0e94c18049af4d9f9907d4e0" +checksum = "46fe8f8603d81ba86327b23a2e9cdf49e1255fb94a4c5f297f6ee0547178ea2c" dependencies = [ "proc-macro2", "quote", @@ -1684,9 +1684,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.110" +version = "1.0.111" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fbd975230bada99c8bb618e0c365c2eefa219158d5c6c29610fd09ff1833257" +checksum = "176e46fa42316f18edd598015a5166857fc835ec732f5215eac6b7bdbf0a84f4" dependencies = [ "itoa", "ryu", @@ -1791,9 +1791,9 @@ checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" [[package]] name = "syn" -version = "2.0.46" +version = "2.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89456b690ff72fddcecf231caedbe615c59480c93358a93dfae7fc29e3ebbf0e" +checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" dependencies = [ "proc-macro2", "quote", diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 6953a318..7cea59b8 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "oxigraph-cli" -version = "0.4.0-alpha.2-dev" +version = "0.4.0-alpha.2" authors = ["Tpt "] license = "MIT OR Apache-2.0" readme = "README.md" @@ -29,7 +29,7 @@ rustls-webpki = ["oxigraph/http-client-rustls-webpki"] anyhow = "1.0.72" oxhttp = { version = "0.2.0-alpha.3", features = ["flate2"] } clap = { version = "4.0", features = ["derive"] } -oxigraph = { version = "0.4.0-alpha.2-dev", path = "../lib" } +oxigraph = { version = "0.4.0-alpha.2", path = "../lib" } rand = "0.8" url = "2.4" oxiri = "0.2.3-alpha.1" diff --git a/js/Cargo.toml b/js/Cargo.toml index 727b15f7..6169004f 100644 --- a/js/Cargo.toml +++ b/js/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "oxigraph-js" -version = "0.4.0-alpha.2-dev" +version = "0.4.0-alpha.2" authors = ["Tpt "] license = "MIT OR Apache-2.0" readme = "README.md" diff --git a/lib/Cargo.toml b/lib/Cargo.toml index 9ea03887..944ecc62 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "oxigraph" -version = "0.4.0-alpha.2-dev" +version = "0.4.0-alpha.2" authors = ["Tpt "] license = "MIT OR Apache-2.0" readme = "README.md" @@ -46,7 +46,7 @@ sparopt = { version = "0.1.0-alpha.1", path = "sparopt", features = ["rdf-star", [target.'cfg(not(target_family = "wasm"))'.dependencies] libc = "0.2.147" -oxrocksdb-sys = { version = "0.4.0-alpha.2-dev", path = "../oxrocksdb-sys" } +oxrocksdb-sys = { version = "0.4.0-alpha.2", path = "../oxrocksdb-sys" } oxhttp = { version = "0.2.0-alpha.3", optional = true } [target.'cfg(all(target_family = "wasm", target_os = "unknown"))'.dependencies] diff --git a/oxrocksdb-sys/Cargo.toml b/oxrocksdb-sys/Cargo.toml index c767fb74..1b0196a3 100644 --- a/oxrocksdb-sys/Cargo.toml +++ b/oxrocksdb-sys/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "oxrocksdb-sys" -version = "0.4.0-alpha.2-dev" +version = "0.4.0-alpha.2" authors = ["Tpt "] license = "GPL-2.0 OR Apache-2.0" repository = "https://github.com/oxigraph/oxigraph/tree/main/oxrocksdb-sys" diff --git a/python/Cargo.toml b/python/Cargo.toml index 4c5607b1..3f386aa9 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyoxigraph" -version = "0.4.0-alpha.2-dev" +version = "0.4.0-alpha.2" authors = ["Tpt "] license = "MIT OR Apache-2.0" readme = "README.md" diff --git a/testsuite/Cargo.toml b/testsuite/Cargo.toml index 79f6b3da..1ddccd11 100644 --- a/testsuite/Cargo.toml +++ b/testsuite/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "oxigraph-testsuite" -version = "0.4.0-alpha.2-dev" +version = "0.0.0" authors = ["Tpt "] license = "MIT OR Apache-2.0" description = """ From c2df0b829d8528218ee77733c0ecf49379fb37cc Mon Sep 17 00:00:00 2001 From: Tpt Date: Mon, 8 Jan 2024 13:41:10 +0100 Subject: [PATCH 173/217] CI: Uploads all Python wheels in one batch Avoids partial releases in Pypi --- .github/workflows/artifacts.yml | 75 ++++++++++++--------------------- 1 file changed, 27 insertions(+), 48 deletions(-) diff --git a/.github/workflows/artifacts.yml b/.github/workflows/artifacts.yml index c23dfc35..d4fe4fb0 100644 --- a/.github/workflows/artifacts.yml +++ b/.github/workflows/artifacts.yml @@ -111,11 +111,6 @@ jobs: python_sdist: runs-on: ubuntu-latest - environment: - name: pypi - url: https://pypi.org/p/pyoxigraph - permissions: - id-token: write steps: - uses: actions/checkout@v3 with: @@ -135,20 +130,11 @@ jobs: - run: maturin sdist -m python/Cargo.toml - uses: actions/upload-artifact@v3 with: - name: pyoxigraph_source + name: pyoxigraph_dist path: target/wheels/*.tar.gz - - uses: pypa/gh-action-pypi-publish@release/v1 - with: - packages-dir: target/wheels - if: github.event_name == 'release' wheel_linux: runs-on: ubuntu-latest - environment: - name: pypi - url: https://pypi.org/p/pyoxigraph - permissions: - id-token: write strategy: matrix: architecture: [ "x86_64", "aarch64" ] @@ -167,20 +153,11 @@ jobs: if: github.event_name == 'release' || matrix.architecture == 'x86_64' - uses: actions/upload-artifact@v3 with: - name: pyoxigraph_wheel_linux_gnu + name: pyoxigraph_dist path: target/wheels/*.whl - - uses: pypa/gh-action-pypi-publish@release/v1 - with: - packages-dir: target/wheels - if: github.event_name == 'release' wheel_linux_musl: runs-on: ubuntu-latest - environment: - name: pypi - url: https://pypi.org/p/pyoxigraph - permissions: - id-token: write strategy: matrix: architecture: [ "x86_64", "aarch64" ] @@ -199,20 +176,11 @@ jobs: if: github.event_name == 'release' || matrix.architecture == 'x86_64' - uses: actions/upload-artifact@v3 with: - name: pyoxigraph_wheel_linux_musl + name: pyoxigraph_dist path: target/wheels/*.whl - - uses: pypa/gh-action-pypi-publish@release/v1 - with: - packages-dir: target/wheels - if: github.event_name == 'release' wheel_mac: runs-on: macos-latest - environment: - name: pypi - url: https://pypi.org/p/pyoxigraph - permissions: - id-token: write env: DEVELOPER_DIR: '/Applications/Xcode.app/Contents/Developer' SDKROOT: '/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk' @@ -246,20 +214,11 @@ jobs: if: github.event_name == 'release' - uses: actions/upload-artifact@v3 with: - name: pyoxigraph_wheel_mac + name: pyoxigraph_dist path: target/wheels/*.whl - - uses: pypa/gh-action-pypi-publish@release/v1 - with: - packages-dir: target/wheels - if: github.event_name == 'release' wheel_windows: runs-on: windows-latest - environment: - name: pypi - url: https://pypi.org/p/pyoxigraph - permissions: - id-token: write steps: - uses: actions/checkout@v3 with: @@ -281,12 +240,32 @@ jobs: - run: maturin build --release -m python/Cargo.toml --features abi3 - uses: actions/upload-artifact@v3 with: - name: pyoxigraph_wheel_windows + name: pyoxigraph_dist path: target/wheels/*.whl + + publish_pypi: + if: github.event_name == 'release' + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/p/pyoxigraph + permissions: + id-token: write + needs: + - python_sdist + - wheel_windows + - wheel_mac + - wheel_linux + - wheel_linux_musl + steps: + - uses: actions/download-artifact@v3 + with: + name: pyoxigraph_dist + path: dist - uses: pypa/gh-action-pypi-publish@release/v1 with: - packages-dir: target/wheels - if: github.event_name == 'release' + packages-dir: dist + skip-existing: true npm_tarball: runs-on: ubuntu-latest From c2040a30fda045f901846a2538aa968c0c82913b Mon Sep 17 00:00:00 2001 From: Tpt Date: Fri, 12 Jan 2024 21:25:47 +0100 Subject: [PATCH 174/217] oxttl: Exposes the base IRI Issue #730 --- .github/workflows/artifacts.yml | 2 +- .github/workflows/tests.yml | 8 ++-- lib/oxttl/src/n3.rs | 84 +++++++++++++++++++++++++++++++++ lib/oxttl/src/terse.rs | 2 +- lib/oxttl/src/toolkit/parser.rs | 9 ++-- lib/oxttl/src/trig.rs | 84 +++++++++++++++++++++++++++++++++ lib/oxttl/src/turtle.rs | 84 +++++++++++++++++++++++++++++++++ lib/src/sparql/eval.rs | 1 + lib/src/storage/small_string.rs | 2 +- 9 files changed, 265 insertions(+), 11 deletions(-) diff --git a/.github/workflows/artifacts.yml b/.github/workflows/artifacts.yml index d4fe4fb0..05579293 100644 --- a/.github/workflows/artifacts.yml +++ b/.github/workflows/artifacts.yml @@ -24,7 +24,7 @@ jobs: with: target: aarch64-unknown-linux-gnu - run: | - sudo apt-get install -y g++-aarch64-linux-gnu + sudo apt-get update && sudo apt-get install -y g++-aarch64-linux-gnu mkdir .cargo echo -e "[target.aarch64-unknown-linux-gnu]\nlinker = \"aarch64-linux-gnu-gcc\"" >> .cargo/config.toml - run: cargo build --release --no-default-features --features rustls-native diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index fe8fa119..689bb227 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -140,7 +140,7 @@ jobs: - uses: ./.github/actions/setup-rust with: target: i686-unknown-linux-gnu - - run: sudo apt-get install -y g++-multilib + - run: sudo apt-get update && sudo apt-get install -y g++-multilib - run: cargo test --target i686-unknown-linux-gnu --no-default-features --features http-client-rustls-native working-directory: ./lib @@ -176,7 +176,7 @@ jobs: - uses: ./.github/actions/setup-rust with: version: nightly - - run: sudo apt-get install -y llvm + - run: sudo apt-get update && sudo apt-get install -y llvm - run: cargo test --tests --target x86_64-unknown-linux-gnu --workspace --exclude pyoxigraph --exclude oxigraph-testsuite --exclude oxigraph-cli env: RUSTFLAGS: -Z sanitizer=address @@ -354,7 +354,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - run: sudo apt-get install -y clang-format + - run: sudo apt-get update && sudo apt-get install -y clang-format - run: clang-format --Werror --dry-run oxrocksdb-sys/api/* fuzz_changes: @@ -425,7 +425,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - run: sudo apt-get install -y shellcheck + - run: sudo apt-get update && sudo apt-get install -y shellcheck - run: git grep -l '^#\( *shellcheck \|!\(/bin/\|/usr/bin/env \)\(sh\|bash\|dash\|ksh\)\)' | xargs shellcheck spec_links: diff --git a/lib/oxttl/src/n3.rs b/lib/oxttl/src/n3.rs index 72db1611..0642d416 100644 --- a/lib/oxttl/src/n3.rs +++ b/lib/oxttl/src/n3.rs @@ -418,6 +418,33 @@ impl FromReadN3Reader { pub fn prefixes(&self) -> &HashMap> { &self.inner.parser.context.prefixes } + + /// The base IRI considered at the current step of the parsing. + /// + /// ``` + /// use oxttl::N3Parser; + /// + /// let file = b"@base . + /// @prefix schema: . + /// a schema:Person ; + /// schema:name \"Foo\" ."; + /// + /// let mut reader = N3Parser::new().parse_read(file.as_ref()); + /// assert!(reader.base_iri().is_none()); // No base at the beginning because none has been given to the parser. + /// + /// reader.next().unwrap()?; // We read the first triple + /// assert_eq!(reader.base_iri(), Some("http://example.com/")); // There is now a base IRI. + /// # Result::<_,Box>::Ok(()) + /// ``` + pub fn base_iri(&self) -> Option<&str> { + self.inner + .parser + .context + .lexer_options + .base_iri + .as_ref() + .map(Iri::as_str) + } } impl Iterator for FromReadN3Reader { @@ -498,6 +525,36 @@ impl FromTokioAsyncReadN3Reader { pub fn prefixes(&self) -> &HashMap> { &self.inner.parser.context.prefixes } + + /// The base IRI considered at the current step of the parsing. + /// + /// ``` + /// use oxttl::N3Parser; + /// + /// # #[tokio::main(flavor = "current_thread")] + /// # async fn main() -> Result<(), oxttl::ParseError> { + /// let file = b"@base . + /// @prefix schema: . + /// a schema:Person ; + /// schema:name \"Foo\" ."; + /// + /// let mut reader = N3Parser::new().parse_tokio_async_read(file.as_ref()); + /// assert!(reader.base_iri().is_none()); // No base IRI at the beginning + /// + /// reader.next().await.unwrap()?; // We read the first triple + /// assert_eq!(reader.base_iri(), Some("http://example.com/")); // There is now a base IRI + /// # Ok(()) + /// # } + /// ``` + pub fn base_iri(&self) -> Option<&str> { + self.inner + .parser + .context + .lexer_options + .base_iri + .as_ref() + .map(Iri::as_str) + } } /// Parses a N3 file by using a low-level API. Can be built using [`N3Parser::parse`]. @@ -592,6 +649,33 @@ impl LowLevelN3Reader { pub fn prefixes(&self) -> &HashMap> { &self.parser.context.prefixes } + + /// The base IRI considered at the current step of the parsing. + /// + /// ``` + /// use oxttl::N3Parser; + /// + /// let file = b"@base . + /// @prefix schema: . + /// a schema:Person ; + /// schema:name \"Foo\" ."; + /// + /// let mut reader = N3Parser::new().parse(); + /// reader.extend_from_slice(file); + /// assert!(reader.base_iri().is_none()); // No base IRI at the beginning + /// + /// reader.read_next().unwrap()?; // We read the first triple + /// assert_eq!(reader.base_iri(), Some("http://example.com/")); // There is now a base IRI + /// # Result::<_,Box>::Ok(()) + /// ``` + pub fn base_iri(&self) -> Option<&str> { + self.parser + .context + .lexer_options + .base_iri + .as_ref() + .map(Iri::as_str) + } } #[derive(Clone)] diff --git a/lib/oxttl/src/terse.rs b/lib/oxttl/src/terse.rs index 6c83fb71..bc092c1f 100644 --- a/lib/oxttl/src/terse.rs +++ b/lib/oxttl/src/terse.rs @@ -22,7 +22,7 @@ pub struct TriGRecognizer { #[allow(clippy::partial_pub_fields)] pub struct TriGRecognizerContext { - lexer_options: N3LexerOptions, + pub lexer_options: N3LexerOptions, pub with_graph_name: bool, #[cfg(feature = "rdf-star")] pub with_quoted_triples: bool, diff --git a/lib/oxttl/src/toolkit/parser.rs b/lib/oxttl/src/toolkit/parser.rs index 7af93752..6314640d 100644 --- a/lib/oxttl/src/toolkit/parser.rs +++ b/lib/oxttl/src/toolkit/parser.rs @@ -110,10 +110,11 @@ impl Parser { } } if self.lexer.is_end() { - let Some(state) = self.state.take() else { - return None; - }; - state.recognize_end(&mut self.context, &mut self.results, &mut self.errors) + self.state.take()?.recognize_end( + &mut self.context, + &mut self.results, + &mut self.errors, + ) } else { return None; } diff --git a/lib/oxttl/src/trig.rs b/lib/oxttl/src/trig.rs index e97cdce3..70d3edb6 100644 --- a/lib/oxttl/src/trig.rs +++ b/lib/oxttl/src/trig.rs @@ -268,6 +268,33 @@ impl FromReadTriGReader { pub fn prefixes(&self) -> &HashMap> { &self.inner.parser.context.prefixes } + + /// The base IRI considered at the current step of the parsing. + /// + /// ``` + /// use oxttl::TriGParser; + /// + /// let file = b"@base . + /// @prefix schema: . + /// a schema:Person ; + /// schema:name \"Foo\" ."; + /// + /// let mut reader = TriGParser::new().parse_read(file.as_ref()); + /// assert!(reader.base_iri().is_none()); // No base at the beginning because none has been given to the parser. + /// + /// reader.next().unwrap()?; // We read the first triple + /// assert_eq!(reader.base_iri(), Some("http://example.com/")); // There is now a base IRI. + /// # Result::<_,Box>::Ok(()) + /// ``` + pub fn base_iri(&self) -> Option<&str> { + self.inner + .parser + .context + .lexer_options + .base_iri + .as_ref() + .map(Iri::as_str) + } } impl Iterator for FromReadTriGReader { @@ -347,6 +374,36 @@ impl FromTokioAsyncReadTriGReader { pub fn prefixes(&self) -> &HashMap> { &self.inner.parser.context.prefixes } + + /// The base IRI considered at the current step of the parsing. + /// + /// ``` + /// use oxttl::TriGParser; + /// + /// # #[tokio::main(flavor = "current_thread")] + /// # async fn main() -> Result<(), oxttl::ParseError> { + /// let file = b"@base . + /// @prefix schema: . + /// a schema:Person ; + /// schema:name \"Foo\" ."; + /// + /// let mut reader = TriGParser::new().parse_tokio_async_read(file.as_ref()); + /// assert!(reader.base_iri().is_none()); // No base IRI at the beginning + /// + /// reader.next().await.unwrap()?; // We read the first triple + /// assert_eq!(reader.base_iri(), Some("http://example.com/")); // There is now a base IRI + /// # Ok(()) + /// # } + /// ``` + pub fn base_iri(&self) -> Option<&str> { + self.inner + .parser + .context + .lexer_options + .base_iri + .as_ref() + .map(Iri::as_str) + } } /// Parses a TriG file by using a low-level API. Can be built using [`TriGParser::parse`]. @@ -440,6 +497,33 @@ impl LowLevelTriGReader { pub fn prefixes(&self) -> &HashMap> { &self.parser.context.prefixes } + + /// The base IRI considered at the current step of the parsing. + /// + /// ``` + /// use oxttl::TriGParser; + /// + /// let file = b"@base . + /// @prefix schema: . + /// a schema:Person ; + /// schema:name \"Foo\" ."; + /// + /// let mut reader = TriGParser::new().parse(); + /// reader.extend_from_slice(file); + /// assert!(reader.base_iri().is_none()); // No base IRI at the beginning + /// + /// reader.read_next().unwrap()?; // We read the first triple + /// assert_eq!(reader.base_iri(), Some("http://example.com/")); // There is now a base IRI + /// # Result::<_,Box>::Ok(()) + /// ``` + pub fn base_iri(&self) -> Option<&str> { + self.parser + .context + .lexer_options + .base_iri + .as_ref() + .map(Iri::as_str) + } } /// A [TriG](https://www.w3.org/TR/trig/) serializer. diff --git a/lib/oxttl/src/turtle.rs b/lib/oxttl/src/turtle.rs index ca0eedb1..0e225611 100644 --- a/lib/oxttl/src/turtle.rs +++ b/lib/oxttl/src/turtle.rs @@ -270,6 +270,33 @@ impl FromReadTurtleReader { pub fn prefixes(&self) -> &HashMap> { &self.inner.parser.context.prefixes } + + /// The base IRI considered at the current step of the parsing. + /// + /// ``` + /// use oxttl::TurtleParser; + /// + /// let file = b"@base . + /// @prefix schema: . + /// a schema:Person ; + /// schema:name \"Foo\" ."; + /// + /// let mut reader = TurtleParser::new().parse_read(file.as_ref()); + /// assert!(reader.base_iri().is_none()); // No base at the beginning because none has been given to the parser. + /// + /// reader.next().unwrap()?; // We read the first triple + /// assert_eq!(reader.base_iri(), Some("http://example.com/")); // There is now a base IRI. + /// # Result::<_,Box>::Ok(()) + /// ``` + pub fn base_iri(&self) -> Option<&str> { + self.inner + .parser + .context + .lexer_options + .base_iri + .as_ref() + .map(Iri::as_str) + } } impl Iterator for FromReadTurtleReader { @@ -349,6 +376,36 @@ impl FromTokioAsyncReadTurtleReader { pub fn prefixes(&self) -> &HashMap> { &self.inner.parser.context.prefixes } + + /// The base IRI considered at the current step of the parsing. + /// + /// ``` + /// use oxttl::TurtleParser; + /// + /// # #[tokio::main(flavor = "current_thread")] + /// # async fn main() -> Result<(), oxttl::ParseError> { + /// let file = b"@base . + /// @prefix schema: . + /// a schema:Person ; + /// schema:name \"Foo\" ."; + /// + /// let mut reader = TurtleParser::new().parse_tokio_async_read(file.as_ref()); + /// assert!(reader.base_iri().is_none()); // No base IRI at the beginning + /// + /// reader.next().await.unwrap()?; // We read the first triple + /// assert_eq!(reader.base_iri(), Some("http://example.com/")); // There is now a base IRI + /// # Ok(()) + /// # } + /// ``` + pub fn base_iri(&self) -> Option<&str> { + self.inner + .parser + .context + .lexer_options + .base_iri + .as_ref() + .map(Iri::as_str) + } } /// Parses a Turtle file by using a low-level API. Can be built using [`TurtleParser::parse`]. @@ -442,6 +499,33 @@ impl LowLevelTurtleReader { pub fn prefixes(&self) -> &HashMap> { &self.parser.context.prefixes } + + /// The base IRI considered at the current step of the parsing. + /// + /// ``` + /// use oxttl::TurtleParser; + /// + /// let file = b"@base . + /// @prefix schema: . + /// a schema:Person ; + /// schema:name \"Foo\" ."; + /// + /// let mut reader = TurtleParser::new().parse(); + /// reader.extend_from_slice(file); + /// assert!(reader.base_iri().is_none()); // No base IRI at the beginning + /// + /// reader.read_next().unwrap()?; // We read the first triple + /// assert_eq!(reader.base_iri(), Some("http://example.com/")); // There is now a base IRI + /// # Result::<_,Box>::Ok(()) + /// ``` + pub fn base_iri(&self) -> Option<&str> { + self.parser + .context + .lexer_options + .base_iri + .as_ref() + .map(Iri::as_str) + } } /// A [Turtle](https://www.w3.org/TR/turtle/) serializer. diff --git a/lib/src/sparql/eval.rs b/lib/src/sparql/eval.rs index 9f317d11..9ae02da1 100644 --- a/lib/src/sparql/eval.rs +++ b/lib/src/sparql/eval.rs @@ -1076,6 +1076,7 @@ impl SimpleEvaluator { inner: spargebra::Query::Select { dataset: None, pattern: graph_pattern.clone(), + #[allow(clippy::useless_asref)] base_iri: self.base_iri.as_ref().map(|iri| iri.as_ref().clone()), }, dataset: QueryDataset::new(), diff --git a/lib/src/storage/small_string.rs b/lib/src/storage/small_string.rs index d5d18987..be836c4d 100644 --- a/lib/src/storage/small_string.rs +++ b/lib/src/storage/small_string.rs @@ -101,7 +101,7 @@ impl fmt::Display for SmallString { impl PartialEq for SmallString { #[inline] fn eq(&self, other: &Self) -> bool { - self.as_str().eq(&**other) + self.as_str() == other.as_str() } } From b08c2010748af4920a818e59e443b6c95939fad0 Mon Sep 17 00:00:00 2001 From: Tpt Date: Sun, 14 Jan 2024 18:10:10 +0100 Subject: [PATCH 175/217] CI: attempt to fix debian compatibility test --- lints/test_debian_compatibility.py | 35 +++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/lints/test_debian_compatibility.py b/lints/test_debian_compatibility.py index ef00126f..6823ac18 100644 --- a/lints/test_debian_compatibility.py +++ b/lints/test_debian_compatibility.py @@ -1,11 +1,20 @@ import json import subprocess from pathlib import Path +from time import sleep +from urllib.error import HTTPError from urllib.request import urlopen TARGET_DEBIAN_VERSIONS = ["sid"] IGNORE_PACKAGES = {"oxigraph-js", "oxigraph-testsuite", "pyoxigraph", "sparql-smith"} -ALLOWED_MISSING_PACKAGES = {"codspeed-criterion-compat", "escargot", "json-event-parser", "oxhttp", "oxiri", "quick-xml"} +ALLOWED_MISSING_PACKAGES = { + "codspeed-criterion-compat", + "escargot", + "json-event-parser", + "oxhttp", + "oxiri", + "quick-xml", +} base_path = Path(__file__).parent.parent @@ -25,6 +34,22 @@ def parse_version(version): return tuple(int(e) for e in version.split("-")[0].split(".")) +def fetch_debian_package_desc(debian_name): + url = f"https://sources.debian.org/api/src/{debian_name}/" + for i in range(0, 10): + try: + with urlopen(url) as response: + return json.loads(response.read().decode()) + except HTTPError as e: + if e.code / 100 == 5: + wait = 2**i + print(f"Error {e} from {url}, retrying after {wait}s") + sleep(wait) + else: + raise e + raise Exception(f"Failed to fetch {url}") + + for package_id in cargo_metadata["workspace_default_members"]: package = package_by_id[package_id] if package["name"] in IGNORE_PACKAGES: @@ -37,11 +62,9 @@ for package_id in cargo_metadata["workspace_default_members"]: continue candidate_debian_name = f"rust-{dependency['name'].replace('_', '-')}" if dependency["name"] not in debian_cache: - with urlopen( - f"https://sources.debian.org/api/src/{candidate_debian_name}/" - ) as response: - debian_package = json.loads(response.read().decode()) - debian_cache[candidate_debian_name] = debian_package + debian_cache[candidate_debian_name] = fetch_debian_package_desc( + candidate_debian_name + ) debian_package = debian_cache[candidate_debian_name] if "error" in debian_package: errors.add(f"No Debian package found for {dependency['name']}") From df040400c5c1b6dee7fc07cfca5b2ee236411c90 Mon Sep 17 00:00:00 2001 From: Tpt Date: Sat, 13 Jan 2024 17:33:08 +0100 Subject: [PATCH 176/217] Turtle: fixes parsing bug with escaped dot at the end of a local name --- Cargo.lock | 14 +++++------ cli/Cargo.toml | 4 ++-- js/Cargo.toml | 2 +- lib/Cargo.toml | 6 ++--- lib/oxrdfio/Cargo.toml | 4 ++-- lib/oxttl/Cargo.toml | 2 +- lib/oxttl/src/lexer.rs | 23 +++++++++++-------- oxrocksdb-sys/Cargo.toml | 2 +- python/Cargo.toml | 2 +- .../parser/escaped_trailing_dot.nq | 1 + .../parser/escaped_trailing_dot.nt | 1 + .../parser/escaped_trailing_dot.trig | 2 ++ .../parser/escaped_trailing_dot.ttl | 2 ++ testsuite/oxigraph-tests/parser/manifest.ttl | 14 +++++++++++ 14 files changed, 52 insertions(+), 27 deletions(-) create mode 100644 testsuite/oxigraph-tests/parser/escaped_trailing_dot.nq create mode 100644 testsuite/oxigraph-tests/parser/escaped_trailing_dot.nt create mode 100644 testsuite/oxigraph-tests/parser/escaped_trailing_dot.trig create mode 100644 testsuite/oxigraph-tests/parser/escaped_trailing_dot.ttl diff --git a/Cargo.lock b/Cargo.lock index 652ddf86..3d54e27c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1033,7 +1033,7 @@ dependencies = [ [[package]] name = "oxigraph" -version = "0.4.0-alpha.2" +version = "0.4.0-alpha.3-dev" dependencies = [ "codspeed-criterion-compat", "digest", @@ -1063,7 +1063,7 @@ dependencies = [ [[package]] name = "oxigraph-cli" -version = "0.4.0-alpha.2" +version = "0.4.0-alpha.3-dev" dependencies = [ "anyhow", "assert_cmd", @@ -1082,7 +1082,7 @@ dependencies = [ [[package]] name = "oxigraph-js" -version = "0.4.0-alpha.2" +version = "0.4.0-alpha.3-dev" dependencies = [ "console_error_panic_hook", "js-sys", @@ -1129,7 +1129,7 @@ dependencies = [ [[package]] name = "oxrdfio" -version = "0.1.0-alpha.1" +version = "0.1.0-alpha.2-dev" dependencies = [ "oxrdf", "oxrdfxml", @@ -1150,7 +1150,7 @@ dependencies = [ [[package]] name = "oxrocksdb-sys" -version = "0.4.0-alpha.2" +version = "0.4.0-alpha.3-dev" dependencies = [ "bindgen", "cc", @@ -1167,7 +1167,7 @@ dependencies = [ [[package]] name = "oxttl" -version = "0.1.0-alpha.1" +version = "0.1.0-alpha.2-dev" dependencies = [ "memchr", "oxilangtag", @@ -1403,7 +1403,7 @@ dependencies = [ [[package]] name = "pyoxigraph" -version = "0.4.0-alpha.2" +version = "0.4.0-alpha.3-dev" dependencies = [ "oxigraph", "pyo3", diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 7cea59b8..befeb158 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "oxigraph-cli" -version = "0.4.0-alpha.2" +version = "0.4.0-alpha.3-dev" authors = ["Tpt "] license = "MIT OR Apache-2.0" readme = "README.md" @@ -29,7 +29,7 @@ rustls-webpki = ["oxigraph/http-client-rustls-webpki"] anyhow = "1.0.72" oxhttp = { version = "0.2.0-alpha.3", features = ["flate2"] } clap = { version = "4.0", features = ["derive"] } -oxigraph = { version = "0.4.0-alpha.2", path = "../lib" } +oxigraph = { version = "0.4.0-alpha.3-dev", path = "../lib" } rand = "0.8" url = "2.4" oxiri = "0.2.3-alpha.1" diff --git a/js/Cargo.toml b/js/Cargo.toml index 6169004f..8caba72e 100644 --- a/js/Cargo.toml +++ b/js/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "oxigraph-js" -version = "0.4.0-alpha.2" +version = "0.4.0-alpha.3-dev" authors = ["Tpt "] license = "MIT OR Apache-2.0" readme = "README.md" diff --git a/lib/Cargo.toml b/lib/Cargo.toml index 944ecc62..8465ce8f 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "oxigraph" -version = "0.4.0-alpha.2" +version = "0.4.0-alpha.3-dev" authors = ["Tpt "] license = "MIT OR Apache-2.0" readme = "README.md" @@ -33,7 +33,7 @@ md-5 = "0.10" oxilangtag = "0.1" oxiri = "0.2.3-alpha.1" oxrdf = { version = "0.2.0-alpha.1", path = "oxrdf", features = ["rdf-star", "oxsdatatypes"] } -oxrdfio = { version = "0.1.0-alpha.1", path = "oxrdfio", features = ["rdf-star"] } +oxrdfio = { version = "0.1.0-alpha.2-dev", path = "oxrdfio", features = ["rdf-star"] } oxsdatatypes = { version = "0.2.0-alpha.1", path = "oxsdatatypes" } rand = "0.8" regex = "1.7" @@ -46,7 +46,7 @@ sparopt = { version = "0.1.0-alpha.1", path = "sparopt", features = ["rdf-star", [target.'cfg(not(target_family = "wasm"))'.dependencies] libc = "0.2.147" -oxrocksdb-sys = { version = "0.4.0-alpha.2", path = "../oxrocksdb-sys" } +oxrocksdb-sys = { version = "0.4.0-alpha.3-dev", path = "../oxrocksdb-sys" } oxhttp = { version = "0.2.0-alpha.3", optional = true } [target.'cfg(all(target_family = "wasm", target_os = "unknown"))'.dependencies] diff --git a/lib/oxrdfio/Cargo.toml b/lib/oxrdfio/Cargo.toml index 6ded48d6..c2930046 100644 --- a/lib/oxrdfio/Cargo.toml +++ b/lib/oxrdfio/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "oxrdfio" -version = "0.1.0-alpha.1" +version = "0.1.0-alpha.2-dev" authors = ["Tpt "] license = "MIT OR Apache-2.0" readme = "README.md" @@ -21,7 +21,7 @@ rdf-star = ["oxrdf/rdf-star", "oxttl/rdf-star"] [dependencies] oxrdf = { version = "0.2.0-alpha.1", path = "../oxrdf" } oxrdfxml = { version = "0.1.0-alpha.1", path = "../oxrdfxml" } -oxttl = { version = "0.1.0-alpha.1", path = "../oxttl" } +oxttl = { version = "0.1.0-alpha.2-dev", path = "../oxttl" } tokio = { version = "1.29", optional = true, features = ["io-util"] } [dev-dependencies] diff --git a/lib/oxttl/Cargo.toml b/lib/oxttl/Cargo.toml index c19cdffe..7f7428f6 100644 --- a/lib/oxttl/Cargo.toml +++ b/lib/oxttl/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "oxttl" -version = "0.1.0-alpha.1" +version = "0.1.0-alpha.2-dev" authors = ["Tpt "] license = "MIT OR Apache-2.0" readme = "README.md" diff --git a/lib/oxttl/src/lexer.rs b/lib/oxttl/src/lexer.rs index ef95938e..9bb8c49e 100644 --- a/lib/oxttl/src/lexer.rs +++ b/lib/oxttl/src/lexer.rs @@ -354,6 +354,7 @@ impl N3Lexer { let mut buffer = None; // Buffer if there are some escaped characters let mut position_that_is_already_in_buffer = 0; let mut might_be_invalid_iri = false; + let mut ends_with_unescaped_dot = 0; loop { if let Some(r) = Self::recognize_unicode_char(&data[i..], i) { match r { @@ -369,6 +370,7 @@ impl N3Lexer { ).into()))); } i += 1; + ends_with_unescaped_dot = 0; } else if c == '\\' { i += 1; let a = char::from(*data.get(i)?); @@ -416,6 +418,7 @@ impl N3Lexer { buffer.push(a); i += 1; position_that_is_already_in_buffer = i; + ends_with_unescaped_dot = 0; } else if i == 0 { if !(Self::is_possible_pn_chars_u(c) || c == ':' || c.is_ascii_digit()) { @@ -427,13 +430,17 @@ impl N3Lexer { || c == ':'; } i += consumed; - } else if Self::is_possible_pn_chars(c) || c == ':' || c == '.' { + } else if Self::is_possible_pn_chars(c) || c == ':' { if !self.unchecked { might_be_invalid_iri |= Self::is_possible_pn_chars_base_but_not_valid_iri(c) || c == ':'; } i += consumed; + ends_with_unescaped_dot = 0; + } else if c == '.' { + i += consumed; + ends_with_unescaped_dot += 1; } else { let buffer = if let Some(mut buffer) = buffer { buffer.push_str( @@ -445,22 +452,20 @@ impl N3Lexer { Err(e) => return Some((i, Err(e))), }, ); - // We do not include the last dot - while buffer.ends_with('.') { + // We do not include the last dots + for _ in 0..ends_with_unescaped_dot { buffer.pop(); - i -= 1; } + i -= ends_with_unescaped_dot; Cow::Owned(buffer) } else { let mut data = match str_from_utf8(&data[..i], 0..i) { Ok(data) => data, Err(e) => return Some((i, Err(e))), }; - // We do not include the last dot - while let Some(d) = data.strip_suffix('.') { - data = d; - i -= 1; - } + // We do not include the last dots + data = &data[..data.len() - ends_with_unescaped_dot]; + i -= ends_with_unescaped_dot; Cow::Borrowed(data) }; return Some((i, Ok((buffer, might_be_invalid_iri)))); diff --git a/oxrocksdb-sys/Cargo.toml b/oxrocksdb-sys/Cargo.toml index 1b0196a3..892243cc 100644 --- a/oxrocksdb-sys/Cargo.toml +++ b/oxrocksdb-sys/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "oxrocksdb-sys" -version = "0.4.0-alpha.2" +version = "0.4.0-alpha.3-dev" authors = ["Tpt "] license = "GPL-2.0 OR Apache-2.0" repository = "https://github.com/oxigraph/oxigraph/tree/main/oxrocksdb-sys" diff --git a/python/Cargo.toml b/python/Cargo.toml index 3f386aa9..3cacf9bc 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyoxigraph" -version = "0.4.0-alpha.2" +version = "0.4.0-alpha.3-dev" authors = ["Tpt "] license = "MIT OR Apache-2.0" readme = "README.md" diff --git a/testsuite/oxigraph-tests/parser/escaped_trailing_dot.nq b/testsuite/oxigraph-tests/parser/escaped_trailing_dot.nq new file mode 100644 index 00000000..d0a345c1 --- /dev/null +++ b/testsuite/oxigraph-tests/parser/escaped_trailing_dot.nq @@ -0,0 +1 @@ + . diff --git a/testsuite/oxigraph-tests/parser/escaped_trailing_dot.nt b/testsuite/oxigraph-tests/parser/escaped_trailing_dot.nt new file mode 100644 index 00000000..d0a345c1 --- /dev/null +++ b/testsuite/oxigraph-tests/parser/escaped_trailing_dot.nt @@ -0,0 +1 @@ + . diff --git a/testsuite/oxigraph-tests/parser/escaped_trailing_dot.trig b/testsuite/oxigraph-tests/parser/escaped_trailing_dot.trig new file mode 100644 index 00000000..9017defa --- /dev/null +++ b/testsuite/oxigraph-tests/parser/escaped_trailing_dot.trig @@ -0,0 +1,2 @@ +@prefix ex: . +ex:s ex:p ex:o\. . diff --git a/testsuite/oxigraph-tests/parser/escaped_trailing_dot.ttl b/testsuite/oxigraph-tests/parser/escaped_trailing_dot.ttl new file mode 100644 index 00000000..9017defa --- /dev/null +++ b/testsuite/oxigraph-tests/parser/escaped_trailing_dot.ttl @@ -0,0 +1,2 @@ +@prefix ex: . +ex:s ex:p ex:o\. . diff --git a/testsuite/oxigraph-tests/parser/manifest.ttl b/testsuite/oxigraph-tests/parser/manifest.ttl index ec90b2bf..af3e80b5 100644 --- a/testsuite/oxigraph-tests/parser/manifest.ttl +++ b/testsuite/oxigraph-tests/parser/manifest.ttl @@ -18,6 +18,8 @@ <#keyword_vs_prefix_ttl> <#keyword_vs_prefix_trig> <#at_keywords_as_lang_tag> + <#escaped_trailing_dot_ttl> + <#escaped_trailing_dot_trig> ) . <#no_end_line_jump> @@ -88,3 +90,15 @@ mf:name "usage of at keywords as language tags" ; mf:action ; mf:result . + +<#escaped_trailing_dot_ttl> + rdf:type rdft:TestTurtleEval ; + mf:name "escaped dot at the end of a local name" ; + mf:action ; + mf:result . + +<#escaped_trailing_dot_trig> + rdf:type rdft:TestTrigEval ; + mf:name "escaped dot at the end of a local name" ; + mf:action ; + mf:result . From d4bfcd3b24ae7cbe5fefed81edbad4c91128d789 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Wed, 17 Jan 2024 23:33:31 -0500 Subject: [PATCH 177/217] Add debug print to test_debian_compatibility --- lints/test_debian_compatibility.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lints/test_debian_compatibility.py b/lints/test_debian_compatibility.py index 6823ac18..2e81a79b 100644 --- a/lints/test_debian_compatibility.py +++ b/lints/test_debian_compatibility.py @@ -46,6 +46,7 @@ def fetch_debian_package_desc(debian_name): print(f"Error {e} from {url}, retrying after {wait}s") sleep(wait) else: + print(f"Failed to fetch debian name {debian_name} from {url}: {e}") raise e raise Exception(f"Failed to fetch {url}") From 51941c0dc576c361730ce4ebf7ad4e680d851351 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Wed, 17 Jan 2024 21:47:05 -0500 Subject: [PATCH 178/217] Simplify complex strings with raw literals, readme fix Also add a minor debugging python statement to linter --- cli/src/main.rs | 40 ++++++++++----------- lib/oxrdf/src/literal.rs | 6 ++-- lib/oxrdfxml/README.md | 32 +++++++++-------- lib/oxrdfxml/src/parser.rs | 60 ++++++++++++++++---------------- lib/oxttl/src/n3.rs | 54 ++++++++++++++-------------- lib/oxttl/src/nquads.rs | 30 ++++++++-------- lib/oxttl/src/ntriples.rs | 32 ++++++++--------- lib/oxttl/src/trig.rs | 54 ++++++++++++++-------------- lib/oxttl/src/turtle.rs | 54 ++++++++++++++-------------- lib/sparesults/src/parser.rs | 10 +++--- lib/sparesults/src/serializer.rs | 16 ++++----- lib/src/sparql/model.rs | 2 +- lib/src/sparql/results.rs | 4 +-- 13 files changed, 198 insertions(+), 196 deletions(-) diff --git a/cli/src/main.rs b/cli/src/main.rs index b0f7a6fa..9e9a0dbd 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -2354,7 +2354,7 @@ mod tests { .build(); ServerTest::new()?.test_body( request, - "{\"head\":{\"vars\":[\"s\",\"p\",\"o\"]},\"results\":{\"bindings\":[]}}", + r#"{"head":{"vars":["s","p","o"]},"results":{"bindings":[]}}"#, ) } @@ -2369,7 +2369,7 @@ mod tests { .build(); ServerTest::new()?.test_body( request, - "{\"head\":{\"vars\":[\"s\",\"p\",\"o\"]},\"results\":{\"bindings\":[]}}", + r#"{"head":{"vars":["s","p","o"]},"results":{"bindings":[]}}"#, ) } @@ -2387,7 +2387,7 @@ mod tests { .build(); ServerTest::new()?.test_body( request, - "{\"head\":{\"vars\":[\"s\",\"p\",\"o\"]},\"results\":{\"bindings\":[]}}", + r#"{"head":{"vars":["s","p","o"]},"results":{"bindings":[]}}"#, ) } @@ -2414,7 +2414,7 @@ mod tests { .build(); ServerTest::new()?.test_body( request, - "{\"head\":{\"vars\":[\"s\",\"p\",\"o\"]},\"results\":{\"bindings\":[]}}", + r#"{"head":{"vars":["s","p","o"]},"results":{"bindings":[]}}"#, ) } @@ -2429,7 +2429,7 @@ mod tests { .build(); ServerTest::new()?.test_body( request, - "{\"head\":{\"vars\":[\"s\",\"p\",\"o\"]},\"results\":{\"bindings\":[]}}", + r#"{"head":{"vars":["s","p","o"]},"results":{"bindings":[]}}"#, ) } #[test] @@ -2679,16 +2679,16 @@ mod tests { let request = Request::builder(Method::PUT, "http://localhost/store/person/1.ttl".parse()?) .with_header(HeaderName::CONTENT_TYPE, "text/turtle; charset=utf-8")? .with_body( - " + r#" @prefix foaf: . @prefix v: . a foaf:Person; foaf:businessCard [ a v:VCard; - v:fn \"John Doe\" + v:fn "John Doe" ]. -", +"#, ); server.test_status(request, Status::CREATED)?; @@ -2717,16 +2717,16 @@ mod tests { let request = Request::builder(Method::PUT, "http://localhost/store/person/1.ttl".parse()?) .with_header(HeaderName::CONTENT_TYPE, "text/turtle; charset=utf-8")? .with_body( - " + r#" @prefix foaf: . @prefix v: . a foaf:Person; foaf:businessCard [ a v:VCard; - v:fn \"Jane Doe\" + v:fn "Jane Doe" ]. -", +"#, ); server.test_status(request, Status::NO_CONTENT)?; @@ -2740,16 +2740,16 @@ mod tests { let request = Request::builder(Method::PUT, "http://localhost/store?default".parse()?) .with_header(HeaderName::CONTENT_TYPE, "text/turtle; charset=utf-8")? .with_body( - " + r#" @prefix foaf: . @prefix v: . [] a foaf:Person; foaf:businessCard [ a v:VCard; - v:given-name \"Alice\" + v:given-name "Alice" ] . -", +"#, ); server.test_status(request, Status::NO_CONTENT)?; // The default graph always exists in Oxigraph @@ -2781,16 +2781,16 @@ mod tests { let request = Request::builder(Method::PUT, "http://localhost/store/person/2.ttl".parse()?) .with_header(HeaderName::CONTENT_TYPE, "text/turtle; charset=utf-8")? .with_body( - " + r#" @prefix foaf: . @prefix v: . [] a foaf:Person; foaf:businessCard [ a v:VCard; - v:given-name \"Alice\" + v:given-name "Alice" ] . -", +"#, ); server.test_status(request, Status::NO_CONTENT)?; @@ -2839,16 +2839,16 @@ mod tests { let request = Request::builder(Method::POST, "http://localhost/store".parse()?) .with_header(HeaderName::CONTENT_TYPE, "text/turtle; charset=utf-8")? .with_body( - " + r#" @prefix foaf: . @prefix v: . [] a foaf:Person; foaf:businessCard [ a v:VCard; - v:given-name \"Alice\" + v:given-name "Alice" ] . -", +"#, ); let response = server.exec(request); assert_eq!(response.status(), Status::CREATED); diff --git a/lib/oxrdf/src/literal.rs b/lib/oxrdf/src/literal.rs index 976b863d..3f2727ca 100644 --- a/lib/oxrdf/src/literal.rs +++ b/lib/oxrdf/src/literal.rs @@ -24,12 +24,12 @@ use std::option::Option; /// ); /// /// assert_eq!( -/// "\"1999-01-01\"^^", +/// r#""1999-01-01"^^"#, /// Literal::new_typed_literal("1999-01-01", xsd::DATE).to_string() /// ); /// /// assert_eq!( -/// "\"foo\"@en", +/// r#""foo"@en"#, /// Literal::new_language_tagged_literal("foo", "en")?.to_string() /// ); /// # Result::<(), LanguageTagParseError>::Ok(()) @@ -436,7 +436,7 @@ impl From for Literal { /// ); /// /// assert_eq!( -/// "\"1999-01-01\"^^", +/// r#""1999-01-01"^^"#, /// LiteralRef::new_typed_literal("1999-01-01", xsd::DATE).to_string() /// ); /// ``` diff --git a/lib/oxrdfxml/README.md b/lib/oxrdfxml/README.md index 8575d19b..29ebb4cf 100644 --- a/lib/oxrdfxml/README.md +++ b/lib/oxrdfxml/README.md @@ -12,31 +12,33 @@ OxRdfXml is a parser and serializer for [RDF/XML](https://www.w3.org/TR/rdf-synt The entry points of this library are the two [`RdfXmlParser`] and [`RdfXmlSerializer`] structs. Usage example counting the number of people in a RDF/XML file: + ```rust use oxrdf::{NamedNodeRef, vocab::rdf}; use oxrdfxml::RdfXmlParser; -let file = b" - - - +fn main() { + let file = br#" + + + Foo - -"; - -let schema_person = NamedNodeRef::new("http://schema.org/Person").unwrap(); -let mut count = 0; -for triple in RdfXmlParser::new().parse_read(file.as_ref()) { - let triple = triple.unwrap(); - if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { - count += 1; + +"#; + + let schema_person = NamedNodeRef::new("http://schema.org/Person").unwrap(); + let mut count = 0; + for triple in RdfXmlParser::new().parse_read(file.as_ref()) { + let triple = triple.unwrap(); + if triple.predicate == rdf::TYPE && triple.object == schema_person.into() { + count += 1; + } } + assert_eq!(2, count); } -assert_eq!(2, count); ``` - ## License This project is licensed under either of diff --git a/lib/oxrdfxml/src/parser.rs b/lib/oxrdfxml/src/parser.rs index ba0b27a1..5a364c78 100644 --- a/lib/oxrdfxml/src/parser.rs +++ b/lib/oxrdfxml/src/parser.rs @@ -29,14 +29,14 @@ use tokio::io::{AsyncRead, BufReader as AsyncBufReader}; /// use oxrdf::{NamedNodeRef, vocab::rdf}; /// use oxrdfxml::RdfXmlParser; /// -/// let file = b" -/// -/// -/// +/// let file = br#" +/// +/// +/// /// Foo /// -/// -/// "; +/// +/// "#; /// /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; /// let mut count = 0; @@ -87,14 +87,14 @@ impl RdfXmlParser { /// use oxrdf::{NamedNodeRef, vocab::rdf}; /// use oxrdfxml::RdfXmlParser; /// - /// let file = b" - /// - /// - /// + /// let file = br#" + /// + /// + /// /// Foo /// - /// - /// "; + /// + /// "#; /// /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; /// let mut count = 0; @@ -124,14 +124,14 @@ impl RdfXmlParser { /// /// # #[tokio::main(flavor = "current_thread")] /// # async fn main() -> Result<(), oxrdfxml::ParseError> { - /// let file = b" - /// - /// - /// + /// let file = br#" + /// + /// + /// /// Foo /// - /// - /// "; + /// + /// "#; /// /// let schema_person = NamedNodeRef::new_unchecked("http://schema.org/Person"); /// let mut count = 0; @@ -182,14 +182,14 @@ impl RdfXmlParser { /// use oxrdf::{NamedNodeRef, vocab::rdf}; /// use oxrdfxml::RdfXmlParser; /// -/// let file = b" -/// -/// -/// +/// let file = br#" +/// +/// +/// /// Foo /// -/// -/// "; +/// +/// "#; /// /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; /// let mut count = 0; @@ -251,14 +251,14 @@ impl FromReadRdfXmlReader { /// /// # #[tokio::main(flavor = "current_thread")] /// # async fn main() -> Result<(), oxrdfxml::ParseError> { -/// let file = b" -/// -/// -/// +/// let file = br#" +/// +/// +/// /// Foo /// -/// -/// "; +/// +/// "#; /// /// let schema_person = NamedNodeRef::new_unchecked("http://schema.org/Person"); /// let mut count = 0; diff --git a/lib/oxttl/src/n3.rs b/lib/oxttl/src/n3.rs index 0642d416..eb1ce55d 100644 --- a/lib/oxttl/src/n3.rs +++ b/lib/oxttl/src/n3.rs @@ -184,12 +184,12 @@ impl From for N3Quad { /// use oxrdf::{NamedNode, vocab::rdf}; /// use oxttl::n3::{N3Parser, N3Term}; /// -/// let file = b"@base . +/// let file = br#"@base . /// @prefix schema: . /// a schema:Person ; -/// schema:name \"Foo\" . +/// schema:name "Foo" . /// a schema:Person ; -/// schema:name \"Bar\" ."; +/// schema:name "Bar" ."#; /// /// let rdf_type = N3Term::NamedNode(rdf::TYPE.into_owned()); /// let schema_person = N3Term::NamedNode(NamedNode::new("http://schema.org/Person")?); @@ -253,12 +253,12 @@ impl N3Parser { /// use oxrdf::NamedNode; /// use oxttl::n3::{N3Parser, N3Term}; /// - /// let file = b"@base . + /// let file = br#"@base . /// @prefix schema: . /// a schema:Person ; - /// schema:name \"Foo\" . + /// schema:name "Foo" . /// a schema:Person ; - /// schema:name \"Bar\" ."; + /// schema:name "Bar" ."#; /// /// let rdf_type = N3Term::NamedNode(NamedNode::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?); /// let schema_person = N3Term::NamedNode(NamedNode::new("http://schema.org/Person")?); @@ -287,12 +287,12 @@ impl N3Parser { /// /// # #[tokio::main(flavor = "current_thread")] /// # async fn main() -> Result<(), oxttl::ParseError> { - /// let file = b"@base . + /// let file = br#"@base . /// @prefix schema: . /// a schema:Person ; - /// schema:name \"Foo\" . + /// schema:name "Foo" . /// a schema:Person ; - /// schema:name \"Bar\" ."; + /// schema:name "Bar" ."#; /// /// let rdf_type = N3Term::NamedNode(rdf::TYPE.into_owned()); /// let schema_person = N3Term::NamedNode(NamedNode::new_unchecked("http://schema.org/Person")); @@ -369,12 +369,12 @@ impl N3Parser { /// use oxrdf::{NamedNode, vocab::rdf}; /// use oxttl::n3::{N3Parser, N3Term}; /// -/// let file = b"@base . +/// let file = br#"@base . /// @prefix schema: . /// a schema:Person ; -/// schema:name \"Foo\" . +/// schema:name "Foo" . /// a schema:Person ; -/// schema:name \"Bar\" ."; +/// schema:name "Bar" ."#; /// /// let rdf_type = N3Term::NamedNode(rdf::TYPE.into_owned()); /// let schema_person = N3Term::NamedNode(NamedNode::new("http://schema.org/Person")?); @@ -403,10 +403,10 @@ impl FromReadN3Reader { /// ``` /// use oxttl::N3Parser; /// - /// let file = b"@base . + /// let file = br#"@base . /// @prefix schema: . /// a schema:Person ; - /// schema:name \"Foo\" ."; + /// schema:name "Foo" ."#; /// /// let mut reader = N3Parser::new().parse_read(file.as_ref()); /// assert!(reader.prefixes().is_empty()); // No prefix at the beginning @@ -424,10 +424,10 @@ impl FromReadN3Reader { /// ``` /// use oxttl::N3Parser; /// - /// let file = b"@base . + /// let file = br#"@base . /// @prefix schema: . /// a schema:Person ; - /// schema:name \"Foo\" ."; + /// schema:name "Foo" ."#; /// /// let mut reader = N3Parser::new().parse_read(file.as_ref()); /// assert!(reader.base_iri().is_none()); // No base at the beginning because none has been given to the parser. @@ -464,12 +464,12 @@ impl Iterator for FromReadN3Reader { /// /// # #[tokio::main(flavor = "current_thread")] /// # async fn main() -> Result<(), oxttl::ParseError> { -/// let file = b"@base . +/// let file = br#"@base . /// @prefix schema: . /// a schema:Person ; -/// schema:name \"Foo\" . +/// schema:name "Foo" . /// a schema:Person ; -/// schema:name \"Bar\" ."; +/// schema:name "Bar" ."#; /// /// let rdf_type = N3Term::NamedNode(rdf::TYPE.into_owned()); /// let schema_person = N3Term::NamedNode(NamedNode::new_unchecked("http://schema.org/Person")); @@ -509,10 +509,10 @@ impl FromTokioAsyncReadN3Reader { /// /// # #[tokio::main(flavor = "current_thread")] /// # async fn main() -> Result<(), oxttl::ParseError> { - /// let file = b"@base . + /// let file = br#"@base . /// @prefix schema: . /// a schema:Person ; - /// schema:name \"Foo\" ."; + /// schema:name "Foo" ."#; /// /// let mut reader = N3Parser::new().parse_tokio_async_read(file.as_ref()); /// assert!(reader.prefixes().is_empty()); // No prefix at the beginning @@ -533,10 +533,10 @@ impl FromTokioAsyncReadN3Reader { /// /// # #[tokio::main(flavor = "current_thread")] /// # async fn main() -> Result<(), oxttl::ParseError> { - /// let file = b"@base . + /// let file = br#"@base . /// @prefix schema: . /// a schema:Person ; - /// schema:name \"Foo\" ."; + /// schema:name "Foo" ."#; /// /// let mut reader = N3Parser::new().parse_tokio_async_read(file.as_ref()); /// assert!(reader.base_iri().is_none()); // No base IRI at the beginning @@ -633,10 +633,10 @@ impl LowLevelN3Reader { /// ``` /// use oxttl::N3Parser; /// - /// let file = b"@base . + /// let file = br#"@base . /// @prefix schema: . /// a schema:Person ; - /// schema:name \"Foo\" ."; + /// schema:name "Foo" ."#; /// /// let mut reader = N3Parser::new().parse(); /// reader.extend_from_slice(file); @@ -655,10 +655,10 @@ impl LowLevelN3Reader { /// ``` /// use oxttl::N3Parser; /// - /// let file = b"@base . + /// let file = br#"@base . /// @prefix schema: . /// a schema:Person ; - /// schema:name \"Foo\" ."; + /// schema:name "Foo" ."#; /// /// let mut reader = N3Parser::new().parse(); /// reader.extend_from_slice(file); diff --git a/lib/oxttl/src/nquads.rs b/lib/oxttl/src/nquads.rs index 9be1843b..d93c7293 100644 --- a/lib/oxttl/src/nquads.rs +++ b/lib/oxttl/src/nquads.rs @@ -19,10 +19,10 @@ use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; /// use oxrdf::{NamedNodeRef, vocab::rdf}; /// use oxttl::NQuadsParser; /// -/// let file = b" . -/// \"Foo\" . +/// let file = br#" . +/// "Foo" . /// . -/// \"Bar\" ."; +/// "Bar" ."#; /// /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; /// let mut count = 0; @@ -76,10 +76,10 @@ impl NQuadsParser { /// use oxrdf::{NamedNodeRef, vocab::rdf}; /// use oxttl::NQuadsParser; /// - /// let file = b" . - /// \"Foo\" . + /// let file = br#" . + /// "Foo" . /// . - /// \"Bar\" ."; + /// "Bar" ."#; /// /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; /// let mut count = 0; @@ -107,10 +107,10 @@ impl NQuadsParser { /// /// # #[tokio::main(flavor = "current_thread")] /// # async fn main() -> Result<(), oxttl::ParseError> { - /// let file = b" . - /// \"Foo\" . + /// let file = br#" . + /// "Foo" . /// . - /// \"Bar\" ."; + /// "Bar" ."#; /// /// let schema_person = NamedNodeRef::new_unchecked("http://schema.org/Person"); /// let mut count = 0; @@ -191,10 +191,10 @@ impl NQuadsParser { /// use oxrdf::{NamedNodeRef, vocab::rdf}; /// use oxttl::NQuadsParser; /// -/// let file = b" . -/// \"Foo\" . +/// let file = br#" . +/// "Foo" . /// . -/// \"Bar\" ."; +/// "Bar" ."#; /// /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; /// let mut count = 0; @@ -229,10 +229,10 @@ impl Iterator for FromReadNQuadsReader { /// /// # #[tokio::main(flavor = "current_thread")] /// # async fn main() -> Result<(), oxttl::ParseError> { -/// let file = b" . -/// \"Foo\" . +/// let file = br#" . +/// "Foo" . /// . -/// \"Bar\" ."; +/// "Bar" ."#; /// /// let schema_person = NamedNodeRef::new_unchecked("http://schema.org/Person"); /// let mut count = 0; diff --git a/lib/oxttl/src/ntriples.rs b/lib/oxttl/src/ntriples.rs index 995643bc..c5ed5b94 100644 --- a/lib/oxttl/src/ntriples.rs +++ b/lib/oxttl/src/ntriples.rs @@ -19,10 +19,10 @@ use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; /// use oxrdf::{NamedNodeRef, vocab::rdf}; /// use oxttl::NTriplesParser; /// -/// let file = b" . -/// \"Foo\" . +/// let file = br#" . +/// "Foo" . /// . -/// \"Bar\" ."; +/// "Bar" ."#; /// /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; /// let mut count = 0; @@ -76,10 +76,10 @@ impl NTriplesParser { /// use oxrdf::{NamedNodeRef, vocab::rdf}; /// use oxttl::NTriplesParser; /// - /// let file = b" . - /// \"Foo\" . + /// let file = br#" . + /// "Foo" . /// . - /// \"Bar\" ."; + /// "Bar" ."#; /// /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; /// let mut count = 0; @@ -107,10 +107,10 @@ impl NTriplesParser { /// /// # #[tokio::main(flavor = "current_thread")] /// # async fn main() -> Result<(), oxttl::ParseError> { - /// let file = b" . - /// \"Foo\" . + /// let file = br#" . + /// "Foo" . /// . - /// \"Bar\" ."; + /// "Bar" ."#; /// /// let schema_person = NamedNodeRef::new_unchecked("http://schema.org/Person"); /// let mut count = 0; @@ -191,10 +191,10 @@ impl NTriplesParser { /// use oxrdf::{NamedNodeRef, vocab::rdf}; /// use oxttl::NTriplesParser; /// -/// let file = b" . -/// \"Foo\" . +/// let file = br#" . +/// "Foo" . /// . -/// \"Bar\" ."; +/// "Bar" ."#; /// /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; /// let mut count = 0; @@ -229,10 +229,10 @@ impl Iterator for FromReadNTriplesReader { /// /// # #[tokio::main(flavor = "current_thread")] /// # async fn main() -> Result<(), oxttl::ParseError> { -/// let file = b" . -/// \"Foo\" . +/// let file = br#" . +/// "Foo" . /// . -/// \"Bar\" ."; +/// "Bar" ."#; /// /// let schema_person = NamedNodeRef::new_unchecked("http://schema.org/Person"); /// let mut count = 0; @@ -565,7 +565,7 @@ mod tests { fn unchecked_parsing() { let triples = NTriplesParser::new() .unchecked() - .parse_read(" \"baz\"@toolonglangtag .".as_bytes()) + .parse_read(r#" "baz"@toolonglangtag ."#.as_bytes()) .collect::, _>>() .unwrap(); assert_eq!( diff --git a/lib/oxttl/src/trig.rs b/lib/oxttl/src/trig.rs index 70d3edb6..08fbf271 100644 --- a/lib/oxttl/src/trig.rs +++ b/lib/oxttl/src/trig.rs @@ -22,12 +22,12 @@ use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; /// use oxrdf::{NamedNodeRef, vocab::rdf}; /// use oxttl::TriGParser; /// -/// let file = b"@base . +/// let file = br#"@base . /// @prefix schema: . /// a schema:Person ; -/// schema:name \"Foo\" . +/// schema:name "Foo" . /// a schema:Person ; -/// schema:name \"Bar\" ."; +/// schema:name "Bar" ."#; /// /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; /// let mut count = 0; @@ -100,12 +100,12 @@ impl TriGParser { /// use oxrdf::{NamedNodeRef, vocab::rdf}; /// use oxttl::TriGParser; /// - /// let file = b"@base . + /// let file = br#"@base . /// @prefix schema: . /// a schema:Person ; - /// schema:name \"Foo\" . + /// schema:name "Foo" . /// a schema:Person ; - /// schema:name \"Bar\" ."; + /// schema:name "Bar" ."#; /// /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; /// let mut count = 0; @@ -133,12 +133,12 @@ impl TriGParser { /// /// # #[tokio::main(flavor = "current_thread")] /// # async fn main() -> Result<(), oxttl::ParseError> { - /// let file = b"@base . + /// let file = br#"@base . /// @prefix schema: . /// a schema:Person ; - /// schema:name \"Foo\" . + /// schema:name "Foo" . /// a schema:Person ; - /// schema:name \"Bar\" ."; + /// schema:name "Bar" ."#; /// /// let schema_person = NamedNodeRef::new_unchecked("http://schema.org/Person"); /// let mut count = 0; @@ -220,12 +220,12 @@ impl TriGParser { /// use oxrdf::{NamedNodeRef, vocab::rdf}; /// use oxttl::TriGParser; /// -/// let file = b"@base . +/// let file = br#"@base . /// @prefix schema: . /// a schema:Person ; -/// schema:name \"Foo\" . +/// schema:name "Foo" . /// a schema:Person ; -/// schema:name \"Bar\" ."; +/// schema:name "Bar" ."#; /// /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; /// let mut count = 0; @@ -253,10 +253,10 @@ impl FromReadTriGReader { /// ``` /// use oxttl::TriGParser; /// - /// let file = b"@base . + /// let file = br#"@base . /// @prefix schema: . /// a schema:Person ; - /// schema:name \"Foo\" ."; + /// schema:name "Foo" ."#; /// /// let mut reader = TriGParser::new().parse_read(file.as_ref()); /// assert!(reader.prefixes().is_empty()); // No prefix at the beginning @@ -274,10 +274,10 @@ impl FromReadTriGReader { /// ``` /// use oxttl::TriGParser; /// - /// let file = b"@base . + /// let file = br#"@base . /// @prefix schema: . /// a schema:Person ; - /// schema:name \"Foo\" ."; + /// schema:name "Foo" ."#; /// /// let mut reader = TriGParser::new().parse_read(file.as_ref()); /// assert!(reader.base_iri().is_none()); // No base at the beginning because none has been given to the parser. @@ -314,12 +314,12 @@ impl Iterator for FromReadTriGReader { /// /// # #[tokio::main(flavor = "current_thread")] /// # async fn main() -> Result<(), oxttl::ParseError> { -/// let file = b"@base . +/// let file = br#"@base . /// @prefix schema: . /// a schema:Person ; -/// schema:name \"Foo\" . +/// schema:name "Foo" . /// a schema:Person ; -/// schema:name \"Bar\" ."; +/// schema:name "Bar" ."#; /// /// let schema_person = NamedNodeRef::new_unchecked("http://schema.org/Person"); /// let mut count = 0; @@ -358,10 +358,10 @@ impl FromTokioAsyncReadTriGReader { /// /// # #[tokio::main(flavor = "current_thread")] /// # async fn main() -> Result<(), oxttl::ParseError> { - /// let file = b"@base . + /// let file = br#"@base . /// @prefix schema: . /// a schema:Person ; - /// schema:name \"Foo\" ."; + /// schema:name "Foo" ."#; /// /// let mut reader = TriGParser::new().parse_tokio_async_read(file.as_ref()); /// assert!(reader.prefixes().is_empty()); // No prefix at the beginning @@ -382,10 +382,10 @@ impl FromTokioAsyncReadTriGReader { /// /// # #[tokio::main(flavor = "current_thread")] /// # async fn main() -> Result<(), oxttl::ParseError> { - /// let file = b"@base . + /// let file = br#"@base . /// @prefix schema: . /// a schema:Person ; - /// schema:name \"Foo\" ."; + /// schema:name "Foo" ."#; /// /// let mut reader = TriGParser::new().parse_tokio_async_read(file.as_ref()); /// assert!(reader.base_iri().is_none()); // No base IRI at the beginning @@ -481,10 +481,10 @@ impl LowLevelTriGReader { /// ``` /// use oxttl::TriGParser; /// - /// let file = b"@base . + /// let file = br#"@base . /// @prefix schema: . /// a schema:Person ; - /// schema:name \"Foo\" ."; + /// schema:name "Foo" ."#; /// /// let mut reader = TriGParser::new().parse(); /// reader.extend_from_slice(file); @@ -503,10 +503,10 @@ impl LowLevelTriGReader { /// ``` /// use oxttl::TriGParser; /// - /// let file = b"@base . + /// let file = br#"@base . /// @prefix schema: . /// a schema:Person ; - /// schema:name \"Foo\" ."; + /// schema:name "Foo" ."#; /// /// let mut reader = TriGParser::new().parse(); /// reader.extend_from_slice(file); diff --git a/lib/oxttl/src/turtle.rs b/lib/oxttl/src/turtle.rs index 0e225611..19590b94 100644 --- a/lib/oxttl/src/turtle.rs +++ b/lib/oxttl/src/turtle.rs @@ -24,12 +24,12 @@ use tokio::io::{AsyncRead, AsyncWrite}; /// use oxrdf::{NamedNodeRef, vocab::rdf}; /// use oxttl::TurtleParser; /// -/// let file = b"@base . +/// let file = br#"@base . /// @prefix schema: . /// a schema:Person ; -/// schema:name \"Foo\" . +/// schema:name "Foo" . /// a schema:Person ; -/// schema:name \"Bar\" ."; +/// schema:name "Bar" ."#; /// /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; /// let mut count = 0; @@ -102,12 +102,12 @@ impl TurtleParser { /// use oxrdf::{NamedNodeRef, vocab::rdf}; /// use oxttl::TurtleParser; /// - /// let file = b"@base . + /// let file = br#"@base . /// @prefix schema: . /// a schema:Person ; - /// schema:name \"Foo\" . + /// schema:name "Foo" . /// a schema:Person ; - /// schema:name \"Bar\" ."; + /// schema:name "Bar" ."#; /// /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; /// let mut count = 0; @@ -135,12 +135,12 @@ impl TurtleParser { /// /// # #[tokio::main(flavor = "current_thread")] /// # async fn main() -> Result<(), oxttl::ParseError> { - /// let file = b"@base . + /// let file = br#"@base . /// @prefix schema: . /// a schema:Person ; - /// schema:name \"Foo\" . + /// schema:name "Foo" . /// a schema:Person ; - /// schema:name \"Bar\" ."; + /// schema:name "Bar" ."#; /// /// let schema_person = NamedNodeRef::new_unchecked("http://schema.org/Person"); /// let mut count = 0; @@ -222,12 +222,12 @@ impl TurtleParser { /// use oxrdf::{NamedNodeRef, vocab::rdf}; /// use oxttl::TurtleParser; /// -/// let file = b"@base . +/// let file = br#"@base . /// @prefix schema: . /// a schema:Person ; -/// schema:name \"Foo\" . +/// schema:name "Foo" . /// a schema:Person ; -/// schema:name \"Bar\" ."; +/// schema:name "Bar" ."#; /// /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; /// let mut count = 0; @@ -255,10 +255,10 @@ impl FromReadTurtleReader { /// ``` /// use oxttl::TurtleParser; /// - /// let file = b"@base . + /// let file = br#"@base . /// @prefix schema: . /// a schema:Person ; - /// schema:name \"Foo\" ."; + /// schema:name "Foo" ."#; /// /// let mut reader = TurtleParser::new().parse_read(file.as_ref()); /// assert!(reader.prefixes().is_empty()); // No prefix at the beginning @@ -276,10 +276,10 @@ impl FromReadTurtleReader { /// ``` /// use oxttl::TurtleParser; /// - /// let file = b"@base . + /// let file = br#"@base . /// @prefix schema: . /// a schema:Person ; - /// schema:name \"Foo\" ."; + /// schema:name "Foo" ."#; /// /// let mut reader = TurtleParser::new().parse_read(file.as_ref()); /// assert!(reader.base_iri().is_none()); // No base at the beginning because none has been given to the parser. @@ -316,12 +316,12 @@ impl Iterator for FromReadTurtleReader { /// /// # #[tokio::main(flavor = "current_thread")] /// # async fn main() -> Result<(), oxttl::ParseError> { -/// let file = b"@base . +/// let file = br#"@base . /// @prefix schema: . /// a schema:Person ; -/// schema:name \"Foo\" . +/// schema:name "Foo" . /// a schema:Person ; -/// schema:name \"Bar\" ."; +/// schema:name "Bar" ."#; /// /// let schema_person = NamedNodeRef::new_unchecked("http://schema.org/Person"); /// let mut count = 0; @@ -360,10 +360,10 @@ impl FromTokioAsyncReadTurtleReader { /// /// # #[tokio::main(flavor = "current_thread")] /// # async fn main() -> Result<(), oxttl::ParseError> { - /// let file = b"@base . + /// let file = br#"@base . /// @prefix schema: . /// a schema:Person ; - /// schema:name \"Foo\" ."; + /// schema:name "Foo" ."#; /// /// let mut reader = TurtleParser::new().parse_tokio_async_read(file.as_ref()); /// assert!(reader.prefixes().is_empty()); // No prefix at the beginning @@ -384,10 +384,10 @@ impl FromTokioAsyncReadTurtleReader { /// /// # #[tokio::main(flavor = "current_thread")] /// # async fn main() -> Result<(), oxttl::ParseError> { - /// let file = b"@base . + /// let file = br#"@base . /// @prefix schema: . /// a schema:Person ; - /// schema:name \"Foo\" ."; + /// schema:name "Foo" ."#; /// /// let mut reader = TurtleParser::new().parse_tokio_async_read(file.as_ref()); /// assert!(reader.base_iri().is_none()); // No base IRI at the beginning @@ -483,10 +483,10 @@ impl LowLevelTurtleReader { /// ``` /// use oxttl::TurtleParser; /// - /// let file = b"@base . + /// let file = br#"@base . /// @prefix schema: . /// a schema:Person ; - /// schema:name \"Foo\" ."; + /// schema:name "Foo" ."#; /// /// let mut reader = TurtleParser::new().parse(); /// reader.extend_from_slice(file); @@ -505,10 +505,10 @@ impl LowLevelTurtleReader { /// ``` /// use oxttl::TurtleParser; /// - /// let file = b"@base . + /// let file = br#"@base . /// @prefix schema: . /// a schema:Person ; - /// schema:name \"Foo\" ."; + /// schema:name "Foo" ."#; /// /// let mut reader = TurtleParser::new().parse(); /// reader.extend_from_slice(file); diff --git a/lib/sparesults/src/parser.rs b/lib/sparesults/src/parser.rs index 0a826e96..3e521fe8 100644 --- a/lib/sparesults/src/parser.rs +++ b/lib/sparesults/src/parser.rs @@ -22,11 +22,11 @@ use std::sync::Arc; /// /// let json_parser = QueryResultsParser::from_format(QueryResultsFormat::Json); /// // boolean -/// if let FromReadQueryResultsReader::Boolean(v) = json_parser.parse_read(b"{\"boolean\":true}".as_slice())? { +/// if let FromReadQueryResultsReader::Boolean(v) = json_parser.parse_read(br#"{"boolean":true}"#.as_slice())? { /// assert_eq!(v, true); /// } /// // solutions -/// if let FromReadQueryResultsReader::Solutions(solutions) = json_parser.parse_read(b"{\"head\":{\"vars\":[\"foo\",\"bar\"]},\"results\":{\"bindings\":[{\"foo\":{\"type\":\"literal\",\"value\":\"test\"}}]}}".as_slice())? { +/// if let FromReadQueryResultsReader::Solutions(solutions) = json_parser.parse_read(br#"{"head":{"vars":["foo","bar"]},"results":{"bindings":[{"foo":{"type":"literal","value":"test"}}]}}"#.as_slice())? { /// assert_eq!(solutions.variables(), &[Variable::new_unchecked("foo"), Variable::new_unchecked("bar")]); /// for solution in solutions { /// assert_eq!(solution?.iter().collect::>(), vec![(&Variable::new_unchecked("foo"), &Literal::from("test").into())]); @@ -57,12 +57,12 @@ impl QueryResultsParser { /// let json_parser = QueryResultsParser::from_format(QueryResultsFormat::Xml); /// /// // boolean - /// if let FromReadQueryResultsReader::Boolean(v) = json_parser.parse_read(b"true".as_slice())? { + /// if let FromReadQueryResultsReader::Boolean(v) = json_parser.parse_read(br#"true"#.as_slice())? { /// assert_eq!(v, true); /// } /// /// // solutions - /// if let FromReadQueryResultsReader::Solutions(solutions) = json_parser.parse_read(b"test".as_slice())? { + /// if let FromReadQueryResultsReader::Solutions(solutions) = json_parser.parse_read(br#"test"#.as_slice())? { /// assert_eq!(solutions.variables(), &[Variable::new_unchecked("foo"), Variable::new_unchecked("bar")]); /// for solution in solutions { /// assert_eq!(solution?.iter().collect::>(), vec![(&Variable::new_unchecked("foo"), &Literal::from("test").into())]); @@ -164,7 +164,7 @@ pub enum FromReadQueryResultsReader { /// use oxrdf::{Literal, Variable}; /// /// let json_parser = QueryResultsParser::from_format(QueryResultsFormat::Json); -/// if let FromReadQueryResultsReader::Solutions(solutions) = json_parser.parse_read(b"{\"head\":{\"vars\":[\"foo\",\"bar\"]},\"results\":{\"bindings\":[{\"foo\":{\"type\":\"literal\",\"value\":\"test\"}}]}}".as_slice())? { +/// if let FromReadQueryResultsReader::Solutions(solutions) = json_parser.parse_read(br#"{"head":{"vars":["foo","bar"]},"results":{"bindings":[{"foo":{"type":"literal","value":"test"}}]}}"#.as_slice())? { /// assert_eq!(solutions.variables(), &[Variable::new_unchecked("foo"), Variable::new_unchecked("bar")]); /// for solution in solutions { /// assert_eq!(solution?.iter().collect::>(), vec![(&Variable::new_unchecked("foo"), &Literal::from("test").into())]); diff --git a/lib/sparesults/src/serializer.rs b/lib/sparesults/src/serializer.rs index 9a4ba143..13c21628 100644 --- a/lib/sparesults/src/serializer.rs +++ b/lib/sparesults/src/serializer.rs @@ -35,14 +35,14 @@ use tokio::io::AsyncWrite; /// // boolean /// let mut buffer = Vec::new(); /// json_serializer.serialize_boolean_to_write(&mut buffer, true)?; -/// assert_eq!(buffer, b"{\"head\":{},\"boolean\":true}"); +/// assert_eq!(buffer, br#"{"head":{},"boolean":true}"#); /// /// // solutions /// let mut buffer = Vec::new(); /// let mut writer = json_serializer.serialize_solutions_to_write(&mut buffer, vec![Variable::new_unchecked("foo"), Variable::new_unchecked("bar")])?; /// writer.write(once((VariableRef::new_unchecked("foo"), LiteralRef::from("test"))))?; /// writer.finish()?; -/// assert_eq!(buffer, b"{\"head\":{\"vars\":[\"foo\",\"bar\"]},\"results\":{\"bindings\":[{\"foo\":{\"type\":\"literal\",\"value\":\"test\"}}]}}"); +/// assert_eq!(buffer, br#"{"head":{"vars":["foo","bar"]},"results":{"bindings":[{"foo":{"type":"literal","value":"test"}}]}}"#); /// # std::io::Result::Ok(()) /// ``` pub struct QueryResultsSerializer { @@ -65,7 +65,7 @@ impl QueryResultsSerializer { /// let xml_serializer = QueryResultsSerializer::from_format(QueryResultsFormat::Xml); /// let mut buffer = Vec::new(); /// xml_serializer.serialize_boolean_to_write(&mut buffer, true)?; - /// assert_eq!(buffer, b"true"); + /// assert_eq!(buffer, br#"true"#); /// # std::io::Result::Ok(()) /// ``` pub fn serialize_boolean_to_write(&self, write: W, value: bool) -> io::Result { @@ -89,7 +89,7 @@ impl QueryResultsSerializer { /// let json_serializer = QueryResultsSerializer::from_format(QueryResultsFormat::Json); /// let mut buffer = Vec::new(); /// json_serializer.serialize_boolean_to_tokio_async_write(&mut buffer, false).await?; - /// assert_eq!(buffer, b"{\"head\":{},\"boolean\":false}"); + /// assert_eq!(buffer, br#"{"head":{},"boolean":false}"r); /// # Ok(()) /// # } /// ``` @@ -134,7 +134,7 @@ impl QueryResultsSerializer { /// let mut writer = xml_serializer.serialize_solutions_to_write(&mut buffer, vec![Variable::new_unchecked("foo"), Variable::new_unchecked("bar")])?; /// writer.write(once((VariableRef::new_unchecked("foo"), LiteralRef::from("test"))))?; /// writer.finish()?; - /// assert_eq!(buffer, b"test"); + /// assert_eq!(buffer, br#"test"#); /// # std::io::Result::Ok(()) /// ``` pub fn serialize_solutions_to_write( @@ -183,7 +183,7 @@ impl QueryResultsSerializer { /// let mut writer = json_serializer.serialize_solutions_to_tokio_async_write(&mut buffer, vec![Variable::new_unchecked("foo"), Variable::new_unchecked("bar")]).await?; /// writer.write(once((VariableRef::new_unchecked("foo"), LiteralRef::from("test")))).await?; /// writer.finish().await?; - /// assert_eq!(buffer, b"{\"head\":{\"vars\":[\"foo\",\"bar\"]},\"results\":{\"bindings\":[{\"foo\":{\"type\":\"literal\",\"value\":\"test\"}}]}}"); + /// assert_eq!(buffer, br#"{"head":{"vars":["foo","bar"]},"results":{"bindings":[{"foo":{"type":"literal","value":"test"}}]}}"#); /// # Ok(()) /// # } /// ``` @@ -280,7 +280,7 @@ impl ToWriteSolutionsWriter { /// writer.write(once((VariableRef::new_unchecked("foo"), LiteralRef::from("test"))))?; /// writer.write(&QuerySolution::from((vec![Variable::new_unchecked("bar")], vec![Some(Literal::from("test").into())])))?; /// writer.finish()?; - /// assert_eq!(buffer, b"{\"head\":{\"vars\":[\"foo\",\"bar\"]},\"results\":{\"bindings\":[{\"foo\":{\"type\":\"literal\",\"value\":\"test\"}},{\"bar\":{\"type\":\"literal\",\"value\":\"test\"}}]}}"); + /// assert_eq!(buffer, br#"{"head":{"vars":["foo","bar"]},"results":{"bindings":[{"foo":{"type":"literal","value":"test"}},{"bar":{"type":"literal","value":"test"}}]}}"#); /// # std::io::Result::Ok(()) /// ``` pub fn write<'a>( @@ -368,7 +368,7 @@ impl ToTokioAsyncWriteSolutionsWriter { /// writer.write(once((VariableRef::new_unchecked("foo"), LiteralRef::from("test")))).await?; /// writer.write(&QuerySolution::from((vec![Variable::new_unchecked("bar")], vec![Some(Literal::from("test").into())]))).await?; /// writer.finish().await?; - /// assert_eq!(buffer, b"{\"head\":{\"vars\":[\"foo\",\"bar\"]},\"results\":{\"bindings\":[{\"foo\":{\"type\":\"literal\",\"value\":\"test\"}},{\"bar\":{\"type\":\"literal\",\"value\":\"test\"}}]}}"); + /// assert_eq!(buffer, br#"{"head":{"vars":["foo","bar"]},"results":{"bindings":[{"foo":{"type":"literal","value":"test"}},{"bar":{"type":"literal","value":"test"}}]}}"#); /// # Ok(()) /// # } /// ``` diff --git a/lib/src/sparql/model.rs b/lib/src/sparql/model.rs index 326e7603..faedc7f5 100644 --- a/lib/src/sparql/model.rs +++ b/lib/src/sparql/model.rs @@ -43,7 +43,7 @@ impl QueryResults { /// /// let mut results = Vec::new(); /// store.query("SELECT ?s WHERE { ?s ?p ?o }")?.write(&mut results, QueryResultsFormat::Json)?; - /// assert_eq!(results, "{\"head\":{\"vars\":[\"s\"]},\"results\":{\"bindings\":[{\"s\":{\"type\":\"uri\",\"value\":\"http://example.com\"}}]}}".as_bytes()); + /// assert_eq!(results, r#"{"head":{"vars":["s"]},"results":{"bindings":[{"s":{"type":"uri","value":"http://example.com"}}]}}"#.as_bytes()); /// # Result::<_,Box>::Ok(()) /// ``` pub fn write( diff --git a/lib/src/sparql/results.rs b/lib/src/sparql/results.rs index bbafe70d..88aff947 100644 --- a/lib/src/sparql/results.rs +++ b/lib/src/sparql/results.rs @@ -30,13 +30,13 @@ //! //! // Let's test with a boolean //! assert_eq!( -//! convert_json_to_tsv(b"{\"boolean\":true}".as_slice()).unwrap(), +//! convert_json_to_tsv(br#"{"boolean":true}"#.as_slice()).unwrap(), //! b"true" //! ); //! //! // And with a set of solutions //! assert_eq!( -//! convert_json_to_tsv(b"{\"head\":{\"vars\":[\"foo\",\"bar\"]},\"results\":{\"bindings\":[{\"foo\":{\"type\":\"literal\",\"value\":\"test\"}}]}}".as_slice()).unwrap(), +//! convert_json_to_tsv(br#"{"head":{"vars":["foo","bar"]},"results":{"bindings":[{"foo":{"type":"literal","value":"test"}}]}}"#.as_slice()).unwrap(), //! b"?foo\t?bar\n\"test\"\t\n" //! ); //! ``` From 2b8df24b8b2afbeb085991f18d99630977e188b2 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Thu, 18 Jan 2024 02:28:30 -0500 Subject: [PATCH 179/217] Use `Self::AssocName` to simplify declarations To keep with DRY principle, I think it makes it a bit less redundant to reuse the Self:: structure in the well known trait implementations - keeps it consistent with the trait decl too. --- js/src/model.rs | 10 +++++----- lib/oxrdf/src/dataset.rs | 12 +++++------ lib/oxrdf/src/graph.rs | 4 ++-- lib/oxrdf/src/interning.rs | 4 ++-- lib/oxrdf/src/parser.rs | 10 +++++----- lib/oxrdfio/src/parser.rs | 2 +- lib/oxrdfxml/src/parser.rs | 2 +- lib/oxsdatatypes/src/boolean.rs | 2 +- lib/oxsdatatypes/src/date_time.rs | 32 +++++++++++++++--------------- lib/oxsdatatypes/src/decimal.rs | 12 +++++------ lib/oxsdatatypes/src/double.rs | 2 +- lib/oxsdatatypes/src/duration.rs | 16 +++++++-------- lib/oxsdatatypes/src/float.rs | 2 +- lib/oxsdatatypes/src/integer.rs | 6 +++--- lib/oxttl/src/lexer.rs | 2 +- lib/oxttl/src/n3.rs | 2 +- lib/oxttl/src/nquads.rs | 2 +- lib/oxttl/src/ntriples.rs | 2 +- lib/oxttl/src/trig.rs | 2 +- lib/oxttl/src/turtle.rs | 2 +- lib/sparesults/src/parser.rs | 2 +- lib/sparesults/src/solution.rs | 14 ++++++------- lib/spargebra/src/query.rs | 6 +++--- lib/spargebra/src/term.rs | 26 ++++++++++++------------ lib/spargebra/src/update.rs | 6 +++--- lib/src/io/format.rs | 4 ++-- lib/src/io/read.rs | 4 ++-- lib/src/sparql/algebra.rs | 12 +++++------ lib/src/sparql/eval.rs | 22 ++++++++++---------- lib/src/sparql/model.rs | 4 ++-- lib/src/sparql/service.rs | 16 ++++----------- lib/src/storage/backend/rocksdb.rs | 4 ++-- lib/src/storage/mod.rs | 6 +++--- lib/src/storage/small_string.rs | 6 +++--- lib/src/store.rs | 4 ++-- testsuite/src/manifest.rs | 4 ++-- testsuite/src/sparql_evaluator.rs | 2 +- 37 files changed, 131 insertions(+), 139 deletions(-) diff --git a/js/src/model.rs b/js/src/model.rs index 323b5978..92d979c8 100644 --- a/js/src/model.rs +++ b/js/src/model.rs @@ -564,7 +564,7 @@ impl From for JsTerm { impl TryFrom for NamedNode { type Error = JsValue; - fn try_from(value: JsTerm) -> Result { + fn try_from(value: JsTerm) -> Result { match value { JsTerm::NamedNode(node) => Ok(node.into()), JsTerm::BlankNode(node) => Err(format_err!( @@ -588,7 +588,7 @@ impl TryFrom for NamedNode { impl TryFrom for NamedOrBlankNode { type Error = JsValue; - fn try_from(value: JsTerm) -> Result { + fn try_from(value: JsTerm) -> Result { match value { JsTerm::NamedNode(node) => Ok(node.into()), JsTerm::BlankNode(node) => Ok(node.into()), @@ -614,7 +614,7 @@ impl TryFrom for NamedOrBlankNode { impl TryFrom for Subject { type Error = JsValue; - fn try_from(value: JsTerm) -> Result { + fn try_from(value: JsTerm) -> Result { match value { JsTerm::NamedNode(node) => Ok(node.into()), JsTerm::BlankNode(node) => Ok(node.into()), @@ -637,7 +637,7 @@ impl TryFrom for Subject { impl TryFrom for Term { type Error = JsValue; - fn try_from(value: JsTerm) -> Result { + fn try_from(value: JsTerm) -> Result { match value { JsTerm::NamedNode(node) => Ok(node.into()), JsTerm::BlankNode(node) => Ok(node.into()), @@ -657,7 +657,7 @@ impl TryFrom for Term { impl TryFrom for GraphName { type Error = JsValue; - fn try_from(value: JsTerm) -> Result { + fn try_from(value: JsTerm) -> Result { match value { JsTerm::NamedNode(node) => Ok(node.into()), JsTerm::BlankNode(node) => Ok(node.into()), diff --git a/lib/oxrdf/src/dataset.rs b/lib/oxrdf/src/dataset.rs index f240a33a..0c1fcbd7 100644 --- a/lib/oxrdf/src/dataset.rs +++ b/lib/oxrdf/src/dataset.rs @@ -927,7 +927,7 @@ impl<'a> IntoIterator for &'a Dataset { type Item = QuadRef<'a>; type IntoIter = Iter<'a>; - fn into_iter(self) -> Iter<'a> { + fn into_iter(self) -> Self::IntoIter { self.iter() } } @@ -1285,7 +1285,7 @@ impl<'a> IntoIterator for GraphView<'a> { type Item = TripleRef<'a>; type IntoIter = GraphViewIter<'a>; - fn into_iter(self) -> GraphViewIter<'a> { + fn into_iter(self) -> Self::IntoIter { self.iter() } } @@ -1294,7 +1294,7 @@ impl<'a, 'b> IntoIterator for &'b GraphView<'a> { type Item = TripleRef<'a>; type IntoIter = GraphViewIter<'a>; - fn into_iter(self) -> GraphViewIter<'a> { + fn into_iter(self) -> Self::IntoIter { self.iter() } } @@ -1496,7 +1496,7 @@ impl<'a> IntoIterator for &'a GraphViewMut<'a> { type Item = TripleRef<'a>; type IntoIter = GraphViewIter<'a>; - fn into_iter(self) -> GraphViewIter<'a> { + fn into_iter(self) -> Self::IntoIter { self.iter() } } @@ -1527,7 +1527,7 @@ pub struct Iter<'a> { impl<'a> Iterator for Iter<'a> { type Item = QuadRef<'a>; - fn next(&mut self) -> Option> { + fn next(&mut self) -> Option { self.inner .next() .map(|(s, p, o, g)| self.dataset.decode_spog((s, p, o, g))) @@ -1551,7 +1551,7 @@ pub struct GraphViewIter<'a> { impl<'a> Iterator for GraphViewIter<'a> { type Item = TripleRef<'a>; - fn next(&mut self) -> Option> { + fn next(&mut self) -> Option { self.inner .next() .map(|(_, s, p, o)| self.dataset.decode_spo((s, p, o))) diff --git a/lib/oxrdf/src/graph.rs b/lib/oxrdf/src/graph.rs index 980e3ebf..3077e5de 100644 --- a/lib/oxrdf/src/graph.rs +++ b/lib/oxrdf/src/graph.rs @@ -229,7 +229,7 @@ impl<'a> IntoIterator for &'a Graph { type Item = TripleRef<'a>; type IntoIter = Iter<'a>; - fn into_iter(self) -> Iter<'a> { + fn into_iter(self) -> Self::IntoIter { self.iter() } } @@ -276,7 +276,7 @@ pub struct Iter<'a> { impl<'a> Iterator for Iter<'a> { type Item = TripleRef<'a>; - fn next(&mut self) -> Option> { + fn next(&mut self) -> Option { self.inner.next() } } diff --git a/lib/oxrdf/src/interning.rs b/lib/oxrdf/src/interning.rs index ef436149..45152b67 100644 --- a/lib/oxrdf/src/interning.rs +++ b/lib/oxrdf/src/interning.rs @@ -510,8 +510,8 @@ struct IdentityHasherBuilder; impl BuildHasher for IdentityHasherBuilder { type Hasher = IdentityHasher; - fn build_hasher(&self) -> IdentityHasher { - IdentityHasher::default() + fn build_hasher(&self) -> Self::Hasher { + Self::Hasher::default() } } diff --git a/lib/oxrdf/src/parser.rs b/lib/oxrdf/src/parser.rs index 4d95072f..f15c0b28 100644 --- a/lib/oxrdf/src/parser.rs +++ b/lib/oxrdf/src/parser.rs @@ -25,7 +25,7 @@ impl FromStr for NamedNode { /// /// assert_eq!(NamedNode::from_str("").unwrap(), NamedNode::new("http://example.com").unwrap()) /// ``` - fn from_str(s: &str) -> Result { + fn from_str(s: &str) -> Result { let (term, left) = read_named_node(s)?; if !left.is_empty() { return Err(TermParseError::msg( @@ -47,7 +47,7 @@ impl FromStr for BlankNode { /// /// assert_eq!(BlankNode::from_str("_:ex").unwrap(), BlankNode::new("ex").unwrap()) /// ``` - fn from_str(s: &str) -> Result { + fn from_str(s: &str) -> Result { let (term, left) = read_blank_node(s)?; if !left.is_empty() { return Err(TermParseError::msg( @@ -75,7 +75,7 @@ impl FromStr for Literal { /// assert_eq!(Literal::from_str("-122.23").unwrap(), Literal::new_typed_literal("-122.23", xsd::DECIMAL)); /// assert_eq!(Literal::from_str("-122e+1").unwrap(), Literal::new_typed_literal("-122e+1", xsd::DOUBLE)); /// ``` - fn from_str(s: &str) -> Result { + fn from_str(s: &str) -> Result { let (term, left) = read_literal(s)?; if !left.is_empty() { return Err(TermParseError::msg("Invalid literal serialization")); @@ -100,7 +100,7 @@ impl FromStr for Term { /// Literal::new_simple_literal("o") /// ).into()); /// ``` - fn from_str(s: &str) -> Result { + fn from_str(s: &str) -> Result { let (term, left) = read_term(s, 0)?; if !left.is_empty() { return Err(TermParseError::msg("Invalid term serialization")); @@ -120,7 +120,7 @@ impl FromStr for Variable { /// /// assert_eq!(Variable::from_str("$foo").unwrap(), Variable::new("foo").unwrap()) /// ``` - fn from_str(s: &str) -> Result { + fn from_str(s: &str) -> Result { if !s.starts_with('?') && !s.starts_with('$') { return Err(TermParseError::msg( "Variable serialization should start with ? or $", diff --git a/lib/oxrdfio/src/parser.rs b/lib/oxrdfio/src/parser.rs index eb60e05e..d5bf196d 100644 --- a/lib/oxrdfio/src/parser.rs +++ b/lib/oxrdfio/src/parser.rs @@ -382,7 +382,7 @@ enum FromReadQuadReaderKind { impl Iterator for FromReadQuadReader { type Item = Result; - fn next(&mut self) -> Option> { + fn next(&mut self) -> Option { Some(match &mut self.parser { FromReadQuadReaderKind::N3(parser) => match parser.next()? { Ok(quad) => self.mapper.map_n3_quad(quad), diff --git a/lib/oxrdfxml/src/parser.rs b/lib/oxrdfxml/src/parser.rs index 5a364c78..69f87711 100644 --- a/lib/oxrdfxml/src/parser.rs +++ b/lib/oxrdfxml/src/parser.rs @@ -212,7 +212,7 @@ pub struct FromReadRdfXmlReader { impl Iterator for FromReadRdfXmlReader { type Item = Result; - fn next(&mut self) -> Option> { + fn next(&mut self) -> Option { loop { if let Some(triple) = self.results.pop() { return Some(Ok(triple)); diff --git a/lib/oxsdatatypes/src/boolean.rs b/lib/oxsdatatypes/src/boolean.rs index 688af076..ad231c47 100644 --- a/lib/oxsdatatypes/src/boolean.rs +++ b/lib/oxsdatatypes/src/boolean.rs @@ -66,7 +66,7 @@ impl FromStr for Boolean { type Err = ParseBoolError; #[inline] - fn from_str(input: &str) -> Result { + fn from_str(input: &str) -> Result { Ok(match input { "true" | "1" => true, "false" | "0" => false, diff --git a/lib/oxsdatatypes/src/date_time.rs b/lib/oxsdatatypes/src/date_time.rs index 3c21cf43..575d06f1 100644 --- a/lib/oxsdatatypes/src/date_time.rs +++ b/lib/oxsdatatypes/src/date_time.rs @@ -256,7 +256,7 @@ impl TryFrom for DateTime { type Error = DateTimeOverflowError; #[inline] - fn try_from(date: Date) -> Result { + fn try_from(date: Date) -> Result { Self::new( date.year(), date.month(), @@ -272,7 +272,7 @@ impl TryFrom for DateTime { impl FromStr for DateTime { type Err = ParseDateTimeError; - fn from_str(input: &str) -> Result { + fn from_str(input: &str) -> Result { ensure_complete(input, date_time_lexical_rep) } } @@ -528,7 +528,7 @@ impl From for Time { impl FromStr for Time { type Err = ParseDateTimeError; - fn from_str(input: &str) -> Result { + fn from_str(input: &str) -> Result { ensure_complete(input, time_lexical_rep) } } @@ -762,7 +762,7 @@ impl TryFrom for Date { type Error = DateTimeOverflowError; #[inline] - fn try_from(date_time: DateTime) -> Result { + fn try_from(date_time: DateTime) -> Result { Self::new( date_time.year(), date_time.month(), @@ -775,7 +775,7 @@ impl TryFrom for Date { impl FromStr for Date { type Err = ParseDateTimeError; - fn from_str(input: &str) -> Result { + fn from_str(input: &str) -> Result { ensure_complete(input, date_lexical_rep) } } @@ -896,7 +896,7 @@ impl TryFrom for GYearMonth { type Error = DateTimeOverflowError; #[inline] - fn try_from(date_time: DateTime) -> Result { + fn try_from(date_time: DateTime) -> Result { Self::new( date_time.year(), date_time.month(), @@ -917,7 +917,7 @@ impl From for GYearMonth { impl FromStr for GYearMonth { type Err = ParseDateTimeError; - fn from_str(input: &str) -> Result { + fn from_str(input: &str) -> Result { ensure_complete(input, g_year_month_lexical_rep) } } @@ -1031,7 +1031,7 @@ impl TryFrom for GYear { type Error = DateTimeOverflowError; #[inline] - fn try_from(date_time: DateTime) -> Result { + fn try_from(date_time: DateTime) -> Result { Self::new(date_time.year(), date_time.timezone_offset()) } } @@ -1041,7 +1041,7 @@ impl TryFrom for GYear { type Error = DateTimeOverflowError; #[inline] - fn try_from(date: Date) -> Result { + fn try_from(date: Date) -> Result { Self::new(date.year(), date.timezone_offset()) } } @@ -1050,7 +1050,7 @@ impl TryFrom for GYear { type Error = DateTimeOverflowError; #[inline] - fn try_from(year_month: GYearMonth) -> Result { + fn try_from(year_month: GYearMonth) -> Result { Self::new(year_month.year(), year_month.timezone_offset()) } } @@ -1058,7 +1058,7 @@ impl TryFrom for GYear { impl FromStr for GYear { type Err = ParseDateTimeError; - fn from_str(input: &str) -> Result { + fn from_str(input: &str) -> Result { ensure_complete(input, g_year_lexical_rep) } } @@ -1186,7 +1186,7 @@ impl From for GMonthDay { impl FromStr for GMonthDay { type Err = ParseDateTimeError; - fn from_str(input: &str) -> Result { + fn from_str(input: &str) -> Result { ensure_complete(input, g_month_day_lexical_rep) } } @@ -1315,7 +1315,7 @@ impl From for GMonth { impl FromStr for GMonth { type Err = ParseDateTimeError; - fn from_str(input: &str) -> Result { + fn from_str(input: &str) -> Result { ensure_complete(input, g_month_lexical_rep) } } @@ -1436,7 +1436,7 @@ impl From for GDay { impl FromStr for GDay { type Err = ParseDateTimeError; - fn from_str(input: &str) -> Result { + fn from_str(input: &str) -> Result { ensure_complete(input, g_day_lexical_rep) } } @@ -1499,7 +1499,7 @@ impl TryFrom for TimezoneOffset { type Error = InvalidTimezoneError; #[inline] - fn try_from(value: DayTimeDuration) -> Result { + fn try_from(value: DayTimeDuration) -> Result { let offset_in_minutes = value.minutes() + value.hours() * 60; let result = Self::new( offset_in_minutes @@ -1519,7 +1519,7 @@ impl TryFrom for TimezoneOffset { type Error = InvalidTimezoneError; #[inline] - fn try_from(value: Duration) -> Result { + fn try_from(value: Duration) -> Result { DayTimeDuration::try_from(value) .map_err(|_| InvalidTimezoneError { offset_in_minutes: 0, diff --git a/lib/oxsdatatypes/src/decimal.rs b/lib/oxsdatatypes/src/decimal.rs index bb2090fd..6a59105e 100644 --- a/lib/oxsdatatypes/src/decimal.rs +++ b/lib/oxsdatatypes/src/decimal.rs @@ -361,7 +361,7 @@ impl TryFrom for Decimal { type Error = TooLargeForDecimalError; #[inline] - fn try_from(value: i128) -> Result { + fn try_from(value: i128) -> Result { Ok(Self { value: value .checked_mul(DECIMAL_PART_POW) @@ -374,7 +374,7 @@ impl TryFrom for Decimal { type Error = TooLargeForDecimalError; #[inline] - fn try_from(value: u128) -> Result { + fn try_from(value: u128) -> Result { Ok(Self { value: i128::try_from(value) .map_err(|_| TooLargeForDecimalError)? @@ -395,7 +395,7 @@ impl TryFrom for Decimal { type Error = TooLargeForDecimalError; #[inline] - fn try_from(value: Float) -> Result { + fn try_from(value: Float) -> Result { Double::from(value).try_into() } } @@ -405,7 +405,7 @@ impl TryFrom for Decimal { #[inline] #[allow(clippy::cast_precision_loss, clippy::cast_possible_truncation)] - fn try_from(value: Double) -> Result { + fn try_from(value: Double) -> Result { let shifted = f64::from(value) * (DECIMAL_PART_POW as f64); if (i128::MIN as f64) <= shifted && shifted <= (i128::MAX as f64) { Ok(Self { @@ -448,7 +448,7 @@ impl TryFrom for Integer { type Error = TooLargeForIntegerError; #[inline] - fn try_from(value: Decimal) -> Result { + fn try_from(value: Decimal) -> Result { Ok(i64::try_from( value .value @@ -464,7 +464,7 @@ impl FromStr for Decimal { type Err = ParseDecimalError; /// Parses decimals lexical mapping - fn from_str(input: &str) -> Result { + fn from_str(input: &str) -> Result { // (\+|-)?([0-9]+(\.[0-9]*)?|\.[0-9]+) let input = input.as_bytes(); if input.is_empty() { diff --git a/lib/oxsdatatypes/src/double.rs b/lib/oxsdatatypes/src/double.rs index a9d19ac5..1a399019 100644 --- a/lib/oxsdatatypes/src/double.rs +++ b/lib/oxsdatatypes/src/double.rs @@ -189,7 +189,7 @@ impl FromStr for Double { type Err = ParseFloatError; #[inline] - fn from_str(input: &str) -> Result { + fn from_str(input: &str) -> Result { Ok(f64::from_str(input)?.into()) } } diff --git a/lib/oxsdatatypes/src/duration.rs b/lib/oxsdatatypes/src/duration.rs index 1cf33ffb..d6790a8b 100644 --- a/lib/oxsdatatypes/src/duration.rs +++ b/lib/oxsdatatypes/src/duration.rs @@ -176,7 +176,7 @@ impl TryFrom for Duration { type Error = DurationOverflowError; #[inline] - fn try_from(value: StdDuration) -> Result { + fn try_from(value: StdDuration) -> Result { Ok(DayTimeDuration::try_from(value)?.into()) } } @@ -184,7 +184,7 @@ impl TryFrom for Duration { impl FromStr for Duration { type Err = ParseDurationError; - fn from_str(input: &str) -> Result { + fn from_str(input: &str) -> Result { let parts = ensure_complete(input, duration_parts)?; if parts.year_month.is_none() && parts.day_time.is_none() { return Err(ParseDurationError::msg("Empty duration")); @@ -394,7 +394,7 @@ impl TryFrom for YearMonthDuration { type Error = DurationOverflowError; #[inline] - fn try_from(value: Duration) -> Result { + fn try_from(value: Duration) -> Result { if value.day_time == DayTimeDuration::default() { Ok(value.year_month) } else { @@ -406,7 +406,7 @@ impl TryFrom for YearMonthDuration { impl FromStr for YearMonthDuration { type Err = ParseDurationError; - fn from_str(input: &str) -> Result { + fn from_str(input: &str) -> Result { let parts = ensure_complete(input, duration_parts)?; if parts.day_time.is_some() { return Err(ParseDurationError::msg( @@ -580,7 +580,7 @@ impl TryFrom for DayTimeDuration { type Error = DurationOverflowError; #[inline] - fn try_from(value: Duration) -> Result { + fn try_from(value: Duration) -> Result { if value.year_month == YearMonthDuration::default() { Ok(value.day_time) } else { @@ -593,7 +593,7 @@ impl TryFrom for DayTimeDuration { type Error = DurationOverflowError; #[inline] - fn try_from(value: StdDuration) -> Result { + fn try_from(value: StdDuration) -> Result { Ok(Self { seconds: Decimal::new( i128::try_from(value.as_nanos()).map_err(|_| DurationOverflowError)?, @@ -608,7 +608,7 @@ impl TryFrom for StdDuration { type Error = DurationOverflowError; #[inline] - fn try_from(value: DayTimeDuration) -> Result { + fn try_from(value: DayTimeDuration) -> Result { if value.seconds.is_negative() { return Err(DurationOverflowError); } @@ -636,7 +636,7 @@ impl TryFrom for StdDuration { impl FromStr for DayTimeDuration { type Err = ParseDurationError; - fn from_str(input: &str) -> Result { + fn from_str(input: &str) -> Result { let parts = ensure_complete(input, duration_parts)?; if parts.year_month.is_some() { return Err(ParseDurationError::msg( diff --git a/lib/oxsdatatypes/src/float.rs b/lib/oxsdatatypes/src/float.rs index af4c66f7..bc0aab75 100644 --- a/lib/oxsdatatypes/src/float.rs +++ b/lib/oxsdatatypes/src/float.rs @@ -179,7 +179,7 @@ impl FromStr for Float { type Err = ParseFloatError; #[inline] - fn from_str(input: &str) -> Result { + fn from_str(input: &str) -> Result { Ok(f32::from_str(input)?.into()) } } diff --git a/lib/oxsdatatypes/src/integer.rs b/lib/oxsdatatypes/src/integer.rs index 376deea6..f2b8506f 100644 --- a/lib/oxsdatatypes/src/integer.rs +++ b/lib/oxsdatatypes/src/integer.rs @@ -228,7 +228,7 @@ impl FromStr for Integer { type Err = ParseIntError; #[inline] - fn from_str(input: &str) -> Result { + fn from_str(input: &str) -> Result { Ok(i64::from_str(input)?.into()) } } @@ -244,7 +244,7 @@ impl TryFrom for Integer { type Error = TooLargeForIntegerError; #[inline] - fn try_from(value: Float) -> Result { + fn try_from(value: Float) -> Result { Decimal::try_from(value) .map_err(|_| TooLargeForIntegerError)? .try_into() @@ -255,7 +255,7 @@ impl TryFrom for Integer { type Error = TooLargeForIntegerError; #[inline] - fn try_from(value: Double) -> Result { + fn try_from(value: Double) -> Result { Decimal::try_from(value) .map_err(|_| TooLargeForIntegerError)? .try_into() diff --git a/lib/oxttl/src/lexer.rs b/lib/oxttl/src/lexer.rs index 9bb8c49e..d4eb024f 100644 --- a/lib/oxttl/src/lexer.rs +++ b/lib/oxttl/src/lexer.rs @@ -56,7 +56,7 @@ impl TokenRecognizer for N3Lexer { &mut self, data: &'a [u8], is_ending: bool, - options: &N3LexerOptions, + options: &Self::Options, ) -> Option<(usize, Result, TokenRecognizerError>)> { match *data.first()? { b'<' => match *data.get(1)? { diff --git a/lib/oxttl/src/n3.rs b/lib/oxttl/src/n3.rs index eb1ce55d..929f5c0f 100644 --- a/lib/oxttl/src/n3.rs +++ b/lib/oxttl/src/n3.rs @@ -450,7 +450,7 @@ impl FromReadN3Reader { impl Iterator for FromReadN3Reader { type Item = Result; - fn next(&mut self) -> Option> { + fn next(&mut self) -> Option { self.inner.next() } } diff --git a/lib/oxttl/src/nquads.rs b/lib/oxttl/src/nquads.rs index d93c7293..f5108828 100644 --- a/lib/oxttl/src/nquads.rs +++ b/lib/oxttl/src/nquads.rs @@ -215,7 +215,7 @@ pub struct FromReadNQuadsReader { impl Iterator for FromReadNQuadsReader { type Item = Result; - fn next(&mut self) -> Option> { + fn next(&mut self) -> Option { self.inner.next() } } diff --git a/lib/oxttl/src/ntriples.rs b/lib/oxttl/src/ntriples.rs index c5ed5b94..4e0f7d7c 100644 --- a/lib/oxttl/src/ntriples.rs +++ b/lib/oxttl/src/ntriples.rs @@ -215,7 +215,7 @@ pub struct FromReadNTriplesReader { impl Iterator for FromReadNTriplesReader { type Item = Result; - fn next(&mut self) -> Option> { + fn next(&mut self) -> Option { Some(self.inner.next()?.map(Into::into)) } } diff --git a/lib/oxttl/src/trig.rs b/lib/oxttl/src/trig.rs index 08fbf271..0dad7fd7 100644 --- a/lib/oxttl/src/trig.rs +++ b/lib/oxttl/src/trig.rs @@ -300,7 +300,7 @@ impl FromReadTriGReader { impl Iterator for FromReadTriGReader { type Item = Result; - fn next(&mut self) -> Option> { + fn next(&mut self) -> Option { self.inner.next() } } diff --git a/lib/oxttl/src/turtle.rs b/lib/oxttl/src/turtle.rs index 19590b94..542afd27 100644 --- a/lib/oxttl/src/turtle.rs +++ b/lib/oxttl/src/turtle.rs @@ -302,7 +302,7 @@ impl FromReadTurtleReader { impl Iterator for FromReadTurtleReader { type Item = Result; - fn next(&mut self) -> Option> { + fn next(&mut self) -> Option { Some(self.inner.next()?.map(Into::into)) } } diff --git a/lib/sparesults/src/parser.rs b/lib/sparesults/src/parser.rs index 3e521fe8..a00d014c 100644 --- a/lib/sparesults/src/parser.rs +++ b/lib/sparesults/src/parser.rs @@ -206,7 +206,7 @@ impl FromReadSolutionsReader { impl Iterator for FromReadSolutionsReader { type Item = Result; - fn next(&mut self) -> Option> { + fn next(&mut self) -> Option { Some( match &mut self.solutions { SolutionsReaderKind::Xml(reader) => reader.read_next(), diff --git a/lib/sparesults/src/solution.rs b/lib/sparesults/src/solution.rs index 3990ab79..0d81adc2 100644 --- a/lib/sparesults/src/solution.rs +++ b/lib/sparesults/src/solution.rs @@ -130,7 +130,7 @@ impl<'a> IntoIterator for &'a QuerySolution { type IntoIter = Iter<'a>; #[inline] - fn into_iter(self) -> Iter<'a> { + fn into_iter(self) -> Self::IntoIter { Iter { inner: self.variables.iter().zip(&self.values), } @@ -142,7 +142,7 @@ impl Index for QuerySolution { #[allow(clippy::panic)] #[inline] - fn index(&self, index: usize) -> &Term { + fn index(&self, index: usize) -> &Self::Output { self.get(index) .unwrap_or_else(|| panic!("The column {index} is not set in this solution")) } @@ -153,7 +153,7 @@ impl Index<&str> for QuerySolution { #[allow(clippy::panic)] #[inline] - fn index(&self, index: &str) -> &Term { + fn index(&self, index: &str) -> &Self::Output { self.get(index) .unwrap_or_else(|| panic!("The variable ?{index} is not set in this solution")) } @@ -164,7 +164,7 @@ impl Index> for QuerySolution { #[allow(clippy::panic)] #[inline] - fn index(&self, index: VariableRef<'_>) -> &Term { + fn index(&self, index: VariableRef<'_>) -> &Self::Output { self.get(index) .unwrap_or_else(|| panic!("The variable {index} is not set in this solution")) } @@ -173,7 +173,7 @@ impl Index for QuerySolution { type Output = Term; #[inline] - fn index(&self, index: Variable) -> &Term { + fn index(&self, index: Variable) -> &Self::Output { self.index(index.as_ref()) } } @@ -182,7 +182,7 @@ impl Index<&Variable> for QuerySolution { type Output = Term; #[inline] - fn index(&self, index: &Variable) -> &Term { + fn index(&self, index: &Variable) -> &Self::Output { self.index(index.as_ref()) } } @@ -228,7 +228,7 @@ impl<'a> Iterator for Iter<'a> { type Item = (&'a Variable, &'a Term); #[inline] - fn next(&mut self) -> Option<(&'a Variable, &'a Term)> { + fn next(&mut self) -> Option { for (variable, value) in &mut self.inner { if let Some(value) = value { return Some((variable, value)); diff --git a/lib/spargebra/src/query.rs b/lib/spargebra/src/query.rs index 6cb7e57b..8716de73 100644 --- a/lib/spargebra/src/query.rs +++ b/lib/spargebra/src/query.rs @@ -275,7 +275,7 @@ impl fmt::Display for Query { impl FromStr for Query { type Err = ParseError; - fn from_str(query: &str) -> Result { + fn from_str(query: &str) -> Result { Self::parse(query, None) } } @@ -283,7 +283,7 @@ impl FromStr for Query { impl<'a> TryFrom<&'a str> for Query { type Error = ParseError; - fn try_from(query: &str) -> Result { + fn try_from(query: &str) -> Result { Self::from_str(query) } } @@ -291,7 +291,7 @@ impl<'a> TryFrom<&'a str> for Query { impl<'a> TryFrom<&'a String> for Query { type Error = ParseError; - fn try_from(query: &String) -> Result { + fn try_from(query: &String) -> Result { Self::from_str(query) } } diff --git a/lib/spargebra/src/term.rs b/lib/spargebra/src/term.rs index 6a33294c..362b3959 100644 --- a/lib/spargebra/src/term.rs +++ b/lib/spargebra/src/term.rs @@ -48,7 +48,7 @@ impl TryFrom for GroundSubject { type Error = (); #[inline] - fn try_from(subject: Subject) -> Result { + fn try_from(subject: Subject) -> Result { match subject { Subject::NamedNode(t) => Ok(t.into()), Subject::BlankNode(_) => Err(()), @@ -62,7 +62,7 @@ impl TryFrom for GroundSubject { type Error = (); #[inline] - fn try_from(term: GroundTerm) -> Result { + fn try_from(term: GroundTerm) -> Result { match term { GroundTerm::NamedNode(t) => Ok(t.into()), GroundTerm::Literal(_) => Err(()), @@ -125,7 +125,7 @@ impl TryFrom for GroundTerm { type Error = (); #[inline] - fn try_from(term: Term) -> Result { + fn try_from(term: Term) -> Result { match term { Term::NamedNode(t) => Ok(t.into()), Term::BlankNode(_) => Err(()), @@ -171,7 +171,7 @@ impl TryFrom for GroundTriple { type Error = (); #[inline] - fn try_from(triple: Triple) -> Result { + fn try_from(triple: Triple) -> Result { Ok(Self { subject: triple.subject.try_into()?, predicate: triple.predicate, @@ -221,7 +221,7 @@ impl TryFrom for GraphName { type Error = (); #[inline] - fn try_from(pattern: GraphNamePattern) -> Result { + fn try_from(pattern: GraphNamePattern) -> Result { match pattern { GraphNamePattern::NamedNode(t) => Ok(t.into()), GraphNamePattern::DefaultGraph => Ok(Self::DefaultGraph), @@ -295,7 +295,7 @@ impl TryFrom for Quad { type Error = (); #[inline] - fn try_from(quad: QuadPattern) -> Result { + fn try_from(quad: QuadPattern) -> Result { Ok(Self { subject: quad.subject.try_into()?, predicate: quad.predicate.try_into()?, @@ -370,7 +370,7 @@ impl TryFrom for GroundQuad { type Error = (); #[inline] - fn try_from(quad: Quad) -> Result { + fn try_from(quad: Quad) -> Result { Ok(Self { subject: quad.subject.try_into()?, predicate: quad.predicate, @@ -425,7 +425,7 @@ impl TryFrom for NamedNode { type Error = (); #[inline] - fn try_from(pattern: NamedNodePattern) -> Result { + fn try_from(pattern: NamedNodePattern) -> Result { match pattern { NamedNodePattern::NamedNode(t) => Ok(t), NamedNodePattern::Variable(_) => Err(()), @@ -559,7 +559,7 @@ impl TryFrom for Subject { type Error = (); #[inline] - fn try_from(term: TermPattern) -> Result { + fn try_from(term: TermPattern) -> Result { match term { TermPattern::NamedNode(t) => Ok(t.into()), TermPattern::BlankNode(t) => Ok(t.into()), @@ -574,7 +574,7 @@ impl TryFrom for Term { type Error = (); #[inline] - fn try_from(pattern: TermPattern) -> Result { + fn try_from(pattern: TermPattern) -> Result { match pattern { TermPattern::NamedNode(t) => Ok(t.into()), TermPattern::BlankNode(t) => Ok(t.into()), @@ -686,7 +686,7 @@ impl TryFrom for GroundTermPattern { type Error = (); #[inline] - fn try_from(pattern: TermPattern) -> Result { + fn try_from(pattern: TermPattern) -> Result { Ok(match pattern { TermPattern::NamedNode(named_node) => named_node.into(), TermPattern::BlankNode(_) => return Err(()), @@ -828,7 +828,7 @@ impl TryFrom for Triple { type Error = (); #[inline] - fn try_from(triple: TriplePattern) -> Result { + fn try_from(triple: TriplePattern) -> Result { Ok(Self { subject: triple.subject.try_into()?, predicate: triple.predicate.try_into()?, @@ -1000,7 +1000,7 @@ impl TryFrom for GroundQuadPattern { type Error = (); #[inline] - fn try_from(pattern: QuadPattern) -> Result { + fn try_from(pattern: QuadPattern) -> Result { Ok(Self { subject: pattern.subject.try_into()?, predicate: pattern.predicate, diff --git a/lib/spargebra/src/update.rs b/lib/spargebra/src/update.rs index e73b234e..e2a2c653 100644 --- a/lib/spargebra/src/update.rs +++ b/lib/spargebra/src/update.rs @@ -70,7 +70,7 @@ impl fmt::Display for Update { impl FromStr for Update { type Err = ParseError; - fn from_str(update: &str) -> Result { + fn from_str(update: &str) -> Result { Self::parse(update, None) } } @@ -78,7 +78,7 @@ impl FromStr for Update { impl<'a> TryFrom<&'a str> for Update { type Error = ParseError; - fn try_from(update: &str) -> Result { + fn try_from(update: &str) -> Result { Self::from_str(update) } } @@ -86,7 +86,7 @@ impl<'a> TryFrom<&'a str> for Update { impl<'a> TryFrom<&'a String> for Update { type Error = ParseError; - fn try_from(update: &String) -> Result { + fn try_from(update: &String) -> Result { Self::from_str(update) } } diff --git a/lib/src/io/format.rs b/lib/src/io/format.rs index 89ba37f9..b07c1709 100644 --- a/lib/src/io/format.rs +++ b/lib/src/io/format.rs @@ -258,7 +258,7 @@ impl TryFrom for GraphFormat { /// Attempts to find a graph format that is a subset of this [`DatasetFormat`]. #[inline] - fn try_from(value: DatasetFormat) -> Result { + fn try_from(value: DatasetFormat) -> Result { match value { DatasetFormat::NQuads => Ok(Self::NTriples), DatasetFormat::TriG => Ok(Self::Turtle), @@ -271,7 +271,7 @@ impl TryFrom for DatasetFormat { /// Attempts to find a dataset format that is a superset of this [`GraphFormat`]. #[inline] - fn try_from(value: GraphFormat) -> Result { + fn try_from(value: GraphFormat) -> Result { match value { GraphFormat::NTriples => Ok(Self::NQuads), GraphFormat::Turtle => Ok(Self::TriG), diff --git a/lib/src/io/read.rs b/lib/src/io/read.rs index 33065615..3400b8e2 100644 --- a/lib/src/io/read.rs +++ b/lib/src/io/read.rs @@ -95,7 +95,7 @@ pub struct TripleReader { impl Iterator for TripleReader { type Item = Result; - fn next(&mut self) -> Option> { + fn next(&mut self) -> Option { Some(self.parser.next()?.map(Into::into).map_err(Into::into)) } } @@ -184,7 +184,7 @@ pub struct QuadReader { impl Iterator for QuadReader { type Item = Result; - fn next(&mut self) -> Option> { + fn next(&mut self) -> Option { Some(self.parser.next()?.map_err(Into::into)) } } diff --git a/lib/src/sparql/algebra.rs b/lib/src/sparql/algebra.rs index d83241c6..b046de80 100644 --- a/lib/src/sparql/algebra.rs +++ b/lib/src/sparql/algebra.rs @@ -65,7 +65,7 @@ impl fmt::Display for Query { impl FromStr for Query { type Err = spargebra::ParseError; - fn from_str(query: &str) -> Result { + fn from_str(query: &str) -> Result { Self::parse(query, None) } } @@ -73,7 +73,7 @@ impl FromStr for Query { impl TryFrom<&str> for Query { type Error = spargebra::ParseError; - fn try_from(query: &str) -> Result { + fn try_from(query: &str) -> Result { Self::from_str(query) } } @@ -81,7 +81,7 @@ impl TryFrom<&str> for Query { impl TryFrom<&String> for Query { type Error = spargebra::ParseError; - fn try_from(query: &String) -> Result { + fn try_from(query: &String) -> Result { Self::from_str(query) } } @@ -158,7 +158,7 @@ impl fmt::Display for Update { impl FromStr for Update { type Err = spargebra::ParseError; - fn from_str(update: &str) -> Result { + fn from_str(update: &str) -> Result { Self::parse(update, None) } } @@ -166,7 +166,7 @@ impl FromStr for Update { impl TryFrom<&str> for Update { type Error = spargebra::ParseError; - fn try_from(update: &str) -> Result { + fn try_from(update: &str) -> Result { Self::from_str(update) } } @@ -174,7 +174,7 @@ impl TryFrom<&str> for Update { impl TryFrom<&String> for Update { type Error = spargebra::ParseError; - fn try_from(update: &String) -> Result { + fn try_from(update: &String) -> Result { Self::from_str(update) } } diff --git a/lib/src/sparql/eval.rs b/lib/src/sparql/eval.rs index 9ae02da1..785c64cd 100644 --- a/lib/src/sparql/eval.rs +++ b/lib/src/sparql/eval.rs @@ -4732,7 +4732,7 @@ struct CartesianProductJoinIterator { impl Iterator for CartesianProductJoinIterator { type Item = Result; - fn next(&mut self) -> Option> { + fn next(&mut self) -> Option { loop { if let Some(result) = self.buffered_results.pop() { return Some(result); @@ -4767,7 +4767,7 @@ struct HashJoinIterator { impl Iterator for HashJoinIterator { type Item = Result; - fn next(&mut self) -> Option> { + fn next(&mut self) -> Option { loop { if let Some(result) = self.buffered_results.pop() { return Some(result); @@ -4806,7 +4806,7 @@ struct HashLeftJoinIterator { impl Iterator for HashLeftJoinIterator { type Item = Result; - fn next(&mut self) -> Option> { + fn next(&mut self) -> Option { loop { if let Some(result) = self.buffered_results.pop() { return Some(result); @@ -4854,7 +4854,7 @@ struct ForLoopLeftJoinIterator { impl Iterator for ForLoopLeftJoinIterator { type Item = Result; - fn next(&mut self) -> Option> { + fn next(&mut self) -> Option { if let Some(tuple) = self.current_right.next() { return Some(tuple); } @@ -4881,7 +4881,7 @@ struct UnionIterator { impl Iterator for UnionIterator { type Item = Result; - fn next(&mut self) -> Option> { + fn next(&mut self) -> Option { loop { if let Some(tuple) = self.current_iterator.next() { return Some(tuple); @@ -4903,7 +4903,7 @@ struct ConsecutiveDeduplication { impl Iterator for ConsecutiveDeduplication { type Item = Result; - fn next(&mut self) -> Option> { + fn next(&mut self) -> Option { // Basic idea. We buffer the previous result and we only emit it when we kow the next one or it's the end loop { if let Some(next) = self.inner.next() { @@ -4944,7 +4944,7 @@ struct ConstructIterator { impl Iterator for ConstructIterator { type Item = Result; - fn next(&mut self) -> Option> { + fn next(&mut self) -> Option { loop { if let Some(result) = self.buffered_results.pop() { return Some(result); @@ -5046,7 +5046,7 @@ struct DescribeIterator { impl Iterator for DescribeIterator { type Item = Result; - fn next(&mut self) -> Option> { + fn next(&mut self) -> Option { loop { if let Some(quad) = self.quads.next() { return Some(match quad { @@ -5097,7 +5097,7 @@ impl, I2: Iterator> Iterator { type Item = (Option, Option); - fn next(&mut self) -> Option<(Option, Option)> { + fn next(&mut self) -> Option { match (self.a.next(), self.b.next()) { (None, None) => None, r => Some(r), @@ -5220,7 +5220,7 @@ impl< { type Item = Result; - fn next(&mut self) -> Option> { + fn next(&mut self) -> Option { loop { if let Some(current) = &mut self.current { if let Some(next) = current.next() { @@ -5629,7 +5629,7 @@ struct StatsIterator { impl Iterator for StatsIterator { type Item = Result; - fn next(&mut self) -> Option> { + fn next(&mut self) -> Option { let start = Timer::now(); let result = self.inner.next(); self.stats.exec_duration.set( diff --git a/lib/src/sparql/model.rs b/lib/src/sparql/model.rs index faedc7f5..ca42db16 100644 --- a/lib/src/sparql/model.rs +++ b/lib/src/sparql/model.rs @@ -221,7 +221,7 @@ impl Iterator for QuerySolutionIter { type Item = Result; #[inline] - fn next(&mut self) -> Option> { + fn next(&mut self) -> Option { self.iter.next() } @@ -253,7 +253,7 @@ impl Iterator for QueryTripleIter { type Item = Result; #[inline] - fn next(&mut self) -> Option> { + fn next(&mut self) -> Option { self.iter.next() } diff --git a/lib/src/sparql/service.rs b/lib/src/sparql/service.rs index 4db172e4..562c1896 100644 --- a/lib/src/sparql/service.rs +++ b/lib/src/sparql/service.rs @@ -24,7 +24,7 @@ use std::time::Duration; /// impl ServiceHandler for TestServiceHandler { /// type Error = EvaluationError; /// -/// fn handle(&self,service_name: NamedNode, query: Query) -> Result { +/// fn handle(&self, service_name: NamedNode, query: Query) -> Result { /// if service_name == "http://example.com/service" { /// self.store.query(query) /// } else { @@ -61,7 +61,7 @@ pub struct EmptyServiceHandler; impl ServiceHandler for EmptyServiceHandler { type Error = EvaluationError; - fn handle(&self, name: NamedNode, _: Query) -> Result { + fn handle(&self, name: NamedNode, _: Query) -> Result { Err(EvaluationError::UnsupportedService(name)) } } @@ -79,11 +79,7 @@ impl ErrorConversionServiceHandler { impl ServiceHandler for ErrorConversionServiceHandler { type Error = EvaluationError; - fn handle( - &self, - service_name: NamedNode, - query: Query, - ) -> Result { + fn handle(&self, service_name: NamedNode, query: Query) -> Result { self.handler .handle(service_name, query) .map_err(|e| EvaluationError::Service(Box::new(e))) @@ -105,11 +101,7 @@ impl SimpleServiceHandler { impl ServiceHandler for SimpleServiceHandler { type Error = EvaluationError; - fn handle( - &self, - service_name: NamedNode, - query: Query, - ) -> Result { + fn handle(&self, service_name: NamedNode, query: Query) -> Result { let (content_type, body) = self .client .post( diff --git a/lib/src/storage/backend/rocksdb.rs b/lib/src/storage/backend/rocksdb.rs index acac6585..670963ba 100644 --- a/lib/src/storage/backend/rocksdb.rs +++ b/lib/src/storage/backend/rocksdb.rs @@ -1157,7 +1157,7 @@ impl Drop for PinnableSlice { impl Deref for PinnableSlice { type Target = [u8]; - fn deref(&self) -> &[u8] { + fn deref(&self) -> &Self::Target { unsafe { let mut len = 0; let val = rocksdb_pinnableslice_value(self.0, &mut len); @@ -1200,7 +1200,7 @@ impl Drop for Buffer { impl Deref for Buffer { type Target = [u8]; - fn deref(&self) -> &[u8] { + fn deref(&self) -> &Self::Target { unsafe { slice::from_raw_parts(self.base, self.len) } } } diff --git a/lib/src/storage/mod.rs b/lib/src/storage/mod.rs index f592aeb9..9dd38ee1 100644 --- a/lib/src/storage/mod.rs +++ b/lib/src/storage/mod.rs @@ -814,7 +814,7 @@ impl ChainedDecodingQuadIterator { impl Iterator for ChainedDecodingQuadIterator { type Item = Result; - fn next(&mut self) -> Option> { + fn next(&mut self) -> Option { if let Some(result) = self.first.next() { Some(result) } else if let Some(second) = self.second.as_mut() { @@ -833,7 +833,7 @@ pub struct DecodingQuadIterator { impl Iterator for DecodingQuadIterator { type Item = Result; - fn next(&mut self) -> Option> { + fn next(&mut self) -> Option { if let Err(e) = self.iter.status() { return Some(Err(e)); } @@ -850,7 +850,7 @@ pub struct DecodingGraphIterator { impl Iterator for DecodingGraphIterator { type Item = Result; - fn next(&mut self) -> Option> { + fn next(&mut self) -> Option { if let Err(e) = self.iter.status() { return Some(Err(e)); } diff --git a/lib/src/storage/small_string.rs b/lib/src/storage/small_string.rs index be836c4d..70a15676 100644 --- a/lib/src/storage/small_string.rs +++ b/lib/src/storage/small_string.rs @@ -65,7 +65,7 @@ impl Deref for SmallString { type Target = str; #[inline] - fn deref(&self) -> &str { + fn deref(&self) -> &Self::Target { self.as_str() } } @@ -146,7 +146,7 @@ impl FromStr for SmallString { type Err = BadSmallStringError; #[inline] - fn from_str(value: &str) -> Result { + fn from_str(value: &str) -> Result { if value.len() <= 15 { let mut inner = [0; 16]; inner[..value.len()].copy_from_slice(value.as_bytes()); @@ -165,7 +165,7 @@ impl<'a> TryFrom<&'a str> for SmallString { type Error = BadSmallStringError; #[inline] - fn try_from(value: &'a str) -> Result { + fn try_from(value: &'a str) -> Result { Self::from_str(value) } } diff --git a/lib/src/store.rs b/lib/src/store.rs index 9b448141..95a25260 100644 --- a/lib/src/store.rs +++ b/lib/src/store.rs @@ -1471,7 +1471,7 @@ pub struct QuadIter { impl Iterator for QuadIter { type Item = Result; - fn next(&mut self) -> Option> { + fn next(&mut self) -> Option { Some(match self.iter.next()? { Ok(quad) => self.reader.decode_quad(&quad), Err(error) => Err(error), @@ -1488,7 +1488,7 @@ pub struct GraphNameIter { impl Iterator for GraphNameIter { type Item = Result; - fn next(&mut self) -> Option> { + fn next(&mut self) -> Option { Some( self.iter .next()? diff --git a/testsuite/src/manifest.rs b/testsuite/src/manifest.rs index 00a0f6a6..97ee0ea7 100644 --- a/testsuite/src/manifest.rs +++ b/testsuite/src/manifest.rs @@ -58,7 +58,7 @@ pub struct TestManifest { impl Iterator for TestManifest { type Item = Result; - fn next(&mut self) -> Option> { + fn next(&mut self) -> Option { loop { if let Some(next) = self.next_test().transpose() { return Some(next); @@ -355,7 +355,7 @@ impl<'a> RdfListIterator<'a> { impl<'a> Iterator for RdfListIterator<'a> { type Item = Term; - fn next(&mut self) -> Option { + fn next(&mut self) -> Option { match self.current_node { Some(current) => { let result = self diff --git a/testsuite/src/sparql_evaluator.rs b/testsuite/src/sparql_evaluator.rs index df9d471e..844158d9 100644 --- a/testsuite/src/sparql_evaluator.rs +++ b/testsuite/src/sparql_evaluator.rs @@ -727,7 +727,7 @@ impl Drop for StoreRef { impl Deref for StoreRef { type Target = Store; - fn deref(&self) -> &Store { + fn deref(&self) -> &Self::Target { &self.store } } From 5f603bc4fe2547fb32375375b286ab0c8c450111 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Wed, 17 Jan 2024 19:42:50 -0500 Subject: [PATCH 180/217] Fix CI status badges --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e035c11b..8f9b7022 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,8 @@ [![Released API docs](https://docs.rs/oxigraph/badge.svg)](https://docs.rs/oxigraph) [![PyPI](https://img.shields.io/pypi/v/pyoxigraph)](https://pypi.org/project/pyoxigraph/) [![npm](https://img.shields.io/npm/v/oxigraph)](https://www.npmjs.com/package/oxigraph) -[![actions status](https://github.com/oxigraph/oxigraph/workflows/build/badge.svg)](https://github.com/oxigraph/oxigraph/actions) +[![tests status](https://github.com/oxigraph/oxigraph/actions/workflows/tests.yml/badge.svg)](https://github.com/oxigraph/oxigraph/actions) +[![artifacts status](https://github.com/oxigraph/oxigraph/actions/workflows/artifacts.yml/badge.svg)](https://github.com/oxigraph/oxigraph/actions) [![dependency status](https://deps.rs/repo/github/oxigraph/oxigraph/status.svg)](https://deps.rs/repo/github/oxigraph/oxigraph) [![Gitter](https://badges.gitter.im/oxigraph/community.svg)](https://gitter.im/oxigraph/community) [![Twitter URL](https://img.shields.io/twitter/url?style=social&url=https%3A%2F%2Ftwitter.com%2Foxigraph)](https://twitter.com/oxigraph) From 405b95b4bdfc2aebabe4ff82dcd2e120528bdc59 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Thu, 18 Jan 2024 02:43:44 -0500 Subject: [PATCH 181/217] Minor linting fixes --- lib/oxsdatatypes/README.md | 2 +- lib/oxsdatatypes/src/duration.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/oxsdatatypes/README.md b/lib/oxsdatatypes/README.md index c7ae8e55..1c3b2c39 100644 --- a/lib/oxsdatatypes/README.md +++ b/lib/oxsdatatypes/README.md @@ -38,7 +38,7 @@ The `DateTime::now()` function needs special OS support. Currently: - If the `custom-now` feature is enabled, a function computing `now` must be set: ```rust - use oxsdatatypes::Duration; + use oxsdatatypes::Duration; #[no_mangle] fn custom_ox_now() -> Duration { diff --git a/lib/oxsdatatypes/src/duration.rs b/lib/oxsdatatypes/src/duration.rs index d6790a8b..8d73cd4a 100644 --- a/lib/oxsdatatypes/src/duration.rs +++ b/lib/oxsdatatypes/src/duration.rs @@ -973,7 +973,7 @@ impl fmt::Display for DurationOverflowError { impl Error for DurationOverflowError {} -/// The year-month and the day-time components of a [`Duration\] have an opposite sign. +/// The year-month and the day-time components of a [`Duration`] have an opposite sign. #[derive(Debug, Clone, Copy)] pub struct OppositeSignInDurationComponentsError; From 5be6f55155329d208632a7b949e378c6e4a253b5 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Thu, 18 Jan 2024 02:45:53 -0500 Subject: [PATCH 182/217] A few more self-fixes These are a bit more questionable but still keep things cleaner a bit, at least in some cases? Most of these were the result of `cargo clippy --fix -- -W clippy::use_self` --- lib/oxrdf/src/parser.rs | 12 ++++++------ lib/oxrdfio/src/error.rs | 6 +++--- lib/oxrdfxml/src/parser.rs | 24 +++++++++++------------ lib/oxsdatatypes/src/date_time.rs | 8 ++++---- lib/oxsdatatypes/src/duration.rs | 24 +++++++++++++---------- lib/oxttl/src/line_formats.rs | 2 +- lib/oxttl/src/n3.rs | 2 +- lib/oxttl/src/terse.rs | 2 +- lib/oxttl/src/toolkit/error.rs | 2 +- lib/sparopt/src/algebra.rs | 32 +++++++++++++++---------------- lib/src/sparql/eval.rs | 6 +++--- lib/src/storage/error.rs | 4 ++-- lib/src/storage/small_string.rs | 4 ++-- 13 files changed, 66 insertions(+), 62 deletions(-) diff --git a/lib/oxrdf/src/parser.rs b/lib/oxrdf/src/parser.rs index f15c0b28..326868f5 100644 --- a/lib/oxrdf/src/parser.rs +++ b/lib/oxrdf/src/parser.rs @@ -28,7 +28,7 @@ impl FromStr for NamedNode { fn from_str(s: &str) -> Result { let (term, left) = read_named_node(s)?; if !left.is_empty() { - return Err(TermParseError::msg( + return Err(Self::Err::msg( "Named node serialization should end with a >", )); } @@ -50,7 +50,7 @@ impl FromStr for BlankNode { fn from_str(s: &str) -> Result { let (term, left) = read_blank_node(s)?; if !left.is_empty() { - return Err(TermParseError::msg( + return Err(Self::Err::msg( "Blank node serialization should not contain whitespaces", )); } @@ -78,7 +78,7 @@ impl FromStr for Literal { fn from_str(s: &str) -> Result { let (term, left) = read_literal(s)?; if !left.is_empty() { - return Err(TermParseError::msg("Invalid literal serialization")); + return Err(Self::Err::msg("Invalid literal serialization")); } Ok(term) } @@ -103,7 +103,7 @@ impl FromStr for Term { fn from_str(s: &str) -> Result { let (term, left) = read_term(s, 0)?; if !left.is_empty() { - return Err(TermParseError::msg("Invalid term serialization")); + return Err(Self::Err::msg("Invalid term serialization")); } Ok(term) } @@ -122,11 +122,11 @@ impl FromStr for Variable { /// ``` fn from_str(s: &str) -> Result { if !s.starts_with('?') && !s.starts_with('$') { - return Err(TermParseError::msg( + return Err(Self::Err::msg( "Variable serialization should start with ? or $", )); } - Self::new(&s[1..]).map_err(|error| TermParseError { + Self::new(&s[1..]).map_err(|error| Self::Err { kind: TermParseErrorKind::Variable { value: s.to_owned(), error, diff --git a/lib/oxrdfio/src/error.rs b/lib/oxrdfio/src/error.rs index 235ba1b7..3b4691f9 100644 --- a/lib/oxrdfio/src/error.rs +++ b/lib/oxrdfio/src/error.rs @@ -42,7 +42,7 @@ impl Error for ParseError { impl From for SyntaxError { #[inline] fn from(error: oxttl::SyntaxError) -> Self { - SyntaxError { + Self { inner: SyntaxErrorKind::Turtle(error), } } @@ -61,7 +61,7 @@ impl From for ParseError { impl From for SyntaxError { #[inline] fn from(error: oxrdfxml::SyntaxError) -> Self { - SyntaxError { + Self { inner: SyntaxErrorKind::RdfXml(error), } } @@ -166,7 +166,7 @@ impl From for io::Error { match error.inner { SyntaxErrorKind::Turtle(error) => error.into(), SyntaxErrorKind::RdfXml(error) => error.into(), - SyntaxErrorKind::Msg { msg } => io::Error::new(io::ErrorKind::InvalidData, msg), + SyntaxErrorKind::Msg { msg } => Self::new(io::ErrorKind::InvalidData, msg), } } } diff --git a/lib/oxrdfxml/src/parser.rs b/lib/oxrdfxml/src/parser.rs index 69f87711..3c6b23ff 100644 --- a/lib/oxrdfxml/src/parser.rs +++ b/lib/oxrdfxml/src/parser.rs @@ -399,23 +399,23 @@ enum RdfXmlState { impl RdfXmlState { fn base_iri(&self) -> Option<&Iri> { match self { - RdfXmlState::Doc { base_iri, .. } - | RdfXmlState::Rdf { base_iri, .. } - | RdfXmlState::NodeElt { base_iri, .. } - | RdfXmlState::PropertyElt { base_iri, .. } - | RdfXmlState::ParseTypeCollectionPropertyElt { base_iri, .. } - | RdfXmlState::ParseTypeLiteralPropertyElt { base_iri, .. } => base_iri.as_ref(), + Self::Doc { base_iri, .. } + | Self::Rdf { base_iri, .. } + | Self::NodeElt { base_iri, .. } + | Self::PropertyElt { base_iri, .. } + | Self::ParseTypeCollectionPropertyElt { base_iri, .. } + | Self::ParseTypeLiteralPropertyElt { base_iri, .. } => base_iri.as_ref(), } } fn language(&self) -> Option<&String> { match self { - RdfXmlState::Doc { .. } => None, - RdfXmlState::Rdf { language, .. } - | RdfXmlState::NodeElt { language, .. } - | RdfXmlState::PropertyElt { language, .. } - | RdfXmlState::ParseTypeCollectionPropertyElt { language, .. } - | RdfXmlState::ParseTypeLiteralPropertyElt { language, .. } => language.as_ref(), + Self::Doc { .. } => None, + Self::Rdf { language, .. } + | Self::NodeElt { language, .. } + | Self::PropertyElt { language, .. } + | Self::ParseTypeCollectionPropertyElt { language, .. } + | Self::ParseTypeLiteralPropertyElt { language, .. } => language.as_ref(), } } } diff --git a/lib/oxsdatatypes/src/date_time.rs b/lib/oxsdatatypes/src/date_time.rs index 575d06f1..d2405692 100644 --- a/lib/oxsdatatypes/src/date_time.rs +++ b/lib/oxsdatatypes/src/date_time.rs @@ -1504,13 +1504,13 @@ impl TryFrom for TimezoneOffset { let result = Self::new( offset_in_minutes .try_into() - .map_err(|_| InvalidTimezoneError { offset_in_minutes })?, + .map_err(|_| Self::Error { offset_in_minutes })?, )?; if DayTimeDuration::from(result) == value { Ok(result) } else { // The value is not an integral number of minutes or overflow problems - Err(InvalidTimezoneError { offset_in_minutes }) + Err(Self::Error { offset_in_minutes }) } } } @@ -1521,7 +1521,7 @@ impl TryFrom for TimezoneOffset { #[inline] fn try_from(value: Duration) -> Result { DayTimeDuration::try_from(value) - .map_err(|_| InvalidTimezoneError { + .map_err(|_| Self::Error { offset_in_minutes: 0, })? .try_into() @@ -2426,7 +2426,7 @@ impl Error for DateTimeOverflowError {} impl From for ParseDateTimeError { fn from(error: DateTimeOverflowError) -> Self { - ParseDateTimeError { + Self { kind: ParseDateTimeErrorKind::Overflow(error), } } diff --git a/lib/oxsdatatypes/src/duration.rs b/lib/oxsdatatypes/src/duration.rs index 8d73cd4a..93dac7d6 100644 --- a/lib/oxsdatatypes/src/duration.rs +++ b/lib/oxsdatatypes/src/duration.rs @@ -187,7 +187,7 @@ impl FromStr for Duration { fn from_str(input: &str) -> Result { let parts = ensure_complete(input, duration_parts)?; if parts.year_month.is_none() && parts.day_time.is_none() { - return Err(ParseDurationError::msg("Empty duration")); + return Err(Self::Err::msg("Empty duration")); } Ok(Self::new( parts.year_month.unwrap_or(0), @@ -409,13 +409,15 @@ impl FromStr for YearMonthDuration { fn from_str(input: &str) -> Result { let parts = ensure_complete(input, duration_parts)?; if parts.day_time.is_some() { - return Err(ParseDurationError::msg( + return Err(Self::Err::msg( "There must not be any day or time component in a yearMonthDuration", )); } - Ok(Self::new(parts.year_month.ok_or( - ParseDurationError::msg("No year and month values found"), - )?)) + Ok(Self::new( + parts + .year_month + .ok_or(Self::Err::msg("No year and month values found"))?, + )) } } @@ -621,7 +623,7 @@ impl TryFrom for StdDuration { .ok_or(DurationOverflowError)? .checked_floor() .ok_or(DurationOverflowError)?; - Ok(StdDuration::new( + Ok(Self::new( secs.as_i128() .try_into() .map_err(|_| DurationOverflowError)?, @@ -639,13 +641,15 @@ impl FromStr for DayTimeDuration { fn from_str(input: &str) -> Result { let parts = ensure_complete(input, duration_parts)?; if parts.year_month.is_some() { - return Err(ParseDurationError::msg( + return Err(Self::Err::msg( "There must not be any year or month component in a dayTimeDuration", )); } - Ok(Self::new(parts.day_time.ok_or(ParseDurationError::msg( - "No day or time values found", - ))?)) + Ok(Self::new( + parts + .day_time + .ok_or(Self::Err::msg("No day or time values found"))?, + )) } } diff --git a/lib/oxttl/src/line_formats.rs b/lib/oxttl/src/line_formats.rs index fc48cd53..e7d39e09 100644 --- a/lib/oxttl/src/line_formats.rs +++ b/lib/oxttl/src/line_formats.rs @@ -274,7 +274,7 @@ impl NQuadsRecognizer { true, Some(b"#"), ), - NQuadsRecognizer { + Self { stack: vec![NQuadsState::ExpectSubject], subjects: Vec::new(), predicates: Vec::new(), diff --git a/lib/oxttl/src/n3.rs b/lib/oxttl/src/n3.rs index 929f5c0f..a1c23f25 100644 --- a/lib/oxttl/src/n3.rs +++ b/lib/oxttl/src/n3.rs @@ -1214,7 +1214,7 @@ impl N3Recognizer { true, Some(b"#"), ), - N3Recognizer { + Self { stack: vec![N3State::N3Doc], terms: Vec::new(), predicates: Vec::new(), diff --git a/lib/oxttl/src/terse.rs b/lib/oxttl/src/terse.rs index bc092c1f..86fad434 100644 --- a/lib/oxttl/src/terse.rs +++ b/lib/oxttl/src/terse.rs @@ -844,7 +844,7 @@ impl TriGRecognizer { true, Some(b"#"), ), - TriGRecognizer { + Self { stack: vec![TriGState::TriGDoc], cur_subject: Vec::new(), cur_predicate: Vec::new(), diff --git a/lib/oxttl/src/toolkit/error.rs b/lib/oxttl/src/toolkit/error.rs index df50b950..e279dab4 100644 --- a/lib/oxttl/src/toolkit/error.rs +++ b/lib/oxttl/src/toolkit/error.rs @@ -72,7 +72,7 @@ impl Error for SyntaxError {} impl From for io::Error { #[inline] fn from(error: SyntaxError) -> Self { - io::Error::new(io::ErrorKind::InvalidData, error) + Self::new(io::ErrorKind::InvalidData, error) } } diff --git a/lib/sparopt/src/algebra.rs b/lib/sparopt/src/algebra.rs index e5cb0952..51ecf6fa 100644 --- a/lib/sparopt/src/algebra.rs +++ b/lib/sparopt/src/algebra.rs @@ -364,25 +364,25 @@ impl Expression { fn returns_boolean(&self) -> bool { match self { - Expression::Or(_) - | Expression::And(_) - | Expression::Equal(_, _) - | Expression::SameTerm(_, _) - | Expression::Greater(_, _) - | Expression::GreaterOrEqual(_, _) - | Expression::Less(_, _) - | Expression::LessOrEqual(_, _) - | Expression::Not(_) - | Expression::Exists(_) - | Expression::Bound(_) - | Expression::FunctionCall( + Self::Or(_) + | Self::And(_) + | Self::Equal(_, _) + | Self::SameTerm(_, _) + | Self::Greater(_, _) + | Self::GreaterOrEqual(_, _) + | Self::Less(_, _) + | Self::LessOrEqual(_, _) + | Self::Not(_) + | Self::Exists(_) + | Self::Bound(_) + | Self::FunctionCall( Function::IsBlank | Function::IsIri | Function::IsLiteral | Function::IsNumeric, _, ) => true, #[cfg(feature = "rdf-star")] - Expression::FunctionCall(Function::IsTriple, _) => true, - Expression::Literal(literal) => literal.datatype() == xsd::BOOLEAN, - Expression::If(_, a, b) => a.returns_boolean() && b.returns_boolean(), + Self::FunctionCall(Function::IsTriple, _) => true, + Self::Literal(literal) => literal.datatype() == xsd::BOOLEAN, + Self::If(_, a, b) => a.returns_boolean() && b.returns_boolean(), _ => false, } } @@ -847,7 +847,7 @@ impl GraphPattern { } } if all.is_empty() { - GraphPattern::empty() + Self::empty() } else { Self::Union { inner: order_vec(all), diff --git a/lib/src/sparql/eval.rs b/lib/src/sparql/eval.rs index 785c64cd..25c4b3cf 100644 --- a/lib/src/sparql/eval.rs +++ b/lib/src/sparql/eval.rs @@ -3892,9 +3892,9 @@ impl TupleSelector { fn get_pattern_value(&self, tuple: &EncodedTuple) -> Option { match self { - TupleSelector::Constant(c) => Some(c.clone()), - TupleSelector::Variable(v) => tuple.get(*v).cloned(), - TupleSelector::TriplePattern(triple) => Some( + Self::Constant(c) => Some(c.clone()), + Self::Variable(v) => tuple.get(*v).cloned(), + Self::TriplePattern(triple) => Some( EncodedTriple { subject: triple.subject.get_pattern_value(tuple)?, predicate: triple.predicate.get_pattern_value(tuple)?, diff --git a/lib/src/storage/error.rs b/lib/src/storage/error.rs index 8c874d77..89895349 100644 --- a/lib/src/storage/error.rs +++ b/lib/src/storage/error.rs @@ -179,7 +179,7 @@ impl From for io::Error { LoaderError::Storage(error) => error.into(), LoaderError::Parsing(error) => error.into(), LoaderError::InvalidBaseIri { .. } => { - io::Error::new(io::ErrorKind::InvalidInput, error.to_string()) + Self::new(io::ErrorKind::InvalidInput, error.to_string()) } } } @@ -242,7 +242,7 @@ impl From for io::Error { SerializerError::Storage(error) => error.into(), SerializerError::Io(error) => error, SerializerError::DatasetFormatExpected(_) => { - io::Error::new(io::ErrorKind::InvalidInput, error.to_string()) + Self::new(io::ErrorKind::InvalidInput, error.to_string()) } } } diff --git a/lib/src/storage/small_string.rs b/lib/src/storage/small_string.rs index 70a15676..c2862ff4 100644 --- a/lib/src/storage/small_string.rs +++ b/lib/src/storage/small_string.rs @@ -153,10 +153,10 @@ impl FromStr for SmallString { inner[15] = value .len() .try_into() - .map_err(|_| BadSmallStringError::TooLong(value.len()))?; + .map_err(|_| Self::Err::TooLong(value.len()))?; Ok(Self { inner }) } else { - Err(BadSmallStringError::TooLong(value.len())) + Err(Self::Err::TooLong(value.len())) } } } From 522bda29066c7c340938979b6f1d8af352230616 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Wed, 17 Jan 2024 22:37:34 -0500 Subject: [PATCH 183/217] Add WKT_LITERAL vocabulary support --- lib/oxrdf/src/vocab.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/oxrdf/src/vocab.rs b/lib/oxrdf/src/vocab.rs index a0c3bd5c..56110c4a 100644 --- a/lib/oxrdf/src/vocab.rs +++ b/lib/oxrdf/src/vocab.rs @@ -231,3 +231,12 @@ pub mod xsd { pub const YEAR_MONTH_DURATION: NamedNodeRef<'_> = NamedNodeRef::new_unchecked("http://www.w3.org/2001/XMLSchema#yearMonthDuration"); } + +pub mod geosparql { + //! [GeoSpatial](https://opengeospatial.github.io/ogc-geosparql/) vocabulary. + use crate::named_node::NamedNodeRef; + + /// Geospatial datatype like `"Point({longitude} {latitude})"^^geo:wktLiteral` + pub const WKT_LITERAL: NamedNodeRef<'_> = + NamedNodeRef::new_unchecked("http://www.opengis.net/ont/geosparql#wktLiteral"); +} From a976eb3efca1184c159202cfd70fd9863b4503e0 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Thu, 18 Jan 2024 03:37:32 -0500 Subject: [PATCH 184/217] Remove use_self allow clippy --- js/src/model.rs | 7 +------ js/src/store.rs | 2 -- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/js/src/model.rs b/js/src/model.rs index 92d979c8..dd8a972c 100644 --- a/js/src/model.rs +++ b/js/src/model.rs @@ -1,9 +1,4 @@ -#![allow( - dead_code, - clippy::inherent_to_string, - clippy::unused_self, - clippy::use_self -)] +#![allow(dead_code, clippy::inherent_to_string, clippy::unused_self)] use crate::format_err; use crate::utils::to_err; diff --git a/js/src/store.rs b/js/src/store.rs index bb0af9e7..0b4b51b8 100644 --- a/js/src/store.rs +++ b/js/src/store.rs @@ -1,5 +1,3 @@ -#![allow(clippy::use_self)] - use crate::format_err; use crate::model::*; use crate::utils::to_err; From f354bc7546dceb5566af5d3edfdca56993e4ce58 Mon Sep 17 00:00:00 2001 From: etiennept <47983879+etiennept@users.noreply.github.com> Date: Thu, 18 Jan 2024 21:45:25 +0100 Subject: [PATCH 185/217] JS: avoids directory copies during build --- js/build_package.js | 10 - js/package-lock.json | 1079 ++++++++++++++++++++++++++++++++++++++++++ js/package.json | 2 +- 3 files changed, 1080 insertions(+), 11 deletions(-) create mode 100644 js/package-lock.json diff --git a/js/build_package.js b/js/build_package.js index e06f84dc..5975aa46 100644 --- a/js/build_package.js +++ b/js/build_package.js @@ -1,16 +1,6 @@ #! /usr/bin/env node const fs = require("fs"); - -// We copy file to the new directory -fs.mkdirSync("pkg"); -for (const file of fs.readdirSync("./pkg-web")) { - fs.copyFileSync(`./pkg-web/${file}`, `./pkg/${file}`); -} -for (const file of fs.readdirSync("./pkg-node")) { - fs.copyFileSync(`./pkg-node/${file}`, `./pkg/${file}`); -} - const pkg = JSON.parse(fs.readFileSync("./pkg/package.json")); pkg.name = "oxigraph"; pkg.main = "node.js"; diff --git a/js/package-lock.json b/js/package-lock.json new file mode 100644 index 00000000..e9c781ae --- /dev/null +++ b/js/package-lock.json @@ -0,0 +1,1079 @@ +{ + "name": "oxigraph_tests", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "oxigraph_tests", + "devDependencies": { + "@biomejs/biome": "^1.0.0", + "@rdfjs/data-model": "^2.0.1", + "mocha": "^10.0.0" + } + }, + "node_modules/@biomejs/biome": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/@biomejs/biome/-/biome-1.5.1.tgz", + "integrity": "sha512-rdMA/N1Zc1nxUtbXMVr+50Sg/Pezz+9qGQa2uyRWFtrCoyr3dv0pVz+0ifGGue18ip50ZH8x2r5CV7zo8Q/0mA==", + "dev": true, + "hasInstallScript": true, + "bin": { + "biome": "bin/biome" + }, + "engines": { + "node": ">=14.*" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/biome" + }, + "optionalDependencies": { + "@biomejs/cli-darwin-arm64": "1.5.1", + "@biomejs/cli-darwin-x64": "1.5.1", + "@biomejs/cli-linux-arm64": "1.5.1", + "@biomejs/cli-linux-arm64-musl": "1.5.1", + "@biomejs/cli-linux-x64": "1.5.1", + "@biomejs/cli-linux-x64-musl": "1.5.1", + "@biomejs/cli-win32-arm64": "1.5.1", + "@biomejs/cli-win32-x64": "1.5.1" + } + }, + "node_modules/@biomejs/cli-darwin-arm64": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-arm64/-/cli-darwin-arm64-1.5.1.tgz", + "integrity": "sha512-E9pLakmSVHP6UH2uqAghqEkr/IHAIDfDyCedqJVnyFc+uufNTHwB8id4XTiWy/eKIdgxHZsTSE+R+W0IqrTNVQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=14.*" + } + }, + "node_modules/@biomejs/cli-darwin-x64": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-x64/-/cli-darwin-x64-1.5.1.tgz", + "integrity": "sha512-8O1F+FcoCi02JlocyilB6R3y3kT9sRkBCRwYddaBIScQe2hCme/mA2rVzrhCCHhskrclJ51GEKjkEORj4/8c2A==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=14.*" + } + }, + "node_modules/@biomejs/cli-linux-arm64": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64/-/cli-linux-arm64-1.5.1.tgz", + "integrity": "sha512-25gwY4FMzmi1Rl6N835raLq7nzTk+PyEQd88k9Em6dqtI4qpljqmZlMmVjOiwXKe3Ee80J/Vlh7BM36lsHUTEg==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=14.*" + } + }, + "node_modules/@biomejs/cli-linux-arm64-musl": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64-musl/-/cli-linux-arm64-musl-1.5.1.tgz", + "integrity": "sha512-Lw9G3LUdhRMp8L8RMeVevnfQCa7luT6ubQ8GRjLju32glxWKefpDrzgfHixGyvTQPlhnYjQ+V8/QQ/I7WPzOoA==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=14.*" + } + }, + "node_modules/@biomejs/cli-linux-x64": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64/-/cli-linux-x64-1.5.1.tgz", + "integrity": "sha512-YDM0gZP4UbAuaBI3DVbUuj5X+Omm6uxzD1Qpc6hcduH1kzXzs9L0ee7cn/kJtNndoXR8MlmUS0O0/wWvZf2YaA==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=14.*" + } + }, + "node_modules/@biomejs/cli-linux-x64-musl": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64-musl/-/cli-linux-x64-musl-1.5.1.tgz", + "integrity": "sha512-5gapxc/VlwTgGRbTc9h8PMTpf8eNahIBauFUGSXncHgayi3VpezKSicgaQ1bb8FahVXf/5eNEVxVARq/or71Ag==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=14.*" + } + }, + "node_modules/@biomejs/cli-win32-arm64": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/@biomejs/cli-win32-arm64/-/cli-win32-arm64-1.5.1.tgz", + "integrity": "sha512-TVpLBOLUMLQmH2VRFBKFr3rgEkr7XvG4QZxHOxWB9Ivc/sQPvg4aHMd8qpgPKXABGUnultyc9t0+WvfIDxuALg==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=14.*" + } + }, + "node_modules/@biomejs/cli-win32-x64": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/@biomejs/cli-win32-x64/-/cli-win32-x64-1.5.1.tgz", + "integrity": "sha512-qx8EKwScZmVYZjMPZ6GF3ZUmgg/N6zqh+d8vHA2E43opNCyqIPTl89sOqkc7zd1CyyABDWxsbqI9Ih6xTT6hnQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=14.*" + } + }, + "node_modules/@rdfjs/data-model": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@rdfjs/data-model/-/data-model-2.0.1.tgz", + "integrity": "sha512-oRDYpy7/fJ9NNjS+M7m+dbnhi4lOWYGbBiM/A+u9bBExnN6ifXUF5mUsFxwZaQulmwTDaMhKERdV6iKTBUMgtw==", + "dev": true, + "bin": { + "rdfjs-data-model-test": "bin/test.js" + } + }, + "node_modules/ansi-colors": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.1.tgz", + "integrity": "sha512-JoX0apGbHaUJBNl6yF+p6JAFYZ666/hhCGKN5t9QFjbJQKUU/g8MNbFDbvfrgKXvI1QpZplPOnwIo99lX/AAmA==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/anymatch": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", + "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", + "dev": true, + "dependencies": { + "normalize-path": "^3.0.0", + "picomatch": "^2.0.4" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "dev": true + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true + }, + "node_modules/binary-extensions": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", + "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/braces": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", + "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", + "dev": true, + "dependencies": { + "fill-range": "^7.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/browser-stdout": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/browser-stdout/-/browser-stdout-1.3.1.tgz", + "integrity": "sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw==", + "dev": true + }, + "node_modules/camelcase": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz", + "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", + "dev": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/chalk/node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/chokidar": { + "version": "3.5.3", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz", + "integrity": "sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==", + "dev": true, + "funding": [ + { + "type": "individual", + "url": "https://paulmillr.com/funding/" + } + ], + "dependencies": { + "anymatch": "~3.1.2", + "braces": "~3.0.2", + "glob-parent": "~5.1.2", + "is-binary-path": "~2.1.0", + "is-glob": "~4.0.1", + "normalize-path": "~3.0.0", + "readdirp": "~3.6.0" + }, + "engines": { + "node": ">= 8.10.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.2" + } + }, + "node_modules/cliui": { + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz", + "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==", + "dev": true, + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.0", + "wrap-ansi": "^7.0.0" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "dev": true + }, + "node_modules/debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dev": true, + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/debug/node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + }, + "node_modules/decamelize": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-4.0.0.tgz", + "integrity": "sha512-9iE1PgSik9HeIIw2JO94IidnE3eBoQrFJ3w7sFuzSX4DpmZ3v5sZpUiV5Swcf6mQEF+Y0ru8Neo+p+nyh2J+hQ==", + "dev": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/diff": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/diff/-/diff-5.0.0.tgz", + "integrity": "sha512-/VTCrvm5Z0JGty/BWHljh+BAiw3IK+2j87NGMu8Nwc/f48WoDAC395uomO9ZD117ZOBaHmkX1oyLvkVM/aIT3w==", + "dev": true, + "engines": { + "node": ">=0.3.1" + } + }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true + }, + "node_modules/escalade": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", + "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "dev": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/fill-range": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", + "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "dev": true, + "dependencies": { + "to-regex-range": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/find-up": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", + "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", + "dev": true, + "dependencies": { + "locate-path": "^6.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/flat": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/flat/-/flat-5.0.2.tgz", + "integrity": "sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ==", + "dev": true, + "bin": { + "flat": "cli.js" + } + }, + "node_modules/fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", + "dev": true + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "dev": true, + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/glob": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.0.tgz", + "integrity": "sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q==", + "dev": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "dev": true, + "dependencies": { + "is-glob": "^4.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/glob/node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/glob/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/he": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", + "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==", + "dev": true, + "bin": { + "he": "bin/he" + } + }, + "node_modules/inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "dev": true, + "dependencies": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true + }, + "node_modules/is-binary-path": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", + "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", + "dev": true, + "dependencies": { + "binary-extensions": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "dev": true, + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "dev": true, + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/is-plain-obj": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-2.1.0.tgz", + "integrity": "sha512-YWnfyRwxL/+SsrWYfOpUtz5b3YD+nyfkHvjbcanzk8zgyO4ASD67uVMRt8k5bM4lLMDnXfriRhOpemw+NfT1eA==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/is-unicode-supported": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-0.1.0.tgz", + "integrity": "sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw==", + "dev": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/js-yaml": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "dev": true, + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/locate-path": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", + "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", + "dev": true, + "dependencies": { + "p-locate": "^5.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/log-symbols": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.1.0.tgz", + "integrity": "sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg==", + "dev": true, + "dependencies": { + "chalk": "^4.1.0", + "is-unicode-supported": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/minimatch": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.0.1.tgz", + "integrity": "sha512-nLDxIFRyhDblz3qMuq+SoRZED4+miJ/G+tdDrjkkkRnjAsBexeGpgjLEQ0blJy7rHhR2b93rhQY4SvyWu9v03g==", + "dev": true, + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/mocha": { + "version": "10.2.0", + "resolved": "https://registry.npmjs.org/mocha/-/mocha-10.2.0.tgz", + "integrity": "sha512-IDY7fl/BecMwFHzoqF2sg/SHHANeBoMMXFlS9r0OXKDssYE1M5O43wUY/9BVPeIvfH2zmEbBfseqN9gBQZzXkg==", + "dev": true, + "dependencies": { + "ansi-colors": "4.1.1", + "browser-stdout": "1.3.1", + "chokidar": "3.5.3", + "debug": "4.3.4", + "diff": "5.0.0", + "escape-string-regexp": "4.0.0", + "find-up": "5.0.0", + "glob": "7.2.0", + "he": "1.2.0", + "js-yaml": "4.1.0", + "log-symbols": "4.1.0", + "minimatch": "5.0.1", + "ms": "2.1.3", + "nanoid": "3.3.3", + "serialize-javascript": "6.0.0", + "strip-json-comments": "3.1.1", + "supports-color": "8.1.1", + "workerpool": "6.2.1", + "yargs": "16.2.0", + "yargs-parser": "20.2.4", + "yargs-unparser": "2.0.0" + }, + "bin": { + "_mocha": "bin/_mocha", + "mocha": "bin/mocha.js" + }, + "engines": { + "node": ">= 14.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/mochajs" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true + }, + "node_modules/nanoid": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.3.tgz", + "integrity": "sha512-p1sjXuopFs0xg+fPASzQ28agW1oHD7xDsd9Xkf3T15H3c/cifrFHVwrh74PdoklAPi+i7MdRsE47vm2r6JoB+w==", + "dev": true, + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dev": true, + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", + "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", + "dev": true, + "dependencies": { + "p-limit": "^3.0.2" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/picomatch": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "dev": true, + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/randombytes": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", + "integrity": "sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==", + "dev": true, + "dependencies": { + "safe-buffer": "^5.1.0" + } + }, + "node_modules/readdirp": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", + "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", + "dev": true, + "dependencies": { + "picomatch": "^2.2.1" + }, + "engines": { + "node": ">=8.10.0" + } + }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/serialize-javascript": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-6.0.0.tgz", + "integrity": "sha512-Qr3TosvguFt8ePWqsvRfrKyQXIiW+nGbYpy8XK24NQHE83caxWt+mIymTT19DGFbNWNLfEwsrkSmN64lVWB9ag==", + "dev": true, + "dependencies": { + "randombytes": "^2.1.0" + } + }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-json-comments": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", + "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", + "dev": true, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/supports-color": { + "version": "8.1.1", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", + "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==", + "dev": true, + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/supports-color?sponsor=1" + } + }, + "node_modules/to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dev": true, + "dependencies": { + "is-number": "^7.0.0" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/workerpool": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-6.2.1.tgz", + "integrity": "sha512-ILEIE97kDZvF9Wb9f6h5aXK4swSlKGUcOEGiIYb2OOu/IrDU9iwj0fD//SsA6E5ibwJxpEvhullJY4Sl4GcpAw==", + "dev": true + }, + "node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "dev": true + }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "dev": true, + "engines": { + "node": ">=10" + } + }, + "node_modules/yargs": { + "version": "16.2.0", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz", + "integrity": "sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==", + "dev": true, + "dependencies": { + "cliui": "^7.0.2", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.0", + "y18n": "^5.0.5", + "yargs-parser": "^20.2.2" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/yargs-parser": { + "version": "20.2.4", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.4.tgz", + "integrity": "sha512-WOkpgNhPTlE73h4VFAFsOnomJVaovO8VqLDzy5saChRBFQFBoMYirowyW+Q9HB4HFF4Z7VZTiG3iSzJJA29yRA==", + "dev": true, + "engines": { + "node": ">=10" + } + }, + "node_modules/yargs-unparser": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/yargs-unparser/-/yargs-unparser-2.0.0.tgz", + "integrity": "sha512-7pRTIA9Qc1caZ0bZ6RYRGbHJthJWuakf+WmHK0rVeLkNrrGhfoabBNdue6kdINI6r4if7ocq9aD/n7xwKOdzOA==", + "dev": true, + "dependencies": { + "camelcase": "^6.0.0", + "decamelize": "^4.0.0", + "flat": "^5.0.2", + "is-plain-obj": "^2.1.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + } + } +} diff --git a/js/package.json b/js/package.json index 72310055..5a81afa5 100644 --- a/js/package.json +++ b/js/package.json @@ -10,7 +10,7 @@ "scripts": { "fmt": "biome format . --write && biome check . --apply-unsafe && biome format . --write", "test": "biome ci . && wasm-pack build --debug --target nodejs && mocha", - "build": "rm -rf pkg && wasm-pack build --release --target web --out-name web --out-dir pkg-web && wasm-pack build --release --target nodejs --out-name node --out-dir pkg-node && node build_package.js && rm -r pkg-web && rm -r pkg-node", + "build": "wasm-pack build --release --target web --out-name web && wasm-pack build --release --target nodejs --out-name node && node build_package.js", "release": "npm run build && npm publish ./pkg", "pack": "npm run build && npm pack ./pkg" }, From d838d55f02c412c186c7b53be4dea9fff92d7f92 Mon Sep 17 00:00:00 2001 From: Tpt Date: Thu, 18 Jan 2024 17:20:18 +0100 Subject: [PATCH 186/217] Uses nightly rustfmt on imports and comments --- cli/src/main.rs | 5 +- lib/oxrdf/src/blank_node.rs | 20 +-- lib/oxrdf/src/dataset.rs | 17 +- lib/oxrdf/src/graph.rs | 7 +- lib/oxrdf/src/literal.rs | 7 +- lib/oxrdf/src/parser.rs | 78 +++++++--- lib/oxrdf/src/triple.rs | 10 +- lib/oxrdf/src/variable.rs | 12 +- lib/oxrdfio/src/format.rs | 10 +- lib/oxrdfio/src/parser.rs | 36 +++-- lib/oxrdfio/src/serializer.rs | 7 +- lib/oxrdfxml/src/parser.rs | 29 ++-- lib/oxsdatatypes/src/date_time.rs | 161 ++++++++++--------- lib/oxsdatatypes/src/decimal.rs | 18 +-- lib/oxsdatatypes/src/double.rs | 24 ++- lib/oxsdatatypes/src/duration.rs | 41 +++-- lib/oxsdatatypes/src/float.rs | 24 ++- lib/oxsdatatypes/src/integer.rs | 7 +- lib/oxttl/src/lexer.rs | 4 +- lib/oxttl/src/line_formats.rs | 6 +- lib/oxttl/src/n3.rs | 42 +++-- lib/oxttl/src/terse.rs | 14 +- lib/oxttl/src/toolkit/lexer.rs | 4 +- lib/oxttl/src/trig.rs | 38 +++-- lib/oxttl/src/turtle.rs | 35 +++-- lib/sparesults/src/csv.rs | 4 +- lib/sparesults/src/format.rs | 21 ++- lib/sparesults/src/json.rs | 3 +- lib/sparesults/src/parser.rs | 36 ++++- lib/sparesults/src/serializer.rs | 34 +++- lib/sparesults/src/solution.rs | 88 +++++++++-- lib/sparesults/src/xml.rs | 7 +- lib/spargebra/src/parser.rs | 18 +-- lib/spargebra/src/query.rs | 5 +- lib/spargebra/src/term.rs | 5 +- lib/sparopt/src/algebra.rs | 4 +- lib/sparopt/src/optimizer.rs | 6 +- lib/sparopt/src/type_inference.rs | 51 +++--- lib/sparql-smith/src/lib.rs | 16 +- lib/src/io/format.rs | 32 +++- lib/src/io/read.rs | 22 ++- lib/src/io/write.rs | 22 ++- lib/src/sparql/algebra.rs | 32 +++- lib/src/sparql/error.rs | 3 +- lib/src/sparql/eval.rs | 21 +-- lib/src/sparql/mod.rs | 15 +- lib/src/sparql/model.rs | 24 ++- lib/src/sparql/service.rs | 20 ++- lib/src/sparql/update.rs | 2 +- lib/src/storage/backend/rocksdb.rs | 14 +- lib/src/storage/error.rs | 3 +- lib/src/storage/mod.rs | 4 +- lib/src/storage/numeric_encoder.rs | 3 +- lib/src/storage/small_string.rs | 3 +- lib/src/store.rs | 241 ++++++++++++++++++----------- python/src/io.rs | 5 - python/src/model.rs | 4 +- python/src/sparql.rs | 19 +-- rustfmt.toml | 10 +- testsuite/src/parser_evaluator.rs | 4 +- testsuite/tests/sparql.rs | 6 +- 61 files changed, 887 insertions(+), 576 deletions(-) diff --git a/cli/src/main.rs b/cli/src/main.rs index 9e9a0dbd..cabf08c4 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -469,7 +469,7 @@ pub fn main() -> anyhow::Result<()> { file.display(), error ) - //TODO: hard fail + // TODO: hard fail } }) } @@ -1845,7 +1845,8 @@ mod tests { use super::*; use anyhow::Result; use assert_cmd::Command; - use assert_fs::{prelude::*, NamedTempFile, TempDir}; + use assert_fs::prelude::*; + use assert_fs::{NamedTempFile, TempDir}; use flate2::write::GzEncoder; use flate2::Compression; use oxhttp::model::Method; diff --git a/lib/oxrdf/src/blank_node.rs b/lib/oxrdf/src/blank_node.rs index e813dd24..9603cd30 100644 --- a/lib/oxrdf/src/blank_node.rs +++ b/lib/oxrdf/src/blank_node.rs @@ -1,8 +1,7 @@ use rand::random; use std::error::Error; -use std::fmt; use std::io::Write; -use std::str; +use std::{fmt, str}; /// An owned RDF [blank node](https://www.w3.org/TR/rdf11-concepts/#dfn-blank-node). /// @@ -15,10 +14,7 @@ use std::str; /// ``` /// use oxrdf::BlankNode; /// -/// assert_eq!( -/// "_:a122", -/// BlankNode::new("a122")?.to_string() -/// ); +/// assert_eq!("_:a122", BlankNode::new("a122")?.to_string()); /// # Result::<_,oxrdf::BlankNodeIdParseError>::Ok(()) /// ``` #[derive(Eq, PartialEq, Debug, Clone, Hash)] @@ -36,7 +32,7 @@ impl BlankNode { /// The blank node identifier must be valid according to N-Triples, Turtle, and SPARQL grammars. /// /// In most cases, it is much more convenient to create a blank node using [`BlankNode::default()`] - ///that creates a random ID that could be easily inlined by Oxigraph stores. + /// that creates a random ID that could be easily inlined by Oxigraph stores. pub fn new(id: impl Into) -> Result { let id = id.into(); validate_blank_node_identifier(&id)?; @@ -133,10 +129,7 @@ impl Default for BlankNode { /// ``` /// use oxrdf::BlankNodeRef; /// -/// assert_eq!( -/// "_:a122", -/// BlankNodeRef::new("a122")?.to_string() -/// ); +/// assert_eq!("_:a122", BlankNodeRef::new("a122")?.to_string()); /// # Result::<_,oxrdf::BlankNodeIdParseError>::Ok(()) /// ``` #[derive(Eq, PartialEq, Debug, Clone, Copy, Hash)] @@ -192,7 +185,10 @@ impl<'a> BlankNodeRef<'a> { /// ``` /// use oxrdf::BlankNode; /// - /// assert_eq!(BlankNode::new_from_unique_id(128).as_ref().unique_id(), Some(128)); + /// assert_eq!( + /// BlankNode::new_from_unique_id(128).as_ref().unique_id(), + /// Some(128) + /// ); /// assert_eq!(BlankNode::new("foo")?.as_ref().unique_id(), None); /// # Result::<_,oxrdf::BlankNodeIdParseError>::Ok(()) /// ``` diff --git a/lib/oxrdf/src/dataset.rs b/lib/oxrdf/src/dataset.rs index 0c1fcbd7..8412a8aa 100644 --- a/lib/oxrdf/src/dataset.rs +++ b/lib/oxrdf/src/dataset.rs @@ -20,19 +20,20 @@ //! assert_eq!(vec![TripleRef::new(ex, ex, ex)], results); //! //! // Print -//! assert_eq!(dataset.to_string(), " .\n"); +//! assert_eq!( +//! dataset.to_string(), +//! " .\n" +//! ); //! # Result::<_,Box>::Ok(()) //! ``` //! //! See also [`Graph`] if you only care about plain triples. use crate::interning::*; -use crate::SubjectRef; use crate::*; use std::cmp::min; use std::collections::hash_map::DefaultHasher; -use std::collections::BTreeSet; -use std::collections::{HashMap, HashSet}; +use std::collections::{BTreeSet, HashMap, HashSet}; use std::fmt; use std::hash::{Hash, Hasher}; @@ -924,8 +925,8 @@ impl PartialEq for Dataset { impl Eq for Dataset {} impl<'a> IntoIterator for &'a Dataset { - type Item = QuadRef<'a>; type IntoIter = Iter<'a>; + type Item = QuadRef<'a>; fn into_iter(self) -> Self::IntoIter { self.iter() @@ -1282,8 +1283,8 @@ impl<'a> GraphView<'a> { } impl<'a> IntoIterator for GraphView<'a> { - type Item = TripleRef<'a>; type IntoIter = GraphViewIter<'a>; + type Item = TripleRef<'a>; fn into_iter(self) -> Self::IntoIter { self.iter() @@ -1291,8 +1292,8 @@ impl<'a> IntoIterator for GraphView<'a> { } impl<'a, 'b> IntoIterator for &'b GraphView<'a> { - type Item = TripleRef<'a>; type IntoIter = GraphViewIter<'a>; + type Item = TripleRef<'a>; fn into_iter(self) -> Self::IntoIter { self.iter() @@ -1493,8 +1494,8 @@ impl<'a, 'b, T: Into>> Extend for GraphViewMut<'a> { } impl<'a> IntoIterator for &'a GraphViewMut<'a> { - type Item = TripleRef<'a>; type IntoIter = GraphViewIter<'a>; + type Item = TripleRef<'a>; fn into_iter(self) -> Self::IntoIter { self.iter() diff --git a/lib/oxrdf/src/graph.rs b/lib/oxrdf/src/graph.rs index 3077e5de..33f67132 100644 --- a/lib/oxrdf/src/graph.rs +++ b/lib/oxrdf/src/graph.rs @@ -16,7 +16,10 @@ //! assert_eq!(vec![triple], results); //! //! // Print -//! assert_eq!(graph.to_string(), " .\n"); +//! assert_eq!( +//! graph.to_string(), +//! " .\n" +//! ); //! # Result::<_,Box>::Ok(()) //! ``` //! @@ -226,8 +229,8 @@ impl PartialEq for Graph { impl Eq for Graph {} impl<'a> IntoIterator for &'a Graph { - type Item = TripleRef<'a>; type IntoIter = Iter<'a>; + type Item = TripleRef<'a>; fn into_iter(self) -> Self::IntoIter { self.iter() diff --git a/lib/oxrdf/src/literal.rs b/lib/oxrdf/src/literal.rs index 3f2727ca..0872fab5 100644 --- a/lib/oxrdf/src/literal.rs +++ b/lib/oxrdf/src/literal.rs @@ -1,6 +1,5 @@ use crate::named_node::NamedNode; -use crate::vocab::rdf; -use crate::vocab::xsd; +use crate::vocab::{rdf, xsd}; use crate::NamedNodeRef; use oxilangtag::{LanguageTag, LanguageTagParseError}; #[cfg(feature = "oxsdatatypes")] @@ -15,8 +14,8 @@ use std::option::Option; /// The default string formatter is returning an N-Triples, Turtle, and SPARQL compatible representation: /// ``` /// # use oxilangtag::LanguageTagParseError; -/// use oxrdf::Literal; /// use oxrdf::vocab::xsd; +/// use oxrdf::Literal; /// /// assert_eq!( /// "\"foo\\nbar\"", @@ -427,8 +426,8 @@ impl From for Literal { /// /// The default string formatter is returning an N-Triples, Turtle, and SPARQL compatible representation: /// ``` -/// use oxrdf::LiteralRef; /// use oxrdf::vocab::xsd; +/// use oxrdf::LiteralRef; /// /// assert_eq!( /// "\"foo\\nbar\"", diff --git a/lib/oxrdf/src/parser.rs b/lib/oxrdf/src/parser.rs index 326868f5..1794540d 100644 --- a/lib/oxrdf/src/parser.rs +++ b/lib/oxrdf/src/parser.rs @@ -5,10 +5,9 @@ use crate::{ }; #[cfg(feature = "rdf-star")] use crate::{Subject, Triple}; -use std::char; use std::error::Error; -use std::fmt; use std::str::{Chars, FromStr}; +use std::{char, fmt}; /// This limit is set in order to avoid stack overflow error when parsing nested triples due to too many recursive calls. /// The actual limit value is a wet finger compromise between not failing to parse valid files and avoiding to trigger stack overflow errors. @@ -23,7 +22,10 @@ impl FromStr for NamedNode { /// use oxrdf::NamedNode; /// use std::str::FromStr; /// - /// assert_eq!(NamedNode::from_str("").unwrap(), NamedNode::new("http://example.com").unwrap()) + /// assert_eq!( + /// NamedNode::from_str("").unwrap(), + /// NamedNode::new("http://example.com").unwrap() + /// ) /// ``` fn from_str(s: &str) -> Result { let (term, left) = read_named_node(s)?; @@ -45,7 +47,10 @@ impl FromStr for BlankNode { /// use oxrdf::BlankNode; /// use std::str::FromStr; /// - /// assert_eq!(BlankNode::from_str("_:ex").unwrap(), BlankNode::new("ex").unwrap()) + /// assert_eq!( + /// BlankNode::from_str("_:ex").unwrap(), + /// BlankNode::new("ex").unwrap() + /// ) /// ``` fn from_str(s: &str) -> Result { let (term, left) = read_blank_node(s)?; @@ -64,16 +69,41 @@ impl FromStr for Literal { /// Parses a literal from its NTriples or Turtle serialization /// /// ``` - /// use oxrdf::{Literal, NamedNode, vocab::xsd}; + /// use oxrdf::vocab::xsd; + /// use oxrdf::{Literal, NamedNode}; /// use std::str::FromStr; /// - /// assert_eq!(Literal::from_str("\"ex\\n\"").unwrap(), Literal::new_simple_literal("ex\n")); - /// assert_eq!(Literal::from_str("\"ex\"@en").unwrap(), Literal::new_language_tagged_literal("ex", "en").unwrap()); - /// assert_eq!(Literal::from_str("\"2020\"^^").unwrap(), Literal::new_typed_literal("2020", NamedNode::new("http://www.w3.org/2001/XMLSchema#gYear").unwrap())); - /// assert_eq!(Literal::from_str("true").unwrap(), Literal::new_typed_literal("true", xsd::BOOLEAN)); - /// assert_eq!(Literal::from_str("+122").unwrap(), Literal::new_typed_literal("+122", xsd::INTEGER)); - /// assert_eq!(Literal::from_str("-122.23").unwrap(), Literal::new_typed_literal("-122.23", xsd::DECIMAL)); - /// assert_eq!(Literal::from_str("-122e+1").unwrap(), Literal::new_typed_literal("-122e+1", xsd::DOUBLE)); + /// assert_eq!( + /// Literal::from_str("\"ex\\n\"").unwrap(), + /// Literal::new_simple_literal("ex\n") + /// ); + /// assert_eq!( + /// Literal::from_str("\"ex\"@en").unwrap(), + /// Literal::new_language_tagged_literal("ex", "en").unwrap() + /// ); + /// assert_eq!( + /// Literal::from_str("\"2020\"^^").unwrap(), + /// Literal::new_typed_literal( + /// "2020", + /// NamedNode::new("http://www.w3.org/2001/XMLSchema#gYear").unwrap() + /// ) + /// ); + /// assert_eq!( + /// Literal::from_str("true").unwrap(), + /// Literal::new_typed_literal("true", xsd::BOOLEAN) + /// ); + /// assert_eq!( + /// Literal::from_str("+122").unwrap(), + /// Literal::new_typed_literal("+122", xsd::INTEGER) + /// ); + /// assert_eq!( + /// Literal::from_str("-122.23").unwrap(), + /// Literal::new_typed_literal("-122.23", xsd::DECIMAL) + /// ); + /// assert_eq!( + /// Literal::from_str("-122e+1").unwrap(), + /// Literal::new_typed_literal("-122e+1", xsd::DOUBLE) + /// ); /// ``` fn from_str(s: &str) -> Result { let (term, left) = read_literal(s)?; @@ -93,12 +123,19 @@ impl FromStr for Term { /// use oxrdf::*; /// use std::str::FromStr; /// - /// assert_eq!(Term::from_str("\"ex\"").unwrap(), Literal::new_simple_literal("ex").into()); - /// assert_eq!(Term::from_str("<< _:s \"o\" >>").unwrap(), Triple::new( - /// BlankNode::new("s").unwrap(), - /// NamedNode::new("http://example.com/p").unwrap(), - /// Literal::new_simple_literal("o") - /// ).into()); + /// assert_eq!( + /// Term::from_str("\"ex\"").unwrap(), + /// Literal::new_simple_literal("ex").into() + /// ); + /// assert_eq!( + /// Term::from_str("<< _:s \"o\" >>").unwrap(), + /// Triple::new( + /// BlankNode::new("s").unwrap(), + /// NamedNode::new("http://example.com/p").unwrap(), + /// Literal::new_simple_literal("o") + /// ) + /// .into() + /// ); /// ``` fn from_str(s: &str) -> Result { let (term, left) = read_term(s, 0)?; @@ -118,7 +155,10 @@ impl FromStr for Variable { /// use oxrdf::Variable; /// use std::str::FromStr; /// - /// assert_eq!(Variable::from_str("$foo").unwrap(), Variable::new("foo").unwrap()) + /// assert_eq!( + /// Variable::from_str("$foo").unwrap(), + /// Variable::new("foo").unwrap() + /// ) /// ``` fn from_str(s: &str) -> Result { if !s.starts_with('?') && !s.starts_with('$') { diff --git a/lib/oxrdf/src/triple.rs b/lib/oxrdf/src/triple.rs index 850b1375..813982d0 100644 --- a/lib/oxrdf/src/triple.rs +++ b/lib/oxrdf/src/triple.rs @@ -698,7 +698,7 @@ impl<'a> From> for Term { /// /// The default string formatter is returning an N-Triples, Turtle, and SPARQL compatible representation: /// ``` -/// use oxrdf::{Triple, NamedNode}; +/// use oxrdf::{NamedNode, Triple}; /// /// assert_eq!( /// " ", @@ -706,7 +706,8 @@ impl<'a> From> for Term { /// subject: NamedNode::new("http://example.com/s")?.into(), /// predicate: NamedNode::new("http://example.com/p")?, /// object: NamedNode::new("http://example.com/o")?.into(), -/// }.to_string() +/// } +/// .to_string() /// ); /// # Result::<_,oxrdf::IriParseError>::Ok(()) /// ``` @@ -769,7 +770,7 @@ impl fmt::Display for Triple { /// /// The default string formatter is returning an N-Triples, Turtle, and SPARQL compatible representation: /// ``` -/// use oxrdf::{TripleRef, NamedNodeRef}; +/// use oxrdf::{NamedNodeRef, TripleRef}; /// /// assert_eq!( /// " ", @@ -777,7 +778,8 @@ impl fmt::Display for Triple { /// subject: NamedNodeRef::new("http://example.com/s")?.into(), /// predicate: NamedNodeRef::new("http://example.com/p")?, /// object: NamedNodeRef::new("http://example.com/o")?.into(), -/// }.to_string() +/// } +/// .to_string() /// ); /// # Result::<_,oxrdf::IriParseError>::Ok(()) /// ``` diff --git a/lib/oxrdf/src/variable.rs b/lib/oxrdf/src/variable.rs index 8bde4d6e..044c73e7 100644 --- a/lib/oxrdf/src/variable.rs +++ b/lib/oxrdf/src/variable.rs @@ -8,10 +8,7 @@ use std::fmt; /// ``` /// use oxrdf::{Variable, VariableNameParseError}; /// -/// assert_eq!( -/// "?foo", -/// Variable::new("foo")?.to_string() -/// ); +/// assert_eq!("?foo", Variable::new("foo")?.to_string()); /// # Result::<_,VariableNameParseError>::Ok(()) /// ``` #[derive(Eq, PartialEq, Ord, PartialOrd, Debug, Clone, Hash)] @@ -67,12 +64,9 @@ impl fmt::Display for Variable { /// /// The default string formatter is returning a SPARQL compatible representation: /// ``` -/// use oxrdf::{VariableRef, VariableNameParseError}; +/// use oxrdf::{VariableNameParseError, VariableRef}; /// -/// assert_eq!( -/// "?foo", -/// VariableRef::new("foo")?.to_string() -/// ); +/// assert_eq!("?foo", VariableRef::new("foo")?.to_string()); /// # Result::<_,VariableNameParseError>::Ok(()) /// ``` #[derive(Eq, PartialEq, Ord, PartialOrd, Debug, Clone, Copy, Hash)] diff --git a/lib/oxrdfio/src/format.rs b/lib/oxrdfio/src/format.rs index cb03a3eb..1cc6aa12 100644 --- a/lib/oxrdfio/src/format.rs +++ b/lib/oxrdfio/src/format.rs @@ -26,7 +26,10 @@ impl RdfFormat { /// ``` /// use oxrdfio::RdfFormat; /// - /// assert_eq!(RdfFormat::NTriples.iri(), "http://www.w3.org/ns/formats/N-Triples") + /// assert_eq!( + /// RdfFormat::NTriples.iri(), + /// "http://www.w3.org/ns/formats/N-Triples" + /// ) /// ``` #[inline] pub const fn iri(self) -> &'static str { @@ -136,7 +139,10 @@ impl RdfFormat { /// ``` /// use oxrdfio::RdfFormat; /// - /// assert_eq!(RdfFormat::from_media_type("text/turtle; charset=utf-8"), Some(RdfFormat::Turtle)) + /// assert_eq!( + /// RdfFormat::from_media_type("text/turtle; charset=utf-8"), + /// Some(RdfFormat::Turtle) + /// ) /// ``` #[inline] pub fn from_media_type(media_type: &str) -> Option { diff --git a/lib/oxrdfio/src/parser.rs b/lib/oxrdfio/src/parser.rs index d5bf196d..0f6d11ac 100644 --- a/lib/oxrdfio/src/parser.rs +++ b/lib/oxrdfio/src/parser.rs @@ -48,7 +48,9 @@ use tokio::io::AsyncRead; /// let file = " ."; /// /// let parser = RdfParser::from_format(RdfFormat::NTriples); -/// let quads = parser.parse_read(file.as_bytes()).collect::,_>>()?; +/// let quads = parser +/// .parse_read(file.as_bytes()) +/// .collect::, _>>()?; /// /// assert_eq!(quads.len(), 1); /// assert_eq!(quads[0].subject.to_string(), ""); @@ -129,9 +131,12 @@ impl RdfParser { /// The format the parser uses. /// /// ``` - /// use oxrdfio::{RdfParser, RdfFormat}; + /// use oxrdfio::{RdfFormat, RdfParser}; /// - /// assert_eq!(RdfParser::from_format(RdfFormat::Turtle).format(), RdfFormat::Turtle); + /// assert_eq!( + /// RdfParser::from_format(RdfFormat::Turtle).format(), + /// RdfFormat::Turtle + /// ); /// ``` pub fn format(&self) -> RdfFormat { match &self.inner { @@ -152,7 +157,9 @@ impl RdfParser { /// let file = "

."; /// /// let parser = RdfParser::from_format(RdfFormat::Turtle).with_base_iri("http://example.com")?; - /// let quads = parser.parse_read(file.as_bytes()).collect::,_>>()?; + /// let quads = parser + /// .parse_read(file.as_bytes()) + /// .collect::, _>>()?; /// /// assert_eq!(quads.len(), 1); /// assert_eq!(quads[0].subject.to_string(), ""); @@ -179,8 +186,11 @@ impl RdfParser { /// /// let file = " ."; /// - /// let parser = RdfParser::from_format(RdfFormat::Turtle).with_default_graph(NamedNode::new("http://example.com/g")?); - /// let quads = parser.parse_read(file.as_bytes()).collect::,_>>()?; + /// let parser = RdfParser::from_format(RdfFormat::Turtle) + /// .with_default_graph(NamedNode::new("http://example.com/g")?); + /// let quads = parser + /// .parse_read(file.as_bytes()) + /// .collect::, _>>()?; /// /// assert_eq!(quads.len(), 1); /// assert_eq!(quads[0].graph_name.to_string(), ""); @@ -221,10 +231,12 @@ impl RdfParser { /// /// let result1 = RdfParser::from_format(RdfFormat::NQuads) /// .rename_blank_nodes() - /// .parse_read(file.as_bytes()).collect::,_>>()?; + /// .parse_read(file.as_bytes()) + /// .collect::, _>>()?; /// let result2 = RdfParser::from_format(RdfFormat::NQuads) /// .rename_blank_nodes() - /// .parse_read(file.as_bytes()).collect::,_>>()?; + /// .parse_read(file.as_bytes()) + /// .collect::, _>>()?; /// assert_ne!(result1, result2); /// # Result::<_,Box>::Ok(()) /// ``` @@ -262,7 +274,9 @@ impl RdfParser { /// let file = " ."; /// /// let parser = RdfParser::from_format(RdfFormat::NTriples); - /// let quads = parser.parse_read(file.as_bytes()).collect::,_>>()?; + /// let quads = parser + /// .parse_read(file.as_bytes()) + /// .collect::, _>>()?; /// /// assert_eq!(quads.len(), 1); /// assert_eq!(quads[0].subject.to_string(), ""); @@ -358,7 +372,9 @@ impl From for RdfParser { /// let file = " ."; /// /// let parser = RdfParser::from_format(RdfFormat::NTriples); -/// let quads = parser.parse_read(file.as_bytes()).collect::,_>>()?; +/// let quads = parser +/// .parse_read(file.as_bytes()) +/// .collect::, _>>()?; /// /// assert_eq!(quads.len(), 1); /// assert_eq!(quads[0].subject.to_string(), ""); diff --git a/lib/oxrdfio/src/serializer.rs b/lib/oxrdfio/src/serializer.rs index cd132cd5..7abf7696 100644 --- a/lib/oxrdfio/src/serializer.rs +++ b/lib/oxrdfio/src/serializer.rs @@ -63,9 +63,12 @@ impl RdfSerializer { /// The format the serializer serializes to. /// /// ``` - /// use oxrdfio::{RdfSerializer, RdfFormat}; + /// use oxrdfio::{RdfFormat, RdfSerializer}; /// - /// assert_eq!(RdfSerializer::from_format(RdfFormat::Turtle).format(), RdfFormat::Turtle); + /// assert_eq!( + /// RdfSerializer::from_format(RdfFormat::Turtle).format(), + /// RdfFormat::Turtle + /// ); /// ``` pub fn format(&self) -> RdfFormat { self.format diff --git a/lib/oxrdfxml/src/parser.rs b/lib/oxrdfxml/src/parser.rs index 3c6b23ff..70ca91aa 100644 --- a/lib/oxrdfxml/src/parser.rs +++ b/lib/oxrdfxml/src/parser.rs @@ -26,7 +26,8 @@ use tokio::io::{AsyncRead, BufReader as AsyncBufReader}; /// /// Count the number of people: /// ``` -/// use oxrdf::{NamedNodeRef, vocab::rdf}; +/// use oxrdf::vocab::rdf; +/// use oxrdf::NamedNodeRef; /// use oxrdfxml::RdfXmlParser; /// /// let file = br#" @@ -84,7 +85,8 @@ impl RdfXmlParser { /// /// Count the number of people: /// ``` - /// use oxrdf::{NamedNodeRef, vocab::rdf}; + /// use oxrdf::vocab::rdf; + /// use oxrdf::NamedNodeRef; /// use oxrdfxml::RdfXmlParser; /// /// let file = br#" @@ -119,7 +121,8 @@ impl RdfXmlParser { /// /// Count the number of people: /// ``` - /// use oxrdf::{NamedNodeRef, vocab::rdf}; + /// use oxrdf::vocab::rdf; + /// use oxrdf::NamedNodeRef; /// use oxrdfxml::RdfXmlParser; /// /// # #[tokio::main(flavor = "current_thread")] @@ -179,7 +182,8 @@ impl RdfXmlParser { /// /// Count the number of people: /// ``` -/// use oxrdf::{NamedNodeRef, vocab::rdf}; +/// use oxrdf::vocab::rdf; +/// use oxrdf::NamedNodeRef; /// use oxrdfxml::RdfXmlParser; /// /// let file = br#" @@ -246,8 +250,9 @@ impl FromReadRdfXmlReader { /// /// Count the number of people: /// ``` -/// use oxrdf::{NamedNodeRef, vocab::rdf}; -/// use oxrdfxml::RdfXmlParser; +/// use oxrdf::vocab::rdf; +/// use oxrdf::NamedNodeRef; +/// use oxrdfxml::RdfXmlParser; /// /// # #[tokio::main(flavor = "current_thread")] /// # async fn main() -> Result<(), oxrdfxml::ParseError> { @@ -368,7 +373,7 @@ enum RdfXmlState { li_counter: u64, }, PropertyElt { - //Resource, Literal or Empty property element + // Resource, Literal or Empty property element iri: NamedNode, base_iri: Option>, language: Option, @@ -392,7 +397,7 @@ enum RdfXmlState { subject: Subject, writer: Writer>, id_attr: Option, - emit: bool, //false for parseTypeOtherPropertyElt support + emit: bool, // false for parseTypeOtherPropertyElt support }, } @@ -523,7 +528,7 @@ impl RdfXmlReader { PropertyElt { subject: Subject }, } - //Literal case + // Literal case if let Some(RdfXmlState::ParseTypeLiteralPropertyElt { writer, .. }) = self.state.last_mut() { let mut clean_event = BytesStart::new( @@ -542,7 +547,7 @@ impl RdfXmlReader { let tag_name = self.resolve_tag_name(event.name())?; - //We read attributes + // We read attributes let (mut language, mut base_iri) = if let Some(current_state) = self.state.last() { ( current_state.language().cloned(), @@ -652,7 +657,7 @@ impl RdfXmlReader { } } - //Parsing with the base URI + // Parsing with the base URI let id_attr = match id_attr { Some(iri) => { let iri = self.resolve_iri(&base_iri, iri)?; @@ -855,7 +860,7 @@ impl RdfXmlReader { event: &BytesEnd<'_>, results: &mut Vec, ) -> Result<(), ParseError> { - //Literal case + // Literal case if self.in_literal_depth > 0 { if let Some(RdfXmlState::ParseTypeLiteralPropertyElt { writer, .. }) = self.state.last_mut() diff --git a/lib/oxsdatatypes/src/date_time.rs b/lib/oxsdatatypes/src/date_time.rs index d2405692..127990df 100644 --- a/lib/oxsdatatypes/src/date_time.rs +++ b/lib/oxsdatatypes/src/date_time.rs @@ -17,6 +17,13 @@ pub struct DateTime { } impl DateTime { + pub const MAX: Self = Self { + timestamp: Timestamp::MAX, + }; + pub const MIN: Self = Self { + timestamp: Timestamp::MIN, + }; + #[inline] pub(super) fn new( year: i64, @@ -241,14 +248,6 @@ impl DateTime { pub fn is_identical_with(self, other: Self) -> bool { self.timestamp.is_identical_with(other.timestamp) } - - pub const MIN: Self = Self { - timestamp: Timestamp::MIN, - }; - - pub const MAX: Self = Self { - timestamp: Timestamp::MAX, - }; } /// Conversion according to [XPath cast rules](https://www.w3.org/TR/xpath-functions-31/#casting-to-datetimes). @@ -317,6 +316,21 @@ pub struct Time { } impl Time { + #[cfg(test)] + const MAX: Self = Self { + timestamp: Timestamp { + value: Decimal::new_from_i128_unchecked(62_230_255_200), + timezone_offset: Some(TimezoneOffset::MIN), + }, + }; + #[cfg(test)] + const MIN: Self = Self { + timestamp: Timestamp { + value: Decimal::new_from_i128_unchecked(62_230_154_400), + timezone_offset: Some(TimezoneOffset::MAX), + }, + }; + #[inline] fn new( mut hour: u8, @@ -493,22 +507,6 @@ impl Time { pub fn is_identical_with(self, other: Self) -> bool { self.timestamp.is_identical_with(other.timestamp) } - - #[cfg(test)] - const MIN: Self = Self { - timestamp: Timestamp { - value: Decimal::new_from_i128_unchecked(62_230_154_400), - timezone_offset: Some(TimezoneOffset::MAX), - }, - }; - - #[cfg(test)] - const MAX: Self = Self { - timestamp: Timestamp { - value: Decimal::new_from_i128_unchecked(62_230_255_200), - timezone_offset: Some(TimezoneOffset::MIN), - }, - }; } /// Conversion according to [XPath cast rules](https://www.w3.org/TR/xpath-functions-31/#casting-to-datetimes). @@ -566,6 +564,19 @@ pub struct Date { } impl Date { + pub const MAX: Self = Self { + timestamp: Timestamp { + value: Decimal::new_from_i128_unchecked(170_141_183_460_469_216_800), + timezone_offset: Some(TimezoneOffset::MAX), + }, + }; + pub const MIN: Self = Self { + timestamp: Timestamp { + value: Decimal::new_from_i128_unchecked(-170_141_183_460_469_216_800), + timezone_offset: Some(TimezoneOffset::MIN), + }, + }; + #[inline] fn new( year: i64, @@ -742,19 +753,6 @@ impl Date { pub fn is_identical_with(self, other: Self) -> bool { self.timestamp.is_identical_with(other.timestamp) } - - pub const MIN: Self = Self { - timestamp: Timestamp { - value: Decimal::new_from_i128_unchecked(-170_141_183_460_469_216_800), - timezone_offset: Some(TimezoneOffset::MIN), - }, - }; - pub const MAX: Self = Self { - timestamp: Timestamp { - value: Decimal::new_from_i128_unchecked(170_141_183_460_469_216_800), - timezone_offset: Some(TimezoneOffset::MAX), - }, - }; } /// Conversion according to [XPath cast rules](https://www.w3.org/TR/xpath-functions-31/#casting-to-datetimes). @@ -805,6 +803,19 @@ pub struct GYearMonth { } impl GYearMonth { + pub const MAX: Self = Self { + timestamp: Timestamp { + value: Decimal::new_from_i128_unchecked(170_141_183_460_469_216_800), + timezone_offset: Some(TimezoneOffset::MAX), + }, + }; + pub const MIN: Self = Self { + timestamp: Timestamp { + value: Decimal::new_from_i128_unchecked(-170_141_183_460_466_970_400), + timezone_offset: Some(TimezoneOffset::MIN), + }, + }; + #[inline] fn new( year: i64, @@ -876,19 +887,6 @@ impl GYearMonth { pub fn is_identical_with(self, other: Self) -> bool { self.timestamp.is_identical_with(other.timestamp) } - - pub const MIN: Self = Self { - timestamp: Timestamp { - value: Decimal::new_from_i128_unchecked(-170_141_183_460_466_970_400), - timezone_offset: Some(TimezoneOffset::MIN), - }, - }; - pub const MAX: Self = Self { - timestamp: Timestamp { - value: Decimal::new_from_i128_unchecked(170_141_183_460_469_216_800), - timezone_offset: Some(TimezoneOffset::MAX), - }, - }; } /// Conversion according to [XPath cast rules](https://www.w3.org/TR/xpath-functions-31/#casting-to-datetimes). @@ -947,6 +945,19 @@ pub struct GYear { } impl GYear { + pub const MAX: Self = Self { + timestamp: Timestamp { + value: Decimal::new_from_i128_unchecked(170_141_183_460_461_440_800), + timezone_offset: Some(TimezoneOffset::MAX), + }, + }; + pub const MIN: Self = Self { + timestamp: Timestamp { + value: Decimal::new_from_i128_unchecked(-170_141_183_460_461_700_000), + timezone_offset: Some(TimezoneOffset::MIN), + }, + }; + #[inline] fn new( year: i64, @@ -1011,19 +1022,6 @@ impl GYear { pub fn is_identical_with(self, other: Self) -> bool { self.timestamp.is_identical_with(other.timestamp) } - - pub const MIN: Self = Self { - timestamp: Timestamp { - value: Decimal::new_from_i128_unchecked(-170_141_183_460_461_700_000), - timezone_offset: Some(TimezoneOffset::MIN), - }, - }; - pub const MAX: Self = Self { - timestamp: Timestamp { - value: Decimal::new_from_i128_unchecked(170_141_183_460_461_440_800), - timezone_offset: Some(TimezoneOffset::MAX), - }, - }; } /// Conversion according to [XPath cast rules](https://www.w3.org/TR/xpath-functions-31/#casting-to-datetimes). @@ -1461,6 +1459,10 @@ pub struct TimezoneOffset { } impl TimezoneOffset { + pub const MAX: Self = Self { offset: 14 * 60 }; + pub const MIN: Self = Self { offset: -14 * 60 }; + pub const UTC: Self = Self { offset: 0 }; + /// From offset in minute with respect to UTC #[inline] pub fn new(offset_in_minutes: i16) -> Result { @@ -1489,10 +1491,6 @@ impl TimezoneOffset { pub fn to_be_bytes(self) -> [u8; 2] { self.offset.to_be_bytes() } - - pub const MIN: Self = Self { offset: -14 * 60 }; - pub const UTC: Self = Self { offset: 0 }; - pub const MAX: Self = Self { offset: 14 * 60 }; } impl TryFrom for TimezoneOffset { @@ -1576,7 +1574,7 @@ impl PartialEq for Timestamp { fn eq(&self, other: &Self) -> bool { match (self.timezone_offset, other.timezone_offset) { (Some(_), Some(_)) | (None, None) => self.value.eq(&other.value), - _ => false, //TODO: implicit timezone + _ => false, // TODO: implicit timezone } } } @@ -1622,6 +1620,15 @@ impl Hash for Timestamp { } impl Timestamp { + pub const MAX: Self = Self { + value: Decimal::MAX, + timezone_offset: Some(TimezoneOffset::MAX), + }; + pub const MIN: Self = Self { + value: Decimal::MIN, + timezone_offset: Some(TimezoneOffset::MIN), + }; + #[inline] fn new(props: &DateTimeSevenPropertyModel) -> Result { Ok(Self { @@ -1790,7 +1797,7 @@ impl Timestamp { (Some(_), Some(_)) | (None, None) => { Some(DayTimeDuration::new(self.value.checked_sub(rhs.value)?)) } - _ => None, //TODO: implicit timezone + _ => None, // TODO: implicit timezone } } @@ -1816,13 +1823,13 @@ impl Timestamp { Self { value: self .value - .checked_add(i64::from(from_timezone.offset) * 60)?, // We keep the literal value + .checked_add(i64::from(from_timezone.offset) * 60)?, /* We keep the literal value */ timezone_offset: None, } } } else if let Some(to_timezone) = timezone_offset { Self { - value: self.value.checked_sub(i64::from(to_timezone.offset) * 60)?, // We keep the literal value + value: self.value.checked_sub(i64::from(to_timezone.offset) * 60)?, /* We keep the literal value */ timezone_offset: Some(to_timezone), } } else { @@ -1851,16 +1858,6 @@ impl Timestamp { pub fn is_identical_with(self, other: Self) -> bool { self.value == other.value && self.timezone_offset == other.timezone_offset } - - pub const MIN: Self = Self { - value: Decimal::MIN, - timezone_offset: Some(TimezoneOffset::MIN), - }; - - pub const MAX: Self = Self { - value: Decimal::MAX, - timezone_offset: Some(TimezoneOffset::MAX), - }; } #[cfg(feature = "custom-now")] @@ -1960,7 +1957,7 @@ fn normalize_second( mi: i64, se: Decimal, ) -> Option<(i64, u8, u8, u8, u8, Decimal)> { - let mi = mi.checked_add(i64::try_from(se.as_i128().checked_div(60)?).ok()?)?; //TODO: good idea? + let mi = mi.checked_add(i64::try_from(se.as_i128().checked_div(60)?).ok()?)?; // TODO: good idea? let se = se.checked_rem(60)?; let (yr, mo, da, hr, mi) = normalize_minute(yr, mo, da, hr, mi)?; Some((yr, mo, da, hr, mi, se)) diff --git a/lib/oxsdatatypes/src/decimal.rs b/lib/oxsdatatypes/src/decimal.rs index 6a59105e..0082ca8a 100644 --- a/lib/oxsdatatypes/src/decimal.rs +++ b/lib/oxsdatatypes/src/decimal.rs @@ -19,6 +19,11 @@ pub struct Decimal { } impl Decimal { + pub const MAX: Self = Self { value: i128::MAX }; + pub const MIN: Self = Self { value: i128::MIN }; + #[cfg(test)] + pub const STEP: Self = Self { value: 1 }; + /// Constructs the decimal i / 10^n #[inline] pub const fn new(i: i128, n: u32) -> Result { @@ -260,13 +265,6 @@ impl Decimal { pub(super) const fn as_i128(self) -> i128 { self.value / DECIMAL_PART_POW } - - pub const MIN: Self = Self { value: i128::MIN }; - - pub const MAX: Self = Self { value: i128::MAX }; - - #[cfg(test)] - pub const STEP: Self = Self { value: 1 }; } impl From for Decimal { @@ -499,7 +497,7 @@ impl FromStr for Decimal { } input = &input[1..]; if input.is_empty() && !with_before_dot { - //We only have a dot + // We only have a dot return Err(PARSE_UNEXPECTED_END); } while input.last() == Some(&b'0') { @@ -520,11 +518,11 @@ impl FromStr for Decimal { } } if exp == 0 { - //Underflow + // Underflow return Err(PARSE_UNDERFLOW); } } else if !with_before_dot { - //It's empty + // It's empty return Err(PARSE_UNEXPECTED_END); } diff --git a/lib/oxsdatatypes/src/double.rs b/lib/oxsdatatypes/src/double.rs index 1a399019..3b58858f 100644 --- a/lib/oxsdatatypes/src/double.rs +++ b/lib/oxsdatatypes/src/double.rs @@ -17,6 +17,16 @@ pub struct Double { } impl Double { + pub const INFINITY: Self = Self { + value: f64::INFINITY, + }; + pub const MAX: Self = Self { value: f64::MAX }; + pub const MIN: Self = Self { value: f64::MIN }; + pub const NAN: Self = Self { value: f64::NAN }; + pub const NEG_INFINITY: Self = Self { + value: f64::NEG_INFINITY, + }; + #[inline] #[must_use] pub fn from_be_bytes(bytes: [u8; 8]) -> Self { @@ -77,20 +87,6 @@ impl Double { pub fn is_identical_with(self, other: Self) -> bool { self.value.to_bits() == other.value.to_bits() } - - pub const MIN: Self = Self { value: f64::MIN }; - - pub const MAX: Self = Self { value: f64::MAX }; - - pub const INFINITY: Self = Self { - value: f64::INFINITY, - }; - - pub const NEG_INFINITY: Self = Self { - value: f64::NEG_INFINITY, - }; - - pub const NAN: Self = Self { value: f64::NAN }; } impl From for f64 { diff --git a/lib/oxsdatatypes/src/duration.rs b/lib/oxsdatatypes/src/duration.rs index 93dac7d6..efb92b62 100644 --- a/lib/oxsdatatypes/src/duration.rs +++ b/lib/oxsdatatypes/src/duration.rs @@ -15,6 +15,15 @@ pub struct Duration { } impl Duration { + pub const MAX: Self = Self { + year_month: YearMonthDuration::MAX, + day_time: DayTimeDuration::MAX, + }; + pub const MIN: Self = Self { + year_month: YearMonthDuration::MIN, + day_time: DayTimeDuration::MIN, + }; + #[inline] pub fn new( months: impl Into, @@ -160,16 +169,6 @@ impl Duration { pub fn is_identical_with(self, other: Self) -> bool { self == other } - - pub const MIN: Self = Self { - year_month: YearMonthDuration::MIN, - day_time: DayTimeDuration::MIN, - }; - - pub const MAX: Self = Self { - year_month: YearMonthDuration::MAX, - day_time: DayTimeDuration::MAX, - }; } impl TryFrom for Duration { @@ -301,6 +300,9 @@ pub struct YearMonthDuration { } impl YearMonthDuration { + pub const MAX: Self = Self { months: i64::MAX }; + pub const MIN: Self = Self { months: i64::MIN }; + #[inline] pub fn new(months: impl Into) -> Self { Self { @@ -374,10 +376,6 @@ impl YearMonthDuration { pub fn is_identical_with(self, other: Self) -> bool { self == other } - - pub const MIN: Self = Self { months: i64::MIN }; - - pub const MAX: Self = Self { months: i64::MAX }; } impl From for Duration { @@ -469,6 +467,13 @@ pub struct DayTimeDuration { } impl DayTimeDuration { + pub const MAX: Self = Self { + seconds: Decimal::MAX, + }; + pub const MIN: Self = Self { + seconds: Decimal::MIN, + }; + #[inline] pub fn new(seconds: impl Into) -> Self { Self { @@ -558,14 +563,6 @@ impl DayTimeDuration { pub fn is_identical_with(self, other: Self) -> bool { self == other } - - pub const MIN: Self = Self { - seconds: Decimal::MIN, - }; - - pub const MAX: Self = Self { - seconds: Decimal::MAX, - }; } impl From for Duration { diff --git a/lib/oxsdatatypes/src/float.rs b/lib/oxsdatatypes/src/float.rs index bc0aab75..c4d08d6c 100644 --- a/lib/oxsdatatypes/src/float.rs +++ b/lib/oxsdatatypes/src/float.rs @@ -17,6 +17,16 @@ pub struct Float { } impl Float { + pub const INFINITY: Self = Self { + value: f32::INFINITY, + }; + pub const MAX: Self = Self { value: f32::MAX }; + pub const MIN: Self = Self { value: f32::MIN }; + pub const NAN: Self = Self { value: f32::NAN }; + pub const NEG_INFINITY: Self = Self { + value: f32::NEG_INFINITY, + }; + #[inline] #[must_use] pub fn from_be_bytes(bytes: [u8; 4]) -> Self { @@ -77,20 +87,6 @@ impl Float { pub fn is_identical_with(self, other: Self) -> bool { self.value.to_bits() == other.value.to_bits() } - - pub const MIN: Self = Self { value: f32::MIN }; - - pub const MAX: Self = Self { value: f32::MAX }; - - pub const INFINITY: Self = Self { - value: f32::INFINITY, - }; - - pub const NEG_INFINITY: Self = Self { - value: f32::NEG_INFINITY, - }; - - pub const NAN: Self = Self { value: f32::NAN }; } impl From for f32 { diff --git a/lib/oxsdatatypes/src/integer.rs b/lib/oxsdatatypes/src/integer.rs index f2b8506f..e76ae62e 100644 --- a/lib/oxsdatatypes/src/integer.rs +++ b/lib/oxsdatatypes/src/integer.rs @@ -14,6 +14,9 @@ pub struct Integer { } impl Integer { + pub const MAX: Self = Self { value: i64::MAX }; + pub const MIN: Self = Self { value: i64::MIN }; + #[inline] #[must_use] pub fn from_be_bytes(bytes: [u8; 8]) -> Self { @@ -134,10 +137,6 @@ impl Integer { pub fn is_identical_with(self, other: Self) -> bool { self == other } - - pub const MIN: Self = Self { value: i64::MIN }; - - pub const MAX: Self = Self { value: i64::MAX }; } impl From for Integer { diff --git a/lib/oxttl/src/lexer.rs b/lib/oxttl/src/lexer.rs index d4eb024f..3fb62845 100644 --- a/lib/oxttl/src/lexer.rs +++ b/lib/oxttl/src/lexer.rs @@ -49,8 +49,8 @@ pub struct N3Lexer { // TODO: simplify by not giving is_end and fail with an "unexpected eof" is none is returned when is_end=true? impl TokenRecognizer for N3Lexer { - type Token<'a> = N3Token<'a>; type Options = N3LexerOptions; + type Token<'a> = N3Token<'a>; fn recognize_next_token<'a>( &mut self, @@ -790,7 +790,7 @@ impl N3Lexer { format!("Unexpected escape character '\\{}'", char::from(c)), ) .into()), - )), //TODO: read until end of string + )), // TODO: read until end of string } } diff --git a/lib/oxttl/src/line_formats.rs b/lib/oxttl/src/line_formats.rs index e7d39e09..5932f7a2 100644 --- a/lib/oxttl/src/line_formats.rs +++ b/lib/oxttl/src/line_formats.rs @@ -39,9 +39,9 @@ enum NQuadsState { } impl RuleRecognizer for NQuadsRecognizer { - type TokenRecognizer = N3Lexer; - type Output = Quad; type Context = NQuadsRecognizerContext; + type Output = Quad; + type TokenRecognizer = N3Lexer; fn error_recovery_state(mut self) -> Self { self.stack.clear(); @@ -251,7 +251,7 @@ impl RuleRecognizer for NQuadsRecognizer { self.emit_quad(results, GraphName::DefaultGraph); errors.push("Triples should be followed by a dot".into()) } - _ => errors.push("Unexpected end".into()), //TODO + _ => errors.push("Unexpected end".into()), // TODO } } diff --git a/lib/oxttl/src/n3.rs b/lib/oxttl/src/n3.rs index a1c23f25..263db936 100644 --- a/lib/oxttl/src/n3.rs +++ b/lib/oxttl/src/n3.rs @@ -181,7 +181,8 @@ impl From for N3Quad { /// /// Count the number of people: /// ``` -/// use oxrdf::{NamedNode, vocab::rdf}; +/// use oxrdf::vocab::rdf; +/// use oxrdf::NamedNode; /// use oxttl::n3::{N3Parser, N3Term}; /// /// let file = br#"@base . @@ -260,7 +261,9 @@ impl N3Parser { /// a schema:Person ; /// schema:name "Bar" ."#; /// - /// let rdf_type = N3Term::NamedNode(NamedNode::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?); + /// let rdf_type = N3Term::NamedNode(NamedNode::new( + /// "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", + /// )?); /// let schema_person = N3Term::NamedNode(NamedNode::new("http://schema.org/Person")?); /// let mut count = 0; /// for triple in N3Parser::new().parse_read(file.as_ref()) { @@ -282,7 +285,8 @@ impl N3Parser { /// /// Count the number of people: /// ``` - /// use oxrdf::{NamedNode, vocab::rdf}; + /// use oxrdf::vocab::rdf; + /// use oxrdf::NamedNode; /// use oxttl::n3::{N3Parser, N3Term}; /// /// # #[tokio::main(flavor = "current_thread")] @@ -322,14 +326,16 @@ impl N3Parser { /// /// Count the number of people: /// ``` - /// use oxrdf::{NamedNode, vocab::rdf}; + /// use oxrdf::vocab::rdf; + /// use oxrdf::NamedNode; /// use oxttl::n3::{N3Parser, N3Term}; /// - /// let file: [&[u8]; 5] = [b"@base ", + /// let file: [&[u8]; 5] = [ + /// b"@base ", /// b". @prefix schema: .", /// b" a schema:Person", /// b" ; schema:name \"Foo\" . ", - /// b" a schema:Person ; schema:name \"Bar\" ." + /// b" a schema:Person ; schema:name \"Bar\" .", /// ]; /// /// let rdf_type = N3Term::NamedNode(rdf::TYPE.into_owned()); @@ -340,7 +346,7 @@ impl N3Parser { /// while !parser.is_end() { /// // We feed more data to the parser /// if let Some(chunk) = file_chunks.next() { - /// parser.extend_from_slice(chunk); + /// parser.extend_from_slice(chunk); /// } else { /// parser.end(); // It's finished /// } @@ -366,7 +372,8 @@ impl N3Parser { /// /// Count the number of people: /// ``` -/// use oxrdf::{NamedNode, vocab::rdf}; +/// use oxrdf::vocab::rdf; +/// use oxrdf::NamedNode; /// use oxttl::n3::{N3Parser, N3Term}; /// /// let file = br#"@base . @@ -459,7 +466,8 @@ impl Iterator for FromReadN3Reader { /// /// Count the number of people: /// ``` -/// use oxrdf::{NamedNode, vocab::rdf}; +/// use oxrdf::vocab::rdf; +/// use oxrdf::NamedNode; /// use oxttl::n3::{N3Parser, N3Term}; /// /// # #[tokio::main(flavor = "current_thread")] @@ -561,14 +569,16 @@ impl FromTokioAsyncReadN3Reader { /// /// Count the number of people: /// ``` -/// use oxrdf::{NamedNode, vocab::rdf}; +/// use oxrdf::vocab::rdf; +/// use oxrdf::NamedNode; /// use oxttl::n3::{N3Parser, N3Term}; /// -/// let file: [&[u8]; 5] = [b"@base ", +/// let file: [&[u8]; 5] = [ +/// b"@base ", /// b". @prefix schema: .", /// b" a schema:Person", /// b" ; schema:name \"Foo\" . ", -/// b" a schema:Person ; schema:name \"Bar\" ." +/// b" a schema:Person ; schema:name \"Bar\" .", /// ]; /// /// let rdf_type = N3Term::NamedNode(rdf::TYPE.into_owned()); @@ -579,7 +589,7 @@ impl FromTokioAsyncReadN3Reader { /// while !parser.is_end() { /// // We feed more data to the parser /// if let Some(chunk) = file_chunks.next() { -/// parser.extend_from_slice(chunk); +/// parser.extend_from_slice(chunk); /// } else { /// parser.end(); // It's finished /// } @@ -697,9 +707,9 @@ struct N3RecognizerContext { } impl RuleRecognizer for N3Recognizer { - type TokenRecognizer = N3Lexer; - type Output = N3Quad; type Context = N3RecognizerContext; + type Output = N3Quad; + type TokenRecognizer = N3Lexer; fn error_recovery_state(mut self) -> Self { self.stack.clear(); @@ -1191,7 +1201,7 @@ impl RuleRecognizer for N3Recognizer { ) { match &*self.stack { [] | [N3State::N3Doc] => (), - _ => errors.push("Unexpected end".into()), //TODO + _ => errors.push("Unexpected end".into()), // TODO } } diff --git a/lib/oxttl/src/terse.rs b/lib/oxttl/src/terse.rs index 86fad434..818524f6 100644 --- a/lib/oxttl/src/terse.rs +++ b/lib/oxttl/src/terse.rs @@ -4,12 +4,10 @@ use crate::lexer::{resolve_local_name, N3Lexer, N3LexerMode, N3LexerOptions, N3T use crate::toolkit::{Lexer, Parser, RuleRecognizer, RuleRecognizerError}; use crate::{MAX_BUFFER_SIZE, MIN_BUFFER_SIZE}; use oxiri::Iri; +use oxrdf::vocab::{rdf, xsd}; #[cfg(feature = "rdf-star")] use oxrdf::Triple; -use oxrdf::{ - vocab::{rdf, xsd}, - BlankNode, GraphName, Literal, NamedNode, NamedOrBlankNode, Quad, Subject, Term, -}; +use oxrdf::{BlankNode, GraphName, Literal, NamedNode, NamedOrBlankNode, Quad, Subject, Term}; use std::collections::HashMap; pub struct TriGRecognizer { @@ -30,9 +28,9 @@ pub struct TriGRecognizerContext { } impl RuleRecognizer for TriGRecognizer { - type TokenRecognizer = N3Lexer; - type Output = Quad; type Context = TriGRecognizerContext; + type Output = Quad; + type TokenRecognizer = N3Lexer; fn error_recovery_state(mut self) -> Self { self.stack.clear(); @@ -784,7 +782,7 @@ impl RuleRecognizer for TriGRecognizer { } } } else if token == N3Token::Punctuation(".") || token == N3Token::Punctuation("}") { - //TODO: be smarter depending if we are in '{' or not + // TODO: be smarter depending if we are in '{' or not self.stack.push(TriGState::TriGDoc); self } else { @@ -819,7 +817,7 @@ impl RuleRecognizer for TriGRecognizer { self.emit_quad(results); errors.push("Triples should be followed by a dot".into()) } - _ => errors.push("Unexpected end".into()), //TODO + _ => errors.push("Unexpected end".into()), // TODO } } diff --git a/lib/oxttl/src/toolkit/lexer.rs b/lib/oxttl/src/toolkit/lexer.rs index 0f7373c2..2406dfb1 100644 --- a/lib/oxttl/src/toolkit/lexer.rs +++ b/lib/oxttl/src/toolkit/lexer.rs @@ -366,7 +366,7 @@ impl Lexer { _ => return Some(()), } i += 1; - //TODO: SIMD + // TODO: SIMD } } else { for c in &self.data[self.position.buffer_offset..] { @@ -376,7 +376,7 @@ impl Lexer { } else { return Some(()); } - //TODO: SIMD + // TODO: SIMD } } Some(()) diff --git a/lib/oxttl/src/trig.rs b/lib/oxttl/src/trig.rs index 0dad7fd7..5a7cdb4a 100644 --- a/lib/oxttl/src/trig.rs +++ b/lib/oxttl/src/trig.rs @@ -6,7 +6,8 @@ use crate::terse::TriGRecognizer; use crate::toolkit::FromTokioAsyncReadIterator; use crate::toolkit::{FromReadIterator, ParseError, Parser, SyntaxError}; use oxiri::{Iri, IriParseError}; -use oxrdf::{vocab::xsd, GraphName, NamedNode, Quad, QuadRef, Subject, TermRef}; +use oxrdf::vocab::xsd; +use oxrdf::{GraphName, NamedNode, Quad, QuadRef, Subject, TermRef}; use std::collections::HashMap; use std::fmt; use std::io::{self, Read, Write}; @@ -19,7 +20,8 @@ use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; /// /// Count the number of people: /// ``` -/// use oxrdf::{NamedNodeRef, vocab::rdf}; +/// use oxrdf::vocab::rdf; +/// use oxrdf::NamedNodeRef; /// use oxttl::TriGParser; /// /// let file = br#"@base . @@ -97,7 +99,8 @@ impl TriGParser { /// /// Count the number of people: /// ``` - /// use oxrdf::{NamedNodeRef, vocab::rdf}; + /// use oxrdf::vocab::rdf; + /// use oxrdf::NamedNodeRef; /// use oxttl::TriGParser; /// /// let file = br#"@base . @@ -128,7 +131,8 @@ impl TriGParser { /// /// Count the number of people: /// ``` - /// use oxrdf::{NamedNodeRef, vocab::rdf}; + /// use oxrdf::vocab::rdf; + /// use oxrdf::NamedNodeRef; /// use oxttl::TriGParser; /// /// # #[tokio::main(flavor = "current_thread")] @@ -167,14 +171,16 @@ impl TriGParser { /// /// Count the number of people: /// ``` - /// use oxrdf::{NamedNodeRef, vocab::rdf}; + /// use oxrdf::vocab::rdf; + /// use oxrdf::NamedNodeRef; /// use oxttl::TriGParser; /// - /// let file: [&[u8]; 5] = [b"@base ", + /// let file: [&[u8]; 5] = [ + /// b"@base ", /// b". @prefix schema: .", /// b" a schema:Person", /// b" ; schema:name \"Foo\" . ", - /// b" a schema:Person ; schema:name \"Bar\" ." + /// b" a schema:Person ; schema:name \"Bar\" .", /// ]; /// /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; @@ -184,7 +190,7 @@ impl TriGParser { /// while !parser.is_end() { /// // We feed more data to the parser /// if let Some(chunk) = file_chunks.next() { - /// parser.extend_from_slice(chunk); + /// parser.extend_from_slice(chunk); /// } else { /// parser.end(); // It's finished /// } @@ -217,7 +223,8 @@ impl TriGParser { /// /// Count the number of people: /// ``` -/// use oxrdf::{NamedNodeRef, vocab::rdf}; +/// use oxrdf::vocab::rdf; +/// use oxrdf::NamedNodeRef; /// use oxttl::TriGParser; /// /// let file = br#"@base . @@ -309,7 +316,8 @@ impl Iterator for FromReadTriGReader { /// /// Count the number of people: /// ``` -/// use oxrdf::{NamedNodeRef, vocab::rdf}; +/// use oxrdf::vocab::rdf; +/// use oxrdf::NamedNodeRef; /// use oxttl::TriGParser; /// /// # #[tokio::main(flavor = "current_thread")] @@ -410,14 +418,16 @@ impl FromTokioAsyncReadTriGReader { /// /// Count the number of people: /// ``` -/// use oxrdf::{NamedNodeRef, vocab::rdf}; +/// use oxrdf::vocab::rdf; +/// use oxrdf::NamedNodeRef; /// use oxttl::TriGParser; /// -/// let file: [&[u8]; 5] = [b"@base ", +/// let file: [&[u8]; 5] = [ +/// b"@base ", /// b". @prefix schema: .", /// b" a schema:Person", /// b" ; schema:name \"Foo\" . ", -/// b" a schema:Person ; schema:name \"Bar\" ." +/// b" a schema:Person ; schema:name \"Bar\" .", /// ]; /// /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; @@ -427,7 +437,7 @@ impl FromTokioAsyncReadTriGReader { /// while !parser.is_end() { /// // We feed more data to the parser /// if let Some(chunk) = file_chunks.next() { -/// parser.extend_from_slice(chunk); +/// parser.extend_from_slice(chunk); /// } else { /// parser.end(); // It's finished /// } diff --git a/lib/oxttl/src/turtle.rs b/lib/oxttl/src/turtle.rs index 542afd27..5a2b67a2 100644 --- a/lib/oxttl/src/turtle.rs +++ b/lib/oxttl/src/turtle.rs @@ -21,7 +21,8 @@ use tokio::io::{AsyncRead, AsyncWrite}; /// /// Count the number of people: /// ``` -/// use oxrdf::{NamedNodeRef, vocab::rdf}; +/// use oxrdf::vocab::rdf; +/// use oxrdf::NamedNodeRef; /// use oxttl::TurtleParser; /// /// let file = br#"@base . @@ -99,7 +100,8 @@ impl TurtleParser { /// /// Count the number of people: /// ``` - /// use oxrdf::{NamedNodeRef, vocab::rdf}; + /// use oxrdf::vocab::rdf; + /// use oxrdf::NamedNodeRef; /// use oxttl::TurtleParser; /// /// let file = br#"@base . @@ -130,7 +132,8 @@ impl TurtleParser { /// /// Count the number of people: /// ``` - /// use oxrdf::{NamedNodeRef, vocab::rdf}; + /// use oxrdf::vocab::rdf; + /// use oxrdf::NamedNodeRef; /// use oxttl::TurtleParser; /// /// # #[tokio::main(flavor = "current_thread")] @@ -169,14 +172,16 @@ impl TurtleParser { /// /// Count the number of people: /// ``` - /// use oxrdf::{NamedNodeRef, vocab::rdf}; + /// use oxrdf::vocab::rdf; + /// use oxrdf::NamedNodeRef; /// use oxttl::TurtleParser; /// - /// let file: [&[u8]; 5] = [b"@base ", + /// let file: [&[u8]; 5] = [ + /// b"@base ", /// b". @prefix schema: .", /// b" a schema:Person", /// b" ; schema:name \"Foo\" . ", - /// b" a schema:Person ; schema:name \"Bar\" ." + /// b" a schema:Person ; schema:name \"Bar\" .", /// ]; /// /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; @@ -186,7 +191,7 @@ impl TurtleParser { /// while !parser.is_end() { /// // We feed more data to the parser /// if let Some(chunk) = file_chunks.next() { - /// parser.extend_from_slice(chunk); + /// parser.extend_from_slice(chunk); /// } else { /// parser.end(); // It's finished /// } @@ -219,7 +224,8 @@ impl TurtleParser { /// /// Count the number of people: /// ``` -/// use oxrdf::{NamedNodeRef, vocab::rdf}; +/// use oxrdf::vocab::rdf; +/// use oxrdf::NamedNodeRef; /// use oxttl::TurtleParser; /// /// let file = br#"@base . @@ -311,7 +317,8 @@ impl Iterator for FromReadTurtleReader { /// /// Count the number of people: /// ``` -/// use oxrdf::{NamedNodeRef, vocab::rdf}; +/// use oxrdf::vocab::rdf; +/// use oxrdf::NamedNodeRef; /// use oxttl::TurtleParser; /// /// # #[tokio::main(flavor = "current_thread")] @@ -412,14 +419,16 @@ impl FromTokioAsyncReadTurtleReader { /// /// Count the number of people: /// ``` -/// use oxrdf::{NamedNodeRef, vocab::rdf}; +/// use oxrdf::vocab::rdf; +/// use oxrdf::NamedNodeRef; /// use oxttl::TurtleParser; /// -/// let file: [&[u8]; 5] = [b"@base ", +/// let file: [&[u8]; 5] = [ +/// b"@base ", /// b". @prefix schema: .", /// b" a schema:Person", /// b" ; schema:name \"Foo\" . ", -/// b" a schema:Person ; schema:name \"Bar\" ." +/// b" a schema:Person ; schema:name \"Bar\" .", /// ]; /// /// let schema_person = NamedNodeRef::new("http://schema.org/Person")?; @@ -429,7 +438,7 @@ impl FromTokioAsyncReadTurtleReader { /// while !parser.is_end() { /// // We feed more data to the parser /// if let Some(chunk) = file_chunks.next() { -/// parser.extend_from_slice(chunk); +/// parser.extend_from_slice(chunk); /// } else { /// parser.end(); // It's finished /// } diff --git a/lib/sparesults/src/csv.rs b/lib/sparesults/src/csv.rs index 985092b4..7cf6059e 100644 --- a/lib/sparesults/src/csv.rs +++ b/lib/sparesults/src/csv.rs @@ -2,8 +2,8 @@ use crate::error::{ParseError, SyntaxError, SyntaxErrorKind, TextPosition}; use memchr::memchr; -use oxrdf::Variable; -use oxrdf::{vocab::xsd, *}; +use oxrdf::vocab::xsd; +use oxrdf::*; use std::io::{self, Read, Write}; use std::str::{self, FromStr}; #[cfg(feature = "async-tokio")] diff --git a/lib/sparesults/src/format.rs b/lib/sparesults/src/format.rs index e7eba74a..982ff11f 100644 --- a/lib/sparesults/src/format.rs +++ b/lib/sparesults/src/format.rs @@ -20,7 +20,10 @@ impl QueryResultsFormat { /// ``` /// use sparesults::QueryResultsFormat; /// - /// assert_eq!(QueryResultsFormat::Json.iri(), "http://www.w3.org/ns/formats/SPARQL_Results_JSON") + /// assert_eq!( + /// QueryResultsFormat::Json.iri(), + /// "http://www.w3.org/ns/formats/SPARQL_Results_JSON" + /// ) /// ``` #[inline] pub fn iri(self) -> &'static str { @@ -31,12 +34,16 @@ impl QueryResultsFormat { Self::Tsv => "http://www.w3.org/ns/formats/SPARQL_Results_TSV", } } + /// The format [IANA media type](https://tools.ietf.org/html/rfc2046). /// /// ``` /// use sparesults::QueryResultsFormat; /// - /// assert_eq!(QueryResultsFormat::Json.media_type(), "application/sparql-results+json") + /// assert_eq!( + /// QueryResultsFormat::Json.media_type(), + /// "application/sparql-results+json" + /// ) /// ``` #[inline] pub fn media_type(self) -> &'static str { @@ -91,7 +98,10 @@ impl QueryResultsFormat { /// ``` /// use sparesults::QueryResultsFormat; /// - /// assert_eq!(QueryResultsFormat::from_media_type("application/sparql-results+json; charset=utf-8"), Some(QueryResultsFormat::Json)) + /// assert_eq!( + /// QueryResultsFormat::from_media_type("application/sparql-results+json; charset=utf-8"), + /// Some(QueryResultsFormat::Json) + /// ) /// ``` #[inline] pub fn from_media_type(media_type: &str) -> Option { @@ -134,7 +144,10 @@ impl QueryResultsFormat { /// ``` /// use sparesults::QueryResultsFormat; /// - /// assert_eq!(QueryResultsFormat::from_extension("json"), Some(QueryResultsFormat::Json)) + /// assert_eq!( + /// QueryResultsFormat::from_extension("json"), + /// Some(QueryResultsFormat::Json) + /// ) /// ``` #[inline] pub fn from_extension(extension: &str) -> Option { diff --git a/lib/sparesults/src/json.rs b/lib/sparesults/src/json.rs index 85b03fcd..2e63fc81 100644 --- a/lib/sparesults/src/json.rs +++ b/lib/sparesults/src/json.rs @@ -5,7 +5,6 @@ use crate::error::{ParseError, SyntaxError}; use json_event_parser::ToTokioAsyncWriteJsonWriter; use json_event_parser::{FromReadJsonReader, JsonEvent, ToWriteJsonWriter}; use oxrdf::vocab::rdf; -use oxrdf::Variable; use oxrdf::*; use std::collections::BTreeMap; use std::io::{self, Read, Write}; @@ -522,7 +521,7 @@ fn read_value( JsonEvent::EndObject => { if let Some(s) = state { if s == State::Value { - state = None; //End of triple + state = None; // End of triple } else { return Err( SyntaxError::msg("Term description values should be string").into() diff --git a/lib/sparesults/src/parser.rs b/lib/sparesults/src/parser.rs index a00d014c..3332335b 100644 --- a/lib/sparesults/src/parser.rs +++ b/lib/sparesults/src/parser.rs @@ -130,8 +130,8 @@ impl From for QueryResultsParser { /// /// Example in TSV (the API is the same for JSON and XML): /// ``` -/// use sparesults::{QueryResultsFormat, QueryResultsParser, FromReadQueryResultsReader}; /// use oxrdf::{Literal, Variable}; +/// use sparesults::{FromReadQueryResultsReader, QueryResultsFormat, QueryResultsParser}; /// /// let json_parser = QueryResultsParser::from_format(QueryResultsFormat::Tsv); /// @@ -141,10 +141,24 @@ impl From for QueryResultsParser { /// } /// /// // solutions -/// if let FromReadQueryResultsReader::Solutions(solutions) = json_parser.parse_read(b"?foo\t?bar\n\"test\"\t".as_slice())? { -/// assert_eq!(solutions.variables(), &[Variable::new_unchecked("foo"), Variable::new_unchecked("bar")]); +/// if let FromReadQueryResultsReader::Solutions(solutions) = +/// json_parser.parse_read(b"?foo\t?bar\n\"test\"\t".as_slice())? +/// { +/// assert_eq!( +/// solutions.variables(), +/// &[ +/// Variable::new_unchecked("foo"), +/// Variable::new_unchecked("bar") +/// ] +/// ); /// for solution in solutions { -/// assert_eq!(solution?.iter().collect::>(), vec![(&Variable::new_unchecked("foo"), &Literal::from("test").into())]); +/// assert_eq!( +/// solution?.iter().collect::>(), +/// vec![( +/// &Variable::new_unchecked("foo"), +/// &Literal::from("test").into() +/// )] +/// ); /// } /// } /// # Result::<(),sparesults::ParseError>::Ok(()) @@ -188,12 +202,20 @@ impl FromReadSolutionsReader { /// /// Example in TSV (the API is the same for JSON and XML): /// ``` - /// use sparesults::{QueryResultsFormat, QueryResultsParser, FromReadQueryResultsReader}; /// use oxrdf::Variable; + /// use sparesults::{FromReadQueryResultsReader, QueryResultsFormat, QueryResultsParser}; /// /// let json_parser = QueryResultsParser::from_format(QueryResultsFormat::Tsv); - /// if let FromReadQueryResultsReader::Solutions(solutions) = json_parser.parse_read(b"?foo\t?bar\n\"ex1\"\t\"ex2\"".as_slice())? { - /// assert_eq!(solutions.variables(), &[Variable::new_unchecked("foo"), Variable::new_unchecked("bar")]); + /// if let FromReadQueryResultsReader::Solutions(solutions) = + /// json_parser.parse_read(b"?foo\t?bar\n\"ex1\"\t\"ex2\"".as_slice())? + /// { + /// assert_eq!( + /// solutions.variables(), + /// &[ + /// Variable::new_unchecked("foo"), + /// Variable::new_unchecked("bar") + /// ] + /// ); /// } /// # Result::<(),sparesults::ParseError>::Ok(()) /// ``` diff --git a/lib/sparesults/src/serializer.rs b/lib/sparesults/src/serializer.rs index 13c21628..1d4a02b1 100644 --- a/lib/sparesults/src/serializer.rs +++ b/lib/sparesults/src/serializer.rs @@ -241,14 +241,23 @@ impl From for QueryResultsSerializer { /// /// Example in TSV (the API is the same for JSON, XML and CSV): /// ``` -/// use sparesults::{QueryResultsFormat, QueryResultsSerializer}; /// use oxrdf::{LiteralRef, Variable, VariableRef}; +/// use sparesults::{QueryResultsFormat, QueryResultsSerializer}; /// use std::iter::once; /// /// let tsv_serializer = QueryResultsSerializer::from_format(QueryResultsFormat::Tsv); /// let mut buffer = Vec::new(); -/// let mut writer = tsv_serializer.serialize_solutions_to_write(&mut buffer, vec![Variable::new_unchecked("foo"), Variable::new_unchecked("bar")])?; -/// writer.write(once((VariableRef::new_unchecked("foo"), LiteralRef::from("test"))))?; +/// let mut writer = tsv_serializer.serialize_solutions_to_write( +/// &mut buffer, +/// vec![ +/// Variable::new_unchecked("foo"), +/// Variable::new_unchecked("bar"), +/// ], +/// )?; +/// writer.write(once(( +/// VariableRef::new_unchecked("foo"), +/// LiteralRef::from("test"), +/// )))?; /// writer.finish()?; /// assert_eq!(buffer, b"?foo\t?bar\n\"test\"\t\n"); /// # std::io::Result::Ok(()) @@ -321,16 +330,29 @@ impl ToWriteSolutionsWriter { /// /// Example in TSV (the API is the same for JSON, CSV and XML): /// ``` -/// use sparesults::{QueryResultsFormat, QueryResultsSerializer}; /// use oxrdf::{LiteralRef, Variable, VariableRef}; +/// use sparesults::{QueryResultsFormat, QueryResultsSerializer}; /// use std::iter::once; /// /// # #[tokio::main(flavor = "current_thread")] /// # async fn main() -> std::io::Result<()> { /// let tsv_serializer = QueryResultsSerializer::from_format(QueryResultsFormat::Tsv); /// let mut buffer = Vec::new(); -/// let mut writer = tsv_serializer.serialize_solutions_to_tokio_async_write(&mut buffer, vec![Variable::new_unchecked("foo"), Variable::new_unchecked("bar")]).await?; -/// writer.write(once((VariableRef::new_unchecked("foo"), LiteralRef::from("test")))).await?; +/// let mut writer = tsv_serializer +/// .serialize_solutions_to_tokio_async_write( +/// &mut buffer, +/// vec![ +/// Variable::new_unchecked("foo"), +/// Variable::new_unchecked("bar"), +/// ], +/// ) +/// .await?; +/// writer +/// .write(once(( +/// VariableRef::new_unchecked("foo"), +/// LiteralRef::from("test"), +/// ))) +/// .await?; /// writer.finish().await?; /// assert_eq!(buffer, b"?foo\t?bar\n\"test\"\t\n"); /// # Ok(()) diff --git a/lib/sparesults/src/solution.rs b/lib/sparesults/src/solution.rs index 0d81adc2..826a9eea 100644 --- a/lib/sparesults/src/solution.rs +++ b/lib/sparesults/src/solution.rs @@ -44,10 +44,16 @@ impl QuerySolution { /// It is also the number of columns in the solutions table. /// /// ``` + /// use oxrdf::{Literal, Variable}; /// use sparesults::QuerySolution; - /// use oxrdf::{Variable, Literal}; /// - /// let solution = QuerySolution::from((vec![Variable::new_unchecked("foo"), Variable::new_unchecked("bar")], vec![Some(Literal::from(1).into()), None])); + /// let solution = QuerySolution::from(( + /// vec![ + /// Variable::new_unchecked("foo"), + /// Variable::new_unchecked("bar"), + /// ], + /// vec![Some(Literal::from(1).into()), None], + /// )); /// assert_eq!(solution.len(), 2); /// ``` #[inline] @@ -58,13 +64,25 @@ impl QuerySolution { /// Is there any variable bound in the table? /// /// ``` + /// use oxrdf::{Literal, Variable}; /// use sparesults::QuerySolution; - /// use oxrdf::{Variable, Literal}; /// - /// let solution = QuerySolution::from((vec![Variable::new_unchecked("foo"), Variable::new_unchecked("bar")], vec![Some(Literal::from(1).into()), None])); + /// let solution = QuerySolution::from(( + /// vec![ + /// Variable::new_unchecked("foo"), + /// Variable::new_unchecked("bar"), + /// ], + /// vec![Some(Literal::from(1).into()), None], + /// )); /// assert!(!solution.is_empty()); /// - /// let empty_solution = QuerySolution::from((vec![Variable::new_unchecked("foo"), Variable::new_unchecked("bar")], vec![None, None])); + /// let empty_solution = QuerySolution::from(( + /// vec![ + /// Variable::new_unchecked("foo"), + /// Variable::new_unchecked("bar"), + /// ], + /// vec![None, None], + /// )); /// assert!(empty_solution.is_empty()); /// ``` #[inline] @@ -75,11 +93,20 @@ impl QuerySolution { /// Returns an iterator over bound variables. /// /// ``` + /// use oxrdf::{Literal, Variable}; /// use sparesults::QuerySolution; - /// use oxrdf::{Variable, Literal}; /// - /// let solution = QuerySolution::from((vec![Variable::new_unchecked("foo"), Variable::new_unchecked("bar")], vec![Some(Literal::from(1).into()), None])); - /// assert_eq!(solution.iter().collect::>(), vec![(&Variable::new_unchecked("foo"), &Literal::from(1).into())]); + /// let solution = QuerySolution::from(( + /// vec![ + /// Variable::new_unchecked("foo"), + /// Variable::new_unchecked("bar"), + /// ], + /// vec![Some(Literal::from(1).into()), None], + /// )); + /// assert_eq!( + /// solution.iter().collect::>(), + /// vec![(&Variable::new_unchecked("foo"), &Literal::from(1).into())] + /// ); /// ``` #[inline] pub fn iter(&self) -> impl Iterator { @@ -89,10 +116,16 @@ impl QuerySolution { /// Returns the ordered slice of variable values. /// /// ``` + /// use oxrdf::{Literal, Variable}; /// use sparesults::QuerySolution; - /// use oxrdf::{Variable, Literal}; /// - /// let solution = QuerySolution::from((vec![Variable::new_unchecked("foo"), Variable::new_unchecked("bar")], vec![Some(Literal::from(1).into()), None])); + /// let solution = QuerySolution::from(( + /// vec![ + /// Variable::new_unchecked("foo"), + /// Variable::new_unchecked("bar"), + /// ], + /// vec![Some(Literal::from(1).into()), None], + /// )); /// assert_eq!(solution.values(), &[Some(Literal::from(1).into()), None]); /// ``` #[inline] @@ -103,11 +136,23 @@ impl QuerySolution { /// Returns the ordered slice of the solution variables, bound or not. /// /// ``` + /// use oxrdf::{Literal, Variable}; /// use sparesults::QuerySolution; - /// use oxrdf::{Variable, Literal}; /// - /// let solution = QuerySolution::from((vec![Variable::new_unchecked("foo"), Variable::new_unchecked("bar")], vec![Some(Literal::from(1).into()), None])); - /// assert_eq!(solution.variables(), &[Variable::new_unchecked("foo"), Variable::new_unchecked("bar")]); + /// let solution = QuerySolution::from(( + /// vec![ + /// Variable::new_unchecked("foo"), + /// Variable::new_unchecked("bar"), + /// ], + /// vec![Some(Literal::from(1).into()), None], + /// )); + /// assert_eq!( + /// solution.variables(), + /// &[ + /// Variable::new_unchecked("foo"), + /// Variable::new_unchecked("bar") + /// ] + /// ); /// ``` #[inline] pub fn variables(&self) -> &[Variable] { @@ -126,8 +171,8 @@ impl>, S: Into>>> From<(V, S)> for Quer } impl<'a> IntoIterator for &'a QuerySolution { - type Item = (&'a Variable, &'a Term); type IntoIter = Iter<'a>; + type Item = (&'a Variable, &'a Term); #[inline] fn into_iter(self) -> Self::IntoIter { @@ -214,11 +259,20 @@ impl fmt::Debug for QuerySolution { /// An iterator over [`QuerySolution`] bound variables. /// /// ``` +/// use oxrdf::{Literal, Variable}; /// use sparesults::QuerySolution; -/// use oxrdf::{Variable, Literal}; /// -/// let solution = QuerySolution::from((vec![Variable::new_unchecked("foo"), Variable::new_unchecked("bar")], vec![Some(Literal::from(1).into()), None])); -/// assert_eq!(solution.iter().collect::>(), vec![(&Variable::new_unchecked("foo"), &Literal::from(1).into())]); +/// let solution = QuerySolution::from(( +/// vec![ +/// Variable::new_unchecked("foo"), +/// Variable::new_unchecked("bar"), +/// ], +/// vec![Some(Literal::from(1).into()), None], +/// )); +/// assert_eq!( +/// solution.iter().collect::>(), +/// vec![(&Variable::new_unchecked("foo"), &Literal::from(1).into())] +/// ); /// ``` pub struct Iter<'a> { inner: Zip, std::slice::Iter<'a, Option>>, diff --git a/lib/sparesults/src/xml.rs b/lib/sparesults/src/xml.rs index 3c2d91a8..c0450fac 100644 --- a/lib/sparesults/src/xml.rs +++ b/lib/sparesults/src/xml.rs @@ -2,7 +2,6 @@ use crate::error::{ParseError, SyntaxError}; use oxrdf::vocab::rdf; -use oxrdf::Variable; use oxrdf::*; use quick_xml::events::{BytesDecl, BytesEnd, BytesStart, BytesText, Event}; use quick_xml::{Reader, Writer}; @@ -245,7 +244,7 @@ impl XmlQueryResultsReader { let mut variables = Vec::default(); let mut state = State::Start; - //Read header + // Read header loop { buffer.clear(); let event = reader.read_event_into(&mut buffer)?; @@ -553,7 +552,7 @@ impl XmlSolutionsReader { } State::BNode => { if term.is_none() { - //We default to a random bnode + // We default to a random bnode term = Some(BlankNode::default().into()) } state = self @@ -563,7 +562,7 @@ impl XmlSolutionsReader { } State::Literal => { if term.is_none() { - //We default to the empty literal + // We default to the empty literal term = Some(build_literal("", lang.take(), datatype.take())?.into()) } state = self diff --git a/lib/spargebra/src/parser.rs b/lib/spargebra/src/parser.rs index 65a251e5..03a71932 100644 --- a/lib/spargebra/src/parser.rs +++ b/lib/spargebra/src/parser.rs @@ -365,7 +365,7 @@ enum PartialGraphPattern { } fn new_join(l: GraphPattern, r: GraphPattern) -> GraphPattern { - //Avoid to output empty BGPs + // Avoid to output empty BGPs if let GraphPattern::Bgp { patterns: pl } = &l { if pl.is_empty() { return r; @@ -449,7 +449,7 @@ fn build_select( let mut p = r#where; let mut with_aggregate = false; - //GROUP BY + // GROUP BY let aggregates = state.aggregates.pop().unwrap_or_default(); if group.is_none() && !aggregates.is_empty() { group = Some((vec![], vec![])); @@ -471,7 +471,7 @@ fn build_select( with_aggregate = true; } - //HAVING + // HAVING if let Some(expr) = having { p = GraphPattern::Filter { expr, @@ -479,12 +479,12 @@ fn build_select( }; } - //VALUES + // VALUES if let Some(data) = values { p = new_join(p, data); } - //SELECT + // SELECT let mut pv = Vec::new(); let with_project = match select.variables { SelectionVariables::Explicit(sel_items) => { @@ -533,7 +533,7 @@ fn build_select( if with_aggregate { return Err("SELECT * is not authorized with GROUP BY"); } - //TODO: is it really useful to do a projection? + // TODO: is it really useful to do a projection? p.on_in_scope_variable(|v| { if !pv.contains(v) { pv.push(v.clone()); @@ -547,7 +547,7 @@ fn build_select( let mut m = p; - //ORDER BY + // ORDER BY if let Some(expression) = order_by { m = GraphPattern::OrderBy { inner: Box::new(m), @@ -555,7 +555,7 @@ fn build_select( }; } - //PROJECT + // PROJECT if with_project { m = GraphPattern::Project { inner: Box::new(m), @@ -568,7 +568,7 @@ fn build_select( SelectionOption::Default => (), } - //OFFSET LIMIT + // OFFSET LIMIT if let Some((start, length)) = offset_limit { m = GraphPattern::Slice { inner: Box::new(m), diff --git a/lib/spargebra/src/query.rs b/lib/spargebra/src/query.rs index 8716de73..5739b7b8 100644 --- a/lib/spargebra/src/query.rs +++ b/lib/spargebra/src/query.rs @@ -13,7 +13,10 @@ use std::str::FromStr; /// let query_str = "SELECT ?s ?p ?o WHERE { ?s ?p ?o . }"; /// let query = Query::parse(query_str, None)?; /// assert_eq!(query.to_string(), query_str); -/// assert_eq!(query.to_sse(), "(project (?s ?p ?o) (bgp (triple ?s ?p ?o)))"); +/// assert_eq!( +/// query.to_sse(), +/// "(project (?s ?p ?o) (bgp (triple ?s ?p ?o)))" +/// ); /// # Ok::<_, spargebra::ParseError>(()) /// ``` #[derive(Eq, PartialEq, Debug, Clone, Hash)] diff --git a/lib/spargebra/src/term.rs b/lib/spargebra/src/term.rs index 362b3959..ba5fb8e3 100644 --- a/lib/spargebra/src/term.rs +++ b/lib/spargebra/src/term.rs @@ -141,7 +141,7 @@ impl TryFrom for GroundTerm { /// The default string formatter is returning a N-Quads representation. /// /// ``` -/// use spargebra::term::{NamedNode, GroundTriple}; +/// use spargebra::term::{GroundTriple, NamedNode}; /// /// assert_eq!( /// " ", @@ -149,7 +149,8 @@ impl TryFrom for GroundTerm { /// subject: NamedNode::new("http://example.com/s")?.into(), /// predicate: NamedNode::new("http://example.com/p")?, /// object: NamedNode::new("http://example.com/o")?.into(), -/// }.to_string() +/// } +/// .to_string() /// ); /// # Result::<_,oxrdf::IriParseError>::Ok(()) /// ``` diff --git a/lib/sparopt/src/algebra.rs b/lib/sparopt/src/algebra.rs index 51ecf6fa..b9bb30f7 100644 --- a/lib/sparopt/src/algebra.rs +++ b/lib/sparopt/src/algebra.rs @@ -197,10 +197,10 @@ impl Expression { xsd::BOOLEAN => match literal.value() { "true" | "1" => Some(true), "false" | "0" => Some(false), - _ => None, //TODO + _ => None, // TODO }, xsd::STRING => Some(!literal.value().is_empty()), - _ => None, //TODO + _ => None, // TODO } } else { None diff --git a/lib/sparopt/src/optimizer.rs b/lib/sparopt/src/optimizer.rs index 5dc9d404..facc5b0c 100644 --- a/lib/sparopt/src/optimizer.rs +++ b/lib/sparopt/src/optimizer.rs @@ -102,7 +102,7 @@ impl Optimizer { let expression = Self::normalize_expression(expression, &inner_types); let expression_type = infer_expression_type(&expression, &inner_types); if expression_type == VariableType::UNDEF { - //TODO: valid? + // TODO: valid? inner } else { GraphPattern::extend(inner, variable, expression) @@ -397,7 +397,7 @@ impl Optimizer { expression, variable, } => { - //TODO: handle the case where the filter overrides an expression variable (should not happen in SPARQL but allowed in the algebra) + // TODO: handle the case where the filter overrides an expression variable (should not happen in SPARQL but allowed in the algebra) let mut inner_filters = Vec::new(); let mut final_filters = Vec::new(); for filter in filters { @@ -735,7 +735,7 @@ fn is_fit_for_for_loop_join( global_input_types: &VariableTypes, entry_types: &VariableTypes, ) -> bool { - //TODO: think more about it + // TODO: think more about it match pattern { GraphPattern::Values { .. } | GraphPattern::QuadPattern { .. } diff --git a/lib/sparopt/src/type_inference.rs b/lib/sparopt/src/type_inference.rs index 161ba58a..d53b63e4 100644 --- a/lib/sparopt/src/type_inference.rs +++ b/lib/sparopt/src/type_inference.rs @@ -49,7 +49,7 @@ pub fn infer_graph_pattern_types( infer_graph_pattern_types(right, infer_graph_pattern_types(left, types)) } GraphPattern::LeftJoin { left, right, .. } => { - let mut right_types = infer_graph_pattern_types(right, types.clone()); //TODO: expression + let mut right_types = infer_graph_pattern_types(right, types.clone()); // TODO: expression for t in right_types.inner.values_mut() { t.undef = true; // Right might be unset } @@ -352,24 +352,14 @@ pub struct VariableType { } impl VariableType { - pub const UNDEF: Self = Self { + const ANY: Self = Self { undef: true, - named_node: false, - blank_node: false, - literal: false, - #[cfg(feature = "rdf-star")] - triple: false, - }; - - const NAMED_NODE: Self = Self { - undef: false, named_node: true, - blank_node: false, - literal: false, + blank_node: true, + literal: true, #[cfg(feature = "rdf-star")] - triple: false, + triple: true, }; - const BLANK_NODE: Self = Self { undef: false, named_node: false, @@ -378,7 +368,6 @@ impl VariableType { #[cfg(feature = "rdf-star")] triple: false, }; - const LITERAL: Self = Self { undef: false, named_node: false, @@ -387,16 +376,14 @@ impl VariableType { #[cfg(feature = "rdf-star")] triple: false, }; - - #[cfg(feature = "rdf-star")] - const TRIPLE: Self = Self { + const NAMED_NODE: Self = Self { undef: false, - named_node: false, + named_node: true, blank_node: false, literal: false, - triple: true, + #[cfg(feature = "rdf-star")] + triple: false, }; - const SUBJECT: Self = Self { undef: false, named_node: true, @@ -405,7 +392,6 @@ impl VariableType { #[cfg(feature = "rdf-star")] triple: true, }; - const TERM: Self = Self { undef: false, named_node: true, @@ -414,14 +400,21 @@ impl VariableType { #[cfg(feature = "rdf-star")] triple: true, }; - - const ANY: Self = Self { + #[cfg(feature = "rdf-star")] + const TRIPLE: Self = Self { + undef: false, + named_node: false, + blank_node: false, + literal: false, + triple: true, + }; + pub const UNDEF: Self = Self { undef: true, - named_node: true, - blank_node: true, - literal: true, + named_node: false, + blank_node: false, + literal: false, #[cfg(feature = "rdf-star")] - triple: true, + triple: false, }; } diff --git a/lib/sparql-smith/src/lib.rs b/lib/sparql-smith/src/lib.rs index 01ca45e5..3bc99c0b 100644 --- a/lib/sparql-smith/src/lib.rs +++ b/lib/sparql-smith/src/lib.rs @@ -44,7 +44,7 @@ struct QueryContent { #[derive(Arbitrary)] enum QueryVariant { Select(SelectQuery), - //TODO: Other variants! + // TODO: Other variants! } impl<'a> Arbitrary<'a> for Query { @@ -246,7 +246,7 @@ impl fmt::Display for GroupCondition { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::BuiltInCall(c) => write!(f, "{c}"), - //Self::FunctionCall(c) => write!(f, "{}", c), + // Self::FunctionCall(c) => write!(f, "{}", c), Self::Projection(e, v) => { if let Some(v) = v { write!(f, "({e} AS {v})") @@ -705,7 +705,7 @@ impl fmt::Display for Constraint { match self { Self::BrackettedExpression(e) => write!(f, "{e}"), Self::BuiltInCall(c) => write!(f, "{c}"), - //Self::FunctionCall(c) => write!(f, "{}", c), + // Self::FunctionCall(c) => write!(f, "{}", c), } } } @@ -1530,7 +1530,7 @@ enum BuiltInCall { IsLiteral(Box), IsNumeric(Box), Exists(ExistsFunc), - NotExists(NotExistsFunc), //TODO: Other functions + NotExists(NotExistsFunc), // TODO: Other functions } impl fmt::Display for BuiltInCall { @@ -1585,15 +1585,15 @@ impl fmt::Display for NotExistsFunc { struct IriOrFunction { // [128] iriOrFunction ::= iri ArgList? iri: Iri, - //TODO args: Option, + // TODO args: Option, } impl fmt::Display for IriOrFunction { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{}", self.iri)?; - /*if let Some(args) = &self.args { - write!(f, "{}", args)?; - }*/ + // if let Some(args) = &self.args { + // write!(f, "{}", args)?; + // } Ok(()) } } diff --git a/lib/src/io/format.rs b/lib/src/io/format.rs index b07c1709..08b61d8a 100644 --- a/lib/src/io/format.rs +++ b/lib/src/io/format.rs @@ -23,7 +23,10 @@ impl GraphFormat { /// ``` /// use oxigraph::io::GraphFormat; /// - /// assert_eq!(GraphFormat::NTriples.iri(), "http://www.w3.org/ns/formats/N-Triples") + /// assert_eq!( + /// GraphFormat::NTriples.iri(), + /// "http://www.w3.org/ns/formats/N-Triples" + /// ) /// ``` #[inline] pub fn iri(self) -> &'static str { @@ -65,6 +68,7 @@ impl GraphFormat { Self::RdfXml => "rdf", } } + /// Looks for a known format from a media type. /// /// It supports some media type aliases. @@ -74,7 +78,10 @@ impl GraphFormat { /// ``` /// use oxigraph::io::GraphFormat; /// - /// assert_eq!(GraphFormat::from_media_type("text/turtle; charset=utf-8"), Some(GraphFormat::Turtle)) + /// assert_eq!( + /// GraphFormat::from_media_type("text/turtle; charset=utf-8"), + /// Some(GraphFormat::Turtle) + /// ) /// ``` #[inline] pub fn from_media_type(media_type: &str) -> Option { @@ -94,7 +101,10 @@ impl GraphFormat { /// ``` /// use oxigraph::io::GraphFormat; /// - /// assert_eq!(GraphFormat::from_extension("nt"), Some(GraphFormat::NTriples)) + /// assert_eq!( + /// GraphFormat::from_extension("nt"), + /// Some(GraphFormat::NTriples) + /// ) /// ``` #[inline] pub fn from_extension(extension: &str) -> Option { @@ -151,7 +161,10 @@ impl DatasetFormat { /// ``` /// use oxigraph::io::DatasetFormat; /// - /// assert_eq!(DatasetFormat::NQuads.iri(), "http://www.w3.org/ns/formats/N-Quads") + /// assert_eq!( + /// DatasetFormat::NQuads.iri(), + /// "http://www.w3.org/ns/formats/N-Quads" + /// ) /// ``` #[inline] pub fn iri(self) -> &'static str { @@ -190,6 +203,7 @@ impl DatasetFormat { Self::TriG => "trig", } } + /// Looks for a known format from a media type. /// /// It supports some media type aliases. @@ -198,7 +212,10 @@ impl DatasetFormat { /// ``` /// use oxigraph::io::DatasetFormat; /// - /// assert_eq!(DatasetFormat::from_media_type("application/n-quads; charset=utf-8"), Some(DatasetFormat::NQuads)) + /// assert_eq!( + /// DatasetFormat::from_media_type("application/n-quads; charset=utf-8"), + /// Some(DatasetFormat::NQuads) + /// ) /// ``` #[inline] pub fn from_media_type(media_type: &str) -> Option { @@ -217,7 +234,10 @@ impl DatasetFormat { /// ``` /// use oxigraph::io::DatasetFormat; /// - /// assert_eq!(DatasetFormat::from_extension("nq"), Some(DatasetFormat::NQuads)) + /// assert_eq!( + /// DatasetFormat::from_extension("nq"), + /// Some(DatasetFormat::NQuads) + /// ) /// ``` #[inline] pub fn from_extension(extension: &str) -> Option { diff --git a/lib/src/io/read.rs b/lib/src/io/read.rs index 3400b8e2..841b166a 100644 --- a/lib/src/io/read.rs +++ b/lib/src/io/read.rs @@ -21,7 +21,9 @@ use std::io::Read; /// let file = " ."; /// /// let parser = GraphParser::from_format(GraphFormat::NTriples); -/// let triples = parser.read_triples(file.as_bytes()).collect::,_>>()?; +/// let triples = parser +/// .read_triples(file.as_bytes()) +/// .collect::, _>>()?; /// /// assert_eq!(triples.len(), 1); /// assert_eq!(triples[0].subject.to_string(), ""); @@ -50,8 +52,11 @@ impl GraphParser { /// /// let file = "

."; /// - /// let parser = GraphParser::from_format(GraphFormat::Turtle).with_base_iri("http://example.com")?; - /// let triples = parser.read_triples(file.as_bytes()).collect::,_>>()?; + /// let parser = + /// GraphParser::from_format(GraphFormat::Turtle).with_base_iri("http://example.com")?; + /// let triples = parser + /// .read_triples(file.as_bytes()) + /// .collect::, _>>()?; /// /// assert_eq!(triples.len(), 1); /// assert_eq!(triples[0].subject.to_string(), ""); @@ -81,7 +86,9 @@ impl GraphParser { /// let file = " ."; /// /// let parser = GraphParser::from_format(GraphFormat::NTriples); -/// let triples = parser.read_triples(file.as_bytes()).collect::,_>>()?; +/// let triples = parser +/// .read_triples(file.as_bytes()) +/// .collect::, _>>()?; /// /// assert_eq!(triples.len(), 1); /// assert_eq!(triples[0].subject.to_string(), ""); @@ -139,8 +146,11 @@ impl DatasetParser { /// /// let file = " {

}"; /// - /// let parser = DatasetParser::from_format(DatasetFormat::TriG).with_base_iri("http://example.com")?; - /// let triples = parser.read_quads(file.as_bytes()).collect::,_>>()?; + /// let parser = + /// DatasetParser::from_format(DatasetFormat::TriG).with_base_iri("http://example.com")?; + /// let triples = parser + /// .read_quads(file.as_bytes()) + /// .collect::, _>>()?; /// /// assert_eq!(triples.len(), 1); /// assert_eq!(triples[0].subject.to_string(), ""); diff --git a/lib/src/io/write.rs b/lib/src/io/write.rs index 7a9007c0..7f27cd9f 100644 --- a/lib/src/io/write.rs +++ b/lib/src/io/write.rs @@ -21,13 +21,16 @@ use std::io::{self, Write}; /// let mut buffer = Vec::new(); /// let mut writer = GraphSerializer::from_format(GraphFormat::NTriples).triple_writer(&mut buffer); /// writer.write(&Triple { -/// subject: NamedNode::new("http://example.com/s")?.into(), -/// predicate: NamedNode::new("http://example.com/p")?, -/// object: NamedNode::new("http://example.com/o")?.into() +/// subject: NamedNode::new("http://example.com/s")?.into(), +/// predicate: NamedNode::new("http://example.com/p")?, +/// object: NamedNode::new("http://example.com/o")?.into(), /// })?; /// writer.finish()?; /// -/// assert_eq!(buffer.as_slice(), " .\n".as_bytes()); +/// assert_eq!( +/// buffer.as_slice(), +/// " .\n".as_bytes() +/// ); /// # Result::<_,Box>::Ok(()) /// ``` #[deprecated(note = "use RdfSerializer instead", since = "0.4.0")] @@ -66,13 +69,16 @@ impl GraphSerializer { /// let mut buffer = Vec::new(); /// let mut writer = GraphSerializer::from_format(GraphFormat::NTriples).triple_writer(&mut buffer); /// writer.write(&Triple { -/// subject: NamedNode::new("http://example.com/s")?.into(), -/// predicate: NamedNode::new("http://example.com/p")?, -/// object: NamedNode::new("http://example.com/o")?.into() +/// subject: NamedNode::new("http://example.com/s")?.into(), +/// predicate: NamedNode::new("http://example.com/p")?, +/// object: NamedNode::new("http://example.com/o")?.into(), /// })?; /// writer.finish()?; /// -/// assert_eq!(buffer.as_slice(), " .\n".as_bytes()); +/// assert_eq!( +/// buffer.as_slice(), +/// " .\n".as_bytes() +/// ); /// # Result::<_,Box>::Ok(()) /// ``` #[must_use] diff --git a/lib/src/sparql/algebra.rs b/lib/src/sparql/algebra.rs index b046de80..819a9bd9 100644 --- a/lib/src/sparql/algebra.rs +++ b/lib/src/sparql/algebra.rs @@ -23,7 +23,10 @@ use std::str::FromStr; /// // We edit the query dataset specification /// let default = vec![NamedNode::new("http://example.com")?.into()]; /// query.dataset_mut().set_default_graph(default.clone()); -/// assert_eq!(query.dataset().default_graph_graphs(), Some(default.as_slice())); +/// assert_eq!( +/// query.dataset().default_graph_graphs(), +/// Some(default.as_slice()) +/// ); /// # Ok::<_, Box>(()) /// ``` #[derive(Eq, PartialEq, Debug, Clone, Hash)] @@ -58,7 +61,7 @@ impl Query { impl fmt::Display for Query { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.inner.fmt(f) //TODO: override + self.inner.fmt(f) // TODO: override } } @@ -217,8 +220,15 @@ impl QueryDataset { /// ``` /// use oxigraph::sparql::Query; /// - /// assert!(Query::parse("SELECT ?s ?p ?o WHERE { ?s ?p ?o . }", None)?.dataset().is_default_dataset()); - /// assert!(!Query::parse("SELECT ?s ?p ?o FROM WHERE { ?s ?p ?o . }", None)?.dataset().is_default_dataset()); + /// assert!(Query::parse("SELECT ?s ?p ?o WHERE { ?s ?p ?o . }", None)? + /// .dataset() + /// .is_default_dataset()); + /// assert!(!Query::parse( + /// "SELECT ?s ?p ?o FROM WHERE { ?s ?p ?o . }", + /// None + /// )? + /// .dataset() + /// .is_default_dataset()); /// /// # Ok::<_, Box>(()) /// ``` @@ -250,7 +260,10 @@ impl QueryDataset { /// let mut query = Query::parse("SELECT ?s ?p ?o WHERE { ?s ?p ?o . }", None)?; /// let default = vec![NamedNode::new("http://example.com")?.into()]; /// query.dataset_mut().set_default_graph(default.clone()); - /// assert_eq!(query.dataset().default_graph_graphs(), Some(default.as_slice())); + /// assert_eq!( + /// query.dataset().default_graph_graphs(), + /// Some(default.as_slice()) + /// ); /// /// # Ok::<_, Box>(()) /// ``` @@ -271,8 +284,13 @@ impl QueryDataset { /// /// let mut query = Query::parse("SELECT ?s ?p ?o WHERE { ?s ?p ?o . }", None)?; /// let named = vec![NamedNode::new("http://example.com")?.into()]; - /// query.dataset_mut().set_available_named_graphs(named.clone()); - /// assert_eq!(query.dataset().available_named_graphs(), Some(named.as_slice())); + /// query + /// .dataset_mut() + /// .set_available_named_graphs(named.clone()); + /// assert_eq!( + /// query.dataset().available_named_graphs(), + /// Some(named.as_slice()) + /// ); /// /// # Ok::<_, Box>(()) /// ``` diff --git a/lib/src/sparql/error.rs b/lib/src/sparql/error.rs index 4728efb7..43234d67 100644 --- a/lib/src/sparql/error.rs +++ b/lib/src/sparql/error.rs @@ -5,8 +5,7 @@ use crate::sparql::ParseError; use crate::storage::StorageError; use std::convert::Infallible; use std::error::Error; -use std::fmt; -use std::io; +use std::{fmt, io}; /// A SPARQL evaluation error. #[derive(Debug)] diff --git a/lib/src/sparql/eval.rs b/lib/src/sparql/eval.rs index 25c4b3cf..19c6884d 100644 --- a/lib/src/sparql/eval.rs +++ b/lib/src/sparql/eval.rs @@ -33,8 +33,7 @@ use std::cmp::Ordering; use std::collections::hash_map::DefaultHasher; use std::collections::{HashMap, HashSet}; use std::hash::{Hash, Hasher}; -use std::iter::Iterator; -use std::iter::{empty, once}; +use std::iter::{empty, once, Iterator}; use std::rc::Rc; use std::sync::Arc; use std::{fmt, io, str}; @@ -112,8 +111,8 @@ impl EncodedTuple { } impl IntoIterator for EncodedTuple { - type Item = Option; type IntoIter = std::vec::IntoIter>; + type Item = Option; fn into_iter(self) -> Self::IntoIter { self.inner.into_iter() @@ -1010,7 +1009,7 @@ impl SimpleEvaluator { } }) .for_each(|tuple| { - //TODO avoid copy for key? + // TODO avoid copy for key? let key = key_variables .iter() .map(|v| tuple.get(*v).cloned()) @@ -3127,7 +3126,7 @@ pub(super) fn compile_pattern(pattern: &str, flags: Option<&str>) -> Option { regex_builder.ignore_whitespace(true); } - _ => (), //TODO: implement q + _ => (), // TODO: implement q } } } @@ -3632,7 +3631,7 @@ fn compare_str_str_id(dataset: &DatasetView, a: &str, b: &StrHash) -> Option Option { - //TODO: optimize? + // TODO: optimize? match value { EncodedTerm::NamedNode { .. } | EncodedTerm::SmallBlankNode { .. } @@ -4387,6 +4386,7 @@ impl PathEvaluator { } } } + fn eval_to_in_unknown_graph( &self, path: &PropertyPath, @@ -4968,7 +4968,7 @@ impl Iterator for ConstructIterator { )); } } - self.bnodes.clear(); //We do not reuse old bnodes + self.bnodes.clear(); // We do not reuse old bnodes } } } @@ -5300,7 +5300,7 @@ impl Accumulator for SumAccumulator { if let Some(sum) = &self.sum { if let Some(operands) = element.and_then(|e| NumericBinaryOperands::new(sum.clone(), e)) { - //TODO: unify with addition? + // TODO: unify with addition? self.sum = match operands { NumericBinaryOperands::Float(v1, v2) => Some((v1 + v2).into()), NumericBinaryOperands::Double(v1, v2) => Some((v1 + v2).into()), @@ -5343,8 +5343,8 @@ impl Accumulator for AvgAccumulator { if self.count == 0 { Some(0.into()) } else { - //TODO: deduplicate? - //TODO: duration? + // TODO: deduplicate? + // TODO: duration? let count = Integer::from(self.count); match sum { EncodedTerm::FloatLiteral(sum) => Some((sum / Float::from(count)).into()), @@ -5584,6 +5584,7 @@ impl EncodedTupleSet { len: 0, } } + fn insert(&mut self, tuple: EncodedTuple) { self.map .entry(self.tuple_key(&tuple)) diff --git a/lib/src/sparql/mod.rs b/lib/src/sparql/mod.rs index f48ea908..ec84467f 100644 --- a/lib/src/sparql/mod.rs +++ b/lib/src/sparql/mod.rs @@ -144,13 +144,13 @@ pub(crate) fn evaluate_query( /// /// Usage example disabling the federated query support: /// ``` -/// use oxigraph::store::Store; /// use oxigraph::sparql::QueryOptions; +/// use oxigraph::store::Store; /// /// let store = Store::new()?; /// store.query_opt( /// "SELECT * WHERE { SERVICE {} }", -/// QueryOptions::default().without_service_handler() +/// QueryOptions::default().without_service_handler(), /// )?; /// # Result::<_,Box>::Ok(()) /// ``` @@ -209,9 +209,9 @@ impl QueryOptions { /// /// Example with a function serializing terms to N-Triples: /// ``` - /// use oxigraph::store::Store; /// use oxigraph::model::*; /// use oxigraph::sparql::{QueryOptions, QueryResults}; + /// use oxigraph::store::Store; /// /// let store = Store::new()?; /// @@ -219,10 +219,13 @@ impl QueryOptions { /// "SELECT ((1) AS ?nt) WHERE {}", /// QueryOptions::default().with_custom_function( /// NamedNode::new("http://www.w3.org/ns/formats/N-Triples")?, - /// |args| args.get(0).map(|t| Literal::from(t.to_string()).into()) - /// ) + /// |args| args.get(0).map(|t| Literal::from(t.to_string()).into()), + /// ), /// )? { - /// assert_eq!(solutions.next().unwrap()?.get("nt"), Some(&Literal::from("\"1\"^^").into())); + /// assert_eq!( + /// solutions.next().unwrap()?.get("nt"), + /// Some(&Literal::from("\"1\"^^").into()) + /// ); /// } /// # Result::<_,Box>::Ok(()) /// ``` diff --git a/lib/src/sparql/model.rs b/lib/src/sparql/model.rs index ca42db16..59c38afd 100644 --- a/lib/src/sparql/model.rs +++ b/lib/src/sparql/model.rs @@ -104,17 +104,24 @@ impl QueryResults { /// This method fails if it is called on the `Solution` or `Boolean` results. /// /// ``` - /// use oxigraph::store::Store; /// use oxigraph::io::RdfFormat; /// use oxigraph::model::*; + /// use oxigraph::store::Store; /// /// let graph = " .\n"; /// /// let store = Store::new()?; - /// store.load_graph(graph.as_bytes(), RdfFormat::NTriples, GraphName::DefaultGraph, None)?; + /// store.load_graph( + /// graph.as_bytes(), + /// RdfFormat::NTriples, + /// GraphName::DefaultGraph, + /// None, + /// )?; /// /// let mut results = Vec::new(); - /// store.query("CONSTRUCT WHERE { ?s ?p ?o }")?.write_graph(&mut results, RdfFormat::NTriples)?; + /// store + /// .query("CONSTRUCT WHERE { ?s ?p ?o }")? + /// .write_graph(&mut results, RdfFormat::NTriples)?; /// assert_eq!(results, graph.as_bytes()); /// # Result::<_,Box>::Ok(()) /// ``` @@ -159,8 +166,8 @@ impl From> for QueryResults { /// An iterator over [`QuerySolution`]s. /// /// ``` -/// use oxigraph::store::Store; /// use oxigraph::sparql::QueryResults; +/// use oxigraph::store::Store; /// /// let store = Store::new()?; /// if let QueryResults::Solutions(solutions) = store.query("SELECT ?s WHERE { ?s ?p ?o }")? { @@ -193,12 +200,15 @@ impl QuerySolutionIter { /// The variables used in the solutions. /// /// ``` - /// use oxigraph::store::Store; /// use oxigraph::sparql::{QueryResults, Variable}; + /// use oxigraph::store::Store; /// /// let store = Store::new()?; /// if let QueryResults::Solutions(solutions) = store.query("SELECT ?s ?o WHERE { ?s ?p ?o }")? { - /// assert_eq!(solutions.variables(), &[Variable::new("s")?, Variable::new("o")?]); + /// assert_eq!( + /// solutions.variables(), + /// &[Variable::new("s")?, Variable::new("o")?] + /// ); /// } /// # Result::<_,Box>::Ok(()) /// ``` @@ -234,8 +244,8 @@ impl Iterator for QuerySolutionIter { /// An iterator over the triples that compose a graph solution. /// /// ``` -/// use oxigraph::store::Store; /// use oxigraph::sparql::QueryResults; +/// use oxigraph::store::Store; /// /// let store = Store::new()?; /// if let QueryResults::Graph(triples) = store.query("CONSTRUCT WHERE { ?s ?p ?o }")? { diff --git a/lib/src/sparql/service.rs b/lib/src/sparql/service.rs index 562c1896..e3dd5602 100644 --- a/lib/src/sparql/service.rs +++ b/lib/src/sparql/service.rs @@ -13,18 +13,22 @@ use std::time::Duration; /// before evaluating a SPARQL query that uses SERVICE calls. /// /// ``` -/// use oxigraph::store::Store; /// use oxigraph::model::*; -/// use oxigraph::sparql::{QueryOptions, QueryResults, ServiceHandler, Query, EvaluationError}; +/// use oxigraph::sparql::{EvaluationError, Query, QueryOptions, QueryResults, ServiceHandler}; +/// use oxigraph::store::Store; /// /// struct TestServiceHandler { -/// store: Store +/// store: Store, /// } /// /// impl ServiceHandler for TestServiceHandler { /// type Error = EvaluationError; /// -/// fn handle(&self, service_name: NamedNode, query: Query) -> Result { +/// fn handle( +/// &self, +/// service_name: NamedNode, +/// query: Query, +/// ) -> Result { /// if service_name == "http://example.com/service" { /// self.store.query(query) /// } else { @@ -35,14 +39,16 @@ use std::time::Duration; /// /// let store = Store::new()?; /// let service = TestServiceHandler { -/// store: Store::new()? +/// store: Store::new()?, /// }; /// let ex = NamedNodeRef::new("http://example.com")?; -/// service.store.insert(QuadRef::new(ex, ex, ex, GraphNameRef::DefaultGraph))?; +/// service +/// .store +/// .insert(QuadRef::new(ex, ex, ex, GraphNameRef::DefaultGraph))?; /// /// if let QueryResults::Solutions(mut solutions) = store.query_opt( /// "SELECT ?s WHERE { SERVICE { ?s ?p ?o } }", -/// QueryOptions::default().with_service_handler(service) +/// QueryOptions::default().with_service_handler(service), /// )? { /// assert_eq!(solutions.next().unwrap()?.get("s"), Some(&ex.into())); /// } diff --git a/lib/src/sparql/update.rs b/lib/src/sparql/update.rs index 2e318c71..967de822 100644 --- a/lib/src/sparql/update.rs +++ b/lib/src/sparql/update.rs @@ -139,7 +139,7 @@ impl<'a, 'b: 'a> SimpleUpdateEvaluator<'a, 'b> { let mut bnodes = HashMap::new(); let (eval, _) = evaluator.graph_pattern_evaluator(&pattern, &mut variables); let tuples = - eval(EncodedTuple::with_capacity(variables.len())).collect::, _>>()?; //TODO: would be much better to stream + eval(EncodedTuple::with_capacity(variables.len())).collect::, _>>()?; // TODO: would be much better to stream for tuple in tuples { for quad in delete { if let Some(quad) = diff --git a/lib/src/storage/backend/rocksdb.rs b/lib/src/storage/backend/rocksdb.rs index 670963ba..fed85421 100644 --- a/lib/src/storage/backend/rocksdb.rs +++ b/lib/src/storage/backend/rocksdb.rs @@ -19,16 +19,14 @@ use std::collections::HashMap; use std::env::temp_dir; use std::error::Error; use std::ffi::{CStr, CString}; -use std::fmt; use std::fs::remove_dir_all; -use std::io; use std::marker::PhantomData; use std::ops::Deref; use std::path::{Path, PathBuf}; use std::rc::{Rc, Weak}; use std::sync::{Arc, OnceLock}; use std::thread::{available_parallelism, yield_now}; -use std::{ptr, slice}; +use std::{fmt, io, ptr, slice}; macro_rules! ffi_result { ( $($function:ident)::*( $arg1:expr $(, $arg:expr)* $(,)? ) ) => {{ @@ -711,7 +709,7 @@ impl Db { column_family: &ColumnFamily, key: &[u8], ) -> Result { - Ok(self.get(column_family, key)?.is_some()) //TODO: optimize + Ok(self.get(column_family, key)?.is_some()) // TODO: optimize } pub fn insert( @@ -970,7 +968,7 @@ impl Reader { column_family: &ColumnFamily, key: &[u8], ) -> Result { - Ok(self.get(column_family, key)?.is_some()) //TODO: optimize + Ok(self.get(column_family, key)?.is_some()) // TODO: optimize } #[allow(clippy::iter_not_returning_iterator)] @@ -983,7 +981,7 @@ impl Reader { column_family: &ColumnFamily, prefix: &[u8], ) -> Result { - //We generate the upper bound + // We generate the upper bound let upper_bound = { let mut bound = prefix.to_vec(); let mut found = false; @@ -1101,7 +1099,7 @@ impl Transaction<'_> { column_family: &ColumnFamily, key: &[u8], ) -> Result { - Ok(self.get_for_update(column_family, key)?.is_some()) //TODO: optimize + Ok(self.get_for_update(column_family, key)?.is_some()) // TODO: optimize } pub fn insert( @@ -1228,7 +1226,7 @@ pub struct Iter { is_currently_valid: bool, _upper_bound: Option>, _reader: Reader, // needed to ensure that DB still lives while iter is used - options: *mut rocksdb_readoptions_t, // needed to ensure that options still lives while iter is used + options: *mut rocksdb_readoptions_t, /* needed to ensure that options still lives while iter is used */ } impl Drop for Iter { diff --git a/lib/src/storage/error.rs b/lib/src/storage/error.rs index 89895349..05076e6e 100644 --- a/lib/src/storage/error.rs +++ b/lib/src/storage/error.rs @@ -1,8 +1,7 @@ use crate::io::{ParseError, RdfFormat}; use oxiri::IriParseError; use std::error::Error; -use std::fmt; -use std::io; +use std::{fmt, io}; /// An error related to storage operations (reads, writes...). #[derive(Debug)] diff --git a/lib/src/storage/mod.rs b/lib/src/storage/mod.rs index 9dd38ee1..ce02449e 100644 --- a/lib/src/storage/mod.rs +++ b/lib/src/storage/mod.rs @@ -579,7 +579,7 @@ impl StorageReader { pub fn named_graphs(&self) -> DecodingGraphIterator { DecodingGraphIterator { - iter: self.reader.iter(&self.storage.graphs_cf).unwrap(), //TODO: propagate error? + iter: self.reader.iter(&self.storage.graphs_cf).unwrap(), // TODO: propagate error? } } @@ -786,7 +786,7 @@ impl StorageReader { #[cfg(target_family = "wasm")] #[allow(clippy::unused_self, clippy::unnecessary_wraps)] pub fn validate(&self) -> Result<(), StorageError> { - Ok(()) //TODO + Ok(()) // TODO } } diff --git a/lib/src/storage/numeric_encoder.rs b/lib/src/storage/numeric_encoder.rs index fd0d7544..a81b76e5 100644 --- a/lib/src/storage/numeric_encoder.rs +++ b/lib/src/storage/numeric_encoder.rs @@ -6,8 +6,7 @@ use crate::storage::small_string::SmallString; use oxsdatatypes::*; use siphasher::sip128::{Hasher128, SipHasher24}; use std::fmt::Debug; -use std::hash::Hash; -use std::hash::Hasher; +use std::hash::{Hash, Hasher}; use std::str; use std::sync::Arc; diff --git a/lib/src/storage/small_string.rs b/lib/src/storage/small_string.rs index c2862ff4..fcd9b227 100644 --- a/lib/src/storage/small_string.rs +++ b/lib/src/storage/small_string.rs @@ -1,11 +1,10 @@ use std::borrow::Borrow; use std::cmp::Ordering; use std::error::Error; -use std::fmt; use std::hash::{Hash, Hasher}; use std::ops::Deref; -use std::str; use std::str::{FromStr, Utf8Error}; +use std::{fmt, str}; /// A small inline string #[derive(Clone, Copy, Default)] diff --git a/lib/src/store.rs b/lib/src/store.rs index 95a25260..5b5a1640 100644 --- a/lib/src/store.rs +++ b/lib/src/store.rs @@ -4,9 +4,9 @@ //! //! Usage example: //! ``` -//! use oxigraph::store::Store; -//! use oxigraph::sparql::QueryResults; //! use oxigraph::model::*; +//! use oxigraph::sparql::QueryResults; +//! use oxigraph::store::Store; //! //! let store = Store::new()?; //! @@ -16,7 +16,7 @@ //! store.insert(&quad)?; //! //! // quad filter -//! let results: Result,_> = store.quads_for_pattern(None, None, None, None).collect(); +//! let results: Result, _> = store.quads_for_pattern(None, None, None, None).collect(); //! assert_eq!(vec![quad], results?); //! //! // SPARQL query @@ -56,9 +56,9 @@ use std::{fmt, str}; /// /// Usage example: /// ``` -/// use oxigraph::store::Store; -/// use oxigraph::sparql::QueryResults; /// use oxigraph::model::*; +/// use oxigraph::sparql::QueryResults; +/// use oxigraph::store::Store; /// # use std::fs::remove_dir_all; /// /// # { @@ -70,7 +70,7 @@ use std::{fmt, str}; /// store.insert(&quad)?; /// /// // quad filter -/// let results: Result,_> = store.quads_for_pattern(None, None, None, None).collect(); +/// let results: Result, _> = store.quads_for_pattern(None, None, None, None).collect(); /// assert_eq!(vec![quad], results?); /// /// // SPARQL query @@ -160,9 +160,9 @@ impl Store { /// /// Usage example: /// ``` - /// use oxigraph::store::Store; /// use oxigraph::model::*; /// use oxigraph::sparql::QueryResults; + /// use oxigraph::store::Store; /// /// let store = Store::new()?; /// @@ -171,8 +171,11 @@ impl Store { /// store.insert(QuadRef::new(ex, ex, ex, GraphNameRef::DefaultGraph))?; /// /// // SPARQL query - /// if let QueryResults::Solutions(mut solutions) = store.query("SELECT ?s WHERE { ?s ?p ?o }")? { - /// assert_eq!(solutions.next().unwrap()?.get("s"), Some(&ex.into_owned().into())); + /// if let QueryResults::Solutions(mut solutions) = store.query("SELECT ?s WHERE { ?s ?p ?o }")? { + /// assert_eq!( + /// solutions.next().unwrap()?.get("s"), + /// Some(&ex.into_owned().into()) + /// ); /// } /// # Result::<_, Box>::Ok(()) /// ``` @@ -187,19 +190,22 @@ impl Store { /// /// Usage example with a custom function serializing terms to N-Triples: /// ``` - /// use oxigraph::store::Store; /// use oxigraph::model::*; /// use oxigraph::sparql::{QueryOptions, QueryResults}; + /// use oxigraph::store::Store; /// /// let store = Store::new()?; /// if let QueryResults::Solutions(mut solutions) = store.query_opt( /// "SELECT ((1) AS ?nt) WHERE {}", /// QueryOptions::default().with_custom_function( /// NamedNode::new("http://www.w3.org/ns/formats/N-Triples")?, - /// |args| args.get(0).map(|t| Literal::from(t.to_string()).into()) - /// ) + /// |args| args.get(0).map(|t| Literal::from(t.to_string()).into()), + /// ), /// )? { - /// assert_eq!(solutions.next().unwrap()?.get("nt"), Some(&Literal::from("\"1\"^^").into())); + /// assert_eq!( + /// solutions.next().unwrap()?.get("nt"), + /// Some(&Literal::from("\"1\"^^").into()) + /// ); /// } /// # Result::<_, Box>::Ok(()) /// ``` @@ -219,14 +225,17 @@ impl Store { /// /// Usage example serialising the explanation with statistics in JSON: /// ``` - /// use oxigraph::store::Store; /// use oxigraph::sparql::{QueryOptions, QueryResults}; + /// use oxigraph::store::Store; /// /// let store = Store::new()?; - /// if let (Ok(QueryResults::Solutions(solutions)), explanation) = store.explain_query_opt("SELECT ?s WHERE { VALUES ?s { 1 2 3 } }", QueryOptions::default(), true)? { + /// if let (Ok(QueryResults::Solutions(solutions)), explanation) = store.explain_query_opt( + /// "SELECT ?s WHERE { VALUES ?s { 1 2 3 } }", + /// QueryOptions::default(), + /// true, + /// )? { /// // We make sure to have read all the solutions - /// for _ in solutions { - /// } + /// for _ in solutions {} /// let mut buf = Vec::new(); /// explanation.write_in_json(&mut buf)?; /// } @@ -245,8 +254,8 @@ impl Store { /// /// Usage example: /// ``` - /// use oxigraph::store::Store; /// use oxigraph::model::*; + /// use oxigraph::store::Store; /// /// let store = Store::new()?; /// @@ -256,7 +265,9 @@ impl Store { /// store.insert(&quad)?; /// /// // quad filter by object - /// let results = store.quads_for_pattern(None, None, Some((&ex).into()), None).collect::,_>>()?; + /// let results = store + /// .quads_for_pattern(None, None, Some((&ex).into()), None) + /// .collect::, _>>()?; /// assert_eq!(vec![quad], results); /// # Result::<_, Box>::Ok(()) /// ``` @@ -283,8 +294,8 @@ impl Store { /// /// Usage example: /// ``` - /// use oxigraph::store::Store; /// use oxigraph::model::*; + /// use oxigraph::store::Store; /// /// let store = Store::new()?; /// @@ -294,7 +305,7 @@ impl Store { /// store.insert(&quad)?; /// /// // quad filter by object - /// let results = store.iter().collect::,_>>()?; + /// let results = store.iter().collect::, _>>()?; /// assert_eq!(vec![quad], results); /// # Result::<_, Box>::Ok(()) /// ``` @@ -306,8 +317,8 @@ impl Store { /// /// Usage example: /// ``` - /// use oxigraph::store::Store; /// use oxigraph::model::*; + /// use oxigraph::store::Store; /// /// let ex = NamedNodeRef::new("http://example.com")?; /// let quad = QuadRef::new(ex, ex, ex, ex); @@ -330,13 +341,13 @@ impl Store { /// /// Usage example: /// ``` - /// use oxigraph::store::Store; /// use oxigraph::model::*; + /// use oxigraph::store::Store; /// /// let ex = NamedNodeRef::new("http://example.com")?; /// let store = Store::new()?; /// store.insert(QuadRef::new(ex, ex, ex, ex))?; - /// store.insert(QuadRef::new(ex, ex, ex, GraphNameRef::DefaultGraph))?; + /// store.insert(QuadRef::new(ex, ex, ex, GraphNameRef::DefaultGraph))?; /// assert_eq!(2, store.len()?); /// # Result::<_, Box>::Ok(()) /// ``` @@ -348,8 +359,8 @@ impl Store { /// /// Usage example: /// ``` - /// use oxigraph::store::Store; /// use oxigraph::model::*; + /// use oxigraph::store::Store; /// /// let store = Store::new()?; /// assert!(store.is_empty()?); @@ -371,8 +382,8 @@ impl Store { /// /// Usage example: /// ``` - /// use oxigraph::store::{StorageError, Store}; /// use oxigraph::model::*; + /// use oxigraph::store::{StorageError, Store}; /// /// let store = Store::new()?; /// let a = NamedNodeRef::new("http://example.com/a")?; @@ -399,13 +410,14 @@ impl Store { /// /// Usage example: /// ``` - /// use oxigraph::store::Store; /// use oxigraph::model::*; + /// use oxigraph::store::Store; /// /// let store = Store::new()?; /// /// // insertion - /// store.update("INSERT DATA { }")?; + /// store + /// .update("INSERT DATA { }")?; /// /// // we inspect the store contents /// let ex = NamedNodeRef::new("http://example.com")?; @@ -504,15 +516,20 @@ impl Store { /// /// Usage example: /// ``` - /// use oxigraph::store::Store; /// use oxigraph::io::RdfFormat; /// use oxigraph::model::*; + /// use oxigraph::store::Store; /// /// let store = Store::new()?; /// /// // insertion /// let file = b" ."; - /// store.load_graph(file.as_ref(), RdfFormat::NTriples, GraphName::DefaultGraph, None)?; + /// store.load_graph( + /// file.as_ref(), + /// RdfFormat::NTriples, + /// GraphName::DefaultGraph, + /// None, + /// )?; /// /// // we inspect the store contents /// let ex = NamedNodeRef::new("http://example.com")?; @@ -547,14 +564,15 @@ impl Store { /// /// Usage example: /// ``` - /// use oxigraph::store::Store; /// use oxigraph::io::RdfFormat; /// use oxigraph::model::*; + /// use oxigraph::store::Store; /// /// let store = Store::new()?; /// /// // insertion - /// let file = b" ."; + /// let file = + /// b" ."; /// store.load_dataset(file.as_ref(), RdfFormat::NQuads, None)?; /// /// // we inspect the store contents @@ -587,8 +605,8 @@ impl Store { /// /// Usage example: /// ``` - /// use oxigraph::store::Store; /// use oxigraph::model::*; + /// use oxigraph::store::Store; /// /// let ex = NamedNodeRef::new("http://example.com")?; /// let quad = QuadRef::new(ex, ex, ex, GraphNameRef::DefaultGraph); @@ -624,8 +642,8 @@ impl Store { /// /// Usage example: /// ``` - /// use oxigraph::store::Store; /// use oxigraph::model::*; + /// use oxigraph::store::Store; /// /// let ex = NamedNodeRef::new("http://example.com")?; /// let quad = QuadRef::new(ex, ex, ex, GraphNameRef::DefaultGraph); @@ -646,10 +664,12 @@ impl Store { /// Dumps the store into a file. /// /// ``` - /// use oxigraph::store::Store; /// use oxigraph::io::RdfFormat; + /// use oxigraph::store::Store; /// - /// let file = " .\n".as_bytes(); + /// let file = + /// " .\n" + /// .as_bytes(); /// /// let store = Store::new()?; /// store.load_from_read(RdfFormat::NQuads, file)?; @@ -678,9 +698,9 @@ impl Store { /// /// Usage example: /// ``` - /// use oxigraph::store::Store; /// use oxigraph::io::RdfFormat; /// use oxigraph::model::*; + /// use oxigraph::store::Store; /// /// let file = " .\n".as_bytes(); /// @@ -709,9 +729,9 @@ impl Store { /// /// Usage example: /// ``` - /// use oxigraph::store::Store; /// use oxigraph::io::RdfFormat; /// use oxigraph::model::*; + /// use oxigraph::store::Store; /// /// let file = " .\n".as_bytes(); /// @@ -736,10 +756,12 @@ impl Store { /// Dumps the store into a file. /// /// ``` - /// use oxigraph::store::Store; /// use oxigraph::io::RdfFormat; + /// use oxigraph::store::Store; /// - /// let file = " .\n".as_bytes(); + /// let file = + /// " .\n" + /// .as_bytes(); /// /// let store = Store::new()?; /// store.load_from_read(RdfFormat::NQuads, file)?; @@ -761,14 +783,17 @@ impl Store { /// /// Usage example: /// ``` - /// use oxigraph::store::Store; /// use oxigraph::model::*; + /// use oxigraph::store::Store; /// /// let ex = NamedNode::new("http://example.com")?; /// let store = Store::new()?; /// store.insert(QuadRef::new(&ex, &ex, &ex, &ex))?; /// store.insert(QuadRef::new(&ex, &ex, &ex, GraphNameRef::DefaultGraph))?; - /// assert_eq!(vec![NamedOrBlankNode::from(ex)], store.named_graphs().collect::,_>>()?); + /// assert_eq!( + /// vec![NamedOrBlankNode::from(ex)], + /// store.named_graphs().collect::, _>>()? + /// ); /// # Result::<_, Box>::Ok(()) /// ``` pub fn named_graphs(&self) -> GraphNameIter { @@ -783,8 +808,8 @@ impl Store { /// /// Usage example: /// ``` - /// use oxigraph::store::Store; /// use oxigraph::model::{NamedNode, QuadRef}; + /// use oxigraph::store::Store; /// /// let ex = NamedNode::new("http://example.com")?; /// let store = Store::new()?; @@ -806,14 +831,17 @@ impl Store { /// /// Usage example: /// ``` - /// use oxigraph::store::Store; /// use oxigraph::model::NamedNodeRef; + /// use oxigraph::store::Store; /// /// let ex = NamedNodeRef::new("http://example.com")?; /// let store = Store::new()?; /// store.insert_named_graph(ex)?; /// - /// assert_eq!(store.named_graphs().collect::,_>>()?, vec![ex.into_owned().into()]); + /// assert_eq!( + /// store.named_graphs().collect::, _>>()?, + /// vec![ex.into_owned().into()] + /// ); /// # Result::<_, Box>::Ok(()) /// ``` pub fn insert_named_graph<'a>( @@ -828,8 +856,8 @@ impl Store { /// /// Usage example: /// ``` - /// use oxigraph::store::Store; /// use oxigraph::model::{NamedNodeRef, QuadRef}; + /// use oxigraph::store::Store; /// /// let ex = NamedNodeRef::new("http://example.com")?; /// let quad = QuadRef::new(ex, ex, ex, ex); @@ -856,8 +884,8 @@ impl Store { /// /// Usage example: /// ``` - /// use oxigraph::store::Store; /// use oxigraph::model::{NamedNodeRef, QuadRef}; + /// use oxigraph::store::Store; /// /// let ex = NamedNodeRef::new("http://example.com")?; /// let quad = QuadRef::new(ex, ex, ex, ex); @@ -882,13 +910,13 @@ impl Store { /// /// Usage example: /// ``` - /// use oxigraph::store::Store; /// use oxigraph::model::*; + /// use oxigraph::store::Store; /// /// let ex = NamedNodeRef::new("http://example.com")?; /// let store = Store::new()?; /// store.insert(QuadRef::new(ex, ex, ex, ex))?; - /// store.insert(QuadRef::new(ex, ex, ex, GraphNameRef::DefaultGraph))?; + /// store.insert(QuadRef::new(ex, ex, ex, GraphNameRef::DefaultGraph))?; /// assert_eq!(2, store.len()?); /// /// store.clear()?; @@ -944,15 +972,18 @@ impl Store { /// /// Usage example: /// ``` - /// use oxigraph::store::Store; /// use oxigraph::io::RdfFormat; /// use oxigraph::model::*; + /// use oxigraph::store::Store; /// /// let store = Store::new()?; /// /// // quads file insertion - /// let file = b" ."; - /// store.bulk_loader().load_from_read(RdfFormat::NQuads, file.as_ref())?; + /// let file = + /// b" ."; + /// store + /// .bulk_loader() + /// .load_from_read(RdfFormat::NQuads, file.as_ref())?; /// /// // we inspect the store contents /// let ex = NamedNodeRef::new("http://example.com")?; @@ -995,16 +1026,23 @@ impl<'a> Transaction<'a> { /// /// Usage example: /// ``` - /// use oxigraph::store::Store; /// use oxigraph::model::*; /// use oxigraph::sparql::{EvaluationError, QueryResults}; + /// use oxigraph::store::Store; /// /// let store = Store::new()?; /// store.transaction(|mut transaction| { - /// if let QueryResults::Solutions(solutions) = transaction.query("SELECT ?s WHERE { ?s ?p ?o }")? { + /// if let QueryResults::Solutions(solutions) = + /// transaction.query("SELECT ?s WHERE { ?s ?p ?o }")? + /// { /// for solution in solutions { - /// if let Some(Term::NamedNode(s)) = solution?.get("s") { - /// transaction.insert(QuadRef::new(s, vocab::rdf::TYPE, NamedNodeRef::new_unchecked("http://example.com"), GraphNameRef::DefaultGraph))?; + /// if let Some(Term::NamedNode(s)) = solution?.get("s") { + /// transaction.insert(QuadRef::new( + /// s, + /// vocab::rdf::TYPE, + /// NamedNodeRef::new_unchecked("http://example.com"), + /// GraphNameRef::DefaultGraph, + /// ))?; /// } /// } /// } @@ -1023,9 +1061,9 @@ impl<'a> Transaction<'a> { /// /// Usage example with a custom function serializing terms to N-Triples: /// ``` - /// use oxigraph::store::Store; /// use oxigraph::model::*; /// use oxigraph::sparql::{EvaluationError, QueryOptions, QueryResults}; + /// use oxigraph::store::Store; /// /// let store = Store::new()?; /// store.transaction(|mut transaction| { @@ -1033,13 +1071,20 @@ impl<'a> Transaction<'a> { /// "SELECT ?s ((?s) AS ?nt) WHERE { ?s ?p ?o }", /// QueryOptions::default().with_custom_function( /// NamedNode::new_unchecked("http://www.w3.org/ns/formats/N-Triples"), - /// |args| args.get(0).map(|t| Literal::from(t.to_string()).into()) - /// ) + /// |args| args.get(0).map(|t| Literal::from(t.to_string()).into()), + /// ), /// )? { /// for solution in solutions { /// let solution = solution?; - /// if let (Some(Term::NamedNode(s)), Some(nt)) = (solution.get("s"), solution.get("nt")) { - /// transaction.insert(QuadRef::new(s, NamedNodeRef::new_unchecked("http://example.com/n-triples-representation"), nt, GraphNameRef::DefaultGraph))?; + /// if let (Some(Term::NamedNode(s)), Some(nt)) = + /// (solution.get("s"), solution.get("nt")) + /// { + /// transaction.insert(QuadRef::new( + /// s, + /// NamedNodeRef::new_unchecked("http://example.com/n-triples-representation"), + /// nt, + /// GraphNameRef::DefaultGraph, + /// ))?; /// } /// } /// } @@ -1060,8 +1105,8 @@ impl<'a> Transaction<'a> { /// /// Usage example: /// ``` - /// use oxigraph::store::{StorageError, Store}; /// use oxigraph::model::*; + /// use oxigraph::store::{StorageError, Store}; /// /// let store = Store::new()?; /// let a = NamedNodeRef::new("http://example.com/a")?; @@ -1123,14 +1168,16 @@ impl<'a> Transaction<'a> { /// /// Usage example: /// ``` - /// use oxigraph::store::Store; /// use oxigraph::model::*; /// use oxigraph::sparql::EvaluationError; + /// use oxigraph::store::Store; /// /// let store = Store::new()?; /// store.transaction(|mut transaction| { /// // insertion - /// transaction.update("INSERT DATA { }")?; + /// transaction.update( + /// "INSERT DATA { }", + /// )?; /// /// // we inspect the store contents /// let ex = NamedNodeRef::new_unchecked("http://example.com"); @@ -1210,16 +1257,21 @@ impl<'a> Transaction<'a> { /// /// Usage example: /// ``` - /// use oxigraph::store::Store; /// use oxigraph::io::RdfFormat; /// use oxigraph::model::*; + /// use oxigraph::store::Store; /// /// let store = Store::new()?; /// /// // insertion /// let file = b" ."; /// store.transaction(|mut transaction| { - /// transaction.load_graph(file.as_ref(), RdfFormat::NTriples, GraphName::DefaultGraph, None) + /// transaction.load_graph( + /// file.as_ref(), + /// RdfFormat::NTriples, + /// GraphName::DefaultGraph, + /// None, + /// ) /// })?; /// /// // we inspect the store contents @@ -1253,14 +1305,15 @@ impl<'a> Transaction<'a> { /// /// Usage example: /// ``` - /// use oxigraph::store::Store; /// use oxigraph::io::RdfFormat; /// use oxigraph::model::*; + /// use oxigraph::store::Store; /// /// let store = Store::new()?; /// /// // insertion - /// let file = b" ."; + /// let file = + /// b" ."; /// store.transaction(|mut transaction| { /// transaction.load_dataset(file.as_ref(), RdfFormat::NQuads, None) /// })?; @@ -1295,16 +1348,14 @@ impl<'a> Transaction<'a> { /// /// Usage example: /// ``` - /// use oxigraph::store::Store; /// use oxigraph::model::*; + /// use oxigraph::store::Store; /// /// let ex = NamedNodeRef::new_unchecked("http://example.com"); /// let quad = QuadRef::new(ex, ex, ex, GraphNameRef::DefaultGraph); /// /// let store = Store::new()?; - /// store.transaction(|mut transaction| { - /// transaction.insert(quad) - /// })?; + /// store.transaction(|mut transaction| transaction.insert(quad))?; /// assert!(store.contains(quad)?); /// # Result::<_,oxigraph::store::StorageError>::Ok(()) /// ``` @@ -1329,8 +1380,8 @@ impl<'a> Transaction<'a> { /// /// Usage example: /// ``` - /// use oxigraph::store::Store; /// use oxigraph::model::*; + /// use oxigraph::store::Store; /// /// let ex = NamedNodeRef::new_unchecked("http://example.com"); /// let quad = QuadRef::new(ex, ex, ex, GraphNameRef::DefaultGraph); @@ -1371,15 +1422,16 @@ impl<'a> Transaction<'a> { /// /// Usage example: /// ``` - /// use oxigraph::store::Store; /// use oxigraph::model::NamedNodeRef; + /// use oxigraph::store::Store; /// /// let ex = NamedNodeRef::new_unchecked("http://example.com"); /// let store = Store::new()?; - /// store.transaction(|mut transaction| { - /// transaction.insert_named_graph(ex) - /// })?; - /// assert_eq!(store.named_graphs().collect::,_>>()?, vec![ex.into_owned().into()]); + /// store.transaction(|mut transaction| transaction.insert_named_graph(ex))?; + /// assert_eq!( + /// store.named_graphs().collect::, _>>()?, + /// vec![ex.into_owned().into()] + /// ); /// # Result::<_,oxigraph::store::StorageError>::Ok(()) /// ``` pub fn insert_named_graph<'b>( @@ -1393,8 +1445,8 @@ impl<'a> Transaction<'a> { /// /// Usage example: /// ``` - /// use oxigraph::store::Store; /// use oxigraph::model::{NamedNodeRef, QuadRef}; + /// use oxigraph::store::Store; /// /// let ex = NamedNodeRef::new_unchecked("http://example.com"); /// let quad = QuadRef::new(ex, ex, ex, ex); @@ -1420,8 +1472,8 @@ impl<'a> Transaction<'a> { /// /// Usage example: /// ``` - /// use oxigraph::store::Store; /// use oxigraph::model::{NamedNodeRef, QuadRef}; + /// use oxigraph::store::Store; /// /// let ex = NamedNodeRef::new_unchecked("http://example.com"); /// let quad = QuadRef::new(ex, ex, ex, ex); @@ -1445,8 +1497,8 @@ impl<'a> Transaction<'a> { /// /// Usage example: /// ``` - /// use oxigraph::store::Store; /// use oxigraph::model::*; + /// use oxigraph::store::Store; /// /// let ex = NamedNodeRef::new_unchecked("http://example.com"); /// let store = Store::new()?; @@ -1518,15 +1570,18 @@ impl Iterator for GraphNameIter { /// /// Usage example with loading a dataset: /// ``` -/// use oxigraph::store::Store; /// use oxigraph::io::RdfFormat; /// use oxigraph::model::*; +/// use oxigraph::store::Store; /// /// let store = Store::new()?; /// /// // quads file insertion -/// let file = b" ."; -/// store.bulk_loader().load_from_read(RdfFormat::NQuads, file.as_ref())?; +/// let file = +/// b" ."; +/// store +/// .bulk_loader() +/// .load_from_read(RdfFormat::NQuads, file.as_ref())?; /// /// // we inspect the store contents /// let ex = NamedNodeRef::new("http://example.com")?; @@ -1684,15 +1739,18 @@ impl BulkLoader { /// /// Usage example: /// ``` - /// use oxigraph::store::Store; /// use oxigraph::io::RdfFormat; /// use oxigraph::model::*; + /// use oxigraph::store::Store; /// /// let store = Store::new()?; /// /// // insertion - /// let file = b" ."; - /// store.bulk_loader().load_dataset(file.as_ref(), RdfFormat::NQuads, None)?; + /// let file = + /// b" ."; + /// store + /// .bulk_loader() + /// .load_dataset(file.as_ref(), RdfFormat::NQuads, None)?; /// /// // we inspect the store contents /// let ex = NamedNodeRef::new("http://example.com")?; @@ -1745,15 +1803,20 @@ impl BulkLoader { /// /// Usage example: /// ``` - /// use oxigraph::store::Store; /// use oxigraph::io::RdfFormat; /// use oxigraph::model::*; + /// use oxigraph::store::Store; /// /// let store = Store::new()?; /// /// // insertion /// let file = b" ."; - /// store.bulk_loader().load_graph(file.as_ref(), RdfFormat::NTriples, GraphName::DefaultGraph, None)?; + /// store.bulk_loader().load_graph( + /// file.as_ref(), + /// RdfFormat::NTriples, + /// GraphName::DefaultGraph, + /// None, + /// )?; /// /// // we inspect the store contents /// let ex = NamedNodeRef::new("http://example.com")?; diff --git a/python/src/io.rs b/python/src/io.rs index 8258fda2..bf3a4383 100644 --- a/python/src/io.rs +++ b/python/src/io.rs @@ -199,31 +199,26 @@ impl PyRdfFormat { const N3: Self = Self { inner: RdfFormat::N3, }; - /// `N-Quads `_ #[classattr] const N_QUADS: Self = Self { inner: RdfFormat::NQuads, }; - /// `N-Triples `_ #[classattr] const N_TRIPLES: Self = Self { inner: RdfFormat::NTriples, }; - /// `RDF/XML `_ #[classattr] const RDF_XML: Self = Self { inner: RdfFormat::RdfXml, }; - /// `TriG `_ #[classattr] const TRIG: Self = Self { inner: RdfFormat::TriG, }; - /// `Turtle `_ #[classattr] const TURTLE: Self = Self { diff --git a/python/src/model.rs b/python/src/model.rs index cf673d65..5933013f 100644 --- a/python/src/model.rs +++ b/python/src/model.rs @@ -6,8 +6,7 @@ use pyo3::prelude::*; use pyo3::types::{PyDict, PyTuple}; use pyo3::PyTypeInfo; use std::collections::hash_map::DefaultHasher; -use std::hash::Hash; -use std::hash::Hasher; +use std::hash::{Hash, Hasher}; use std::vec::IntoIter; /// An RDF `node identified by an IRI `_. @@ -345,7 +344,6 @@ impl PyLiteral { /// >>> Literal('example', language='en').language /// 'en' /// >>> Literal('example').language - /// #[getter] fn language(&self) -> Option<&str> { self.inner.language() diff --git a/python/src/sparql.rs b/python/src/sparql.rs index c79a6b7e..383a7413 100644 --- a/python/src/sparql.rs +++ b/python/src/sparql.rs @@ -515,29 +515,26 @@ pub struct PyQueryResultsFormat { #[pymethods] impl PyQueryResultsFormat { - /// `SPARQL Query Results XML Format `_ + /// `SPARQL Query Results CSV Format `_ #[classattr] - const XML: Self = Self { - inner: QueryResultsFormat::Xml, + const CSV: Self = Self { + inner: QueryResultsFormat::Csv, }; - /// `SPARQL Query Results JSON Format `_ #[classattr] const JSON: Self = Self { inner: QueryResultsFormat::Json, }; - - /// `SPARQL Query Results CSV Format `_ - #[classattr] - const CSV: Self = Self { - inner: QueryResultsFormat::Csv, - }; - /// `SPARQL Query Results TSV Format `_ #[classattr] const TSV: Self = Self { inner: QueryResultsFormat::Tsv, }; + /// `SPARQL Query Results XML Format `_ + #[classattr] + const XML: Self = Self { + inner: QueryResultsFormat::Xml, + }; /// :return: the format canonical IRI according to the `Unique URIs for file formats registry `_. /// :rtype: str diff --git a/rustfmt.toml b/rustfmt.toml index d51f7320..5fce0595 100644 --- a/rustfmt.toml +++ b/rustfmt.toml @@ -1,3 +1,11 @@ +force_explicit_abi = true +format_code_in_doc_comments = true +format_macro_matchers = true +imports_granularity = "Module" newline_style = "Unix" +normalize_comments = true +normalize_doc_attributes = true +reorder_impl_items = true +group_imports = "One" use_field_init_shorthand = true -use_try_shorthand = true \ No newline at end of file +use_try_shorthand = true diff --git a/testsuite/src/parser_evaluator.rs b/testsuite/src/parser_evaluator.rs index 0d3a22e6..830607b0 100644 --- a/testsuite/src/parser_evaluator.rs +++ b/testsuite/src/parser_evaluator.rs @@ -77,11 +77,11 @@ pub fn register_parser_tests(evaluator: &mut TestEvaluator) { ); evaluator.register( "https://w3c.github.io/rdf-canon/tests/vocab#RDFC10NegativeEvalTest", - |_| Ok(()), //TODO: not a proper implementation + |_| Ok(()), // TODO: not a proper implementation ); evaluator.register( "https://w3c.github.io/rdf-canon/tests/vocab#RDFC10MapTest", - |_| Ok(()), //TODO: not a proper implementation + |_| Ok(()), // TODO: not a proper implementation ); evaluator.register( "https://github.com/oxigraph/oxigraph/tests#TestNTripleRecovery", diff --git a/testsuite/tests/sparql.rs b/testsuite/tests/sparql.rs index eafb80fd..6d2a6f7d 100644 --- a/testsuite/tests/sparql.rs +++ b/testsuite/tests/sparql.rs @@ -8,7 +8,7 @@ fn sparql10_w3c_query_syntax_testsuite() -> Result<()> { check_testsuite( "https://w3c.github.io/rdf-tests/sparql/sparql10/manifest-syntax.ttl", &[ - "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/syntax-sparql3/manifest#syn-bad-26", // tokenizer + "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/syntax-sparql3/manifest#syn-bad-26", /* tokenizer */ ], ) } @@ -53,9 +53,9 @@ fn sparql11_query_w3c_evaluation_testsuite() -> Result<()> { check_testsuite( "https://w3c.github.io/rdf-tests/sparql/sparql11/manifest-sparql11-query.ttl", &[ - //BNODE() scope is currently wrong + // BNODE() scope is currently wrong "http://www.w3.org/2009/sparql/docs/tests/data-sparql11/functions/manifest#bnode01", - //SERVICE name from a BGP + // SERVICE name from a BGP "http://www.w3.org/2009/sparql/docs/tests/data-sparql11/service/manifest#service5", ], ) From 185d83838cb1bc1e25497e20da9bb06e02143cf5 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Fri, 19 Jan 2024 14:44:28 -0500 Subject: [PATCH 187/217] Linting: Impl ordering, Self refs For consistency, ordered implementation of a traits the same way as they are declared. Used `Self::` in a few last spots --- lib/oxrdf/src/dataset.rs | 8 ++++---- lib/oxrdf/src/graph.rs | 2 +- lib/oxttl/src/lexer.rs | 4 ++-- lib/oxttl/src/line_formats.rs | 14 +++++++------- lib/oxttl/src/n3.rs | 12 ++++++------ lib/oxttl/src/terse.rs | 12 ++++++------ lib/sparesults/src/solution.rs | 2 +- lib/src/sparql/eval.rs | 2 +- 8 files changed, 28 insertions(+), 28 deletions(-) diff --git a/lib/oxrdf/src/dataset.rs b/lib/oxrdf/src/dataset.rs index 8412a8aa..ed6249a4 100644 --- a/lib/oxrdf/src/dataset.rs +++ b/lib/oxrdf/src/dataset.rs @@ -925,8 +925,8 @@ impl PartialEq for Dataset { impl Eq for Dataset {} impl<'a> IntoIterator for &'a Dataset { - type IntoIter = Iter<'a>; type Item = QuadRef<'a>; + type IntoIter = Iter<'a>; fn into_iter(self) -> Self::IntoIter { self.iter() @@ -1283,8 +1283,8 @@ impl<'a> GraphView<'a> { } impl<'a> IntoIterator for GraphView<'a> { - type IntoIter = GraphViewIter<'a>; type Item = TripleRef<'a>; + type IntoIter = GraphViewIter<'a>; fn into_iter(self) -> Self::IntoIter { self.iter() @@ -1292,8 +1292,8 @@ impl<'a> IntoIterator for GraphView<'a> { } impl<'a, 'b> IntoIterator for &'b GraphView<'a> { - type IntoIter = GraphViewIter<'a>; type Item = TripleRef<'a>; + type IntoIter = GraphViewIter<'a>; fn into_iter(self) -> Self::IntoIter { self.iter() @@ -1494,8 +1494,8 @@ impl<'a, 'b, T: Into>> Extend for GraphViewMut<'a> { } impl<'a> IntoIterator for &'a GraphViewMut<'a> { - type IntoIter = GraphViewIter<'a>; type Item = TripleRef<'a>; + type IntoIter = GraphViewIter<'a>; fn into_iter(self) -> Self::IntoIter { self.iter() diff --git a/lib/oxrdf/src/graph.rs b/lib/oxrdf/src/graph.rs index 33f67132..5459b65c 100644 --- a/lib/oxrdf/src/graph.rs +++ b/lib/oxrdf/src/graph.rs @@ -229,8 +229,8 @@ impl PartialEq for Graph { impl Eq for Graph {} impl<'a> IntoIterator for &'a Graph { - type IntoIter = Iter<'a>; type Item = TripleRef<'a>; + type IntoIter = Iter<'a>; fn into_iter(self) -> Self::IntoIter { self.iter() diff --git a/lib/oxttl/src/lexer.rs b/lib/oxttl/src/lexer.rs index 3fb62845..eee7cd60 100644 --- a/lib/oxttl/src/lexer.rs +++ b/lib/oxttl/src/lexer.rs @@ -49,15 +49,15 @@ pub struct N3Lexer { // TODO: simplify by not giving is_end and fail with an "unexpected eof" is none is returned when is_end=true? impl TokenRecognizer for N3Lexer { - type Options = N3LexerOptions; type Token<'a> = N3Token<'a>; + type Options = N3LexerOptions; fn recognize_next_token<'a>( &mut self, data: &'a [u8], is_ending: bool, options: &Self::Options, - ) -> Option<(usize, Result, TokenRecognizerError>)> { + ) -> Option<(usize, Result, TokenRecognizerError>)> { match *data.first()? { b'<' => match *data.get(1)? { b'<' => Some((2, Ok(N3Token::Punctuation("<<")))), diff --git a/lib/oxttl/src/line_formats.rs b/lib/oxttl/src/line_formats.rs index 5932f7a2..730da18a 100644 --- a/lib/oxttl/src/line_formats.rs +++ b/lib/oxttl/src/line_formats.rs @@ -39,9 +39,9 @@ enum NQuadsState { } impl RuleRecognizer for NQuadsRecognizer { - type Context = NQuadsRecognizerContext; - type Output = Quad; type TokenRecognizer = N3Lexer; + type Output = Quad; + type Context = NQuadsRecognizerContext; fn error_recovery_state(mut self) -> Self { self.stack.clear(); @@ -54,8 +54,8 @@ impl RuleRecognizer for NQuadsRecognizer { fn recognize_next( mut self, token: N3Token<'_>, - context: &mut NQuadsRecognizerContext, - results: &mut Vec, + context: &mut Self::Context, + results: &mut Vec, errors: &mut Vec, ) -> Self { if let Some(state) = self.stack.pop() { @@ -235,8 +235,8 @@ impl RuleRecognizer for NQuadsRecognizer { fn recognize_end( mut self, - _context: &mut NQuadsRecognizerContext, - results: &mut Vec, + _context: &mut Self::Context, + results: &mut Vec, errors: &mut Vec, ) { match &*self.stack { @@ -255,7 +255,7 @@ impl RuleRecognizer for NQuadsRecognizer { } } - fn lexer_options(context: &NQuadsRecognizerContext) -> &N3LexerOptions { + fn lexer_options(context: &Self::Context) -> &N3LexerOptions { &context.lexer_options } } diff --git a/lib/oxttl/src/n3.rs b/lib/oxttl/src/n3.rs index 263db936..516144a3 100644 --- a/lib/oxttl/src/n3.rs +++ b/lib/oxttl/src/n3.rs @@ -707,9 +707,9 @@ struct N3RecognizerContext { } impl RuleRecognizer for N3Recognizer { - type Context = N3RecognizerContext; - type Output = N3Quad; type TokenRecognizer = N3Lexer; + type Output = N3Quad; + type Context = N3RecognizerContext; fn error_recovery_state(mut self) -> Self { self.stack.clear(); @@ -722,8 +722,8 @@ impl RuleRecognizer for N3Recognizer { fn recognize_next( mut self, token: N3Token<'_>, - context: &mut N3RecognizerContext, - results: &mut Vec, + context: &mut Self::Context, + results: &mut Vec, errors: &mut Vec, ) -> Self { while let Some(rule) = self.stack.pop() { @@ -1195,7 +1195,7 @@ impl RuleRecognizer for N3Recognizer { fn recognize_end( self, - _state: &mut N3RecognizerContext, + _state: &mut Self::Context, _results: &mut Vec, errors: &mut Vec, ) { @@ -1205,7 +1205,7 @@ impl RuleRecognizer for N3Recognizer { } } - fn lexer_options(context: &N3RecognizerContext) -> &N3LexerOptions { + fn lexer_options(context: &Self::Context) -> &N3LexerOptions { &context.lexer_options } } diff --git a/lib/oxttl/src/terse.rs b/lib/oxttl/src/terse.rs index 818524f6..d94f742d 100644 --- a/lib/oxttl/src/terse.rs +++ b/lib/oxttl/src/terse.rs @@ -28,9 +28,9 @@ pub struct TriGRecognizerContext { } impl RuleRecognizer for TriGRecognizer { - type Context = TriGRecognizerContext; - type Output = Quad; type TokenRecognizer = N3Lexer; + type Output = Quad; + type Context = TriGRecognizerContext; fn error_recovery_state(mut self) -> Self { self.stack.clear(); @@ -44,8 +44,8 @@ impl RuleRecognizer for TriGRecognizer { fn recognize_next( mut self, token: N3Token<'_>, - context: &mut TriGRecognizerContext, - results: &mut Vec, + context: &mut Self::Context, + results: &mut Vec, errors: &mut Vec, ) -> Self { if let Some(rule) = self.stack.pop() { @@ -792,7 +792,7 @@ impl RuleRecognizer for TriGRecognizer { fn recognize_end( mut self, - _context: &mut TriGRecognizerContext, + _context: &mut Self::Context, results: &mut Vec, errors: &mut Vec, ) { @@ -821,7 +821,7 @@ impl RuleRecognizer for TriGRecognizer { } } - fn lexer_options(context: &TriGRecognizerContext) -> &N3LexerOptions { + fn lexer_options(context: &Self::Context) -> &N3LexerOptions { &context.lexer_options } } diff --git a/lib/sparesults/src/solution.rs b/lib/sparesults/src/solution.rs index 826a9eea..a1364861 100644 --- a/lib/sparesults/src/solution.rs +++ b/lib/sparesults/src/solution.rs @@ -171,8 +171,8 @@ impl>, S: Into>>> From<(V, S)> for Quer } impl<'a> IntoIterator for &'a QuerySolution { - type IntoIter = Iter<'a>; type Item = (&'a Variable, &'a Term); + type IntoIter = Iter<'a>; #[inline] fn into_iter(self) -> Self::IntoIter { diff --git a/lib/src/sparql/eval.rs b/lib/src/sparql/eval.rs index 19c6884d..174f41fa 100644 --- a/lib/src/sparql/eval.rs +++ b/lib/src/sparql/eval.rs @@ -111,8 +111,8 @@ impl EncodedTuple { } impl IntoIterator for EncodedTuple { - type IntoIter = std::vec::IntoIter>; type Item = Option; + type IntoIter = std::vec::IntoIter>; fn into_iter(self) -> Self::IntoIter { self.inner.into_iter() From 6494ba6e31ef80992d175c0e89285abd3c2e5b59 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Fri, 19 Jan 2024 15:49:37 -0500 Subject: [PATCH 188/217] keep concrete types --- lib/oxttl/src/lexer.rs | 2 +- lib/oxttl/src/line_formats.rs | 10 +++++----- lib/oxttl/src/n3.rs | 8 ++++---- lib/oxttl/src/terse.rs | 8 ++++---- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/lib/oxttl/src/lexer.rs b/lib/oxttl/src/lexer.rs index eee7cd60..fd8341a2 100644 --- a/lib/oxttl/src/lexer.rs +++ b/lib/oxttl/src/lexer.rs @@ -57,7 +57,7 @@ impl TokenRecognizer for N3Lexer { data: &'a [u8], is_ending: bool, options: &Self::Options, - ) -> Option<(usize, Result, TokenRecognizerError>)> { + ) -> Option<(usize, Result, TokenRecognizerError>)> { match *data.first()? { b'<' => match *data.get(1)? { b'<' => Some((2, Ok(N3Token::Punctuation("<<")))), diff --git a/lib/oxttl/src/line_formats.rs b/lib/oxttl/src/line_formats.rs index 730da18a..e522bd53 100644 --- a/lib/oxttl/src/line_formats.rs +++ b/lib/oxttl/src/line_formats.rs @@ -54,8 +54,8 @@ impl RuleRecognizer for NQuadsRecognizer { fn recognize_next( mut self, token: N3Token<'_>, - context: &mut Self::Context, - results: &mut Vec, + context: &mut NQuadsRecognizerContext, + results: &mut Vec, errors: &mut Vec, ) -> Self { if let Some(state) = self.stack.pop() { @@ -235,8 +235,8 @@ impl RuleRecognizer for NQuadsRecognizer { fn recognize_end( mut self, - _context: &mut Self::Context, - results: &mut Vec, + _context: &mut NQuadsRecognizerContext, + results: &mut Vec, errors: &mut Vec, ) { match &*self.stack { @@ -255,7 +255,7 @@ impl RuleRecognizer for NQuadsRecognizer { } } - fn lexer_options(context: &Self::Context) -> &N3LexerOptions { + fn lexer_options(context: &NQuadsRecognizerContext) -> &N3LexerOptions { &context.lexer_options } } diff --git a/lib/oxttl/src/n3.rs b/lib/oxttl/src/n3.rs index 516144a3..d4241b7d 100644 --- a/lib/oxttl/src/n3.rs +++ b/lib/oxttl/src/n3.rs @@ -722,8 +722,8 @@ impl RuleRecognizer for N3Recognizer { fn recognize_next( mut self, token: N3Token<'_>, - context: &mut Self::Context, - results: &mut Vec, + context: &mut N3RecognizerContext, + results: &mut Vec, errors: &mut Vec, ) -> Self { while let Some(rule) = self.stack.pop() { @@ -1195,7 +1195,7 @@ impl RuleRecognizer for N3Recognizer { fn recognize_end( self, - _state: &mut Self::Context, + _state: &mut N3RecognizerContext, _results: &mut Vec, errors: &mut Vec, ) { @@ -1205,7 +1205,7 @@ impl RuleRecognizer for N3Recognizer { } } - fn lexer_options(context: &Self::Context) -> &N3LexerOptions { + fn lexer_options(context: &N3RecognizerContext) -> &N3LexerOptions { &context.lexer_options } } diff --git a/lib/oxttl/src/terse.rs b/lib/oxttl/src/terse.rs index d94f742d..7aa3e103 100644 --- a/lib/oxttl/src/terse.rs +++ b/lib/oxttl/src/terse.rs @@ -44,8 +44,8 @@ impl RuleRecognizer for TriGRecognizer { fn recognize_next( mut self, token: N3Token<'_>, - context: &mut Self::Context, - results: &mut Vec, + context: &mut TriGRecognizerContext, + results: &mut Vec, errors: &mut Vec, ) -> Self { if let Some(rule) = self.stack.pop() { @@ -792,7 +792,7 @@ impl RuleRecognizer for TriGRecognizer { fn recognize_end( mut self, - _context: &mut Self::Context, + _context: &mut TriGRecognizerContext, results: &mut Vec, errors: &mut Vec, ) { @@ -821,7 +821,7 @@ impl RuleRecognizer for TriGRecognizer { } } - fn lexer_options(context: &Self::Context) -> &N3LexerOptions { + fn lexer_options(context: &TriGRecognizerContext) -> &N3LexerOptions { &context.lexer_options } } From 54489aacfbecf04238651be425d1a32ee77b0be2 Mon Sep 17 00:00:00 2001 From: Tpt Date: Thu, 18 Jan 2024 21:46:13 +0100 Subject: [PATCH 189/217] oxttl and oxrdfio: improves prefixes and base_iri getters --- lib/oxrdfio/src/parser.rs | 191 +++++++++++++++++++++++++++++++++++++- lib/oxttl/src/n3.rs | 68 +++++++++++--- lib/oxttl/src/terse.rs | 9 +- lib/oxttl/src/trig.rs | 68 +++++++++++--- lib/oxttl/src/turtle.rs | 68 +++++++++++--- 5 files changed, 355 insertions(+), 49 deletions(-) diff --git a/lib/oxrdfio/src/parser.rs b/lib/oxrdfio/src/parser.rs index 0f6d11ac..766be9ce 100644 --- a/lib/oxrdfio/src/parser.rs +++ b/lib/oxrdfio/src/parser.rs @@ -8,7 +8,7 @@ use oxrdfxml::FromTokioAsyncReadRdfXmlReader; use oxrdfxml::{FromReadRdfXmlReader, RdfXmlParser}; #[cfg(feature = "async-tokio")] use oxttl::n3::FromTokioAsyncReadN3Reader; -use oxttl::n3::{FromReadN3Reader, N3Parser, N3Quad, N3Term}; +use oxttl::n3::{FromReadN3Reader, N3Parser, N3PrefixesIter, N3Quad, N3Term}; #[cfg(feature = "async-tokio")] use oxttl::nquads::FromTokioAsyncReadNQuadsReader; use oxttl::nquads::{FromReadNQuadsReader, NQuadsParser}; @@ -17,10 +17,10 @@ use oxttl::ntriples::FromTokioAsyncReadNTriplesReader; use oxttl::ntriples::{FromReadNTriplesReader, NTriplesParser}; #[cfg(feature = "async-tokio")] use oxttl::trig::FromTokioAsyncReadTriGReader; -use oxttl::trig::{FromReadTriGReader, TriGParser}; +use oxttl::trig::{FromReadTriGReader, TriGParser, TriGPrefixesIter}; #[cfg(feature = "async-tokio")] use oxttl::turtle::FromTokioAsyncReadTurtleReader; -use oxttl::turtle::{FromReadTurtleReader, TurtleParser}; +use oxttl::turtle::{FromReadTurtleReader, TurtleParser, TurtlePrefixesIter}; use std::collections::HashMap; use std::io::Read; #[cfg(feature = "async-tokio")] @@ -428,6 +428,77 @@ impl Iterator for FromReadQuadReader { } } +impl FromReadQuadReader { + /// The list of IRI prefixes considered at the current step of the parsing. + /// + /// This method returns (prefix name, prefix value) tuples. + /// It is empty at the beginning of the parsing and gets updated when prefixes are encountered. + /// It should be full at the end of the parsing (but if a prefix is overridden, only the latest version will be returned). + /// + /// An empty iterator is return if the format does not support prefixes. + /// + /// ``` + /// use oxrdfio::{RdfFormat, RdfParser}; + /// + /// let file = br#"@base . + /// @prefix schema: . + /// a schema:Person ; + /// schema:name "Foo" ."#; + /// + /// let mut reader = RdfParser::from_format(RdfFormat::Turtle).parse_read(file.as_slice()); + /// assert!(reader.prefixes().collect::>().is_empty()); // No prefix at the beginning + /// + /// reader.next().unwrap()?; // We read the first triple + /// assert_eq!( + /// reader.prefixes().collect::>(), + /// [("schema", "http://schema.org/")] + /// ); // There are now prefixes + /// # Result::<_,Box>::Ok(()) + /// ``` + pub fn prefixes(&self) -> PrefixesIter<'_> { + PrefixesIter { + inner: match &self.parser { + FromReadQuadReaderKind::N3(p) => PrefixesIterKind::N3(p.prefixes()), + FromReadQuadReaderKind::TriG(p) => PrefixesIterKind::TriG(p.prefixes()), + FromReadQuadReaderKind::Turtle(p) => PrefixesIterKind::Turtle(p.prefixes()), + FromReadQuadReaderKind::NQuads(_) + | FromReadQuadReaderKind::NTriples(_) + | FromReadQuadReaderKind::RdfXml(_) => PrefixesIterKind::None, /* TODO: implement for RDF/XML */ + }, + } + } + + /// The base IRI considered at the current step of the parsing. + /// + /// `None` is returned if no base IRI is set or the format does not support base IRIs. + /// + /// ``` + /// use oxrdfio::{RdfFormat, RdfParser}; + /// + /// let file = br#"@base . + /// @prefix schema: . + /// a schema:Person ; + /// schema:name "Foo" ."#; + /// + /// let mut reader = RdfParser::from_format(RdfFormat::Turtle).parse_read(file.as_slice()); + /// assert!(reader.base_iri().is_none()); // No base at the beginning because none has been given to the parser. + /// + /// reader.next().unwrap()?; // We read the first triple + /// assert_eq!(reader.base_iri(), Some("http://example.com/")); // There is now a base IRI. + /// # Result::<_,Box>::Ok(()) + /// ``` + pub fn base_iri(&self) -> Option<&str> { + match &self.parser { + FromReadQuadReaderKind::N3(p) => p.base_iri(), + FromReadQuadReaderKind::TriG(p) => p.base_iri(), + FromReadQuadReaderKind::Turtle(p) => p.base_iri(), + FromReadQuadReaderKind::NQuads(_) + | FromReadQuadReaderKind::NTriples(_) + | FromReadQuadReaderKind::RdfXml(_) => None, // TODO: implement for RDF/XML + } + } +} + /// Parses a RDF file from a Tokio [`AsyncRead`] implementation. Can be built using [`RdfParser::parse_tokio_async_read`]. /// /// Reads are buffered. @@ -494,6 +565,120 @@ impl FromTokioAsyncReadQuadReader { }, }) } + + /// The list of IRI prefixes considered at the current step of the parsing. + /// + /// This method returns (prefix name, prefix value) tuples. + /// It is empty at the beginning of the parsing and gets updated when prefixes are encountered. + /// It should be full at the end of the parsing (but if a prefix is overridden, only the latest version will be returned). + /// + /// An empty iterator is return if the format does not support prefixes. + /// + /// ``` + /// use oxrdfio::{RdfFormat, RdfParser}; + /// + /// # #[tokio::main(flavor = "current_thread")] + /// # async fn main() -> Result<(), oxttl::ParseError> { + /// let file = br#"@base . + /// @prefix schema: . + /// a schema:Person ; + /// schema:name "Foo" ."#; + /// + /// let mut reader = RdfParser::from_format(RdfFormat::Turtle).parse_read(file.as_slice()); + /// assert_eq!(reader.prefixes().collect::>(), []); // No prefix at the beginning + /// + /// reader.next().await.unwrap()?; // We read the first triple + /// assert_eq!( + /// reader.prefixes().collect::>(), + /// [("schema", "http://schema.org/")] + /// ); // There are now prefixes + /// # Ok(()) + /// # } + /// ``` + pub fn prefixes(&self) -> PrefixesIter<'_> { + PrefixesIter { + inner: match &self.parser { + FromReadQuadReaderKind::N3(p) => PrefixesIterKind::N3(p.prefixes()), + FromReadQuadReaderKind::TriG(p) => PrefixesIterKind::TriG(p.prefixes()), + FromReadQuadReaderKind::Turtle(p) => PrefixesIterKind::Turtle(p.prefixes()), + FromReadQuadReaderKind::NQuads(_) + | FromReadQuadReaderKind::NTriples(_) + | FromReadQuadReaderKind::RdfXml(_) => PrefixesIterKind::None, /* TODO: implement for RDF/XML */ + }, + } + } + + /// The base IRI considered at the current step of the parsing. + /// + /// `None` is returned if no base IRI is set or the format does not support base IRIs. + /// + /// ``` + /// use oxrdfio::{RdfFormat, RdfParser}; + /// + /// # #[tokio::main(flavor = "current_thread")] + /// # async fn main() -> Result<(), oxttl::ParseError> { + /// let file = br#"@base . + /// @prefix schema: . + /// a schema:Person ; + /// schema:name "Foo" ."#; + /// + /// let mut reader = + /// RdfParser::from_format(RdfFormat::Turtle).parse_tokio_async_read(file.as_slice()); + /// assert!(reader.base_iri().is_none()); // No base IRI at the beginning + /// + /// reader.next().await.unwrap()?; // We read the first triple + /// assert_eq!(reader.base_iri(), Some("http://example.com/")); // There is now a base IRI + /// # Ok(()) + /// # } + /// ``` + pub fn base_iri(&self) -> Option<&str> { + match &self.parser { + FromReadQuadReaderKind::N3(p) => p.base_iri(), + FromReadQuadReaderKind::TriG(p) => p.base_iri(), + FromReadQuadReaderKind::Turtle(p) => p.base_iri(), + FromReadQuadReaderKind::NQuads(_) + | FromReadQuadReaderKind::NTriples(_) + | FromReadQuadReaderKind::RdfXml(_) => None, // TODO: implement for RDF/XML + } + } +} + +/// Iterator on the file prefixes. +/// +/// See [`FromReadQuadReader::prefixes`]. +pub struct PrefixesIter<'a> { + inner: PrefixesIterKind<'a>, +} + +enum PrefixesIterKind<'a> { + Turtle(TurtlePrefixesIter<'a>), + TriG(TriGPrefixesIter<'a>), + N3(N3PrefixesIter<'a>), + None, +} + +impl<'a> Iterator for PrefixesIter<'a> { + type Item = (&'a str, &'a str); + + #[inline] + fn next(&mut self) -> Option { + match &mut self.inner { + PrefixesIterKind::Turtle(iter) => iter.next(), + PrefixesIterKind::TriG(iter) => iter.next(), + PrefixesIterKind::N3(iter) => iter.next(), + PrefixesIterKind::None => None, + } + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + match &self.inner { + PrefixesIterKind::Turtle(iter) => iter.size_hint(), + PrefixesIterKind::TriG(iter) => iter.size_hint(), + PrefixesIterKind::N3(iter) => iter.size_hint(), + PrefixesIterKind::None => (0, Some(0)), + } + } } struct QuadMapper { diff --git a/lib/oxttl/src/n3.rs b/lib/oxttl/src/n3.rs index d4241b7d..233edccc 100644 --- a/lib/oxttl/src/n3.rs +++ b/lib/oxttl/src/n3.rs @@ -15,6 +15,7 @@ use oxrdf::{ BlankNode, GraphName, Literal, NamedNode, NamedNodeRef, NamedOrBlankNode, Quad, Subject, Term, Variable, }; +use std::collections::hash_map::Iter; use std::collections::HashMap; use std::fmt; use std::io::Read; @@ -403,7 +404,7 @@ pub struct FromReadN3Reader { impl FromReadN3Reader { /// The list of IRI prefixes considered at the current step of the parsing. /// - /// This method returns the mapping from prefix name to prefix value. + /// This method returns (prefix name, prefix value) tuples. /// It is empty at the beginning of the parsing and gets updated when prefixes are encountered. /// It should be full at the end of the parsing (but if a prefix is overridden, only the latest version will be returned). /// @@ -416,14 +417,19 @@ impl FromReadN3Reader { /// schema:name "Foo" ."#; /// /// let mut reader = N3Parser::new().parse_read(file.as_ref()); - /// assert!(reader.prefixes().is_empty()); // No prefix at the beginning + /// assert_eq!(reader.prefixes().collect::>(), []); // No prefix at the beginning /// /// reader.next().unwrap()?; // We read the first triple - /// assert_eq!(reader.prefixes()["schema"], "http://schema.org/"); // There are now prefixes + /// assert_eq!( + /// reader.prefixes().collect::>(), + /// [("schema", "http://schema.org/")] + /// ); // There are now prefixes /// # Result::<_,Box>::Ok(()) /// ``` - pub fn prefixes(&self) -> &HashMap> { - &self.inner.parser.context.prefixes + pub fn prefixes(&self) -> N3PrefixesIter<'_> { + N3PrefixesIter { + inner: self.inner.parser.context.prefixes.iter(), + } } /// The base IRI considered at the current step of the parsing. @@ -508,7 +514,7 @@ impl FromTokioAsyncReadN3Reader { /// The list of IRI prefixes considered at the current step of the parsing. /// - /// This method returns the mapping from prefix name to prefix value. + /// This method returns (prefix name, prefix value) tuples. /// It is empty at the beginning of the parsing and gets updated when prefixes are encountered. /// It should be full at the end of the parsing (but if a prefix is overridden, only the latest version will be returned). /// @@ -523,15 +529,20 @@ impl FromTokioAsyncReadN3Reader { /// schema:name "Foo" ."#; /// /// let mut reader = N3Parser::new().parse_tokio_async_read(file.as_ref()); - /// assert!(reader.prefixes().is_empty()); // No prefix at the beginning + /// assert_eq!(reader.prefixes().collect::>(), []); // No prefix at the beginning /// /// reader.next().await.unwrap()?; // We read the first triple - /// assert_eq!(reader.prefixes()["schema"], "http://schema.org/"); // There are now prefixes + /// assert_eq!( + /// reader.prefixes().collect::>(), + /// [("schema", "http://schema.org/")] + /// ); // There are now prefixes /// # Ok(()) /// # } /// ``` - pub fn prefixes(&self) -> &HashMap> { - &self.inner.parser.context.prefixes + pub fn prefixes(&self) -> N3PrefixesIter<'_> { + N3PrefixesIter { + inner: self.inner.parser.context.prefixes.iter(), + } } /// The base IRI considered at the current step of the parsing. @@ -636,7 +647,7 @@ impl LowLevelN3Reader { /// The list of IRI prefixes considered at the current step of the parsing. /// - /// This method returns the mapping from prefix name to prefix value. + /// This method returns (prefix name, prefix value) tuples. /// It is empty at the beginning of the parsing and gets updated when prefixes are encountered. /// It should be full at the end of the parsing (but if a prefix is overridden, only the latest version will be returned). /// @@ -650,14 +661,19 @@ impl LowLevelN3Reader { /// /// let mut reader = N3Parser::new().parse(); /// reader.extend_from_slice(file); - /// assert!(reader.prefixes().is_empty()); // No prefix at the beginning + /// assert_eq!(reader.prefixes().collect::>(), []); // No prefix at the beginning /// /// reader.read_next().unwrap()?; // We read the first triple - /// assert_eq!(reader.prefixes()["schema"], "http://schema.org/"); // There are now prefixes + /// assert_eq!( + /// reader.prefixes().collect::>(), + /// [("schema", "http://schema.org/")] + /// ); // There are now prefixes /// # Result::<_,Box>::Ok(()) /// ``` - pub fn prefixes(&self) -> &HashMap> { - &self.parser.context.prefixes + pub fn prefixes(&self) -> N3PrefixesIter<'_> { + N3PrefixesIter { + inner: self.parser.context.prefixes.iter(), + } } /// The base IRI considered at the current step of the parsing. @@ -1299,3 +1315,25 @@ enum N3State { FormulaContent, FormulaContentExpectDot, } + +/// Iterator on the file prefixes. +/// +/// See [`LowLevelN3Reader::prefixes`]. +pub struct N3PrefixesIter<'a> { + inner: Iter<'a, String, Iri>, +} + +impl<'a> Iterator for N3PrefixesIter<'a> { + type Item = (&'a str, &'a str); + + #[inline] + fn next(&mut self) -> Option { + let (key, value) = self.inner.next()?; + Some((key.as_str(), value.as_str())) + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + self.inner.size_hint() + } +} diff --git a/lib/oxttl/src/terse.rs b/lib/oxttl/src/terse.rs index 7aa3e103..a8a72c1a 100644 --- a/lib/oxttl/src/terse.rs +++ b/lib/oxttl/src/terse.rs @@ -8,6 +8,7 @@ use oxrdf::vocab::{rdf, xsd}; #[cfg(feature = "rdf-star")] use oxrdf::Triple; use oxrdf::{BlankNode, GraphName, Literal, NamedNode, NamedOrBlankNode, Quad, Subject, Term}; +use std::collections::hash_map::Iter; use std::collections::HashMap; pub struct TriGRecognizer { @@ -24,7 +25,13 @@ pub struct TriGRecognizerContext { pub with_graph_name: bool, #[cfg(feature = "rdf-star")] pub with_quoted_triples: bool, - pub prefixes: HashMap>, + prefixes: HashMap>, +} + +impl TriGRecognizerContext { + pub fn prefixes(&self) -> Iter<'_, String, Iri> { + self.prefixes.iter() + } } impl RuleRecognizer for TriGRecognizer { diff --git a/lib/oxttl/src/trig.rs b/lib/oxttl/src/trig.rs index 5a7cdb4a..77decffd 100644 --- a/lib/oxttl/src/trig.rs +++ b/lib/oxttl/src/trig.rs @@ -8,6 +8,7 @@ use crate::toolkit::{FromReadIterator, ParseError, Parser, SyntaxError}; use oxiri::{Iri, IriParseError}; use oxrdf::vocab::xsd; use oxrdf::{GraphName, NamedNode, Quad, QuadRef, Subject, TermRef}; +use std::collections::hash_map::Iter; use std::collections::HashMap; use std::fmt; use std::io::{self, Read, Write}; @@ -253,7 +254,7 @@ pub struct FromReadTriGReader { impl FromReadTriGReader { /// The list of IRI prefixes considered at the current step of the parsing. /// - /// This method returns the mapping from prefix name to prefix value. + /// This method returns (prefix name, prefix value) tuples. /// It is empty at the beginning of the parsing and gets updated when prefixes are encountered. /// It should be full at the end of the parsing (but if a prefix is overridden, only the latest version will be returned). /// @@ -266,14 +267,19 @@ impl FromReadTriGReader { /// schema:name "Foo" ."#; /// /// let mut reader = TriGParser::new().parse_read(file.as_ref()); - /// assert!(reader.prefixes().is_empty()); // No prefix at the beginning + /// assert_eq!(reader.prefixes().collect::>(), []); // No prefix at the beginning /// /// reader.next().unwrap()?; // We read the first triple - /// assert_eq!(reader.prefixes()["schema"], "http://schema.org/"); // There are now prefixes + /// assert_eq!( + /// reader.prefixes().collect::>(), + /// [("schema", "http://schema.org/")] + /// ); // There are now prefixes /// # Result::<_,Box>::Ok(()) /// ``` - pub fn prefixes(&self) -> &HashMap> { - &self.inner.parser.context.prefixes + pub fn prefixes(&self) -> TriGPrefixesIter<'_> { + TriGPrefixesIter { + inner: self.inner.parser.context.prefixes(), + } } /// The base IRI considered at the current step of the parsing. @@ -357,7 +363,7 @@ impl FromTokioAsyncReadTriGReader { /// The list of IRI prefixes considered at the current step of the parsing. /// - /// This method returns the mapping from prefix name to prefix value. + /// This method returns (prefix name, prefix value) tuples. /// It is empty at the beginning of the parsing and gets updated when prefixes are encountered. /// It should be full at the end of the parsing (but if a prefix is overridden, only the latest version will be returned). /// @@ -372,15 +378,20 @@ impl FromTokioAsyncReadTriGReader { /// schema:name "Foo" ."#; /// /// let mut reader = TriGParser::new().parse_tokio_async_read(file.as_ref()); - /// assert!(reader.prefixes().is_empty()); // No prefix at the beginning + /// assert_eq!(reader.prefixes().collect::>(), []); // No prefix at the beginning /// /// reader.next().await.unwrap()?; // We read the first triple - /// assert_eq!(reader.prefixes()["schema"], "http://schema.org/"); // There are now prefixes + /// assert_eq!( + /// reader.prefixes().collect::>(), + /// [("schema", "http://schema.org/")] + /// ); // There are now prefixes /// # Ok(()) /// # } /// ``` - pub fn prefixes(&self) -> &HashMap> { - &self.inner.parser.context.prefixes + pub fn prefixes(&self) -> TriGPrefixesIter<'_> { + TriGPrefixesIter { + inner: self.inner.parser.context.prefixes(), + } } /// The base IRI considered at the current step of the parsing. @@ -484,7 +495,7 @@ impl LowLevelTriGReader { /// The list of IRI prefixes considered at the current step of the parsing. /// - /// This method returns the mapping from prefix name to prefix value. + /// This method returns (prefix name, prefix value) tuples. /// It is empty at the beginning of the parsing and gets updated when prefixes are encountered. /// It should be full at the end of the parsing (but if a prefix is overridden, only the latest version will be returned). /// @@ -498,14 +509,19 @@ impl LowLevelTriGReader { /// /// let mut reader = TriGParser::new().parse(); /// reader.extend_from_slice(file); - /// assert!(reader.prefixes().is_empty()); // No prefix at the beginning + /// assert_eq!(reader.prefixes().collect::>(), []); // No prefix at the beginning /// /// reader.read_next().unwrap()?; // We read the first triple - /// assert_eq!(reader.prefixes()["schema"], "http://schema.org/"); // There are now prefixes + /// assert_eq!( + /// reader.prefixes().collect::>(), + /// [("schema", "http://schema.org/")] + /// ); // There are now prefixes /// # Result::<_,Box>::Ok(()) /// ``` - pub fn prefixes(&self) -> &HashMap> { - &self.parser.context.prefixes + pub fn prefixes(&self) -> TriGPrefixesIter<'_> { + TriGPrefixesIter { + inner: self.parser.context.prefixes(), + } } /// The base IRI considered at the current step of the parsing. @@ -536,6 +552,28 @@ impl LowLevelTriGReader { } } +/// Iterator on the file prefixes. +/// +/// See [`LowLevelTriGReader::prefixes`]. +pub struct TriGPrefixesIter<'a> { + inner: Iter<'a, String, Iri>, +} + +impl<'a> Iterator for TriGPrefixesIter<'a> { + type Item = (&'a str, &'a str); + + #[inline] + fn next(&mut self) -> Option { + let (key, value) = self.inner.next()?; + Some((key.as_str(), value.as_str())) + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + self.inner.size_hint() + } +} + /// A [TriG](https://www.w3.org/TR/trig/) serializer. /// /// Support for [TriG-star](https://w3c.github.io/rdf-star/cg-spec/2021-12-17.html#trig-star) is available behind the `rdf-star` feature. diff --git a/lib/oxttl/src/turtle.rs b/lib/oxttl/src/turtle.rs index 5a2b67a2..f5193059 100644 --- a/lib/oxttl/src/turtle.rs +++ b/lib/oxttl/src/turtle.rs @@ -10,6 +10,7 @@ use crate::trig::ToTokioAsyncWriteTriGWriter; use crate::trig::{LowLevelTriGWriter, ToWriteTriGWriter, TriGSerializer}; use oxiri::{Iri, IriParseError}; use oxrdf::{GraphNameRef, Triple, TripleRef}; +use std::collections::hash_map::Iter; use std::collections::HashMap; use std::io::{self, Read, Write}; #[cfg(feature = "async-tokio")] @@ -254,7 +255,7 @@ pub struct FromReadTurtleReader { impl FromReadTurtleReader { /// The list of IRI prefixes considered at the current step of the parsing. /// - /// This method returns the mapping from prefix name to prefix value. + /// This method returns (prefix name, prefix value) tuples. /// It is empty at the beginning of the parsing and gets updated when prefixes are encountered. /// It should be full at the end of the parsing (but if a prefix is overridden, only the latest version will be returned). /// @@ -267,14 +268,19 @@ impl FromReadTurtleReader { /// schema:name "Foo" ."#; /// /// let mut reader = TurtleParser::new().parse_read(file.as_ref()); - /// assert!(reader.prefixes().is_empty()); // No prefix at the beginning + /// assert!(reader.prefixes().collect::>().is_empty()); // No prefix at the beginning /// /// reader.next().unwrap()?; // We read the first triple - /// assert_eq!(reader.prefixes()["schema"], "http://schema.org/"); // There are now prefixes + /// assert_eq!( + /// reader.prefixes().collect::>(), + /// [("schema", "http://schema.org/")] + /// ); // There are now prefixes /// # Result::<_,Box>::Ok(()) /// ``` - pub fn prefixes(&self) -> &HashMap> { - &self.inner.parser.context.prefixes + pub fn prefixes(&self) -> TurtlePrefixesIter<'_> { + TurtlePrefixesIter { + inner: self.inner.parser.context.prefixes(), + } } /// The base IRI considered at the current step of the parsing. @@ -358,7 +364,7 @@ impl FromTokioAsyncReadTurtleReader { /// The list of IRI prefixes considered at the current step of the parsing. /// - /// This method returns the mapping from prefix name to prefix value. + /// This method returns (prefix name, prefix value) tuples. /// It is empty at the beginning of the parsing and gets updated when prefixes are encountered. /// It should be full at the end of the parsing (but if a prefix is overridden, only the latest version will be returned). /// @@ -373,15 +379,20 @@ impl FromTokioAsyncReadTurtleReader { /// schema:name "Foo" ."#; /// /// let mut reader = TurtleParser::new().parse_tokio_async_read(file.as_ref()); - /// assert!(reader.prefixes().is_empty()); // No prefix at the beginning + /// assert_eq!(reader.prefixes().collect::>(), []); // No prefix at the beginning /// /// reader.next().await.unwrap()?; // We read the first triple - /// assert_eq!(reader.prefixes()["schema"], "http://schema.org/"); // There are now prefixes + /// assert_eq!( + /// reader.prefixes().collect::>(), + /// [("schema", "http://schema.org/")] + /// ); // There are now prefixes /// # Ok(()) /// # } /// ``` - pub fn prefixes(&self) -> &HashMap> { - &self.inner.parser.context.prefixes + pub fn prefixes(&self) -> TurtlePrefixesIter<'_> { + TurtlePrefixesIter { + inner: self.inner.parser.context.prefixes(), + } } /// The base IRI considered at the current step of the parsing. @@ -485,7 +496,7 @@ impl LowLevelTurtleReader { /// The list of IRI prefixes considered at the current step of the parsing. /// - /// This method returns the mapping from prefix name to prefix value. + /// This method returns (prefix name, prefix value) tuples. /// It is empty at the beginning of the parsing and gets updated when prefixes are encountered. /// It should be full at the end of the parsing (but if a prefix is overridden, only the latest version will be returned). /// @@ -499,14 +510,19 @@ impl LowLevelTurtleReader { /// /// let mut reader = TurtleParser::new().parse(); /// reader.extend_from_slice(file); - /// assert!(reader.prefixes().is_empty()); // No prefix at the beginning + /// assert_eq!(reader.prefixes().collect::>(), []); // No prefix at the beginning /// /// reader.read_next().unwrap()?; // We read the first triple - /// assert_eq!(reader.prefixes()["schema"], "http://schema.org/"); // There are now prefixes + /// assert_eq!( + /// reader.prefixes().collect::>(), + /// [("schema", "http://schema.org/")] + /// ); // There are now prefixes /// # Result::<_,Box>::Ok(()) /// ``` - pub fn prefixes(&self) -> &HashMap> { - &self.parser.context.prefixes + pub fn prefixes(&self) -> TurtlePrefixesIter<'_> { + TurtlePrefixesIter { + inner: self.parser.context.prefixes(), + } } /// The base IRI considered at the current step of the parsing. @@ -537,6 +553,28 @@ impl LowLevelTurtleReader { } } +/// Iterator on the file prefixes. +/// +/// See [`LowLevelTurtleReader::prefixes`]. +pub struct TurtlePrefixesIter<'a> { + inner: Iter<'a, String, Iri>, +} + +impl<'a> Iterator for TurtlePrefixesIter<'a> { + type Item = (&'a str, &'a str); + + #[inline] + fn next(&mut self) -> Option { + let (key, value) = self.inner.next()?; + Some((key.as_str(), value.as_str())) + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + self.inner.size_hint() + } +} + /// A [Turtle](https://www.w3.org/TR/turtle/) serializer. /// /// Support for [Turtle-star](https://w3c.github.io/rdf-star/cg-spec/2021-12-17.html#turtle-star) is available behind the `rdf-star` feature. From 18bf38370190a6aa486da42fc971307859637544 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Fri, 19 Jan 2024 14:58:46 -0500 Subject: [PATCH 190/217] Remove tabs from BNF comments Minor cleanup on the BNF comments - get rid of tabs and align them --- lib/oxsdatatypes/src/duration.rs | 20 +-- lib/oxttl/src/lexer.rs | 43 +++-- lib/oxttl/src/n3.rs | 60 +++---- lib/oxttl/src/terse.rs | 68 ++++---- lib/oxttl/src/trig.rs | 8 +- lib/sparesults/src/csv.rs | 8 +- lib/sparql-smith/src/lib.rs | 290 +++++++++++++++---------------- 7 files changed, 248 insertions(+), 249 deletions(-) diff --git a/lib/oxsdatatypes/src/duration.rs b/lib/oxsdatatypes/src/duration.rs index efb92b62..863fcb38 100644 --- a/lib/oxsdatatypes/src/duration.rs +++ b/lib/oxsdatatypes/src/duration.rs @@ -713,16 +713,16 @@ impl PartialOrd for YearMonthDuration { } } -// [6] duYearFrag ::= unsignedNoDecimalPtNumeral 'Y' -// [7] duMonthFrag ::= unsignedNoDecimalPtNumeral 'M' -// [8] duDayFrag ::= unsignedNoDecimalPtNumeral 'D' -// [9] duHourFrag ::= unsignedNoDecimalPtNumeral 'H' -// [10] duMinuteFrag ::= unsignedNoDecimalPtNumeral 'M' -// [11] duSecondFrag ::= (unsignedNoDecimalPtNumeral | unsignedDecimalPtNumeral) 'S' -// [12] duYearMonthFrag ::= (duYearFrag duMonthFrag?) | duMonthFrag -// [13] duTimeFrag ::= 'T' ((duHourFrag duMinuteFrag? duSecondFrag?) | (duMinuteFrag duSecondFrag?) | duSecondFrag) -// [14] duDayTimeFrag ::= (duDayFrag duTimeFrag?) | duTimeFrag -// [15] durationLexicalRep ::= '-'? 'P' ((duYearMonthFrag duDayTimeFrag?) | duDayTimeFrag) +// [6] duYearFrag ::= unsignedNoDecimalPtNumeral 'Y' +// [7] duMonthFrag ::= unsignedNoDecimalPtNumeral 'M' +// [8] duDayFrag ::= unsignedNoDecimalPtNumeral 'D' +// [9] duHourFrag ::= unsignedNoDecimalPtNumeral 'H' +// [10] duMinuteFrag ::= unsignedNoDecimalPtNumeral 'M' +// [11] duSecondFrag ::= (unsignedNoDecimalPtNumeral | unsignedDecimalPtNumeral) 'S' +// [12] duYearMonthFrag ::= (duYearFrag duMonthFrag?) | duMonthFrag +// [13] duTimeFrag ::= 'T' ((duHourFrag duMinuteFrag? duSecondFrag?) | (duMinuteFrag duSecondFrag?) | duSecondFrag) +// [14] duDayTimeFrag ::= (duDayFrag duTimeFrag?) | duTimeFrag +// [15] durationLexicalRep ::= '-'? 'P' ((duYearMonthFrag duDayTimeFrag?) | duDayTimeFrag) struct DurationParts { year_month: Option, day_time: Option, diff --git a/lib/oxttl/src/lexer.rs b/lib/oxttl/src/lexer.rs index fd8341a2..d11bd2fe 100644 --- a/lib/oxttl/src/lexer.rs +++ b/lib/oxttl/src/lexer.rs @@ -179,7 +179,7 @@ impl N3Lexer { data: &[u8], options: &N3LexerOptions, ) -> Option<(usize, Result, TokenRecognizerError>)> { - // [18] IRIREF ::= '<' ([^#x00-#x20<>"{}|^`\] | UCHAR)* '>' /* #x00=NULL #01-#x1F=control codes #x20=space */ + // [18] IRIREF ::= '<' ([^#x00-#x20<>"{}|^`\] | UCHAR)* '>' /* #x00=NULL #01-#x1F=control codes #x20=space */ let mut string = Vec::new(); let mut i = 1; loop { @@ -238,10 +238,9 @@ impl N3Lexer { data: &'a [u8], is_ending: bool, ) -> Option<(usize, Result, TokenRecognizerError>)> { - // [139s] PNAME_NS ::= PN_PREFIX? ':' - // [140s] PNAME_LN ::= PNAME_NS PN_LOCAL - - // [167s] PN_PREFIX ::= PN_CHARS_BASE ((PN_CHARS | '.')* PN_CHARS)? + // [139s] PNAME_NS ::= PN_PREFIX? ':' + // [140s] PNAME_LN ::= PNAME_NS PN_LOCAL + // [167s] PN_PREFIX ::= PN_CHARS_BASE ((PN_CHARS | '.')* PN_CHARS)? let mut i = 0; loop { if let Some(r) = Self::recognize_unicode_char(&data[i..], i) { @@ -330,7 +329,7 @@ impl N3Lexer { data: &'a [u8], is_ending: bool, ) -> Option<(usize, Result, TokenRecognizerError>)> { - // [36] QUICK_VAR_NAME ::= "?" PN_LOCAL + // [36] QUICK_VAR_NAME ::= "?" PN_LOCAL let (consumed, result) = self.recognize_optional_pn_local(&data[1..], is_ending)?; Some(( consumed + 1, @@ -349,7 +348,7 @@ impl N3Lexer { data: &'a [u8], is_ending: bool, ) -> Option<(usize, Result<(Cow<'a, str>, bool), TokenRecognizerError>)> { - // [168s] PN_LOCAL ::= (PN_CHARS_U | ':' | [0-9] | PLX) ((PN_CHARS | '.' | ':' | PLX)* (PN_CHARS | ':' | PLX))? + // [168s] PN_LOCAL ::= (PN_CHARS_U | ':' | [0-9] | PLX) ((PN_CHARS | '.' | ':' | PLX)* (PN_CHARS | ':' | PLX))? let mut i = 0; let mut buffer = None; // Buffer if there are some escaped characters let mut position_that_is_already_in_buffer = 0; @@ -503,7 +502,7 @@ impl N3Lexer { fn recognize_blank_node_label( data: &[u8], ) -> Option<(usize, Result, TokenRecognizerError>)> { - // [141s] BLANK_NODE_LABEL ::= '_:' (PN_CHARS_U | [0-9]) ((PN_CHARS | '.')* PN_CHARS)? + // [141s] BLANK_NODE_LABEL ::= '_:' (PN_CHARS_U | [0-9]) ((PN_CHARS | '.')* PN_CHARS)? let mut i = 2; loop { match Self::recognize_unicode_char(&data[i..], i)? { @@ -548,7 +547,7 @@ impl N3Lexer { &self, data: &'a [u8], ) -> Option<(usize, Result, TokenRecognizerError>)> { - // [144s] LANGTAG ::= '@' [a-zA-Z]+ ('-' [a-zA-Z0-9]+)* + // [144s] LANGTAG ::= '@' [a-zA-Z]+ ('-' [a-zA-Z0-9]+)* let mut is_last_block_empty = true; for (i, c) in data[1..].iter().enumerate() { if c.is_ascii_alphabetic() { @@ -588,8 +587,8 @@ impl N3Lexer { data: &[u8], delimiter: u8, ) -> Option<(usize, Result, TokenRecognizerError>)> { - // [22] STRING_LITERAL_QUOTE ::= '"' ([^#x22#x5C#xA#xD] | ECHAR | UCHAR)* '"' /* #x22=" #x5C=\ #xA=new line #xD=carriage return */ - // [23] STRING_LITERAL_SINGLE_QUOTE ::= "'" ([^#x27#x5C#xA#xD] | ECHAR | UCHAR)* "'" /* #x27=' #x5C=\ #xA=new line #xD=carriage return */ + // [22] STRING_LITERAL_QUOTE ::= '"' ([^#x22#x5C#xA#xD] | ECHAR | UCHAR)* '"' /* #x22=" #x5C=\ #xA=new line #xD=carriage return */ + // [23] STRING_LITERAL_SINGLE_QUOTE ::= "'" ([^#x27#x5C#xA#xD] | ECHAR | UCHAR)* "'" /* #x27=' #x5C=\ #xA=new line #xD=carriage return */ let mut string = String::new(); let mut i = 1; loop { @@ -626,8 +625,8 @@ impl N3Lexer { data: &[u8], delimiter: u8, ) -> Option<(usize, Result, TokenRecognizerError>)> { - // [24] STRING_LITERAL_LONG_SINGLE_QUOTE ::= "'''" (("'" | "''")? ([^'\] | ECHAR | UCHAR))* "'''" - // [25] STRING_LITERAL_LONG_QUOTE ::= '"""' (('"' | '""')? ([^"\] | ECHAR | UCHAR))* '"""' + // [24] STRING_LITERAL_LONG_SINGLE_QUOTE ::= "'''" (("'" | "''")? ([^'\] | ECHAR | UCHAR))* "'''" + // [25] STRING_LITERAL_LONG_QUOTE ::= '"""' (('"' | '""')? ([^"\] | ECHAR | UCHAR))* '"""' let mut string = String::new(); let mut i = 3; loop { @@ -661,10 +660,10 @@ impl N3Lexer { } fn recognize_number(data: &[u8]) -> Option<(usize, Result, TokenRecognizerError>)> { - // [19] INTEGER ::= [+-]? [0-9]+ - // [20] DECIMAL ::= [+-]? [0-9]* '.' [0-9]+ - // [21] DOUBLE ::= [+-]? ([0-9]+ '.' [0-9]* EXPONENT | '.' [0-9]+ EXPONENT | [0-9]+ EXPONENT) - // [154s] EXPONENT ::= [eE] [+-]? [0-9]+ + // [19] INTEGER ::= [+-]? [0-9]+ + // [20] DECIMAL ::= [+-]? [0-9]* '.' [0-9]+ + // [21] DOUBLE ::= [+-]? ([0-9]+ '.' [0-9]* EXPONENT | '.' [0-9]+ EXPONENT | [0-9]+ EXPONENT) + // [154s] EXPONENT ::= [eE] [+-]? [0-9]+ let mut i = 0; let c = *data.first()?; if matches!(c, b'+' | b'-') { @@ -764,8 +763,8 @@ impl N3Lexer { position: usize, with_echar: bool, ) -> Option<(usize, Result)> { - // [26] UCHAR ::= '\u' HEX HEX HEX HEX | '\U' HEX HEX HEX HEX HEX HEX HEX HEX - // [159s] ECHAR ::= '\' [tbnrf"'\] + // [26] UCHAR ::= '\u' HEX HEX HEX HEX | '\U' HEX HEX HEX HEX HEX HEX HEX HEX + // [159s] ECHAR ::= '\' [tbnrf"'\] match *data.get(1)? { b'u' => match Self::recognize_hex_char(&data[2..], 4, 'u', position) { Ok(c) => Some((5, Ok(c?))), @@ -895,7 +894,7 @@ impl N3Lexer { ) } - // [157s] PN_CHARS_BASE ::= [A-Z] | [a-z] | [#x00C0-#x00D6] | [#x00D8-#x00F6] | [#x00F8-#x02FF] | [#x0370-#x037D] | [#x037F-#x1FFF] | [#x200C-#x200D] | [#x2070-#x218F] | [#x2C00-#x2FEF] | [#x3001-#xD7FF] | [#xF900-#xFDCF] | [#xFDF0-#xFFFD] | [#x10000-#xEFFFF] + // [157s] PN_CHARS_BASE ::= [A-Z] | [a-z] | [#x00C0-#x00D6] | [#x00D8-#x00F6] | [#x00F8-#x02FF] | [#x0370-#x037D] | [#x037F-#x1FFF] | [#x200C-#x200D] | [#x2070-#x218F] | [#x2C00-#x2FEF] | [#x3001-#xD7FF] | [#xF900-#xFDCF] | [#xFDF0-#xFFFD] | [#x10000-#xEFFFF] fn is_possible_pn_chars_base(c: char) -> bool { matches!(c, 'A'..='Z' @@ -914,12 +913,12 @@ impl N3Lexer { | '\u{10000}'..='\u{EFFFF}') } - // [158s] PN_CHARS_U ::= PN_CHARS_BASE | '_' | ':' + // [158s] PN_CHARS_U ::= PN_CHARS_BASE | '_' | ':' fn is_possible_pn_chars_u(c: char) -> bool { Self::is_possible_pn_chars_base(c) || c == '_' } - // [160s] PN_CHARS ::= PN_CHARS_U | '-' | [0-9] | #x00B7 | [#x0300-#x036F] | [#x203F-#x2040] + // [160s] PN_CHARS ::= PN_CHARS_U | '-' | [0-9] | #x00B7 | [#x0300-#x036F] | [#x203F-#x2040] fn is_possible_pn_chars(c: char) -> bool { Self::is_possible_pn_chars_u(c) || matches!(c, diff --git a/lib/oxttl/src/n3.rs b/lib/oxttl/src/n3.rs index 233edccc..8b70a01e 100644 --- a/lib/oxttl/src/n3.rs +++ b/lib/oxttl/src/n3.rs @@ -744,14 +744,14 @@ impl RuleRecognizer for N3Recognizer { ) -> Self { while let Some(rule) = self.stack.pop() { match rule { - // [1] n3Doc ::= ( ( n3Statement ".") | sparqlDirective) * - // [2] n3Statement ::= n3Directive | triples - // [3] n3Directive ::= prefixID | base - // [4] sparqlDirective ::= sparqlBase | sparqlPrefix - // [5] sparqlBase ::= BASE IRIREF - // [6] sparqlPrefix ::= PREFIX PNAME_NS IRIREF - // [7] prefixID ::= "@prefix" PNAME_NS IRIREF - // [8] base ::= "@base" IRIREF + // [1] n3Doc ::= ( ( n3Statement ".") | sparqlDirective) * + // [2] n3Statement ::= n3Directive | triples + // [3] n3Directive ::= prefixID | base + // [4] sparqlDirective ::= sparqlBase | sparqlPrefix + // [5] sparqlBase ::= BASE IRIREF + // [6] sparqlPrefix ::= PREFIX PNAME_NS IRIREF + // [7] prefixID ::= "@prefix" PNAME_NS IRIREF + // [8] base ::= "@base" IRIREF N3State::N3Doc => { self.stack.push(N3State::N3Doc); match token { @@ -818,7 +818,7 @@ impl RuleRecognizer for N3Recognizer { } _ => self.error(errors, "The PREFIX declaration should be followed by a prefix and its value as an IRI"), }, - // [9] triples ::= subject predicateObjectList? + // [9] triples ::= subject predicateObjectList? N3State::Triples => { self.stack.push(N3State::TriplesMiddle); self.stack.push(N3State::Path); @@ -830,7 +830,7 @@ impl RuleRecognizer for N3Recognizer { N3State::TriplesEnd => { self.terms.pop(); } - // [10] predicateObjectList ::= verb objectList ( ";" ( verb objectList) ? ) * + // [10] predicateObjectList ::= verb objectList ( ";" ( verb objectList) ? ) * N3State::PredicateObjectList => { self.stack.push(N3State::PredicateObjectListEnd); self.stack.push(N3State::ObjectsList); @@ -851,7 +851,7 @@ impl RuleRecognizer for N3Recognizer { self.stack.push(N3State::ObjectsList); self.stack.push(N3State::Verb); }, - // [11] objectList ::= object ( "," object) * + // [11] objectList ::= object ( "," object) * N3State::ObjectsList => { self.stack.push(N3State::ObjectsListEnd); self.stack.push(N3State::Path); @@ -877,8 +877,8 @@ impl RuleRecognizer for N3Recognizer { return self; } } - // [12] verb ::= predicate | "a" | ( "has" expression) | ( "is" expression "of") | "=" | "<=" | "=>" - // [14] predicate ::= expression | ( "<-" expression) + // [12] verb ::= predicate | "a" | ( "has" expression) | ( "is" expression "of") | "=" | "<=" | "=>" + // [14] predicate ::= expression | ( "<-" expression) N3State::Verb => match token { N3Token::PlainKeyword("a") => { self.predicates.push(Predicate::Regular(rdf::TYPE.into())); @@ -931,10 +931,10 @@ impl RuleRecognizer for N3Recognizer { self.error(errors, "The keyword 'is' should be followed by a predicate then by the keyword 'of'") } }, - // [13] subject ::= expression - // [15] object ::= expression - // [16] expression ::= path - // [17] path ::= pathItem ( ( "!" path) | ( "^" path) ) ? + // [13] subject ::= expression + // [15] object ::= expression + // [16] expression ::= path + // [17] path ::= pathItem ( ( "!" path) | ( "^" path) ) ? N3State::Path => { self.stack.push(N3State::PathFollowUp); self.stack.push(N3State::PathItem); @@ -960,18 +960,18 @@ impl RuleRecognizer for N3Recognizer { self.terms.push(current.into()); self.stack.push(N3State::PathFollowUp); } - // [18] pathItem ::= iri | blankNode | quickVar | collection | blankNodePropertyList | iriPropertyList | literal | formula - // [19] literal ::= rdfLiteral | numericLiteral | BOOLEAN_LITERAL - // [20] blankNodePropertyList ::= "[" predicateObjectList "]" - // [21] iriPropertyList ::= IPLSTART iri predicateObjectList "]" - // [22] collection ::= "(" object* ")" - // [23] formula ::= "{" formulaContent? "}" - // [25] numericLiteral ::= DOUBLE | DECIMAL | INTEGER - // [26] rdfLiteral ::= STRING ( LANGTAG | ( "^^" iri) ) ? - // [27] iri ::= IRIREF | prefixedName - // [28] prefixedName ::= PNAME_LN | PNAME_NS - // [29] blankNode ::= BLANK_NODE_LABEL | ANON - // [30] quickVar ::= QUICK_VAR_NAME + // [18] pathItem ::= iri | blankNode | quickVar | collection | blankNodePropertyList | iriPropertyList | literal | formula + // [19] literal ::= rdfLiteral | numericLiteral | BOOLEAN_LITERAL + // [20] blankNodePropertyList ::= "[" predicateObjectList "]" + // [21] iriPropertyList ::= IPLSTART iri predicateObjectList "]" + // [22] collection ::= "(" object* ")" + // [23] formula ::= "{" formulaContent? "}" + // [25] numericLiteral ::= DOUBLE | DECIMAL | INTEGER + // [26] rdfLiteral ::= STRING ( LANGTAG | ( "^^" iri) ) ? + // [27] iri ::= IRIREF | prefixedName + // [28] prefixedName ::= PNAME_LN | PNAME_NS + // [29] blankNode ::= BLANK_NODE_LABEL | ANON + // [30] quickVar ::= QUICK_VAR_NAME N3State::PathItem => { return match token { N3Token::IriRef(iri) => { @@ -1149,7 +1149,7 @@ impl RuleRecognizer for N3Recognizer { } } } - // [24] formulaContent ::= ( n3Statement ( "." formulaContent? ) ? ) | ( sparqlDirective formulaContent? ) + // [24] formulaContent ::= ( n3Statement ( "." formulaContent? ) ? ) | ( sparqlDirective formulaContent? ) N3State::FormulaContent => { match token { N3Token::Punctuation("}") => { diff --git a/lib/oxttl/src/terse.rs b/lib/oxttl/src/terse.rs index a8a72c1a..c233c735 100644 --- a/lib/oxttl/src/terse.rs +++ b/lib/oxttl/src/terse.rs @@ -57,13 +57,13 @@ impl RuleRecognizer for TriGRecognizer { ) -> Self { if let Some(rule) = self.stack.pop() { match rule { - // [1g] trigDoc ::= (directive | block)* - // [2g] block ::= triplesOrGraph | wrappedGraph | triples2 | "GRAPH" labelOrSubject wrappedGraph - // [3] directive ::= prefixID | base | sparqlPrefix | sparqlBase - // [4] prefixID ::= '@prefix' PNAME_NS IRIREF '.' - // [5] base ::= '@base' IRIREF '.' - // [5s] sparqlPrefix ::= "PREFIX" PNAME_NS IRIREF - // [6s] sparqlBase ::= "BASE" IRIREF + // [1g] trigDoc ::= (directive | block)* + // [2g] block ::= triplesOrGraph | wrappedGraph | triples2 | "GRAPH" labelOrSubject wrappedGraph + // [3] directive ::= prefixID | base | sparqlPrefix | sparqlBase + // [4] prefixID ::= '@prefix' PNAME_NS IRIREF '.' + // [5] base ::= '@base' IRIREF '.' + // [5s] sparqlPrefix ::= "PREFIX" PNAME_NS IRIREF + // [6s] sparqlBase ::= "BASE" IRIREF TriGState::TriGDoc => { self.cur_graph = GraphName::DefaultGraph; self.stack.push(TriGState::TriGDoc); @@ -142,8 +142,8 @@ impl RuleRecognizer for TriGRecognizer { } } _ => self.error(errors, "The PREFIX declaration should be followed by a prefix and its value as an IRI"), }, - // [3g] triplesOrGraph ::= labelOrSubject ( wrappedGraph | predicateObjectList '.' ) | quotedTriple predicateObjectList '.' - // [4g] triples2 ::= blankNodePropertyList predicateObjectList? '.' | collection predicateObjectList '.' + // [3g] triplesOrGraph ::= labelOrSubject ( wrappedGraph | predicateObjectList '.' ) | quotedTriple predicateObjectList '.' + // [4g] triples2 ::= blankNodePropertyList predicateObjectList? '.' | collection predicateObjectList '.' TriGState::TriplesOrGraph => match token { N3Token::IriRef(iri) => { self.stack.push(TriGState::WrappedGraphOrPredicateObjectList { @@ -267,8 +267,8 @@ impl RuleRecognizer for TriGRecognizer { self.recognize_next(token, context,results, errors) } } - // [5g] wrappedGraph ::= '{' triplesBlock? '}' - // [6g] triplesBlock ::= triples ('.' triplesBlock?)? + // [5g] wrappedGraph ::= '{' triplesBlock? '}' + // [6g] triplesBlock ::= triples ('.' triplesBlock?)? TriGState::WrappedGraph => if token == N3Token::Punctuation("{") { self.stack.push(TriGState::WrappedGraphPossibleEnd); self.stack.push(TriGState::Triples); @@ -293,8 +293,8 @@ impl RuleRecognizer for TriGRecognizer { } } } - // [6] triples ::= subject predicateObjectList | blankNodePropertyList predicateObjectList? - // [10] subject ::= iri | BlankNode | collection | quotedTriple + // [6] triples ::= subject predicateObjectList | blankNodePropertyList predicateObjectList? + // [10] subject ::= iri | BlankNode | collection | quotedTriple TriGState::Triples => match token { N3Token::Punctuation("}") => { self.recognize_next(token, context,results, errors) // Early end @@ -348,7 +348,7 @@ impl RuleRecognizer for TriGRecognizer { self.stack.push(TriGState::PredicateObjectList); self.recognize_next(token, context,results, errors) } - // [7g] labelOrSubject ::= iri | BlankNode + // [7g] labelOrSubject ::= iri | BlankNode TriGState::GraphName => match token { N3Token::IriRef(iri) => { self.cur_graph = NamedNode::new_unchecked(iri).into(); @@ -379,7 +379,7 @@ impl RuleRecognizer for TriGRecognizer { } else { self.error(errors, "Anonymous blank node with a property list are not allowed as graph name") } - // [7] predicateObjectList ::= verb objectList (';' (verb objectList)?)* + // [7] predicateObjectList ::= verb objectList (';' (verb objectList)?)* TriGState::PredicateObjectList => { self.stack.push(TriGState::PredicateObjectListEnd); self.stack.push(TriGState::ObjectsList); @@ -406,8 +406,8 @@ impl RuleRecognizer for TriGRecognizer { self.stack.push(TriGState::Verb); self.recognize_next(token, context,results, errors) }, - // [8] objectList ::= object annotation? ( ',' object annotation? )* - // [30t] annotation ::= '{|' predicateObjectList '|}' + // [8] objectList ::= object annotation? ( ',' object annotation? )* + // [30t] annotation ::= '{|' predicateObjectList '|}' TriGState::ObjectsList => { self.stack.push(TriGState::ObjectsListEnd); self.stack.push(TriGState::Object); @@ -457,8 +457,8 @@ impl RuleRecognizer for TriGRecognizer { } else { self.recognize_next(token, context,results, errors) }, - // [9] verb ::= predicate | 'a' - // [11] predicate ::= iri + // [9] verb ::= predicate | 'a' + // [11] predicate ::= iri TriGState::Verb => match token { N3Token::PlainKeyword("a") => { self.cur_predicate.push(rdf::TYPE.into()); @@ -479,18 +479,18 @@ impl RuleRecognizer for TriGRecognizer { self.error(errors, "TOKEN is not a valid predicate") } } - // [12] object ::= iri | BlankNode | collection | blankNodePropertyList | literal | quotedTriple - // [13] literal ::= RDFLiteral | NumericLiteral | BooleanLiteral - // [14] blank ::= BlankNode | collection - // [15] blankNodePropertyList ::= '[' predicateObjectList ']' - // [16] collection ::= '(' object* ')' - // [17] NumericLiteral ::= INTEGER | DECIMAL | DOUBLE - // [128s] RDFLiteral ::= String (LANGTAG | '^^' iri)? - // [133s] BooleanLiteral ::= 'true' | 'false' - // [18] String ::= STRING_LITERAL_QUOTE | STRING_LITERAL_SINGLE_QUOTE | STRING_LITERAL_LONG_SINGLE_QUOTE | STRING_LITERAL_LONG_QUOTE - // [135s] iri ::= IRIREF | PrefixedName - // [136s] PrefixedName ::= PNAME_LN | PNAME_NS - // [137s] BlankNode ::= BLANK_NODE_LABEL | ANON + // [12] object ::= iri | BlankNode | collection | blankNodePropertyList | literal | quotedTriple + // [13] literal ::= RDFLiteral | NumericLiteral | BooleanLiteral + // [14] blank ::= BlankNode | collection + // [15] blankNodePropertyList ::= '[' predicateObjectList ']' + // [16] collection ::= '(' object* ')' + // [17] NumericLiteral ::= INTEGER | DECIMAL | DOUBLE + // [128s] RDFLiteral ::= String (LANGTAG | '^^' iri)? + // [133s] BooleanLiteral ::= 'true' | 'false' + // [18] String ::= STRING_LITERAL_QUOTE | STRING_LITERAL_SINGLE_QUOTE | STRING_LITERAL_LONG_SINGLE_QUOTE | STRING_LITERAL_LONG_QUOTE + // [135s] iri ::= IRIREF | PrefixedName + // [136s] PrefixedName ::= PNAME_LN | PNAME_NS + // [137s] BlankNode ::= BLANK_NODE_LABEL | ANON TriGState::Object => match token { N3Token::IriRef(iri) => { self.cur_object.push(NamedNode::new_unchecked(iri).into()); @@ -661,7 +661,7 @@ impl RuleRecognizer for TriGRecognizer { } } } - // [27t] quotedTriple ::= '<<' qtSubject verb qtObject '>>' + // [27t] quotedTriple ::= '<<' qtSubject verb qtObject '>>' #[cfg(feature = "rdf-star")] TriGState::SubjectQuotedTripleEnd => { let triple = Triple::new( @@ -693,7 +693,7 @@ impl RuleRecognizer for TriGRecognizer { self.error(errors, "Expecting '>>' to close a quoted triple, found TOKEN") } } - // [28t] qtSubject ::= iri | BlankNode | quotedTriple + // [28t] qtSubject ::= iri | BlankNode | quotedTriple #[cfg(feature = "rdf-star")] TriGState::QuotedSubject => match token { N3Token::Punctuation("[") => { @@ -725,7 +725,7 @@ impl RuleRecognizer for TriGRecognizer { } _ => self.error(errors, "TOKEN is not a valid RDF quoted triple subject: TOKEN") } - // [29t] qtObject ::= iri | BlankNode | literal | quotedTriple + // [29t] qtObject ::= iri | BlankNode | literal | quotedTriple #[cfg(feature = "rdf-star")] TriGState::QuotedObject => match token { N3Token::Punctuation("[") => { diff --git a/lib/oxttl/src/trig.rs b/lib/oxttl/src/trig.rs index 77decffd..a27dd24d 100644 --- a/lib/oxttl/src/trig.rs +++ b/lib/oxttl/src/trig.rs @@ -940,7 +940,7 @@ fn is_turtle_boolean(value: &str) -> bool { } fn is_turtle_integer(value: &str) -> bool { - // [19] INTEGER ::= [+-]? [0-9]+ + // [19] INTEGER ::= [+-]? [0-9]+ let mut value = value.as_bytes(); if let Some(v) = value.strip_prefix(b"+") { value = v; @@ -951,7 +951,7 @@ fn is_turtle_integer(value: &str) -> bool { } fn is_turtle_decimal(value: &str) -> bool { - // [20] DECIMAL ::= [+-]? [0-9]* '.' [0-9]+ + // [20] DECIMAL ::= [+-]? [0-9]* '.' [0-9]+ let mut value = value.as_bytes(); if let Some(v) = value.strip_prefix(b"+") { value = v; @@ -968,8 +968,8 @@ fn is_turtle_decimal(value: &str) -> bool { } fn is_turtle_double(value: &str) -> bool { - // [21] DOUBLE ::= [+-]? ([0-9]+ '.' [0-9]* EXPONENT | '.' [0-9]+ EXPONENT | [0-9]+ EXPONENT) - // [154s] EXPONENT ::= [eE] [+-]? [0-9]+ + // [21] DOUBLE ::= [+-]? ([0-9]+ '.' [0-9]* EXPONENT | '.' [0-9]+ EXPONENT | [0-9]+ EXPONENT) + // [154s] EXPONENT ::= [eE] [+-]? [0-9]+ let mut value = value.as_bytes(); if let Some(v) = value.strip_prefix(b"+") { value = v; diff --git a/lib/sparesults/src/csv.rs b/lib/sparesults/src/csv.rs index 7cf6059e..bd2fe4b1 100644 --- a/lib/sparesults/src/csv.rs +++ b/lib/sparesults/src/csv.rs @@ -359,7 +359,7 @@ fn is_turtle_boolean(value: &str) -> bool { } fn is_turtle_integer(value: &str) -> bool { - // [19] INTEGER ::= [+-]? [0-9]+ + // [19] INTEGER ::= [+-]? [0-9]+ let mut value = value.as_bytes(); if let Some(v) = value.strip_prefix(b"+") { value = v; @@ -370,7 +370,7 @@ fn is_turtle_integer(value: &str) -> bool { } fn is_turtle_decimal(value: &str) -> bool { - // [20] DECIMAL ::= [+-]? [0-9]* '.' [0-9]+ + // [20] DECIMAL ::= [+-]? [0-9]* '.' [0-9]+ let mut value = value.as_bytes(); if let Some(v) = value.strip_prefix(b"+") { value = v; @@ -387,8 +387,8 @@ fn is_turtle_decimal(value: &str) -> bool { } fn is_turtle_double(value: &str) -> bool { - // [21] DOUBLE ::= [+-]? ([0-9]+ '.' [0-9]* EXPONENT | '.' [0-9]+ EXPONENT | [0-9]+ EXPONENT) - // [154s] EXPONENT ::= [eE] [+-]? [0-9]+ + // [21] DOUBLE ::= [+-]? ([0-9]+ '.' [0-9]* EXPONENT | '.' [0-9]+ EXPONENT | [0-9]+ EXPONENT) + // [154s] EXPONENT ::= [eE] [+-]? [0-9]+ let mut value = value.as_bytes(); if let Some(v) = value.strip_prefix(b"+") { value = v; diff --git a/lib/sparql-smith/src/lib.rs b/lib/sparql-smith/src/lib.rs index 3bc99c0b..a57fe006 100644 --- a/lib/sparql-smith/src/lib.rs +++ b/lib/sparql-smith/src/lib.rs @@ -35,8 +35,8 @@ pub struct Query { #[derive(Arbitrary)] struct QueryContent { - // [1] QueryUnit ::= Query - // [2] Query ::= Prologue ( SelectQuery | ConstructQuery | DescribeQuery | AskQuery ) ValuesClause + // [1] QueryUnit ::= Query + // [2] Query ::= Prologue ( SelectQuery | ConstructQuery | DescribeQuery | AskQuery ) ValuesClause variant: QueryVariant, values_clause: ValuesClause, } @@ -82,7 +82,7 @@ impl fmt::Debug for Query { #[derive(Arbitrary)] struct SelectQuery { - // [7] SelectQuery ::= SelectClause DatasetClause* WhereClause SolutionModifier + // [7] SelectQuery ::= SelectClause DatasetClause* WhereClause SolutionModifier select_clause: SelectClause, where_clause: WhereClause, solution_modifier: SolutionModifier, @@ -100,7 +100,7 @@ impl fmt::Display for SelectQuery { #[derive(Arbitrary)] struct SubSelect { - // [8] SubSelect ::= SelectClause WhereClause SolutionModifier ValuesClause + // [8] SubSelect ::= SelectClause WhereClause SolutionModifier ValuesClause select_clause: SelectClause, where_clause: WhereClause, solution_modifier: SolutionModifier, @@ -119,7 +119,7 @@ impl fmt::Display for SubSelect { #[derive(Arbitrary)] struct SelectClause { - // [9] SelectClause ::= 'SELECT' ( 'DISTINCT' | 'REDUCED' )? ( ( Var | ( '(' Expression 'AS' Var ')' ) )+ | '*' ) + // [9] SelectClause ::= 'SELECT' ( 'DISTINCT' | 'REDUCED' )? ( ( Var | ( '(' Expression 'AS' Var ')' ) )+ | '*' ) option: Option, values: SelectValues, } @@ -171,7 +171,7 @@ impl fmt::Display for SelectClause { #[derive(Arbitrary)] struct WhereClause { - // [17] WhereClause ::= 'WHERE'? GroupGraphPattern + // [17] WhereClause ::= 'WHERE'? GroupGraphPattern with_where: bool, group_graph_pattern: GroupGraphPattern, } @@ -187,7 +187,7 @@ impl fmt::Display for WhereClause { #[derive(Arbitrary)] struct SolutionModifier { - // [18] SolutionModifier ::= GroupClause? HavingClause? OrderClause? LimitOffsetClauses? + // [18] SolutionModifier ::= GroupClause? HavingClause? OrderClause? LimitOffsetClauses? group: Option, having: Option, #[cfg(feature = "order")] @@ -218,7 +218,7 @@ impl fmt::Display for SolutionModifier { #[derive(Arbitrary)] struct GroupClause { - // [19] GroupClause ::= 'GROUP' 'BY' GroupCondition+ + // [19] GroupClause ::= 'GROUP' 'BY' GroupCondition+ start: GroupCondition, others: Vec, } @@ -235,7 +235,7 @@ impl fmt::Display for GroupClause { #[derive(Arbitrary)] enum GroupCondition { - // [20] GroupCondition ::= BuiltInCall | FunctionCall | '(' Expression ( 'AS' Var )? ')' | Var + // [20] GroupCondition ::= BuiltInCall | FunctionCall | '(' Expression ( 'AS' Var )? ')' | Var BuiltInCall(BuiltInCall), // TODO FunctionCall(FunctionCall) Projection(Expression, Option), @@ -261,7 +261,7 @@ impl fmt::Display for GroupCondition { #[derive(Arbitrary)] struct HavingClause { - // [21] HavingClause ::= 'HAVING' HavingCondition+ + // [21] HavingClause ::= 'HAVING' HavingCondition+ start: HavingCondition, others: Vec, } @@ -276,13 +276,13 @@ impl fmt::Display for HavingClause { } } -// [22] HavingCondition ::= Constraint +// [22] HavingCondition ::= Constraint type HavingCondition = Constraint; #[cfg(feature = "order")] #[derive(Arbitrary)] struct OrderClause { - // [23] OrderClause ::= 'ORDER' 'BY' OrderCondition+ + // [23] OrderClause ::= 'ORDER' 'BY' OrderCondition+ start: OrderCondition, others: Vec, } @@ -301,7 +301,7 @@ impl fmt::Display for OrderClause { #[cfg(feature = "order")] #[derive(Arbitrary)] enum OrderCondition { - // [24] OrderCondition ::= ( ( 'ASC' | 'DESC' ) BrackettedExpression ) | ( Constraint | Var ) + // [24] OrderCondition ::= ( ( 'ASC' | 'DESC' ) BrackettedExpression ) | ( Constraint | Var ) BrackettedExpression { is_asc: bool, inner: BrackettedExpression, @@ -330,7 +330,7 @@ impl fmt::Display for OrderCondition { #[cfg(feature = "limit-offset")] #[derive(Arbitrary)] enum LimitOffsetClauses { - // [25] LimitOffsetClauses ::= LimitClause OffsetClause? | OffsetClause LimitClause? + // [25] LimitOffsetClauses ::= LimitClause OffsetClause? | OffsetClause LimitClause? LimitOffset(LimitClause, Option), OffsetLimit(OffsetClause, Option), } @@ -350,7 +350,7 @@ impl fmt::Display for LimitOffsetClauses { #[cfg(feature = "limit-offset")] #[derive(Arbitrary)] struct LimitClause { - // [26] LimitClause ::= 'LIMIT' INTEGER + // [26] LimitClause ::= 'LIMIT' INTEGER value: u8, } @@ -364,7 +364,7 @@ impl fmt::Display for LimitClause { #[cfg(feature = "limit-offset")] #[derive(Arbitrary)] struct OffsetClause { - // [27] OffsetClause ::= 'OFFSET' INTEGER + // [27] OffsetClause ::= 'OFFSET' INTEGER value: u8, } @@ -377,7 +377,7 @@ impl fmt::Display for OffsetClause { #[derive(Arbitrary)] struct ValuesClause { - // [28] ValuesClause ::= ( 'VALUES' DataBlock )? + // [28] ValuesClause ::= ( 'VALUES' DataBlock )? value: Option, } @@ -393,7 +393,7 @@ impl fmt::Display for ValuesClause { #[derive(Arbitrary)] enum GroupGraphPattern { - // [53] GroupGraphPattern ::= '{' ( SubSelect | GroupGraphPatternSub ) '}' + // [53] GroupGraphPattern ::= '{' ( SubSelect | GroupGraphPatternSub ) '}' GroupGraphPatternSub(GroupGraphPatternSub), SubSelect(Box), } @@ -411,7 +411,7 @@ impl fmt::Display for GroupGraphPattern { #[derive(Arbitrary)] struct GroupGraphPatternSub { - // [54] GroupGraphPatternSub ::= TriplesBlock? ( GraphPatternNotTriples '.'? TriplesBlock? )* + // [54] GroupGraphPatternSub ::= TriplesBlock? ( GraphPatternNotTriples '.'? TriplesBlock? )* start: Option, others: Vec, } @@ -443,7 +443,7 @@ impl fmt::Display for GroupGraphPatternSub { #[derive(Arbitrary)] struct TriplesBlock { - // [55] TriplesBlock ::= TriplesSameSubjectPath ( '.' TriplesBlock? )? + // [55] TriplesBlock ::= TriplesSameSubjectPath ( '.' TriplesBlock? )? start: TriplesSameSubjectPath, end: Option>>, } @@ -463,7 +463,7 @@ impl fmt::Display for TriplesBlock { #[derive(Arbitrary)] enum GraphPatternNotTriples { - // [56] GraphPatternNotTriples ::= GroupOrUnionGraphPattern | OptionalGraphPattern | MinusGraphPattern | GraphGraphPattern | ServiceGraphPattern | Filter | Bind | InlineData + // [56] GraphPatternNotTriples ::= GroupOrUnionGraphPattern | OptionalGraphPattern | MinusGraphPattern | GraphGraphPattern | ServiceGraphPattern | Filter | Bind | InlineData GroupOrUnion(GroupOrUnionGraphPattern), Optional(OptionalGraphPattern), Minus(MinusGraphPattern), @@ -493,7 +493,7 @@ impl fmt::Display for GraphPatternNotTriples { #[derive(Arbitrary)] struct OptionalGraphPattern { - // [57] OptionalGraphPattern ::= 'OPTIONAL' GroupGraphPattern + // [57] OptionalGraphPattern ::= 'OPTIONAL' GroupGraphPattern inner: GroupGraphPattern, } @@ -505,7 +505,7 @@ impl fmt::Display for OptionalGraphPattern { #[derive(Arbitrary)] struct LateralGraphPattern { - // [] LateralGraphPattern ::= 'LATERAL' GroupGraphPattern + // [] LateralGraphPattern ::= 'LATERAL' GroupGraphPattern inner: GroupGraphPattern, } @@ -517,7 +517,7 @@ impl fmt::Display for LateralGraphPattern { #[derive(Arbitrary)] struct GraphGraphPattern { - // [58] GraphGraphPattern ::= 'GRAPH' VarOrIri GroupGraphPattern + // [58] GraphGraphPattern ::= 'GRAPH' VarOrIri GroupGraphPattern graph: VarOrIri, inner: GroupGraphPattern, } @@ -530,7 +530,7 @@ impl fmt::Display for GraphGraphPattern { #[derive(Arbitrary)] struct Bind { - // [60] Bind ::= 'BIND' '(' Expression 'AS' Var ')' + // [60] Bind ::= 'BIND' '(' Expression 'AS' Var ')' expression: Expression, var: Var, } @@ -543,7 +543,7 @@ impl fmt::Display for Bind { #[derive(Arbitrary)] struct InlineData { - // [61] InlineData ::= 'VALUES' DataBlock + // [61] InlineData ::= 'VALUES' DataBlock inner: DataBlock, } @@ -555,7 +555,7 @@ impl fmt::Display for InlineData { #[derive(Arbitrary)] enum DataBlock { - // [62] DataBlock ::= InlineDataOneVar | InlineDataFull + // [62] DataBlock ::= InlineDataOneVar | InlineDataFull OneVar(InlineDataOneVar), Full(InlineDataFull), } @@ -571,7 +571,7 @@ impl fmt::Display for DataBlock { #[derive(Arbitrary)] struct InlineDataOneVar { - // [63] InlineDataOneVar ::= Var '{' DataBlockValue* '}' + // [63] InlineDataOneVar ::= Var '{' DataBlockValue* '}' var: Var, values: Vec, } @@ -587,7 +587,7 @@ impl fmt::Display for InlineDataOneVar { } struct InlineDataFull { - // [64] InlineDataFull ::= ( NIL | '(' Var* ')' ) '{' ( '(' DataBlockValue* ')' | NIL )* '}' + // [64] InlineDataFull ::= ( NIL | '(' Var* ')' ) '{' ( '(' DataBlockValue* ')' | NIL )* '}' vars: Vec, values: Vec>, } @@ -635,7 +635,7 @@ impl fmt::Display for InlineDataFull { #[derive(Arbitrary)] enum DataBlockValue { - // [65] DataBlockValue ::= iri | RDFLiteral | NumericLiteral | BooleanLiteral | 'UNDEF' + // [65] DataBlockValue ::= iri | RDFLiteral | NumericLiteral | BooleanLiteral | 'UNDEF' Iri(Iri), Literal(Literal), Undef, @@ -653,7 +653,7 @@ impl fmt::Display for DataBlockValue { #[derive(Arbitrary)] struct MinusGraphPattern { - // [66] MinusGraphPattern ::= 'MINUS' GroupGraphPattern + // [66] MinusGraphPattern ::= 'MINUS' GroupGraphPattern inner: GroupGraphPattern, } @@ -665,7 +665,7 @@ impl fmt::Display for MinusGraphPattern { #[derive(Arbitrary)] struct GroupOrUnionGraphPattern { - // [67] GroupOrUnionGraphPattern ::= GroupGraphPattern ( 'UNION' GroupGraphPattern )* + // [67] GroupOrUnionGraphPattern ::= GroupGraphPattern ( 'UNION' GroupGraphPattern )* start: GroupGraphPattern, others: Vec, } @@ -682,7 +682,7 @@ impl fmt::Display for GroupOrUnionGraphPattern { #[derive(Arbitrary)] struct Filter { - // [68] Filter ::= 'FILTER' Constraint + // [68] Filter ::= 'FILTER' Constraint constraint: Constraint, } @@ -694,7 +694,7 @@ impl fmt::Display for Filter { #[derive(Arbitrary)] enum Constraint { - // [69] Constraint ::= BrackettedExpression | BuiltInCall | FunctionCall + // [69] Constraint ::= BrackettedExpression | BuiltInCall | FunctionCall BrackettedExpression(BrackettedExpression), BuiltInCall(BuiltInCall), // TODO FunctionCall(FunctionCall), @@ -712,7 +712,7 @@ impl fmt::Display for Constraint { #[derive(Arbitrary)] struct FunctionCall { - // [70] FunctionCall ::= iri ArgList + // [70] FunctionCall ::= iri ArgList iri: Iri, args: ArgList, } @@ -725,7 +725,7 @@ impl fmt::Display for FunctionCall { #[derive(Arbitrary)] enum ArgList { - // [71] ArgList ::= NIL | '(' 'DISTINCT'? Expression ( ',' Expression )* ')' + // [71] ArgList ::= NIL | '(' 'DISTINCT'? Expression ( ',' Expression )* ')' Nil, NotNil { // TODO: DISTINCT @@ -749,7 +749,7 @@ impl fmt::Display for ArgList { #[derive(Arbitrary)] struct ExpressionList { - // [72] ExpressionList ::= NIL | '(' Expression ( ',' Expression )* ')' + // [72] ExpressionList ::= NIL | '(' Expression ( ',' Expression )* ')' inner: Vec, } @@ -768,7 +768,7 @@ impl fmt::Display for ExpressionList { #[derive(Arbitrary)] struct PropertyListNotEmpty { - // [77] PropertyListNotEmpty ::= Verb ObjectList ( ';' ( Verb ObjectList )? )* + // [77] PropertyListNotEmpty ::= Verb ObjectList ( ';' ( Verb ObjectList )? )* start_predicate: Verb, start_object: Box, others: Vec>, @@ -795,7 +795,7 @@ impl fmt::Display for PropertyListNotEmpty { #[derive(Arbitrary)] enum Verb { - // [78] Verb ::= VarOrIri | 'a' + // [78] Verb ::= VarOrIri | 'a' VarOrIri(VarOrIri), A, } @@ -811,7 +811,7 @@ impl fmt::Display for Verb { #[derive(Arbitrary)] struct ObjectList { - // [79] ObjectList ::= Object ( ',' Object )* + // [79] ObjectList ::= Object ( ',' Object )* start: Object, others: Vec, } @@ -827,12 +827,12 @@ impl fmt::Display for ObjectList { } } -// [80] Object ::= GraphNode +// [80] Object ::= GraphNode type Object = GraphNode; #[derive(Arbitrary)] enum TriplesSameSubjectPath { - // [81] TriplesSameSubjectPath ::= VarOrTerm PropertyListPathNotEmpty | TriplesNodePath PropertyListPath + // [81] TriplesSameSubjectPath ::= VarOrTerm PropertyListPathNotEmpty | TriplesNodePath PropertyListPath Atomic { subject: VarOrTerm, predicate_object: PropertyListPathNotEmpty, @@ -864,7 +864,7 @@ impl fmt::Display for TriplesSameSubjectPath { #[derive(Arbitrary)] struct PropertyListPath { - // [82] PropertyListPath ::= PropertyListPathNotEmpty? + // [82] PropertyListPath ::= PropertyListPathNotEmpty? inner: Option, } @@ -880,7 +880,7 @@ impl fmt::Display for PropertyListPath { #[derive(Arbitrary)] struct PropertyListPathNotEmpty { - // [83] PropertyListPathNotEmpty ::= ( VerbPath | VerbSimple ) ObjectListPath ( ';' ( ( VerbPath | VerbSimple ) ObjectListPath )? )* + // [83] PropertyListPathNotEmpty ::= ( VerbPath | VerbSimple ) ObjectListPath ( ';' ( ( VerbPath | VerbSimple ) ObjectListPath )? )* start_predicate: PropertyListPathNotEmptyVerb, start_object: Box, others: Vec>, @@ -919,15 +919,15 @@ impl fmt::Display for PropertyListPathNotEmpty { } } -// [84] VerbPath ::= Path +// [84] VerbPath ::= Path type VerbPath = Path; -// [85] VerbSimple ::= Var +// [85] VerbSimple ::= Var type VerbSimple = Var; #[derive(Arbitrary)] struct ObjectListPath { - // [86] ObjectListPath ::= ObjectPath ( ',' ObjectPath )* + // [86] ObjectListPath ::= ObjectPath ( ',' ObjectPath )* start: ObjectPath, others: Vec, } @@ -942,15 +942,15 @@ impl fmt::Display for ObjectListPath { } } -// [87] ObjectPath ::= GraphNodePath +// [87] ObjectPath ::= GraphNodePath type ObjectPath = GraphNodePath; -// [88] Path ::= PathAlternative +// [88] Path ::= PathAlternative type Path = PathAlternative; #[derive(Arbitrary)] struct PathAlternative { - // [89] PathAlternative ::= PathSequence ( '|' PathSequence )* + // [89] PathAlternative ::= PathSequence ( '|' PathSequence )* start: PathSequence, others: Vec, } @@ -967,7 +967,7 @@ impl fmt::Display for PathAlternative { #[derive(Arbitrary)] struct PathSequence { - // [90] PathSequence ::= PathEltOrInverse ( '/' PathEltOrInverse )* + // [90] PathSequence ::= PathEltOrInverse ( '/' PathEltOrInverse )* start: PathEltOrInverse, others: Vec, } @@ -984,7 +984,7 @@ impl fmt::Display for PathSequence { #[derive(Arbitrary)] struct PathElt { - // [91] PathElt ::= PathPrimary PathMod? + // [91] PathElt ::= PathPrimary PathMod? path: PathPrimary, mode: Option, } @@ -1001,7 +1001,7 @@ impl fmt::Display for PathElt { #[derive(Arbitrary)] enum PathEltOrInverse { - // [92] PathEltOrInverse ::= PathElt | '^' PathElt + // [92] PathEltOrInverse ::= PathElt | '^' PathElt PathElt(PathElt), Inverse(PathElt), } @@ -1017,7 +1017,7 @@ impl fmt::Display for PathEltOrInverse { #[derive(Arbitrary)] enum PathMod { - // [93] PathMod ::= '?' | '*' | '+' + // [93] PathMod ::= '?' | '*' | '+' ZeroOrOne, ZeroOrMore, OneOrMore, @@ -1035,7 +1035,7 @@ impl fmt::Display for PathMod { #[derive(Arbitrary)] enum PathPrimary { - // [94] PathPrimary ::= iri | 'a' | '!' PathNegatedPropertySet | '(' Path ')' + // [94] PathPrimary ::= iri | 'a' | '!' PathNegatedPropertySet | '(' Path ')' Iri(Iri), A, Negated(PathNegatedPropertySet), @@ -1055,7 +1055,7 @@ impl fmt::Display for PathPrimary { #[derive(Arbitrary)] enum PathNegatedPropertySet { - // [95] PathNegatedPropertySet ::= PathOneInPropertySet | '(' ( PathOneInPropertySet ( '|' PathOneInPropertySet )* )? ')' + // [95] PathNegatedPropertySet ::= PathOneInPropertySet | '(' ( PathOneInPropertySet ( '|' PathOneInPropertySet )* )? ')' Single(PathOneInPropertySet), Multiple { start: PathOneInPropertySet, @@ -1080,7 +1080,7 @@ impl fmt::Display for PathNegatedPropertySet { #[derive(Arbitrary)] enum PathOneInPropertySet { - // [96] PathOneInPropertySet ::= iri | 'a' | '^' ( iri | 'a' ) + // [96] PathOneInPropertySet ::= iri | 'a' | '^' ( iri | 'a' ) Iri(Iri), A, NegatedIri(Iri), @@ -1100,7 +1100,7 @@ impl fmt::Display for PathOneInPropertySet { #[derive(Arbitrary)] enum TriplesNode { - // [98] TriplesNode ::= Collection | BlankNodePropertyList + // [98] TriplesNode ::= Collection | BlankNodePropertyList Collection(Collection), BlankNodePropertyList(BlankNodePropertyList), } @@ -1116,7 +1116,7 @@ impl fmt::Display for TriplesNode { #[derive(Arbitrary)] struct BlankNodePropertyList { - // [99] BlankNodePropertyList ::= '[' PropertyListNotEmpty ']' + // [99] BlankNodePropertyList ::= '[' PropertyListNotEmpty ']' inner: PropertyListNotEmpty, } @@ -1128,7 +1128,7 @@ impl fmt::Display for BlankNodePropertyList { #[derive(Arbitrary)] enum TriplesNodePath { - // [100] TriplesNodePath ::= CollectionPath | BlankNodePropertyListPath + // [100] TriplesNodePath ::= CollectionPath | BlankNodePropertyListPath CollectionPath(CollectionPath), BlankNodePropertyListPath(BlankNodePropertyListPath), } @@ -1144,7 +1144,7 @@ impl fmt::Display for TriplesNodePath { #[derive(Arbitrary)] struct BlankNodePropertyListPath { - // [101] BlankNodePropertyListPath ::= '[' PropertyListPathNotEmpty ']' + // [101] BlankNodePropertyListPath ::= '[' PropertyListPathNotEmpty ']' inner: PropertyListPathNotEmpty, } @@ -1156,7 +1156,7 @@ impl fmt::Display for BlankNodePropertyListPath { #[derive(Arbitrary)] struct Collection { - // [102] Collection ::= '(' GraphNode+ ')' + // [102] Collection ::= '(' GraphNode+ ')' start: Box, others: Vec, } @@ -1173,7 +1173,7 @@ impl fmt::Display for Collection { #[derive(Arbitrary)] struct CollectionPath { - // [103] CollectionPath ::= '(' GraphNodePath+ ')' + // [103] CollectionPath ::= '(' GraphNodePath+ ')' start: Box, others: Vec, } @@ -1190,7 +1190,7 @@ impl fmt::Display for CollectionPath { #[derive(Arbitrary)] enum GraphNode { - // [104] GraphNode ::= VarOrTerm | TriplesNode + // [104] GraphNode ::= VarOrTerm | TriplesNode VarOrTerm(VarOrTerm), TriplesNode(TriplesNode), } @@ -1206,7 +1206,7 @@ impl fmt::Display for GraphNode { #[derive(Arbitrary)] enum GraphNodePath { - // [105] GraphNodePath ::= VarOrTerm | TriplesNodePath + // [105] GraphNodePath ::= VarOrTerm | TriplesNodePath VarOrTerm(VarOrTerm), TriplesNodePath(TriplesNodePath), } @@ -1222,7 +1222,7 @@ impl fmt::Display for GraphNodePath { #[derive(Arbitrary)] enum VarOrTerm { - // [106] VarOrTerm ::= Var | GraphTerm + // [106] VarOrTerm ::= Var | GraphTerm Var(Var), GraphTerm(GraphTerm), } @@ -1238,7 +1238,7 @@ impl fmt::Display for VarOrTerm { #[derive(Arbitrary)] enum VarOrIri { - // [107] VarOrIri ::= Var | iri + // [107] VarOrIri ::= Var | iri Var(Var), Iri(Iri), } @@ -1253,7 +1253,7 @@ impl fmt::Display for VarOrIri { } struct Var { - // [108] Var ::= VAR1 | VAR2 + // [108] Var ::= VAR1 | VAR2 value: u8, } @@ -1277,7 +1277,7 @@ impl fmt::Display for Var { #[derive(Arbitrary)] enum GraphTerm { - // [109] GraphTerm ::= iri | RDFLiteral | NumericLiteral | BooleanLiteral | BlankNode | NIL + // [109] GraphTerm ::= iri | RDFLiteral | NumericLiteral | BooleanLiteral | BlankNode | NIL Iri(Iri), Literal(Literal), Nil, @@ -1294,12 +1294,12 @@ impl fmt::Display for GraphTerm { } } -// [110] Expression ::= ConditionalOrExpression +// [110] Expression ::= ConditionalOrExpression type Expression = ConditionalOrExpression; #[derive(Arbitrary)] struct ConditionalOrExpression { - // [111] ConditionalOrExpression ::= ConditionalAndExpression ( '||' ConditionalAndExpression )* + // [111] ConditionalOrExpression ::= ConditionalAndExpression ( '||' ConditionalAndExpression )* start: ConditionalAndExpression, others: Vec, } @@ -1316,7 +1316,7 @@ impl fmt::Display for ConditionalOrExpression { #[derive(Arbitrary)] struct ConditionalAndExpression { - // [112] ConditionalAndExpression ::= ValueLogical ( '&&' ValueLogical )* + // [112] ConditionalAndExpression ::= ValueLogical ( '&&' ValueLogical )* start: ValueLogical, others: Vec, } @@ -1331,12 +1331,12 @@ impl fmt::Display for ConditionalAndExpression { } } -// [113] ValueLogical ::= RelationalExpression +// [113] ValueLogical ::= RelationalExpression type ValueLogical = RelationalExpression; #[derive(Arbitrary)] enum RelationalExpression { - // [114] RelationalExpression ::= NumericExpression ( '=' NumericExpression | '!=' NumericExpression | '<' NumericExpression | '>' NumericExpression | '<=' NumericExpression | '>=' NumericExpression | 'IN' ExpressionList | 'NOT' 'IN' ExpressionList )? + // [114] RelationalExpression ::= NumericExpression ( '=' NumericExpression | '!=' NumericExpression | '<' NumericExpression | '>' NumericExpression | '<=' NumericExpression | '>=' NumericExpression | 'IN' ExpressionList | 'NOT' 'IN' ExpressionList )? Base(NumericExpression), Equal(NumericExpression, NumericExpression), NotEqual(NumericExpression, NumericExpression), @@ -1364,12 +1364,12 @@ impl fmt::Display for RelationalExpression { } } -// [115] NumericExpression ::= AdditiveExpression +// [115] NumericExpression ::= AdditiveExpression type NumericExpression = AdditiveExpression; #[derive(Arbitrary)] enum AdditiveExpression { - // [116] AdditiveExpression ::= MultiplicativeExpression ( '+' MultiplicativeExpression | '-' MultiplicativeExpression | ( NumericLiteralPositive | NumericLiteralNegative ) ( ( '*' UnaryExpression ) | ( '/' UnaryExpression ) )* )* + // [116] AdditiveExpression ::= MultiplicativeExpression ( '+' MultiplicativeExpression | '-' MultiplicativeExpression | ( NumericLiteralPositive | NumericLiteralNegative ) ( ( '*' UnaryExpression ) | ( '/' UnaryExpression ) )* )* Base(MultiplicativeExpression), Plus(MultiplicativeExpression, MultiplicativeExpression), Minus(MultiplicativeExpression, MultiplicativeExpression), // TODO: Prefix + and - @@ -1387,7 +1387,7 @@ impl fmt::Display for AdditiveExpression { #[derive(Arbitrary)] enum MultiplicativeExpression { - // [117] MultiplicativeExpression ::= UnaryExpression ( '*' UnaryExpression | '/' UnaryExpression )* + // [117] MultiplicativeExpression ::= UnaryExpression ( '*' UnaryExpression | '/' UnaryExpression )* Base(UnaryExpression), Mul(UnaryExpression, UnaryExpression), Div(UnaryExpression, UnaryExpression), @@ -1405,7 +1405,7 @@ impl fmt::Display for MultiplicativeExpression { #[derive(Arbitrary)] enum UnaryExpression { - // [118] UnaryExpression ::= '!' PrimaryExpression | '+' PrimaryExpression | '-' PrimaryExpression | PrimaryExpression + // [118] UnaryExpression ::= '!' PrimaryExpression | '+' PrimaryExpression | '-' PrimaryExpression | PrimaryExpression Not(PrimaryExpression), Plus(PrimaryExpression), Minus(PrimaryExpression), @@ -1425,7 +1425,7 @@ impl fmt::Display for UnaryExpression { #[derive(Arbitrary)] enum PrimaryExpression { - // [119] PrimaryExpression ::= BrackettedExpression | BuiltInCall | iriOrFunction | RDFLiteral | NumericLiteral | BooleanLiteral | Var + // [119] PrimaryExpression ::= BrackettedExpression | BuiltInCall | iriOrFunction | RDFLiteral | NumericLiteral | BooleanLiteral | Var Bracketted(BrackettedExpression), BuiltInCall(BuiltInCall), IriOrFunction(IriOrFunction), @@ -1447,7 +1447,7 @@ impl fmt::Display for PrimaryExpression { #[derive(Arbitrary)] struct BrackettedExpression { - // [120] BrackettedExpression ::= '(' Expression ')' + // [120] BrackettedExpression ::= '(' Expression ')' inner: Box, } @@ -1459,61 +1459,61 @@ impl fmt::Display for BrackettedExpression { #[derive(Arbitrary)] enum BuiltInCall { - // [121] BuiltInCall ::= Aggregate - // | 'STR' '(' Expression ')' - // | 'LANG' '(' Expression ')' - // | 'LANGMATCHES' '(' Expression ',' Expression ')' - // | 'DATATYPE' '(' Expression ')' - // | 'BOUND' '(' Var ')' - // | 'IRI' '(' Expression ')' - // | 'URI' '(' Expression ')' - // | 'BNODE' ( '(' Expression ')' | NIL ) - // | 'RAND' NIL - // | 'ABS' '(' Expression ')' - // | 'CEIL' '(' Expression ')' - // | 'FLOOR' '(' Expression ')' - // | 'ROUND' '(' Expression ')' - // | 'CONCAT' ExpressionList - // | SubstringExpression - // | 'STRLEN' '(' Expression ')' - // | StrReplaceExpression - // | 'UCASE' '(' Expression ')' - // | 'LCASE' '(' Expression ')' - // | 'ENCODE_FOR_URI' '(' Expression ')' - // | 'CONTAINS' '(' Expression ',' Expression ')' - // | 'STRSTARTS' '(' Expression ',' Expression ')' - // | 'STRENDS' '(' Expression ',' Expression ')' - // | 'STRBEFORE' '(' Expression ',' Expression ')' - // | 'STRAFTER' '(' Expression ',' Expression ')' - // | 'YEAR' '(' Expression ')' - // | 'MONTH' '(' Expression ')' - // | 'DAY' '(' Expression ')' - // | 'HOURS' '(' Expression ')' - // | 'MINUTES' '(' Expression ')' - // | 'SECONDS' '(' Expression ')' - // | 'TIMEZONE' '(' Expression ')' - // | 'TZ' '(' Expression ')' - // | 'NOW' NIL - // | 'UUID' NIL - // | 'STRUUID' NIL - // | 'MD5' '(' Expression ')' - // | 'SHA1' '(' Expression ')' - // | 'SHA256' '(' Expression ')' - // | 'SHA384' '(' Expression ')' - // | 'SHA512' '(' Expression ')' - // | 'COALESCE' ExpressionList - // | 'IF' '(' Expression ',' Expression ',' Expression ')' - // | 'STRLANG' '(' Expression ',' Expression ')' - // | 'STRDT' '(' Expression ',' Expression ')' - // | 'sameTerm' '(' Expression ',' Expression ')' - // | 'isIRI' '(' Expression ')' - // | 'isURI' '(' Expression ')' - // | 'isBLANK' '(' Expression ')' - // | 'isLITERAL' '(' Expression ')' - // | 'isNUMERIC' '(' Expression ')' - // | RegexExpression - // | ExistsFunc - // | NotExistsFunc + // [121] BuiltInCall ::= Aggregate + // | 'STR' '(' Expression ')' + // | 'LANG' '(' Expression ')' + // | 'LANGMATCHES' '(' Expression ',' Expression ')' + // | 'DATATYPE' '(' Expression ')' + // | 'BOUND' '(' Var ')' + // | 'IRI' '(' Expression ')' + // | 'URI' '(' Expression ')' + // | 'BNODE' ( '(' Expression ')' | NIL ) + // | 'RAND' NIL + // | 'ABS' '(' Expression ')' + // | 'CEIL' '(' Expression ')' + // | 'FLOOR' '(' Expression ')' + // | 'ROUND' '(' Expression ')' + // | 'CONCAT' ExpressionList + // | SubstringExpression + // | 'STRLEN' '(' Expression ')' + // | StrReplaceExpression + // | 'UCASE' '(' Expression ')' + // | 'LCASE' '(' Expression ')' + // | 'ENCODE_FOR_URI' '(' Expression ')' + // | 'CONTAINS' '(' Expression ',' Expression ')' + // | 'STRSTARTS' '(' Expression ',' Expression ')' + // | 'STRENDS' '(' Expression ',' Expression ')' + // | 'STRBEFORE' '(' Expression ',' Expression ')' + // | 'STRAFTER' '(' Expression ',' Expression ')' + // | 'YEAR' '(' Expression ')' + // | 'MONTH' '(' Expression ')' + // | 'DAY' '(' Expression ')' + // | 'HOURS' '(' Expression ')' + // | 'MINUTES' '(' Expression ')' + // | 'SECONDS' '(' Expression ')' + // | 'TIMEZONE' '(' Expression ')' + // | 'TZ' '(' Expression ')' + // | 'NOW' NIL + // | 'UUID' NIL + // | 'STRUUID' NIL + // | 'MD5' '(' Expression ')' + // | 'SHA1' '(' Expression ')' + // | 'SHA256' '(' Expression ')' + // | 'SHA384' '(' Expression ')' + // | 'SHA512' '(' Expression ')' + // | 'COALESCE' ExpressionList + // | 'IF' '(' Expression ',' Expression ',' Expression ')' + // | 'STRLANG' '(' Expression ',' Expression ')' + // | 'STRDT' '(' Expression ',' Expression ')' + // | 'sameTerm' '(' Expression ',' Expression ')' + // | 'isIRI' '(' Expression ')' + // | 'isURI' '(' Expression ')' + // | 'isBLANK' '(' Expression ')' + // | 'isLITERAL' '(' Expression ')' + // | 'isNUMERIC' '(' Expression ')' + // | RegexExpression + // | ExistsFunc + // | NotExistsFunc Str(Box), Lang(Box), Datatype(Box), @@ -1559,7 +1559,7 @@ impl fmt::Display for BuiltInCall { #[derive(Arbitrary)] struct ExistsFunc { - // [125] ExistsFunc ::= 'EXISTS' GroupGraphPattern + // [125] ExistsFunc ::= 'EXISTS' GroupGraphPattern pattern: GroupGraphPattern, } @@ -1571,7 +1571,7 @@ impl fmt::Display for ExistsFunc { #[derive(Arbitrary)] struct NotExistsFunc { - // [126] NotExistsFunc ::= 'NOT' 'EXISTS' GroupGraphPattern + // [126] NotExistsFunc ::= 'NOT' 'EXISTS' GroupGraphPattern pattern: GroupGraphPattern, } @@ -1583,7 +1583,7 @@ impl fmt::Display for NotExistsFunc { #[derive(Arbitrary)] struct IriOrFunction { - // [128] iriOrFunction ::= iri ArgList? + // [128] iriOrFunction ::= iri ArgList? iri: Iri, // TODO args: Option, } @@ -1599,12 +1599,12 @@ impl fmt::Display for IriOrFunction { } struct Literal { - // [129] RDFLiteral ::= String ( LANGTAG | ( '^^' iri ) )? - // [130] NumericLiteral ::= NumericLiteralUnsigned | NumericLiteralPositive | NumericLiteralNegative - // [131] NumericLiteralUnsigned ::= INTEGER | DECIMAL | DOUBLE - // [132] NumericLiteralPositive ::= INTEGER_POSITIVE | DECIMAL_POSITIVE | DOUBLE_POSITIVE - // [133] NumericLiteralNegative ::= INTEGER_NEGATIVE | DECIMAL_NEGATIVE | DOUBLE_NEGATIVE - // [134] BooleanLiteral ::= 'true' | 'false' + // [129] RDFLiteral ::= String ( LANGTAG | ( '^^' iri ) )? + // [130] NumericLiteral ::= NumericLiteralUnsigned | NumericLiteralPositive | NumericLiteralNegative + // [131] NumericLiteralUnsigned ::= INTEGER | DECIMAL | DOUBLE + // [132] NumericLiteralPositive ::= INTEGER_POSITIVE | DECIMAL_POSITIVE | DOUBLE_POSITIVE + // [133] NumericLiteralNegative ::= INTEGER_NEGATIVE | DECIMAL_NEGATIVE | DOUBLE_NEGATIVE + // [134] BooleanLiteral ::= 'true' | 'false' value: &'static str, } @@ -1627,7 +1627,7 @@ impl fmt::Display for Literal { } struct Iri { - // [136] iri ::= IRIREF | PrefixedName + // [136] iri ::= IRIREF | PrefixedName value: u8, } From 0a7cea5e25631720679b51fead3df5133263fbc0 Mon Sep 17 00:00:00 2001 From: Tpt Date: Tue, 23 Jan 2024 09:09:17 +0100 Subject: [PATCH 191/217] Updates dependencies --- Cargo.lock | 172 +++++++++++++++++++----------------------- js/package-lock.json | 70 ++++++++--------- oxrocksdb-sys/rocksdb | 2 +- 3 files changed, 114 insertions(+), 130 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3d54e27c..c9fb9e90 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -34,9 +34,9 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "anstream" -version = "0.6.5" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d664a92ecae85fd0a7392615844904654d1d5f5514837f471ddef4a057aba1b6" +checksum = "6e2e1ebcb11de5c03c67de28a7df593d32191b44939c482e97702baaaa6ab6a5" dependencies = [ "anstyle", "anstyle-parse", @@ -97,9 +97,9 @@ dependencies = [ [[package]] name = "assert_cmd" -version = "2.0.12" +version = "2.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88903cb14723e4d4003335bb7f8a14f27691649105346a0f0957466c096adfe6" +checksum = "00ad3f3a942eee60335ab4342358c161ee296829e0d16ff42fc1d6cb07815467" dependencies = [ "anstyle", "bstr", @@ -112,9 +112,9 @@ dependencies = [ [[package]] name = "assert_fs" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adc5d78e9048d836d12a0c0040ca5f45b18a94d204b4ba4f677a8a7de162426b" +checksum = "2cd762e110c8ed629b11b6cde59458cc1c71de78ebbcc30099fc8e0403a2a2ec" dependencies = [ "anstyle", "doc-comment", @@ -148,17 +148,17 @@ dependencies = [ [[package]] name = "base64" -version = "0.21.5" +version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" [[package]] name = "bindgen" -version = "0.69.1" +version = "0.69.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ffcebc3849946a7170a05992aac39da343a90676ab392c51a4280981d6379c2" +checksum = "a4c69fae65a523209d34240b60abe0c42d33d1045d445c0839d8a4894a736e2d" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "cexpr", "clang-sys", "lazy_static", @@ -183,9 +183,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.1" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" +checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" [[package]] name = "block-buffer" @@ -290,9 +290,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.13" +version = "4.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52bdc885e4cacc7f7c9eedc1ef6da641603180c783c41a15c264944deeaab642" +checksum = "1e578d6ec4194633722ccf9544794b71b1385c3c027efe0c55db226fc880865c" dependencies = [ "clap_builder", "clap_derive", @@ -300,9 +300,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.12" +version = "4.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb7fb5e4e979aec3be7791562fcba452f94ad85e954da024396433e0e25a79e9" +checksum = "4df4df40ec50c46000231c914968278b1eb05098cf8f1b3a518a95030e71d1c7" dependencies = [ "anstream", "anstyle", @@ -422,7 +422,7 @@ dependencies = [ "clap", "criterion-plot", "is-terminal", - "itertools 0.10.5", + "itertools", "num-traits", "once_cell", "oorandom", @@ -443,39 +443,33 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" dependencies = [ "cast", - "itertools 0.10.5", + "itertools", ] [[package]] name = "crossbeam-deque" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fca89a0e215bab21874660c67903c5f143333cab1da83d041c7ded6053774751" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" dependencies = [ - "cfg-if", "crossbeam-epoch", "crossbeam-utils", ] [[package]] name = "crossbeam-epoch" -version = "0.9.17" +version = "0.9.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e3681d554572a651dda4186cd47240627c3d0114d45a95f6ad27f2f22e7548d" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" dependencies = [ - "autocfg", - "cfg-if", "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.18" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3a430a770ebd84726f584a90ee7f020d28db52c6d02138900f22341f866d39c" -dependencies = [ - "cfg-if", -] +checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" [[package]] name = "crypto-common" @@ -627,9 +621,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f" +checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5" dependencies = [ "cfg-if", "js-sys", @@ -665,11 +659,11 @@ dependencies = [ [[package]] name = "globwalk" -version = "0.8.1" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93e3af942408868f6934a7b85134a3230832b9977cf66125df2f9edcfce4ddcc" +checksum = "0bf760ebf69878d9fd8f110c89703d90ce35095324d1f1edcb595c63945ee757" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.4.2", "ignore", "walkdir", ] @@ -688,9 +682,9 @@ checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] name = "hermit-abi" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" +checksum = "5d3d0e0f38255e7fa3cf31335b3a56f05febd18025f4db5ef7a0cfb4f8da651f" [[package]] name = "hex" @@ -765,15 +759,6 @@ dependencies = [ "either", ] -[[package]] -name = "itertools" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" -dependencies = [ - "either", -] - [[package]] name = "itoa" version = "1.0.10" @@ -791,9 +776,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.66" +version = "0.3.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cee9c64da59eae3b50095c18d3e74f8b73c0b86d2792824ff01bbce68ba229ca" +checksum = "9a1d36f1235bc969acba30b7f5990b864423a6068a10f7c90ae8f0112e3a59d1" dependencies = [ "wasm-bindgen", ] @@ -847,9 +832,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456" +checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" [[package]] name = "lock_api" @@ -973,11 +958,11 @@ checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" [[package]] name = "openssl" -version = "0.10.62" +version = "0.10.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cde4d2d9200ad5909f8dac647e29482e07c3a35de8a13fce7c9c7747ad9f671" +checksum = "15c9d69dd87a29568d4d017cfe8ec518706046a05184e5aea92d0af890b803c8" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "cfg-if", "foreign-types", "libc", @@ -1005,9 +990,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.98" +version = "0.9.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1665caf8ab2dc9aef43d1c0023bd904633a6a05cb30b0ad59bec2ae986e57a7" +checksum = "22e1bf214306098e4832460f797824c05d25aacdf896f64a985fb0fd992454ae" dependencies = [ "cc", "libc", @@ -1246,9 +1231,9 @@ checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" [[package]] name = "pkg-config" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69d3587f8a9e599cc7ec2c00e331f71c4e69a5f9a4b8a6efd5b07466b9736f9a" +checksum = "2900ede94e305130c13ddd391e0ab7cbaeb783945ae07a279c268cb05109c6cb" [[package]] name = "plotters" @@ -1292,14 +1277,13 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "predicates" -version = "3.0.4" +version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dfc28575c2e3f19cb3c73b93af36460ae898d426eba6fc15b9bd2a5220758a0" +checksum = "68b87bfd4605926cdfefc1c3b5f8fe560e3feca9d5552cf68c466d3d8236c7e8" dependencies = [ "anstyle", "difflib", "float-cmp", - "itertools 0.11.0", "normalize-line-endings", "predicates-core", "regex", @@ -1333,9 +1317,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.76" +version = "1.0.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95fc56cda0b5c3325f5fbbd7ff9fda9e02bb00bb3dac51252d2f1bfa1cb8cc8c" +checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae" dependencies = [ "unicode-ident", ] @@ -1460,9 +1444,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.8.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1" +checksum = "fa7237101a77a10773db45d62004a272517633fbcc3df19d96455ede1122e051" dependencies = [ "either", "rayon-core", @@ -1470,9 +1454,9 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.12.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" dependencies = [ "crossbeam-deque", "crossbeam-utils", @@ -1489,9 +1473,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.2" +version = "1.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" +checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15" dependencies = [ "aho-corasick", "memchr", @@ -1501,9 +1485,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" +checksum = "3b7fa1134405e2ec9353fd416b17f8dacd46c473d7d3fd1cf202706a14eb792a" dependencies = [ "aho-corasick", "memchr", @@ -1544,11 +1528,11 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "rustix" -version = "0.38.28" +version = "0.38.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72e572a5e8ca657d7366229cdde4bd14c4eb5499a9573d4d366fe1b599daa316" +checksum = "322394588aaf33c24007e8bb3238ee3e4c5c09c084ab32bc73890b99ff326bca" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "errno", "libc", "linux-raw-sys", @@ -1557,9 +1541,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.22.1" +version = "0.22.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe6b63262c9fcac8659abfaa96cac103d28166d3ff3eaf8f412e19f3ae9e5a48" +checksum = "e87c9956bd9807afa1f77e0f7594af32566e830e088a5576d27c5b6f30f49d41" dependencies = [ "log", "ring", @@ -1717,9 +1701,9 @@ dependencies = [ [[package]] name = "shlex" -version = "1.2.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7cee0529a6d40f580e7a5e6c495c8fbfe21b7b52795ed4bb5e62cdf92bc6380" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "siphasher" @@ -1729,9 +1713,9 @@ checksum = "54ac45299ccbd390721be55b412d41931911f654fa99e2cb8bfb57184b2061fe" [[package]] name = "smallvec" -version = "1.11.2" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970" +checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" [[package]] name = "sparesults" @@ -1930,9 +1914,9 @@ checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "unicode-bidi" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f2528f27a9eb2b21e69c95319b30bd0efd85d09c379741b0f78ea1d86be2416" +checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" [[package]] name = "unicode-ident" @@ -2023,9 +2007,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.89" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ed0d4f68a3015cc185aff4db9506a015f4b96f95303897bfa23f846db54064e" +checksum = "b1223296a201415c7fad14792dbefaace9bd52b62d33453ade1c5b5f07555406" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -2033,9 +2017,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.89" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b56f625e64f3a1084ded111c4d5f477df9f8c92df113852fa5a374dbda78826" +checksum = "fcdc935b63408d58a32f8cc9738a0bffd8f05cc7c002086c6ef20b7312ad9dcd" dependencies = [ "bumpalo", "log", @@ -2048,9 +2032,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.89" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0162dbf37223cd2afce98f3d0785506dcb8d266223983e4b5b525859e6e182b2" +checksum = "3e4c238561b2d428924c49815533a8b9121c664599558a5d9ec51f8a1740a999" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2058,9 +2042,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.89" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283" +checksum = "bae1abb6806dc1ad9e560ed242107c0f6c84335f1749dd4e8ddb012ebd5e25a7" dependencies = [ "proc-macro2", "quote", @@ -2071,15 +2055,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.89" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ab9b36309365056cd639da3134bf87fa8f3d86008abf99e612384a6eecd459f" +checksum = "4d91413b1c31d7539ba5ef2451af3f0b833a005eb27a631cec32bc0635a8602b" [[package]] name = "web-sys" -version = "0.3.66" +version = "0.3.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50c24a44ec86bb68fbecd1b3efed7e85ea5621b39b35ef2766b66cd984f8010f" +checksum = "58cd2333b6e0be7a39605f0e255892fd7418a682d8da8fe042fe25128794d2ed" dependencies = [ "js-sys", "wasm-bindgen", diff --git a/js/package-lock.json b/js/package-lock.json index e9c781ae..3f28849a 100644 --- a/js/package-lock.json +++ b/js/package-lock.json @@ -12,9 +12,9 @@ } }, "node_modules/@biomejs/biome": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/@biomejs/biome/-/biome-1.5.1.tgz", - "integrity": "sha512-rdMA/N1Zc1nxUtbXMVr+50Sg/Pezz+9qGQa2uyRWFtrCoyr3dv0pVz+0ifGGue18ip50ZH8x2r5CV7zo8Q/0mA==", + "version": "1.5.3", + "resolved": "https://registry.npmjs.org/@biomejs/biome/-/biome-1.5.3.tgz", + "integrity": "sha512-yvZCa/g3akwTaAQ7PCwPWDCkZs3Qa5ONg/fgOUT9e6wAWsPftCjLQFPXBeGxPK30yZSSpgEmRCfpGTmVbUjGgg==", "dev": true, "hasInstallScript": true, "bin": { @@ -28,20 +28,20 @@ "url": "https://opencollective.com/biome" }, "optionalDependencies": { - "@biomejs/cli-darwin-arm64": "1.5.1", - "@biomejs/cli-darwin-x64": "1.5.1", - "@biomejs/cli-linux-arm64": "1.5.1", - "@biomejs/cli-linux-arm64-musl": "1.5.1", - "@biomejs/cli-linux-x64": "1.5.1", - "@biomejs/cli-linux-x64-musl": "1.5.1", - "@biomejs/cli-win32-arm64": "1.5.1", - "@biomejs/cli-win32-x64": "1.5.1" + "@biomejs/cli-darwin-arm64": "1.5.3", + "@biomejs/cli-darwin-x64": "1.5.3", + "@biomejs/cli-linux-arm64": "1.5.3", + "@biomejs/cli-linux-arm64-musl": "1.5.3", + "@biomejs/cli-linux-x64": "1.5.3", + "@biomejs/cli-linux-x64-musl": "1.5.3", + "@biomejs/cli-win32-arm64": "1.5.3", + "@biomejs/cli-win32-x64": "1.5.3" } }, "node_modules/@biomejs/cli-darwin-arm64": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-arm64/-/cli-darwin-arm64-1.5.1.tgz", - "integrity": "sha512-E9pLakmSVHP6UH2uqAghqEkr/IHAIDfDyCedqJVnyFc+uufNTHwB8id4XTiWy/eKIdgxHZsTSE+R+W0IqrTNVQ==", + "version": "1.5.3", + "resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-arm64/-/cli-darwin-arm64-1.5.3.tgz", + "integrity": "sha512-ImU7mh1HghEDyqNmxEZBoMPr8SxekkZuYcs+gynKlNW+TALQs7swkERiBLkG9NR0K1B3/2uVzlvYowXrmlW8hw==", "cpu": [ "arm64" ], @@ -55,9 +55,9 @@ } }, "node_modules/@biomejs/cli-darwin-x64": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-x64/-/cli-darwin-x64-1.5.1.tgz", - "integrity": "sha512-8O1F+FcoCi02JlocyilB6R3y3kT9sRkBCRwYddaBIScQe2hCme/mA2rVzrhCCHhskrclJ51GEKjkEORj4/8c2A==", + "version": "1.5.3", + "resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-x64/-/cli-darwin-x64-1.5.3.tgz", + "integrity": "sha512-vCdASqYnlpq/swErH7FD6nrFz0czFtK4k/iLgj0/+VmZVjineFPgevOb+Sr9vz0tk0GfdQO60bSpI74zU8M9Dw==", "cpu": [ "x64" ], @@ -71,9 +71,9 @@ } }, "node_modules/@biomejs/cli-linux-arm64": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64/-/cli-linux-arm64-1.5.1.tgz", - "integrity": "sha512-25gwY4FMzmi1Rl6N835raLq7nzTk+PyEQd88k9Em6dqtI4qpljqmZlMmVjOiwXKe3Ee80J/Vlh7BM36lsHUTEg==", + "version": "1.5.3", + "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64/-/cli-linux-arm64-1.5.3.tgz", + "integrity": "sha512-cupBQv0sNF1OKqBfx7EDWMSsKwRrBUZfjXawT4s6hKV6ALq7p0QzWlxr/sDmbKMLOaLQtw2Qgu/77N9rm+f9Rg==", "cpu": [ "arm64" ], @@ -87,9 +87,9 @@ } }, "node_modules/@biomejs/cli-linux-arm64-musl": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64-musl/-/cli-linux-arm64-musl-1.5.1.tgz", - "integrity": "sha512-Lw9G3LUdhRMp8L8RMeVevnfQCa7luT6ubQ8GRjLju32glxWKefpDrzgfHixGyvTQPlhnYjQ+V8/QQ/I7WPzOoA==", + "version": "1.5.3", + "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64-musl/-/cli-linux-arm64-musl-1.5.3.tgz", + "integrity": "sha512-DYuMizUYUBYfS0IHGjDrOP1RGipqWfMGEvNEJ398zdtmCKLXaUvTimiox5dvx4X15mBK5M2m8wgWUgOP1giUpQ==", "cpu": [ "arm64" ], @@ -103,9 +103,9 @@ } }, "node_modules/@biomejs/cli-linux-x64": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64/-/cli-linux-x64-1.5.1.tgz", - "integrity": "sha512-YDM0gZP4UbAuaBI3DVbUuj5X+Omm6uxzD1Qpc6hcduH1kzXzs9L0ee7cn/kJtNndoXR8MlmUS0O0/wWvZf2YaA==", + "version": "1.5.3", + "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64/-/cli-linux-x64-1.5.3.tgz", + "integrity": "sha512-YQrSArQvcv4FYsk7Q91Yv4uuu5F8hJyORVcv3zsjCLGkjIjx2RhjYLpTL733SNL7v33GmOlZY0eFR1ko38tuUw==", "cpu": [ "x64" ], @@ -119,9 +119,9 @@ } }, "node_modules/@biomejs/cli-linux-x64-musl": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64-musl/-/cli-linux-x64-musl-1.5.1.tgz", - "integrity": "sha512-5gapxc/VlwTgGRbTc9h8PMTpf8eNahIBauFUGSXncHgayi3VpezKSicgaQ1bb8FahVXf/5eNEVxVARq/or71Ag==", + "version": "1.5.3", + "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64-musl/-/cli-linux-x64-musl-1.5.3.tgz", + "integrity": "sha512-UUHiAnlDqr2Y/LpvshBFhUYMWkl2/Jn+bi3U6jKuav0qWbbBKU/ByHgR4+NBxpKBYoCtWxhnmatfH1bpPIuZMw==", "cpu": [ "x64" ], @@ -135,9 +135,9 @@ } }, "node_modules/@biomejs/cli-win32-arm64": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/@biomejs/cli-win32-arm64/-/cli-win32-arm64-1.5.1.tgz", - "integrity": "sha512-TVpLBOLUMLQmH2VRFBKFr3rgEkr7XvG4QZxHOxWB9Ivc/sQPvg4aHMd8qpgPKXABGUnultyc9t0+WvfIDxuALg==", + "version": "1.5.3", + "resolved": "https://registry.npmjs.org/@biomejs/cli-win32-arm64/-/cli-win32-arm64-1.5.3.tgz", + "integrity": "sha512-HxatYH7vf/kX9nrD+pDYuV2GI9GV8EFo6cfKkahAecTuZLPxryHx1WEfJthp5eNsE0+09STGkKIKjirP0ufaZA==", "cpu": [ "arm64" ], @@ -151,9 +151,9 @@ } }, "node_modules/@biomejs/cli-win32-x64": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/@biomejs/cli-win32-x64/-/cli-win32-x64-1.5.1.tgz", - "integrity": "sha512-qx8EKwScZmVYZjMPZ6GF3ZUmgg/N6zqh+d8vHA2E43opNCyqIPTl89sOqkc7zd1CyyABDWxsbqI9Ih6xTT6hnQ==", + "version": "1.5.3", + "resolved": "https://registry.npmjs.org/@biomejs/cli-win32-x64/-/cli-win32-x64-1.5.3.tgz", + "integrity": "sha512-fMvbSouZEASU7mZH8SIJSANDm5OqsjgtVXlbUqxwed6BP7uuHRSs396Aqwh2+VoW8fwTpp6ybIUoC9FrzB0kyA==", "cpu": [ "x64" ], diff --git a/oxrocksdb-sys/rocksdb b/oxrocksdb-sys/rocksdb index d9f9b9a7..2234753c 160000 --- a/oxrocksdb-sys/rocksdb +++ b/oxrocksdb-sys/rocksdb @@ -1 +1 @@ -Subproject commit d9f9b9a759821252261151c3ca371947734da720 +Subproject commit 2234753cc7b7bd86cf7461f8189a29b2441af51d From ef765666bef2f7b19071056d2fcc78414d2acd95 Mon Sep 17 00:00:00 2001 From: Tpt Date: Fri, 19 Jan 2024 22:14:24 +0100 Subject: [PATCH 192/217] Serialization: allows to set prefixes --- .github/workflows/tests.yml | 4 + Cargo.lock | 10 +- cli/src/main.rs | 19 +- fuzz/fuzz_targets/trig.rs | 43 +++-- lib/Cargo.toml | 8 +- lib/oxrdf/Cargo.toml | 2 +- lib/oxrdfio/Cargo.toml | 4 +- lib/oxrdfio/src/parser.rs | 26 +-- lib/oxrdfio/src/serializer.rs | 164 ++++++++++------ lib/oxrdfxml/Cargo.toml | 4 +- lib/oxrdfxml/src/serializer.rs | 209 ++++++++++++++------ lib/oxttl/Cargo.toml | 2 +- lib/oxttl/src/lexer.rs | 8 +- lib/oxttl/src/line_formats.rs | 4 +- lib/oxttl/src/n3.rs | 4 +- lib/oxttl/src/nquads.rs | 2 +- lib/oxttl/src/ntriples.rs | 2 +- lib/oxttl/src/terse.rs | 4 +- lib/oxttl/src/trig.rs | 341 ++++++++++++++++++++++++++------- lib/oxttl/src/turtle.rs | 108 +++++++---- lib/sparesults/Cargo.toml | 4 +- lib/sparesults/src/xml.rs | 7 +- lib/spargebra/Cargo.toml | 4 +- lib/sparopt/Cargo.toml | 6 +- 24 files changed, 695 insertions(+), 294 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 689bb227..76074704 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -42,8 +42,12 @@ jobs: working-directory: ./lib/oxttl - run: cargo clippy --all-targets -- -D warnings -D clippy::all working-directory: ./lib/oxrdfio + - run: cargo clippy --all-targets --features async-tokio -- -D warnings -D clippy::all + working-directory: ./lib/oxrdfio - run: cargo clippy --all-targets -- -D warnings -D clippy::all working-directory: ./lib/sparesults + - run: cargo clippy --all-targets --features async-tokio -- -D warnings -D clippy::all + working-directory: ./lib/sparesults - run: cargo clippy --all-targets -- -D warnings -D clippy::all working-directory: ./lib/spargebra - run: cargo clippy --all-targets -- -D warnings -D clippy::all diff --git a/Cargo.lock b/Cargo.lock index c9fb9e90..70b1abfb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1104,7 +1104,7 @@ checksum = "b225dad32cfaa43a960b93f01fa7f87528ac07e794b80f6d9a0153e0222557e2" [[package]] name = "oxrdf" -version = "0.2.0-alpha.1" +version = "0.2.0-alpha.2-dev" dependencies = [ "oxilangtag", "oxiri", @@ -1124,7 +1124,7 @@ dependencies = [ [[package]] name = "oxrdfxml" -version = "0.1.0-alpha.1" +version = "0.1.0-alpha.2-dev" dependencies = [ "oxilangtag", "oxiri", @@ -1719,7 +1719,7 @@ checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" [[package]] name = "sparesults" -version = "0.2.0-alpha.1" +version = "0.2.0-alpha.2-dev" dependencies = [ "json-event-parser", "memchr", @@ -1730,7 +1730,7 @@ dependencies = [ [[package]] name = "spargebra" -version = "0.3.0-alpha.1" +version = "0.3.0-alpha.2-dev" dependencies = [ "oxilangtag", "oxiri", @@ -1741,7 +1741,7 @@ dependencies = [ [[package]] name = "sparopt" -version = "0.1.0-alpha.1" +version = "0.1.0-alpha.2-dev" dependencies = [ "oxrdf", "rand", diff --git a/cli/src/main.rs b/cli/src/main.rs index cabf08c4..cff5cf79 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -714,7 +714,7 @@ pub fn main() -> anyhow::Result<()> { let to_format = if let Some(format) = to_format { rdf_format_from_name(&format)? - } else if let Some(file) = &from_file { + } else if let Some(file) = &to_file { rdf_format_from_path(file)? } else { bail!("The --to-format option must be set when writing to stdout") @@ -826,14 +826,21 @@ fn dump( fn do_convert( parser: RdfParser, read: R, - serializer: RdfSerializer, + mut serializer: RdfSerializer, write: W, lenient: bool, from_graph: &Option, default_graph: &GraphName, ) -> anyhow::Result { + let mut parser = parser.parse_read(read); + let first = parser.next(); // We read the first element to get prefixes + for (prefix_name, prefix_iri) in parser.prefixes() { + serializer = serializer + .with_prefix(prefix_name, prefix_iri) + .with_context(|| format!("Invalid IRI for prefix {prefix_name}: {prefix_iri}"))?; + } let mut writer = serializer.serialize_to_write(write); - for quad_result in parser.parse_read(read) { + for quad_result in first.into_iter().chain(parser) { match quad_result { Ok(mut quad) => { if let Some(from_graph) = from_graph { @@ -2239,8 +2246,8 @@ mod tests { #[test] fn cli_convert_file() -> Result<()> { let input_file = NamedTempFile::new("input.ttl")?; - input_file.write_str("

.")?; - let output_file = NamedTempFile::new("output.nt")?; + input_file.write_str("@prefix schema: .\n a schema:Person ;\n\tschema:name \"Foo Bar\"@en .\n")?; + let output_file = NamedTempFile::new("output.rdf")?; cli_command()? .arg("convert") .arg("--from-file") @@ -2252,7 +2259,7 @@ mod tests { .assert() .success(); output_file - .assert(" .\n"); + .assert("\n\n\t\n\t\tFoo Bar\n\t\n"); Ok(()) } diff --git a/fuzz/fuzz_targets/trig.rs b/fuzz/fuzz_targets/trig.rs index 1ce03d1b..64253e3c 100644 --- a/fuzz/fuzz_targets/trig.rs +++ b/fuzz/fuzz_targets/trig.rs @@ -7,7 +7,7 @@ use oxttl::{TriGParser, TriGSerializer}; fn parse<'a>( chunks: impl IntoIterator, unchecked: bool, -) -> (Vec, Vec) { +) -> (Vec, Vec, Vec<(String, String)>) { let mut quads = Vec::new(); let mut errors = Vec::new(); let mut parser = TriGParser::new() @@ -35,7 +35,14 @@ fn parse<'a>( } } assert!(reader.is_end()); - (quads, errors) + ( + quads, + errors, + reader + .prefixes() + .map(|(k, v)| (k.to_owned(), v.to_owned())) + .collect(), + ) } fn count_triple_blank_nodes(triple: &Triple) -> usize { @@ -62,8 +69,12 @@ fn count_quad_blank_nodes(quad: &Quad) -> usize { }) + usize::from(matches!(quad.graph_name, GraphName::BlankNode(_))) } -fn serialize_quads(quads: &[Quad]) -> Vec { - let mut writer = TriGSerializer::new().serialize_to_write(Vec::new()); +fn serialize_quads(quads: &[Quad], prefixes: Vec<(String, String)>) -> Vec { + let mut serializer = TriGSerializer::new(); + for (prefix_name, prefix_iri) in prefixes { + serializer = serializer.with_prefix(prefix_name, prefix_iri).unwrap(); + } + let mut writer = serializer.serialize_to_write(Vec::new()); for quad in quads { writer.write_quad(quad).unwrap(); } @@ -72,9 +83,9 @@ fn serialize_quads(quads: &[Quad]) -> Vec { fuzz_target!(|data: &[u8]| { // We parse with splitting - let (quads, errors) = parse(data.split(|c| *c == 0xFF), false); + let (quads, errors, prefixes) = parse(data.split(|c| *c == 0xFF), false); // We parse without splitting - let (quads_without_split, errors_without_split) = parse( + let (quads_without_split, errors_without_split, _) = parse( [data .iter() .copied() @@ -83,7 +94,7 @@ fuzz_target!(|data: &[u8]| { .as_slice()], false, ); - let (quads_unchecked, errors_unchecked) = parse(data.split(|c| *c == 0xFF), true); + let (quads_unchecked, errors_unchecked, _) = parse(data.split(|c| *c == 0xFF), true); if errors.is_empty() { assert!(errors_unchecked.is_empty()); } @@ -94,16 +105,16 @@ fuzz_target!(|data: &[u8]| { quads, quads_without_split, "With split:\n{}\nWithout split:\n{}", - String::from_utf8_lossy(&serialize_quads(&quads)), - String::from_utf8_lossy(&serialize_quads(&quads_without_split)) + String::from_utf8_lossy(&serialize_quads(&quads, Vec::new())), + String::from_utf8_lossy(&serialize_quads(&quads_without_split, Vec::new())) ); if errors.is_empty() { assert_eq!( quads, quads_unchecked, "Validating:\n{}\nUnchecked:\n{}", - String::from_utf8_lossy(&serialize_quads(&quads)), - String::from_utf8_lossy(&serialize_quads(&quads_unchecked)) + String::from_utf8_lossy(&serialize_quads(&quads, Vec::new())), + String::from_utf8_lossy(&serialize_quads(&quads_unchecked, Vec::new())) ); } } else if bnodes_count <= 4 { @@ -115,8 +126,8 @@ fuzz_target!(|data: &[u8]| { dataset_with_split, dataset_without_split, "With split:\n{}\nWithout split:\n{}", - String::from_utf8_lossy(&serialize_quads(&quads)), - String::from_utf8_lossy(&serialize_quads(&quads_without_split)) + String::from_utf8_lossy(&serialize_quads(&quads, Vec::new())), + String::from_utf8_lossy(&serialize_quads(&quads_without_split, Vec::new())) ); if errors.is_empty() { if errors.is_empty() { @@ -126,8 +137,8 @@ fuzz_target!(|data: &[u8]| { dataset_with_split, dataset_unchecked, "Validating:\n{}\nUnchecked:\n{}", - String::from_utf8_lossy(&serialize_quads(&quads)), - String::from_utf8_lossy(&serialize_quads(&quads_unchecked)) + String::from_utf8_lossy(&serialize_quads(&quads, Vec::new())), + String::from_utf8_lossy(&serialize_quads(&quads_unchecked, Vec::new())) ); } } @@ -135,7 +146,7 @@ fuzz_target!(|data: &[u8]| { assert_eq!(errors, errors_without_split); // We serialize - let new_serialization = serialize_quads(&quads); + let new_serialization = serialize_quads(&quads, prefixes); // We parse the serialization let new_quads = TriGParser::new() diff --git a/lib/Cargo.toml b/lib/Cargo.toml index 8465ce8f..44005104 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -32,7 +32,7 @@ json-event-parser = "0.2.0-alpha.2" md-5 = "0.10" oxilangtag = "0.1" oxiri = "0.2.3-alpha.1" -oxrdf = { version = "0.2.0-alpha.1", path = "oxrdf", features = ["rdf-star", "oxsdatatypes"] } +oxrdf = { version = "0.2.0-alpha.2-dev", path = "oxrdf", features = ["rdf-star", "oxsdatatypes"] } oxrdfio = { version = "0.1.0-alpha.2-dev", path = "oxrdfio", features = ["rdf-star"] } oxsdatatypes = { version = "0.2.0-alpha.1", path = "oxsdatatypes" } rand = "0.8" @@ -40,9 +40,9 @@ regex = "1.7" sha1 = "0.10" sha2 = "0.10" siphasher = ">=0.3, <2.0" -sparesults = { version = "0.2.0-alpha.1", path = "sparesults", features = ["rdf-star"] } -spargebra = { version = "0.3.0-alpha.1", path = "spargebra", features = ["rdf-star", "sep-0002", "sep-0006"] } -sparopt = { version = "0.1.0-alpha.1", path = "sparopt", features = ["rdf-star", "sep-0002", "sep-0006"] } +sparesults = { version = "0.2.0-alpha.2-dev", path = "sparesults", features = ["rdf-star"] } +spargebra = { version = "0.3.0-alpha.2-dev", path = "spargebra", features = ["rdf-star", "sep-0002", "sep-0006"] } +sparopt = { version = "0.1.0-alpha.2-dev", path = "sparopt", features = ["rdf-star", "sep-0002", "sep-0006"] } [target.'cfg(not(target_family = "wasm"))'.dependencies] libc = "0.2.147" diff --git a/lib/oxrdf/Cargo.toml b/lib/oxrdf/Cargo.toml index fdf94a51..c52b13ca 100644 --- a/lib/oxrdf/Cargo.toml +++ b/lib/oxrdf/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "oxrdf" -version = "0.2.0-alpha.1" +version = "0.2.0-alpha.2-dev" authors = ["Tpt "] license = "MIT OR Apache-2.0" readme = "README.md" diff --git a/lib/oxrdfio/Cargo.toml b/lib/oxrdfio/Cargo.toml index c2930046..027c5eba 100644 --- a/lib/oxrdfio/Cargo.toml +++ b/lib/oxrdfio/Cargo.toml @@ -19,8 +19,8 @@ async-tokio = ["dep:tokio", "oxrdfxml/async-tokio", "oxttl/async-tokio"] rdf-star = ["oxrdf/rdf-star", "oxttl/rdf-star"] [dependencies] -oxrdf = { version = "0.2.0-alpha.1", path = "../oxrdf" } -oxrdfxml = { version = "0.1.0-alpha.1", path = "../oxrdfxml" } +oxrdf = { version = "0.2.0-alpha.2-dev", path = "../oxrdf" } +oxrdfxml = { version = "0.1.0-alpha.2-dev", path = "../oxrdfxml" } oxttl = { version = "0.1.0-alpha.2-dev", path = "../oxttl" } tokio = { version = "1.29", optional = true, features = ["io-util"] } diff --git a/lib/oxrdfio/src/parser.rs b/lib/oxrdfio/src/parser.rs index 766be9ce..d1536141 100644 --- a/lib/oxrdfio/src/parser.rs +++ b/lib/oxrdfio/src/parser.rs @@ -598,12 +598,14 @@ impl FromTokioAsyncReadQuadReader { pub fn prefixes(&self) -> PrefixesIter<'_> { PrefixesIter { inner: match &self.parser { - FromReadQuadReaderKind::N3(p) => PrefixesIterKind::N3(p.prefixes()), - FromReadQuadReaderKind::TriG(p) => PrefixesIterKind::TriG(p.prefixes()), - FromReadQuadReaderKind::Turtle(p) => PrefixesIterKind::Turtle(p.prefixes()), - FromReadQuadReaderKind::NQuads(_) - | FromReadQuadReaderKind::NTriples(_) - | FromReadQuadReaderKind::RdfXml(_) => PrefixesIterKind::None, /* TODO: implement for RDF/XML */ + FromTokioAsyncReadQuadReaderKind::N3(p) => PrefixesIterKind::N3(p.prefixes()), + FromTokioAsyncReadQuadReaderKind::TriG(p) => PrefixesIterKind::TriG(p.prefixes()), + FromTokioAsyncReadQuadReaderKind::Turtle(p) => { + PrefixesIterKind::Turtle(p.prefixes()) + } + FromTokioAsyncReadQuadReaderKind::NQuads(_) + | FromTokioAsyncReadQuadReaderKind::NTriples(_) + | FromTokioAsyncReadQuadReaderKind::RdfXml(_) => PrefixesIterKind::None, /* TODO: implement for RDF/XML */ }, } } @@ -633,12 +635,12 @@ impl FromTokioAsyncReadQuadReader { /// ``` pub fn base_iri(&self) -> Option<&str> { match &self.parser { - FromReadQuadReaderKind::N3(p) => p.base_iri(), - FromReadQuadReaderKind::TriG(p) => p.base_iri(), - FromReadQuadReaderKind::Turtle(p) => p.base_iri(), - FromReadQuadReaderKind::NQuads(_) - | FromReadQuadReaderKind::NTriples(_) - | FromReadQuadReaderKind::RdfXml(_) => None, // TODO: implement for RDF/XML + FromTokioAsyncReadQuadReaderKind::N3(p) => p.base_iri(), + FromTokioAsyncReadQuadReaderKind::TriG(p) => p.base_iri(), + FromTokioAsyncReadQuadReaderKind::Turtle(p) => p.base_iri(), + FromTokioAsyncReadQuadReaderKind::NQuads(_) + | FromTokioAsyncReadQuadReaderKind::NTriples(_) + | FromTokioAsyncReadQuadReaderKind::RdfXml(_) => None, // TODO: implement for RDF/XML } } } diff --git a/lib/oxrdfio/src/serializer.rs b/lib/oxrdfio/src/serializer.rs index 7abf7696..3b32dc32 100644 --- a/lib/oxrdfio/src/serializer.rs +++ b/lib/oxrdfio/src/serializer.rs @@ -1,7 +1,7 @@ //! Utilities to write RDF graphs and datasets. use crate::format::RdfFormat; -use oxrdf::{GraphNameRef, QuadRef, TripleRef}; +use oxrdf::{GraphNameRef, IriParseError, QuadRef, TripleRef}; #[cfg(feature = "async-tokio")] use oxrdfxml::ToTokioAsyncWriteRdfXmlWriter; use oxrdfxml::{RdfXmlSerializer, ToWriteRdfXmlWriter}; @@ -35,29 +35,44 @@ use tokio::io::AsyncWrite; /// use oxrdfio::{RdfFormat, RdfSerializer}; /// use oxrdf::{Quad, NamedNode}; /// -/// let mut buffer = Vec::new(); -/// let mut writer = RdfSerializer::from_format(RdfFormat::NQuads).serialize_to_write(&mut buffer); +/// let mut writer = RdfSerializer::from_format(RdfFormat::NQuads).serialize_to_write(Vec::new()); /// writer.write_quad(&Quad { /// subject: NamedNode::new("http://example.com/s")?.into(), /// predicate: NamedNode::new("http://example.com/p")?, /// object: NamedNode::new("http://example.com/o")?.into(), /// graph_name: NamedNode::new("http://example.com/g")?.into() /// })?; -/// writer.finish()?; -/// -/// assert_eq!(buffer.as_slice(), " .\n".as_bytes()); +/// assert_eq!(writer.finish()?, b" .\n"); /// # Result::<_,Box>::Ok(()) /// ``` #[must_use] pub struct RdfSerializer { - format: RdfFormat, + inner: RdfSerializerKind, +} + +enum RdfSerializerKind { + NQuads(NQuadsSerializer), + NTriples(NTriplesSerializer), + RdfXml(RdfXmlSerializer), + TriG(TriGSerializer), + Turtle(TurtleSerializer), } impl RdfSerializer { /// Builds a serializer for the given format #[inline] pub fn from_format(format: RdfFormat) -> Self { - Self { format } + Self { + inner: match format { + RdfFormat::NQuads => RdfSerializerKind::NQuads(NQuadsSerializer::new()), + RdfFormat::NTriples => RdfSerializerKind::NTriples(NTriplesSerializer::new()), + RdfFormat::RdfXml => RdfSerializerKind::RdfXml(RdfXmlSerializer::new()), + RdfFormat::TriG => RdfSerializerKind::TriG(TriGSerializer::new()), + RdfFormat::Turtle | RdfFormat::N3 => { + RdfSerializerKind::Turtle(TurtleSerializer::new()) + } + }, + } } /// The format the serializer serializes to. @@ -71,7 +86,56 @@ impl RdfSerializer { /// ); /// ``` pub fn format(&self) -> RdfFormat { - self.format + match &self.inner { + RdfSerializerKind::NQuads(_) => RdfFormat::NQuads, + RdfSerializerKind::NTriples(_) => RdfFormat::NTriples, + RdfSerializerKind::RdfXml(_) => RdfFormat::RdfXml, + RdfSerializerKind::TriG(_) => RdfFormat::TriG, + RdfSerializerKind::Turtle(_) => RdfFormat::Turtle, + } + } + + /// If the format supports it, sets a prefix. + /// + /// ``` + /// use oxrdf::vocab::rdf; + /// use oxrdf::{NamedNodeRef, TripleRef}; + /// use oxrdfio::{RdfFormat, RdfSerializer}; + /// + /// let mut writer = RdfSerializer::from_format(RdfFormat::Turtle) + /// .with_prefix("schema", "http://schema.org/")? + /// .serialize_to_write(Vec::new()); + /// writer.write_triple(TripleRef { + /// subject: NamedNodeRef::new("http://example.com/s")?.into(), + /// predicate: rdf::TYPE.into(), + /// object: NamedNodeRef::new("http://schema.org/Person")?.into(), + /// })?; + /// assert_eq!( + /// writer.finish()?, + /// b"@prefix schema: .\n a schema:Person .\n" + /// ); + /// # Result::<_,Box>::Ok(()) + /// ``` + #[inline] + pub fn with_prefix( + mut self, + prefix_name: impl Into, + prefix_iri: impl Into, + ) -> Result { + self.inner = match self.inner { + RdfSerializerKind::NQuads(s) => RdfSerializerKind::NQuads(s), + RdfSerializerKind::NTriples(s) => RdfSerializerKind::NTriples(s), + RdfSerializerKind::RdfXml(s) => { + RdfSerializerKind::RdfXml(s.with_prefix(prefix_name, prefix_iri)?) + } + RdfSerializerKind::TriG(s) => { + RdfSerializerKind::TriG(s.with_prefix(prefix_name, prefix_iri)?) + } + RdfSerializerKind::Turtle(s) => { + RdfSerializerKind::Turtle(s.with_prefix(prefix_name, prefix_iri)?) + } + }; + Ok(self) } /// Writes to a [`Write`] implementation. @@ -88,36 +152,33 @@ impl RdfSerializer { /// use oxrdfio::{RdfFormat, RdfSerializer}; /// use oxrdf::{Quad, NamedNode}; /// - /// let mut buffer = Vec::new(); - /// let mut writer = RdfSerializer::from_format(RdfFormat::NQuads).serialize_to_write(&mut buffer); + /// let mut writer = RdfSerializer::from_format(RdfFormat::NQuads).serialize_to_write(Vec::new()); /// writer.write_quad(&Quad { /// subject: NamedNode::new("http://example.com/s")?.into(), /// predicate: NamedNode::new("http://example.com/p")?, /// object: NamedNode::new("http://example.com/o")?.into(), /// graph_name: NamedNode::new("http://example.com/g")?.into() /// })?; - /// writer.finish()?; - /// - /// assert_eq!(buffer.as_slice(), " .\n".as_bytes()); + /// assert_eq!(writer.finish()?, b" .\n"); /// # Result::<_,Box>::Ok(()) /// ``` pub fn serialize_to_write(self, write: W) -> ToWriteQuadWriter { ToWriteQuadWriter { - formatter: match self.format { - RdfFormat::NQuads => { - ToWriteQuadWriterKind::NQuads(NQuadsSerializer::new().serialize_to_write(write)) + formatter: match self.inner { + RdfSerializerKind::NQuads(s) => { + ToWriteQuadWriterKind::NQuads(s.serialize_to_write(write)) } - RdfFormat::NTriples => ToWriteQuadWriterKind::NTriples( - NTriplesSerializer::new().serialize_to_write(write), - ), - RdfFormat::RdfXml => { - ToWriteQuadWriterKind::RdfXml(RdfXmlSerializer::new().serialize_to_write(write)) + RdfSerializerKind::NTriples(s) => { + ToWriteQuadWriterKind::NTriples(s.serialize_to_write(write)) } - RdfFormat::TriG => { - ToWriteQuadWriterKind::TriG(TriGSerializer::new().serialize_to_write(write)) + RdfSerializerKind::RdfXml(s) => { + ToWriteQuadWriterKind::RdfXml(s.serialize_to_write(write)) } - RdfFormat::Turtle | RdfFormat::N3 => { - ToWriteQuadWriterKind::Turtle(TurtleSerializer::new().serialize_to_write(write)) + RdfSerializerKind::TriG(s) => { + ToWriteQuadWriterKind::TriG(s.serialize_to_write(write)) + } + RdfSerializerKind::Turtle(s) => { + ToWriteQuadWriterKind::Turtle(s.serialize_to_write(write)) } }, } @@ -139,17 +200,14 @@ impl RdfSerializer { /// /// # #[tokio::main(flavor = "current_thread")] /// # async fn main() -> std::io::Result<()> { - /// let mut buffer = Vec::new(); - /// let mut writer = RdfSerializer::from_format(RdfFormat::NQuads).serialize_to_tokio_async_write(&mut buffer); + /// let mut writer = RdfSerializer::from_format(RdfFormat::NQuads).serialize_to_tokio_async_write(Vec::new()); /// writer.write_quad(&Quad { /// subject: NamedNode::new_unchecked("http://example.com/s").into(), /// predicate: NamedNode::new_unchecked("http://example.com/p"), /// object: NamedNode::new_unchecked("http://example.com/o").into(), /// graph_name: NamedNode::new_unchecked("http://example.com/g").into() /// }).await?; - /// writer.finish().await?; - /// - /// assert_eq!(buffer.as_slice(), " .\n".as_bytes()); + /// assert_eq!(writer.finish().await?, " .\n"); /// # Ok(()) /// # } /// ``` @@ -159,22 +217,22 @@ impl RdfSerializer { write: W, ) -> ToTokioAsyncWriteQuadWriter { ToTokioAsyncWriteQuadWriter { - formatter: match self.format { - RdfFormat::NQuads => ToTokioAsyncWriteQuadWriterKind::NQuads( - NQuadsSerializer::new().serialize_to_tokio_async_write(write), - ), - RdfFormat::NTriples => ToTokioAsyncWriteQuadWriterKind::NTriples( - NTriplesSerializer::new().serialize_to_tokio_async_write(write), - ), - RdfFormat::RdfXml => ToTokioAsyncWriteQuadWriterKind::RdfXml( - RdfXmlSerializer::new().serialize_to_tokio_async_write(write), - ), - RdfFormat::TriG => ToTokioAsyncWriteQuadWriterKind::TriG( - TriGSerializer::new().serialize_to_tokio_async_write(write), - ), - RdfFormat::Turtle | RdfFormat::N3 => ToTokioAsyncWriteQuadWriterKind::Turtle( - TurtleSerializer::new().serialize_to_tokio_async_write(write), + formatter: match self.inner { + RdfSerializerKind::NQuads(s) => { + ToTokioAsyncWriteQuadWriterKind::NQuads(s.serialize_to_tokio_async_write(write)) + } + RdfSerializerKind::NTriples(s) => ToTokioAsyncWriteQuadWriterKind::NTriples( + s.serialize_to_tokio_async_write(write), ), + RdfSerializerKind::RdfXml(s) => { + ToTokioAsyncWriteQuadWriterKind::RdfXml(s.serialize_to_tokio_async_write(write)) + } + RdfSerializerKind::TriG(s) => { + ToTokioAsyncWriteQuadWriterKind::TriG(s.serialize_to_tokio_async_write(write)) + } + RdfSerializerKind::Turtle(s) => { + ToTokioAsyncWriteQuadWriterKind::Turtle(s.serialize_to_tokio_async_write(write)) + } }, } } @@ -202,17 +260,14 @@ impl From for RdfSerializer { /// use oxrdfio::{RdfFormat, RdfSerializer}; /// use oxrdf::{Quad, NamedNode}; /// -/// let mut buffer = Vec::new(); -/// let mut writer = RdfSerializer::from_format(RdfFormat::NQuads).serialize_to_write(&mut buffer); +/// let mut writer = RdfSerializer::from_format(RdfFormat::NQuads).serialize_to_write(Vec::new()); /// writer.write_quad(&Quad { /// subject: NamedNode::new("http://example.com/s")?.into(), /// predicate: NamedNode::new("http://example.com/p")?, /// object: NamedNode::new("http://example.com/o")?.into(), /// graph_name: NamedNode::new("http://example.com/g")?.into(), /// })?; -/// writer.finish()?; -/// -/// assert_eq!(buffer.as_slice(), " .\n".as_bytes()); +/// assert_eq!(writer.finish()?, b" .\n"); /// # Result::<_,Box>::Ok(()) /// ``` #[must_use] @@ -277,17 +332,14 @@ impl ToWriteQuadWriter { /// /// # #[tokio::main(flavor = "current_thread")] /// # async fn main() -> std::io::Result<()> { -/// let mut buffer = Vec::new(); -/// let mut writer = RdfSerializer::from_format(RdfFormat::NQuads).serialize_to_tokio_async_write(&mut buffer); +/// let mut writer = RdfSerializer::from_format(RdfFormat::NQuads).serialize_to_tokio_async_write(Vec::new()); /// writer.write_quad(&Quad { /// subject: NamedNode::new_unchecked("http://example.com/s").into(), /// predicate: NamedNode::new_unchecked("http://example.com/p"), /// object: NamedNode::new_unchecked("http://example.com/o").into(), /// graph_name: NamedNode::new_unchecked("http://example.com/g").into() /// }).await?; -/// writer.finish().await?; -/// -/// assert_eq!(buffer.as_slice(), " .\n".as_bytes()); +/// assert_eq!(writer.finish().await?, " .\n"); /// # Ok(()) /// # } /// ``` diff --git a/lib/oxrdfxml/Cargo.toml b/lib/oxrdfxml/Cargo.toml index 2906ac0e..6546b809 100644 --- a/lib/oxrdfxml/Cargo.toml +++ b/lib/oxrdfxml/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "oxrdfxml" -version = "0.1.0-alpha.1" +version = "0.1.0-alpha.2-dev" authors = ["Tpt "] license = "MIT OR Apache-2.0" readme = "README.md" @@ -18,7 +18,7 @@ default = [] async-tokio = ["dep:tokio", "quick-xml/async-tokio"] [dependencies] -oxrdf = { version = "0.2.0-alpha.1", path = "../oxrdf" } +oxrdf = { version = "0.2.0-alpha.2-dev", path = "../oxrdf" } oxilangtag = "0.1" oxiri = "0.2.3-alpha.1" quick-xml = ">=0.29, <0.32" diff --git a/lib/oxrdfxml/src/serializer.rs b/lib/oxrdfxml/src/serializer.rs index a19856f0..742b026e 100644 --- a/lib/oxrdfxml/src/serializer.rs +++ b/lib/oxrdfxml/src/serializer.rs @@ -1,8 +1,11 @@ use crate::utils::*; -use oxrdf::{Subject, SubjectRef, TermRef, TripleRef}; -use quick_xml::events::*; +use oxiri::{Iri, IriParseError}; +use oxrdf::vocab::rdf; +use oxrdf::{NamedNodeRef, Subject, SubjectRef, TermRef, TripleRef}; +use quick_xml::events::{BytesDecl, BytesEnd, BytesStart, BytesText, Event}; use quick_xml::Writer; use std::borrow::Cow; +use std::collections::BTreeMap; use std::io; use std::io::Write; use std::sync::Arc; @@ -12,30 +15,52 @@ use tokio::io::AsyncWrite; /// A [RDF/XML](https://www.w3.org/TR/rdf-syntax-grammar/) serializer. /// /// ``` -/// use oxrdf::{NamedNodeRef, TripleRef}; +/// use oxrdf::{LiteralRef, NamedNodeRef, TripleRef}; /// use oxrdfxml::RdfXmlSerializer; /// -/// let mut writer = RdfXmlSerializer::new().serialize_to_write(Vec::new()); +/// let mut writer = RdfXmlSerializer::new().with_prefix("schema", "http://schema.org/")?.serialize_to_write(Vec::new()); /// writer.write_triple(TripleRef::new( /// NamedNodeRef::new("http://example.com#me")?, /// NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?, /// NamedNodeRef::new("http://schema.org/Person")?, /// ))?; +/// writer.write_triple(TripleRef::new( +/// NamedNodeRef::new("http://example.com#me")?, +/// NamedNodeRef::new("http://schema.org/name")?, +/// LiteralRef::new_language_tagged_literal_unchecked("Foo Bar", "en"), +/// ))?; /// assert_eq!( -/// b"\n\n\t\n\t\t\n\t\n", +/// b"\n\n\t\n\t\tFoo Bar\n\t\n", /// writer.finish()?.as_slice() /// ); /// # Result::<_,Box>::Ok(()) /// ``` #[derive(Default)] #[must_use] -pub struct RdfXmlSerializer; +pub struct RdfXmlSerializer { + prefixes: BTreeMap, +} impl RdfXmlSerializer { /// Builds a new [`RdfXmlSerializer`]. #[inline] pub fn new() -> Self { - Self + Self { + prefixes: BTreeMap::new(), + } + } + + #[inline] + pub fn with_prefix( + mut self, + prefix_name: impl Into, + prefix_iri: impl Into, + ) -> Result { + self.prefixes.insert( + Iri::parse(prefix_iri.into())?.into_inner(), + prefix_name.into(), + ); + Ok(self) } /// Writes a RDF/XML file to a [`Write`] implementation. @@ -43,17 +68,22 @@ impl RdfXmlSerializer { /// This writer does unbuffered writes. /// /// ``` - /// use oxrdf::{NamedNodeRef, TripleRef}; + /// use oxrdf::{LiteralRef, NamedNodeRef, TripleRef}; /// use oxrdfxml::RdfXmlSerializer; /// - /// let mut writer = RdfXmlSerializer::new().serialize_to_write(Vec::new()); + /// let mut writer = RdfXmlSerializer::new().with_prefix("schema", "http://schema.org/")?.serialize_to_write(Vec::new()); /// writer.write_triple(TripleRef::new( /// NamedNodeRef::new("http://example.com#me")?, /// NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?, /// NamedNodeRef::new("http://schema.org/Person")?, /// ))?; + /// writer.write_triple(TripleRef::new( + /// NamedNodeRef::new("http://example.com#me")?, + /// NamedNodeRef::new("http://schema.org/name")?, + /// LiteralRef::new_language_tagged_literal_unchecked("Foo Bar", "en"), + /// ))?; /// assert_eq!( - /// b"\n\n\t\n\t\t\n\t\n", + /// b"\n\n\t\n\t\tFoo Bar\n\t\n", /// writer.finish()?.as_slice() /// ); /// # Result::<_,Box>::Ok(()) @@ -62,9 +92,7 @@ impl RdfXmlSerializer { pub fn serialize_to_write(self, write: W) -> ToWriteRdfXmlWriter { ToWriteRdfXmlWriter { writer: Writer::new_with_indent(write, b'\t', 1), - inner: InnerRdfXmlWriter { - current_subject: None, - }, + inner: self.inner_writer(), } } @@ -73,19 +101,24 @@ impl RdfXmlSerializer { /// This writer does unbuffered writes. /// /// ``` - /// use oxrdf::{NamedNodeRef, TripleRef}; + /// use oxrdf::{NamedNodeRef, TripleRef, LiteralRef}; /// use oxrdfxml::RdfXmlSerializer; /// /// # #[tokio::main(flavor = "current_thread")] - /// # async fn main() -> std::io::Result<()> { - /// let mut writer = RdfXmlSerializer::new().serialize_to_tokio_async_write(Vec::new()); + /// # async fn main() -> Result<(), Box> { + /// let mut writer = RdfXmlSerializer::new().with_prefix("schema", "http://schema.org/")?.serialize_to_tokio_async_write(Vec::new()); /// writer.write_triple(TripleRef::new( - /// NamedNodeRef::new_unchecked("http://example.com#me"), - /// NamedNodeRef::new_unchecked("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), - /// NamedNodeRef::new_unchecked("http://schema.org/Person"), + /// NamedNodeRef::new("http://example.com#me")?, + /// NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?, + /// NamedNodeRef::new("http://schema.org/Person")?, + /// )).await?; + /// writer.write_triple(TripleRef::new( + /// NamedNodeRef::new("http://example.com#me")?, + /// NamedNodeRef::new("http://schema.org/name")?, + /// LiteralRef::new_language_tagged_literal_unchecked("Foo Bar", "en"), /// )).await?; /// assert_eq!( - /// b"\n\n\t\n\t\t\n\t\n", + /// b"\n\n\t\n\t\tFoo Bar\n\t\n", /// writer.finish().await?.as_slice() /// ); /// # Ok(()) @@ -99,9 +132,19 @@ impl RdfXmlSerializer { ) -> ToTokioAsyncWriteRdfXmlWriter { ToTokioAsyncWriteRdfXmlWriter { writer: Writer::new_with_indent(write, b'\t', 1), - inner: InnerRdfXmlWriter { - current_subject: None, - }, + inner: self.inner_writer(), + } + } + + fn inner_writer(mut self) -> InnerRdfXmlWriter { + self.prefixes.insert( + "http://www.w3.org/1999/02/22-rdf-syntax-ns#".into(), + "rdf".into(), + ); + InnerRdfXmlWriter { + current_subject: None, + current_resource_tag: None, + prefixes: self.prefixes, } } } @@ -109,17 +152,22 @@ impl RdfXmlSerializer { /// Writes a RDF/XML file to a [`Write`] implementation. Can be built using [`RdfXmlSerializer::serialize_to_write`]. /// /// ``` -/// use oxrdf::{NamedNodeRef, TripleRef}; +/// use oxrdf::{LiteralRef, NamedNodeRef, TripleRef}; /// use oxrdfxml::RdfXmlSerializer; /// -/// let mut writer = RdfXmlSerializer::new().serialize_to_write(Vec::new()); +/// let mut writer = RdfXmlSerializer::new().with_prefix("schema", "http://schema.org/")?.serialize_to_write(Vec::new()); /// writer.write_triple(TripleRef::new( /// NamedNodeRef::new("http://example.com#me")?, /// NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?, /// NamedNodeRef::new("http://schema.org/Person")?, /// ))?; +/// writer.write_triple(TripleRef::new( +/// NamedNodeRef::new("http://example.com#me")?, +/// NamedNodeRef::new("http://schema.org/name")?, +/// LiteralRef::new_language_tagged_literal_unchecked("Foo Bar", "en"), +/// ))?; /// assert_eq!( -/// b"\n\n\t\n\t\t\n\t\n", +/// b"\n\n\t\n\t\tFoo Bar\n\t\n", /// writer.finish()?.as_slice() /// ); /// # Result::<_,Box>::Ok(()) @@ -158,19 +206,24 @@ impl ToWriteRdfXmlWriter { /// Writes a RDF/XML file to a [`AsyncWrite`] implementation. Can be built using [`RdfXmlSerializer::serialize_to_tokio_async_write`]. /// /// ``` -/// use oxrdf::{NamedNodeRef, TripleRef}; +/// use oxrdf::{NamedNodeRef, TripleRef, LiteralRef}; /// use oxrdfxml::RdfXmlSerializer; /// /// # #[tokio::main(flavor = "current_thread")] -/// # async fn main() -> std::io::Result<()> { -/// let mut writer = RdfXmlSerializer::new().serialize_to_tokio_async_write(Vec::new()); +/// # async fn main() -> Result<(), Box> { +/// let mut writer = RdfXmlSerializer::new().with_prefix("schema", "http://schema.org/")?.serialize_to_tokio_async_write(Vec::new()); /// writer.write_triple(TripleRef::new( -/// NamedNodeRef::new_unchecked("http://example.com#me"), -/// NamedNodeRef::new_unchecked("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), -/// NamedNodeRef::new_unchecked("http://schema.org/Person"), +/// NamedNodeRef::new("http://example.com#me")?, +/// NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?, +/// NamedNodeRef::new("http://schema.org/Person")?, +/// )).await?; +/// writer.write_triple(TripleRef::new( +/// NamedNodeRef::new("http://example.com#me")?, +/// NamedNodeRef::new("http://schema.org/name")?, +/// LiteralRef::new_language_tagged_literal_unchecked("Foo Bar", "en"), /// )).await?; /// assert_eq!( -/// b"\n\n\t\n\t\t\n\t\n", +/// b"\n\n\t\n\t\tFoo Bar\n\t\n", /// writer.finish().await?.as_slice() /// ); /// # Ok(()) @@ -214,6 +267,8 @@ impl ToTokioAsyncWriteRdfXmlWriter { pub struct InnerRdfXmlWriter { current_subject: Option, + current_resource_tag: Option, + prefixes: BTreeMap, } impl InnerRdfXmlWriter { @@ -224,17 +279,36 @@ impl InnerRdfXmlWriter { output: &mut Vec>, ) -> io::Result<()> { if self.current_subject.is_none() { - Self::write_start(output); + self.write_start(output); } let triple = t.into(); // We open a new rdf:Description if useful if self.current_subject.as_ref().map(Subject::as_ref) != Some(triple.subject) { if self.current_subject.is_some() { - output.push(Event::End(BytesEnd::new("rdf:Description"))); + output.push(Event::End( + self.current_resource_tag + .take() + .map_or_else(|| BytesEnd::new("rdf:Description"), BytesEnd::new), + )); } + self.current_subject = Some(triple.subject.into_owned()); - let mut description_open = BytesStart::new("rdf:Description"); + let (mut description_open, with_type_tag) = if triple.predicate == rdf::TYPE { + if let TermRef::NamedNode(t) = triple.object { + let (prop_qname, prop_xmlns) = self.uri_to_qname_and_xmlns(t); + let mut description_open = BytesStart::new(prop_qname.clone()); + if let Some(prop_xmlns) = prop_xmlns { + description_open.push_attribute(prop_xmlns); + } + self.current_resource_tag = Some(prop_qname.into_owned()); + (description_open, true) + } else { + (BytesStart::new("rdf:Description"), false) + } + } else { + (BytesStart::new("rdf:Description"), false) + }; match triple.subject { SubjectRef::NamedNode(node) => { description_open.push_attribute(("rdf:about", node.as_str())) @@ -250,20 +324,12 @@ impl InnerRdfXmlWriter { } } output.push(Event::Start(description_open)); + if with_type_tag { + return Ok(()); // No need for a value + } } - self.current_subject = Some(triple.subject.into_owned()); - let (prop_prefix, prop_value) = split_iri(triple.predicate.as_str()); - let (prop_qname, prop_xmlns) = - if prop_prefix == "http://www.w3.org/1999/02/22-rdf-syntax-ns#" { - (Cow::Owned(format!("rdf:{prop_value}")), None) - } else if prop_prefix == "http://www.w3.org/2000/xmlns/" { - (Cow::Owned(format!("xmlns:{prop_value}")), None) - } else if prop_value.is_empty() { - (Cow::Borrowed("p:"), Some(("xmlns:p", prop_prefix))) - } else { - (Cow::Borrowed(prop_value), Some(("xmlns", prop_prefix))) - }; + let (prop_qname, prop_xmlns) = self.uri_to_qname_and_xmlns(triple.predicate); let mut property_open = BytesStart::new(prop_qname.clone()); if let Some(prop_xmlns) = prop_xmlns { property_open.push_attribute(prop_xmlns); @@ -302,29 +368,58 @@ impl InnerRdfXmlWriter { Ok(()) } - fn write_start(output: &mut Vec>) { + fn write_start(&self, output: &mut Vec>) { output.push(Event::Decl(BytesDecl::new("1.0", Some("UTF-8"), None))); let mut rdf_open = BytesStart::new("rdf:RDF"); - rdf_open.push_attribute(("xmlns:rdf", "http://www.w3.org/1999/02/22-rdf-syntax-ns#")); + for (prefix_value, prefix_name) in &self.prefixes { + rdf_open.push_attribute(( + format!("xmlns:{prefix_name}").as_str(), + prefix_value.as_str(), + )); + } output.push(Event::Start(rdf_open)) } - fn finish(&self, output: &mut Vec>) { + fn finish(&mut self, output: &mut Vec>) { if self.current_subject.is_some() { - output.push(Event::End(BytesEnd::new("rdf:Description"))); + output.push(Event::End( + self.current_resource_tag + .take() + .map_or_else(|| BytesEnd::new("rdf:Description"), BytesEnd::new), + )); } else { - Self::write_start(output); + self.write_start(output); } output.push(Event::End(BytesEnd::new("rdf:RDF"))); } + + fn uri_to_qname_and_xmlns<'a>( + &self, + uri: NamedNodeRef<'a>, + ) -> (Cow<'a, str>, Option<(&'a str, &'a str)>) { + let (prop_prefix, prop_value) = split_iri(uri.as_str()); + if let Some(prop_prefix) = self.prefixes.get(prop_prefix) { + ( + if prop_prefix.is_empty() { + Cow::Borrowed(prop_value) + } else { + Cow::Owned(format!("{prop_prefix}:{prop_value}")) + }, + None, + ) + } else if prop_prefix == "http://www.w3.org/2000/xmlns/" { + (Cow::Owned(format!("xmlns:{prop_value}")), None) + } else if prop_value.is_empty() { + (Cow::Borrowed("p:"), Some(("xmlns:p", prop_prefix))) + } else { + (Cow::Borrowed(prop_value), Some(("xmlns", prop_prefix))) + } + } } fn map_err(error: quick_xml::Error) -> io::Error { if let quick_xml::Error::Io(error) = error { - match Arc::try_unwrap(error) { - Ok(error) => error, - Err(error) => io::Error::new(error.kind(), error), - } + Arc::try_unwrap(error).unwrap_or_else(|error| io::Error::new(error.kind(), error)) } else { io::Error::new(io::ErrorKind::Other, error) } diff --git a/lib/oxttl/Cargo.toml b/lib/oxttl/Cargo.toml index 7f7428f6..25d87b48 100644 --- a/lib/oxttl/Cargo.toml +++ b/lib/oxttl/Cargo.toml @@ -20,7 +20,7 @@ async-tokio = ["dep:tokio"] [dependencies] memchr = "2.5" -oxrdf = { version = "0.2.0-alpha.1", path = "../oxrdf" } +oxrdf = { version = "0.2.0-alpha.2-dev", path = "../oxrdf" } oxiri = "0.2.3-alpha.1" oxilangtag = "0.1" tokio = { version = "1.29", optional = true, features = ["io-util"] } diff --git a/lib/oxttl/src/lexer.rs b/lib/oxttl/src/lexer.rs index d11bd2fe..1eac849e 100644 --- a/lib/oxttl/src/lexer.rs +++ b/lib/oxttl/src/lexer.rs @@ -49,14 +49,14 @@ pub struct N3Lexer { // TODO: simplify by not giving is_end and fail with an "unexpected eof" is none is returned when is_end=true? impl TokenRecognizer for N3Lexer { - type Token<'a> = N3Token<'a>; type Options = N3LexerOptions; + type Token<'a> = N3Token<'a>; fn recognize_next_token<'a>( &mut self, data: &'a [u8], is_ending: bool, - options: &Self::Options, + options: &N3LexerOptions, ) -> Option<(usize, Result, TokenRecognizerError>)> { match *data.first()? { b'<' => match *data.get(1)? { @@ -914,12 +914,12 @@ impl N3Lexer { } // [158s] PN_CHARS_U ::= PN_CHARS_BASE | '_' | ':' - fn is_possible_pn_chars_u(c: char) -> bool { + pub(super) fn is_possible_pn_chars_u(c: char) -> bool { Self::is_possible_pn_chars_base(c) || c == '_' } // [160s] PN_CHARS ::= PN_CHARS_U | '-' | [0-9] | #x00B7 | [#x0300-#x036F] | [#x203F-#x2040] - fn is_possible_pn_chars(c: char) -> bool { + pub(crate) fn is_possible_pn_chars(c: char) -> bool { Self::is_possible_pn_chars_u(c) || matches!(c, '-' | '0'..='9' | '\u{00B7}' | '\u{0300}'..='\u{036F}' | '\u{203F}'..='\u{2040}') diff --git a/lib/oxttl/src/line_formats.rs b/lib/oxttl/src/line_formats.rs index e522bd53..5932f7a2 100644 --- a/lib/oxttl/src/line_formats.rs +++ b/lib/oxttl/src/line_formats.rs @@ -39,9 +39,9 @@ enum NQuadsState { } impl RuleRecognizer for NQuadsRecognizer { - type TokenRecognizer = N3Lexer; - type Output = Quad; type Context = NQuadsRecognizerContext; + type Output = Quad; + type TokenRecognizer = N3Lexer; fn error_recovery_state(mut self) -> Self { self.stack.clear(); diff --git a/lib/oxttl/src/n3.rs b/lib/oxttl/src/n3.rs index 8b70a01e..59ba9cf3 100644 --- a/lib/oxttl/src/n3.rs +++ b/lib/oxttl/src/n3.rs @@ -723,9 +723,9 @@ struct N3RecognizerContext { } impl RuleRecognizer for N3Recognizer { - type TokenRecognizer = N3Lexer; - type Output = N3Quad; type Context = N3RecognizerContext; + type Output = N3Quad; + type TokenRecognizer = N3Lexer; fn error_recovery_state(mut self) -> Self { self.stack.clear(); diff --git a/lib/oxttl/src/nquads.rs b/lib/oxttl/src/nquads.rs index f5108828..0ae22119 100644 --- a/lib/oxttl/src/nquads.rs +++ b/lib/oxttl/src/nquads.rs @@ -441,7 +441,7 @@ impl NQuadsSerializer { /// # Result::<_,Box>::Ok(()) /// ``` #[allow(clippy::unused_self)] - pub fn serialize(&self) -> LowLevelNQuadsWriter { + pub fn serialize(self) -> LowLevelNQuadsWriter { LowLevelNQuadsWriter } } diff --git a/lib/oxttl/src/ntriples.rs b/lib/oxttl/src/ntriples.rs index 4e0f7d7c..686907dc 100644 --- a/lib/oxttl/src/ntriples.rs +++ b/lib/oxttl/src/ntriples.rs @@ -437,7 +437,7 @@ impl NTriplesSerializer { /// # Result::<_,Box>::Ok(()) /// ``` #[allow(clippy::unused_self)] - pub fn serialize(&self) -> LowLevelNTriplesWriter { + pub fn serialize(self) -> LowLevelNTriplesWriter { LowLevelNTriplesWriter } } diff --git a/lib/oxttl/src/terse.rs b/lib/oxttl/src/terse.rs index c233c735..ebeff436 100644 --- a/lib/oxttl/src/terse.rs +++ b/lib/oxttl/src/terse.rs @@ -35,9 +35,9 @@ impl TriGRecognizerContext { } impl RuleRecognizer for TriGRecognizer { - type TokenRecognizer = N3Lexer; - type Output = Quad; type Context = TriGRecognizerContext; + type Output = Quad; + type TokenRecognizer = N3Lexer; fn error_recovery_state(mut self) -> Self { self.stack.clear(); diff --git a/lib/oxttl/src/trig.rs b/lib/oxttl/src/trig.rs index a27dd24d..aca75110 100644 --- a/lib/oxttl/src/trig.rs +++ b/lib/oxttl/src/trig.rs @@ -1,15 +1,18 @@ //! A [TriG](https://www.w3.org/TR/trig/) streaming parser implemented by [`TriGParser`] //! and a serializer implemented by [`TriGSerializer`]. +use crate::lexer::N3Lexer; use crate::terse::TriGRecognizer; #[cfg(feature = "async-tokio")] use crate::toolkit::FromTokioAsyncReadIterator; use crate::toolkit::{FromReadIterator, ParseError, Parser, SyntaxError}; use oxiri::{Iri, IriParseError}; -use oxrdf::vocab::xsd; -use oxrdf::{GraphName, NamedNode, Quad, QuadRef, Subject, TermRef}; +use oxrdf::vocab::{rdf, xsd}; +use oxrdf::{ + GraphName, GraphNameRef, LiteralRef, NamedNode, NamedNodeRef, Quad, QuadRef, Subject, TermRef, +}; use std::collections::hash_map::Iter; -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; use std::fmt; use std::io::{self, Read, Write}; #[cfg(feature = "async-tokio")] @@ -582,7 +585,9 @@ impl<'a> Iterator for TriGPrefixesIter<'a> { /// use oxrdf::{NamedNodeRef, QuadRef}; /// use oxttl::TriGSerializer; /// -/// let mut writer = TriGSerializer::new().serialize_to_write(Vec::new()); +/// let mut writer = TriGSerializer::new() +/// .with_prefix("schema", "http://schema.org/")? +/// .serialize_to_write(Vec::new()); /// writer.write_quad(QuadRef::new( /// NamedNodeRef::new("http://example.com#me")?, /// NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?, @@ -590,20 +595,37 @@ impl<'a> Iterator for TriGPrefixesIter<'a> { /// NamedNodeRef::new("http://example.com")?, /// ))?; /// assert_eq!( -/// b" {\n\t .\n}\n", +/// b"@prefix schema: .\n {\n\t a schema:Person .\n}\n", /// writer.finish()?.as_slice() /// ); /// # Result::<_,Box>::Ok(()) /// ``` #[derive(Default)] #[must_use] -pub struct TriGSerializer; +pub struct TriGSerializer { + prefixes: BTreeMap, +} impl TriGSerializer { /// Builds a new [`TriGSerializer`]. #[inline] pub fn new() -> Self { - Self + Self { + prefixes: BTreeMap::new(), + } + } + + #[inline] + pub fn with_prefix( + mut self, + prefix_name: impl Into, + prefix_iri: impl Into, + ) -> Result { + self.prefixes.insert( + Iri::parse(prefix_iri.into())?.into_inner(), + prefix_name.into(), + ); + Ok(self) } /// Writes a TriG file to a [`Write`] implementation. @@ -612,7 +634,9 @@ impl TriGSerializer { /// use oxrdf::{NamedNodeRef, QuadRef}; /// use oxttl::TriGSerializer; /// - /// let mut writer = TriGSerializer::new().serialize_to_write(Vec::new()); + /// let mut writer = TriGSerializer::new() + /// .with_prefix("schema", "http://schema.org/")? + /// .serialize_to_write(Vec::new()); /// writer.write_quad(QuadRef::new( /// NamedNodeRef::new("http://example.com#me")?, /// NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?, @@ -620,7 +644,7 @@ impl TriGSerializer { /// NamedNodeRef::new("http://example.com")?, /// ))?; /// assert_eq!( - /// b" {\n\t .\n}\n", + /// b"@prefix schema: .\n {\n\t a schema:Person .\n}\n", /// writer.finish()?.as_slice() /// ); /// # Result::<_,Box>::Ok(()) @@ -639,16 +663,20 @@ impl TriGSerializer { /// use oxttl::TriGSerializer; /// /// # #[tokio::main(flavor = "current_thread")] - /// # async fn main() -> std::io::Result<()> { - /// let mut writer = TriGSerializer::new().serialize_to_tokio_async_write(Vec::new()); - /// writer.write_quad(QuadRef::new( - /// NamedNodeRef::new_unchecked("http://example.com#me"), - /// NamedNodeRef::new_unchecked("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), - /// NamedNodeRef::new_unchecked("http://schema.org/Person"), - /// NamedNodeRef::new_unchecked("http://example.com"), - /// )).await?; + /// # async fn main() -> Result<(), Box> { + /// let mut writer = TriGSerializer::new() + /// .with_prefix("schema", "http://schema.org/")? + /// .serialize_to_tokio_async_write(Vec::new()); + /// writer + /// .write_quad(QuadRef::new( + /// NamedNodeRef::new_unchecked("http://example.com#me"), + /// NamedNodeRef::new_unchecked("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), + /// NamedNodeRef::new_unchecked("http://schema.org/Person"), + /// NamedNodeRef::new_unchecked("http://example.com"), + /// )) + /// .await?; /// assert_eq!( - /// b" {\n\t .\n}\n", + /// b"@prefix schema: .\n {\n\t a schema:Person .\n}\n", /// writer.finish().await?.as_slice() /// ); /// # Ok(()) @@ -673,23 +701,29 @@ impl TriGSerializer { /// use oxttl::TriGSerializer; /// /// let mut buf = Vec::new(); - /// let mut writer = TriGSerializer::new().serialize(); - /// writer.write_quad(QuadRef::new( - /// NamedNodeRef::new("http://example.com#me")?, - /// NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?, - /// NamedNodeRef::new("http://schema.org/Person")?, - /// NamedNodeRef::new("http://example.com")?, - /// ), &mut buf)?; + /// let mut writer = TriGSerializer::new() + /// .with_prefix("schema", "http://schema.org/")? + /// .serialize(); + /// writer.write_quad( + /// QuadRef::new( + /// NamedNodeRef::new("http://example.com#me")?, + /// NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?, + /// NamedNodeRef::new("http://schema.org/Person")?, + /// NamedNodeRef::new("http://example.com")?, + /// ), + /// &mut buf, + /// )?; /// writer.finish(&mut buf)?; /// assert_eq!( - /// b" {\n\t .\n}\n", + /// b"@prefix schema: .\n {\n\t a schema:Person .\n}\n", /// buf.as_slice() /// ); /// # Result::<_,Box>::Ok(()) /// ``` - #[allow(clippy::unused_self)] - pub fn serialize(&self) -> LowLevelTriGWriter { + pub fn serialize(self) -> LowLevelTriGWriter { LowLevelTriGWriter { + prefixes: self.prefixes, + prelude_written: false, current_graph_name: GraphName::DefaultGraph, current_subject_predicate: None, } @@ -702,7 +736,9 @@ impl TriGSerializer { /// use oxrdf::{NamedNodeRef, QuadRef}; /// use oxttl::TriGSerializer; /// -/// let mut writer = TriGSerializer::new().serialize_to_write(Vec::new()); +/// let mut writer = TriGSerializer::new() +/// .with_prefix("schema", "http://schema.org/")? +/// .serialize_to_write(Vec::new()); /// writer.write_quad(QuadRef::new( /// NamedNodeRef::new("http://example.com#me")?, /// NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?, @@ -710,7 +746,7 @@ impl TriGSerializer { /// NamedNodeRef::new("http://example.com")?, /// ))?; /// assert_eq!( -/// b" {\n\t .\n}\n", +/// b"@prefix schema: .\n {\n\t a schema:Person .\n}\n", /// writer.finish()?.as_slice() /// ); /// # Result::<_,Box>::Ok(()) @@ -741,16 +777,20 @@ impl ToWriteTriGWriter { /// use oxttl::TriGSerializer; /// /// # #[tokio::main(flavor = "current_thread")] -/// # async fn main() -> std::io::Result<()> { -/// let mut writer = TriGSerializer::new().serialize_to_tokio_async_write(Vec::new()); -/// writer.write_quad(QuadRef::new( -/// NamedNodeRef::new_unchecked("http://example.com#me"), -/// NamedNodeRef::new_unchecked("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), -/// NamedNodeRef::new_unchecked("http://schema.org/Person"), -/// NamedNodeRef::new_unchecked("http://example.com"), -/// )).await?; +/// # async fn main() -> Result<(), Box> { +/// let mut writer = TriGSerializer::new() +/// .with_prefix("schema", "http://schema.org/")? +/// .serialize_to_tokio_async_write(Vec::new()); +/// writer +/// .write_quad(QuadRef::new( +/// NamedNodeRef::new_unchecked("http://example.com#me"), +/// NamedNodeRef::new_unchecked("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), +/// NamedNodeRef::new_unchecked("http://schema.org/Person"), +/// NamedNodeRef::new_unchecked("http://example.com"), +/// )) +/// .await?; /// assert_eq!( -/// b" {\n\t .\n}\n", +/// b"@prefix schema: .\n {\n\t a schema:Person .\n}\n", /// writer.finish().await?.as_slice() /// ); /// # Ok(()) @@ -790,21 +830,28 @@ impl ToTokioAsyncWriteTriGWriter { /// use oxttl::TriGSerializer; /// /// let mut buf = Vec::new(); -/// let mut writer = TriGSerializer::new().serialize(); -/// writer.write_quad(QuadRef::new( -/// NamedNodeRef::new("http://example.com#me")?, -/// NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?, -/// NamedNodeRef::new("http://schema.org/Person")?, -/// NamedNodeRef::new("http://example.com")?, -/// ), &mut buf)?; +/// let mut writer = TriGSerializer::new() +/// .with_prefix("schema", "http://schema.org/")? +/// .serialize(); +/// writer.write_quad( +/// QuadRef::new( +/// NamedNodeRef::new("http://example.com#me")?, +/// NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?, +/// NamedNodeRef::new("http://schema.org/Person")?, +/// NamedNodeRef::new("http://example.com")?, +/// ), +/// &mut buf, +/// )?; /// writer.finish(&mut buf)?; /// assert_eq!( -/// b" {\n\t .\n}\n", +/// b"@prefix schema: .\n {\n\t a schema:Person .\n}\n", /// buf.as_slice() /// ); /// # Result::<_,Box>::Ok(()) /// ``` pub struct LowLevelTriGWriter { + prefixes: BTreeMap, + prelude_written: bool, current_graph_name: GraphName, current_subject_predicate: Option<(Subject, NamedNode)>, } @@ -816,6 +863,12 @@ impl LowLevelTriGWriter { q: impl Into>, mut write: impl Write, ) -> io::Result<()> { + if !self.prelude_written { + self.prelude_written = true; + for (prefix_iri, prefix_name) in &self.prefixes { + writeln!(write, "@prefix {prefix_name}: <{prefix_iri}> .")?; + } + } let q = q.into(); if q.graph_name == self.current_graph_name.as_ref() { if let Some((current_subject, current_predicate)) = @@ -824,7 +877,7 @@ impl LowLevelTriGWriter { if q.subject == current_subject.as_ref() { if q.predicate == current_predicate { self.current_subject_predicate = Some((current_subject, current_predicate)); - write!(write, " , {}", TurtleTerm(q.object)) + write!(write, " , {}", self.term(q.object)) } else { self.current_subject_predicate = Some((current_subject, q.predicate.into_owned())); @@ -832,7 +885,12 @@ impl LowLevelTriGWriter { if !self.current_graph_name.is_default_graph() { write!(write, "\t")?; } - write!(write, "\t{} {}", q.predicate, TurtleTerm(q.object)) + write!( + write, + "\t{} {}", + self.predicate(q.predicate), + self.term(q.object) + ) } } else { self.current_subject_predicate = @@ -844,9 +902,9 @@ impl LowLevelTriGWriter { write!( write, "{} {} {}", - TurtleTerm(q.subject.into()), - q.predicate, - TurtleTerm(q.object) + self.term(q.subject), + self.predicate(q.predicate), + self.term(q.object) ) } } else { @@ -858,9 +916,9 @@ impl LowLevelTriGWriter { write!( write, "{} {} {}", - TurtleTerm(q.subject.into()), - q.predicate, - TurtleTerm(q.object) + self.term(q.subject), + self.predicate(q.predicate), + self.term(q.object) ) } } else { @@ -873,20 +931,42 @@ impl LowLevelTriGWriter { self.current_graph_name = q.graph_name.into_owned(); self.current_subject_predicate = Some((q.subject.into_owned(), q.predicate.into_owned())); - if !self.current_graph_name.is_default_graph() { - writeln!(write, "{} {{", q.graph_name)?; - write!(write, "\t")?; + match self.current_graph_name.as_ref() { + GraphNameRef::NamedNode(g) => { + writeln!(write, "{} {{", self.term(g))?; + write!(write, "\t")?; + } + GraphNameRef::BlankNode(g) => { + writeln!(write, "{} {{", self.term(g))?; + write!(write, "\t")?; + } + GraphNameRef::DefaultGraph => (), } + write!( write, "{} {} {}", - TurtleTerm(q.subject.into()), - q.predicate, - TurtleTerm(q.object) + self.term(q.subject), + self.predicate(q.predicate), + self.term(q.object) ) } } + fn predicate<'a>(&'a self, named_node: impl Into>) -> TurtlePredicate<'a> { + TurtlePredicate { + named_node: named_node.into(), + prefixes: &self.prefixes, + } + } + + fn term<'a>(&'a self, term: impl Into>) -> TurtleTerm<'a> { + TurtleTerm { + term: term.into(), + prefixes: &self.prefixes, + } + } + /// Finishes to write the file. pub fn finish(&mut self, mut write: impl Write) -> io::Result<()> { if self.current_subject_predicate.is_some() { @@ -899,12 +979,43 @@ impl LowLevelTriGWriter { } } -struct TurtleTerm<'a>(TermRef<'a>); +struct TurtlePredicate<'a> { + named_node: NamedNodeRef<'a>, + prefixes: &'a BTreeMap, +} + +impl<'a> fmt::Display for TurtlePredicate<'a> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if self.named_node == rdf::TYPE { + write!(f, "a") + } else { + TurtleTerm { + term: self.named_node.into(), + prefixes: self.prefixes, + } + .fmt(f) + } + } +} + +struct TurtleTerm<'a> { + term: TermRef<'a>, + prefixes: &'a BTreeMap, +} impl<'a> fmt::Display for TurtleTerm<'a> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self.0 { - TermRef::NamedNode(v) => write!(f, "{v}"), + match self.term { + TermRef::NamedNode(v) => { + for (prefix_iri, prefix_name) in self.prefixes { + if let Some(local_name) = v.as_str().strip_prefix(prefix_iri) { + if let Some(escaped_local_name) = escape_local_name(local_name) { + return write!(f, "{prefix_name}:{escaped_local_name}"); + } + } + } + write!(f, "{v}") + } TermRef::BlankNode(v) => write!(f, "{v}"), TermRef::Literal(v) => { let value = v.value(); @@ -917,8 +1028,18 @@ impl<'a> fmt::Display for TurtleTerm<'a> { }; if inline { write!(f, "{value}") - } else { + } else if v.is_plain() { write!(f, "{v}") + } else { + write!( + f, + "{}^^{}", + LiteralRef::new_simple_literal(v.value()), + TurtleTerm { + term: v.datatype().into(), + prefixes: self.prefixes + } + ) } } #[cfg(feature = "rdf-star")] @@ -926,9 +1047,18 @@ impl<'a> fmt::Display for TurtleTerm<'a> { write!( f, "<< {} {} {} >>", - TurtleTerm(t.subject.as_ref().into()), - t.predicate, - TurtleTerm(t.object.as_ref()) + TurtleTerm { + term: t.subject.as_ref().into(), + prefixes: self.prefixes + }, + TurtleTerm { + term: t.predicate.as_ref().into(), + prefixes: self.prefixes + }, + TurtleTerm { + term: t.object.as_ref(), + prefixes: self.prefixes + } ) } } @@ -1004,6 +1134,61 @@ fn is_turtle_double(value: &str) -> bool { (with_before || with_after) && !value.is_empty() && value.iter().all(u8::is_ascii_digit) } +fn escape_local_name(value: &str) -> Option { + // TODO: PLX + // [168s] PN_LOCAL ::= (PN_CHARS_U | ':' | [0-9] | PLX) ((PN_CHARS | '.' | ':' | PLX)* (PN_CHARS | ':' | PLX))? + let mut output = String::with_capacity(value.len()); + let mut chars = value.chars(); + let first = chars.next()?; + if N3Lexer::is_possible_pn_chars_u(first) || first == ':' || first.is_ascii_digit() { + output.push(first); + } else if can_be_escaped_in_local_name(first) { + output.push('\\'); + output.push(first); + } else { + return None; + } + + while let Some(c) = chars.next() { + if N3Lexer::is_possible_pn_chars(c) || c == ':' || (c == '.' && !chars.as_str().is_empty()) + { + output.push(c); + } else if can_be_escaped_in_local_name(c) { + output.push('\\'); + output.push(c); + } else { + return None; + } + } + + Some(output) +} + +fn can_be_escaped_in_local_name(c: char) -> bool { + matches!( + c, + '_' | '~' + | '.' + | '-' + | '!' + | '$' + | '&' + | '\'' + | '(' + | ')' + | '*' + | '+' + | ',' + | ';' + | '=' + | '/' + | '?' + | '#' + | '@' + | '%' + ) +} + #[cfg(test)] mod tests { #![allow(clippy::panic_in_result_fn)] @@ -1014,11 +1199,20 @@ mod tests { #[test] fn test_write() -> io::Result<()> { - let mut writer = TriGSerializer::new().serialize_to_write(Vec::new()); + let mut writer = TriGSerializer::new() + .with_prefix("ex", "http://example.com/") + .unwrap() + .serialize_to_write(Vec::new()); writer.write_quad(QuadRef::new( NamedNodeRef::new_unchecked("http://example.com/s"), NamedNodeRef::new_unchecked("http://example.com/p"), - NamedNodeRef::new_unchecked("http://example.com/o"), + NamedNodeRef::new_unchecked("http://example.com/o."), + NamedNodeRef::new_unchecked("http://example.com/g"), + ))?; + writer.write_quad(QuadRef::new( + NamedNodeRef::new_unchecked("http://example.com/s"), + NamedNodeRef::new_unchecked("http://example.com/p"), + NamedNodeRef::new_unchecked("http://example.com/o{o}"), NamedNodeRef::new_unchecked("http://example.com/g"), ))?; writer.write_quad(QuadRef::new( @@ -1047,11 +1241,14 @@ mod tests { ))?; writer.write_quad(QuadRef::new( BlankNodeRef::new_unchecked("b"), - NamedNodeRef::new_unchecked("http://example.com/p2"), + NamedNodeRef::new_unchecked("http://example.org/p2"), LiteralRef::new_typed_literal("false", xsd::BOOLEAN), NamedNodeRef::new_unchecked("http://example.com/g2"), ))?; - assert_eq!(String::from_utf8(writer.finish()?).unwrap(), " {\n\t , \"foo\" ;\n\t\t \"foo\"@en .\n\t_:b _:b2 .\n}\n_:b true .\n {\n\t_:b false .\n}\n"); + assert_eq!( + String::from_utf8(writer.finish()?).unwrap(), + "@prefix ex: .\nex:g {\n\tex:s ex:p ex:o\\. , , \"foo\" ;\n\t\tex:p2 \"foo\"@en .\n\t_:b ex:p2 _:b2 .\n}\n_:b ex:p2 true .\nex:g2 {\n\t_:b false .\n}\n" + ); Ok(()) } } diff --git a/lib/oxttl/src/turtle.rs b/lib/oxttl/src/turtle.rs index f5193059..0cc9fd77 100644 --- a/lib/oxttl/src/turtle.rs +++ b/lib/oxttl/src/turtle.rs @@ -583,14 +583,16 @@ impl<'a> Iterator for TurtlePrefixesIter<'a> { /// use oxrdf::{NamedNodeRef, TripleRef}; /// use oxttl::TurtleSerializer; /// -/// let mut writer = TurtleSerializer::new().serialize_to_write(Vec::new()); +/// let mut writer = TurtleSerializer::new() +/// .with_prefix("schema", "http://schema.org/")? +/// .serialize_to_write(Vec::new()); /// writer.write_triple(TripleRef::new( /// NamedNodeRef::new("http://example.com#me")?, /// NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?, /// NamedNodeRef::new("http://schema.org/Person")?, /// ))?; /// assert_eq!( -/// b" .\n", +/// b"@prefix schema: .\n a schema:Person .\n", /// writer.finish()?.as_slice() /// ); /// # Result::<_,Box>::Ok(()) @@ -608,20 +610,32 @@ impl TurtleSerializer { Self::default() } + #[inline] + pub fn with_prefix( + mut self, + prefix_name: impl Into, + prefix_iri: impl Into, + ) -> Result { + self.inner = self.inner.with_prefix(prefix_name, prefix_iri)?; + Ok(self) + } + /// Writes a Turtle file to a [`Write`] implementation. /// /// ``` /// use oxrdf::{NamedNodeRef, TripleRef}; /// use oxttl::TurtleSerializer; /// - /// let mut writer = TurtleSerializer::new().serialize_to_write(Vec::new()); + /// let mut writer = TurtleSerializer::new() + /// .with_prefix("schema", "http://schema.org/")? + /// .serialize_to_write(Vec::new()); /// writer.write_triple(TripleRef::new( /// NamedNodeRef::new("http://example.com#me")?, /// NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?, /// NamedNodeRef::new("http://schema.org/Person")?, /// ))?; /// assert_eq!( - /// b" .\n", + /// b"@prefix schema: .\n a schema:Person .\n", /// writer.finish()?.as_slice() /// ); /// # Result::<_,Box>::Ok(()) @@ -639,15 +653,19 @@ impl TurtleSerializer { /// use oxttl::TurtleSerializer; /// /// # #[tokio::main(flavor = "current_thread")] - /// # async fn main() -> std::io::Result<()> { - /// let mut writer = TurtleSerializer::new().serialize_to_tokio_async_write(Vec::new()); - /// writer.write_triple(TripleRef::new( - /// NamedNodeRef::new_unchecked("http://example.com#me"), - /// NamedNodeRef::new_unchecked("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), - /// NamedNodeRef::new_unchecked("http://schema.org/Person"), - /// )).await?; + /// # async fn main() -> Result<(),Box> { + /// let mut writer = TurtleSerializer::new() + /// .with_prefix("schema", "http://schema.org/")? + /// .serialize_to_tokio_async_write(Vec::new()); + /// writer + /// .write_triple(TripleRef::new( + /// NamedNodeRef::new_unchecked("http://example.com#me"), + /// NamedNodeRef::new_unchecked("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), + /// NamedNodeRef::new_unchecked("http://schema.org/Person"), + /// )) + /// .await?; /// assert_eq!( - /// b" .\n", + /// b"@prefix schema: .\n a schema:Person .\n", /// writer.finish().await?.as_slice() /// ); /// # Ok(()) @@ -670,20 +688,25 @@ impl TurtleSerializer { /// use oxttl::TurtleSerializer; /// /// let mut buf = Vec::new(); - /// let mut writer = TurtleSerializer::new().serialize(); - /// writer.write_triple(TripleRef::new( - /// NamedNodeRef::new("http://example.com#me")?, - /// NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?, - /// NamedNodeRef::new("http://schema.org/Person")?, - /// ), &mut buf)?; + /// let mut writer = TurtleSerializer::new() + /// .with_prefix("schema", "http://schema.org/")? + /// .serialize(); + /// writer.write_triple( + /// TripleRef::new( + /// NamedNodeRef::new("http://example.com#me")?, + /// NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?, + /// NamedNodeRef::new("http://schema.org/Person")?, + /// ), + /// &mut buf, + /// )?; /// writer.finish(&mut buf)?; /// assert_eq!( - /// b" .\n", + /// b"@prefix schema: .\n a schema:Person .\n", /// buf.as_slice() /// ); /// # Result::<_,Box>::Ok(()) /// ``` - pub fn serialize(&self) -> LowLevelTurtleWriter { + pub fn serialize(self) -> LowLevelTurtleWriter { LowLevelTurtleWriter { inner: self.inner.serialize(), } @@ -696,14 +719,16 @@ impl TurtleSerializer { /// use oxrdf::{NamedNodeRef, TripleRef}; /// use oxttl::TurtleSerializer; /// -/// let mut writer = TurtleSerializer::new().serialize_to_write(Vec::new()); +/// let mut writer = TurtleSerializer::new() +/// .with_prefix("schema", "http://schema.org/")? +/// .serialize_to_write(Vec::new()); /// writer.write_triple(TripleRef::new( /// NamedNodeRef::new("http://example.com#me")?, /// NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?, /// NamedNodeRef::new("http://schema.org/Person")?, /// ))?; /// assert_eq!( -/// b" .\n", +/// b"@prefix schema: .\n a schema:Person .\n", /// writer.finish()?.as_slice() /// ); /// # Result::<_,Box>::Ok(()) @@ -733,15 +758,19 @@ impl ToWriteTurtleWriter { /// use oxttl::TurtleSerializer; /// /// # #[tokio::main(flavor = "current_thread")] -/// # async fn main() -> std::io::Result<()> { -/// let mut writer = TurtleSerializer::new().serialize_to_tokio_async_write(Vec::new()); -/// writer.write_triple(TripleRef::new( -/// NamedNodeRef::new_unchecked("http://example.com#me"), -/// NamedNodeRef::new_unchecked("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), -/// NamedNodeRef::new_unchecked("http://schema.org/Person") -/// )).await?; +/// # async fn main() -> Result<(), Box> { +/// let mut writer = TurtleSerializer::new() +/// .with_prefix("schema", "http://schema.org/")? +/// .serialize_to_tokio_async_write(Vec::new()); +/// writer +/// .write_triple(TripleRef::new( +/// NamedNodeRef::new_unchecked("http://example.com#me"), +/// NamedNodeRef::new_unchecked("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), +/// NamedNodeRef::new_unchecked("http://schema.org/Person"), +/// )) +/// .await?; /// assert_eq!( -/// b" .\n", +/// b"@prefix schema: .\n a schema:Person .\n", /// writer.finish().await?.as_slice() /// ); /// # Ok(()) @@ -775,15 +804,20 @@ impl ToTokioAsyncWriteTurtleWriter { /// use oxttl::TurtleSerializer; /// /// let mut buf = Vec::new(); -/// let mut writer = TurtleSerializer::new().serialize(); -/// writer.write_triple(TripleRef::new( -/// NamedNodeRef::new("http://example.com#me")?, -/// NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?, -/// NamedNodeRef::new("http://schema.org/Person")?, -/// ), &mut buf)?; +/// let mut writer = TurtleSerializer::new() +/// .with_prefix("schema", "http://schema.org/")? +/// .serialize(); +/// writer.write_triple( +/// TripleRef::new( +/// NamedNodeRef::new("http://example.com#me")?, +/// NamedNodeRef::new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")?, +/// NamedNodeRef::new("http://schema.org/Person")?, +/// ), +/// &mut buf, +/// )?; /// writer.finish(&mut buf)?; /// assert_eq!( -/// b" .\n", +/// b"@prefix schema: .\n a schema:Person .\n", /// buf.as_slice() /// ); /// # Result::<_,Box>::Ok(()) diff --git a/lib/sparesults/Cargo.toml b/lib/sparesults/Cargo.toml index 0ef5eb23..1bfdf4ab 100644 --- a/lib/sparesults/Cargo.toml +++ b/lib/sparesults/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sparesults" -version = "0.2.0-alpha.1" +version = "0.2.0-alpha.2-dev" authors = ["Tpt "] license = "MIT OR Apache-2.0" readme = "README.md" @@ -21,7 +21,7 @@ async-tokio = ["dep:tokio", "quick-xml/async-tokio", "json-event-parser/async-to [dependencies] json-event-parser = "0.2.0-alpha.2" memchr = "2.5" -oxrdf = { version = "0.2.0-alpha.1", path = "../oxrdf" } +oxrdf = { version = "0.2.0-alpha.2-dev", path = "../oxrdf" } quick-xml = ">=0.29, <0.32" tokio = { version = "1.29", optional = true, features = ["io-util"] } diff --git a/lib/sparesults/src/xml.rs b/lib/sparesults/src/xml.rs index c0450fac..fb038d2d 100644 --- a/lib/sparesults/src/xml.rs +++ b/lib/sparesults/src/xml.rs @@ -665,10 +665,9 @@ fn decode<'a, T>( fn map_xml_error(error: quick_xml::Error) -> io::Error { match error { - quick_xml::Error::Io(error) => match Arc::try_unwrap(error) { - Ok(error) => error, - Err(error) => io::Error::new(error.kind(), error), - }, + quick_xml::Error::Io(error) => { + Arc::try_unwrap(error).unwrap_or_else(|error| io::Error::new(error.kind(), error)) + } quick_xml::Error::UnexpectedEof(_) => io::Error::new(io::ErrorKind::UnexpectedEof, error), _ => io::Error::new(io::ErrorKind::InvalidData, error), } diff --git a/lib/spargebra/Cargo.toml b/lib/spargebra/Cargo.toml index 620bb041..09d42f88 100644 --- a/lib/spargebra/Cargo.toml +++ b/lib/spargebra/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "spargebra" -version = "0.3.0-alpha.1" +version = "0.3.0-alpha.2-dev" authors = ["Tpt "] license = "MIT OR Apache-2.0" readme = "README.md" @@ -24,7 +24,7 @@ peg = "0.8" rand = "0.8" oxiri = "0.2.3-alpha.1" oxilangtag = "0.1" -oxrdf = { version = "0.2.0-alpha.1", path = "../oxrdf" } +oxrdf = { version = "0.2.0-alpha.2-dev", path = "../oxrdf" } [lints] workspace = true diff --git a/lib/sparopt/Cargo.toml b/lib/sparopt/Cargo.toml index 27698646..bdf55805 100644 --- a/lib/sparopt/Cargo.toml +++ b/lib/sparopt/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sparopt" -version = "0.1.0-alpha.1" +version = "0.1.0-alpha.2-dev" authors = ["Tpt "] license = "MIT OR Apache-2.0" readme = "README.md" @@ -20,9 +20,9 @@ sep-0002 = ["spargebra/sep-0002"] sep-0006 = ["spargebra/sep-0006"] [dependencies] -oxrdf = { version = "0.2.0-alpha.1", path = "../oxrdf" } +oxrdf = { version = "0.2.0-alpha.2-dev", path = "../oxrdf" } rand = "0.8" -spargebra = { version = "0.3.0-alpha.1", path = "../spargebra" } +spargebra = { version = "0.3.0-alpha.2-dev", path = "../spargebra" } [lints] workspace = true From 46d3ed3f99f5140f635284de5941bada05c75c32 Mon Sep 17 00:00:00 2001 From: Tpt Date: Wed, 24 Jan 2024 09:16:13 +0100 Subject: [PATCH 193/217] Removes all debuginfo from release build Smaller binaries, most of them where stripped out anyway --- Cargo.toml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 3ec94c5e..507f5023 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -206,7 +206,9 @@ zero_sized_map_values = "warn" [profile.release] lto = true codegen-units = 1 +strip = "debuginfo" [profile.release.package.oxigraph-js] +codegen-units = 1 opt-level = "z" - +strip = "debuginfo" From 2a81106c34a8dc48adcb4375b3cfa253c903c355 Mon Sep 17 00:00:00 2001 From: Tpt Date: Wed, 24 Jan 2024 17:34:27 +0100 Subject: [PATCH 194/217] Python: use rustls by default on anything that is not Windows/macOS/iOS --- python/Cargo.toml | 10 ++++++---- python/README.md | 4 ++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/python/Cargo.toml b/python/Cargo.toml index 3cacf9bc..a63d6c89 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -18,15 +18,17 @@ name = "pyoxigraph" doctest = false [features] -default = ["native-tls"] abi3 = ["pyo3/abi3-py38"] -native-tls = ["oxigraph/http-client-native-tls"] rocksdb-pkg-config = ["oxigraph/rocksdb-pkg-config"] -rustls = ["oxigraph/http-client-rustls-native"] [dependencies] -oxigraph.path = "../lib" pyo3 = { version = "0.20.1", features = ["extension-module"] } +[target.'cfg(any(target_family = "windows", target_os = "macos", target_os = "ios"))'.dependencies] +oxigraph = { path = "../lib", features = ["http-client-native-tls"] } + +[target.'cfg(not(any(target_family = "windows", target_os = "macos", target_os = "ios")))'.dependencies] +oxigraph = { path = "../lib", features = ["http-client-rustls-native"] } + [lints] workspace = true diff --git a/python/README.md b/python/README.md index c8478a60..d9fae275 100644 --- a/python/README.md +++ b/python/README.md @@ -32,8 +32,8 @@ Pyoxigraph documentation is [available on the Oxigraph website](https://pyoxigra To build and install the development version of pyoxigraph you need to clone this git repository including submodules (`git clone --recursive https://github.com/oxigraph/oxigraph.git`) and to run `pip install .` in the `python` directory (the one this README is in). -Note that by default the installation will not use [cpython stable ABI](https://docs.python.org/3/c-api/stable.html) and will rely on the host TLS implementation. -Use `--features abi3` feature to use cpython stable ABI and use `--no-default-features --features rustls` to use [rustls](https://github.com/rustls/rustls) with the system certificates. +Note that by default the installation will not use [cpython stable ABI](https://docs.python.org/3/c-api/stable.html). +Use `--features abi3` feature to use cpython stable ABI. ## Help From ec030fb652731e3061fede89f1232d8a0bb12c5e Mon Sep 17 00:00:00 2001 From: Tpt Date: Sun, 21 Jan 2024 21:34:48 +0100 Subject: [PATCH 195/217] Python: test Pyodide wheel --- .github/workflows/tests.yml | 48 +++++++++++++++++++++++++++++++++++++ python/Cargo.toml | 5 +++- python/src/store.rs | 17 +++++++++++++ python/tests/test_store.py | 6 +++++ 4 files changed, 75 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 76074704..d1fcb1bc 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -88,6 +88,24 @@ jobs: component: clippy - run: cargo clippy --lib --tests --target wasm32-wasi -- -D warnings -D clippy::all working-directory: ./lib + - run: cargo clippy --target wasm32-wasi --features abi3 --no-default-features -- -D warnings -D clippy::all + working-directory: ./python + + clippy_wasm_emscripten: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + submodules: true + - uses: ./.github/actions/setup-rust + with: + version: 1.74.1 + target: wasm32-unknown-emscripten + component: clippy + - run: cargo clippy --lib --tests --target wasm32-unknown-emscripten -- -D warnings -D clippy::all + working-directory: ./lib + - run: cargo clippy --target wasm32-unknown-emscripten --features abi3 -- -D warnings -D clippy::all + working-directory: ./python clippy_wasm_unknown: runs-on: ubuntu-latest @@ -346,6 +364,36 @@ jobs: - run: python -m unittest working-directory: ./python/tests + python_pyodide: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + submodules: true + - uses: actions/setup-python@v4 + with: + python-version: "3.11" + cache: pip + cache-dependency-path: '**/requirements.dev.txt' + - uses: ./.github/actions/setup-rust + with: + version: nightly + target: wasm32-unknown-emscripten + - run: | + pip install pyodide-build + echo EMSCRIPTEN_VERSION=$(pyodide config get emscripten_version) >> $GITHUB_ENV + - uses: mymindstorm/setup-emsdk@v13 + with: + version: ${{ env.EMSCRIPTEN_VERSION }} + - run: pyodide build + working-directory: ./python + - run: | + pyodide venv venv + source venv/bin/activate + pip install --no-index --find-links=../dist/ pyoxigraph + python -m unittest + working-directory: ./python/tests + typos: runs-on: ubuntu-latest steps: diff --git a/python/Cargo.toml b/python/Cargo.toml index a63d6c89..94c5bbf1 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -27,7 +27,10 @@ pyo3 = { version = "0.20.1", features = ["extension-module"] } [target.'cfg(any(target_family = "windows", target_os = "macos", target_os = "ios"))'.dependencies] oxigraph = { path = "../lib", features = ["http-client-native-tls"] } -[target.'cfg(not(any(target_family = "windows", target_os = "macos", target_os = "ios")))'.dependencies] +[target.'cfg(target_family = "wasm")'.dependencies] +oxigraph.path = "../lib" + +[target.'cfg(not(any(target_family = "windows", target_os = "macos", target_os = "ios", target_family = "wasm")))'.dependencies] oxigraph = { path = "../lib", features = ["http-client-rustls-native"] } [lints] diff --git a/python/src/store.rs b/python/src/store.rs index ccc56e35..cbb4f975 100644 --- a/python/src/store.rs +++ b/python/src/store.rs @@ -48,6 +48,7 @@ pub struct PyStore { #[pymethods] impl PyStore { + #[cfg(not(target_family = "wasm"))] #[new] #[pyo3(signature = (path = None))] fn new(path: Option, py: Python<'_>) -> PyResult { @@ -63,6 +64,16 @@ impl PyStore { }) } + #[cfg(target_family = "wasm")] + #[new] + fn new(py: Python<'_>) -> PyResult { + py.allow_threads(|| { + Ok(Self { + inner: Store::new().map_err(map_storage_error)?, + }) + }) + } + /// Opens a read-only store from disk. /// /// Opening as read-only while having an other process writing the database is undefined behavior. @@ -73,6 +84,7 @@ impl PyStore { /// :return: the opened store. /// :rtype: Store /// :raises OSError: if the target directory contains invalid data or could not be accessed. + #[cfg(not(target_family = "wasm"))] #[staticmethod] fn read_only(path: &str, py: Python<'_>) -> PyResult { py.allow_threads(|| { @@ -97,6 +109,7 @@ impl PyStore { /// :return: the opened store. /// :rtype: Store /// :raises OSError: if the target directories contain invalid data or could not be accessed. + #[cfg(not(target_family = "wasm"))] #[staticmethod] #[pyo3(signature = (primary_path, secondary_path = None))] fn secondary( @@ -173,6 +186,7 @@ impl PyStore { /// >>> store.bulk_extend([Quad(NamedNode('http://example.com'), NamedNode('http://example.com/p'), Literal('1'), NamedNode('http://example.com/g'))]) /// >>> list(store) /// [ predicate= object=> graph_name=>] + #[cfg(not(target_family = "wasm"))] fn bulk_extend(&self, quads: &PyAny) -> PyResult<()> { self.inner .bulk_loader() @@ -695,6 +709,7 @@ impl PyStore { /// /// :rtype: None /// :raises OSError: if an error happens during the flush. + #[cfg(not(target_family = "wasm"))] fn flush(&self, py: Python<'_>) -> PyResult<()> { py.allow_threads(|| self.inner.flush().map_err(map_storage_error)) } @@ -705,6 +720,7 @@ impl PyStore { /// /// :rtype: None /// :raises OSError: if an error happens during the optimization. + #[cfg(not(target_family = "wasm"))] fn optimize(&self, py: Python<'_>) -> PyResult<()> { py.allow_threads(|| self.inner.optimize().map_err(map_storage_error)) } @@ -730,6 +746,7 @@ impl PyStore { /// :type target_directory: str or os.PathLike[str] /// :rtype: None /// :raises OSError: if an error happens during the backup. + #[cfg(not(target_family = "wasm"))] fn backup(&self, target_directory: PathBuf, py: Python<'_>) -> PyResult<()> { py.allow_threads(|| { self.inner diff --git a/python/tests/test_store.py b/python/tests/test_store.py index 9b662ad6..abda28ca 100644 --- a/python/tests/test_store.py +++ b/python/tests/test_store.py @@ -1,4 +1,5 @@ import gc +import sys import unittest from io import BytesIO, StringIO, UnsupportedOperation from pathlib import Path @@ -26,6 +27,7 @@ bar = NamedNode("http://bar") baz = NamedNode("http://baz") triple = Triple(foo, foo, foo) graph = NamedNode("http://graph") +is_wasm = sys.platform == "emscripten" class TestStore(unittest.TestCase): @@ -49,6 +51,7 @@ class TestStore(unittest.TestCase): ) self.assertEqual(len(store), 2) + @unittest.skipIf(is_wasm, "Not supported with WASM") def test_bulk_extend(self) -> None: store = Store() store.bulk_extend( @@ -244,6 +247,7 @@ class TestStore(unittest.TestCase): store.update("DELETE WHERE { ?v ?v ?v }") self.assertEqual(len(store), 0) + @unittest.skipIf(is_wasm, "Not supported with WASM") def test_update_load(self) -> None: store = Store() store.update("LOAD ") @@ -381,6 +385,7 @@ class TestStore(unittest.TestCase): self.assertEqual(list(store.named_graphs()), []) self.assertEqual(list(store), []) + @unittest.skipIf(is_wasm, "Not supported with WASM") def test_read_only(self) -> None: quad = Quad(foo, bar, baz, graph) with TemporaryDirectory() as dir: @@ -391,6 +396,7 @@ class TestStore(unittest.TestCase): store = Store.read_only(dir) self.assertEqual(list(store), [quad]) + @unittest.skipIf(is_wasm, "Not supported with WASM") def test_secondary(self) -> None: quad = Quad(foo, bar, baz, graph) with TemporaryDirectory() as dir: From 2b6ac5c1954f1b531f3f57ed8a35425e795f2019 Mon Sep 17 00:00:00 2001 From: Tpt Date: Wed, 24 Jan 2024 21:46:21 +0100 Subject: [PATCH 196/217] Release v0.4.0-alpha.3 --- .github/workflows/manylinux_build.sh | 10 +++++----- .github/workflows/musllinux_build.sh | 6 +++--- CHANGELOG.md | 15 +++++++++++++++ cli/Cargo.toml | 4 ++-- js/Cargo.toml | 2 +- lib/Cargo.toml | 15 +++++++-------- lib/oxrdf/Cargo.toml | 2 +- lib/oxrdfio/Cargo.toml | 8 ++++---- lib/oxrdfxml/Cargo.toml | 4 ++-- lib/oxttl/Cargo.toml | 4 ++-- lib/sparesults/Cargo.toml | 4 ++-- lib/spargebra/Cargo.toml | 4 ++-- lib/sparopt/Cargo.toml | 6 +++--- oxrocksdb-sys/Cargo.toml | 2 +- python/Cargo.toml | 2 +- 15 files changed, 51 insertions(+), 37 deletions(-) diff --git a/.github/workflows/manylinux_build.sh b/.github/workflows/manylinux_build.sh index 48c69a2a..cf56e41d 100644 --- a/.github/workflows/manylinux_build.sh +++ b/.github/workflows/manylinux_build.sh @@ -11,14 +11,14 @@ cd python python3.12 -m venv venv source venv/bin/activate pip install -r requirements.dev.txt -maturin develop --release --no-default-features --features rustls +maturin develop --release python generate_stubs.py pyoxigraph pyoxigraph.pyi --ruff -maturin build --release --no-default-features --features abi3 --features rustls --compatibility manylinux2014 +maturin build --release --features abi3 --compatibility manylinux2014 if [ %for_each_version% ]; then for VERSION in 8 9 10 11 12; do - maturin build --release --no-default-features --features rustls --interpreter "python3.$VERSION" --compatibility manylinux2014 + maturin build --release --interpreter "python3.$VERSION" --compatibility manylinux2014 done for VERSION in 9 10; do - maturin build --release --no-default-features --features rustls --interpreter "pypy3.$VERSION" --compatibility manylinux2014 + maturin build --release --interpreter "pypy3.$VERSION" --compatibility manylinux2014 done -fi \ No newline at end of file +fi diff --git a/.github/workflows/musllinux_build.sh b/.github/workflows/musllinux_build.sh index 17d1233d..75d51ddf 100644 --- a/.github/workflows/musllinux_build.sh +++ b/.github/workflows/musllinux_build.sh @@ -9,11 +9,11 @@ cd python python3.12 -m venv venv source venv/bin/activate pip install -r requirements.dev.txt -maturin develop --release --no-default-features --features rustls +maturin develop --release python generate_stubs.py pyoxigraph pyoxigraph.pyi --ruff -maturin build --release --no-default-features --features abi3 --features rustls --compatibility musllinux_1_2 +maturin build --release --features abi3 --compatibility musllinux_1_2 if [ %for_each_version% ]; then for VERSION in 8 9 10 11 12; do - maturin build --release --no-default-features --features rustls --interpreter "python3.$VERSION" --compatibility musllinux_1_2 + maturin build --release --interpreter "python3.$VERSION" --compatibility musllinux_1_2 done fi diff --git a/CHANGELOG.md b/CHANGELOG.md index 2da6d30a..d52008ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,18 @@ +## [0.4.0-alpha.3] - 2024-01-25 + +### Added +- `oxttl`: expose base IRIs. +- `oxttl`: allows to inject prefixes for serialization. +- `oxrdf`: `vocab::geosparql::WKT_LITERAL`. + +### Changed +- Turtle: Fixes parsing bug with escaped dot at the end of local name. +- `oxttl`: Changes `prefixes` getter return type. +- JS: simplify build. +- Python: uses rustls by default all platforms that are not Windows/macOS/iOS/WASM. +- Strips debug info of the Rust std library in release build. + + ## [0.4.0-alpha.2] - 2024-01-08 ### Added diff --git a/cli/Cargo.toml b/cli/Cargo.toml index befeb158..0c00e383 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "oxigraph-cli" -version = "0.4.0-alpha.3-dev" +version = "0.4.0-alpha.3" authors = ["Tpt "] license = "MIT OR Apache-2.0" readme = "README.md" @@ -29,7 +29,7 @@ rustls-webpki = ["oxigraph/http-client-rustls-webpki"] anyhow = "1.0.72" oxhttp = { version = "0.2.0-alpha.3", features = ["flate2"] } clap = { version = "4.0", features = ["derive"] } -oxigraph = { version = "0.4.0-alpha.3-dev", path = "../lib" } +oxigraph = { version = "0.4.0-alpha.3", path = "../lib" } rand = "0.8" url = "2.4" oxiri = "0.2.3-alpha.1" diff --git a/js/Cargo.toml b/js/Cargo.toml index 8caba72e..16764596 100644 --- a/js/Cargo.toml +++ b/js/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "oxigraph-js" -version = "0.4.0-alpha.3-dev" +version = "0.4.0-alpha.3" authors = ["Tpt "] license = "MIT OR Apache-2.0" readme = "README.md" diff --git a/lib/Cargo.toml b/lib/Cargo.toml index 44005104..12a62f22 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "oxigraph" -version = "0.4.0-alpha.3-dev" +version = "0.4.0-alpha.3" authors = ["Tpt "] license = "MIT OR Apache-2.0" readme = "README.md" @@ -32,21 +32,21 @@ json-event-parser = "0.2.0-alpha.2" md-5 = "0.10" oxilangtag = "0.1" oxiri = "0.2.3-alpha.1" -oxrdf = { version = "0.2.0-alpha.2-dev", path = "oxrdf", features = ["rdf-star", "oxsdatatypes"] } -oxrdfio = { version = "0.1.0-alpha.2-dev", path = "oxrdfio", features = ["rdf-star"] } +oxrdf = { version = "0.2.0-alpha.2", path = "oxrdf", features = ["rdf-star", "oxsdatatypes"] } +oxrdfio = { version = "0.1.0-alpha.2", path = "oxrdfio", features = ["rdf-star"] } oxsdatatypes = { version = "0.2.0-alpha.1", path = "oxsdatatypes" } rand = "0.8" regex = "1.7" sha1 = "0.10" sha2 = "0.10" siphasher = ">=0.3, <2.0" -sparesults = { version = "0.2.0-alpha.2-dev", path = "sparesults", features = ["rdf-star"] } -spargebra = { version = "0.3.0-alpha.2-dev", path = "spargebra", features = ["rdf-star", "sep-0002", "sep-0006"] } -sparopt = { version = "0.1.0-alpha.2-dev", path = "sparopt", features = ["rdf-star", "sep-0002", "sep-0006"] } +sparesults = { version = "0.2.0-alpha.2", path = "sparesults", features = ["rdf-star"] } +spargebra = { version = "0.3.0-alpha.2", path = "spargebra", features = ["rdf-star", "sep-0002", "sep-0006"] } +sparopt = { version = "0.1.0-alpha.2", path = "sparopt", features = ["rdf-star", "sep-0002", "sep-0006"] } [target.'cfg(not(target_family = "wasm"))'.dependencies] libc = "0.2.147" -oxrocksdb-sys = { version = "0.4.0-alpha.3-dev", path = "../oxrocksdb-sys" } +oxrocksdb-sys = { version = "0.4.0-alpha.3", path = "../oxrocksdb-sys" } oxhttp = { version = "0.2.0-alpha.3", optional = true } [target.'cfg(all(target_family = "wasm", target_os = "unknown"))'.dependencies] @@ -62,7 +62,6 @@ zstd = ">=0.12, <0.14" workspace = true [package.metadata.docs.rs] -all-features = true rustdoc-args = ["--cfg", "docsrs"] [[bench]] diff --git a/lib/oxrdf/Cargo.toml b/lib/oxrdf/Cargo.toml index c52b13ca..93fbf18d 100644 --- a/lib/oxrdf/Cargo.toml +++ b/lib/oxrdf/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "oxrdf" -version = "0.2.0-alpha.2-dev" +version = "0.2.0-alpha.2" authors = ["Tpt "] license = "MIT OR Apache-2.0" readme = "README.md" diff --git a/lib/oxrdfio/Cargo.toml b/lib/oxrdfio/Cargo.toml index 027c5eba..1d10c046 100644 --- a/lib/oxrdfio/Cargo.toml +++ b/lib/oxrdfio/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "oxrdfio" -version = "0.1.0-alpha.2-dev" +version = "0.1.0-alpha.2" authors = ["Tpt "] license = "MIT OR Apache-2.0" readme = "README.md" @@ -19,9 +19,9 @@ async-tokio = ["dep:tokio", "oxrdfxml/async-tokio", "oxttl/async-tokio"] rdf-star = ["oxrdf/rdf-star", "oxttl/rdf-star"] [dependencies] -oxrdf = { version = "0.2.0-alpha.2-dev", path = "../oxrdf" } -oxrdfxml = { version = "0.1.0-alpha.2-dev", path = "../oxrdfxml" } -oxttl = { version = "0.1.0-alpha.2-dev", path = "../oxttl" } +oxrdf = { version = "0.2.0-alpha.2", path = "../oxrdf" } +oxrdfxml = { version = "0.1.0-alpha.2", path = "../oxrdfxml" } +oxttl = { version = "0.1.0-alpha.2", path = "../oxttl" } tokio = { version = "1.29", optional = true, features = ["io-util"] } [dev-dependencies] diff --git a/lib/oxrdfxml/Cargo.toml b/lib/oxrdfxml/Cargo.toml index 6546b809..c1a6e18f 100644 --- a/lib/oxrdfxml/Cargo.toml +++ b/lib/oxrdfxml/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "oxrdfxml" -version = "0.1.0-alpha.2-dev" +version = "0.1.0-alpha.2" authors = ["Tpt "] license = "MIT OR Apache-2.0" readme = "README.md" @@ -18,7 +18,7 @@ default = [] async-tokio = ["dep:tokio", "quick-xml/async-tokio"] [dependencies] -oxrdf = { version = "0.2.0-alpha.2-dev", path = "../oxrdf" } +oxrdf = { version = "0.2.0-alpha.2", path = "../oxrdf" } oxilangtag = "0.1" oxiri = "0.2.3-alpha.1" quick-xml = ">=0.29, <0.32" diff --git a/lib/oxttl/Cargo.toml b/lib/oxttl/Cargo.toml index 25d87b48..d18cb7e0 100644 --- a/lib/oxttl/Cargo.toml +++ b/lib/oxttl/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "oxttl" -version = "0.1.0-alpha.2-dev" +version = "0.1.0-alpha.2" authors = ["Tpt "] license = "MIT OR Apache-2.0" readme = "README.md" @@ -20,7 +20,7 @@ async-tokio = ["dep:tokio"] [dependencies] memchr = "2.5" -oxrdf = { version = "0.2.0-alpha.2-dev", path = "../oxrdf" } +oxrdf = { version = "0.2.0-alpha.2", path = "../oxrdf" } oxiri = "0.2.3-alpha.1" oxilangtag = "0.1" tokio = { version = "1.29", optional = true, features = ["io-util"] } diff --git a/lib/sparesults/Cargo.toml b/lib/sparesults/Cargo.toml index 1bfdf4ab..35d03deb 100644 --- a/lib/sparesults/Cargo.toml +++ b/lib/sparesults/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sparesults" -version = "0.2.0-alpha.2-dev" +version = "0.2.0-alpha.2" authors = ["Tpt "] license = "MIT OR Apache-2.0" readme = "README.md" @@ -21,7 +21,7 @@ async-tokio = ["dep:tokio", "quick-xml/async-tokio", "json-event-parser/async-to [dependencies] json-event-parser = "0.2.0-alpha.2" memchr = "2.5" -oxrdf = { version = "0.2.0-alpha.2-dev", path = "../oxrdf" } +oxrdf = { version = "0.2.0-alpha.2", path = "../oxrdf" } quick-xml = ">=0.29, <0.32" tokio = { version = "1.29", optional = true, features = ["io-util"] } diff --git a/lib/spargebra/Cargo.toml b/lib/spargebra/Cargo.toml index 09d42f88..ac6c049b 100644 --- a/lib/spargebra/Cargo.toml +++ b/lib/spargebra/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "spargebra" -version = "0.3.0-alpha.2-dev" +version = "0.3.0-alpha.2" authors = ["Tpt "] license = "MIT OR Apache-2.0" readme = "README.md" @@ -24,7 +24,7 @@ peg = "0.8" rand = "0.8" oxiri = "0.2.3-alpha.1" oxilangtag = "0.1" -oxrdf = { version = "0.2.0-alpha.2-dev", path = "../oxrdf" } +oxrdf = { version = "0.2.0-alpha.2", path = "../oxrdf" } [lints] workspace = true diff --git a/lib/sparopt/Cargo.toml b/lib/sparopt/Cargo.toml index bdf55805..c0f397ab 100644 --- a/lib/sparopt/Cargo.toml +++ b/lib/sparopt/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sparopt" -version = "0.1.0-alpha.2-dev" +version = "0.1.0-alpha.2" authors = ["Tpt "] license = "MIT OR Apache-2.0" readme = "README.md" @@ -20,9 +20,9 @@ sep-0002 = ["spargebra/sep-0002"] sep-0006 = ["spargebra/sep-0006"] [dependencies] -oxrdf = { version = "0.2.0-alpha.2-dev", path = "../oxrdf" } +oxrdf = { version = "0.2.0-alpha.2", path = "../oxrdf" } rand = "0.8" -spargebra = { version = "0.3.0-alpha.2-dev", path = "../spargebra" } +spargebra = { version = "0.3.0-alpha.2", path = "../spargebra" } [lints] workspace = true diff --git a/oxrocksdb-sys/Cargo.toml b/oxrocksdb-sys/Cargo.toml index 892243cc..3af73776 100644 --- a/oxrocksdb-sys/Cargo.toml +++ b/oxrocksdb-sys/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "oxrocksdb-sys" -version = "0.4.0-alpha.3-dev" +version = "0.4.0-alpha.3" authors = ["Tpt "] license = "GPL-2.0 OR Apache-2.0" repository = "https://github.com/oxigraph/oxigraph/tree/main/oxrocksdb-sys" diff --git a/python/Cargo.toml b/python/Cargo.toml index 94c5bbf1..31d1f812 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyoxigraph" -version = "0.4.0-alpha.3-dev" +version = "0.4.0-alpha.3" authors = ["Tpt "] license = "MIT OR Apache-2.0" readme = "README.md" From c0d245871c7d32b044ac04756299a1c225b823d1 Mon Sep 17 00:00:00 2001 From: Tpt Date: Sat, 27 Jan 2024 18:03:44 +0100 Subject: [PATCH 197/217] Simplifies the reexport of sub crates --- Cargo.lock | 24 ++++++++++++------------ lib/oxrdf/src/dataset.rs | 8 ++++---- lib/oxrdf/src/graph.rs | 2 +- lib/sparesults/src/solution.rs | 2 +- lib/src/io/mod.rs | 5 +---- lib/src/model.rs | 7 +------ lib/src/sparql/eval.rs | 2 +- lib/src/sparql/results.rs | 5 +---- 8 files changed, 22 insertions(+), 33 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 70b1abfb..b63ea4a0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1018,7 +1018,7 @@ dependencies = [ [[package]] name = "oxigraph" -version = "0.4.0-alpha.3-dev" +version = "0.4.0-alpha.3" dependencies = [ "codspeed-criterion-compat", "digest", @@ -1048,7 +1048,7 @@ dependencies = [ [[package]] name = "oxigraph-cli" -version = "0.4.0-alpha.3-dev" +version = "0.4.0-alpha.3" dependencies = [ "anyhow", "assert_cmd", @@ -1067,7 +1067,7 @@ dependencies = [ [[package]] name = "oxigraph-js" -version = "0.4.0-alpha.3-dev" +version = "0.4.0-alpha.3" dependencies = [ "console_error_panic_hook", "js-sys", @@ -1104,7 +1104,7 @@ checksum = "b225dad32cfaa43a960b93f01fa7f87528ac07e794b80f6d9a0153e0222557e2" [[package]] name = "oxrdf" -version = "0.2.0-alpha.2-dev" +version = "0.2.0-alpha.2" dependencies = [ "oxilangtag", "oxiri", @@ -1114,7 +1114,7 @@ dependencies = [ [[package]] name = "oxrdfio" -version = "0.1.0-alpha.2-dev" +version = "0.1.0-alpha.2" dependencies = [ "oxrdf", "oxrdfxml", @@ -1124,7 +1124,7 @@ dependencies = [ [[package]] name = "oxrdfxml" -version = "0.1.0-alpha.2-dev" +version = "0.1.0-alpha.2" dependencies = [ "oxilangtag", "oxiri", @@ -1135,7 +1135,7 @@ dependencies = [ [[package]] name = "oxrocksdb-sys" -version = "0.4.0-alpha.3-dev" +version = "0.4.0-alpha.3" dependencies = [ "bindgen", "cc", @@ -1152,7 +1152,7 @@ dependencies = [ [[package]] name = "oxttl" -version = "0.1.0-alpha.2-dev" +version = "0.1.0-alpha.2" dependencies = [ "memchr", "oxilangtag", @@ -1387,7 +1387,7 @@ dependencies = [ [[package]] name = "pyoxigraph" -version = "0.4.0-alpha.3-dev" +version = "0.4.0-alpha.3" dependencies = [ "oxigraph", "pyo3", @@ -1719,7 +1719,7 @@ checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" [[package]] name = "sparesults" -version = "0.2.0-alpha.2-dev" +version = "0.2.0-alpha.2" dependencies = [ "json-event-parser", "memchr", @@ -1730,7 +1730,7 @@ dependencies = [ [[package]] name = "spargebra" -version = "0.3.0-alpha.2-dev" +version = "0.3.0-alpha.2" dependencies = [ "oxilangtag", "oxiri", @@ -1741,7 +1741,7 @@ dependencies = [ [[package]] name = "sparopt" -version = "0.1.0-alpha.2-dev" +version = "0.1.0-alpha.2" dependencies = [ "oxrdf", "rand", diff --git a/lib/oxrdf/src/dataset.rs b/lib/oxrdf/src/dataset.rs index ed6249a4..8412a8aa 100644 --- a/lib/oxrdf/src/dataset.rs +++ b/lib/oxrdf/src/dataset.rs @@ -925,8 +925,8 @@ impl PartialEq for Dataset { impl Eq for Dataset {} impl<'a> IntoIterator for &'a Dataset { - type Item = QuadRef<'a>; type IntoIter = Iter<'a>; + type Item = QuadRef<'a>; fn into_iter(self) -> Self::IntoIter { self.iter() @@ -1283,8 +1283,8 @@ impl<'a> GraphView<'a> { } impl<'a> IntoIterator for GraphView<'a> { - type Item = TripleRef<'a>; type IntoIter = GraphViewIter<'a>; + type Item = TripleRef<'a>; fn into_iter(self) -> Self::IntoIter { self.iter() @@ -1292,8 +1292,8 @@ impl<'a> IntoIterator for GraphView<'a> { } impl<'a, 'b> IntoIterator for &'b GraphView<'a> { - type Item = TripleRef<'a>; type IntoIter = GraphViewIter<'a>; + type Item = TripleRef<'a>; fn into_iter(self) -> Self::IntoIter { self.iter() @@ -1494,8 +1494,8 @@ impl<'a, 'b, T: Into>> Extend for GraphViewMut<'a> { } impl<'a> IntoIterator for &'a GraphViewMut<'a> { - type Item = TripleRef<'a>; type IntoIter = GraphViewIter<'a>; + type Item = TripleRef<'a>; fn into_iter(self) -> Self::IntoIter { self.iter() diff --git a/lib/oxrdf/src/graph.rs b/lib/oxrdf/src/graph.rs index 5459b65c..33f67132 100644 --- a/lib/oxrdf/src/graph.rs +++ b/lib/oxrdf/src/graph.rs @@ -229,8 +229,8 @@ impl PartialEq for Graph { impl Eq for Graph {} impl<'a> IntoIterator for &'a Graph { - type Item = TripleRef<'a>; type IntoIter = Iter<'a>; + type Item = TripleRef<'a>; fn into_iter(self) -> Self::IntoIter { self.iter() diff --git a/lib/sparesults/src/solution.rs b/lib/sparesults/src/solution.rs index a1364861..826a9eea 100644 --- a/lib/sparesults/src/solution.rs +++ b/lib/sparesults/src/solution.rs @@ -171,8 +171,8 @@ impl>, S: Into>>> From<(V, S)> for Quer } impl<'a> IntoIterator for &'a QuerySolution { - type Item = (&'a Variable, &'a Term); type IntoIter = Iter<'a>; + type Item = (&'a Variable, &'a Term); #[inline] fn into_iter(self) -> Self::IntoIter { diff --git a/lib/src/io/mod.rs b/lib/src/io/mod.rs index 883abb2a..1b15bc8e 100644 --- a/lib/src/io/mod.rs +++ b/lib/src/io/mod.rs @@ -36,7 +36,4 @@ pub use self::format::{DatasetFormat, GraphFormat}; pub use self::read::{DatasetParser, GraphParser}; #[allow(deprecated)] pub use self::write::{DatasetSerializer, GraphSerializer}; -pub use oxrdfio::{ - FromReadQuadReader, ParseError, RdfFormat, RdfParser, RdfSerializer, SyntaxError, TextPosition, - ToWriteQuadWriter, -}; +pub use oxrdfio::*; diff --git a/lib/src/model.rs b/lib/src/model.rs index ef8cbd1d..3a9fd053 100644 --- a/lib/src/model.rs +++ b/lib/src/model.rs @@ -17,9 +17,4 @@ //! assert_eq!(vec![triple], results); //! ``` -pub use oxrdf::{ - dataset, graph, vocab, BlankNode, BlankNodeIdParseError, BlankNodeRef, Dataset, Graph, - GraphName, GraphNameRef, IriParseError, LanguageTagParseError, Literal, LiteralRef, NamedNode, - NamedNodeRef, NamedOrBlankNode, NamedOrBlankNodeRef, Quad, QuadRef, Subject, SubjectRef, Term, - TermParseError, TermRef, Triple, TripleRef, -}; +pub use oxrdf::*; diff --git a/lib/src/sparql/eval.rs b/lib/src/sparql/eval.rs index 174f41fa..19c6884d 100644 --- a/lib/src/sparql/eval.rs +++ b/lib/src/sparql/eval.rs @@ -111,8 +111,8 @@ impl EncodedTuple { } impl IntoIterator for EncodedTuple { - type Item = Option; type IntoIter = std::vec::IntoIter>; + type Item = Option; fn into_iter(self) -> Self::IntoIter { self.inner.into_iter() diff --git a/lib/src/sparql/results.rs b/lib/src/sparql/results.rs index 88aff947..00f8cc3d 100644 --- a/lib/src/sparql/results.rs +++ b/lib/src/sparql/results.rs @@ -41,7 +41,4 @@ //! ); //! ``` -pub use sparesults::{ - FromReadQueryResultsReader, FromReadSolutionsReader, ParseError, QueryResultsFormat, - QueryResultsParser, QueryResultsSerializer, SyntaxError, TextPosition, -}; +pub use sparesults::*; From 1e4326a2c5605d224c41eb02ab8ce9d2f12913a6 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Wed, 7 Feb 2024 01:21:33 -0500 Subject: [PATCH 198/217] Optimize format performance As seen in the https://rust.godbolt.org/z/Y8djWsq1P - write! macro produces significantly more code than a write_str call, so this change should have somewhat better performance. To my knowledge, a lot of ppl tried to solve this optimization in the compiler, but no luck yet, so may help compiler ourselves for now. --- lib/oxrdf/src/blank_node.rs | 2 +- lib/oxrdf/src/triple.rs | 2 +- lib/oxrdf/src/variable.rs | 2 +- lib/oxrdfio/src/error.rs | 2 +- lib/oxsdatatypes/src/date_time.rs | 14 +- lib/oxsdatatypes/src/decimal.rs | 10 +- lib/oxsdatatypes/src/duration.rs | 16 +- lib/oxsdatatypes/src/integer.rs | 2 +- lib/oxttl/src/trig.rs | 4 +- lib/spargebra/src/algebra.rs | 518 +++++++++++++++--------------- lib/spargebra/src/query.rs | 56 ++-- lib/spargebra/src/term.rs | 64 ++-- lib/spargebra/src/update.rs | 84 ++--- lib/sparql-smith/src/lib.rs | 68 ++-- lib/src/sparql/error.rs | 6 +- testsuite/src/sparql_evaluator.rs | 4 +- 16 files changed, 428 insertions(+), 426 deletions(-) diff --git a/lib/oxrdf/src/blank_node.rs b/lib/oxrdf/src/blank_node.rs index 9603cd30..ce07226b 100644 --- a/lib/oxrdf/src/blank_node.rs +++ b/lib/oxrdf/src/blank_node.rs @@ -351,7 +351,7 @@ pub struct BlankNodeIdParseError; impl fmt::Display for BlankNodeIdParseError { #[inline] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "The blank node identifier is invalid") + f.write_str("The blank node identifier is invalid") } } diff --git a/lib/oxrdf/src/triple.rs b/lib/oxrdf/src/triple.rs index 813982d0..76039644 100644 --- a/lib/oxrdf/src/triple.rs +++ b/lib/oxrdf/src/triple.rs @@ -983,7 +983,7 @@ impl fmt::Display for GraphNameRef<'_> { match self { Self::NamedNode(node) => node.fmt(f), Self::BlankNode(node) => node.fmt(f), - Self::DefaultGraph => write!(f, "DEFAULT"), + Self::DefaultGraph => f.write_str("DEFAULT"), } } } diff --git a/lib/oxrdf/src/variable.rs b/lib/oxrdf/src/variable.rs index 044c73e7..30af6c0f 100644 --- a/lib/oxrdf/src/variable.rs +++ b/lib/oxrdf/src/variable.rs @@ -218,7 +218,7 @@ pub struct VariableNameParseError; impl fmt::Display for VariableNameParseError { #[inline] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "The variable name is invalid") + f.write_str("The variable name is invalid") } } diff --git a/lib/oxrdfio/src/error.rs b/lib/oxrdfio/src/error.rs index 3b4691f9..78f9b998 100644 --- a/lib/oxrdfio/src/error.rs +++ b/lib/oxrdfio/src/error.rs @@ -144,7 +144,7 @@ impl fmt::Display for SyntaxError { match &self.inner { SyntaxErrorKind::Turtle(e) => e.fmt(f), SyntaxErrorKind::RdfXml(e) => e.fmt(f), - SyntaxErrorKind::Msg { msg } => write!(f, "{msg}"), + SyntaxErrorKind::Msg { msg } => f.write_str(msg), } } } diff --git a/lib/oxsdatatypes/src/date_time.rs b/lib/oxsdatatypes/src/date_time.rs index 127990df..1d165bbb 100644 --- a/lib/oxsdatatypes/src/date_time.rs +++ b/lib/oxsdatatypes/src/date_time.rs @@ -281,7 +281,7 @@ impl fmt::Display for DateTime { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let year = self.year(); if year < 0 { - write!(f, "-")?; + f.write_str("-")?; } let second = self.second(); write!( @@ -783,7 +783,7 @@ impl fmt::Display for Date { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let year = self.year(); if year < 0 { - write!(f, "-")?; + f.write_str("-")?; } write!(f, "{:04}-{:02}-{:02}", year.abs(), self.month(), self.day())?; if let Some(timezone_offset) = self.timezone_offset() { @@ -925,7 +925,7 @@ impl fmt::Display for GYearMonth { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let year = self.year(); if year < 0 { - write!(f, "-")?; + f.write_str("-")?; } write!(f, "{:04}-{:02}", year.abs(), self.month())?; if let Some(timezone_offset) = self.timezone_offset() { @@ -1066,7 +1066,7 @@ impl fmt::Display for GYear { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let year = self.year(); if year < 0 { - write!(f, "-")?; + f.write_str("-")?; } write!(f, "{:04}", year.abs())?; if let Some(timezone_offset) = self.timezone_offset() { @@ -1544,7 +1544,7 @@ impl fmt::Display for TimezoneOffset { #[inline] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self.offset { - 0 => write!(f, "Z"), + 0 => f.write_str("Z"), offset if offset < 0 => write!(f, "-{:02}:{:02}", -offset / 60, -offset % 60), offset => write!(f, "+{:02}:{:02}", offset / 60, offset % 60), } @@ -2060,7 +2060,7 @@ impl fmt::Display for ParseDateTimeError { } ParseDateTimeErrorKind::Overflow(error) => error.fmt(f), ParseDateTimeErrorKind::InvalidTimezone(error) => error.fmt(f), - ParseDateTimeErrorKind::Message(msg) => write!(f, "{msg}"), + ParseDateTimeErrorKind::Message(msg) => f.write_str(msg), } } } @@ -2415,7 +2415,7 @@ pub struct DateTimeOverflowError; impl fmt::Display for DateTimeOverflowError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "overflow during xsd:dateTime computation") + f.write_str("overflow during xsd:dateTime computation") } } diff --git a/lib/oxsdatatypes/src/decimal.rs b/lib/oxsdatatypes/src/decimal.rs index 0082ca8a..b526e541 100644 --- a/lib/oxsdatatypes/src/decimal.rs +++ b/lib/oxsdatatypes/src/decimal.rs @@ -638,10 +638,10 @@ const PARSE_UNEXPECTED_END: ParseDecimalError = ParseDecimalError { impl fmt::Display for ParseDecimalError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self.kind { - DecimalParseErrorKind::Overflow => write!(f, "Value overflow"), - DecimalParseErrorKind::Underflow => write!(f, "Value underflow"), - DecimalParseErrorKind::UnexpectedChar => write!(f, "Unexpected character"), - DecimalParseErrorKind::UnexpectedEnd => write!(f, "Unexpected end of string"), + DecimalParseErrorKind::Overflow => f.write_str("Value overflow"), + DecimalParseErrorKind::Underflow => f.write_str("Value underflow"), + DecimalParseErrorKind::UnexpectedChar => f.write_str("Unexpected character"), + DecimalParseErrorKind::UnexpectedEnd => f.write_str("Unexpected end of string"), } } } @@ -664,7 +664,7 @@ pub struct TooLargeForDecimalError; impl fmt::Display for TooLargeForDecimalError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "Value too large for xsd:decimal internal representation") + f.write_str("Value too large for xsd:decimal internal representation") } } diff --git a/lib/oxsdatatypes/src/duration.rs b/lib/oxsdatatypes/src/duration.rs index 863fcb38..35f00bce 100644 --- a/lib/oxsdatatypes/src/duration.rs +++ b/lib/oxsdatatypes/src/duration.rs @@ -206,12 +206,12 @@ impl fmt::Display for Duration { return Err(fmt::Error); // Not able to format with only a part of the duration that is negative } if ym < 0 || ss < 0.into() { - write!(f, "-")?; + f.write_str("-")?; } - write!(f, "P")?; + f.write_str("P")?; if ym == 0 && ss == 0.into() { - return write!(f, "T0S"); + return f.write_str("T0S"); } { @@ -245,7 +245,7 @@ impl fmt::Display for Duration { } if h != 0 || m != 0 || s != 0.into() { - write!(f, "T")?; + f.write_str("T")?; if h != 0 { write!(f, "{}H", h.abs())?; } @@ -423,7 +423,7 @@ impl fmt::Display for YearMonthDuration { #[inline] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { if self.months == 0 { - write!(f, "P0M") + f.write_str("P0M") } else { Duration::from(*self).fmt(f) } @@ -948,7 +948,7 @@ const OVERFLOW_ERROR: ParseDurationError = ParseDurationError { impl fmt::Display for ParseDurationError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.msg) + f.write_str(self.msg) } } @@ -968,7 +968,7 @@ pub struct DurationOverflowError; impl fmt::Display for DurationOverflowError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "overflow during xsd:duration computation") + f.write_str("overflow during xsd:duration computation") } } @@ -980,7 +980,7 @@ pub struct OppositeSignInDurationComponentsError; impl fmt::Display for OppositeSignInDurationComponentsError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "The xsd:yearMonthDuration and xsd:dayTimeDuration components of a xsd:duration can't have opposite sign") + f.write_str("The xsd:yearMonthDuration and xsd:dayTimeDuration components of a xsd:duration can't have opposite sign") } } diff --git a/lib/oxsdatatypes/src/integer.rs b/lib/oxsdatatypes/src/integer.rs index e76ae62e..b28638d2 100644 --- a/lib/oxsdatatypes/src/integer.rs +++ b/lib/oxsdatatypes/src/integer.rs @@ -269,7 +269,7 @@ pub struct TooLargeForIntegerError; impl fmt::Display for TooLargeForIntegerError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "Value too large for xsd:integer internal representation") + f.write_str("Value too large for xsd:integer internal representation") } } diff --git a/lib/oxttl/src/trig.rs b/lib/oxttl/src/trig.rs index aca75110..21434ed4 100644 --- a/lib/oxttl/src/trig.rs +++ b/lib/oxttl/src/trig.rs @@ -987,7 +987,7 @@ struct TurtlePredicate<'a> { impl<'a> fmt::Display for TurtlePredicate<'a> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { if self.named_node == rdf::TYPE { - write!(f, "a") + f.write_str("a") } else { TurtleTerm { term: self.named_node.into(), @@ -1027,7 +1027,7 @@ impl<'a> fmt::Display for TurtleTerm<'a> { _ => false, }; if inline { - write!(f, "{value}") + f.write_str(value) } else if v.is_plain() { write!(f, "{v}") } else { diff --git a/lib/spargebra/src/algebra.rs b/lib/spargebra/src/algebra.rs index 37d1da8a..9b38b71f 100644 --- a/lib/spargebra/src/algebra.rs +++ b/lib/spargebra/src/algebra.rs @@ -23,45 +23,45 @@ impl PropertyPathExpression { match self { Self::NamedNode(p) => write!(f, "{p}"), Self::Reverse(p) => { - write!(f, "(reverse ")?; + f.write_str("(reverse ")?; p.fmt_sse(f)?; - write!(f, ")") + f.write_str(")") } Self::Alternative(a, b) => { - write!(f, "(alt ")?; + f.write_str("(alt ")?; a.fmt_sse(f)?; - write!(f, " ")?; + f.write_str(" ")?; b.fmt_sse(f)?; - write!(f, ")") + f.write_str(")") } Self::Sequence(a, b) => { - write!(f, "(seq ")?; + f.write_str("(seq ")?; a.fmt_sse(f)?; - write!(f, " ")?; + f.write_str(" ")?; b.fmt_sse(f)?; - write!(f, ")") + f.write_str(")") } Self::ZeroOrMore(p) => { - write!(f, "(path* ")?; + f.write_str("(path* ")?; p.fmt_sse(f)?; - write!(f, ")") + f.write_str(")") } Self::OneOrMore(p) => { - write!(f, "(path+ ")?; + f.write_str("(path+ ")?; p.fmt_sse(f)?; - write!(f, ")") + f.write_str(")") } Self::ZeroOrOne(p) => { - write!(f, "(path? ")?; + f.write_str("(path? ")?; p.fmt_sse(f)?; - write!(f, ")") + f.write_str(")") } Self::NegatedPropertySet(p) => { - write!(f, "(notoneof")?; + f.write_str("(notoneof")?; for p in p { write!(f, " {p}")?; } - write!(f, ")") + f.write_str(")") } } } @@ -78,14 +78,14 @@ impl fmt::Display for PropertyPathExpression { Self::OneOrMore(p) => write!(f, "({p})+"), Self::ZeroOrOne(p) => write!(f, "({p})?"), Self::NegatedPropertySet(p) => { - write!(f, "!(")?; + f.write_str("!(")?; for (i, c) in p.iter().enumerate() { if i > 0 { - write!(f, " | ")?; + f.write_str(" | ")?; } write!(f, "{c}")?; } - write!(f, ")") + f.write_str(")") } } } @@ -161,13 +161,13 @@ impl Expression { Self::Less(a, b) => fmt_sse_binary_expression(f, "<", a, b), Self::LessOrEqual(a, b) => fmt_sse_binary_expression(f, "<=", a, b), Self::In(a, b) => { - write!(f, "(in ")?; + f.write_str("(in ")?; a.fmt_sse(f)?; for p in b { - write!(f, " ")?; + f.write_str(" ")?; p.fmt_sse(f)?; } - write!(f, ")") + f.write_str(")") } Self::Add(a, b) => fmt_sse_binary_expression(f, "+", a, b), Self::Subtract(a, b) => fmt_sse_binary_expression(f, "-", a, b), @@ -177,38 +177,38 @@ impl Expression { Self::UnaryMinus(e) => fmt_sse_unary_expression(f, "-", e), Self::Not(e) => fmt_sse_unary_expression(f, "!", e), Self::FunctionCall(function, parameters) => { - write!(f, "( ")?; + f.write_str("( ")?; function.fmt_sse(f)?; for p in parameters { - write!(f, " ")?; + f.write_str(" ")?; p.fmt_sse(f)?; } - write!(f, ")") + f.write_str(")") } Self::Exists(p) => { - write!(f, "(exists ")?; + f.write_str("(exists ")?; p.fmt_sse(f)?; - write!(f, ")") + f.write_str(")") } Self::Bound(v) => { write!(f, "(bound {v})") } Self::If(a, b, c) => { - write!(f, "(if ")?; + f.write_str("(if ")?; a.fmt_sse(f)?; - write!(f, " ")?; + f.write_str(" ")?; b.fmt_sse(f)?; - write!(f, " ")?; + f.write_str(" ")?; c.fmt_sse(f)?; - write!(f, ")") + f.write_str(")") } Self::Coalesce(parameters) => { - write!(f, "(coalesce")?; + f.write_str("(coalesce")?; for p in parameters { - write!(f, " ")?; + f.write_str(" ")?; p.fmt_sse(f)?; } - write!(f, ")") + f.write_str(")") } } } @@ -239,7 +239,7 @@ impl fmt::Display for Expression { Self::In(a, b) => { write!(f, "({a} IN ")?; write_arg_list(b, f)?; - write!(f, ")") + f.write_str(")") } Self::Add(a, b) => { write!(f, "{a} + {b}") @@ -267,7 +267,7 @@ impl fmt::Display for Expression { Self::Exists(p) => write!(f, "EXISTS {{ {p} }}"), Self::If(a, b, c) => write!(f, "IF({a}, {b}, {c})"), Self::Coalesce(parameters) => { - write!(f, "COALESCE")?; + f.write_str("COALESCE")?; write_arg_list(parameters, f) } } @@ -305,16 +305,16 @@ fn write_arg_list( params: impl IntoIterator, f: &mut fmt::Formatter<'_>, ) -> fmt::Result { - write!(f, "(")?; + f.write_str("(")?; let mut cont = false; for p in params { if cont { - write!(f, ", ")?; + f.write_str(", ")?; } p.fmt(f)?; cont = true; } - write!(f, ")") + f.write_str(")") } /// A function name. @@ -385,64 +385,64 @@ impl Function { /// Formats using the [SPARQL S-Expression syntax](https://jena.apache.org/documentation/notes/sse.html). pub(crate) fn fmt_sse(&self, f: &mut impl fmt::Write) -> fmt::Result { match self { - Self::Str => write!(f, "str"), - Self::Lang => write!(f, "lang"), - Self::LangMatches => write!(f, "langmatches"), - Self::Datatype => write!(f, "datatype"), - Self::Iri => write!(f, "iri"), - Self::BNode => write!(f, "bnode"), - Self::Rand => write!(f, "rand"), - Self::Abs => write!(f, "abs"), - Self::Ceil => write!(f, "ceil"), - Self::Floor => write!(f, "floor"), - Self::Round => write!(f, "round"), - Self::Concat => write!(f, "concat"), - Self::SubStr => write!(f, "substr"), - Self::StrLen => write!(f, "strlen"), - Self::Replace => write!(f, "replace"), - Self::UCase => write!(f, "ucase"), - Self::LCase => write!(f, "lcase"), - Self::EncodeForUri => write!(f, "encode_for_uri"), - Self::Contains => write!(f, "contains"), - Self::StrStarts => write!(f, "strstarts"), - Self::StrEnds => write!(f, "strends"), - Self::StrBefore => write!(f, "strbefore"), - Self::StrAfter => write!(f, "strafter"), - Self::Year => write!(f, "year"), - Self::Month => write!(f, "month"), - Self::Day => write!(f, "day"), - Self::Hours => write!(f, "hours"), - Self::Minutes => write!(f, "minutes"), - Self::Seconds => write!(f, "seconds"), - Self::Timezone => write!(f, "timezone"), - Self::Tz => write!(f, "tz"), - Self::Now => write!(f, "now"), - Self::Uuid => write!(f, "uuid"), - Self::StrUuid => write!(f, "struuid"), - Self::Md5 => write!(f, "md5"), - Self::Sha1 => write!(f, "sha1"), - Self::Sha256 => write!(f, "sha256"), - Self::Sha384 => write!(f, "sha384"), - Self::Sha512 => write!(f, "sha512"), - Self::StrLang => write!(f, "strlang"), - Self::StrDt => write!(f, "strdt"), - Self::IsIri => write!(f, "isiri"), - Self::IsBlank => write!(f, "isblank"), - Self::IsLiteral => write!(f, "isliteral"), - Self::IsNumeric => write!(f, "isnumeric"), - Self::Regex => write!(f, "regex"), + Self::Str => f.write_str("str"), + Self::Lang => f.write_str("lang"), + Self::LangMatches => f.write_str("langmatches"), + Self::Datatype => f.write_str("datatype"), + Self::Iri => f.write_str("iri"), + Self::BNode => f.write_str("bnode"), + Self::Rand => f.write_str("rand"), + Self::Abs => f.write_str("abs"), + Self::Ceil => f.write_str("ceil"), + Self::Floor => f.write_str("floor"), + Self::Round => f.write_str("round"), + Self::Concat => f.write_str("concat"), + Self::SubStr => f.write_str("substr"), + Self::StrLen => f.write_str("strlen"), + Self::Replace => f.write_str("replace"), + Self::UCase => f.write_str("ucase"), + Self::LCase => f.write_str("lcase"), + Self::EncodeForUri => f.write_str("encode_for_uri"), + Self::Contains => f.write_str("contains"), + Self::StrStarts => f.write_str("strstarts"), + Self::StrEnds => f.write_str("strends"), + Self::StrBefore => f.write_str("strbefore"), + Self::StrAfter => f.write_str("strafter"), + Self::Year => f.write_str("year"), + Self::Month => f.write_str("month"), + Self::Day => f.write_str("day"), + Self::Hours => f.write_str("hours"), + Self::Minutes => f.write_str("minutes"), + Self::Seconds => f.write_str("seconds"), + Self::Timezone => f.write_str("timezone"), + Self::Tz => f.write_str("tz"), + Self::Now => f.write_str("now"), + Self::Uuid => f.write_str("uuid"), + Self::StrUuid => f.write_str("struuid"), + Self::Md5 => f.write_str("md5"), + Self::Sha1 => f.write_str("sha1"), + Self::Sha256 => f.write_str("sha256"), + Self::Sha384 => f.write_str("sha384"), + Self::Sha512 => f.write_str("sha512"), + Self::StrLang => f.write_str("strlang"), + Self::StrDt => f.write_str("strdt"), + Self::IsIri => f.write_str("isiri"), + Self::IsBlank => f.write_str("isblank"), + Self::IsLiteral => f.write_str("isliteral"), + Self::IsNumeric => f.write_str("isnumeric"), + Self::Regex => f.write_str("regex"), #[cfg(feature = "rdf-star")] - Self::Triple => write!(f, "triple"), + Self::Triple => f.write_str("triple"), #[cfg(feature = "rdf-star")] - Self::Subject => write!(f, "subject"), + Self::Subject => f.write_str("subject"), #[cfg(feature = "rdf-star")] - Self::Predicate => write!(f, "predicate"), + Self::Predicate => f.write_str("predicate"), #[cfg(feature = "rdf-star")] - Self::Object => write!(f, "object"), + Self::Object => f.write_str("object"), #[cfg(feature = "rdf-star")] - Self::IsTriple => write!(f, "istriple"), + Self::IsTriple => f.write_str("istriple"), #[cfg(feature = "sep-0002")] - Self::Adjust => write!(f, "adjust"), + Self::Adjust => f.write_str("adjust"), Self::Custom(iri) => write!(f, "{iri}"), } } @@ -451,64 +451,64 @@ impl Function { impl fmt::Display for Function { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - Self::Str => write!(f, "STR"), - Self::Lang => write!(f, "LANG"), - Self::LangMatches => write!(f, "LANGMATCHES"), - Self::Datatype => write!(f, "DATATYPE"), - Self::Iri => write!(f, "IRI"), - Self::BNode => write!(f, "BNODE"), - Self::Rand => write!(f, "RAND"), - Self::Abs => write!(f, "ABS"), - Self::Ceil => write!(f, "CEIL"), - Self::Floor => write!(f, "FLOOR"), - Self::Round => write!(f, "ROUND"), - Self::Concat => write!(f, "CONCAT"), - Self::SubStr => write!(f, "SUBSTR"), - Self::StrLen => write!(f, "STRLEN"), - Self::Replace => write!(f, "REPLACE"), - Self::UCase => write!(f, "UCASE"), - Self::LCase => write!(f, "LCASE"), - Self::EncodeForUri => write!(f, "ENCODE_FOR_URI"), - Self::Contains => write!(f, "CONTAINS"), - Self::StrStarts => write!(f, "STRSTARTS"), - Self::StrEnds => write!(f, "STRENDS"), - Self::StrBefore => write!(f, "STRBEFORE"), - Self::StrAfter => write!(f, "STRAFTER"), - Self::Year => write!(f, "YEAR"), - Self::Month => write!(f, "MONTH"), - Self::Day => write!(f, "DAY"), - Self::Hours => write!(f, "HOURS"), - Self::Minutes => write!(f, "MINUTES"), - Self::Seconds => write!(f, "SECONDS"), - Self::Timezone => write!(f, "TIMEZONE"), - Self::Tz => write!(f, "TZ"), - Self::Now => write!(f, "NOW"), - Self::Uuid => write!(f, "UUID"), - Self::StrUuid => write!(f, "STRUUID"), - Self::Md5 => write!(f, "MD5"), - Self::Sha1 => write!(f, "SHA1"), - Self::Sha256 => write!(f, "SHA256"), - Self::Sha384 => write!(f, "SHA384"), - Self::Sha512 => write!(f, "SHA512"), - Self::StrLang => write!(f, "STRLANG"), - Self::StrDt => write!(f, "STRDT"), - Self::IsIri => write!(f, "isIRI"), - Self::IsBlank => write!(f, "isBLANK"), - Self::IsLiteral => write!(f, "isLITERAL"), - Self::IsNumeric => write!(f, "isNUMERIC"), - Self::Regex => write!(f, "REGEX"), + Self::Str => f.write_str("STR"), + Self::Lang => f.write_str("LANG"), + Self::LangMatches => f.write_str("LANGMATCHES"), + Self::Datatype => f.write_str("DATATYPE"), + Self::Iri => f.write_str("IRI"), + Self::BNode => f.write_str("BNODE"), + Self::Rand => f.write_str("RAND"), + Self::Abs => f.write_str("ABS"), + Self::Ceil => f.write_str("CEIL"), + Self::Floor => f.write_str("FLOOR"), + Self::Round => f.write_str("ROUND"), + Self::Concat => f.write_str("CONCAT"), + Self::SubStr => f.write_str("SUBSTR"), + Self::StrLen => f.write_str("STRLEN"), + Self::Replace => f.write_str("REPLACE"), + Self::UCase => f.write_str("UCASE"), + Self::LCase => f.write_str("LCASE"), + Self::EncodeForUri => f.write_str("ENCODE_FOR_URI"), + Self::Contains => f.write_str("CONTAINS"), + Self::StrStarts => f.write_str("STRSTARTS"), + Self::StrEnds => f.write_str("STRENDS"), + Self::StrBefore => f.write_str("STRBEFORE"), + Self::StrAfter => f.write_str("STRAFTER"), + Self::Year => f.write_str("YEAR"), + Self::Month => f.write_str("MONTH"), + Self::Day => f.write_str("DAY"), + Self::Hours => f.write_str("HOURS"), + Self::Minutes => f.write_str("MINUTES"), + Self::Seconds => f.write_str("SECONDS"), + Self::Timezone => f.write_str("TIMEZONE"), + Self::Tz => f.write_str("TZ"), + Self::Now => f.write_str("NOW"), + Self::Uuid => f.write_str("UUID"), + Self::StrUuid => f.write_str("STRUUID"), + Self::Md5 => f.write_str("MD5"), + Self::Sha1 => f.write_str("SHA1"), + Self::Sha256 => f.write_str("SHA256"), + Self::Sha384 => f.write_str("SHA384"), + Self::Sha512 => f.write_str("SHA512"), + Self::StrLang => f.write_str("STRLANG"), + Self::StrDt => f.write_str("STRDT"), + Self::IsIri => f.write_str("isIRI"), + Self::IsBlank => f.write_str("isBLANK"), + Self::IsLiteral => f.write_str("isLITERAL"), + Self::IsNumeric => f.write_str("isNUMERIC"), + Self::Regex => f.write_str("REGEX"), #[cfg(feature = "rdf-star")] - Self::Triple => write!(f, "TRIPLE"), + Self::Triple => f.write_str("TRIPLE"), #[cfg(feature = "rdf-star")] - Self::Subject => write!(f, "SUBJECT"), + Self::Subject => f.write_str("SUBJECT"), #[cfg(feature = "rdf-star")] - Self::Predicate => write!(f, "PREDICATE"), + Self::Predicate => f.write_str("PREDICATE"), #[cfg(feature = "rdf-star")] - Self::Object => write!(f, "OBJECT"), + Self::Object => f.write_str("OBJECT"), #[cfg(feature = "rdf-star")] - Self::IsTriple => write!(f, "isTRIPLE"), + Self::IsTriple => f.write_str("isTRIPLE"), #[cfg(feature = "sep-0002")] - Self::Adjust => write!(f, "ADJUST"), + Self::Adjust => f.write_str("ADJUST"), Self::Custom(iri) => iri.fmt(f), } } @@ -665,29 +665,29 @@ impl fmt::Display for GraphPattern { variables, bindings, } => { - write!(f, "VALUES ( ")?; + f.write_str("VALUES ( ")?; for var in variables { write!(f, "{var} ")?; } - write!(f, ") {{ ")?; + f.write_str(") { ")?; for row in bindings { - write!(f, "( ")?; + f.write_str("( ")?; for val in row { match val { Some(val) => write!(f, "{val} "), - None => write!(f, "UNDEF "), + None => f.write_str("UNDEF "), }?; } - write!(f, ") ")?; + f.write_str(") ")?; } - write!(f, " }}") + f.write_str(" }") } Self::Group { inner, variables, aggregates, } => { - write!(f, "{{SELECT")?; + f.write_str("{SELECT")?; for (a, v) in aggregates { write!(f, " ({v} AS {a})")?; } @@ -696,12 +696,12 @@ impl fmt::Display for GraphPattern { } write!(f, " WHERE {{ {inner} }}")?; if !variables.is_empty() { - write!(f, " GROUP BY")?; + f.write_str(" GROUP BY")?; for v in variables { write!(f, " {v}")?; } } - write!(f, "}}") + f.write_str("}") } p => write!( f, @@ -728,76 +728,76 @@ impl GraphPattern { pub(crate) fn fmt_sse(&self, f: &mut impl fmt::Write) -> fmt::Result { match self { Self::Bgp { patterns } => { - write!(f, "(bgp")?; + f.write_str("(bgp")?; for pattern in patterns { - write!(f, " ")?; + f.write_str(" ")?; pattern.fmt_sse(f)?; } - write!(f, ")") + f.write_str(")") } Self::Path { subject, path, object, } => { - write!(f, "(path ")?; + f.write_str("(path ")?; subject.fmt_sse(f)?; - write!(f, " ")?; + f.write_str(" ")?; path.fmt_sse(f)?; - write!(f, " ")?; + f.write_str(" ")?; object.fmt_sse(f)?; - write!(f, ")") + f.write_str(")") } Self::Join { left, right } => { - write!(f, "(join ")?; + f.write_str("(join ")?; left.fmt_sse(f)?; - write!(f, " ")?; + f.write_str(" ")?; right.fmt_sse(f)?; - write!(f, ")") + f.write_str(")") } Self::LeftJoin { left, right, expression, } => { - write!(f, "(leftjoin ")?; + f.write_str("(leftjoin ")?; left.fmt_sse(f)?; - write!(f, " ")?; + f.write_str(" ")?; right.fmt_sse(f)?; if let Some(expr) = expression { - write!(f, " ")?; + f.write_str(" ")?; expr.fmt_sse(f)?; } - write!(f, ")") + f.write_str(")") } #[cfg(feature = "sep-0006")] Self::Lateral { left, right } => { - write!(f, "(lateral ")?; + f.write_str("(lateral ")?; left.fmt_sse(f)?; - write!(f, " ")?; + f.write_str(" ")?; right.fmt_sse(f)?; - write!(f, ")") + f.write_str(")") } Self::Filter { expr, inner } => { - write!(f, "(filter ")?; + f.write_str("(filter ")?; expr.fmt_sse(f)?; - write!(f, " ")?; + f.write_str(" ")?; inner.fmt_sse(f)?; - write!(f, ")") + f.write_str(")") } Self::Union { left, right } => { - write!(f, "(union ")?; + f.write_str("(union ")?; left.fmt_sse(f)?; - write!(f, " ")?; + f.write_str(" ")?; right.fmt_sse(f)?; - write!(f, ")") + f.write_str(")") } Self::Graph { name, inner } => { - write!(f, "(graph ")?; + f.write_str("(graph ")?; name.fmt_sse(f)?; - write!(f, " ")?; + f.write_str(" ")?; inner.fmt_sse(f)?; - write!(f, ")") + f.write_str(")") } Self::Extend { inner, @@ -806,109 +806,109 @@ impl GraphPattern { } => { write!(f, "(extend (({variable} ")?; expression.fmt_sse(f)?; - write!(f, ")) ")?; + f.write_str(")) ")?; inner.fmt_sse(f)?; - write!(f, ")") + f.write_str(")") } Self::Minus { left, right } => { - write!(f, "(minus ")?; + f.write_str("(minus ")?; left.fmt_sse(f)?; - write!(f, " ")?; + f.write_str(" ")?; right.fmt_sse(f)?; - write!(f, ")") + f.write_str(")") } Self::Service { name, inner, silent, } => { - write!(f, "(service ")?; + f.write_str("(service ")?; if *silent { - write!(f, "silent ")?; + f.write_str("silent ")?; } name.fmt_sse(f)?; - write!(f, " ")?; + f.write_str(" ")?; inner.fmt_sse(f)?; - write!(f, ")") + f.write_str(")") } Self::Group { inner, variables, aggregates, } => { - write!(f, "(group (")?; + f.write_str("(group (")?; for (i, v) in variables.iter().enumerate() { if i > 0 { - write!(f, " ")?; + f.write_str(" ")?; } write!(f, "{v}")?; } - write!(f, ") (")?; + f.write_str(") (")?; for (i, (v, a)) in aggregates.iter().enumerate() { if i > 0 { - write!(f, " ")?; + f.write_str(" ")?; } - write!(f, "(")?; + f.write_str("(")?; a.fmt_sse(f)?; write!(f, " {v})")?; } - write!(f, ") ")?; + f.write_str(") ")?; inner.fmt_sse(f)?; - write!(f, ")") + f.write_str(")") } Self::Values { variables, bindings, } => { - write!(f, "(table (vars")?; + f.write_str("(table (vars")?; for var in variables { write!(f, " {var}")?; } - write!(f, ")")?; + f.write_str(")")?; for row in bindings { - write!(f, " (row")?; + f.write_str(" (row")?; for (value, var) in row.iter().zip(variables) { if let Some(value) = value { write!(f, " ({var} {value})")?; } } - write!(f, ")")?; + f.write_str(")")?; } - write!(f, ")") + f.write_str(")") } Self::OrderBy { inner, expression } => { - write!(f, "(order (")?; + f.write_str("(order (")?; for (i, c) in expression.iter().enumerate() { if i > 0 { - write!(f, " ")?; + f.write_str(" ")?; } c.fmt_sse(f)?; } - write!(f, ") ")?; + f.write_str(") ")?; inner.fmt_sse(f)?; - write!(f, ")") + f.write_str(")") } Self::Project { inner, variables } => { - write!(f, "(project (")?; + f.write_str("(project (")?; for (i, v) in variables.iter().enumerate() { if i > 0 { - write!(f, " ")?; + f.write_str(" ")?; } write!(f, "{v}")?; } - write!(f, ") ")?; + f.write_str(") ")?; inner.fmt_sse(f)?; - write!(f, ")") + f.write_str(")") } Self::Distinct { inner } => { - write!(f, "(distinct ")?; + f.write_str("(distinct ")?; inner.fmt_sse(f)?; - write!(f, ")") + f.write_str(")") } Self::Reduced { inner } => { - write!(f, "(reduced ")?; + f.write_str("(reduced ")?; inner.fmt_sse(f)?; - write!(f, ")") + f.write_str(")") } Self::Slice { inner, @@ -921,7 +921,7 @@ impl GraphPattern { write!(f, "(slice {start} _ ")?; } inner.fmt_sse(f)?; - write!(f, ")") + f.write_str(")") } } } @@ -1074,15 +1074,15 @@ impl<'a> fmt::Display for SparqlGraphRootPattern<'a> { child = inner; } p => { - write!(f, "SELECT")?; + f.write_str("SELECT")?; if distinct { - write!(f, " DISTINCT")?; + f.write_str(" DISTINCT")?; } if reduced { - write!(f, " REDUCED")?; + f.write_str(" REDUCED")?; } if project.is_empty() { - write!(f, " *")?; + f.write_str(" *")?; } else { for v in project { write!(f, " {v}")?; @@ -1093,7 +1093,7 @@ impl<'a> fmt::Display for SparqlGraphRootPattern<'a> { } write!(f, " WHERE {{ {p} }}")?; if let Some(order) = order { - write!(f, " ORDER BY")?; + f.write_str(" ORDER BY")?; for c in order { write!(f, " {c}")?; } @@ -1128,11 +1128,11 @@ impl AggregateExpression { pub(crate) fn fmt_sse(&self, f: &mut impl fmt::Write) -> fmt::Result { match self { Self::CountSolutions { distinct } => { - write!(f, "(count")?; + f.write_str("(count")?; if *distinct { - write!(f, " distinct")?; + f.write_str(" distinct")?; } - write!(f, ")") + f.write_str(")") } Self::FunctionCall { name: @@ -1142,9 +1142,9 @@ impl AggregateExpression { expr, distinct, } => { - write!(f, "(group_concat ")?; + f.write_str("(group_concat ")?; if *distinct { - write!(f, "distinct ")?; + f.write_str("distinct ")?; } expr.fmt_sse(f)?; write!(f, " {})", LiteralRef::new_simple_literal(separator)) @@ -1154,14 +1154,14 @@ impl AggregateExpression { expr, distinct, } => { - write!(f, "(")?; + f.write_str("(")?; name.fmt_sse(f)?; - write!(f, " ")?; + f.write_str(" ")?; if *distinct { - write!(f, "distinct ")?; + f.write_str("distinct ")?; } expr.fmt_sse(f)?; - write!(f, ")") + f.write_str(")") } } } @@ -1172,9 +1172,9 @@ impl fmt::Display for AggregateExpression { match self { Self::CountSolutions { distinct } => { if *distinct { - write!(f, "COUNT(DISTINCT *)") + f.write_str("COUNT(DISTINCT *)") } else { - write!(f, "COUNT(*)") + f.write_str("COUNT(*)") } } Self::FunctionCall { @@ -1242,13 +1242,13 @@ impl AggregateFunction { /// Formats using the [SPARQL S-Expression syntax](https://jena.apache.org/documentation/notes/sse.html). pub(crate) fn fmt_sse(&self, f: &mut impl fmt::Write) -> fmt::Result { match self { - Self::Count => write!(f, "count"), - Self::Sum => write!(f, "sum"), - Self::Avg => write!(f, "avg"), - Self::Min => write!(f, "min"), - Self::Max => write!(f, "max"), - Self::GroupConcat { .. } => write!(f, "group_concat"), - Self::Sample => write!(f, "sample"), + Self::Count => f.write_str("count"), + Self::Sum => f.write_str("sum"), + Self::Avg => f.write_str("avg"), + Self::Min => f.write_str("min"), + Self::Max => f.write_str("max"), + Self::GroupConcat { .. } => f.write_str("group_concat"), + Self::Sample => f.write_str("sample"), Self::Custom(iri) => write!(f, "{iri}"), } } @@ -1257,13 +1257,13 @@ impl AggregateFunction { impl fmt::Display for AggregateFunction { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - Self::Count => write!(f, "COUNT"), - Self::Sum => write!(f, "SUM"), - Self::Avg => write!(f, "AVG"), - Self::Min => write!(f, "MIN"), - Self::Max => write!(f, "MAX"), - Self::GroupConcat { .. } => write!(f, "GROUP_CONCAT"), - Self::Sample => write!(f, "SAMPLE"), + Self::Count => f.write_str("COUNT"), + Self::Sum => f.write_str("SUM"), + Self::Avg => f.write_str("AVG"), + Self::Min => f.write_str("MIN"), + Self::Max => f.write_str("MAX"), + Self::GroupConcat { .. } => f.write_str("GROUP_CONCAT"), + Self::Sample => f.write_str("SAMPLE"), Self::Custom(iri) => iri.fmt(f), } } @@ -1283,14 +1283,14 @@ impl OrderExpression { pub(crate) fn fmt_sse(&self, f: &mut impl fmt::Write) -> fmt::Result { match self { Self::Asc(e) => { - write!(f, "(asc ")?; + f.write_str("(asc ")?; e.fmt_sse(f)?; - write!(f, ")") + f.write_str(")") } Self::Desc(e) => { - write!(f, "(desc ")?; + f.write_str("(desc ")?; e.fmt_sse(f)?; - write!(f, ")") + f.write_str(")") } } } @@ -1315,22 +1315,22 @@ pub struct QueryDataset { impl QueryDataset { /// Formats using the [SPARQL S-Expression syntax](https://jena.apache.org/documentation/notes/sse.html). pub(crate) fn fmt_sse(&self, f: &mut impl fmt::Write) -> fmt::Result { - write!(f, "(")?; + f.write_str("(")?; for (i, graph_name) in self.default.iter().enumerate() { if i > 0 { - write!(f, " ")?; + f.write_str(" ")?; } write!(f, "{graph_name}")?; } if let Some(named) = &self.named { for (i, graph_name) in named.iter().enumerate() { if !self.default.is_empty() || i > 0 { - write!(f, " ")?; + f.write_str(" ")?; } write!(f, "(named {graph_name})")?; } } - write!(f, ")") + f.write_str(")") } } @@ -1364,9 +1364,9 @@ impl GraphTarget { pub(crate) fn fmt_sse(&self, f: &mut impl fmt::Write) -> fmt::Result { match self { Self::NamedNode(node) => write!(f, "{node}"), - Self::DefaultGraph => write!(f, "default"), - Self::NamedGraphs => write!(f, "named"), - Self::AllGraphs => write!(f, "all"), + Self::DefaultGraph => f.write_str("default"), + Self::NamedGraphs => f.write_str("named"), + Self::AllGraphs => f.write_str("all"), } } } @@ -1375,9 +1375,9 @@ impl fmt::Display for GraphTarget { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::NamedNode(node) => write!(f, "GRAPH {node}"), - Self::DefaultGraph => write!(f, "DEFAULT"), - Self::NamedGraphs => write!(f, "NAMED"), - Self::AllGraphs => write!(f, "ALL"), + Self::DefaultGraph => f.write_str("DEFAULT"), + Self::NamedGraphs => f.write_str("NAMED"), + Self::AllGraphs => f.write_str("ALL"), } } } @@ -1401,7 +1401,7 @@ impl From for GraphTarget { fn fmt_sse_unary_expression(f: &mut impl fmt::Write, name: &str, e: &Expression) -> fmt::Result { write!(f, "({name} ")?; e.fmt_sse(f)?; - write!(f, ")") + f.write_str(")") } #[inline] @@ -1413,7 +1413,7 @@ fn fmt_sse_binary_expression( ) -> fmt::Result { write!(f, "({name} ")?; a.fmt_sse(f)?; - write!(f, " ")?; + f.write_str(" ")?; b.fmt_sse(f)?; - write!(f, ")") + f.write_str(")") } diff --git a/lib/spargebra/src/query.rs b/lib/spargebra/src/query.rs index 5739b7b8..780455f7 100644 --- a/lib/spargebra/src/query.rs +++ b/lib/spargebra/src/query.rs @@ -86,16 +86,16 @@ impl Query { write!(f, "(base <{base_iri}> ")?; } if let Some(dataset) = dataset { - write!(f, "(dataset ")?; + f.write_str("(dataset ")?; dataset.fmt_sse(f)?; - write!(f, " ")?; + f.write_str(" ")?; } pattern.fmt_sse(f)?; if dataset.is_some() { - write!(f, ")")?; + f.write_str(")")?; } if base_iri.is_some() { - write!(f, ")")?; + f.write_str(")")?; } Ok(()) } @@ -108,26 +108,26 @@ impl Query { if let Some(base_iri) = base_iri { write!(f, "(base <{base_iri}> ")?; } - write!(f, "(construct (")?; + f.write_str("(construct (")?; for (i, t) in template.iter().enumerate() { if i > 0 { - write!(f, " ")?; + f.write_str(" ")?; } t.fmt_sse(f)?; } - write!(f, ") ")?; + f.write_str(") ")?; if let Some(dataset) = dataset { - write!(f, "(dataset ")?; + f.write_str("(dataset ")?; dataset.fmt_sse(f)?; - write!(f, " ")?; + f.write_str(" ")?; } pattern.fmt_sse(f)?; if dataset.is_some() { - write!(f, ")")?; + f.write_str(")")?; } - write!(f, ")")?; + f.write_str(")")?; if base_iri.is_some() { - write!(f, ")")?; + f.write_str(")")?; } Ok(()) } @@ -139,19 +139,19 @@ impl Query { if let Some(base_iri) = base_iri { write!(f, "(base <{base_iri}> ")?; } - write!(f, "(describe ")?; + f.write_str("(describe ")?; if let Some(dataset) = dataset { - write!(f, "(dataset ")?; + f.write_str("(dataset ")?; dataset.fmt_sse(f)?; - write!(f, " ")?; + f.write_str(" ")?; } pattern.fmt_sse(f)?; if dataset.is_some() { - write!(f, ")")?; + f.write_str(")")?; } - write!(f, ")")?; + f.write_str(")")?; if base_iri.is_some() { - write!(f, ")")?; + f.write_str(")")?; } Ok(()) } @@ -163,19 +163,19 @@ impl Query { if let Some(base_iri) = base_iri { write!(f, "(base <{base_iri}> ")?; } - write!(f, "(ask ")?; + f.write_str("(ask ")?; if let Some(dataset) = dataset { - write!(f, "(dataset ")?; + f.write_str("(dataset ")?; dataset.fmt_sse(f)?; - write!(f, " ")?; + f.write_str(" ")?; } pattern.fmt_sse(f)?; if dataset.is_some() { - write!(f, ")")?; + f.write_str(")")?; } - write!(f, ")")?; + f.write_str(")")?; if base_iri.is_some() { - write!(f, ")")?; + f.write_str(")")?; } Ok(()) } @@ -212,11 +212,11 @@ impl fmt::Display for Query { if let Some(base_iri) = base_iri { writeln!(f, "BASE <{base_iri}>")?; } - write!(f, "CONSTRUCT {{ ")?; + f.write_str("CONSTRUCT { ")?; for triple in template { write!(f, "{triple} . ")?; } - write!(f, "}}")?; + f.write_str("}")?; if let Some(dataset) = dataset { dataset.fmt(f)?; } @@ -237,7 +237,7 @@ impl fmt::Display for Query { if let Some(base_iri) = base_iri { writeln!(f, "BASE <{}>", base_iri.as_str())?; } - write!(f, "DESCRIBE *")?; + f.write_str("DESCRIBE *")?; if let Some(dataset) = dataset { dataset.fmt(f)?; } @@ -258,7 +258,7 @@ impl fmt::Display for Query { if let Some(base_iri) = base_iri { writeln!(f, "BASE <{base_iri}>")?; } - write!(f, "ASK")?; + f.write_str("ASK")?; if let Some(dataset) = dataset { dataset.fmt(f)?; } diff --git a/lib/spargebra/src/term.rs b/lib/spargebra/src/term.rs index ba5fb8e3..43a22efe 100644 --- a/lib/spargebra/src/term.rs +++ b/lib/spargebra/src/term.rs @@ -196,7 +196,7 @@ impl GraphName { pub(crate) fn fmt_sse(&self, f: &mut impl Write) -> fmt::Result { match self { Self::NamedNode(node) => write!(f, "{node}"), - Self::DefaultGraph => write!(f, "default"), + Self::DefaultGraph => f.write_str("default"), } } } @@ -206,7 +206,7 @@ impl fmt::Display for GraphName { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::NamedNode(node) => node.fmt(f), - Self::DefaultGraph => write!(f, "DEFAULT"), + Self::DefaultGraph => f.write_str("DEFAULT"), } } } @@ -261,9 +261,9 @@ impl Quad { /// Formats using the [SPARQL S-Expression syntax](https://jena.apache.org/documentation/notes/sse.html). pub(crate) fn fmt_sse(&self, f: &mut impl Write) -> fmt::Result { if self.graph_name != GraphName::DefaultGraph { - write!(f, "(graph ")?; + f.write_str("(graph ")?; self.graph_name.fmt_sse(f)?; - write!(f, " (")?; + f.write_str(" (")?; } write!( f, @@ -271,7 +271,7 @@ impl Quad { self.subject, self.predicate, self.object )?; if self.graph_name != GraphName::DefaultGraph { - write!(f, "))")?; + f.write_str("))")?; } Ok(()) } @@ -336,9 +336,9 @@ impl GroundQuad { /// Formats using the [SPARQL S-Expression syntax](https://jena.apache.org/documentation/notes/sse.html). pub(crate) fn fmt_sse(&self, f: &mut impl Write) -> fmt::Result { if self.graph_name != GraphName::DefaultGraph { - write!(f, "(graph ")?; + f.write_str("(graph ")?; self.graph_name.fmt_sse(f)?; - write!(f, " (")?; + f.write_str(" (")?; } write!( f, @@ -346,7 +346,7 @@ impl GroundQuad { self.subject, self.predicate, self.object )?; if self.graph_name != GraphName::DefaultGraph { - write!(f, "))")?; + f.write_str("))")?; } Ok(()) } @@ -712,7 +712,7 @@ impl GraphNamePattern { pub(crate) fn fmt_sse(&self, f: &mut impl Write) -> fmt::Result { match self { Self::NamedNode(node) => write!(f, "{node}"), - Self::DefaultGraph => write!(f, "default"), + Self::DefaultGraph => f.write_str("default"), Self::Variable(var) => write!(f, "{var}"), } } @@ -723,7 +723,7 @@ impl fmt::Display for GraphNamePattern { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::NamedNode(node) => node.fmt(f), - Self::DefaultGraph => write!(f, "DEFAULT"), + Self::DefaultGraph => f.write_str("DEFAULT"), Self::Variable(var) => var.fmt(f), } } @@ -786,13 +786,13 @@ impl TriplePattern { /// Formats using the [SPARQL S-Expression syntax](https://jena.apache.org/documentation/notes/sse.html). pub(crate) fn fmt_sse(&self, f: &mut impl Write) -> fmt::Result { - write!(f, "(triple ")?; + f.write_str("(triple ")?; self.subject.fmt_sse(f)?; - write!(f, " ")?; + f.write_str(" ")?; self.predicate.fmt_sse(f)?; - write!(f, " ")?; + f.write_str(" ")?; self.object.fmt_sse(f)?; - write!(f, ")") + f.write_str(")") } } @@ -850,13 +850,13 @@ impl GroundTriplePattern { /// Formats using the [SPARQL S-Expression syntax](https://jena.apache.org/documentation/notes/sse.html). #[allow(dead_code)] pub(crate) fn fmt_sse(&self, f: &mut impl Write) -> fmt::Result { - write!(f, "(triple ")?; + f.write_str("(triple ")?; self.subject.fmt_sse(f)?; - write!(f, " ")?; + f.write_str(" ")?; self.predicate.fmt_sse(f)?; - write!(f, " ")?; + f.write_str(" ")?; self.object.fmt_sse(f)?; - write!(f, ")") + f.write_str(")") } } @@ -918,19 +918,19 @@ impl QuadPattern { /// Formats using the [SPARQL S-Expression syntax](https://jena.apache.org/documentation/notes/sse.html). pub(crate) fn fmt_sse(&self, f: &mut impl Write) -> fmt::Result { if self.graph_name != GraphNamePattern::DefaultGraph { - write!(f, "(graph ")?; + f.write_str("(graph ")?; self.graph_name.fmt_sse(f)?; - write!(f, " (")?; + f.write_str(" (")?; } - write!(f, "(triple ")?; + f.write_str("(triple ")?; self.subject.fmt_sse(f)?; - write!(f, " ")?; + f.write_str(" ")?; self.predicate.fmt_sse(f)?; - write!(f, " ")?; + f.write_str(" ")?; self.object.fmt_sse(f)?; - write!(f, ")")?; + f.write_str(")")?; if self.graph_name != GraphNamePattern::DefaultGraph { - write!(f, "))")?; + f.write_str("))")?; } Ok(()) } @@ -964,19 +964,19 @@ impl GroundQuadPattern { /// Formats using the [SPARQL S-Expression syntax](https://jena.apache.org/documentation/notes/sse.html). pub(crate) fn fmt_sse(&self, f: &mut impl Write) -> fmt::Result { if self.graph_name != GraphNamePattern::DefaultGraph { - write!(f, "(graph ")?; + f.write_str("(graph ")?; self.graph_name.fmt_sse(f)?; - write!(f, " (")?; + f.write_str(" (")?; } - write!(f, "(triple ")?; + f.write_str("(triple ")?; self.subject.fmt_sse(f)?; - write!(f, " ")?; + f.write_str(" ")?; self.predicate.fmt_sse(f)?; - write!(f, " ")?; + f.write_str(" ")?; self.object.fmt_sse(f)?; - write!(f, ")")?; + f.write_str(")")?; if self.graph_name != GraphNamePattern::DefaultGraph { - write!(f, "))")?; + f.write_str("))")?; } Ok(()) } diff --git a/lib/spargebra/src/update.rs b/lib/spargebra/src/update.rs index e2a2c653..76c5c65d 100644 --- a/lib/spargebra/src/update.rs +++ b/lib/spargebra/src/update.rs @@ -42,14 +42,14 @@ impl Update { if let Some(base_iri) = &self.base_iri { write!(f, "(base <{base_iri}> ")?; } - write!(f, "(update")?; + f.write_str("(update")?; for op in &self.operations { - write!(f, " ")?; + f.write_str(" ")?; op.fmt_sse(f)?; } - write!(f, ")")?; + f.write_str(")")?; if self.base_iri.is_some() { - write!(f, ")")?; + f.write_str(")")?; } Ok(()) } @@ -124,24 +124,24 @@ impl GraphUpdateOperation { fn fmt_sse(&self, f: &mut impl fmt::Write) -> fmt::Result { match self { Self::InsertData { data } => { - write!(f, "(insertData (")?; + f.write_str("(insertData (")?; for (i, t) in data.iter().enumerate() { if i > 0 { - write!(f, " ")?; + f.write_str(" ")?; } t.fmt_sse(f)?; } - write!(f, "))") + f.write_str("))") } Self::DeleteData { data } => { - write!(f, "(deleteData (")?; + f.write_str("(deleteData (")?; for (i, t) in data.iter().enumerate() { if i > 0 { - write!(f, " ")?; + f.write_str(" ")?; } t.fmt_sse(f)?; } - write!(f, "))") + f.write_str("))") } Self::DeleteInsert { delete, @@ -149,73 +149,73 @@ impl GraphUpdateOperation { using, pattern, } => { - write!(f, "(modify ")?; + f.write_str("(modify ")?; if let Some(using) = using { - write!(f, " (using ")?; + f.write_str(" (using ")?; using.fmt_sse(f)?; - write!(f, " ")?; + f.write_str(" ")?; pattern.fmt_sse(f)?; - write!(f, ")")?; + f.write_str(")")?; } else { pattern.fmt_sse(f)?; } if !delete.is_empty() { - write!(f, " (delete (")?; + f.write_str(" (delete (")?; for (i, t) in delete.iter().enumerate() { if i > 0 { - write!(f, " ")?; + f.write_str(" ")?; } t.fmt_sse(f)?; } - write!(f, "))")?; + f.write_str("))")?; } if !insert.is_empty() { - write!(f, " (insert (")?; + f.write_str(" (insert (")?; for (i, t) in insert.iter().enumerate() { if i > 0 { - write!(f, " ")?; + f.write_str(" ")?; } t.fmt_sse(f)?; } - write!(f, "))")?; + f.write_str("))")?; } - write!(f, ")") + f.write_str(")") } Self::Load { silent, source, destination, } => { - write!(f, "(load ")?; + f.write_str("(load ")?; if *silent { - write!(f, "silent ")?; + f.write_str("silent ")?; } write!(f, "{source} ")?; destination.fmt_sse(f)?; - write!(f, ")") + f.write_str(")") } Self::Clear { silent, graph } => { - write!(f, "(clear ")?; + f.write_str("(clear ")?; if *silent { - write!(f, "silent ")?; + f.write_str("silent ")?; } graph.fmt_sse(f)?; - write!(f, ")") + f.write_str(")") } Self::Create { silent, graph } => { - write!(f, "(create ")?; + f.write_str("(create ")?; if *silent { - write!(f, "silent ")?; + f.write_str("silent ")?; } write!(f, "{graph})") } Self::Drop { silent, graph } => { - write!(f, "(drop ")?; + f.write_str("(drop ")?; if *silent { - write!(f, "silent ")?; + f.write_str("silent ")?; } graph.fmt_sse(f)?; - write!(f, ")") + f.write_str(")") } } } @@ -227,12 +227,12 @@ impl fmt::Display for GraphUpdateOperation { Self::InsertData { data } => { writeln!(f, "INSERT DATA {{")?; write_quads(data, f)?; - write!(f, "}}") + f.write_str("}") } Self::DeleteData { data } => { writeln!(f, "DELETE DATA {{")?; write_ground_quads(data, f)?; - write!(f, "}}") + f.write_str("}") } Self::DeleteInsert { delete, @@ -278,9 +278,9 @@ impl fmt::Display for GraphUpdateOperation { source, destination, } => { - write!(f, "LOAD ")?; + f.write_str("LOAD ")?; if *silent { - write!(f, "SILENT ")?; + f.write_str("SILENT ")?; } write!(f, "{source}")?; if destination != &GraphName::DefaultGraph { @@ -289,23 +289,23 @@ impl fmt::Display for GraphUpdateOperation { Ok(()) } Self::Clear { silent, graph } => { - write!(f, "CLEAR ")?; + f.write_str("CLEAR ")?; if *silent { - write!(f, "SILENT ")?; + f.write_str("SILENT ")?; } write!(f, "{graph}") } Self::Create { silent, graph } => { - write!(f, "CREATE ")?; + f.write_str("CREATE ")?; if *silent { - write!(f, "SILENT ")?; + f.write_str("SILENT ")?; } write!(f, "GRAPH {graph}") } Self::Drop { silent, graph } => { - write!(f, "DROP ")?; + f.write_str("DROP ")?; if *silent { - write!(f, "SILENT ")?; + f.write_str("SILENT ")?; } write!(f, "{graph}") } diff --git a/lib/sparql-smith/src/lib.rs b/lib/sparql-smith/src/lib.rs index a57fe006..7266f67e 100644 --- a/lib/sparql-smith/src/lib.rs +++ b/lib/sparql-smith/src/lib.rs @@ -147,15 +147,15 @@ enum SelectProjection { impl fmt::Display for SelectClause { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "SELECT")?; + f.write_str("SELECT")?; if let Some(option) = &self.option { match option { - SelectOption::Distinct => write!(f, " DISTINCT"), - SelectOption::Reduced => write!(f, " REDUCED"), + SelectOption::Distinct => f.write_str(" DISTINCT"), + SelectOption::Reduced => f.write_str(" REDUCED"), }?; } match &self.values { - SelectValues::Star => write!(f, " *"), + SelectValues::Star => f.write_str(" *"), SelectValues::Projection { start, others } => { for e in once(start).chain(others) { match e { @@ -179,7 +179,7 @@ struct WhereClause { impl fmt::Display for WhereClause { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { if self.with_where { - write!(f, " WHERE ")?; + f.write_str(" WHERE ")?; } write!(f, "{}", self.group_graph_pattern) } @@ -400,12 +400,12 @@ enum GroupGraphPattern { impl fmt::Display for GroupGraphPattern { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, " {{ ")?; + f.write_str(" { ")?; match self { Self::GroupGraphPatternSub(p) => write!(f, "{p}"), Self::SubSelect(s) => write!(f, "{s}"), }?; - write!(f, " }} ") + f.write_str(" } ") } } @@ -431,7 +431,7 @@ impl fmt::Display for GroupGraphPatternSub { for other in &self.others { write!(f, "{}", other.start)?; if other.with_dot { - write!(f, " . ")?; + f.write_str(" . ")?; } if let Some(end) = &other.end { write!(f, "{end}")?; @@ -452,7 +452,7 @@ impl fmt::Display for TriplesBlock { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{}", self.start)?; if let Some(end) = &self.end { - write!(f, " . ")?; + f.write_str(" . ")?; if let Some(end) = end { write!(f, "{end}")?; } @@ -617,19 +617,19 @@ impl<'a> Arbitrary<'a> for InlineDataFull { impl fmt::Display for InlineDataFull { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "( ")?; + f.write_str("( ")?; for v in &self.vars { write!(f, " {v}")?; } - write!(f, " ) {{")?; + f.write_str(" ) {")?; for vs in &self.values { - write!(f, " (")?; + f.write_str(" (")?; for v in vs { write!(f, " {v}")?; } - write!(f, " )")?; + f.write_str(" )")?; } - write!(f, " }}") + f.write_str(" }") } } @@ -646,7 +646,7 @@ impl fmt::Display for DataBlockValue { match self { Self::Iri(i) => write!(f, "{i}"), Self::Literal(l) => write!(f, "{l}"), - Self::Undef => write!(f, "UNDEF"), + Self::Undef => f.write_str("UNDEF"), } } } @@ -736,14 +736,14 @@ enum ArgList { impl fmt::Display for ArgList { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "(")?; + f.write_str("(")?; if let Self::NotNil { start, others } = self { write!(f, "{start}")?; for e in others { write!(f, ", {e}")?; } } - write!(f, ")") + f.write_str(")") } } @@ -755,14 +755,14 @@ struct ExpressionList { impl fmt::Display for ExpressionList { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "(")?; + f.write_str("(")?; for (i, e) in self.inner.iter().enumerate() { if i > 0 { - write!(f, ", ")?; + f.write_str(", ")?; } write!(f, "{e}")?; } - write!(f, ")") + f.write_str(")") } } @@ -784,7 +784,7 @@ impl fmt::Display for PropertyListNotEmpty { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{} {}", self.start_predicate, self.start_object)?; for other in &self.others { - write!(f, " ; ")?; + f.write_str(" ; ")?; if let Some(e) = other { write!(f, "{} {}", e.predicate, e.object)?; } @@ -804,7 +804,7 @@ impl fmt::Display for Verb { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::VarOrIri(iri) => write!(f, "{iri}"), - Self::A => write!(f, " a "), + Self::A => f.write_str(" a "), } } } @@ -820,7 +820,7 @@ impl fmt::Display for ObjectList { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{}", self.start)?; for other in &self.others { - write!(f, " , ")?; + f.write_str(" , ")?; write!(f, "{other}")?; } Ok(()) @@ -906,7 +906,7 @@ impl fmt::Display for PropertyListPathNotEmpty { }?; write!(f, "{}", self.start_object)?; for other in &self.others { - write!(f, " ; ")?; + f.write_str(" ; ")?; if let Some(e) = other { match &e.predicate { PropertyListPathNotEmptyVerb::VerbPath(p) => write!(f, "{p}"), @@ -1026,9 +1026,9 @@ enum PathMod { impl fmt::Display for PathMod { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - Self::ZeroOrOne => write!(f, " ? "), - Self::ZeroOrMore => write!(f, " * "), - Self::OneOrMore => write!(f, " + "), + Self::ZeroOrOne => f.write_str(" ? "), + Self::ZeroOrMore => f.write_str(" * "), + Self::OneOrMore => f.write_str(" + "), } } } @@ -1046,7 +1046,7 @@ impl fmt::Display for PathPrimary { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::Iri(iri) => write!(f, "{iri}"), - Self::A => write!(f, " a "), + Self::A => f.write_str(" a "), Self::Negated(n) => write!(f, "!{n}"), Self::Child(c) => write!(f, "({c})"), } @@ -1072,7 +1072,7 @@ impl fmt::Display for PathNegatedPropertySet { for other in others { write!(f, " | {other}")?; } - write!(f, " ) ") + f.write_str(" ) ") } } } @@ -1091,9 +1091,9 @@ impl fmt::Display for PathOneInPropertySet { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::Iri(iri) => write!(f, "{iri}"), - Self::A => write!(f, " a "), + Self::A => f.write_str(" a "), Self::NegatedIri(iri) => write!(f, "^{iri}"), - Self::NegatedA => write!(f, " ^a "), + Self::NegatedA => f.write_str(" ^a "), } } } @@ -1167,7 +1167,7 @@ impl fmt::Display for Collection { for e in &self.others { write!(f, " {e}")?; } - write!(f, " )") + f.write_str(" )") } } @@ -1184,7 +1184,7 @@ impl fmt::Display for CollectionPath { for e in &self.others { write!(f, " {e}")?; } - write!(f, " )") + f.write_str(" )") } } @@ -1289,7 +1289,7 @@ impl fmt::Display for GraphTerm { match self { Self::Iri(iri) => write!(f, "{iri}"), Self::Literal(l) => write!(f, "{l}"), - Self::Nil => write!(f, " () "), + Self::Nil => f.write_str(" () "), } } } diff --git a/lib/src/sparql/error.rs b/lib/src/sparql/error.rs index 43234d67..a8627790 100644 --- a/lib/src/sparql/error.rs +++ b/lib/src/sparql/error.rs @@ -51,7 +51,9 @@ impl fmt::Display for EvaluationError { Self::Service(error) => error.fmt(f), Self::GraphAlreadyExists(graph) => write!(f, "The graph {graph} already exists"), Self::GraphDoesNotExist(graph) => write!(f, "The graph {graph} does not exist"), - Self::UnboundService => write!(f, "The variable encoding the service name is unbound"), + Self::UnboundService => { + f.write_str("The variable encoding the service name is unbound") + } Self::UnsupportedService(service) => { write!(f, "The service {service} is not supported") } @@ -62,7 +64,7 @@ impl fmt::Display for EvaluationError { f, "The service is not returning solutions but a boolean or a graph" ), - Self::NotAGraph => write!(f, "The query results are not a RDF graph"), + Self::NotAGraph => f.write_str("The query results are not a RDF graph"), } } } diff --git a/testsuite/src/sparql_evaluator.rs b/testsuite/src/sparql_evaluator.rs index 844158d9..b68c4c2b 100644 --- a/testsuite/src/sparql_evaluator.rs +++ b/testsuite/src/sparql_evaluator.rs @@ -629,12 +629,12 @@ fn solutions_to_string(solutions: Vec>, ordered: bool) -> .into_iter() .map(|mut s| { let mut out = String::new(); - write!(&mut out, "{{").unwrap(); + out.write_str("{").unwrap(); s.sort_unstable_by(|(v1, _), (v2, _)| v1.cmp(v2)); for (variable, value) in s { write!(&mut out, "{variable} = {value} ").unwrap(); } - write!(&mut out, "}}").unwrap(); + out.write_str("}").unwrap(); out }) .collect::>(); From 1e37577b717a25e2b4db57d4f6dcb909a76fccd8 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Wed, 7 Feb 2024 00:49:16 -0500 Subject: [PATCH 199/217] Optimize some code, lints * A few match simplifications * Order trait impl to match trait itself * A few unneeded parenthesis * One dup code de-duplication with a new func --- lib/oxrdf/src/dataset.rs | 8 +++--- lib/oxrdf/src/graph.rs | 2 +- lib/oxrdfxml/src/error.rs | 14 +++++----- lib/oxsdatatypes/src/decimal.rs | 9 ++++--- lib/oxttl/src/lexer.rs | 2 +- lib/oxttl/src/line_formats.rs | 4 +-- lib/oxttl/src/n3.rs | 4 +-- lib/oxttl/src/terse.rs | 4 +-- lib/oxttl/src/toolkit/error.rs | 2 +- lib/sparesults/src/error.rs | 14 +++++----- lib/sparesults/src/solution.rs | 2 +- lib/spargebra/src/parser.rs | 46 +++++++++++++-------------------- lib/src/sparql/eval.rs | 2 +- 13 files changed, 51 insertions(+), 62 deletions(-) diff --git a/lib/oxrdf/src/dataset.rs b/lib/oxrdf/src/dataset.rs index 8412a8aa..ed6249a4 100644 --- a/lib/oxrdf/src/dataset.rs +++ b/lib/oxrdf/src/dataset.rs @@ -925,8 +925,8 @@ impl PartialEq for Dataset { impl Eq for Dataset {} impl<'a> IntoIterator for &'a Dataset { - type IntoIter = Iter<'a>; type Item = QuadRef<'a>; + type IntoIter = Iter<'a>; fn into_iter(self) -> Self::IntoIter { self.iter() @@ -1283,8 +1283,8 @@ impl<'a> GraphView<'a> { } impl<'a> IntoIterator for GraphView<'a> { - type IntoIter = GraphViewIter<'a>; type Item = TripleRef<'a>; + type IntoIter = GraphViewIter<'a>; fn into_iter(self) -> Self::IntoIter { self.iter() @@ -1292,8 +1292,8 @@ impl<'a> IntoIterator for GraphView<'a> { } impl<'a, 'b> IntoIterator for &'b GraphView<'a> { - type IntoIter = GraphViewIter<'a>; type Item = TripleRef<'a>; + type IntoIter = GraphViewIter<'a>; fn into_iter(self) -> Self::IntoIter { self.iter() @@ -1494,8 +1494,8 @@ impl<'a, 'b, T: Into>> Extend for GraphViewMut<'a> { } impl<'a> IntoIterator for &'a GraphViewMut<'a> { - type IntoIter = GraphViewIter<'a>; type Item = TripleRef<'a>; + type IntoIter = GraphViewIter<'a>; fn into_iter(self) -> Self::IntoIter { self.iter() diff --git a/lib/oxrdf/src/graph.rs b/lib/oxrdf/src/graph.rs index 33f67132..5459b65c 100644 --- a/lib/oxrdf/src/graph.rs +++ b/lib/oxrdf/src/graph.rs @@ -229,8 +229,8 @@ impl PartialEq for Graph { impl Eq for Graph {} impl<'a> IntoIterator for &'a Graph { - type IntoIter = Iter<'a>; type Item = TripleRef<'a>; + type IntoIter = Iter<'a>; fn into_iter(self) -> Self::IntoIter { self.iter() diff --git a/lib/oxrdfxml/src/error.rs b/lib/oxrdfxml/src/error.rs index cb9eb9c4..be2e161d 100644 --- a/lib/oxrdfxml/src/error.rs +++ b/lib/oxrdfxml/src/error.rs @@ -61,10 +61,9 @@ impl From for ParseError { #[inline] fn from(error: quick_xml::Error) -> Self { match error { - quick_xml::Error::Io(error) => Self::Io(match Arc::try_unwrap(error) { - Ok(error) => error, - Err(error) => io::Error::new(error.kind(), error), - }), + quick_xml::Error::Io(error) => { + Self::Io(Arc::try_unwrap(error).unwrap_or_else(|e| io::Error::new(e.kind(), e))) + } _ => Self::Syntax(SyntaxError { inner: SyntaxErrorKind::Xml(error), }), @@ -137,10 +136,9 @@ impl From for io::Error { fn from(error: SyntaxError) -> Self { match error.inner { SyntaxErrorKind::Xml(error) => match error { - quick_xml::Error::Io(error) => match Arc::try_unwrap(error) { - Ok(error) => error, - Err(error) => Self::new(error.kind(), error), - }, + quick_xml::Error::Io(error) => { + Arc::try_unwrap(error).unwrap_or_else(|e| Self::new(e.kind(), e)) + } quick_xml::Error::UnexpectedEof(error) => { Self::new(io::ErrorKind::UnexpectedEof, error) } diff --git a/lib/oxsdatatypes/src/decimal.rs b/lib/oxsdatatypes/src/decimal.rs index b526e541..ee84b604 100644 --- a/lib/oxsdatatypes/src/decimal.rs +++ b/lib/oxsdatatypes/src/decimal.rs @@ -882,14 +882,16 @@ mod tests { ); assert_eq!(Decimal::MAX.checked_round(), None); assert_eq!( - (Decimal::MAX.checked_sub(Decimal::from_str("0.5")?)) + Decimal::MAX + .checked_sub(Decimal::from_str("0.5")?) .unwrap() .checked_round(), Some(Decimal::from_str("170141183460469231731")?) ); assert_eq!(Decimal::MIN.checked_round(), None); assert_eq!( - (Decimal::MIN.checked_add(Decimal::from_str("0.5")?)) + Decimal::MIN + .checked_add(Decimal::from_str("0.5")?) .unwrap() .checked_round(), Some(Decimal::from_str("-170141183460469231731")?) @@ -958,7 +960,8 @@ mod tests { ); assert_eq!(Decimal::MIN.checked_floor(), None); assert_eq!( - (Decimal::MIN.checked_add(Decimal::from_str("1")?)) + Decimal::MIN + .checked_add(Decimal::from_str("1")?) .unwrap() .checked_floor(), Some(Decimal::from_str("-170141183460469231731")?) diff --git a/lib/oxttl/src/lexer.rs b/lib/oxttl/src/lexer.rs index 1eac849e..f60c230e 100644 --- a/lib/oxttl/src/lexer.rs +++ b/lib/oxttl/src/lexer.rs @@ -49,8 +49,8 @@ pub struct N3Lexer { // TODO: simplify by not giving is_end and fail with an "unexpected eof" is none is returned when is_end=true? impl TokenRecognizer for N3Lexer { - type Options = N3LexerOptions; type Token<'a> = N3Token<'a>; + type Options = N3LexerOptions; fn recognize_next_token<'a>( &mut self, diff --git a/lib/oxttl/src/line_formats.rs b/lib/oxttl/src/line_formats.rs index 5932f7a2..e522bd53 100644 --- a/lib/oxttl/src/line_formats.rs +++ b/lib/oxttl/src/line_formats.rs @@ -39,9 +39,9 @@ enum NQuadsState { } impl RuleRecognizer for NQuadsRecognizer { - type Context = NQuadsRecognizerContext; - type Output = Quad; type TokenRecognizer = N3Lexer; + type Output = Quad; + type Context = NQuadsRecognizerContext; fn error_recovery_state(mut self) -> Self { self.stack.clear(); diff --git a/lib/oxttl/src/n3.rs b/lib/oxttl/src/n3.rs index 59ba9cf3..8b70a01e 100644 --- a/lib/oxttl/src/n3.rs +++ b/lib/oxttl/src/n3.rs @@ -723,9 +723,9 @@ struct N3RecognizerContext { } impl RuleRecognizer for N3Recognizer { - type Context = N3RecognizerContext; - type Output = N3Quad; type TokenRecognizer = N3Lexer; + type Output = N3Quad; + type Context = N3RecognizerContext; fn error_recovery_state(mut self) -> Self { self.stack.clear(); diff --git a/lib/oxttl/src/terse.rs b/lib/oxttl/src/terse.rs index ebeff436..c233c735 100644 --- a/lib/oxttl/src/terse.rs +++ b/lib/oxttl/src/terse.rs @@ -35,9 +35,9 @@ impl TriGRecognizerContext { } impl RuleRecognizer for TriGRecognizer { - type Context = TriGRecognizerContext; - type Output = Quad; type TokenRecognizer = N3Lexer; + type Output = Quad; + type Context = TriGRecognizerContext; fn error_recovery_state(mut self) -> Self { self.stack.clear(); diff --git a/lib/oxttl/src/toolkit/error.rs b/lib/oxttl/src/toolkit/error.rs index e279dab4..2f632352 100644 --- a/lib/oxttl/src/toolkit/error.rs +++ b/lib/oxttl/src/toolkit/error.rs @@ -78,7 +78,7 @@ impl From for io::Error { /// A parsing error. /// -/// It is the union of [`SyntaxError`] and [`std::io::Error`]. +/// It is the union of [`SyntaxError`] and [`io::Error`]. #[derive(Debug)] pub enum ParseError { /// I/O error during parsing (file not found...). diff --git a/lib/sparesults/src/error.rs b/lib/sparesults/src/error.rs index b8fe2eff..8ece28d0 100644 --- a/lib/sparesults/src/error.rs +++ b/lib/sparesults/src/error.rs @@ -70,10 +70,9 @@ impl From for ParseError { #[inline] fn from(error: quick_xml::Error) -> Self { match error { - quick_xml::Error::Io(error) => Self::Io(match Arc::try_unwrap(error) { - Ok(error) => error, - Err(error) => io::Error::new(error.kind(), error), - }), + quick_xml::Error::Io(error) => { + Self::Io(Arc::try_unwrap(error).unwrap_or_else(|e| io::Error::new(e.kind(), e))) + } _ => Self::Syntax(SyntaxError { inner: SyntaxErrorKind::Xml(error), }), @@ -184,10 +183,9 @@ impl From for io::Error { match error.inner { SyntaxErrorKind::Json(error) => Self::new(io::ErrorKind::InvalidData, error), SyntaxErrorKind::Xml(error) => match error { - quick_xml::Error::Io(error) => match Arc::try_unwrap(error) { - Ok(error) => error, - Err(error) => Self::new(error.kind(), error), - }, + quick_xml::Error::Io(error) => { + Arc::try_unwrap(error).unwrap_or_else(|e| Self::new(e.kind(), e)) + } quick_xml::Error::UnexpectedEof(error) => { Self::new(io::ErrorKind::UnexpectedEof, error) } diff --git a/lib/sparesults/src/solution.rs b/lib/sparesults/src/solution.rs index 826a9eea..a1364861 100644 --- a/lib/sparesults/src/solution.rs +++ b/lib/sparesults/src/solution.rs @@ -171,8 +171,8 @@ impl>, S: Into>>> From<(V, S)> for Quer } impl<'a> IntoIterator for &'a QuerySolution { - type IntoIter = Iter<'a>; type Item = (&'a Variable, &'a Term); + type IntoIter = Iter<'a>; #[inline] fn into_iter(self) -> Self::IntoIter { diff --git a/lib/spargebra/src/parser.rs b/lib/spargebra/src/parser.rs index 03a71932..bcb4d491 100644 --- a/lib/spargebra/src/parser.rs +++ b/lib/spargebra/src/parser.rs @@ -17,20 +17,7 @@ use std::{char, fmt}; /// Parses a SPARQL query with an optional base IRI to resolve relative IRIs in the query. pub fn parse_query(query: &str, base_iri: Option<&str>) -> Result { - let mut state = ParserState { - base_iri: if let Some(base_iri) = base_iri { - Some(Iri::parse(base_iri.to_owned()).map_err(|e| ParseError { - inner: ParseErrorKind::InvalidBaseIri(e), - })?) - } else { - None - }, - namespaces: HashMap::default(), - used_bnodes: HashSet::default(), - currently_used_bnodes: HashSet::default(), - aggregates: Vec::new(), - }; - + let mut state = ParserState::from_base_iri(base_iri)?; parser::QueryUnit(query, &mut state).map_err(|e| ParseError { inner: ParseErrorKind::Parser(e), }) @@ -38,20 +25,7 @@ pub fn parse_query(query: &str, base_iri: Option<&str>) -> Result) -> Result { - let mut state = ParserState { - base_iri: if let Some(base_iri) = base_iri { - Some(Iri::parse(base_iri.to_owned()).map_err(|e| ParseError { - inner: ParseErrorKind::InvalidBaseIri(e), - })?) - } else { - None - }, - namespaces: HashMap::default(), - used_bnodes: HashSet::default(), - currently_used_bnodes: HashSet::default(), - aggregates: Vec::new(), - }; - + let mut state = ParserState::from_base_iri(base_iri)?; let operations = parser::UpdateInit(update, &mut state).map_err(|e| ParseError { inner: ParseErrorKind::Parser(e), })?; @@ -720,6 +694,22 @@ pub struct ParserState { } impl ParserState { + pub(crate) fn from_base_iri(base_iri: Option<&str>) -> Result { + Ok(Self { + base_iri: if let Some(base_iri) = base_iri { + Some(Iri::parse(base_iri.to_owned()).map_err(|e| ParseError { + inner: ParseErrorKind::InvalidBaseIri(e), + })?) + } else { + None + }, + namespaces: HashMap::default(), + used_bnodes: HashSet::default(), + currently_used_bnodes: HashSet::default(), + aggregates: Vec::new(), + }) + } + fn parse_iri(&self, iri: String) -> Result, IriParseError> { if let Some(base_iri) = &self.base_iri { base_iri.resolve(&iri) diff --git a/lib/src/sparql/eval.rs b/lib/src/sparql/eval.rs index 19c6884d..174f41fa 100644 --- a/lib/src/sparql/eval.rs +++ b/lib/src/sparql/eval.rs @@ -111,8 +111,8 @@ impl EncodedTuple { } impl IntoIterator for EncodedTuple { - type IntoIter = std::vec::IntoIter>; type Item = Option; + type IntoIter = std::vec::IntoIter>; fn into_iter(self) -> Self::IntoIter { self.inner.into_iter() From c15233e9646e7a196dc7731549cb97374841230d Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Wed, 7 Feb 2024 03:07:33 -0500 Subject: [PATCH 200/217] do not order trait methods --- rustfmt.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/rustfmt.toml b/rustfmt.toml index 5fce0595..751c75da 100644 --- a/rustfmt.toml +++ b/rustfmt.toml @@ -5,7 +5,6 @@ imports_granularity = "Module" newline_style = "Unix" normalize_comments = true normalize_doc_attributes = true -reorder_impl_items = true group_imports = "One" use_field_init_shorthand = true use_try_shorthand = true From d49fb477677cacf2271f8c5e27d7c5346ec04b95 Mon Sep 17 00:00:00 2001 From: Tpt Date: Sat, 3 Feb 2024 15:31:34 +0100 Subject: [PATCH 201/217] Adds a link to RDFa and JSON-LD parsers in Rust --- lib/oxrdfio/README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/oxrdfio/README.md b/lib/oxrdfio/README.md index 921c8365..72238f86 100644 --- a/lib/oxrdfio/README.md +++ b/lib/oxrdfio/README.md @@ -47,6 +47,9 @@ for quad in RdfParser::from_format(RdfFormat::Turtle).parse_read(turtle_file.as_ assert_eq!(writer.finish().unwrap(), ntriples_file); ``` +Parsers for other RDF formats exists in Rust like [graph-rdfa-processor](https://github.com/nbittich/graph-rdfa-processor) for RDFa and [json-ld](https://github.com/timothee-haudebourg/json-ld) for JSON-LD. + + ## License This project is licensed under either of From 70a4ff231b65b710edd56a4f9dbb3f51bedb6ac3 Mon Sep 17 00:00:00 2001 From: Tpt Date: Thu, 1 Feb 2024 19:04:33 +0100 Subject: [PATCH 202/217] Runs SPARQL 1.2 testsuite --- testsuite/rdf-tests | 2 +- testsuite/tests/sparql.rs | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/testsuite/rdf-tests b/testsuite/rdf-tests index 35fb5dd6..8bd6d803 160000 --- a/testsuite/rdf-tests +++ b/testsuite/rdf-tests @@ -1 +1 @@ -Subproject commit 35fb5dd65b9d504770008a85b4ff577cc581fcee +Subproject commit 8bd6d803093bc80750c0d01343166c9742ddd08c diff --git a/testsuite/tests/sparql.rs b/testsuite/tests/sparql.rs index 6d2a6f7d..0de42891 100644 --- a/testsuite/tests/sparql.rs +++ b/testsuite/tests/sparql.rs @@ -104,6 +104,17 @@ fn sparql11_tsv_w3c_evaluation_testsuite() -> Result<()> { ) } +#[test] +fn sparql12_w3c_testsuite() -> Result<()> { + check_testsuite( + "https://w3c.github.io/rdf-tests/sparql/sparql12/manifest.ttl", + &[ + // Literal normalization + "https://w3c.github.io/rdf-tests/sparql/sparql12/grouping#group01", + ], + ) +} + #[test] fn sparql_star_syntax_testsuite() -> Result<()> { check_testsuite( From 0b1aabfcdd1c93efd307e6f87d7cd690628bece2 Mon Sep 17 00:00:00 2001 From: Tpt Date: Wed, 7 Feb 2024 17:59:15 +0100 Subject: [PATCH 203/217] Moves main crate to lib/oxigraph and centralizes dependencies --- .github/workflows/tests.yml | 11 +-- Cargo.lock | 10 +- Cargo.toml | 65 ++++++++++++- cli/Cargo.toml | 37 +++---- fuzz/Cargo.toml | 6 +- js/Cargo.toml | 19 ++-- lib/Cargo.toml | 69 -------------- lib/README.md | 90 +++--------------- lib/oxigraph/Cargo.toml | 69 ++++++++++++++ lib/oxigraph/README.md | 77 +++++++++++++++ lib/{ => oxigraph}/benches/store.rs | 0 lib/{ => oxigraph}/src/io/format.rs | 0 lib/{ => oxigraph}/src/io/mod.rs | 0 lib/{ => oxigraph}/src/io/read.rs | 0 lib/{ => oxigraph}/src/io/write.rs | 0 lib/{ => oxigraph}/src/lib.rs | 0 lib/{ => oxigraph}/src/model.rs | 0 lib/{ => oxigraph}/src/sparql/algebra.rs | 0 lib/{ => oxigraph}/src/sparql/dataset.rs | 0 lib/{ => oxigraph}/src/sparql/error.rs | 0 lib/{ => oxigraph}/src/sparql/eval.rs | 0 lib/{ => oxigraph}/src/sparql/http/dummy.rs | 0 lib/{ => oxigraph}/src/sparql/http/mod.rs | 0 lib/{ => oxigraph}/src/sparql/http/simple.rs | 0 lib/{ => oxigraph}/src/sparql/mod.rs | 0 lib/{ => oxigraph}/src/sparql/model.rs | 0 lib/{ => oxigraph}/src/sparql/results.rs | 0 lib/{ => oxigraph}/src/sparql/service.rs | 0 lib/{ => oxigraph}/src/sparql/update.rs | 0 .../src/storage/backend/fallback.rs | 0 lib/{ => oxigraph}/src/storage/backend/mod.rs | 0 .../src/storage/backend/rocksdb.rs | 0 .../src/storage/binary_encoder.rs | 0 lib/{ => oxigraph}/src/storage/error.rs | 0 lib/{ => oxigraph}/src/storage/mod.rs | 0 .../src/storage/numeric_encoder.rs | 0 .../src/storage/small_string.rs | 0 lib/{ => oxigraph}/src/store.rs | 0 .../tests/rocksdb_bc_data/000003.log | Bin .../tests/rocksdb_bc_data/CURRENT | 0 .../tests/rocksdb_bc_data/IDENTITY | 0 lib/{ => oxigraph}/tests/rocksdb_bc_data/LOCK | 0 .../tests/rocksdb_bc_data/MANIFEST-000004 | Bin .../tests/rocksdb_bc_data/OPTIONS-000026 | 0 lib/{ => oxigraph}/tests/store.rs | 0 lib/oxrdf/Cargo.toml | 16 ++-- lib/oxrdfio/Cargo.toml | 18 ++-- lib/oxrdfxml/Cargo.toml | 20 ++-- lib/oxsdatatypes/Cargo.toml | 10 +- lib/oxttl/Cargo.toml | 20 ++-- lib/sparesults/Cargo.toml | 20 ++-- lib/spargebra/Cargo.toml | 18 ++-- lib/sparopt/Cargo.toml | 14 +-- lib/sparql-smith/Cargo.toml | 10 +- oxrocksdb-sys/Cargo.toml | 16 ++-- python/Cargo.lock | 1 - python/Cargo.toml | 19 ++-- testsuite/Cargo.toml | 26 ++--- 58 files changed, 366 insertions(+), 295 deletions(-) delete mode 100644 lib/Cargo.toml create mode 100644 lib/oxigraph/Cargo.toml create mode 100644 lib/oxigraph/README.md rename lib/{ => oxigraph}/benches/store.rs (100%) rename lib/{ => oxigraph}/src/io/format.rs (100%) rename lib/{ => oxigraph}/src/io/mod.rs (100%) rename lib/{ => oxigraph}/src/io/read.rs (100%) rename lib/{ => oxigraph}/src/io/write.rs (100%) rename lib/{ => oxigraph}/src/lib.rs (100%) rename lib/{ => oxigraph}/src/model.rs (100%) rename lib/{ => oxigraph}/src/sparql/algebra.rs (100%) rename lib/{ => oxigraph}/src/sparql/dataset.rs (100%) rename lib/{ => oxigraph}/src/sparql/error.rs (100%) rename lib/{ => oxigraph}/src/sparql/eval.rs (100%) rename lib/{ => oxigraph}/src/sparql/http/dummy.rs (100%) rename lib/{ => oxigraph}/src/sparql/http/mod.rs (100%) rename lib/{ => oxigraph}/src/sparql/http/simple.rs (100%) rename lib/{ => oxigraph}/src/sparql/mod.rs (100%) rename lib/{ => oxigraph}/src/sparql/model.rs (100%) rename lib/{ => oxigraph}/src/sparql/results.rs (100%) rename lib/{ => oxigraph}/src/sparql/service.rs (100%) rename lib/{ => oxigraph}/src/sparql/update.rs (100%) rename lib/{ => oxigraph}/src/storage/backend/fallback.rs (100%) rename lib/{ => oxigraph}/src/storage/backend/mod.rs (100%) rename lib/{ => oxigraph}/src/storage/backend/rocksdb.rs (100%) rename lib/{ => oxigraph}/src/storage/binary_encoder.rs (100%) rename lib/{ => oxigraph}/src/storage/error.rs (100%) rename lib/{ => oxigraph}/src/storage/mod.rs (100%) rename lib/{ => oxigraph}/src/storage/numeric_encoder.rs (100%) rename lib/{ => oxigraph}/src/storage/small_string.rs (100%) rename lib/{ => oxigraph}/src/store.rs (100%) rename lib/{ => oxigraph}/tests/rocksdb_bc_data/000003.log (100%) rename lib/{ => oxigraph}/tests/rocksdb_bc_data/CURRENT (100%) rename lib/{ => oxigraph}/tests/rocksdb_bc_data/IDENTITY (100%) rename lib/{ => oxigraph}/tests/rocksdb_bc_data/LOCK (100%) rename lib/{ => oxigraph}/tests/rocksdb_bc_data/MANIFEST-000004 (100%) rename lib/{ => oxigraph}/tests/rocksdb_bc_data/OPTIONS-000026 (100%) rename lib/{ => oxigraph}/tests/store.rs (100%) delete mode 120000 python/Cargo.lock diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d1fcb1bc..42dbae54 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -53,7 +53,7 @@ jobs: - run: cargo clippy --all-targets -- -D warnings -D clippy::all working-directory: ./lib/sparopt - run: cargo clippy --all-targets -- -D warnings -D clippy::all - working-directory: ./lib + working-directory: ./lib/oxigraph - run: cargo clippy --all-targets -- -D warnings -D clippy::all working-directory: ./python - run: cargo clippy --all-targets -- -D warnings -D clippy::all @@ -87,7 +87,7 @@ jobs: target: wasm32-wasi component: clippy - run: cargo clippy --lib --tests --target wasm32-wasi -- -D warnings -D clippy::all - working-directory: ./lib + working-directory: ./lib/oxigraph - run: cargo clippy --target wasm32-wasi --features abi3 --no-default-features -- -D warnings -D clippy::all working-directory: ./python @@ -103,7 +103,7 @@ jobs: target: wasm32-unknown-emscripten component: clippy - run: cargo clippy --lib --tests --target wasm32-unknown-emscripten -- -D warnings -D clippy::all - working-directory: ./lib + working-directory: ./lib/oxigraph - run: cargo clippy --target wasm32-unknown-emscripten --features abi3 -- -D warnings -D clippy::all working-directory: ./python @@ -119,7 +119,7 @@ jobs: target: wasm32-unknown-unknown component: clippy - run: cargo clippy --lib --tests --target wasm32-unknown-unknown --features getrandom/custom --features oxsdatatypes/custom-now -- -D warnings -D clippy::all - working-directory: ./lib + working-directory: ./lib/oxigraph deny: runs-on: ubuntu-latest @@ -164,7 +164,7 @@ jobs: target: i686-unknown-linux-gnu - run: sudo apt-get update && sudo apt-get install -y g++-multilib - run: cargo test --target i686-unknown-linux-gnu --no-default-features --features http-client-rustls-native - working-directory: ./lib + working-directory: ./lib/oxigraph test_linux_msv: runs-on: ubuntu-latest @@ -250,7 +250,6 @@ jobs: with: version: 1.74.1 - run: cargo doc - working-directory: ./lib env: RUSTDOCFLAGS: -D warnings diff --git a/Cargo.lock b/Cargo.lock index b63ea4a0..e33d7c52 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1018,7 +1018,7 @@ dependencies = [ [[package]] name = "oxigraph" -version = "0.4.0-alpha.3" +version = "0.4.0-alpha.3-dev" dependencies = [ "codspeed-criterion-compat", "digest", @@ -1048,7 +1048,7 @@ dependencies = [ [[package]] name = "oxigraph-cli" -version = "0.4.0-alpha.3" +version = "0.4.0-alpha.3-dev" dependencies = [ "anyhow", "assert_cmd", @@ -1067,7 +1067,7 @@ dependencies = [ [[package]] name = "oxigraph-js" -version = "0.4.0-alpha.3" +version = "0.4.0-alpha.3-dev" dependencies = [ "console_error_panic_hook", "js-sys", @@ -1135,7 +1135,7 @@ dependencies = [ [[package]] name = "oxrocksdb-sys" -version = "0.4.0-alpha.3" +version = "0.4.0-alpha.3-dev" dependencies = [ "bindgen", "cc", @@ -1387,7 +1387,7 @@ dependencies = [ [[package]] name = "pyoxigraph" -version = "0.4.0-alpha.3" +version = "0.4.0-alpha.3-dev" dependencies = [ "oxigraph", "pyo3", diff --git a/Cargo.toml b/Cargo.toml index 507f5023..91b1b7ec 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,24 +1,81 @@ [workspace] members = [ + "cli", "js", - "lib", + "lib/oxigraph", "lib/oxrdf", "lib/oxrdfio", "lib/oxrdfxml", "lib/oxsdatatypes", "lib/oxttl", - "lib/spargebra", "lib/sparesults", + "lib/spargebra", "lib/sparopt", "lib/sparql-smith", "oxrocksdb-sys", "python", - "cli", "testsuite" ] resolver = "2" -# TODO: use workspace.package shared metadata when https://github.com/obi1kenobi/cargo-semver-checks/issues/462 will be fixed +[workspace.package] +version = "0.4.0-alpha.3-dev" +authors = ["Tpt "] +license = "MIT OR Apache-2.0" +edition = "2021" +rust-version = "1.70" + +[workspace.dependencies] +anyhow = "1.0.72" +arbitrary = "1.3" +assert_cmd = "2.0" +assert_fs = "1.0" +bindgen = ">=0.60, <0.70" +cc = "1.0.73" +clap = "4.0" +codspeed-criterion-compat = "2.3.3" +console_error_panic_hook = "0.1.7" +digest = "0.10" +escargot = "0.5" +flate2 = "1.0" +getrandom = "0.2.8" +hex = "0.4" +js-sys = "0.3.60" +json-event-parser = "0.2.0-alpha.2" +libc = "0.2.147" +md-5 = "0.10" +memchr = "2.5" +oxhttp = "0.2.0-alpha.3" +oxigraph = { version = "0.4.0-alpha.3-dev", path = "./lib/oxigraph" } +oxilangtag = "0.1" +oxiri = "0.2.3-alpha.1" +oxrdf = { version = "0.2.0-alpha.2", path = "lib/oxrdf" } +oxrdfdatatypes = { version = "0.2.0-alpha.1", path = "lib/oxsdatatypes" } +oxrdfio = { version = "0.1.0-alpha.2", path = "lib/oxrdfio" } +oxrdfxml = { version = "0.1.0-alpha.2", path = "lib/oxrdfxml" } +oxrocksdb-sys = { version = "0.4.0-alpha.3-dev", path = "./oxrocksdb-sys" } +oxsdatatypes = { version = "0.2.0-alpha.1", path = "lib/oxsdatatypes" } +oxttl = { version = "0.1.0-alpha.2", path = "lib/oxttl" } +peg = "0.8" +pkg-config = "0.3.25" +predicates = ">=2.0, <4.0" +pyo3 = "0.20.1" +quick-xml = ">=0.29, <0.32" +rand = "0.8" +rayon-core = "1.11" +regex = "1.7" +sha1 = "0.10" +sha2 = "0.10" +siphasher = ">=0.3, <2.0" +sparesults = { version = "0.2.0-alpha.2", path = "lib/sparesults" } +spargebra = { version = "0.3.0-alpha.2", path = "lib/spargebra" } +sparopt = { version = "0.1.0-alpha.2", path = "lib/sparopt"} +text-diff = "0.4" +time = "0.3" +tokio = "1.29" +url = "2.4" +wasm-bindgen = "0.2.83" +zstd = ">=0.12, <0.14" [workspace.lints.rust] absolute_paths_not_starting_with_crate = "warn" diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 0c00e383..88cef55c 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -1,8 +1,8 @@ [package] name = "oxigraph-cli" -version = "0.4.0-alpha.3" -authors = ["Tpt "] -license = "MIT OR Apache-2.0" +version.workspace = true +authors.workspace = true +license.workspace = true readme = "README.md" keywords = ["RDF", "SPARQL", "graph-database", "database"] categories = ["command-line-utilities", "database"] @@ -11,12 +11,13 @@ homepage = "https://oxigraph.org/cli/" description = """ Oxigraph command line toolkit and SPARQL HTTP server """ -edition = "2021" -rust-version = "1.70" +edition.workspace = true +rust-version.workspace = true [[bin]] name = "oxigraph" path = "src/main.rs" +doc = false [features] default = ["native-tls"] @@ -26,21 +27,21 @@ rustls-native = ["oxigraph/http-client-rustls-native"] rustls-webpki = ["oxigraph/http-client-rustls-webpki"] [dependencies] -anyhow = "1.0.72" -oxhttp = { version = "0.2.0-alpha.3", features = ["flate2"] } -clap = { version = "4.0", features = ["derive"] } -oxigraph = { version = "0.4.0-alpha.3", path = "../lib" } -rand = "0.8" -url = "2.4" -oxiri = "0.2.3-alpha.1" -flate2 = "1.0" -rayon-core = "1.11" +anyhow.workspace = true +clap = { workspace = true, features = ["derive"] } +flate2.workspace = true +oxhttp = { workspace = true, features = ["flate2"] } +oxigraph.workspace = true +oxiri.workspace = true +rand.workspace = true +rayon-core.workspace = true +url.workspace = true [dev-dependencies] -assert_cmd = "2.0" -assert_fs = "1.0" -escargot = "0.5" -predicates = ">=2.0, <4.0" +assert_cmd.workspace = true +assert_fs.workspace = true +escargot.workspace = true +predicates.workspace = true [lints] workspace = true diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index de97438f..8ce8daf3 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -10,13 +10,13 @@ cargo-fuzz = true [dependencies] anyhow = "1.0.72" libfuzzer-sys = "0.4" +oxigraph = { path = "../lib/oxigraph" } oxrdf = { path = "../lib/oxrdf", features = ["rdf-star"] } -oxttl = { path = "../lib/oxttl", features = ["rdf-star"] } oxrdfxml = { path = "../lib/oxrdfxml" } -spargebra = { path = "../lib/spargebra", features = ["rdf-star", "sep-0006"] } +oxttl = { path = "../lib/oxttl", features = ["rdf-star"] } sparesults = { path = "../lib/sparesults", features = ["rdf-star"] } +spargebra = { path = "../lib/spargebra", features = ["rdf-star", "sep-0006"] } sparql-smith = { path = "../lib/sparql-smith", features = ["sep-0006"] } -oxigraph = { path = "../lib" } [profile.release] codegen-units = 1 diff --git a/js/Cargo.toml b/js/Cargo.toml index 16764596..0b5156b3 100644 --- a/js/Cargo.toml +++ b/js/Cargo.toml @@ -1,25 +1,26 @@ [package] name = "oxigraph-js" -version = "0.4.0-alpha.3" -authors = ["Tpt "] -license = "MIT OR Apache-2.0" +version.workspace = true +authors.workspace = true +license.workspace = true readme = "README.md" keywords = ["RDF", "N-Triples", "Turtle", "RDF/XML", "SPARQL"] repository = "https://github.com/oxigraph/oxigraph/tree/main/js" description = "JavaScript bindings of Oxigraph" -edition = "2021" -rust-version = "1.70" +edition.workspace = true +rust-version.workspace = true publish = false [lib] crate-type = ["cdylib"] name = "oxigraph" +doc = false [dependencies] -oxigraph = { path = "../lib", features = ["js"] } -wasm-bindgen = "0.2.83" -js-sys = "0.3.60" -console_error_panic_hook = "0.1.7" +console_error_panic_hook.workspace = true +js-sys.workspace = true +oxigraph = { workspace = true, features = ["js"] } +wasm-bindgen.workspace = true [lints] workspace = true diff --git a/lib/Cargo.toml b/lib/Cargo.toml deleted file mode 100644 index 12a62f22..00000000 --- a/lib/Cargo.toml +++ /dev/null @@ -1,69 +0,0 @@ -[package] -name = "oxigraph" -version = "0.4.0-alpha.3" -authors = ["Tpt "] -license = "MIT OR Apache-2.0" -readme = "README.md" -keywords = ["RDF", "SPARQL", "graph-database", "database"] -categories = ["database-implementations"] -repository = "https://github.com/oxigraph/oxigraph/tree/main/lib" -homepage = "https://oxigraph.org/" -documentation = "https://docs.rs/oxigraph" -description = """ -a SPARQL database and RDF toolkit -""" -edition = "2021" -rust-version = "1.70" - -[features] -default = [] -js = ["getrandom/js", "oxsdatatypes/js", "js-sys"] -http-client = ["oxhttp"] -http-client-native-tls = ["http-client", "oxhttp/native-tls"] -http-client-rustls-webpki = ["http-client", "oxhttp/rustls-webpki"] -http-client-rustls-native = ["http-client", "oxhttp/rustls-native"] -rocksdb-pkg-config = ["oxrocksdb-sys/pkg-config"] -rocksdb-debug = [] - -[dependencies] -digest = "0.10" -hex = "0.4" -json-event-parser = "0.2.0-alpha.2" -md-5 = "0.10" -oxilangtag = "0.1" -oxiri = "0.2.3-alpha.1" -oxrdf = { version = "0.2.0-alpha.2", path = "oxrdf", features = ["rdf-star", "oxsdatatypes"] } -oxrdfio = { version = "0.1.0-alpha.2", path = "oxrdfio", features = ["rdf-star"] } -oxsdatatypes = { version = "0.2.0-alpha.1", path = "oxsdatatypes" } -rand = "0.8" -regex = "1.7" -sha1 = "0.10" -sha2 = "0.10" -siphasher = ">=0.3, <2.0" -sparesults = { version = "0.2.0-alpha.2", path = "sparesults", features = ["rdf-star"] } -spargebra = { version = "0.3.0-alpha.2", path = "spargebra", features = ["rdf-star", "sep-0002", "sep-0006"] } -sparopt = { version = "0.1.0-alpha.2", path = "sparopt", features = ["rdf-star", "sep-0002", "sep-0006"] } - -[target.'cfg(not(target_family = "wasm"))'.dependencies] -libc = "0.2.147" -oxrocksdb-sys = { version = "0.4.0-alpha.3", path = "../oxrocksdb-sys" } -oxhttp = { version = "0.2.0-alpha.3", optional = true } - -[target.'cfg(all(target_family = "wasm", target_os = "unknown"))'.dependencies] -getrandom = "0.2.8" -js-sys = { version = "0.3.60", optional = true } - -[target.'cfg(not(target_family = "wasm"))'.dev-dependencies] -codspeed-criterion-compat = "2.3.3" -oxhttp = "0.2.0-alpha.3" -zstd = ">=0.12, <0.14" - -[lints] -workspace = true - -[package.metadata.docs.rs] -rustdoc-args = ["--cfg", "docsrs"] - -[[bench]] -name = "store" -harness = false diff --git a/lib/README.md b/lib/README.md index 4e4f72df..47d5eb4c 100644 --- a/lib/README.md +++ b/lib/README.md @@ -1,77 +1,13 @@ -Oxigraph -======== - -[![Latest Version](https://img.shields.io/crates/v/oxigraph.svg)](https://crates.io/crates/oxigraph) -[![Released API docs](https://docs.rs/oxigraph/badge.svg)](https://docs.rs/oxigraph) -[![Crates.io downloads](https://img.shields.io/crates/d/oxigraph)](https://crates.io/crates/oxigraph) -[![actions status](https://github.com/oxigraph/oxigraph/workflows/build/badge.svg)](https://github.com/oxigraph/oxigraph/actions) -[![Gitter](https://badges.gitter.im/oxigraph/community.svg)](https://gitter.im/oxigraph/community) - -Oxigraph is a graph database library implementing the [SPARQL](https://www.w3.org/TR/sparql11-overview/) standard. - -Its goal is to provide a compliant, safe and fast on-disk graph database. -It also provides a set of utility functions for reading, writing, and processing RDF files. - -Oxigraph is in heavy development and SPARQL query evaluation has not been optimized yet. - -Oxigraph also provides [a CLI tool](https://crates.io/crates/oxigraph-cli) and [a Python library](https://pyoxigraph.readthedocs.io/) based on this library. - - -Oxigraph implements the following specifications: -* [SPARQL 1.1 Query](https://www.w3.org/TR/sparql11-query/), [SPARQL 1.1 Update](https://www.w3.org/TR/sparql11-update/), and [SPARQL 1.1 Federated Query](https://www.w3.org/TR/sparql11-federated-query/). -* [Turtle](https://www.w3.org/TR/turtle/), [TriG](https://www.w3.org/TR/trig/), [N-Triples](https://www.w3.org/TR/n-triples/), [N-Quads](https://www.w3.org/TR/n-quads/), and [RDF XML](https://www.w3.org/TR/rdf-syntax-grammar/) RDF serialization formats for both data ingestion and retrieval using the [Rio library](https://github.com/oxigraph/rio). -* [SPARQL Query Results XML Format](https://www.w3.org/TR/rdf-sparql-XMLres/), [SPARQL 1.1 Query Results JSON Format](https://www.w3.org/TR/sparql11-results-json/) and [SPARQL 1.1 Query Results CSV and TSV Formats](https://www.w3.org/TR/sparql11-results-csv-tsv/). - -A preliminary benchmark [is provided](../bench/README.md). Oxigraph internal design [is described on the wiki](https://github.com/oxigraph/oxigraph/wiki/Architecture). - -The main entry point of Oxigraph is the [`Store`](store::Store) struct: -```rust -use oxigraph::store::Store; -use oxigraph::model::*; -use oxigraph::sparql::QueryResults; - -let store = Store::new().unwrap(); - -// insertion -let ex = NamedNode::new("http://example.com").unwrap(); -let quad = Quad::new(ex.clone(), ex.clone(), ex.clone(), GraphName::DefaultGraph); -store.insert(&quad).unwrap(); - -// quad filter -let results = store.quads_for_pattern(Some(ex.as_ref().into()), None, None, None).collect::,_>>().unwrap(); -assert_eq!(vec![quad], results); - -// SPARQL query -if let QueryResults::Solutions(mut solutions) = store.query("SELECT ?s WHERE { ?s ?p ?o }").unwrap() { - assert_eq!(solutions.next().unwrap().unwrap().get("s"), Some(&ex.into())); -} -``` - -Some parts of this library are available as standalone crates: -* [`oxrdf`](https://crates.io/crates/oxrdf), datastructures encoding RDF basic concepts (the [`oxigraph::model`](crate::model) module). -* [`oxrdfio`](https://crates.io/crates/oxrdfio), a unified parser and serializer API for RDF formats (the [`oxigraph::io`](crate::io) module). It itself relies on: - * [`oxttl`](https://crates.io/crates/oxttl), N-Triple, N-Quad, Turtle, TriG and N3 parsing and serialization. - * [`oxrdfxml`](https://crates.io/crates/oxrdfxml), RDF/XML parsing and serialization. -* [`spargebra`](https://crates.io/crates/spargebra), a SPARQL parser. -* [`sparesults`](https://crates.io/crates/sparesults), parsers and serializers for SPARQL result formats. -* [`sparopt`](https://crates.io/crates/sparesults), a SPARQL optimizer. -* [`oxsdatatypes`](https://crates.io/crates/oxsdatatypes), an implementation of some XML Schema datatypes. - -To build the library locally, don't forget to clone the submodules using `git clone --recursive https://github.com/oxigraph/oxigraph.git` to clone the repository including submodules or `git submodule update --init` to add submodules to the already cloned repository. - - -## License - -This project is licensed under either of - -* Apache License, Version 2.0, ([LICENSE-APACHE](../LICENSE-APACHE) or - ``) -* MIT license ([LICENSE-MIT](../LICENSE-MIT) or - ``) - -at your option. - - -### Contribution - -Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in Oxigraph by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. +Oxigraph Rust crates +==================== + +Oxigraph is implemented in Rust. +It is composed on a main library, [`oxigraph`](./oxigraph) and a set of smaller crates used by the `oxigraph` crate: +* [`oxrdf`](./oxrdf), datastructures encoding RDF basic concepts (the `model` module of the `oxigraph` crate). +* [`oxrdfio`](./oxrdfio), a unified parser and serializer API for RDF formats (the `io` module of the `oxigraph` crate). It itself relies on: + * [`oxttl`](./oxttl), N-Triple, N-Quad, Turtle, TriG and N3 parsing and serialization. + * [`oxrdfxml`](./oxrdfxml), RDF/XML parsing and serialization. +* [`spargebra`](./spargebra), a SPARQL parser. +* [`sparesults`](./sparesults), parsers and serializers for SPARQL result formats (the `sparql::results` module of the `oxigraph` crate). +* [`sparopt`](./sparesults), a SPARQL optimizer. +* [`oxsdatatypes`](./oxsdatatypes), an implementation of some XML Schema datatypes. diff --git a/lib/oxigraph/Cargo.toml b/lib/oxigraph/Cargo.toml new file mode 100644 index 00000000..13f14476 --- /dev/null +++ b/lib/oxigraph/Cargo.toml @@ -0,0 +1,69 @@ +[package] +name = "oxigraph" +version.workspace = true +authors.workspace = true +license.workspace = true +readme = "README.md" +keywords = ["RDF", "SPARQL", "graph-database", "database"] +categories = ["database-implementations"] +repository = "https://github.com/oxigraph/oxigraph/tree/main/lib/oxigraph" +homepage = "https://oxigraph.org/" +documentation = "https://docs.rs/oxigraph" +description = """ +a SPARQL database and RDF toolkit +""" +edition.workspace = true +rust-version.workspace = true + +[features] +default = [] +js = ["getrandom/js", "oxsdatatypes/js", "js-sys"] +http-client = ["oxhttp"] +http-client-native-tls = ["http-client", "oxhttp/native-tls"] +http-client-rustls-webpki = ["http-client", "oxhttp/rustls-webpki"] +http-client-rustls-native = ["http-client", "oxhttp/rustls-native"] +rocksdb-pkg-config = ["oxrocksdb-sys/pkg-config"] +rocksdb-debug = [] + +[dependencies] +digest.workspace = true +hex.workspace = true +json-event-parser.workspace = true +md-5.workspace = true +oxilangtag.workspace = true +oxiri.workspace = true +oxrdf = { workspace = true, features = ["rdf-star", "oxsdatatypes"] } +oxrdfio = { workspace = true, features = ["rdf-star"] } +oxsdatatypes.workspace = true +rand.workspace = true +regex.workspace = true +sha1.workspace = true +sha2.workspace = true +siphasher.workspace = true +sparesults = { workspace = true, features = ["rdf-star"] } +spargebra = { workspace = true, features = ["rdf-star", "sep-0002", "sep-0006"] } +sparopt = { workspace = true, features = ["rdf-star", "sep-0002", "sep-0006"] } + +[target.'cfg(not(target_family = "wasm"))'.dependencies] +libc.workspace = true +oxhttp = { workspace = true, optional = true } +oxrocksdb-sys.workspace = true + +[target.'cfg(all(target_family = "wasm", target_os = "unknown"))'.dependencies] +getrandom.workspace = true +js-sys = { workspace = true, optional = true } + +[target.'cfg(not(target_family = "wasm"))'.dev-dependencies] +codspeed-criterion-compat.workspace = true +oxhttp.workspace = true +zstd.workspace = true + +[lints] +workspace = true + +[package.metadata.docs.rs] +rustdoc-args = ["--cfg", "docsrs"] + +[[bench]] +name = "store" +harness = false diff --git a/lib/oxigraph/README.md b/lib/oxigraph/README.md new file mode 100644 index 00000000..98cd1c8b --- /dev/null +++ b/lib/oxigraph/README.md @@ -0,0 +1,77 @@ +Oxigraph +======== + +[![Latest Version](https://img.shields.io/crates/v/oxigraph.svg)](https://crates.io/crates/oxigraph) +[![Released API docs](https://docs.rs/oxigraph/badge.svg)](https://docs.rs/oxigraph) +[![Crates.io downloads](https://img.shields.io/crates/d/oxigraph)](https://crates.io/crates/oxigraph) +[![actions status](https://github.com/oxigraph/oxigraph/workflows/build/badge.svg)](https://github.com/oxigraph/oxigraph/actions) +[![Gitter](https://badges.gitter.im/oxigraph/community.svg)](https://gitter.im/oxigraph/community) + +Oxigraph is a graph database library implementing the [SPARQL](https://www.w3.org/TR/sparql11-overview/) standard. + +Its goal is to provide a compliant, safe and fast on-disk graph database. +It also provides a set of utility functions for reading, writing, and processing RDF files. + +Oxigraph is in heavy development and SPARQL query evaluation has not been optimized yet. + +Oxigraph also provides [a CLI tool](https://crates.io/crates/oxigraph-cli) and [a Python library](https://pyoxigraph.readthedocs.io/) based on this library. + + +Oxigraph implements the following specifications: +* [SPARQL 1.1 Query](https://www.w3.org/TR/sparql11-query/), [SPARQL 1.1 Update](https://www.w3.org/TR/sparql11-update/), and [SPARQL 1.1 Federated Query](https://www.w3.org/TR/sparql11-federated-query/). +* [Turtle](https://www.w3.org/TR/turtle/), [TriG](https://www.w3.org/TR/trig/), [N-Triples](https://www.w3.org/TR/n-triples/), [N-Quads](https://www.w3.org/TR/n-quads/), and [RDF XML](https://www.w3.org/TR/rdf-syntax-grammar/) RDF serialization formats for both data ingestion and retrieval using the [Rio library](https://github.com/oxigraph/rio). +* [SPARQL Query Results XML Format](https://www.w3.org/TR/rdf-sparql-XMLres/), [SPARQL 1.1 Query Results JSON Format](https://www.w3.org/TR/sparql11-results-json/) and [SPARQL 1.1 Query Results CSV and TSV Formats](https://www.w3.org/TR/sparql11-results-csv-tsv/). + +A preliminary benchmark [is provided](../bench/README.md). Oxigraph internal design [is described on the wiki](https://github.com/oxigraph/oxigraph/wiki/Architecture). + +The main entry point of Oxigraph is the [`Store`](store::Store) struct: +```rust +use oxigraph::store::Store; +use oxigraph::model::*; +use oxigraph::sparql::QueryResults; + +let store = Store::new().unwrap(); + +// insertion +let ex = NamedNode::new("http://example.com").unwrap(); +let quad = Quad::new(ex.clone(), ex.clone(), ex.clone(), GraphName::DefaultGraph); +store.insert(&quad).unwrap(); + +// quad filter +let results = store.quads_for_pattern(Some(ex.as_ref().into()), None, None, None).collect::,_>>().unwrap(); +assert_eq!(vec![quad], results); + +// SPARQL query +if let QueryResults::Solutions(mut solutions) = store.query("SELECT ?s WHERE { ?s ?p ?o }").unwrap() { + assert_eq!(solutions.next().unwrap().unwrap().get("s"), Some(&ex.into())); +} +``` + +It is based on these crates that can be used separately: +* [`oxrdf`](https://crates.io/crates/oxrdf), datastructures encoding RDF basic concepts (the [`oxigraph::model`](crate::model) module). +* [`oxrdfio`](https://crates.io/crates/oxrdfio), a unified parser and serializer API for RDF formats (the [`oxigraph::io`](crate::io) module). It itself relies on: + * [`oxttl`](https://crates.io/crates/oxttl), N-Triple, N-Quad, Turtle, TriG and N3 parsing and serialization. + * [`oxrdfxml`](https://crates.io/crates/oxrdfxml), RDF/XML parsing and serialization. +* [`spargebra`](https://crates.io/crates/spargebra), a SPARQL parser. +* [`sparesults`](https://crates.io/crates/sparesults), parsers and serializers for SPARQL result formats (the [`oxigraph::sparql::results`](crate::sparql::results) module). +* [`sparopt`](https://crates.io/crates/sparesults), a SPARQL optimizer. +* [`oxsdatatypes`](https://crates.io/crates/oxsdatatypes), an implementation of some XML Schema datatypes. + +To build the library locally, don't forget to clone the submodules using `git clone --recursive https://github.com/oxigraph/oxigraph.git` to clone the repository including submodules or `git submodule update --init` to add submodules to the already cloned repository. + + +## License + +This project is licensed under either of + +* Apache License, Version 2.0, ([LICENSE-APACHE](../LICENSE-APACHE) or + ``) +* MIT license ([LICENSE-MIT](../LICENSE-MIT) or + ``) + +at your option. + + +### Contribution + +Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in Oxigraph by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. diff --git a/lib/benches/store.rs b/lib/oxigraph/benches/store.rs similarity index 100% rename from lib/benches/store.rs rename to lib/oxigraph/benches/store.rs diff --git a/lib/src/io/format.rs b/lib/oxigraph/src/io/format.rs similarity index 100% rename from lib/src/io/format.rs rename to lib/oxigraph/src/io/format.rs diff --git a/lib/src/io/mod.rs b/lib/oxigraph/src/io/mod.rs similarity index 100% rename from lib/src/io/mod.rs rename to lib/oxigraph/src/io/mod.rs diff --git a/lib/src/io/read.rs b/lib/oxigraph/src/io/read.rs similarity index 100% rename from lib/src/io/read.rs rename to lib/oxigraph/src/io/read.rs diff --git a/lib/src/io/write.rs b/lib/oxigraph/src/io/write.rs similarity index 100% rename from lib/src/io/write.rs rename to lib/oxigraph/src/io/write.rs diff --git a/lib/src/lib.rs b/lib/oxigraph/src/lib.rs similarity index 100% rename from lib/src/lib.rs rename to lib/oxigraph/src/lib.rs diff --git a/lib/src/model.rs b/lib/oxigraph/src/model.rs similarity index 100% rename from lib/src/model.rs rename to lib/oxigraph/src/model.rs diff --git a/lib/src/sparql/algebra.rs b/lib/oxigraph/src/sparql/algebra.rs similarity index 100% rename from lib/src/sparql/algebra.rs rename to lib/oxigraph/src/sparql/algebra.rs diff --git a/lib/src/sparql/dataset.rs b/lib/oxigraph/src/sparql/dataset.rs similarity index 100% rename from lib/src/sparql/dataset.rs rename to lib/oxigraph/src/sparql/dataset.rs diff --git a/lib/src/sparql/error.rs b/lib/oxigraph/src/sparql/error.rs similarity index 100% rename from lib/src/sparql/error.rs rename to lib/oxigraph/src/sparql/error.rs diff --git a/lib/src/sparql/eval.rs b/lib/oxigraph/src/sparql/eval.rs similarity index 100% rename from lib/src/sparql/eval.rs rename to lib/oxigraph/src/sparql/eval.rs diff --git a/lib/src/sparql/http/dummy.rs b/lib/oxigraph/src/sparql/http/dummy.rs similarity index 100% rename from lib/src/sparql/http/dummy.rs rename to lib/oxigraph/src/sparql/http/dummy.rs diff --git a/lib/src/sparql/http/mod.rs b/lib/oxigraph/src/sparql/http/mod.rs similarity index 100% rename from lib/src/sparql/http/mod.rs rename to lib/oxigraph/src/sparql/http/mod.rs diff --git a/lib/src/sparql/http/simple.rs b/lib/oxigraph/src/sparql/http/simple.rs similarity index 100% rename from lib/src/sparql/http/simple.rs rename to lib/oxigraph/src/sparql/http/simple.rs diff --git a/lib/src/sparql/mod.rs b/lib/oxigraph/src/sparql/mod.rs similarity index 100% rename from lib/src/sparql/mod.rs rename to lib/oxigraph/src/sparql/mod.rs diff --git a/lib/src/sparql/model.rs b/lib/oxigraph/src/sparql/model.rs similarity index 100% rename from lib/src/sparql/model.rs rename to lib/oxigraph/src/sparql/model.rs diff --git a/lib/src/sparql/results.rs b/lib/oxigraph/src/sparql/results.rs similarity index 100% rename from lib/src/sparql/results.rs rename to lib/oxigraph/src/sparql/results.rs diff --git a/lib/src/sparql/service.rs b/lib/oxigraph/src/sparql/service.rs similarity index 100% rename from lib/src/sparql/service.rs rename to lib/oxigraph/src/sparql/service.rs diff --git a/lib/src/sparql/update.rs b/lib/oxigraph/src/sparql/update.rs similarity index 100% rename from lib/src/sparql/update.rs rename to lib/oxigraph/src/sparql/update.rs diff --git a/lib/src/storage/backend/fallback.rs b/lib/oxigraph/src/storage/backend/fallback.rs similarity index 100% rename from lib/src/storage/backend/fallback.rs rename to lib/oxigraph/src/storage/backend/fallback.rs diff --git a/lib/src/storage/backend/mod.rs b/lib/oxigraph/src/storage/backend/mod.rs similarity index 100% rename from lib/src/storage/backend/mod.rs rename to lib/oxigraph/src/storage/backend/mod.rs diff --git a/lib/src/storage/backend/rocksdb.rs b/lib/oxigraph/src/storage/backend/rocksdb.rs similarity index 100% rename from lib/src/storage/backend/rocksdb.rs rename to lib/oxigraph/src/storage/backend/rocksdb.rs diff --git a/lib/src/storage/binary_encoder.rs b/lib/oxigraph/src/storage/binary_encoder.rs similarity index 100% rename from lib/src/storage/binary_encoder.rs rename to lib/oxigraph/src/storage/binary_encoder.rs diff --git a/lib/src/storage/error.rs b/lib/oxigraph/src/storage/error.rs similarity index 100% rename from lib/src/storage/error.rs rename to lib/oxigraph/src/storage/error.rs diff --git a/lib/src/storage/mod.rs b/lib/oxigraph/src/storage/mod.rs similarity index 100% rename from lib/src/storage/mod.rs rename to lib/oxigraph/src/storage/mod.rs diff --git a/lib/src/storage/numeric_encoder.rs b/lib/oxigraph/src/storage/numeric_encoder.rs similarity index 100% rename from lib/src/storage/numeric_encoder.rs rename to lib/oxigraph/src/storage/numeric_encoder.rs diff --git a/lib/src/storage/small_string.rs b/lib/oxigraph/src/storage/small_string.rs similarity index 100% rename from lib/src/storage/small_string.rs rename to lib/oxigraph/src/storage/small_string.rs diff --git a/lib/src/store.rs b/lib/oxigraph/src/store.rs similarity index 100% rename from lib/src/store.rs rename to lib/oxigraph/src/store.rs diff --git a/lib/tests/rocksdb_bc_data/000003.log b/lib/oxigraph/tests/rocksdb_bc_data/000003.log similarity index 100% rename from lib/tests/rocksdb_bc_data/000003.log rename to lib/oxigraph/tests/rocksdb_bc_data/000003.log diff --git a/lib/tests/rocksdb_bc_data/CURRENT b/lib/oxigraph/tests/rocksdb_bc_data/CURRENT similarity index 100% rename from lib/tests/rocksdb_bc_data/CURRENT rename to lib/oxigraph/tests/rocksdb_bc_data/CURRENT diff --git a/lib/tests/rocksdb_bc_data/IDENTITY b/lib/oxigraph/tests/rocksdb_bc_data/IDENTITY similarity index 100% rename from lib/tests/rocksdb_bc_data/IDENTITY rename to lib/oxigraph/tests/rocksdb_bc_data/IDENTITY diff --git a/lib/tests/rocksdb_bc_data/LOCK b/lib/oxigraph/tests/rocksdb_bc_data/LOCK similarity index 100% rename from lib/tests/rocksdb_bc_data/LOCK rename to lib/oxigraph/tests/rocksdb_bc_data/LOCK diff --git a/lib/tests/rocksdb_bc_data/MANIFEST-000004 b/lib/oxigraph/tests/rocksdb_bc_data/MANIFEST-000004 similarity index 100% rename from lib/tests/rocksdb_bc_data/MANIFEST-000004 rename to lib/oxigraph/tests/rocksdb_bc_data/MANIFEST-000004 diff --git a/lib/tests/rocksdb_bc_data/OPTIONS-000026 b/lib/oxigraph/tests/rocksdb_bc_data/OPTIONS-000026 similarity index 100% rename from lib/tests/rocksdb_bc_data/OPTIONS-000026 rename to lib/oxigraph/tests/rocksdb_bc_data/OPTIONS-000026 diff --git a/lib/tests/store.rs b/lib/oxigraph/tests/store.rs similarity index 100% rename from lib/tests/store.rs rename to lib/oxigraph/tests/store.rs diff --git a/lib/oxrdf/Cargo.toml b/lib/oxrdf/Cargo.toml index 93fbf18d..5acda6b6 100644 --- a/lib/oxrdf/Cargo.toml +++ b/lib/oxrdf/Cargo.toml @@ -1,8 +1,8 @@ [package] name = "oxrdf" version = "0.2.0-alpha.2" -authors = ["Tpt "] -license = "MIT OR Apache-2.0" +authors.workspace = true +license.workspace = true readme = "README.md" keywords = ["RDF"] repository = "https://github.com/oxigraph/oxigraph/tree/main/lib/oxrdf" @@ -10,18 +10,18 @@ description = """ A library providing basic data structures related to RDF """ documentation = "https://docs.rs/oxrdf" -edition = "2021" -rust-version = "1.70" +edition.workspace = true +rust-version.workspace = true [features] default = [] rdf-star = [] [dependencies] -rand = "0.8" -oxilangtag = "0.1" -oxiri = "0.2.3-alpha.1" -oxsdatatypes = { version = "0.2.0-alpha.1", path = "../oxsdatatypes", optional = true } +oxilangtag.workspace = true +oxiri.workspace = true +oxsdatatypes = { workspace = true, optional = true } +rand.workspace = true [lints] workspace = true diff --git a/lib/oxrdfio/Cargo.toml b/lib/oxrdfio/Cargo.toml index 1d10c046..84e398f1 100644 --- a/lib/oxrdfio/Cargo.toml +++ b/lib/oxrdfio/Cargo.toml @@ -1,8 +1,8 @@ [package] name = "oxrdfio" version = "0.1.0-alpha.2" -authors = ["Tpt "] -license = "MIT OR Apache-2.0" +authors.workspace = true +license.workspace = true readme = "README.md" keywords = ["RDF"] repository = "https://github.com/oxigraph/oxigraph/tree/master/lib/oxrdfxml" @@ -10,8 +10,8 @@ documentation = "https://docs.rs/oxrdfio" description = """ Parser and serializer for various RDF formats """ -edition = "2021" -rust-version = "1.70" +edition.workspace = true +rust-version.workspace = true [features] default = [] @@ -19,13 +19,13 @@ async-tokio = ["dep:tokio", "oxrdfxml/async-tokio", "oxttl/async-tokio"] rdf-star = ["oxrdf/rdf-star", "oxttl/rdf-star"] [dependencies] -oxrdf = { version = "0.2.0-alpha.2", path = "../oxrdf" } -oxrdfxml = { version = "0.1.0-alpha.2", path = "../oxrdfxml" } -oxttl = { version = "0.1.0-alpha.2", path = "../oxttl" } -tokio = { version = "1.29", optional = true, features = ["io-util"] } +oxrdf.workspace = true +oxrdfxml.workspace = true +oxttl.workspace = true +tokio = { workspace = true, optional = true, features = ["io-util"] } [dev-dependencies] -tokio = { version = "1.29", features = ["rt", "macros"] } +tokio = { workspace = true, features = ["rt", "macros"] } [lints] workspace = true diff --git a/lib/oxrdfxml/Cargo.toml b/lib/oxrdfxml/Cargo.toml index c1a6e18f..2ed9b248 100644 --- a/lib/oxrdfxml/Cargo.toml +++ b/lib/oxrdfxml/Cargo.toml @@ -1,8 +1,8 @@ [package] name = "oxrdfxml" version = "0.1.0-alpha.2" -authors = ["Tpt "] -license = "MIT OR Apache-2.0" +authors.workspace = true +license.workspace = true readme = "README.md" keywords = ["RDFXML", "XML", "RDF"] repository = "https://github.com/oxigraph/oxigraph/tree/master/lib/oxrdfxml" @@ -10,22 +10,22 @@ description = """ Parser and serializer for the RDF/XML format """ documentation = "https://docs.rs/oxrdfxml" -edition = "2021" -rust-version = "1.70" +edition.workspace = true +rust-version.workspace = true [features] default = [] async-tokio = ["dep:tokio", "quick-xml/async-tokio"] [dependencies] -oxrdf = { version = "0.2.0-alpha.2", path = "../oxrdf" } -oxilangtag = "0.1" -oxiri = "0.2.3-alpha.1" -quick-xml = ">=0.29, <0.32" -tokio = { version = "1.29", optional = true, features = ["io-util"] } +oxilangtag.workspace = true +oxiri.workspace = true +oxrdf.workspace = true +quick-xml.workspace = true +tokio = { workspace = true, optional = true, features = ["io-util"] } [dev-dependencies] -tokio = { version = "1.29", features = ["rt", "macros"] } +tokio = { workspace = true, features = ["rt", "macros"] } [lints] workspace = true diff --git a/lib/oxsdatatypes/Cargo.toml b/lib/oxsdatatypes/Cargo.toml index f7bc1f31..fb63c125 100644 --- a/lib/oxsdatatypes/Cargo.toml +++ b/lib/oxsdatatypes/Cargo.toml @@ -1,8 +1,8 @@ [package] name = "oxsdatatypes" version = "0.2.0-alpha.1" -authors = ["Tpt "] -license = "MIT OR Apache-2.0" +authors.workspace = true +license.workspace = true readme = "README.md" keywords = ["XSD"] repository = "https://github.com/oxigraph/oxigraph/tree/main/lib/oxsdatatypes" @@ -10,15 +10,15 @@ description = """ An implementation of some XSD datatypes for SPARQL implementations """ documentation = "https://docs.rs/oxsdatatypes" -edition = "2021" -rust-version = "1.70" +edition.workspace = true +rust-version.workspace = true [features] js = ["js-sys"] custom-now = [] [target.'cfg(all(target_family = "wasm", target_os = "unknown"))'.dependencies] -js-sys = { version = "0.3.60", optional = true } +js-sys = { workspace = true, optional = true } [lints] workspace = true diff --git a/lib/oxttl/Cargo.toml b/lib/oxttl/Cargo.toml index d18cb7e0..6f4f48ff 100644 --- a/lib/oxttl/Cargo.toml +++ b/lib/oxttl/Cargo.toml @@ -1,8 +1,8 @@ [package] name = "oxttl" version = "0.1.0-alpha.2" -authors = ["Tpt "] -license = "MIT OR Apache-2.0" +authors.workspace = true +license.workspace = true readme = "README.md" keywords = ["N-Triples", "N-Quads", "Turtle", "TriG", "N3"] repository = "https://github.com/oxigraph/oxigraph/tree/master/lib/oxttl" @@ -10,8 +10,8 @@ description = """ Parser and serializer for languages related to RDF Turtle (N-Triples, N-Quads, Turtle, TriG and N3) """ documentation = "https://docs.rs/oxttl" -edition = "2021" -rust-version = "1.70" +edition.workspace = true +rust-version.workspace = true [features] default = [] @@ -19,14 +19,14 @@ rdf-star = ["oxrdf/rdf-star"] async-tokio = ["dep:tokio"] [dependencies] -memchr = "2.5" -oxrdf = { version = "0.2.0-alpha.2", path = "../oxrdf" } -oxiri = "0.2.3-alpha.1" -oxilangtag = "0.1" -tokio = { version = "1.29", optional = true, features = ["io-util"] } +memchr.workspace = true +oxrdf.workspace = true +oxiri.workspace = true +oxilangtag.workspace = true +tokio = { workspace = true, optional = true, features = ["io-util"] } [dev-dependencies] -tokio = { version = "1.29", features = ["rt", "macros"] } +tokio = { workspace = true, features = ["rt", "macros"] } [lints] workspace = true diff --git a/lib/sparesults/Cargo.toml b/lib/sparesults/Cargo.toml index 35d03deb..75c5a0bb 100644 --- a/lib/sparesults/Cargo.toml +++ b/lib/sparesults/Cargo.toml @@ -1,8 +1,8 @@ [package] name = "sparesults" version = "0.2.0-alpha.2" -authors = ["Tpt "] -license = "MIT OR Apache-2.0" +authors.workspace = true +license.workspace = true readme = "README.md" keywords = ["SPARQL"] repository = "https://github.com/oxigraph/oxigraph/tree/main/lib/sparesults" @@ -10,8 +10,8 @@ description = """ SPARQL query results formats parsers and serializers """ documentation = "https://docs.rs/sparesults" -edition = "2021" -rust-version = "1.70" +edition.workspace = true +rust-version.workspace = true [features] default = [] @@ -19,14 +19,14 @@ rdf-star = ["oxrdf/rdf-star"] async-tokio = ["dep:tokio", "quick-xml/async-tokio", "json-event-parser/async-tokio"] [dependencies] -json-event-parser = "0.2.0-alpha.2" -memchr = "2.5" -oxrdf = { version = "0.2.0-alpha.2", path = "../oxrdf" } -quick-xml = ">=0.29, <0.32" -tokio = { version = "1.29", optional = true, features = ["io-util"] } +json-event-parser.workspace = true +memchr.workspace = true +oxrdf.workspace = true +quick-xml.workspace = true +tokio = { workspace = true, optional = true, features = ["io-util"] } [dev-dependencies] -tokio = { version = "1.29", features = ["rt", "macros"] } +tokio = { workspace = true, features = ["rt", "macros"] } [lints] workspace = true diff --git a/lib/spargebra/Cargo.toml b/lib/spargebra/Cargo.toml index ac6c049b..24acb482 100644 --- a/lib/spargebra/Cargo.toml +++ b/lib/spargebra/Cargo.toml @@ -1,8 +1,8 @@ [package] name = "spargebra" version = "0.3.0-alpha.2" -authors = ["Tpt "] -license = "MIT OR Apache-2.0" +authors.workspace = true +license.workspace = true readme = "README.md" keywords = ["SPARQL"] repository = "https://github.com/oxigraph/oxigraph/tree/main/lib/spargebra" @@ -10,8 +10,8 @@ documentation = "https://docs.rs/spargebra" description = """ A SPARQL parser """ -edition = "2021" -rust-version = "1.70" +edition.workspace = true +rust-version.workspace = true [features] default = [] @@ -20,11 +20,11 @@ sep-0002 = [] sep-0006 = [] [dependencies] -peg = "0.8" -rand = "0.8" -oxiri = "0.2.3-alpha.1" -oxilangtag = "0.1" -oxrdf = { version = "0.2.0-alpha.2", path = "../oxrdf" } +oxilangtag.workspace = true +oxiri.workspace = true +oxrdf.workspace = true +peg.workspace = true +rand.workspace = true [lints] workspace = true diff --git a/lib/sparopt/Cargo.toml b/lib/sparopt/Cargo.toml index c0f397ab..541e2ea3 100644 --- a/lib/sparopt/Cargo.toml +++ b/lib/sparopt/Cargo.toml @@ -1,8 +1,8 @@ [package] name = "sparopt" version = "0.1.0-alpha.2" -authors = ["Tpt "] -license = "MIT OR Apache-2.0" +authors.workspace = true +license.workspace = true readme = "README.md" keywords = ["SPARQL"] repository = "https://github.com/oxigraph/oxigraph/tree/main/lib/sparopt" @@ -10,8 +10,8 @@ documentation = "https://docs.rs/sparopt" description = """ A SPARQL optimizer """ -edition = "2021" -rust-version = "1.70" +edition.workspace = true +rust-version.workspace = true [features] default = [] @@ -20,9 +20,9 @@ sep-0002 = ["spargebra/sep-0002"] sep-0006 = ["spargebra/sep-0006"] [dependencies] -oxrdf = { version = "0.2.0-alpha.2", path = "../oxrdf" } -rand = "0.8" -spargebra = { version = "0.3.0-alpha.2", path = "../spargebra" } +oxrdf.workspace = true +rand.workspace = true +spargebra.workspace = true [lints] workspace = true diff --git a/lib/sparql-smith/Cargo.toml b/lib/sparql-smith/Cargo.toml index b18b94df..5d0a5680 100644 --- a/lib/sparql-smith/Cargo.toml +++ b/lib/sparql-smith/Cargo.toml @@ -1,8 +1,8 @@ [package] name = "sparql-smith" version = "0.1.0-alpha.5" -authors = ["Tpt "] -license = "MIT OR Apache-2.0" +authors.workspace = true +license.workspace = true readme = "README.md" keywords = ["SPARQL"] repository = "https://github.com/oxigraph/oxigraph/tree/main/lib/sparql-smith" @@ -10,8 +10,8 @@ documentation = "https://docs.rs/sparql-smith" description = """ A SPARQL test cases generator """ -edition = "2021" -rust-version = "1.70" +edition.workspace = true +rust-version.workspace = true [features] default = [] @@ -23,4 +23,4 @@ sep-0006 = [] workspace = true [dependencies] -arbitrary = { version = "1.3", features = ["derive"] } +arbitrary = { workspace = true, features = ["derive"] } diff --git a/oxrocksdb-sys/Cargo.toml b/oxrocksdb-sys/Cargo.toml index 3af73776..1c99afd2 100644 --- a/oxrocksdb-sys/Cargo.toml +++ b/oxrocksdb-sys/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "oxrocksdb-sys" -version = "0.4.0-alpha.3" -authors = ["Tpt "] +version.workspace = true +authors.workspace = true license = "GPL-2.0 OR Apache-2.0" repository = "https://github.com/oxigraph/oxigraph/tree/main/oxrocksdb-sys" readme = "README.md" @@ -9,8 +9,8 @@ description = """ Rust bindings for RocksDB for Oxigraph usage. """ documentation = "https://docs.rs/oxrocksdb-sys" -edition = "2021" -rust-version = "1.70" +edition.workspace = true +rust-version.workspace = true build = "build.rs" links = "rocksdb" @@ -18,9 +18,9 @@ links = "rocksdb" pkg-config = ["dep:pkg-config"] [dependencies] -libc = "0.2.147" +libc.workspace = true [build-dependencies] -pkg-config = { version = "0.3.25", optional = true } -bindgen = ">=0.60, <0.70" -cc = { version = "1.0.73", features = ["parallel"] } +bindgen.workspace = true +cc = { workspace = true, features = ["parallel"] } +pkg-config = { workspace = true, optional = true } diff --git a/python/Cargo.lock b/python/Cargo.lock deleted file mode 120000 index 09fdd3b6..00000000 --- a/python/Cargo.lock +++ /dev/null @@ -1 +0,0 @@ -../Cargo.lock \ No newline at end of file diff --git a/python/Cargo.toml b/python/Cargo.toml index 31d1f812..e1abf1be 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -1,37 +1,38 @@ [package] name = "pyoxigraph" -version = "0.4.0-alpha.3" -authors = ["Tpt "] -license = "MIT OR Apache-2.0" +version.workspace = true +authors.workspace = true +license.workspace = true readme = "README.md" keywords = ["RDF", "SPARQL", "graph-database", "database"] repository = "https://github.com/oxigraph/oxigraph/tree/main/python" homepage = "https://pyoxigraph.readthedocs.io/" description = "Python bindings of Oxigraph, a SPARQL database and RDF toolkit" -edition = "2021" -rust-version = "1.70" +edition.workspace = true +rust-version.workspace = true publish = false [lib] crate-type = ["cdylib"] name = "pyoxigraph" doctest = false +doc = false [features] abi3 = ["pyo3/abi3-py38"] rocksdb-pkg-config = ["oxigraph/rocksdb-pkg-config"] [dependencies] -pyo3 = { version = "0.20.1", features = ["extension-module"] } +pyo3 = { workspace = true, features = ["extension-module"] } [target.'cfg(any(target_family = "windows", target_os = "macos", target_os = "ios"))'.dependencies] -oxigraph = { path = "../lib", features = ["http-client-native-tls"] } +oxigraph = { workspace = true, features = ["http-client-native-tls"] } [target.'cfg(target_family = "wasm")'.dependencies] -oxigraph.path = "../lib" +oxigraph.workspace = true [target.'cfg(not(any(target_family = "windows", target_os = "macos", target_os = "ios", target_family = "wasm")))'.dependencies] -oxigraph = { path = "../lib", features = ["http-client-rustls-native"] } +oxigraph = { workspace = true, features = ["http-client-rustls-native"] } [lints] workspace = true diff --git a/testsuite/Cargo.toml b/testsuite/Cargo.toml index 1ddccd11..6218880d 100644 --- a/testsuite/Cargo.toml +++ b/testsuite/Cargo.toml @@ -1,27 +1,27 @@ [package] name = "oxigraph-testsuite" version = "0.0.0" -authors = ["Tpt "] -license = "MIT OR Apache-2.0" +authors.workspace = true +license.workspace = true description = """ Implementation of W3C testsuites for Oxigraph """ -edition = "2021" -rust-version = "1.70" +edition.workspace = true +rust-version.workspace = true publish = false [dependencies] -anyhow = "1.0.72" -clap = { version = "4.0", features = ["derive"] } -oxigraph.path = "../lib" -oxttl.path = "../lib/oxttl" -sparopt.path = "../lib/sparopt" -spargebra.path = "../lib/spargebra" -text-diff = "0.4" -time = { version = "0.3", features = ["formatting"] } +anyhow.workspace = true +clap = { workspace = true, features = ["derive"] } +oxigraph.workspace = true +oxttl.workspace = true +spargebra.workspace = true +sparopt.workspace = true +text-diff.workspace = true +time = { workspace = true, features = ["formatting"] } [dev-dependencies] -codspeed-criterion-compat = "2.3.3" +codspeed-criterion-compat.workspace = true [lints] workspace = true From a924df0e0a901d51f9fd95189da18f3999c4eb98 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Thu, 8 Feb 2024 04:38:41 -0500 Subject: [PATCH 204/217] Clean workspace dependency list and updates dependencies --- Cargo.lock | 156 ++++++++++++++++++++++++------------------- Cargo.toml | 24 ++++--- lib/oxrdf/Cargo.toml | 2 +- testsuite/Cargo.toml | 2 +- 4 files changed, 103 insertions(+), 81 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e33d7c52..5245a514 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -48,9 +48,9 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.4" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87" +checksum = "8901269c6307e8d93993578286ac0edf7f195079ffff5ebdeea6a59ffb7e36bc" [[package]] name = "anstyle-parse" @@ -154,17 +154,17 @@ checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" [[package]] name = "bindgen" -version = "0.69.2" +version = "0.69.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4c69fae65a523209d34240b60abe0c42d33d1045d445c0839d8a4894a736e2d" +checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0" dependencies = [ "bitflags 2.4.2", "cexpr", "clang-sys", + "itertools 0.12.1", "lazy_static", "lazycell", "log", - "peeking_take_while", "prettyplease", "proc-macro2", "quote", @@ -252,9 +252,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "ciborium" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "effd91f6c78e5a4ace8a5d3c0b6bfaec9e2baaef55f3efc00e45fb2e477ee926" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" dependencies = [ "ciborium-io", "ciborium-ll", @@ -263,15 +263,15 @@ dependencies = [ [[package]] name = "ciborium-io" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdf919175532b369853f5d5e20b26b43112613fd6fe7aee757e35f7a44642656" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" [[package]] name = "ciborium-ll" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "defaa24ecc093c77630e6c15e17c51f5e187bf35ee514f4e2d67baaa96dae22b" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" dependencies = [ "ciborium-io", "half", @@ -422,7 +422,7 @@ dependencies = [ "clap", "criterion-plot", "is-terminal", - "itertools", + "itertools 0.10.5", "num-traits", "once_cell", "oorandom", @@ -443,7 +443,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" dependencies = [ "cast", - "itertools", + "itertools 0.10.5", ] [[package]] @@ -471,6 +471,12 @@ version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" +[[package]] +name = "crunchy" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" + [[package]] name = "crypto-common" version = "0.1.6" @@ -670,9 +676,13 @@ dependencies = [ [[package]] name = "half" -version = "1.8.2" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" +checksum = "bc52e53916c08643f1b56ec082790d1e86a32e58dc5268f897f313fbae7b4872" +dependencies = [ + "cfg-if", + "crunchy", +] [[package]] name = "heck" @@ -682,9 +692,9 @@ checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] name = "hermit-abi" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d3d0e0f38255e7fa3cf31335b3a56f05febd18025f4db5ef7a0cfb4f8da651f" +checksum = "d0c62115964e08cb8039170eb33c1d0e2388a256930279edca206fff675f82c3" [[package]] name = "hex" @@ -759,6 +769,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.10" @@ -776,9 +795,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.67" +version = "0.3.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a1d36f1235bc969acba30b7f5990b864423a6068a10f7c90ae8f0112e3a59d1" +checksum = "406cda4b368d531c842222cf9d2600a9a4acce8d29423695379c6868a143a9ee" dependencies = [ "wasm-bindgen", ] @@ -816,9 +835,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.152" +version = "0.2.153" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13e3bf6590cbc649f4d1a3eefc9d5d6eb746f5200ffb04e5e142700b8faa56e7" +checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" [[package]] name = "libloading" @@ -885,9 +904,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" dependencies = [ "adler", ] @@ -926,11 +945,17 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + [[package]] name = "num-traits" -version = "0.2.17" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" +checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a" dependencies = [ "autocfg", ] @@ -1184,12 +1209,6 @@ dependencies = [ "windows-targets 0.48.5", ] -[[package]] -name = "peeking_take_while" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" - [[package]] name = "peg" version = "0.8.2" @@ -1485,9 +1504,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b7fa1134405e2ec9353fd416b17f8dacd46c473d7d3fd1cf202706a14eb792a" +checksum = "5bb987efffd3c6d0d8f5f89510bb458559eab11e4f869acb20bf845e016259cd" dependencies = [ "aho-corasick", "memchr", @@ -1528,9 +1547,9 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "rustix" -version = "0.38.30" +version = "0.38.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "322394588aaf33c24007e8bb3238ee3e4c5c09c084ab32bc73890b99ff326bca" +checksum = "6ea3e1a662af26cd7a3ba09c0297a31af215563ecf42817c98df621387f4e949" dependencies = [ "bitflags 2.4.2", "errno", @@ -1578,15 +1597,15 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e9d979b3ce68192e42760c7810125eb6cf2ea10efae545a156063e61f314e2a" +checksum = "0a716eb65e3158e90e17cd93d855216e27bde02745ab842f2cab4a39dba1bacf" [[package]] name = "rustls-webpki" -version = "0.102.1" +version = "0.102.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef4ca26037c909dedb327b48c3327d0ba91d3dd3c4e05dad328f210ffb68e95b" +checksum = "faaa0a62740bedb9b2ef5afa303da42764c012f743917351dc9a237ea1663610" dependencies = [ "ring", "rustls-pki-types", @@ -1648,18 +1667,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.195" +version = "1.0.196" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63261df402c67811e9ac6def069e4786148c4563f4b50fd4bf30aa370d626b02" +checksum = "870026e60fa08c69f064aa766c10f10b1d62db9ccd4d0abb206472bee0ce3b32" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.195" +version = "1.0.196" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46fe8f8603d81ba86327b23a2e9cdf49e1255fb94a4c5f297f6ee0547178ea2c" +checksum = "33c85360c95e7d137454dc81d9a4ed2b8efd8fbe19cee57357b32b9771fccb67" dependencies = [ "proc-macro2", "quote", @@ -1668,9 +1687,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.111" +version = "1.0.113" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "176e46fa42316f18edd598015a5166857fc835ec732f5215eac6b7bdbf0a84f4" +checksum = "69801b70b1c3dac963ecb03a364ba0ceda9cf60c71cfe475e99864759c8b8a79" dependencies = [ "itoa", "ryu", @@ -1792,13 +1811,12 @@ checksum = "69758bda2e78f098e4ccb393021a0963bb3442eac05f135c30f61b7370bbafae" [[package]] name = "tempfile" -version = "3.9.0" +version = "3.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01ce4141aa927a6d1bd34a041795abd0db1cccba5d5f24b009f694bdf3a1f3fa" +checksum = "a365e8cd18e44762ef95d87f284f4b5cd04107fec2ff3052bd6a3e6069669e67" dependencies = [ "cfg-if", "fastrand", - "redox_syscall", "rustix", "windows-sys 0.52.0", ] @@ -1831,12 +1849,13 @@ dependencies = [ [[package]] name = "time" -version = "0.3.31" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f657ba42c3f86e7680e53c8cd3af8abbe56b5491790b46e22e19c0d57463583e" +checksum = "c8248b6521bb14bc45b4067159b9b6ad792e2d6d754d6c41fb50e29fefe38749" dependencies = [ "deranged", "itoa", + "num-conv", "powerfmt", "serde", "time-core", @@ -1851,10 +1870,11 @@ checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26197e33420244aeb70c3e8c78376ca46571bc4e701e4791c2cd9f57dcb3a43f" +checksum = "7ba3a3ef41e6672a2f0f001392bb5dcd3ff0a9992d618ca761a11c3121547774" dependencies = [ + "num-conv", "time-core", ] @@ -1885,9 +1905,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.35.1" +version = "1.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c89b4efa943be685f629b149f53829423f8f5531ea21249408e8e2f8671ec104" +checksum = "61285f6515fa018fb2d1e46eb21223fff441ee8db5d0f1435e8ab4f5cdb80931" dependencies = [ "backtrace", "bytes", @@ -2007,9 +2027,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.90" +version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1223296a201415c7fad14792dbefaace9bd52b62d33453ade1c5b5f07555406" +checksum = "c1e124130aee3fb58c5bdd6b639a0509486b0338acaaae0c84a5124b0f588b7f" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -2017,9 +2037,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.90" +version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcdc935b63408d58a32f8cc9738a0bffd8f05cc7c002086c6ef20b7312ad9dcd" +checksum = "c9e7e1900c352b609c8488ad12639a311045f40a35491fb69ba8c12f758af70b" dependencies = [ "bumpalo", "log", @@ -2032,9 +2052,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.90" +version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e4c238561b2d428924c49815533a8b9121c664599558a5d9ec51f8a1740a999" +checksum = "b30af9e2d358182b5c7449424f017eba305ed32a7010509ede96cdc4696c46ed" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2042,9 +2062,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.90" +version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bae1abb6806dc1ad9e560ed242107c0f6c84335f1749dd4e8ddb012ebd5e25a7" +checksum = "642f325be6301eb8107a83d12a8ac6c1e1c54345a7ef1a9261962dfefda09e66" dependencies = [ "proc-macro2", "quote", @@ -2055,15 +2075,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.90" +version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d91413b1c31d7539ba5ef2451af3f0b833a005eb27a631cec32bc0635a8602b" +checksum = "4f186bd2dcf04330886ce82d6f33dd75a7bfcf69ecf5763b89fcde53b6ac9838" [[package]] name = "web-sys" -version = "0.3.67" +version = "0.3.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58cd2333b6e0be7a39605f0e255892fd7418a682d8da8fe042fe25128794d2ed" +checksum = "96565907687f7aceb35bc5fc03770a8a0471d82e479f25832f54a0e3f4b28446" dependencies = [ "js-sys", "wasm-bindgen", @@ -2071,9 +2091,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.26.0" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0de2cfda980f21be5a7ed2eadb3e6fe074d56022bea2cdeb1a62eb220fc04188" +checksum = "b3de34ae270483955a94f4b21bdaaeb83d508bb84a01435f393818edb0012009" dependencies = [ "rustls-pki-types", ] diff --git a/Cargo.toml b/Cargo.toml index 91b1b7ec..7cbc711b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,16 +46,8 @@ libc = "0.2.147" md-5 = "0.10" memchr = "2.5" oxhttp = "0.2.0-alpha.3" -oxigraph = { version = "0.4.0-alpha.3-dev", path = "./lib/oxigraph" } oxilangtag = "0.1" oxiri = "0.2.3-alpha.1" -oxrdf = { version = "0.2.0-alpha.2", path = "lib/oxrdf" } -oxrdfdatatypes = { version = "0.2.0-alpha.1", path = "lib/oxsdatatypes" } -oxrdfio = { version = "0.1.0-alpha.2", path = "lib/oxrdfio" } -oxrdfxml = { version = "0.1.0-alpha.2", path = "lib/oxrdfxml" } -oxrocksdb-sys = { version = "0.4.0-alpha.3-dev", path = "./oxrocksdb-sys" } -oxsdatatypes = { version = "0.2.0-alpha.1", path = "lib/oxsdatatypes" } -oxttl = { version = "0.1.0-alpha.2", path = "lib/oxttl" } peg = "0.8" pkg-config = "0.3.25" predicates = ">=2.0, <4.0" @@ -67,9 +59,6 @@ regex = "1.7" sha1 = "0.10" sha2 = "0.10" siphasher = ">=0.3, <2.0" -sparesults = { version = "0.2.0-alpha.2", path = "lib/sparesults" } -spargebra = { version = "0.3.0-alpha.2", path = "lib/spargebra" } -sparopt = { version = "0.1.0-alpha.2", path = "lib/sparopt"} text-diff = "0.4" time = "0.3" tokio = "1.29" @@ -77,6 +66,19 @@ url = "2.4" wasm-bindgen = "0.2.83" zstd = ">=0.12, <0.14" +# Internal dependencies +oxigraph = { version = "0.4.0-alpha.3-dev", path = "lib/oxigraph" } +oxrdf = { version = "0.2.0-alpha.2", path = "lib/oxrdf" } +oxrdfio = { version = "0.1.0-alpha.2", path = "lib/oxrdfio" } +oxrdfxml = { version = "0.1.0-alpha.2", path = "lib/oxrdfxml" } +oxrocksdb-sys = { version = "0.4.0-alpha.3-dev", path = "./oxrocksdb-sys" } +oxsdatatypes = { version = "0.2.0-alpha.1", path = "lib/oxsdatatypes" } +oxttl = { version = "0.1.0-alpha.2", path = "lib/oxttl" } +sparesults = { version = "0.2.0-alpha.2", path = "lib/sparesults" } +spargebra = { version = "0.3.0-alpha.2", path = "lib/spargebra" } +sparopt = { version = "0.1.0-alpha.2", path = "lib/sparopt" } +sparql-smith = { version = "0.1.0-alpha.5", path = "lib/sparql-smith" } + [workspace.lints.rust] absolute_paths_not_starting_with_crate = "warn" elided_lifetimes_in_paths = "warn" diff --git a/lib/oxrdf/Cargo.toml b/lib/oxrdf/Cargo.toml index 5acda6b6..90b2bb12 100644 --- a/lib/oxrdf/Cargo.toml +++ b/lib/oxrdf/Cargo.toml @@ -28,4 +28,4 @@ workspace = true [package.metadata.docs.rs] all-features = true -rustdoc-args = ["--cfg", "docsrs"] \ No newline at end of file +rustdoc-args = ["--cfg", "docsrs"] diff --git a/testsuite/Cargo.toml b/testsuite/Cargo.toml index 6218880d..1302d710 100644 --- a/testsuite/Cargo.toml +++ b/testsuite/Cargo.toml @@ -28,4 +28,4 @@ workspace = true [[bench]] name = "parser" -harness = false \ No newline at end of file +harness = false From be26d210f1282fe8e05e54551a611300f7255ff3 Mon Sep 17 00:00:00 2001 From: Tpt Date: Thu, 8 Feb 2024 18:45:23 +0100 Subject: [PATCH 205/217] Removes unused StrLookup::contains_str --- lib/oxigraph/src/sparql/dataset.rs | 4 ---- lib/oxigraph/src/storage/binary_encoder.rs | 4 ---- lib/oxigraph/src/storage/mod.rs | 4 ---- lib/oxigraph/src/storage/numeric_encoder.rs | 2 -- 4 files changed, 14 deletions(-) diff --git a/lib/oxigraph/src/sparql/dataset.rs b/lib/oxigraph/src/sparql/dataset.rs index bf7e6195..3253be18 100644 --- a/lib/oxigraph/src/sparql/dataset.rs +++ b/lib/oxigraph/src/sparql/dataset.rs @@ -176,10 +176,6 @@ impl StrLookup for DatasetView { self.reader.get_str(key)? }) } - - fn contains_str(&self, key: &StrHash) -> Result { - Ok(self.extra.borrow().contains_key(key) || self.reader.contains_str(key)?) - } } struct EncodedDatasetSpec { diff --git a/lib/oxigraph/src/storage/binary_encoder.rs b/lib/oxigraph/src/storage/binary_encoder.rs index c06a8d7e..25626571 100644 --- a/lib/oxigraph/src/storage/binary_encoder.rs +++ b/lib/oxigraph/src/storage/binary_encoder.rs @@ -650,10 +650,6 @@ mod tests { fn get_str(&self, key: &StrHash) -> Result, StorageError> { Ok(self.id2str.borrow().get(key).cloned()) } - - fn contains_str(&self, key: &StrHash) -> Result { - Ok(self.id2str.borrow().contains_key(key)) - } } impl MemoryStrStore { diff --git a/lib/oxigraph/src/storage/mod.rs b/lib/oxigraph/src/storage/mod.rs index ce02449e..1f77d1b0 100644 --- a/lib/oxigraph/src/storage/mod.rs +++ b/lib/oxigraph/src/storage/mod.rs @@ -864,10 +864,6 @@ impl StrLookup for StorageReader { fn get_str(&self, key: &StrHash) -> Result, StorageError> { self.get_str(key) } - - fn contains_str(&self, key: &StrHash) -> Result { - self.contains_str(key) - } } pub struct StorageWriter<'a> { diff --git a/lib/oxigraph/src/storage/numeric_encoder.rs b/lib/oxigraph/src/storage/numeric_encoder.rs index a81b76e5..e730a163 100644 --- a/lib/oxigraph/src/storage/numeric_encoder.rs +++ b/lib/oxigraph/src/storage/numeric_encoder.rs @@ -701,8 +701,6 @@ impl From> for EncodedQuad { pub trait StrLookup { fn get_str(&self, key: &StrHash) -> Result, StorageError>; - - fn contains_str(&self, key: &StrHash) -> Result; } pub fn insert_term Result<(), StorageError>>( From 9e3758e2c9ef00c7cc97de05d6fa22a2fc6060c3 Mon Sep 17 00:00:00 2001 From: Tpt Date: Thu, 8 Feb 2024 18:33:52 +0100 Subject: [PATCH 206/217] Makes QueryResults::write return the Write impl --- lib/oxigraph/src/sparql/model.rs | 49 ++++++++++++++------------------ 1 file changed, 21 insertions(+), 28 deletions(-) diff --git a/lib/oxigraph/src/sparql/model.rs b/lib/oxigraph/src/sparql/model.rs index 59c38afd..051f5136 100644 --- a/lib/oxigraph/src/sparql/model.rs +++ b/lib/oxigraph/src/sparql/model.rs @@ -41,23 +41,21 @@ impl QueryResults { /// let ex = NamedNodeRef::new("http://example.com")?; /// store.insert(QuadRef::new(ex, ex, ex, GraphNameRef::DefaultGraph))?; /// - /// let mut results = Vec::new(); - /// store.query("SELECT ?s WHERE { ?s ?p ?o }")?.write(&mut results, QueryResultsFormat::Json)?; - /// assert_eq!(results, r#"{"head":{"vars":["s"]},"results":{"bindings":[{"s":{"type":"uri","value":"http://example.com"}}]}}"#.as_bytes()); + /// let results = store.query("SELECT ?s WHERE { ?s ?p ?o }")?; + /// assert_eq!( + /// results.write(Vec::new(), QueryResultsFormat::Json)?, + /// r#"{"head":{"vars":["s"]},"results":{"bindings":[{"s":{"type":"uri","value":"http://example.com"}}]}}"#.as_bytes() + /// ); /// # Result::<_,Box>::Ok(()) /// ``` - pub fn write( + pub fn write( self, - write: impl Write, + write: W, format: QueryResultsFormat, - ) -> Result<(), EvaluationError> { + ) -> Result { let serializer = QueryResultsSerializer::from_format(format); match self { - Self::Boolean(value) => { - serializer - .serialize_boolean_to_write(write, value) - .map_err(EvaluationError::ResultsSerialization)?; - } + Self::Boolean(value) => serializer.serialize_boolean_to_write(write, value), Self::Solutions(solutions) => { let mut writer = serializer .serialize_solutions_to_write(write, solutions.variables().to_vec()) @@ -67,9 +65,7 @@ impl QueryResults { .write(&solution?) .map_err(EvaluationError::ResultsSerialization)?; } - writer - .finish() - .map_err(EvaluationError::ResultsSerialization)?; + writer.finish() } Self::Graph(triples) => { let s = VariableRef::new_unchecked("subject"); @@ -91,12 +87,10 @@ impl QueryResults { ]) .map_err(EvaluationError::ResultsSerialization)?; } - writer - .finish() - .map_err(EvaluationError::ResultsSerialization)?; + writer.finish() } } - Ok(()) + .map_err(EvaluationError::ResultsSerialization) } /// Writes the graph query results. @@ -118,18 +112,18 @@ impl QueryResults { /// None, /// )?; /// - /// let mut results = Vec::new(); - /// store - /// .query("CONSTRUCT WHERE { ?s ?p ?o }")? - /// .write_graph(&mut results, RdfFormat::NTriples)?; - /// assert_eq!(results, graph.as_bytes()); + /// let results = store.query("CONSTRUCT WHERE { ?s ?p ?o }")?; + /// assert_eq!( + /// results.write_graph(Vec::new(), RdfFormat::NTriples)?, + /// graph.as_bytes() + /// ); /// # Result::<_,Box>::Ok(()) /// ``` - pub fn write_graph( + pub fn write_graph( self, - write: impl Write, + write: W, format: impl Into, - ) -> Result<(), EvaluationError> { + ) -> Result { if let Self::Graph(triples) = self { let mut writer = RdfSerializer::from_format(format.into()).serialize_to_write(write); for triple in triples { @@ -139,8 +133,7 @@ impl QueryResults { } writer .finish() - .map_err(EvaluationError::ResultsSerialization)?; - Ok(()) + .map_err(EvaluationError::ResultsSerialization) } else { Err(EvaluationError::NotAGraph) } From 089875ad21a359f3a9edc7ac7d153d5f758214b7 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Thu, 8 Feb 2024 03:34:12 -0500 Subject: [PATCH 207/217] A few more minor lints, keyword fix keywords must not have any special chars --- cli/src/main.rs | 2 +- fuzz/fuzz_targets/sparql_eval.rs | 2 +- fuzz/src/result_format.rs | 4 ++-- js/Cargo.toml | 2 +- lib/oxigraph/src/sparql/error.rs | 7 +++---- lib/oxrdfxml/src/parser.rs | 32 +++++++++++--------------------- lib/sparesults/src/serializer.rs | 6 ++++-- lib/spargebra/src/algebra.rs | 2 +- lib/sparql-smith/src/lib.rs | 8 ++++---- testsuite/src/manifest.rs | 2 +- 10 files changed, 29 insertions(+), 38 deletions(-) diff --git a/cli/src/main.rs b/cli/src/main.rs index cff5cf79..99a9015d 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -923,7 +923,7 @@ fn serve(store: Store, bind: &str, read_only: bool, cors: bool) -> anyhow::Resul let server = server.spawn()?; #[cfg(target_os = "linux")] systemd_notify_ready()?; - eprintln!("Listening for requests at http://{}", &bind); + eprintln!("Listening for requests at http://{bind}"); server.join()?; Ok(()) } diff --git a/fuzz/fuzz_targets/sparql_eval.rs b/fuzz/fuzz_targets/sparql_eval.rs index 24c8a176..142f0be9 100644 --- a/fuzz/fuzz_targets/sparql_eval.rs +++ b/fuzz/fuzz_targets/sparql_eval.rs @@ -47,7 +47,7 @@ fn query_solutions_key(iter: QuerySolutionIter, is_reduced: bool) -> String { let mut b = t .unwrap() .iter() - .map(|(var, val)| format!("{}: {}", var, val)) + .map(|(var, val)| format!("{var}: {val}")) .collect::>(); b.sort_unstable(); b.join(" ") diff --git a/fuzz/src/result_format.rs b/fuzz/src/result_format.rs index 5c6b59ae..d6e15c54 100644 --- a/fuzz/src/result_format.rs +++ b/fuzz/src/result_format.rs @@ -33,13 +33,13 @@ pub fn fuzz_result_format(format: QueryResultsFormat, data: &[u8]) { // And to parse again if let FromReadQueryResultsReader::Solutions(roundtrip_solutions) = parser .parse_read(serialized.as_bytes()) - .with_context(|| format!("Parsing {:?}", &serialized)) + .with_context(|| format!("Parsing {serialized:?}")) .unwrap() { assert_eq!( roundtrip_solutions .collect::, _>>() - .with_context(|| format!("Parsing {:?}", &serialized)) + .with_context(|| format!("Parsing {serialized:?}")) .unwrap(), solutions ) diff --git a/js/Cargo.toml b/js/Cargo.toml index 0b5156b3..8e62e147 100644 --- a/js/Cargo.toml +++ b/js/Cargo.toml @@ -4,7 +4,7 @@ version.workspace = true authors.workspace = true license.workspace = true readme = "README.md" -keywords = ["RDF", "N-Triples", "Turtle", "RDF/XML", "SPARQL"] +keywords = ["RDF", "N-Triples", "Turtle", "XML", "SPARQL"] repository = "https://github.com/oxigraph/oxigraph/tree/main/js" description = "JavaScript bindings of Oxigraph" edition.workspace = true diff --git a/lib/oxigraph/src/sparql/error.rs b/lib/oxigraph/src/sparql/error.rs index a8627790..b3516d8e 100644 --- a/lib/oxigraph/src/sparql/error.rs +++ b/lib/oxigraph/src/sparql/error.rs @@ -60,10 +60,9 @@ impl fmt::Display for EvaluationError { Self::UnsupportedContentType(content_type) => { write!(f, "The content media type {content_type} is not supported") } - Self::ServiceDoesNotReturnSolutions => write!( - f, - "The service is not returning solutions but a boolean or a graph" - ), + Self::ServiceDoesNotReturnSolutions => { + f.write_str("The service is not returning solutions but a boolean or a graph") + } Self::NotAGraph => f.write_str("The query results are not a RDF graph"), } } diff --git a/lib/oxrdfxml/src/parser.rs b/lib/oxrdfxml/src/parser.rs index 70ca91aa..a4e35784 100644 --- a/lib/oxrdfxml/src/parser.rs +++ b/lib/oxrdfxml/src/parser.rs @@ -600,11 +600,9 @@ impl RdfXmlReader { if *attribute_url == *RDF_ID { let mut id = self.convert_attribute(&attribute)?; if !is_nc_name(&id) { - return Err(SyntaxError::msg(format!( - "{} is not a valid rdf:ID value", - &id - )) - .into()); + return Err( + SyntaxError::msg(format!("{id} is not a valid rdf:ID value")).into(), + ); } id.insert(0, '#'); id_attr = Some(id); @@ -612,8 +610,7 @@ impl RdfXmlReader { let bag_id = self.convert_attribute(&attribute)?; if !is_nc_name(&bag_id) { return Err(SyntaxError::msg(format!( - "{} is not a valid rdf:bagID value", - &bag_id + "{bag_id} is not a valid rdf:bagID value" )) .into()); } @@ -621,8 +618,7 @@ impl RdfXmlReader { let id = self.convert_attribute(&attribute)?; if !is_nc_name(&id) { return Err(SyntaxError::msg(format!( - "{} is not a valid rdf:nodeID value", - &id + "{id} is not a valid rdf:nodeID value" )) .into()); } @@ -644,8 +640,7 @@ impl RdfXmlReader { type_attr = Some(attribute); } else if RESERVED_RDF_ATTRIBUTES.contains(&&*attribute_url) { return Err(SyntaxError::msg(format!( - "{} is not a valid attribute", - &attribute_url + "{attribute_url} is not a valid attribute" )) .into()); } else { @@ -663,8 +658,7 @@ impl RdfXmlReader { let iri = self.resolve_iri(&base_iri, iri)?; if self.known_rdf_id.contains(iri.as_str()) { return Err(SyntaxError::msg(format!( - "{} has already been used as rdf:ID value", - &iri + "{iri} has already been used as rdf:ID value" )) .into()); } @@ -718,8 +712,7 @@ impl RdfXmlReader { RdfXmlState::Rdf { base_iri, language } } else if RESERVED_RDF_ELEMENTS.contains(&&*tag_name) { return Err(SyntaxError::msg(format!( - "Invalid node element tag name: {}", - &tag_name + "Invalid node element tag name: {tag_name}" )) .into()); } else { @@ -739,8 +732,7 @@ impl RdfXmlReader { RdfXmlNextProduction::NodeElt => { if RESERVED_RDF_ELEMENTS.contains(&&*tag_name) { return Err(SyntaxError::msg(format!( - "Invalid property element tag name: {}", - &tag_name + "Invalid property element tag name: {tag_name}" )) .into()); } @@ -761,8 +753,7 @@ impl RdfXmlReader { let Some(RdfXmlState::NodeElt { li_counter, .. }) = self.state.last_mut() else { return Err(SyntaxError::msg(format!( - "Invalid property element tag name: {}", - &tag_name + "Invalid property element tag name: {tag_name}" )) .into()); }; @@ -774,8 +765,7 @@ impl RdfXmlReader { || *tag_name == *RDF_DESCRIPTION { return Err(SyntaxError::msg(format!( - "Invalid property element tag name: {}", - &tag_name + "Invalid property element tag name: {tag_name}" )) .into()); } else { diff --git a/lib/sparesults/src/serializer.rs b/lib/sparesults/src/serializer.rs index 1d4a02b1..ecc730cd 100644 --- a/lib/sparesults/src/serializer.rs +++ b/lib/sparesults/src/serializer.rs @@ -88,8 +88,10 @@ impl QueryResultsSerializer { /// # async fn main() -> std::io::Result<()> { /// let json_serializer = QueryResultsSerializer::from_format(QueryResultsFormat::Json); /// let mut buffer = Vec::new(); - /// json_serializer.serialize_boolean_to_tokio_async_write(&mut buffer, false).await?; - /// assert_eq!(buffer, br#"{"head":{},"boolean":false}"r); + /// json_serializer + /// .serialize_boolean_to_tokio_async_write(&mut buffer, false) + /// .await?; + /// assert_eq!(buffer, br#"{"head":{},"boolean":false}"#); /// # Ok(()) /// # } /// ``` diff --git a/lib/spargebra/src/algebra.rs b/lib/spargebra/src/algebra.rs index 9b38b71f..a261b2dc 100644 --- a/lib/spargebra/src/algebra.rs +++ b/lib/spargebra/src/algebra.rs @@ -640,7 +640,7 @@ impl fmt::Display for GraphPattern { Self::Filter { expr, inner } => { write!(f, "{inner} FILTER({expr})") } - Self::Union { left, right } => write!(f, "{{ {left} }} UNION {{ {right} }}",), + Self::Union { left, right } => write!(f, "{{ {left} }} UNION {{ {right} }}"), Self::Graph { name, inner } => { write!(f, "GRAPH {name} {{ {inner} }}") } diff --git a/lib/sparql-smith/src/lib.rs b/lib/sparql-smith/src/lib.rs index 7266f67e..6b114d68 100644 --- a/lib/sparql-smith/src/lib.rs +++ b/lib/sparql-smith/src/lib.rs @@ -246,7 +246,7 @@ impl fmt::Display for GroupCondition { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::BuiltInCall(c) => write!(f, "{c}"), - // Self::FunctionCall(c) => write!(f, "{}", c), + // Self::FunctionCall(c) => write!(f, "{c}"), Self::Projection(e, v) => { if let Some(v) = v { write!(f, "({e} AS {v})") @@ -549,7 +549,7 @@ struct InlineData { impl fmt::Display for InlineData { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "VALUES {}", &self.inner) + write!(f, "VALUES {}", self.inner) } } @@ -705,7 +705,7 @@ impl fmt::Display for Constraint { match self { Self::BrackettedExpression(e) => write!(f, "{e}"), Self::BuiltInCall(c) => write!(f, "{c}"), - // Self::FunctionCall(c) => write!(f, "{}", c), + // Self::FunctionCall(c) => write!(f, "{c}"), } } } @@ -1592,7 +1592,7 @@ impl fmt::Display for IriOrFunction { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{}", self.iri)?; // if let Some(args) = &self.args { - // write!(f, "{}", args)?; + // write!(f, "{args}")?; // } Ok(()) } diff --git a/testsuite/src/manifest.rs b/testsuite/src/manifest.rs index 97ee0ea7..b911a630 100644 --- a/testsuite/src/manifest.rs +++ b/testsuite/src/manifest.rs @@ -34,7 +34,7 @@ impl fmt::Display for Test { write!(f, " on file \"{action}\"")?; } if let Some(query) = &self.query { - write!(f, " on query {}", &query)?; + write!(f, " on query {query}")?; } for data in &self.data { write!(f, " with data {data}")?; From f5de5d3e980e8b200d579f81fa89995ca04357fb Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Thu, 8 Feb 2024 14:55:39 -0500 Subject: [PATCH 208/217] use github action to install cargo tools --- .github/workflows/artifacts.yml | 3 ++- .github/workflows/tests.yml | 21 ++++++++++++--------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/.github/workflows/artifacts.yml b/.github/workflows/artifacts.yml index 05579293..662ae259 100644 --- a/.github/workflows/artifacts.yml +++ b/.github/workflows/artifacts.yml @@ -274,7 +274,8 @@ jobs: with: submodules: true - uses: ./.github/actions/setup-rust - - run: cargo install wasm-pack || true + - uses: taiki-e/install-action@v2 + with: { tool: wasm-pack } - uses: actions/setup-node@v3 with: node-version: 16 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 42dbae54..b936522e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -125,8 +125,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-rust - - run: cargo install cargo-deny || true + - uses: taiki-e/install-action@v2 + with: { tool: cargo-deny } - run: cargo deny check semver_checks: @@ -136,7 +136,8 @@ jobs: with: submodules: true - uses: ./.github/actions/setup-rust - - run: cargo install cargo-semver-checks || true + - uses: taiki-e/install-action@v2 + with: { tool: cargo-semver-checks } - uses: actions/cache@v3 with: path: rocksdb @@ -236,8 +237,8 @@ jobs: - uses: ./.github/actions/setup-rust with: target: wasm32-wasi - - uses: taiki-e/install-action@wasmtime - - run: cargo install cargo-wasi || true + - uses: taiki-e/install-action@v2 + with: { tool: "wasmtime,cargo-wasi" } - run: cargo wasi test --workspace --exclude oxigraph-js --exclude oxigraph-cli --exclude oxigraph-testsuite --exclude oxrocksdb-sys --exclude pyoxigraph rustdoc: @@ -258,7 +259,8 @@ jobs: steps: - uses: actions/checkout@v3 - uses: ./.github/actions/setup-rust - - run: cargo install wasm-pack || true + - uses: taiki-e/install-action@v2 + with: { tool: wasm-pack } - uses: actions/setup-node@v3 with: node-version: 16 @@ -397,8 +399,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-rust - - run: cargo install typos-cli || true + - uses: taiki-e/install-action@v2 + with: { tool: typos-cli } - run: typos clang_fmt: @@ -499,7 +501,8 @@ jobs: with: submodules: true - uses: ./.github/actions/setup-rust - - run: cargo install cargo-codspeed || true + - uses: taiki-e/install-action@v2 + with: { tool: cargo-codspeed } - run: cargo codspeed build -p oxigraph --features http-client-native-tls - run: cargo codspeed build -p oxigraph-testsuite - uses: CodSpeedHQ/action@v2 From 1c3f05483643583fe244d16ee77c2d046b60827c Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Thu, 8 Feb 2024 16:41:41 -0500 Subject: [PATCH 209/217] Convert error to thiserror This converts just one `SerializerError` to use `thiserror` crate, removing some code. --- Cargo.lock | 21 +++++++++++++ Cargo.toml | 1 + lib/oxigraph/Cargo.toml | 1 + lib/oxigraph/src/storage/error.rs | 49 +++++-------------------------- 4 files changed, 30 insertions(+), 42 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5245a514..d729862c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1068,6 +1068,7 @@ dependencies = [ "sparesults", "spargebra", "sparopt", + "thiserror", "zstd", ] @@ -1847,6 +1848,26 @@ dependencies = [ "term", ] +[[package]] +name = "thiserror" +version = "1.0.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d54378c645627613241d077a3a79db965db602882668f9136ac42af9ecb730ad" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa0faa943b50f3db30a20aa7e265dbc66076993efed8463e8de414e5d06d3471" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "time" version = "0.3.34" diff --git a/Cargo.toml b/Cargo.toml index 7cbc711b..1c0cc9e6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -60,6 +60,7 @@ sha1 = "0.10" sha2 = "0.10" siphasher = ">=0.3, <2.0" text-diff = "0.4" +thiserror = "1" time = "0.3" tokio = "1.29" url = "2.4" diff --git a/lib/oxigraph/Cargo.toml b/lib/oxigraph/Cargo.toml index 13f14476..cefb5755 100644 --- a/lib/oxigraph/Cargo.toml +++ b/lib/oxigraph/Cargo.toml @@ -43,6 +43,7 @@ siphasher.workspace = true sparesults = { workspace = true, features = ["rdf-star"] } spargebra = { workspace = true, features = ["rdf-star", "sep-0002", "sep-0006"] } sparopt = { workspace = true, features = ["rdf-star", "sep-0002", "sep-0006"] } +thiserror.workspace = true [target.'cfg(not(target_family = "wasm"))'.dependencies] libc.workspace = true diff --git a/lib/oxigraph/src/storage/error.rs b/lib/oxigraph/src/storage/error.rs index 05076e6e..d0a67522 100644 --- a/lib/oxigraph/src/storage/error.rs +++ b/lib/oxigraph/src/storage/error.rs @@ -2,6 +2,7 @@ use crate::io::{ParseError, RdfFormat}; use oxiri::IriParseError; use std::error::Error; use std::{fmt, io}; +use thiserror::Error; /// An error related to storage operations (reads, writes...). #[derive(Debug)] @@ -185,55 +186,19 @@ impl From for io::Error { } /// An error raised while writing a file from a [`Store`](crate::store::Store). -#[derive(Debug)] +#[derive(Debug, Error)] pub enum SerializerError { /// An error raised while writing the content. - Io(io::Error), + #[error(transparent)] + Io(#[from] io::Error), /// An error raised during the lookup in the store. - Storage(StorageError), + #[error(transparent)] + Storage(#[from] StorageError), /// A format compatible with [RDF dataset](https://www.w3.org/TR/rdf11-concepts/#dfn-rdf-dataset) is required. + #[error("A RDF format supporting datasets was expected, {0} found")] DatasetFormatExpected(RdfFormat), } -impl fmt::Display for SerializerError { - #[inline] - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Io(e) => e.fmt(f), - Self::Storage(e) => e.fmt(f), - Self::DatasetFormatExpected(format) => write!( - f, - "A RDF format supporting datasets was expected, {format} found" - ), - } - } -} - -impl Error for SerializerError { - #[inline] - fn source(&self) -> Option<&(dyn Error + 'static)> { - match self { - Self::Io(e) => Some(e), - Self::Storage(e) => Some(e), - Self::DatasetFormatExpected(_) => None, - } - } -} - -impl From for SerializerError { - #[inline] - fn from(error: io::Error) -> Self { - Self::Io(error) - } -} - -impl From for SerializerError { - #[inline] - fn from(error: StorageError) -> Self { - Self::Storage(error) - } -} - impl From for io::Error { #[inline] fn from(error: SerializerError) -> Self { From 655ecd3e91f21ef8ab9ce06e91015c7af85f797e Mon Sep 17 00:00:00 2001 From: Tpt Date: Sat, 10 Feb 2024 17:20:23 +0100 Subject: [PATCH 210/217] Convert error to thiserror Co-authored-by: Yuri Astrakhan --- Cargo.lock | 31 ++-- Cargo.toml | 2 +- lib/oxigraph/src/sparql/error.rs | 110 +++----------- lib/oxigraph/src/storage/backend/fallback.rs | 2 +- lib/oxigraph/src/storage/backend/rocksdb.rs | 12 +- lib/oxigraph/src/storage/error.rs | 146 +++++-------------- lib/oxigraph/src/storage/numeric_encoder.rs | 30 +--- lib/oxigraph/src/storage/small_string.rs | 30 +--- lib/oxrdf/Cargo.toml | 1 + lib/oxrdf/src/blank_node.rs | 13 +- lib/oxrdf/src/parser.rs | 78 ++++------ lib/oxrdf/src/variable.rs | 13 +- lib/oxrdfio/Cargo.toml | 1 + lib/oxrdfio/src/error.rs | 106 +++----------- lib/oxrdfxml/Cargo.toml | 1 + lib/oxrdfxml/src/error.rs | 106 +++----------- lib/oxrdfxml/src/parser.rs | 20 ++- lib/oxsdatatypes/Cargo.toml | 3 + lib/oxsdatatypes/src/date_time.rs | 82 +++-------- lib/oxsdatatypes/src/decimal.rs | 60 +++----- lib/oxsdatatypes/src/duration.rs | 35 +---- lib/oxsdatatypes/src/integer.rs | 12 +- lib/oxttl/Cargo.toml | 1 + lib/oxttl/src/toolkit/error.rs | 47 +----- lib/sparesults/Cargo.toml | 1 + lib/sparesults/src/error.rs | 88 +++-------- lib/spargebra/Cargo.toml | 1 + lib/spargebra/src/parser.rs | 56 ++----- 28 files changed, 273 insertions(+), 815 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d729862c..85d35123 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -290,9 +290,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.18" +version = "4.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e578d6ec4194633722ccf9544794b71b1385c3c027efe0c55db226fc880865c" +checksum = "80c21025abd42669a92efc996ef13cfb2c5c627858421ea58d5c3b331a6c134f" dependencies = [ "clap_builder", "clap_derive", @@ -300,9 +300,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.18" +version = "4.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4df4df40ec50c46000231c914968278b1eb05098cf8f1b3a518a95030e71d1c7" +checksum = "458bf1f341769dfcf849846f65dffdf9146daa56bcd2a47cb4e1de9915567c99" dependencies = [ "anstream", "anstyle", @@ -312,9 +312,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.4.7" +version = "4.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442" +checksum = "307bc0538d5f0f83b8248db3087aa92fe504e4691294d0c96c0eabc33f47ba47" dependencies = [ "heck", "proc-macro2", @@ -324,9 +324,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" +checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce" [[package]] name = "codspeed" @@ -786,9 +786,9 @@ checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" [[package]] name = "jobserver" -version = "0.1.27" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c37f63953c4c63420ed5fd3d6d398c719489b9f872b9fa683262f8edd363c7d" +checksum = "ab46a6e9526ddef3ae7f787c06f0f2600639ba80ea3eade3d8e670a2230f51d6" dependencies = [ "libc", ] @@ -1136,6 +1136,7 @@ dependencies = [ "oxiri", "oxsdatatypes", "rand", + "thiserror", ] [[package]] @@ -1145,6 +1146,7 @@ dependencies = [ "oxrdf", "oxrdfxml", "oxttl", + "thiserror", "tokio", ] @@ -1156,6 +1158,7 @@ dependencies = [ "oxiri", "oxrdf", "quick-xml", + "thiserror", "tokio", ] @@ -1174,6 +1177,7 @@ name = "oxsdatatypes" version = "0.2.0-alpha.1" dependencies = [ "js-sys", + "thiserror", ] [[package]] @@ -1184,6 +1188,7 @@ dependencies = [ "oxilangtag", "oxiri", "oxrdf", + "thiserror", "tokio", ] @@ -1745,6 +1750,7 @@ dependencies = [ "memchr", "oxrdf", "quick-xml", + "thiserror", "tokio", ] @@ -1757,6 +1763,7 @@ dependencies = [ "oxrdf", "peg", "rand", + "thiserror", ] [[package]] @@ -1783,9 +1790,9 @@ checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" [[package]] name = "strsim" -version = "0.10.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +checksum = "5ee073c9e4cd00e28217186dbe12796d692868f432bf2e97ee73bed0c56dfa01" [[package]] name = "subtle" diff --git a/Cargo.toml b/Cargo.toml index 1c0cc9e6..203ae7a1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -60,7 +60,7 @@ sha1 = "0.10" sha2 = "0.10" siphasher = ">=0.3, <2.0" text-diff = "0.4" -thiserror = "1" +thiserror = "1.0.50" time = "0.3" tokio = "1.29" url = "2.4" diff --git a/lib/oxigraph/src/sparql/error.rs b/lib/oxigraph/src/sparql/error.rs index b3516d8e..94ee6f57 100644 --- a/lib/oxigraph/src/sparql/error.rs +++ b/lib/oxigraph/src/sparql/error.rs @@ -5,93 +5,53 @@ use crate::sparql::ParseError; use crate::storage::StorageError; use std::convert::Infallible; use std::error::Error; -use std::{fmt, io}; +use std::io; /// A SPARQL evaluation error. -#[derive(Debug)] +#[derive(Debug, thiserror::Error)] #[non_exhaustive] pub enum EvaluationError { /// An error in SPARQL parsing. - Parsing(ParseError), + #[error(transparent)] + Parsing(#[from] ParseError), /// An error from the storage. - Storage(StorageError), + #[error(transparent)] + Storage(#[from] StorageError), /// An error while parsing an external RDF file. - GraphParsing(RdfParseError), + #[error(transparent)] + GraphParsing(#[from] RdfParseError), /// An error while parsing an external result file (likely from a federated query). - ResultsParsing(ResultsParseError), + #[error(transparent)] + ResultsParsing(#[from] ResultsParseError), /// An error returned during results serialization. - ResultsSerialization(io::Error), + #[error(transparent)] + ResultsSerialization(#[from] io::Error), /// Error during `SERVICE` evaluation - Service(Box), + #[error("{0}")] + Service(#[source] Box), /// Error when `CREATE` tries to create an already existing graph + #[error("The graph {0} already exists")] GraphAlreadyExists(NamedNode), /// Error when `DROP` or `CLEAR` tries to remove a not existing graph + #[error("The graph {0} does not exist")] GraphDoesNotExist(NamedNode), /// The variable storing the `SERVICE` name is unbound + #[error("The variable encoding the service name is unbound")] UnboundService, /// The given `SERVICE` is not supported + #[error("The service {0} is not supported")] UnsupportedService(NamedNode), /// The given content media type returned from an HTTP response is not supported (`SERVICE` and `LOAD`) + #[error("The content media type {0} is not supported")] UnsupportedContentType(String), /// The `SERVICE` call has not returns solutions + #[error("The service is not returning solutions but a boolean or a graph")] ServiceDoesNotReturnSolutions, /// The results are not a RDF graph + #[error("The query results are not a RDF graph")] NotAGraph, } -impl fmt::Display for EvaluationError { - #[inline] - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Parsing(error) => error.fmt(f), - Self::Storage(error) => error.fmt(f), - Self::GraphParsing(error) => error.fmt(f), - Self::ResultsParsing(error) => error.fmt(f), - Self::ResultsSerialization(error) => error.fmt(f), - Self::Service(error) => error.fmt(f), - Self::GraphAlreadyExists(graph) => write!(f, "The graph {graph} already exists"), - Self::GraphDoesNotExist(graph) => write!(f, "The graph {graph} does not exist"), - Self::UnboundService => { - f.write_str("The variable encoding the service name is unbound") - } - Self::UnsupportedService(service) => { - write!(f, "The service {service} is not supported") - } - Self::UnsupportedContentType(content_type) => { - write!(f, "The content media type {content_type} is not supported") - } - Self::ServiceDoesNotReturnSolutions => { - f.write_str("The service is not returning solutions but a boolean or a graph") - } - Self::NotAGraph => f.write_str("The query results are not a RDF graph"), - } - } -} - -impl Error for EvaluationError { - #[inline] - fn source(&self) -> Option<&(dyn Error + 'static)> { - match self { - Self::Parsing(e) => Some(e), - Self::Storage(e) => Some(e), - Self::GraphParsing(e) => Some(e), - Self::ResultsParsing(e) => Some(e), - Self::ResultsSerialization(e) => Some(e), - Self::Service(e) => { - let e = Box::as_ref(e); - Some(e) - } - Self::GraphAlreadyExists(_) - | Self::GraphDoesNotExist(_) - | Self::UnboundService - | Self::UnsupportedService(_) - | Self::UnsupportedContentType(_) - | Self::ServiceDoesNotReturnSolutions - | Self::NotAGraph => None, - } - } -} - impl From for EvaluationError { #[inline] fn from(error: Infallible) -> Self { @@ -99,34 +59,6 @@ impl From for EvaluationError { } } -impl From for EvaluationError { - #[inline] - fn from(error: ParseError) -> Self { - Self::Parsing(error) - } -} - -impl From for EvaluationError { - #[inline] - fn from(error: StorageError) -> Self { - Self::Storage(error) - } -} - -impl From for EvaluationError { - #[inline] - fn from(error: RdfParseError) -> Self { - Self::GraphParsing(error) - } -} - -impl From for EvaluationError { - #[inline] - fn from(error: ResultsParseError) -> Self { - Self::ResultsParsing(error) - } -} - impl From for io::Error { #[inline] fn from(error: EvaluationError) -> Self { diff --git a/lib/oxigraph/src/storage/backend/fallback.rs b/lib/oxigraph/src/storage/backend/fallback.rs index c2eb17ec..7214851e 100644 --- a/lib/oxigraph/src/storage/backend/fallback.rs +++ b/lib/oxigraph/src/storage/backend/fallback.rs @@ -36,7 +36,7 @@ impl Db { if self.0.read().unwrap().contains_key(&column_family) { Ok(column_family) } else { - Err(CorruptionError::msg(format!("Column family {name} does not exist")).into()) + Err(CorruptionError::from_missing_column_family_name(name).into()) } } diff --git a/lib/oxigraph/src/storage/backend/rocksdb.rs b/lib/oxigraph/src/storage/backend/rocksdb.rs index fed85421..14c45d0a 100644 --- a/lib/oxigraph/src/storage/backend/rocksdb.rs +++ b/lib/oxigraph/src/storage/backend/rocksdb.rs @@ -548,7 +548,7 @@ impl Db { return Ok(ColumnFamily(*cf_handle)); } } - Err(CorruptionError::msg(format!("Column family {name} does not exist")).into()) + Err(CorruptionError::from_missing_column_family_name(name).into()) } #[must_use] @@ -1314,6 +1314,8 @@ impl SstFileWriter { } } +#[derive(thiserror::Error)] +#[error("{}", self.message())] struct ErrorStatus(rocksdb_status_t); unsafe impl Send for ErrorStatus {} @@ -1352,14 +1354,6 @@ impl fmt::Debug for ErrorStatus { } } -impl fmt::Display for ErrorStatus { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(self.message()) - } -} - -impl Error for ErrorStatus {} - impl From for StorageError { fn from(status: ErrorStatus) -> Self { if status.0.code == rocksdb_status_code_t_rocksdb_status_code_io_error { diff --git a/lib/oxigraph/src/storage/error.rs b/lib/oxigraph/src/storage/error.rs index d0a67522..f2f27cdb 100644 --- a/lib/oxigraph/src/storage/error.rs +++ b/lib/oxigraph/src/storage/error.rs @@ -1,48 +1,23 @@ use crate::io::{ParseError, RdfFormat}; +use crate::storage::numeric_encoder::EncodedTerm; use oxiri::IriParseError; +use oxrdf::TermRef; use std::error::Error; -use std::{fmt, io}; -use thiserror::Error; +use std::io; /// An error related to storage operations (reads, writes...). -#[derive(Debug)] +#[derive(Debug, thiserror::Error)] #[non_exhaustive] pub enum StorageError { /// Error from the OS I/O layer. - Io(io::Error), + #[error(transparent)] + Io(#[from] io::Error), /// Error related to data corruption. - Corruption(CorruptionError), + #[error(transparent)] + Corruption(#[from] CorruptionError), #[doc(hidden)] - Other(Box), -} - -impl fmt::Display for StorageError { - #[inline] - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Io(e) => e.fmt(f), - Self::Corruption(e) => e.fmt(f), - Self::Other(e) => e.fmt(f), - } - } -} - -impl Error for StorageError { - #[inline] - fn source(&self) -> Option<&(dyn Error + 'static)> { - match self { - Self::Io(e) => Some(e), - Self::Corruption(e) => Some(e), - Self::Other(e) => Some(e.as_ref()), - } - } -} - -impl From for StorageError { - #[inline] - fn from(error: io::Error) -> Self { - Self::Io(error) - } + #[error("{0}")] + Other(#[source] Box), } impl From for io::Error { @@ -57,59 +32,42 @@ impl From for io::Error { } /// An error return if some content in the database is corrupted. -#[derive(Debug)] -pub struct CorruptionError { - inner: CorruptionErrorKind, -} +#[derive(Debug, thiserror::Error)] +#[error(transparent)] +pub struct CorruptionError(#[from] CorruptionErrorKind); -#[derive(Debug)] +/// An error return if some content in the database is corrupted. +#[derive(Debug, thiserror::Error)] enum CorruptionErrorKind { + #[error("{0}")] Msg(String), - Other(Box), + #[error("{0}")] + Other(#[source] Box), } impl CorruptionError { /// Builds an error from a printable error message. #[inline] pub(crate) fn new(error: impl Into>) -> Self { - Self { - inner: CorruptionErrorKind::Other(error.into()), - } - } - - /// Builds an error from a printable error message. - #[inline] - pub(crate) fn msg(msg: impl Into) -> Self { - Self { - inner: CorruptionErrorKind::Msg(msg.into()), - } + Self(CorruptionErrorKind::Other(error.into())) } -} -impl fmt::Display for CorruptionError { #[inline] - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match &self.inner { - CorruptionErrorKind::Msg(e) => e.fmt(f), - CorruptionErrorKind::Other(e) => e.fmt(f), - } + pub(crate) fn from_encoded_term(encoded: &EncodedTerm, term: &TermRef<'_>) -> Self { + // TODO: eventually use a dedicated error enum value + Self::msg(format!("Invalid term encoding {encoded:?} for {term}")) } -} -impl Error for CorruptionError { #[inline] - fn source(&self) -> Option<&(dyn Error + 'static)> { - match &self.inner { - CorruptionErrorKind::Msg(_) => None, - CorruptionErrorKind::Other(e) => Some(e.as_ref()), - } + pub(crate) fn from_missing_column_family_name(name: &'static str) -> Self { + // TODO: eventually use a dedicated error enum value + Self::msg(format!("Column family {name} does not exist")) } -} -impl From for StorageError { + /// Builds an error from a printable error message. #[inline] - fn from(error: CorruptionError) -> Self { - Self::Corruption(error) + pub(crate) fn msg(msg: impl Into) -> Self { + Self(CorruptionErrorKind::Msg(msg.into())) } } @@ -121,57 +79,25 @@ impl From for io::Error { } /// An error raised while loading a file into a [`Store`](crate::store::Store). -#[derive(Debug)] +#[derive(Debug, thiserror::Error)] pub enum LoaderError { /// An error raised while reading the file. - Parsing(ParseError), + #[error(transparent)] + Parsing(#[from] ParseError), /// An error raised during the insertion in the store. - Storage(StorageError), + #[error(transparent)] + Storage(#[from] StorageError), /// The base IRI is invalid. + #[error("Invalid base IRI '{iri}': {error}")] InvalidBaseIri { /// The IRI itself. iri: String, /// The parsing error. + #[source] error: IriParseError, }, } -impl fmt::Display for LoaderError { - #[inline] - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Parsing(e) => e.fmt(f), - Self::Storage(e) => e.fmt(f), - Self::InvalidBaseIri { iri, error } => write!(f, "Invalid base IRI '{iri}': {error}"), - } - } -} - -impl Error for LoaderError { - #[inline] - fn source(&self) -> Option<&(dyn Error + 'static)> { - match self { - Self::Parsing(e) => Some(e), - Self::Storage(e) => Some(e), - Self::InvalidBaseIri { error, .. } => Some(error), - } - } -} - -impl From for LoaderError { - #[inline] - fn from(error: ParseError) -> Self { - Self::Parsing(error) - } -} - -impl From for LoaderError { - #[inline] - fn from(error: StorageError) -> Self { - Self::Storage(error) - } -} - impl From for io::Error { #[inline] fn from(error: LoaderError) -> Self { @@ -186,7 +112,7 @@ impl From for io::Error { } /// An error raised while writing a file from a [`Store`](crate::store::Store). -#[derive(Debug, Error)] +#[derive(Debug, thiserror::Error)] pub enum SerializerError { /// An error raised while writing the content. #[error(transparent)] diff --git a/lib/oxigraph/src/storage/numeric_encoder.rs b/lib/oxigraph/src/storage/numeric_encoder.rs index e730a163..bf4b070b 100644 --- a/lib/oxigraph/src/storage/numeric_encoder.rs +++ b/lib/oxigraph/src/storage/numeric_encoder.rs @@ -713,19 +713,13 @@ pub fn insert_term Result<(), StorageError>>( if let EncodedTerm::NamedNode { iri_id } = encoded { insert_str(iri_id, node.as_str()) } else { - Err( - CorruptionError::new(format!("Invalid term encoding {encoded:?} for {term}")) - .into(), - ) + Err(CorruptionError::from_encoded_term(encoded, &term).into()) } } TermRef::BlankNode(node) => match encoded { EncodedTerm::BigBlankNode { id_id } => insert_str(id_id, node.as_str()), EncodedTerm::SmallBlankNode(..) | EncodedTerm::NumericalBlankNode { .. } => Ok(()), - _ => Err( - CorruptionError::new(format!("Invalid term encoding {encoded:?} for {term}")) - .into(), - ), + _ => Err(CorruptionError::from_encoded_term(encoded, &term).into()), }, TermRef::Literal(literal) => match encoded { EncodedTerm::BigStringLiteral { value_id } @@ -736,10 +730,7 @@ pub fn insert_term Result<(), StorageError>>( if let Some(language) = literal.language() { insert_str(language_id, language) } else { - Err(CorruptionError::new(format!( - "Invalid term encoding {encoded:?} for {term}" - )) - .into()) + Err(CorruptionError::from_encoded_term(encoded, &term).into()) } } EncodedTerm::BigBigLangStringLiteral { @@ -750,10 +741,7 @@ pub fn insert_term Result<(), StorageError>>( if let Some(language) = literal.language() { insert_str(language_id, language) } else { - Err(CorruptionError::new(format!( - "Invalid term encoding {encoded:?} for {term}" - )) - .into()) + Err(CorruptionError::from_encoded_term(encoded, &term).into()) } } EncodedTerm::SmallTypedLiteral { datatype_id, .. } => { @@ -784,10 +772,7 @@ pub fn insert_term Result<(), StorageError>>( | EncodedTerm::DurationLiteral(..) | EncodedTerm::YearMonthDurationLiteral(..) | EncodedTerm::DayTimeDurationLiteral(..) => Ok(()), - _ => Err( - CorruptionError::new(format!("Invalid term encoding {encoded:?} for {term}")) - .into(), - ), + _ => Err(CorruptionError::from_encoded_term(encoded, &term).into()), }, TermRef::Triple(triple) => { if let EncodedTerm::Triple(encoded) = encoded { @@ -799,10 +784,7 @@ pub fn insert_term Result<(), StorageError>>( )?; insert_term(triple.object.as_ref(), &encoded.object, insert_str) } else { - Err( - CorruptionError::new(format!("Invalid term encoding {encoded:?} for {term}")) - .into(), - ) + Err(CorruptionError::from_encoded_term(encoded, &term).into()) } } } diff --git a/lib/oxigraph/src/storage/small_string.rs b/lib/oxigraph/src/storage/small_string.rs index fcd9b227..355606da 100644 --- a/lib/oxigraph/src/storage/small_string.rs +++ b/lib/oxigraph/src/storage/small_string.rs @@ -1,6 +1,5 @@ use std::borrow::Borrow; use std::cmp::Ordering; -use std::error::Error; use std::hash::{Hash, Hasher}; use std::ops::Deref; use std::str::{FromStr, Utf8Error}; @@ -169,31 +168,10 @@ impl<'a> TryFrom<&'a str> for SmallString { } } -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, thiserror::Error)] pub enum BadSmallStringError { + #[error("small strings could only contain at most 15 characters, found {0}")] TooLong(usize), - BadUtf8(Utf8Error), -} - -impl fmt::Display for BadSmallStringError { - #[inline] - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::TooLong(v) => write!( - f, - "small strings could only contain at most 15 characters, found {v}" - ), - Self::BadUtf8(e) => e.fmt(f), - } - } -} - -impl Error for BadSmallStringError { - #[inline] - fn source(&self) -> Option<&(dyn Error + 'static)> { - match self { - Self::TooLong(_) => None, - Self::BadUtf8(e) => Some(e), - } - } + #[error(transparent)] + BadUtf8(#[from] Utf8Error), } diff --git a/lib/oxrdf/Cargo.toml b/lib/oxrdf/Cargo.toml index 90b2bb12..6fec2123 100644 --- a/lib/oxrdf/Cargo.toml +++ b/lib/oxrdf/Cargo.toml @@ -22,6 +22,7 @@ oxilangtag.workspace = true oxiri.workspace = true oxsdatatypes = { workspace = true, optional = true } rand.workspace = true +thiserror.workspace = true [lints] workspace = true diff --git a/lib/oxrdf/src/blank_node.rs b/lib/oxrdf/src/blank_node.rs index ce07226b..6d565fda 100644 --- a/lib/oxrdf/src/blank_node.rs +++ b/lib/oxrdf/src/blank_node.rs @@ -1,5 +1,4 @@ use rand::random; -use std::error::Error; use std::io::Write; use std::{fmt, str}; @@ -345,18 +344,10 @@ fn to_integer_id(id: &str) -> Option { } /// An error raised during [`BlankNode`] IDs validation. -#[derive(Debug)] +#[derive(Debug, thiserror::Error)] +#[error("The blank node identifier is invalid")] pub struct BlankNodeIdParseError; -impl fmt::Display for BlankNodeIdParseError { - #[inline] - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str("The blank node identifier is invalid") - } -} - -impl Error for BlankNodeIdParseError {} - #[cfg(test)] mod tests { #![allow(clippy::panic_in_result_fn)] diff --git a/lib/oxrdf/src/parser.rs b/lib/oxrdf/src/parser.rs index 1794540d..a531a86e 100644 --- a/lib/oxrdf/src/parser.rs +++ b/lib/oxrdf/src/parser.rs @@ -5,9 +5,8 @@ use crate::{ }; #[cfg(feature = "rdf-star")] use crate::{Subject, Triple}; -use std::error::Error; +use std::char; use std::str::{Chars, FromStr}; -use std::{char, fmt}; /// This limit is set in order to avoid stack overflow error when parsing nested triples due to too many recursive calls. /// The actual limit value is a wet finger compromise between not failing to parse valid files and avoiding to trigger stack overflow errors. @@ -166,11 +165,11 @@ impl FromStr for Variable { "Variable serialization should start with ? or $", )); } - Self::new(&s[1..]).map_err(|error| Self::Err { - kind: TermParseErrorKind::Variable { + Self::new(&s[1..]).map_err(|error| { + TermParseError(TermParseErrorKind::Variable { value: s.to_owned(), error, - }, + }) }) } } @@ -183,11 +182,11 @@ fn read_named_node(s: &str) -> Result<(NamedNode, &str), TermParseError> { .ok_or_else(|| TermParseError::msg("Named node serialization should end with a >"))?; let (value, remain) = remain.split_at(end); let remain = &remain[1..]; - let term = NamedNode::new(value).map_err(|error| TermParseError { - kind: TermParseErrorKind::Iri { + let term = NamedNode::new(value).map_err(|error| { + TermParseError(TermParseErrorKind::Iri { value: value.to_owned(), error, - }, + }) })?; Ok((term, remain)) } else { @@ -207,11 +206,11 @@ fn read_blank_node(s: &str) -> Result<(BlankNode, &str), TermParseError> { }) .unwrap_or(remain.len()); let (value, remain) = remain.split_at(end); - let term = BlankNode::new(value).map_err(|error| TermParseError { - kind: TermParseErrorKind::BlankNode { + let term = BlankNode::new(value).map_err(|error| { + TermParseError(TermParseErrorKind::BlankNode { value: value.to_owned(), error, - }, + }) })?; Ok((term, remain)) } else { @@ -237,11 +236,11 @@ fn read_literal(s: &str) -> Result<(Literal, &str), TermParseError> { let (language, remain) = remain.split_at(end); Ok(( Literal::new_language_tagged_literal(value, language).map_err( - |error| TermParseError { - kind: TermParseErrorKind::LanguageTag { + |error| { + TermParseError(TermParseErrorKind::LanguageTag { value: language.to_owned(), error, - }, + }) }, )?, remain, @@ -421,61 +420,36 @@ fn read_hexa_char(input: &mut Chars<'_>, len: usize) -> Result) -> fmt::Result { - match &self.kind { - TermParseErrorKind::Iri { error, value } => { - write!(f, "Error while parsing the named node '{value}': {error}") - } - TermParseErrorKind::BlankNode { error, value } => { - write!(f, "Error while parsing the blank node '{value}': {error}") - } - TermParseErrorKind::LanguageTag { error, value } => { - write!(f, "Error while parsing the language tag '{value}': {error}") - } - TermParseErrorKind::Variable { error, value } => { - write!(f, "Error while parsing the variable '{value}': {error}") - } - TermParseErrorKind::Msg { msg } => f.write_str(msg), - } - } -} - -impl Error for TermParseError {} - impl TermParseError { pub(crate) fn msg(msg: &'static str) -> Self { - Self { - kind: TermParseErrorKind::Msg { msg }, - } + Self(TermParseErrorKind::Msg(msg)) } } diff --git a/lib/oxrdf/src/variable.rs b/lib/oxrdf/src/variable.rs index 30af6c0f..36dcb3e4 100644 --- a/lib/oxrdf/src/variable.rs +++ b/lib/oxrdf/src/variable.rs @@ -1,5 +1,4 @@ use std::cmp::Ordering; -use std::error::Error; use std::fmt; /// A [SPARQL query](https://www.w3.org/TR/sparql11-query/) owned variable. @@ -212,14 +211,6 @@ fn validate_variable_identifier(id: &str) -> Result<(), VariableNameParseError> } /// An error raised during [`Variable`] name validation. -#[derive(Debug)] +#[derive(Debug, thiserror::Error)] +#[error("The variable name is invalid")] pub struct VariableNameParseError; - -impl fmt::Display for VariableNameParseError { - #[inline] - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str("The variable name is invalid") - } -} - -impl Error for VariableNameParseError {} diff --git a/lib/oxrdfio/Cargo.toml b/lib/oxrdfio/Cargo.toml index 84e398f1..05069df3 100644 --- a/lib/oxrdfio/Cargo.toml +++ b/lib/oxrdfio/Cargo.toml @@ -22,6 +22,7 @@ rdf-star = ["oxrdf/rdf-star", "oxttl/rdf-star"] oxrdf.workspace = true oxrdfxml.workspace = true oxttl.workspace = true +thiserror.workspace = true tokio = { workspace = true, optional = true, features = ["io-util"] } [dev-dependencies] diff --git a/lib/oxrdfio/src/error.rs b/lib/oxrdfio/src/error.rs index 78f9b998..cd2c4449 100644 --- a/lib/oxrdfio/src/error.rs +++ b/lib/oxrdfio/src/error.rs @@ -1,50 +1,27 @@ -use std::error::Error; +use std::io; use std::ops::Range; -use std::{fmt, io}; /// Error returned during RDF format parsing. -#[derive(Debug)] +#[derive(Debug, thiserror::Error)] pub enum ParseError { /// I/O error during parsing (file not found...). - Io(io::Error), + #[error(transparent)] + Io(#[from] io::Error), /// An error in the file syntax. - Syntax(SyntaxError), + #[error(transparent)] + Syntax(#[from] SyntaxError), } impl ParseError { pub(crate) fn msg(msg: &'static str) -> Self { - Self::Syntax(SyntaxError { - inner: SyntaxErrorKind::Msg { msg }, - }) - } -} - -impl fmt::Display for ParseError { - #[inline] - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Io(e) => e.fmt(f), - Self::Syntax(e) => e.fmt(f), - } - } -} - -impl Error for ParseError { - #[inline] - fn source(&self) -> Option<&(dyn Error + 'static)> { - match self { - Self::Io(e) => Some(e), - Self::Syntax(e) => Some(e), - } + Self::Syntax(SyntaxError(SyntaxErrorKind::Msg(msg))) } } impl From for SyntaxError { #[inline] fn from(error: oxttl::SyntaxError) -> Self { - Self { - inner: SyntaxErrorKind::Turtle(error), - } + Self(SyntaxErrorKind::Turtle(error)) } } @@ -61,9 +38,7 @@ impl From for ParseError { impl From for SyntaxError { #[inline] fn from(error: oxrdfxml::SyntaxError) -> Self { - Self { - inner: SyntaxErrorKind::RdfXml(error), - } + Self(SyntaxErrorKind::RdfXml(error)) } } @@ -77,20 +52,6 @@ impl From for ParseError { } } -impl From for ParseError { - #[inline] - fn from(error: io::Error) -> Self { - Self::Io(error) - } -} - -impl From for ParseError { - #[inline] - fn from(error: SyntaxError) -> Self { - Self::Syntax(error) - } -} - impl From for io::Error { #[inline] fn from(error: ParseError) -> Self { @@ -102,23 +63,26 @@ impl From for io::Error { } /// An error in the syntax of the parsed file. -#[derive(Debug)] -pub struct SyntaxError { - inner: SyntaxErrorKind, -} +#[derive(Debug, thiserror::Error)] +#[error(transparent)] +pub struct SyntaxError(#[from] SyntaxErrorKind); -#[derive(Debug)] +/// An error in the syntax of the parsed file. +#[derive(Debug, thiserror::Error)] enum SyntaxErrorKind { - Turtle(oxttl::SyntaxError), - RdfXml(oxrdfxml::SyntaxError), - Msg { msg: &'static str }, + #[error(transparent)] + Turtle(#[from] oxttl::SyntaxError), + #[error(transparent)] + RdfXml(#[from] oxrdfxml::SyntaxError), + #[error("{0}")] + Msg(&'static str), } impl SyntaxError { /// The location of the error inside of the file. #[inline] pub fn location(&self) -> Option> { - match &self.inner { + match &self.0 { SyntaxErrorKind::Turtle(e) => { let location = e.location(); Some( @@ -133,29 +97,7 @@ impl SyntaxError { }, ) } - SyntaxErrorKind::RdfXml(_) | SyntaxErrorKind::Msg { .. } => None, - } - } -} - -impl fmt::Display for SyntaxError { - #[inline] - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match &self.inner { - SyntaxErrorKind::Turtle(e) => e.fmt(f), - SyntaxErrorKind::RdfXml(e) => e.fmt(f), - SyntaxErrorKind::Msg { msg } => f.write_str(msg), - } - } -} - -impl Error for SyntaxError { - #[inline] - fn source(&self) -> Option<&(dyn Error + 'static)> { - match &self.inner { - SyntaxErrorKind::Turtle(e) => Some(e), - SyntaxErrorKind::RdfXml(e) => Some(e), - SyntaxErrorKind::Msg { .. } => None, + SyntaxErrorKind::RdfXml(_) | SyntaxErrorKind::Msg(_) => None, } } } @@ -163,10 +105,10 @@ impl Error for SyntaxError { impl From for io::Error { #[inline] fn from(error: SyntaxError) -> Self { - match error.inner { + match error.0 { SyntaxErrorKind::Turtle(error) => error.into(), SyntaxErrorKind::RdfXml(error) => error.into(), - SyntaxErrorKind::Msg { msg } => Self::new(io::ErrorKind::InvalidData, msg), + SyntaxErrorKind::Msg(msg) => Self::new(io::ErrorKind::InvalidData, msg), } } } diff --git a/lib/oxrdfxml/Cargo.toml b/lib/oxrdfxml/Cargo.toml index 2ed9b248..efc71e62 100644 --- a/lib/oxrdfxml/Cargo.toml +++ b/lib/oxrdfxml/Cargo.toml @@ -22,6 +22,7 @@ oxilangtag.workspace = true oxiri.workspace = true oxrdf.workspace = true quick-xml.workspace = true +thiserror.workspace = true tokio = { workspace = true, optional = true, features = ["io-util"] } [dev-dependencies] diff --git a/lib/oxrdfxml/src/error.rs b/lib/oxrdfxml/src/error.rs index be2e161d..abc7c55c 100644 --- a/lib/oxrdfxml/src/error.rs +++ b/lib/oxrdfxml/src/error.rs @@ -1,50 +1,17 @@ use oxilangtag::LanguageTagParseError; use oxiri::IriParseError; -use std::error::Error; +use std::io; use std::sync::Arc; -use std::{fmt, io}; /// Error returned during RDF/XML parsing. -#[derive(Debug)] +#[derive(Debug, thiserror::Error)] pub enum ParseError { /// I/O error during parsing (file not found...). - Io(io::Error), + #[error(transparent)] + Io(#[from] io::Error), /// An error in the file syntax. - Syntax(SyntaxError), -} - -impl fmt::Display for ParseError { - #[inline] - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Io(e) => e.fmt(f), - Self::Syntax(e) => e.fmt(f), - } - } -} - -impl Error for ParseError { - #[inline] - fn source(&self) -> Option<&(dyn Error + 'static)> { - match self { - Self::Io(e) => Some(e), - Self::Syntax(e) => Some(e), - } - } -} - -impl From for ParseError { - #[inline] - fn from(error: io::Error) -> Self { - Self::Io(error) - } -} - -impl From for ParseError { - #[inline] - fn from(error: SyntaxError) -> Self { - Self::Syntax(error) - } + #[error(transparent)] + Syntax(#[from] SyntaxError), } impl From for io::Error { @@ -64,77 +31,48 @@ impl From for ParseError { quick_xml::Error::Io(error) => { Self::Io(Arc::try_unwrap(error).unwrap_or_else(|e| io::Error::new(e.kind(), e))) } - _ => Self::Syntax(SyntaxError { - inner: SyntaxErrorKind::Xml(error), - }), + _ => Self::Syntax(SyntaxError(SyntaxErrorKind::Xml(error))), } } } /// An error in the syntax of the parsed file. -#[derive(Debug)] -pub struct SyntaxError { - pub(crate) inner: SyntaxErrorKind, -} +#[derive(Debug, thiserror::Error)] +#[error(transparent)] +pub struct SyntaxError(#[from] pub(crate) SyntaxErrorKind); -#[derive(Debug)] +#[derive(Debug, thiserror::Error)] pub enum SyntaxErrorKind { - Xml(quick_xml::Error), + #[error(transparent)] + Xml(#[from] quick_xml::Error), + #[error("error while parsing IRI '{iri}': {error}")] InvalidIri { iri: String, + #[source] error: IriParseError, }, + #[error("error while parsing language tag '{tag}': {error}")] InvalidLanguageTag { tag: String, + #[source] error: LanguageTagParseError, }, - Msg { - msg: String, - }, + #[error("{0}")] + Msg(String), } impl SyntaxError { /// Builds an error from a printable error message. #[inline] pub(crate) fn msg(msg: impl Into) -> Self { - Self { - inner: SyntaxErrorKind::Msg { msg: msg.into() }, - } - } -} - -impl fmt::Display for SyntaxError { - #[inline] - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match &self.inner { - SyntaxErrorKind::Xml(error) => error.fmt(f), - SyntaxErrorKind::InvalidIri { iri, error } => { - write!(f, "error while parsing IRI '{iri}': {error}") - } - SyntaxErrorKind::InvalidLanguageTag { tag, error } => { - write!(f, "error while parsing language tag '{tag}': {error}") - } - SyntaxErrorKind::Msg { msg } => f.write_str(msg), - } - } -} - -impl Error for SyntaxError { - #[inline] - fn source(&self) -> Option<&(dyn Error + 'static)> { - match &self.inner { - SyntaxErrorKind::Xml(error) => Some(error), - SyntaxErrorKind::InvalidIri { error, .. } => Some(error), - SyntaxErrorKind::InvalidLanguageTag { error, .. } => Some(error), - SyntaxErrorKind::Msg { .. } => None, - } + Self(SyntaxErrorKind::Msg(msg.into())) } } impl From for io::Error { #[inline] fn from(error: SyntaxError) -> Self { - match error.inner { + match error.0 { SyntaxErrorKind::Xml(error) => match error { quick_xml::Error::Io(error) => { Arc::try_unwrap(error).unwrap_or_else(|e| Self::new(e.kind(), e)) @@ -144,7 +82,7 @@ impl From for io::Error { } _ => Self::new(io::ErrorKind::InvalidData, error), }, - SyntaxErrorKind::Msg { msg } => Self::new(io::ErrorKind::InvalidData, msg), + SyntaxErrorKind::Msg(msg) => Self::new(io::ErrorKind::InvalidData, msg), _ => Self::new(io::ErrorKind::InvalidData, error), } } diff --git a/lib/oxrdfxml/src/parser.rs b/lib/oxrdfxml/src/parser.rs index a4e35784..a952ad02 100644 --- a/lib/oxrdfxml/src/parser.rs +++ b/lib/oxrdfxml/src/parser.rs @@ -575,8 +575,8 @@ impl RdfXmlReader { tag } else { LanguageTag::parse(tag.to_ascii_lowercase()) - .map_err(|error| SyntaxError { - inner: SyntaxErrorKind::InvalidLanguageTag { tag, error }, + .map_err(|error| { + SyntaxError(SyntaxErrorKind::InvalidLanguageTag { tag, error }) })? .into_inner() }); @@ -588,9 +588,7 @@ impl RdfXmlReader { } else { Iri::parse(iri.clone()) } - .map_err(|error| SyntaxError { - inner: SyntaxErrorKind::InvalidIri { iri, error }, - })?, + .map_err(|error| SyntaxError(SyntaxErrorKind::InvalidIri { iri, error }))?, ) } else { // We ignore other xml attributes @@ -1169,11 +1167,11 @@ impl RdfXmlReader { } else { base_iri.resolve(&relative_iri) } - .map_err(|error| SyntaxError { - inner: SyntaxErrorKind::InvalidIri { + .map_err(|error| { + SyntaxError(SyntaxErrorKind::InvalidIri { iri: relative_iri, error, - }, + }) })? .into_inner(), )) @@ -1187,11 +1185,11 @@ impl RdfXmlReader { relative_iri } else { Iri::parse(relative_iri.clone()) - .map_err(|error| SyntaxError { - inner: SyntaxErrorKind::InvalidIri { + .map_err(|error| { + SyntaxError(SyntaxErrorKind::InvalidIri { iri: relative_iri, error, - }, + }) })? .into_inner() })) diff --git a/lib/oxsdatatypes/Cargo.toml b/lib/oxsdatatypes/Cargo.toml index fb63c125..d0e8aafd 100644 --- a/lib/oxsdatatypes/Cargo.toml +++ b/lib/oxsdatatypes/Cargo.toml @@ -17,6 +17,9 @@ rust-version.workspace = true js = ["js-sys"] custom-now = [] +[dependencies] +thiserror.workspace = true + [target.'cfg(all(target_family = "wasm", target_os = "unknown"))'.dependencies] js-sys = { workspace = true, optional = true } diff --git a/lib/oxsdatatypes/src/date_time.rs b/lib/oxsdatatypes/src/date_time.rs index 1d165bbb..d508c0dd 100644 --- a/lib/oxsdatatypes/src/date_time.rs +++ b/lib/oxsdatatypes/src/date_time.rs @@ -2,7 +2,6 @@ use crate::{DayTimeDuration, Decimal, Duration, YearMonthDuration}; use std::cmp::{min, Ordering}; -use std::error::Error; use std::fmt; use std::hash::{Hash, Hasher}; use std::str::FromStr; @@ -2039,42 +2038,28 @@ fn time_on_timeline(props: &DateTimeSevenPropertyModel) -> Option { } /// A parsing error -#[derive(Debug, Clone)] -pub struct ParseDateTimeError { - kind: ParseDateTimeErrorKind, -} +#[derive(Debug, thiserror::Error)] +#[error(transparent)] +pub struct ParseDateTimeError(#[from] ParseDateTimeErrorKind); -#[derive(Debug, Clone)] +#[derive(Debug, Clone, thiserror::Error)] enum ParseDateTimeErrorKind { + #[error("{day} is not a valid day of {month}")] InvalidDayOfMonth { day: u8, month: u8 }, - Overflow(DateTimeOverflowError), + #[error(transparent)] + Overflow(#[from] DateTimeOverflowError), + #[error(transparent)] InvalidTimezone(InvalidTimezoneError), + #[error("{0}")] Message(&'static str), } -impl fmt::Display for ParseDateTimeError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match &self.kind { - ParseDateTimeErrorKind::InvalidDayOfMonth { day, month } => { - write!(f, "{day} is not a valid day of {month}") - } - ParseDateTimeErrorKind::Overflow(error) => error.fmt(f), - ParseDateTimeErrorKind::InvalidTimezone(error) => error.fmt(f), - ParseDateTimeErrorKind::Message(msg) => f.write_str(msg), - } - } -} - impl ParseDateTimeError { const fn msg(message: &'static str) -> Self { - Self { - kind: ParseDateTimeErrorKind::Message(message), - } + Self(ParseDateTimeErrorKind::Message(message)) } } -impl Error for ParseDateTimeError {} - // [16] dateTimeLexicalRep ::= yearFrag '-' monthFrag '-' dayFrag 'T' ((hourFrag ':' minuteFrag ':' secondFrag) | endOfDayFrag) timezoneFrag? fn date_time_lexical_rep(input: &str) -> Result<(DateTime, &str), ParseDateTimeError> { let (year, input) = year_frag(input)?; @@ -2326,11 +2311,8 @@ fn timezone_frag(input: &str) -> Result<(TimezoneOffset, &str), ParseDateTimeErr } Ok(( - TimezoneOffset::new(sign * (hours * 60 + i16::from(minutes))).map_err(|e| { - ParseDateTimeError { - kind: ParseDateTimeErrorKind::InvalidTimezone(e), - } - })?, + TimezoneOffset::new(sign * (hours * 60 + i16::from(minutes))) + .map_err(|e| ParseDateTimeError(ParseDateTimeErrorKind::InvalidTimezone(e)))?, input, )) } @@ -2400,9 +2382,9 @@ fn optional_end( fn validate_day_of_month(year: Option, month: u8, day: u8) -> Result<(), ParseDateTimeError> { // Constraint: Day-of-month Values if day > days_in_month(year, month) { - return Err(ParseDateTimeError { - kind: ParseDateTimeErrorKind::InvalidDayOfMonth { day, month }, - }); + return Err(ParseDateTimeError( + ParseDateTimeErrorKind::InvalidDayOfMonth { day, month }, + )); } Ok(()) } @@ -2410,51 +2392,33 @@ fn validate_day_of_month(year: Option, month: u8, day: u8) -> Result<(), Pa /// An overflow during [`DateTime`]-related operations. /// /// Matches XPath [`FODT0001` error](https://www.w3.org/TR/xpath-functions-31/#ERRFODT0001). -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, thiserror::Error)] +#[error("overflow during xsd:dateTime computation")] pub struct DateTimeOverflowError; -impl fmt::Display for DateTimeOverflowError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str("overflow during xsd:dateTime computation") - } -} - -impl Error for DateTimeOverflowError {} - impl From for ParseDateTimeError { fn from(error: DateTimeOverflowError) -> Self { - Self { - kind: ParseDateTimeErrorKind::Overflow(error), - } + Self(ParseDateTimeErrorKind::Overflow(error)) } } /// The value provided as timezone is not valid. /// /// Matches XPath [`FODT0003` error](https://www.w3.org/TR/xpath-functions-31/#ERRFODT0003). -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, thiserror::Error)] +#[error("invalid timezone offset {}:{}", + self.offset_in_minutes / 60, + self.offset_in_minutes.abs() % 60)] pub struct InvalidTimezoneError { offset_in_minutes: i64, } -impl fmt::Display for InvalidTimezoneError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!( - f, - "invalid timezone offset {}:{}", - self.offset_in_minutes / 60, - self.offset_in_minutes.abs() % 60 - ) - } -} - -impl Error for InvalidTimezoneError {} - #[cfg(test)] mod tests { #![allow(clippy::panic_in_result_fn)] use super::*; + use std::error::Error; #[test] fn from_str() -> Result<(), ParseDateTimeError> { diff --git a/lib/oxsdatatypes/src/decimal.rs b/lib/oxsdatatypes/src/decimal.rs index ee84b604..72740dc1 100644 --- a/lib/oxsdatatypes/src/decimal.rs +++ b/lib/oxsdatatypes/src/decimal.rs @@ -1,5 +1,4 @@ use crate::{Boolean, Double, Float, Integer, TooLargeForIntegerError}; -use std::error::Error; use std::fmt; use std::fmt::Write; use std::str::FromStr; @@ -609,67 +608,42 @@ impl fmt::Display for Decimal { } /// An error when parsing a [`Decimal`]. -#[derive(Debug, Clone)] -pub struct ParseDecimalError { - kind: DecimalParseErrorKind, -} +#[derive(Debug, thiserror::Error)] +#[error(transparent)] +pub struct ParseDecimalError(#[from] DecimalParseErrorKind); -#[derive(Debug, Clone)] +#[derive(Debug, Clone, thiserror::Error)] enum DecimalParseErrorKind { + #[error("Value overflow")] Overflow, + #[error("Value underflow")] Underflow, + #[error("Unexpected character")] UnexpectedChar, + #[error("Unexpected end of string")] UnexpectedEnd, } -const PARSE_OVERFLOW: ParseDecimalError = ParseDecimalError { - kind: DecimalParseErrorKind::Overflow, -}; -const PARSE_UNDERFLOW: ParseDecimalError = ParseDecimalError { - kind: DecimalParseErrorKind::Underflow, -}; -const PARSE_UNEXPECTED_CHAR: ParseDecimalError = ParseDecimalError { - kind: DecimalParseErrorKind::UnexpectedChar, -}; -const PARSE_UNEXPECTED_END: ParseDecimalError = ParseDecimalError { - kind: DecimalParseErrorKind::UnexpectedEnd, -}; - -impl fmt::Display for ParseDecimalError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self.kind { - DecimalParseErrorKind::Overflow => f.write_str("Value overflow"), - DecimalParseErrorKind::Underflow => f.write_str("Value underflow"), - DecimalParseErrorKind::UnexpectedChar => f.write_str("Unexpected character"), - DecimalParseErrorKind::UnexpectedEnd => f.write_str("Unexpected end of string"), - } - } -} - -impl Error for ParseDecimalError {} +const PARSE_OVERFLOW: ParseDecimalError = ParseDecimalError(DecimalParseErrorKind::Overflow); +const PARSE_UNDERFLOW: ParseDecimalError = ParseDecimalError(DecimalParseErrorKind::Underflow); +const PARSE_UNEXPECTED_CHAR: ParseDecimalError = + ParseDecimalError(DecimalParseErrorKind::UnexpectedChar); +const PARSE_UNEXPECTED_END: ParseDecimalError = + ParseDecimalError(DecimalParseErrorKind::UnexpectedEnd); impl From for ParseDecimalError { fn from(_: TooLargeForDecimalError) -> Self { - Self { - kind: DecimalParseErrorKind::Overflow, - } + Self(DecimalParseErrorKind::Overflow) } } /// The input is too large to fit into a [`Decimal`]. /// /// Matches XPath [`FOCA0001` error](https://www.w3.org/TR/xpath-functions-31/#ERRFOCA0001). -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, thiserror::Error)] +#[error("Value too large for xsd:decimal internal representation")] pub struct TooLargeForDecimalError; -impl fmt::Display for TooLargeForDecimalError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str("Value too large for xsd:decimal internal representation") - } -} - -impl Error for TooLargeForDecimalError {} - #[cfg(test)] mod tests { #![allow(clippy::panic_in_result_fn)] diff --git a/lib/oxsdatatypes/src/duration.rs b/lib/oxsdatatypes/src/duration.rs index 35f00bce..fe0514ab 100644 --- a/lib/oxsdatatypes/src/duration.rs +++ b/lib/oxsdatatypes/src/duration.rs @@ -1,6 +1,5 @@ use crate::{DateTime, Decimal}; use std::cmp::Ordering; -use std::error::Error; use std::fmt; use std::str::FromStr; use std::time::Duration as StdDuration; @@ -937,7 +936,8 @@ fn decimal_prefix(input: &str) -> (&str, &str) { } /// A parsing error -#[derive(Debug, Clone)] +#[derive(Debug, Clone, thiserror::Error)] +#[error("{msg}")] pub struct ParseDurationError { msg: &'static str, } @@ -946,46 +946,24 @@ const OVERFLOW_ERROR: ParseDurationError = ParseDurationError { msg: "Overflow error", }; -impl fmt::Display for ParseDurationError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(self.msg) - } -} - impl ParseDurationError { const fn msg(msg: &'static str) -> Self { Self { msg } } } -impl Error for ParseDurationError {} - /// An overflow during [`Duration`]-related operations. /// /// Matches XPath [`FODT0002` error](https://www.w3.org/TR/xpath-functions-31/#ERRFODT0002). -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, thiserror::Error)] +#[error("overflow during xsd:duration computation")] pub struct DurationOverflowError; -impl fmt::Display for DurationOverflowError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str("overflow during xsd:duration computation") - } -} - -impl Error for DurationOverflowError {} - /// The year-month and the day-time components of a [`Duration`] have an opposite sign. -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, thiserror::Error)] +#[error("The xsd:yearMonthDuration and xsd:dayTimeDuration components of a xsd:duration can't have opposite sign")] pub struct OppositeSignInDurationComponentsError; -impl fmt::Display for OppositeSignInDurationComponentsError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str("The xsd:yearMonthDuration and xsd:dayTimeDuration components of a xsd:duration can't have opposite sign") - } -} - -impl Error for OppositeSignInDurationComponentsError {} - impl From for ParseDurationError { #[inline] fn from(_: OppositeSignInDurationComponentsError) -> Self { @@ -1000,6 +978,7 @@ mod tests { #![allow(clippy::panic_in_result_fn)] use super::*; + use std::error::Error; #[test] fn from_str() -> Result<(), ParseDurationError> { diff --git a/lib/oxsdatatypes/src/integer.rs b/lib/oxsdatatypes/src/integer.rs index b28638d2..d0a693ea 100644 --- a/lib/oxsdatatypes/src/integer.rs +++ b/lib/oxsdatatypes/src/integer.rs @@ -1,5 +1,4 @@ use crate::{Boolean, Decimal, Double, Float}; -use std::error::Error; use std::fmt; use std::num::ParseIntError; use std::str::FromStr; @@ -264,17 +263,10 @@ impl TryFrom for Integer { /// The input is too large to fit into an [`Integer`]. /// /// Matches XPath [`FOCA0003` error](https://www.w3.org/TR/xpath-functions-31/#ERRFOCA0003). -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, thiserror::Error)] +#[error("Value too large for xsd:integer internal representation")] pub struct TooLargeForIntegerError; -impl fmt::Display for TooLargeForIntegerError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str("Value too large for xsd:integer internal representation") - } -} - -impl Error for TooLargeForIntegerError {} - #[cfg(test)] mod tests { #![allow(clippy::panic_in_result_fn)] diff --git a/lib/oxttl/Cargo.toml b/lib/oxttl/Cargo.toml index 6f4f48ff..0a4bc3ab 100644 --- a/lib/oxttl/Cargo.toml +++ b/lib/oxttl/Cargo.toml @@ -23,6 +23,7 @@ memchr.workspace = true oxrdf.workspace = true oxiri.workspace = true oxilangtag.workspace = true +thiserror.workspace = true tokio = { workspace = true, optional = true, features = ["io-util"] } [dev-dependencies] diff --git a/lib/oxttl/src/toolkit/error.rs b/lib/oxttl/src/toolkit/error.rs index 2f632352..33f2a916 100644 --- a/lib/oxttl/src/toolkit/error.rs +++ b/lib/oxttl/src/toolkit/error.rs @@ -1,4 +1,3 @@ -use std::error::Error; use std::ops::Range; use std::{fmt, io}; @@ -13,7 +12,7 @@ pub struct TextPosition { /// An error in the syntax of the parsed file. /// /// It is composed of a message and a byte range in the input. -#[derive(Debug)] +#[derive(Debug, thiserror::Error)] pub struct SyntaxError { pub(super) location: Range, pub(super) message: String, @@ -67,8 +66,6 @@ impl fmt::Display for SyntaxError { } } -impl Error for SyntaxError {} - impl From for io::Error { #[inline] fn from(error: SyntaxError) -> Self { @@ -79,46 +76,14 @@ impl From for io::Error { /// A parsing error. /// /// It is the union of [`SyntaxError`] and [`io::Error`]. -#[derive(Debug)] +#[derive(Debug, thiserror::Error)] pub enum ParseError { /// I/O error during parsing (file not found...). - Io(io::Error), + #[error(transparent)] + Io(#[from] io::Error), /// An error in the file syntax. - Syntax(SyntaxError), -} - -impl fmt::Display for ParseError { - #[inline] - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Io(e) => e.fmt(f), - Self::Syntax(e) => e.fmt(f), - } - } -} - -impl Error for ParseError { - #[inline] - fn source(&self) -> Option<&(dyn Error + 'static)> { - Some(match self { - Self::Io(e) => e, - Self::Syntax(e) => e, - }) - } -} - -impl From for ParseError { - #[inline] - fn from(error: SyntaxError) -> Self { - Self::Syntax(error) - } -} - -impl From for ParseError { - #[inline] - fn from(error: io::Error) -> Self { - Self::Io(error) - } + #[error(transparent)] + Syntax(#[from] SyntaxError), } impl From for io::Error { diff --git a/lib/sparesults/Cargo.toml b/lib/sparesults/Cargo.toml index 75c5a0bb..5eae3746 100644 --- a/lib/sparesults/Cargo.toml +++ b/lib/sparesults/Cargo.toml @@ -23,6 +23,7 @@ json-event-parser.workspace = true memchr.workspace = true oxrdf.workspace = true quick-xml.workspace = true +thiserror.workspace = true tokio = { workspace = true, optional = true, features = ["io-util"] } [dev-dependencies] diff --git a/lib/sparesults/src/error.rs b/lib/sparesults/src/error.rs index 8ece28d0..cc5a1e4a 100644 --- a/lib/sparesults/src/error.rs +++ b/lib/sparesults/src/error.rs @@ -1,50 +1,17 @@ use oxrdf::TermParseError; -use std::error::Error; +use std::io; use std::ops::Range; use std::sync::Arc; -use std::{fmt, io}; /// Error returned during SPARQL result formats format parsing. -#[derive(Debug)] +#[derive(Debug, thiserror::Error)] pub enum ParseError { /// I/O error during parsing (file not found...). - Io(io::Error), + #[error(transparent)] + Io(#[from] io::Error), /// An error in the file syntax. - Syntax(SyntaxError), -} - -impl fmt::Display for ParseError { - #[inline] - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Io(e) => e.fmt(f), - Self::Syntax(e) => e.fmt(f), - } - } -} - -impl Error for ParseError { - #[inline] - fn source(&self) -> Option<&(dyn Error + 'static)> { - match self { - Self::Io(e) => Some(e), - Self::Syntax(e) => Some(e), - } - } -} - -impl From for ParseError { - #[inline] - fn from(error: io::Error) -> Self { - Self::Io(error) - } -} - -impl From for ParseError { - #[inline] - fn from(error: SyntaxError) -> Self { - Self::Syntax(error) - } + #[error(transparent)] + Syntax(#[from] SyntaxError), } impl From for io::Error { @@ -81,20 +48,27 @@ impl From for ParseError { } /// An error in the syntax of the parsed file. -#[derive(Debug)] +#[derive(Debug, thiserror::Error)] +#[error(transparent)] pub struct SyntaxError { + #[from] pub(crate) inner: SyntaxErrorKind, } -#[derive(Debug)] +#[derive(Debug, thiserror::Error)] pub(crate) enum SyntaxErrorKind { - Json(json_event_parser::SyntaxError), - Xml(quick_xml::Error), + #[error(transparent)] + Json(#[from] json_event_parser::SyntaxError), + #[error(transparent)] + Xml(#[from] quick_xml::Error), + #[error("Error {error} on '{term}' in line {}", location.start.line + 1)] Term { + #[source] error: TermParseError, term: String, location: Range, }, + #[error("{msg}")] Msg { msg: String, location: Option>, @@ -149,34 +123,6 @@ impl SyntaxError { } } -impl fmt::Display for SyntaxError { - #[inline] - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match &self.inner { - SyntaxErrorKind::Json(e) => e.fmt(f), - SyntaxErrorKind::Xml(e) => e.fmt(f), - SyntaxErrorKind::Term { - error, - term, - location, - } => write!(f, "{error} on '{term}' in line {}", location.start.line + 1), - SyntaxErrorKind::Msg { msg, .. } => f.write_str(msg), - } - } -} - -impl Error for SyntaxError { - #[inline] - fn source(&self) -> Option<&(dyn Error + 'static)> { - match &self.inner { - SyntaxErrorKind::Json(e) => Some(e), - SyntaxErrorKind::Xml(e) => Some(e), - SyntaxErrorKind::Term { error, .. } => Some(error), - SyntaxErrorKind::Msg { .. } => None, - } - } -} - impl From for io::Error { #[inline] fn from(error: SyntaxError) -> Self { diff --git a/lib/spargebra/Cargo.toml b/lib/spargebra/Cargo.toml index 24acb482..f367c57c 100644 --- a/lib/spargebra/Cargo.toml +++ b/lib/spargebra/Cargo.toml @@ -25,6 +25,7 @@ oxiri.workspace = true oxrdf.workspace = true peg.workspace = true rand.workspace = true +thiserror.workspace = true [lints] workspace = true diff --git a/lib/spargebra/src/parser.rs b/lib/spargebra/src/parser.rs index bcb4d491..a0a8e9d1 100644 --- a/lib/spargebra/src/parser.rs +++ b/lib/spargebra/src/parser.rs @@ -9,26 +9,22 @@ use oxrdf::vocab::{rdf, xsd}; use peg::parser; use peg::str::LineCol; use rand::random; +use std::char; use std::collections::{HashMap, HashSet}; -use std::error::Error; use std::mem::take; use std::str::FromStr; -use std::{char, fmt}; /// Parses a SPARQL query with an optional base IRI to resolve relative IRIs in the query. pub fn parse_query(query: &str, base_iri: Option<&str>) -> Result { let mut state = ParserState::from_base_iri(base_iri)?; - parser::QueryUnit(query, &mut state).map_err(|e| ParseError { - inner: ParseErrorKind::Parser(e), - }) + parser::QueryUnit(query, &mut state).map_err(|e| ParseError(ParseErrorKind::Parser(e))) } /// Parses a SPARQL update with an optional base IRI to resolve relative IRIs in the query. pub fn parse_update(update: &str, base_iri: Option<&str>) -> Result { let mut state = ParserState::from_base_iri(base_iri)?; - let operations = parser::UpdateInit(update, &mut state).map_err(|e| ParseError { - inner: ParseErrorKind::Parser(e), - })?; + let operations = parser::UpdateInit(update, &mut state) + .map_err(|e| ParseError(ParseErrorKind::Parser(e)))?; Ok(Update { operations, base_iri: state.base_iri, @@ -36,37 +32,16 @@ pub fn parse_update(update: &str, base_iri: Option<&str>) -> Result), -} - -impl fmt::Display for ParseError { - #[inline] - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match &self.inner { - ParseErrorKind::InvalidBaseIri(e) => { - write!(f, "Invalid SPARQL base IRI provided: {e}") - } - ParseErrorKind::Parser(e) => e.fmt(f), - } - } -} - -impl Error for ParseError { - #[inline] - fn source(&self) -> Option<&(dyn Error + 'static)> { - match &self.inner { - ParseErrorKind::InvalidBaseIri(e) => Some(e), - ParseErrorKind::Parser(e) => Some(e), - } - } + #[error("Invalid SPARQL base IRI provided: {0}")] + InvalidBaseIri(#[from] IriParseError), + #[error(transparent)] + Parser(#[from] peg::error::ParseError), } struct AnnotatedTerm { @@ -697,9 +672,10 @@ impl ParserState { pub(crate) fn from_base_iri(base_iri: Option<&str>) -> Result { Ok(Self { base_iri: if let Some(base_iri) = base_iri { - Some(Iri::parse(base_iri.to_owned()).map_err(|e| ParseError { - inner: ParseErrorKind::InvalidBaseIri(e), - })?) + Some( + Iri::parse(base_iri.to_owned()) + .map_err(|e| ParseError(ParseErrorKind::InvalidBaseIri(e)))?, + ) } else { None }, From 0400f0491559ac6552c779be8dc495a5a096a393 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Sat, 10 Feb 2024 12:13:09 -0500 Subject: [PATCH 211/217] Error renaming ``` enum sparesults::error::ParseError -> QueryResultsParseError struct sparesults::error::SyntaxError -> QueryResultsSyntaxError Inlined inner enum oxrdfxml::error::ParseError -> RdfXmlParseError struct oxrdfxml::error::SyntaxError -> RdfXmlSyntaxError enum oxttl::toolkit::error::ParseError -> TurtleParseError struct oxttl::toolkit::error::SyntaxError -> TurtleSyntaxError enum oxrdfio::error::ParseError -> RdfParseError struct oxrdfio::error::SyntaxError -> RdfSyntaxError struct spargebra::parser::ParseError -> SparqlSyntaxError enum spargebra::parser::ParseErrorKind Parser -> Syntax ``` --- Cargo.toml | 6 +- lib/oxigraph/src/io/read.rs | 6 +- lib/oxigraph/src/sparql/algebra.rs | 24 +++-- lib/oxigraph/src/sparql/error.rs | 8 +- lib/oxigraph/src/sparql/mod.rs | 2 +- lib/oxigraph/src/sparql/model.rs | 9 +- lib/oxigraph/src/storage/error.rs | 4 +- lib/oxigraph/src/store.rs | 6 +- lib/oxrdfio/Cargo.toml | 2 +- lib/oxrdfio/src/error.rs | 52 ++++----- lib/oxrdfio/src/lib.rs | 2 +- lib/oxrdfio/src/parser.rs | 38 +++---- lib/oxrdfxml/Cargo.toml | 2 +- lib/oxrdfxml/src/error.rs | 24 ++--- lib/oxrdfxml/src/lib.rs | 2 +- lib/oxrdfxml/src/parser.rs | 124 +++++++++++---------- lib/oxttl/Cargo.toml | 2 +- lib/oxttl/src/lib.rs | 2 +- lib/oxttl/src/n3.rs | 18 ++-- lib/oxttl/src/nquads.rs | 12 +-- lib/oxttl/src/ntriples.rs | 12 +-- lib/oxttl/src/toolkit/error.rs | 24 ++--- lib/oxttl/src/toolkit/lexer.rs | 11 +- lib/oxttl/src/toolkit/mod.rs | 2 +- lib/oxttl/src/toolkit/parser.rs | 14 +-- lib/oxttl/src/trig.rs | 16 +-- lib/oxttl/src/turtle.rs | 16 +-- lib/sparesults/src/csv.rs | 52 ++++----- lib/sparesults/src/error.rs | 65 +++++------ lib/sparesults/src/json.rs | 167 +++++++++++++++++++---------- lib/sparesults/src/lib.rs | 2 +- lib/sparesults/src/parser.rs | 20 ++-- lib/sparesults/src/xml.rs | 83 +++++++------- lib/spargebra/src/lib.rs | 2 +- lib/spargebra/src/parser.rs | 16 +-- lib/spargebra/src/query.rs | 12 +-- lib/spargebra/src/update.rs | 12 +-- python/src/io.rs | 8 +- python/src/sparql.rs | 13 ++- 39 files changed, 481 insertions(+), 411 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 203ae7a1..7e0fe900 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -70,11 +70,11 @@ zstd = ">=0.12, <0.14" # Internal dependencies oxigraph = { version = "0.4.0-alpha.3-dev", path = "lib/oxigraph" } oxrdf = { version = "0.2.0-alpha.2", path = "lib/oxrdf" } -oxrdfio = { version = "0.1.0-alpha.2", path = "lib/oxrdfio" } -oxrdfxml = { version = "0.1.0-alpha.2", path = "lib/oxrdfxml" } +oxrdfio = { version = "0.1.0-alpha.3-dev", path = "lib/oxrdfio" } +oxrdfxml = { version = "0.1.0-alpha.3-dev", path = "lib/oxrdfxml" } oxrocksdb-sys = { version = "0.4.0-alpha.3-dev", path = "./oxrocksdb-sys" } oxsdatatypes = { version = "0.2.0-alpha.1", path = "lib/oxsdatatypes" } -oxttl = { version = "0.1.0-alpha.2", path = "lib/oxttl" } +oxttl = { version = "0.1.0-alpha.3-dev", path = "lib/oxttl" } sparesults = { version = "0.2.0-alpha.2", path = "lib/sparesults" } spargebra = { version = "0.3.0-alpha.2", path = "lib/spargebra" } sparopt = { version = "0.1.0-alpha.2", path = "lib/sparopt" } diff --git a/lib/oxigraph/src/io/read.rs b/lib/oxigraph/src/io/read.rs index 841b166a..c1c3b332 100644 --- a/lib/oxigraph/src/io/read.rs +++ b/lib/oxigraph/src/io/read.rs @@ -5,7 +5,7 @@ use crate::io::{DatasetFormat, GraphFormat}; use crate::model::*; use oxiri::IriParseError; -use oxrdfio::{FromReadQuadReader, ParseError, RdfParser}; +use oxrdfio::{FromReadQuadReader, RdfParseError, RdfParser}; use std::io::Read; /// Parsers for RDF graph serialization formats. @@ -100,7 +100,7 @@ pub struct TripleReader { } impl Iterator for TripleReader { - type Item = Result; + type Item = Result; fn next(&mut self) -> Option { Some(self.parser.next()?.map(Into::into).map_err(Into::into)) @@ -192,7 +192,7 @@ pub struct QuadReader { } impl Iterator for QuadReader { - type Item = Result; + type Item = Result; fn next(&mut self) -> Option { Some(self.parser.next()?.map_err(Into::into)) diff --git a/lib/oxigraph/src/sparql/algebra.rs b/lib/oxigraph/src/sparql/algebra.rs index 819a9bd9..41689904 100644 --- a/lib/oxigraph/src/sparql/algebra.rs +++ b/lib/oxigraph/src/sparql/algebra.rs @@ -38,7 +38,10 @@ pub struct Query { impl Query { /// Parses a SPARQL query with an optional base IRI to resolve relative IRIs in the query. - pub fn parse(query: &str, base_iri: Option<&str>) -> Result { + pub fn parse( + query: &str, + base_iri: Option<&str>, + ) -> Result { let start = Timer::now(); let query = Self::from(spargebra::Query::parse(query, base_iri)?); Ok(Self { @@ -66,7 +69,7 @@ impl fmt::Display for Query { } impl FromStr for Query { - type Err = spargebra::ParseError; + type Err = spargebra::SparqlSyntaxError; fn from_str(query: &str) -> Result { Self::parse(query, None) @@ -74,7 +77,7 @@ impl FromStr for Query { } impl TryFrom<&str> for Query { - type Error = spargebra::ParseError; + type Error = spargebra::SparqlSyntaxError; fn try_from(query: &str) -> Result { Self::from_str(query) @@ -82,7 +85,7 @@ impl TryFrom<&str> for Query { } impl TryFrom<&String> for Query { - type Error = spargebra::ParseError; + type Error = spargebra::SparqlSyntaxError; fn try_from(query: &String) -> Result { Self::from_str(query) @@ -113,7 +116,7 @@ impl From for Query { /// let update = Update::parse(update_str, None)?; /// /// assert_eq!(update.to_string().trim(), update_str); -/// # Ok::<_, oxigraph::sparql::ParseError>(()) +/// # Ok::<_, oxigraph::sparql::SparqlSyntaxError>(()) /// ``` #[derive(Eq, PartialEq, Debug, Clone, Hash)] pub struct Update { @@ -123,7 +126,10 @@ pub struct Update { impl Update { /// Parses a SPARQL update with an optional base IRI to resolve relative IRIs in the query. - pub fn parse(update: &str, base_iri: Option<&str>) -> Result { + pub fn parse( + update: &str, + base_iri: Option<&str>, + ) -> Result { let update = spargebra::Update::parse(update, base_iri)?; Ok(Self { using_datasets: update @@ -159,7 +165,7 @@ impl fmt::Display for Update { } impl FromStr for Update { - type Err = spargebra::ParseError; + type Err = spargebra::SparqlSyntaxError; fn from_str(update: &str) -> Result { Self::parse(update, None) @@ -167,7 +173,7 @@ impl FromStr for Update { } impl TryFrom<&str> for Update { - type Error = spargebra::ParseError; + type Error = spargebra::SparqlSyntaxError; fn try_from(update: &str) -> Result { Self::from_str(update) @@ -175,7 +181,7 @@ impl TryFrom<&str> for Update { } impl TryFrom<&String> for Update { - type Error = spargebra::ParseError; + type Error = spargebra::SparqlSyntaxError; fn try_from(update: &String) -> Result { Self::from_str(update) diff --git a/lib/oxigraph/src/sparql/error.rs b/lib/oxigraph/src/sparql/error.rs index 94ee6f57..38731ded 100644 --- a/lib/oxigraph/src/sparql/error.rs +++ b/lib/oxigraph/src/sparql/error.rs @@ -1,7 +1,7 @@ -use crate::io::ParseError as RdfParseError; +use crate::io::RdfParseError; use crate::model::NamedNode; -use crate::sparql::results::ParseError as ResultsParseError; -use crate::sparql::ParseError; +use crate::sparql::results::QueryResultsParseError as ResultsParseError; +use crate::sparql::SparqlSyntaxError; use crate::storage::StorageError; use std::convert::Infallible; use std::error::Error; @@ -13,7 +13,7 @@ use std::io; pub enum EvaluationError { /// An error in SPARQL parsing. #[error(transparent)] - Parsing(#[from] ParseError), + Parsing(#[from] SparqlSyntaxError), /// An error from the storage. #[error(transparent)] Storage(#[from] StorageError), diff --git a/lib/oxigraph/src/sparql/mod.rs b/lib/oxigraph/src/sparql/mod.rs index ec84467f..089f84be 100644 --- a/lib/oxigraph/src/sparql/mod.rs +++ b/lib/oxigraph/src/sparql/mod.rs @@ -25,7 +25,7 @@ use crate::storage::StorageReader; use json_event_parser::{JsonEvent, ToWriteJsonWriter}; pub use oxrdf::{Variable, VariableNameParseError}; use oxsdatatypes::{DayTimeDuration, Float}; -pub use spargebra::ParseError; +pub use spargebra::SparqlSyntaxError; use sparopt::algebra::GraphPattern; use sparopt::Optimizer; use std::collections::HashMap; diff --git a/lib/oxigraph/src/sparql/model.rs b/lib/oxigraph/src/sparql/model.rs index 051f5136..f3624ad5 100644 --- a/lib/oxigraph/src/sparql/model.rs +++ b/lib/oxigraph/src/sparql/model.rs @@ -2,8 +2,8 @@ use crate::io::{RdfFormat, RdfSerializer}; use crate::model::*; use crate::sparql::error::EvaluationError; use crate::sparql::results::{ - FromReadQueryResultsReader, FromReadSolutionsReader, ParseError, QueryResultsFormat, - QueryResultsParser, QueryResultsSerializer, + FromReadQueryResultsReader, FromReadSolutionsReader, QueryResultsFormat, + QueryResultsParseError, QueryResultsParser, QueryResultsSerializer, }; use oxrdf::{Variable, VariableRef}; pub use sparesults::QuerySolution; @@ -22,7 +22,10 @@ pub enum QueryResults { impl QueryResults { /// Reads a SPARQL query results serialization. - pub fn read(read: impl Read + 'static, format: QueryResultsFormat) -> Result { + pub fn read( + read: impl Read + 'static, + format: QueryResultsFormat, + ) -> Result { Ok(QueryResultsParser::from_format(format) .parse_read(read)? .into()) diff --git a/lib/oxigraph/src/storage/error.rs b/lib/oxigraph/src/storage/error.rs index f2f27cdb..d58d0316 100644 --- a/lib/oxigraph/src/storage/error.rs +++ b/lib/oxigraph/src/storage/error.rs @@ -1,4 +1,4 @@ -use crate::io::{ParseError, RdfFormat}; +use crate::io::{RdfFormat, RdfParseError}; use crate::storage::numeric_encoder::EncodedTerm; use oxiri::IriParseError; use oxrdf::TermRef; @@ -83,7 +83,7 @@ impl From for io::Error { pub enum LoaderError { /// An error raised while reading the file. #[error(transparent)] - Parsing(#[from] ParseError), + Parsing(#[from] RdfParseError), /// An error raised during the insertion in the store. #[error(transparent)] Storage(#[from] StorageError), diff --git a/lib/oxigraph/src/store.rs b/lib/oxigraph/src/store.rs index 5b5a1640..d84f5728 100644 --- a/lib/oxigraph/src/store.rs +++ b/lib/oxigraph/src/store.rs @@ -26,7 +26,7 @@ //! # Result::<_, Box>::Ok(()) //! ``` #[cfg(not(target_family = "wasm"))] -use crate::io::ParseError; +use crate::io::RdfParseError; use crate::io::{RdfFormat, RdfParser, RdfSerializer}; use crate::model::*; use crate::sparql::{ @@ -1592,7 +1592,7 @@ impl Iterator for GraphNameIter { #[must_use] pub struct BulkLoader { storage: StorageBulkLoader, - on_parse_error: Option Result<(), ParseError>>>, + on_parse_error: Option Result<(), RdfParseError>>>, } #[cfg(not(target_family = "wasm"))] @@ -1647,7 +1647,7 @@ impl BulkLoader { /// By default the parsing fails. pub fn on_parse_error( mut self, - callback: impl Fn(ParseError) -> Result<(), ParseError> + 'static, + callback: impl Fn(RdfParseError) -> Result<(), RdfParseError> + 'static, ) -> Self { self.on_parse_error = Some(Box::new(callback)); self diff --git a/lib/oxrdfio/Cargo.toml b/lib/oxrdfio/Cargo.toml index 05069df3..427fa559 100644 --- a/lib/oxrdfio/Cargo.toml +++ b/lib/oxrdfio/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "oxrdfio" -version = "0.1.0-alpha.2" +version = "0.1.0-alpha.3-dev" authors.workspace = true license.workspace = true readme = "README.md" diff --git a/lib/oxrdfio/src/error.rs b/lib/oxrdfio/src/error.rs index cd2c4449..a2d14845 100644 --- a/lib/oxrdfio/src/error.rs +++ b/lib/oxrdfio/src/error.rs @@ -3,61 +3,61 @@ use std::ops::Range; /// Error returned during RDF format parsing. #[derive(Debug, thiserror::Error)] -pub enum ParseError { +pub enum RdfParseError { /// I/O error during parsing (file not found...). #[error(transparent)] Io(#[from] io::Error), /// An error in the file syntax. #[error(transparent)] - Syntax(#[from] SyntaxError), + Syntax(#[from] RdfSyntaxError), } -impl ParseError { +impl RdfParseError { pub(crate) fn msg(msg: &'static str) -> Self { - Self::Syntax(SyntaxError(SyntaxErrorKind::Msg(msg))) + Self::Syntax(RdfSyntaxError(SyntaxErrorKind::Msg(msg))) } } -impl From for SyntaxError { +impl From for RdfSyntaxError { #[inline] - fn from(error: oxttl::SyntaxError) -> Self { + fn from(error: oxttl::TurtleSyntaxError) -> Self { Self(SyntaxErrorKind::Turtle(error)) } } -impl From for ParseError { +impl From for RdfParseError { #[inline] - fn from(error: oxttl::ParseError) -> Self { + fn from(error: oxttl::TurtleParseError) -> Self { match error { - oxttl::ParseError::Syntax(e) => Self::Syntax(e.into()), - oxttl::ParseError::Io(e) => Self::Io(e), + oxttl::TurtleParseError::Syntax(e) => Self::Syntax(e.into()), + oxttl::TurtleParseError::Io(e) => Self::Io(e), } } } -impl From for SyntaxError { +impl From for RdfSyntaxError { #[inline] - fn from(error: oxrdfxml::SyntaxError) -> Self { + fn from(error: oxrdfxml::RdfXmlSyntaxError) -> Self { Self(SyntaxErrorKind::RdfXml(error)) } } -impl From for ParseError { +impl From for RdfParseError { #[inline] - fn from(error: oxrdfxml::ParseError) -> Self { + fn from(error: oxrdfxml::RdfXmlParseError) -> Self { match error { - oxrdfxml::ParseError::Syntax(e) => Self::Syntax(e.into()), - oxrdfxml::ParseError::Io(e) => Self::Io(e), + oxrdfxml::RdfXmlParseError::Syntax(e) => Self::Syntax(e.into()), + oxrdfxml::RdfXmlParseError::Io(e) => Self::Io(e), } } } -impl From for io::Error { +impl From for io::Error { #[inline] - fn from(error: ParseError) -> Self { + fn from(error: RdfParseError) -> Self { match error { - ParseError::Io(error) => error, - ParseError::Syntax(error) => error.into(), + RdfParseError::Io(error) => error, + RdfParseError::Syntax(error) => error.into(), } } } @@ -65,20 +65,20 @@ impl From for io::Error { /// An error in the syntax of the parsed file. #[derive(Debug, thiserror::Error)] #[error(transparent)] -pub struct SyntaxError(#[from] SyntaxErrorKind); +pub struct RdfSyntaxError(#[from] SyntaxErrorKind); /// An error in the syntax of the parsed file. #[derive(Debug, thiserror::Error)] enum SyntaxErrorKind { #[error(transparent)] - Turtle(#[from] oxttl::SyntaxError), + Turtle(#[from] oxttl::TurtleSyntaxError), #[error(transparent)] - RdfXml(#[from] oxrdfxml::SyntaxError), + RdfXml(#[from] oxrdfxml::RdfXmlSyntaxError), #[error("{0}")] Msg(&'static str), } -impl SyntaxError { +impl RdfSyntaxError { /// The location of the error inside of the file. #[inline] pub fn location(&self) -> Option> { @@ -102,9 +102,9 @@ impl SyntaxError { } } -impl From for io::Error { +impl From for io::Error { #[inline] - fn from(error: SyntaxError) -> Self { + fn from(error: RdfSyntaxError) -> Self { match error.0 { SyntaxErrorKind::Turtle(error) => error.into(), SyntaxErrorKind::RdfXml(error) => error.into(), diff --git a/lib/oxrdfio/src/lib.rs b/lib/oxrdfio/src/lib.rs index 4b51dcf8..b0873f79 100644 --- a/lib/oxrdfio/src/lib.rs +++ b/lib/oxrdfio/src/lib.rs @@ -9,7 +9,7 @@ mod format; mod parser; mod serializer; -pub use error::{ParseError, SyntaxError, TextPosition}; +pub use error::{RdfParseError, RdfSyntaxError, TextPosition}; pub use format::RdfFormat; #[cfg(feature = "async-tokio")] pub use parser::FromTokioAsyncReadQuadReader; diff --git a/lib/oxrdfio/src/parser.rs b/lib/oxrdfio/src/parser.rs index d1536141..5e8f1b8d 100644 --- a/lib/oxrdfio/src/parser.rs +++ b/lib/oxrdfio/src/parser.rs @@ -1,6 +1,6 @@ //! Utilities to read RDF graphs and datasets. -pub use crate::error::ParseError; +pub use crate::error::RdfParseError; use crate::format::RdfFormat; use oxrdf::{BlankNode, GraphName, IriParseError, Quad, Subject, Term, Triple}; #[cfg(feature = "async-tokio")] @@ -310,7 +310,7 @@ impl RdfParser { /// use oxrdfio::{RdfFormat, RdfParser}; /// /// # #[tokio::main(flavor = "current_thread")] - /// # async fn main() -> Result<(), oxrdfio::ParseError> { + /// # async fn main() -> Result<(), oxrdfio::RdfParseError> { /// let file = " ."; /// /// let parser = RdfParser::from_format(RdfFormat::NTriples); @@ -396,7 +396,7 @@ enum FromReadQuadReaderKind { } impl Iterator for FromReadQuadReader { - type Item = Result; + type Item = Result; fn next(&mut self) -> Option { Some(match &mut self.parser { @@ -507,7 +507,7 @@ impl FromReadQuadReader { /// use oxrdfio::{RdfFormat, RdfParser}; /// /// # #[tokio::main(flavor = "current_thread")] -/// # async fn main() -> Result<(), oxrdfio::ParseError> { +/// # async fn main() -> Result<(), oxrdfio::RdfParseError> { /// let file = " ."; /// /// let parser = RdfParser::from_format(RdfFormat::NTriples); @@ -537,7 +537,7 @@ enum FromTokioAsyncReadQuadReaderKind { #[cfg(feature = "async-tokio")] impl FromTokioAsyncReadQuadReader { - pub async fn next(&mut self) -> Option> { + pub async fn next(&mut self) -> Option> { Some(match &mut self.parser { FromTokioAsyncReadQuadReaderKind::N3(parser) => match parser.next().await? { Ok(quad) => self.mapper.map_n3_quad(quad), @@ -578,7 +578,7 @@ impl FromTokioAsyncReadQuadReader { /// use oxrdfio::{RdfFormat, RdfParser}; /// /// # #[tokio::main(flavor = "current_thread")] - /// # async fn main() -> Result<(), oxttl::ParseError> { + /// # async fn main() -> Result<(), oxttl::TurtleParseError> { /// let file = br#"@base . /// @prefix schema: . /// a schema:Person ; @@ -618,7 +618,7 @@ impl FromTokioAsyncReadQuadReader { /// use oxrdfio::{RdfFormat, RdfParser}; /// /// # #[tokio::main(flavor = "current_thread")] - /// # async fn main() -> Result<(), oxttl::ParseError> { + /// # async fn main() -> Result<(), oxttl::TurtleParseError> { /// let file = br#"@base . /// @prefix schema: . /// a schema:Person ; @@ -728,18 +728,18 @@ impl QuadMapper { } } - fn map_graph_name(&mut self, graph_name: GraphName) -> Result { + fn map_graph_name(&mut self, graph_name: GraphName) -> Result { match graph_name { GraphName::NamedNode(node) => { if self.without_named_graphs { - Err(ParseError::msg("Named graphs are not allowed")) + Err(RdfParseError::msg("Named graphs are not allowed")) } else { Ok(node.into()) } } GraphName::BlankNode(node) => { if self.without_named_graphs { - Err(ParseError::msg("Named graphs are not allowed")) + Err(RdfParseError::msg("Named graphs are not allowed")) } else { Ok(self.map_blank_node(node).into()) } @@ -748,7 +748,7 @@ impl QuadMapper { } } - fn map_quad(&mut self, quad: Quad) -> Result { + fn map_quad(&mut self, quad: Quad) -> Result { Ok(Quad { subject: self.map_subject(quad.subject), predicate: quad.predicate, @@ -761,33 +761,33 @@ impl QuadMapper { self.map_triple(triple).in_graph(self.default_graph.clone()) } - fn map_n3_quad(&mut self, quad: N3Quad) -> Result { + fn map_n3_quad(&mut self, quad: N3Quad) -> Result { Ok(Quad { subject: match quad.subject { N3Term::NamedNode(s) => Ok(s.into()), N3Term::BlankNode(s) => Ok(self.map_blank_node(s).into()), - N3Term::Literal(_) => Err(ParseError::msg( + N3Term::Literal(_) => Err(RdfParseError::msg( "literals are not allowed in regular RDF subjects", )), #[cfg(feature = "rdf-star")] N3Term::Triple(s) => Ok(self.map_triple(*s).into()), - N3Term::Variable(_) => Err(ParseError::msg( + N3Term::Variable(_) => Err(RdfParseError::msg( "variables are not allowed in regular RDF subjects", )), }?, predicate: match quad.predicate { N3Term::NamedNode(p) => Ok(p), - N3Term::BlankNode(_) => Err(ParseError::msg( + N3Term::BlankNode(_) => Err(RdfParseError::msg( "blank nodes are not allowed in regular RDF predicates", )), - N3Term::Literal(_) => Err(ParseError::msg( + N3Term::Literal(_) => Err(RdfParseError::msg( "literals are not allowed in regular RDF predicates", )), #[cfg(feature = "rdf-star")] - N3Term::Triple(_) => Err(ParseError::msg( + N3Term::Triple(_) => Err(RdfParseError::msg( "quoted triples are not allowed in regular RDF predicates", )), - N3Term::Variable(_) => Err(ParseError::msg( + N3Term::Variable(_) => Err(RdfParseError::msg( "variables are not allowed in regular RDF predicates", )), }?, @@ -797,7 +797,7 @@ impl QuadMapper { N3Term::Literal(o) => Ok(o.into()), #[cfg(feature = "rdf-star")] N3Term::Triple(o) => Ok(self.map_triple(*o).into()), - N3Term::Variable(_) => Err(ParseError::msg( + N3Term::Variable(_) => Err(RdfParseError::msg( "variables are not allowed in regular RDF objects", )), }?, diff --git a/lib/oxrdfxml/Cargo.toml b/lib/oxrdfxml/Cargo.toml index efc71e62..97dcee82 100644 --- a/lib/oxrdfxml/Cargo.toml +++ b/lib/oxrdfxml/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "oxrdfxml" -version = "0.1.0-alpha.2" +version = "0.1.0-alpha.3-dev" authors.workspace = true license.workspace = true readme = "README.md" diff --git a/lib/oxrdfxml/src/error.rs b/lib/oxrdfxml/src/error.rs index abc7c55c..9a59a766 100644 --- a/lib/oxrdfxml/src/error.rs +++ b/lib/oxrdfxml/src/error.rs @@ -5,33 +5,33 @@ use std::sync::Arc; /// Error returned during RDF/XML parsing. #[derive(Debug, thiserror::Error)] -pub enum ParseError { +pub enum RdfXmlParseError { /// I/O error during parsing (file not found...). #[error(transparent)] Io(#[from] io::Error), /// An error in the file syntax. #[error(transparent)] - Syntax(#[from] SyntaxError), + Syntax(#[from] RdfXmlSyntaxError), } -impl From for io::Error { +impl From for io::Error { #[inline] - fn from(error: ParseError) -> Self { + fn from(error: RdfXmlParseError) -> Self { match error { - ParseError::Io(error) => error, - ParseError::Syntax(error) => error.into(), + RdfXmlParseError::Io(error) => error, + RdfXmlParseError::Syntax(error) => error.into(), } } } -impl From for ParseError { +impl From for RdfXmlParseError { #[inline] fn from(error: quick_xml::Error) -> Self { match error { quick_xml::Error::Io(error) => { Self::Io(Arc::try_unwrap(error).unwrap_or_else(|e| io::Error::new(e.kind(), e))) } - _ => Self::Syntax(SyntaxError(SyntaxErrorKind::Xml(error))), + _ => Self::Syntax(RdfXmlSyntaxError(SyntaxErrorKind::Xml(error))), } } } @@ -39,7 +39,7 @@ impl From for ParseError { /// An error in the syntax of the parsed file. #[derive(Debug, thiserror::Error)] #[error(transparent)] -pub struct SyntaxError(#[from] pub(crate) SyntaxErrorKind); +pub struct RdfXmlSyntaxError(#[from] pub(crate) SyntaxErrorKind); #[derive(Debug, thiserror::Error)] pub enum SyntaxErrorKind { @@ -61,7 +61,7 @@ pub enum SyntaxErrorKind { Msg(String), } -impl SyntaxError { +impl RdfXmlSyntaxError { /// Builds an error from a printable error message. #[inline] pub(crate) fn msg(msg: impl Into) -> Self { @@ -69,9 +69,9 @@ impl SyntaxError { } } -impl From for io::Error { +impl From for io::Error { #[inline] - fn from(error: SyntaxError) -> Self { + fn from(error: RdfXmlSyntaxError) -> Self { match error.0 { SyntaxErrorKind::Xml(error) => match error { quick_xml::Error::Io(error) => { diff --git a/lib/oxrdfxml/src/lib.rs b/lib/oxrdfxml/src/lib.rs index bcc87308..2354101a 100644 --- a/lib/oxrdfxml/src/lib.rs +++ b/lib/oxrdfxml/src/lib.rs @@ -9,7 +9,7 @@ mod parser; mod serializer; mod utils; -pub use error::{ParseError, SyntaxError}; +pub use error::{RdfXmlParseError, RdfXmlSyntaxError}; #[cfg(feature = "async-tokio")] pub use parser::FromTokioAsyncReadRdfXmlReader; pub use parser::{FromReadRdfXmlReader, RdfXmlParser}; diff --git a/lib/oxrdfxml/src/parser.rs b/lib/oxrdfxml/src/parser.rs index a952ad02..d11a1f99 100644 --- a/lib/oxrdfxml/src/parser.rs +++ b/lib/oxrdfxml/src/parser.rs @@ -1,4 +1,4 @@ -use crate::error::{ParseError, SyntaxError, SyntaxErrorKind}; +use crate::error::{RdfXmlParseError, RdfXmlSyntaxError, SyntaxErrorKind}; use crate::utils::*; use oxilangtag::LanguageTag; use oxiri::{Iri, IriParseError}; @@ -126,7 +126,7 @@ impl RdfXmlParser { /// use oxrdfxml::RdfXmlParser; /// /// # #[tokio::main(flavor = "current_thread")] - /// # async fn main() -> Result<(), oxrdfxml::ParseError> { + /// # async fn main() -> Result<(), oxrdfxml::RdfXmlParseError> { /// let file = br#" /// /// @@ -214,7 +214,7 @@ pub struct FromReadRdfXmlReader { } impl Iterator for FromReadRdfXmlReader { - type Item = Result; + type Item = Result; fn next(&mut self) -> Option { loop { @@ -236,7 +236,7 @@ impl FromReadRdfXmlReader { self.reader.reader.buffer_position() } - fn parse_step(&mut self) -> Result<(), ParseError> { + fn parse_step(&mut self) -> Result<(), RdfXmlParseError> { self.reader_buffer.clear(); let event = self .reader @@ -255,7 +255,7 @@ impl FromReadRdfXmlReader { /// use oxrdfxml::RdfXmlParser; /// /// # #[tokio::main(flavor = "current_thread")] -/// # async fn main() -> Result<(), oxrdfxml::ParseError> { +/// # async fn main() -> Result<(), oxrdfxml::RdfXmlParseError> { /// let file = br#" /// /// @@ -289,7 +289,7 @@ pub struct FromTokioAsyncReadRdfXmlReader { #[cfg(feature = "async-tokio")] impl FromTokioAsyncReadRdfXmlReader { /// Reads the next triple or returns `None` if the file is finished. - pub async fn next(&mut self) -> Option> { + pub async fn next(&mut self) -> Option> { loop { if let Some(triple) = self.results.pop() { return Some(Ok(triple)); @@ -307,7 +307,7 @@ impl FromTokioAsyncReadRdfXmlReader { self.reader.reader.buffer_position() } - async fn parse_step(&mut self) -> Result<(), ParseError> { + async fn parse_step(&mut self) -> Result<(), RdfXmlParseError> { self.reader_buffer.clear(); let event = self .reader @@ -440,20 +440,21 @@ impl RdfXmlReader { &mut self, event: Event<'_>, results: &mut Vec, - ) -> Result<(), ParseError> { + ) -> Result<(), RdfXmlParseError> { match event { Event::Start(event) => self.parse_start_event(&event, results), Event::End(event) => self.parse_end_event(&event, results), - Event::Empty(_) => { - Err(SyntaxError::msg("The expand_empty_elements option must be enabled").into()) - } + Event::Empty(_) => Err(RdfXmlSyntaxError::msg( + "The expand_empty_elements option must be enabled", + ) + .into()), Event::Text(event) => self.parse_text_event(&event), Event::CData(event) => self.parse_text_event(&event.escape()?), Event::Comment(_) | Event::PI(_) => Ok(()), Event::Decl(decl) => { if let Some(encoding) = decl.encoding() { if !is_utf8(&encoding?) { - return Err(SyntaxError::msg( + return Err(RdfXmlSyntaxError::msg( "Only UTF-8 is supported by the RDF/XML parser", ) .into()); @@ -469,7 +470,7 @@ impl RdfXmlReader { } } - fn parse_doctype(&mut self, dt: &BytesText<'_>) -> Result<(), ParseError> { + fn parse_doctype(&mut self, dt: &BytesText<'_>) -> Result<(), RdfXmlParseError> { // we extract entities for input in self .reader @@ -481,20 +482,20 @@ impl RdfXmlReader { if let Some(input) = input.strip_prefix("!ENTITY") { let input = input.trim_start().strip_prefix('%').unwrap_or(input); let (entity_name, input) = input.trim_start().split_once(|c: char| c.is_ascii_whitespace()).ok_or_else(|| { - SyntaxError::msg( + RdfXmlSyntaxError::msg( "').ok_or_else(|| { - SyntaxError::msg("") + RdfXmlSyntaxError::msg("") })?; // Resolves custom entities within the current entity definition. @@ -511,7 +512,7 @@ impl RdfXmlReader { &mut self, event: &BytesStart<'_>, results: &mut Vec, - ) -> Result<(), ParseError> { + ) -> Result<(), RdfXmlParseError> { #[derive(PartialEq, Eq)] enum RdfXmlParseType { Default, @@ -576,7 +577,10 @@ impl RdfXmlReader { } else { LanguageTag::parse(tag.to_ascii_lowercase()) .map_err(|error| { - SyntaxError(SyntaxErrorKind::InvalidLanguageTag { tag, error }) + RdfXmlSyntaxError(SyntaxErrorKind::InvalidLanguageTag { + tag, + error, + }) })? .into_inner() }); @@ -588,7 +592,9 @@ impl RdfXmlReader { } else { Iri::parse(iri.clone()) } - .map_err(|error| SyntaxError(SyntaxErrorKind::InvalidIri { iri, error }))?, + .map_err(|error| { + RdfXmlSyntaxError(SyntaxErrorKind::InvalidIri { iri, error }) + })?, ) } else { // We ignore other xml attributes @@ -598,16 +604,17 @@ impl RdfXmlReader { if *attribute_url == *RDF_ID { let mut id = self.convert_attribute(&attribute)?; if !is_nc_name(&id) { - return Err( - SyntaxError::msg(format!("{id} is not a valid rdf:ID value")).into(), - ); + return Err(RdfXmlSyntaxError::msg(format!( + "{id} is not a valid rdf:ID value" + )) + .into()); } id.insert(0, '#'); id_attr = Some(id); } else if *attribute_url == *RDF_BAG_ID { let bag_id = self.convert_attribute(&attribute)?; if !is_nc_name(&bag_id) { - return Err(SyntaxError::msg(format!( + return Err(RdfXmlSyntaxError::msg(format!( "{bag_id} is not a valid rdf:bagID value" )) .into()); @@ -615,7 +622,7 @@ impl RdfXmlReader { } else if *attribute_url == *RDF_NODE_ID { let id = self.convert_attribute(&attribute)?; if !is_nc_name(&id) { - return Err(SyntaxError::msg(format!( + return Err(RdfXmlSyntaxError::msg(format!( "{id} is not a valid rdf:nodeID value" )) .into()); @@ -637,7 +644,7 @@ impl RdfXmlReader { } else if attribute_url == rdf::TYPE.as_str() { type_attr = Some(attribute); } else if RESERVED_RDF_ATTRIBUTES.contains(&&*attribute_url) { - return Err(SyntaxError::msg(format!( + return Err(RdfXmlSyntaxError::msg(format!( "{attribute_url} is not a valid attribute" )) .into()); @@ -655,7 +662,7 @@ impl RdfXmlReader { Some(iri) => { let iri = self.resolve_iri(&base_iri, iri)?; if self.known_rdf_id.contains(iri.as_str()) { - return Err(SyntaxError::msg(format!( + return Err(RdfXmlSyntaxError::msg(format!( "{iri} has already been used as rdf:ID value" )) .into()); @@ -694,13 +701,14 @@ impl RdfXmlReader { }, Some(RdfXmlState::ParseTypeLiteralPropertyElt { .. }) => { return Err( - SyntaxError::msg("ParseTypeLiteralPropertyElt production children should never be considered as a RDF/XML content").into() + RdfXmlSyntaxError::msg("ParseTypeLiteralPropertyElt production children should never be considered as a RDF/XML content").into() ); } None => { - return Err( - SyntaxError::msg("No state in the stack: the XML is not balanced").into(), - ); + return Err(RdfXmlSyntaxError::msg( + "No state in the stack: the XML is not balanced", + ) + .into()); } }; @@ -709,7 +717,7 @@ impl RdfXmlReader { if *tag_name == *RDF_RDF { RdfXmlState::Rdf { base_iri, language } } else if RESERVED_RDF_ELEMENTS.contains(&&*tag_name) { - return Err(SyntaxError::msg(format!( + return Err(RdfXmlSyntaxError::msg(format!( "Invalid node element tag name: {tag_name}" )) .into()); @@ -729,7 +737,7 @@ impl RdfXmlReader { } RdfXmlNextProduction::NodeElt => { if RESERVED_RDF_ELEMENTS.contains(&&*tag_name) { - return Err(SyntaxError::msg(format!( + return Err(RdfXmlSyntaxError::msg(format!( "Invalid property element tag name: {tag_name}" )) .into()); @@ -750,7 +758,7 @@ impl RdfXmlReader { let iri = if *tag_name == *RDF_LI { let Some(RdfXmlState::NodeElt { li_counter, .. }) = self.state.last_mut() else { - return Err(SyntaxError::msg(format!( + return Err(RdfXmlSyntaxError::msg(format!( "Invalid property element tag name: {tag_name}" )) .into()); @@ -762,7 +770,7 @@ impl RdfXmlReader { } else if RESERVED_RDF_ELEMENTS.contains(&&*tag_name) || *tag_name == *RDF_DESCRIPTION { - return Err(SyntaxError::msg(format!( + return Err(RdfXmlSyntaxError::msg(format!( "Invalid property element tag name: {tag_name}" )) .into()); @@ -780,7 +788,7 @@ impl RdfXmlReader { (Some(resource_attr), None) => Subject::from(resource_attr), (None, Some(node_id_attr)) => node_id_attr.into(), (None, None) => BlankNode::default().into(), - (Some(_), Some(_)) => return Err(SyntaxError::msg("Not both rdf:resource and rdf:nodeID could be set at the same time").into()) + (Some(_), Some(_)) => return Err(RdfXmlSyntaxError::msg("Not both rdf:resource and rdf:nodeID could be set at the same time").into()) }; Self::emit_property_attrs(&object, property_attrs, &language, results); if let Some(type_attr) = type_attr { @@ -847,7 +855,7 @@ impl RdfXmlReader { &mut self, event: &BytesEnd<'_>, results: &mut Vec, - ) -> Result<(), ParseError> { + ) -> Result<(), RdfXmlParseError> { // Literal case if self.in_literal_depth > 0 { if let Some(RdfXmlState::ParseTypeLiteralPropertyElt { writer, .. }) = @@ -867,7 +875,7 @@ impl RdfXmlReader { Ok(()) } - fn parse_text_event(&mut self, event: &BytesText<'_>) -> Result<(), ParseError> { + fn parse_text_event(&mut self, event: &BytesText<'_>) -> Result<(), RdfXmlParseError> { let text = event.unescape_with(|e| self.resolve_entity(e))?.to_string(); match self.state.last_mut() { Some(RdfXmlState::PropertyElt { object, .. }) => { @@ -884,18 +892,18 @@ impl RdfXmlReader { if event.iter().copied().all(is_whitespace) { Ok(()) } else { - Err(SyntaxError::msg(format!("Unexpected text event: '{text}'")).into()) + Err(RdfXmlSyntaxError::msg(format!("Unexpected text event: '{text}'")).into()) } } } } - fn resolve_tag_name(&self, qname: QName<'_>) -> Result { + fn resolve_tag_name(&self, qname: QName<'_>) -> Result { let (namespace, local_name) = self.reader.resolve_element(qname); self.resolve_ns_name(namespace, local_name) } - fn resolve_attribute_name(&self, qname: QName<'_>) -> Result { + fn resolve_attribute_name(&self, qname: QName<'_>) -> Result { let (namespace, local_name) = self.reader.resolve_attribute(qname); self.resolve_ns_name(namespace, local_name) } @@ -904,7 +912,7 @@ impl RdfXmlReader { &self, namespace: ResolveResult<'_>, local_name: LocalName<'_>, - ) -> Result { + ) -> Result { match namespace { ResolveResult::Bound(ns) => { let mut value = Vec::with_capacity(ns.as_ref().len() + local_name.as_ref().len()); @@ -917,9 +925,9 @@ impl RdfXmlReader { .to_string()) } ResolveResult::Unbound => { - Err(SyntaxError::msg("XML namespaces are required in RDF/XML").into()) + Err(RdfXmlSyntaxError::msg("XML namespaces are required in RDF/XML").into()) } - ResolveResult::Unknown(v) => Err(SyntaxError::msg(format!( + ResolveResult::Unknown(v) => Err(RdfXmlSyntaxError::msg(format!( "Unknown prefix {}:", self.reader.decoder().decode(&v)? )) @@ -938,24 +946,24 @@ impl RdfXmlReader { type_attr: Option, property_attrs: Vec<(NamedNode, String)>, results: &mut Vec, - ) -> Result { + ) -> Result { let subject = match (id_attr, node_id_attr, about_attr) { (Some(id_attr), None, None) => Subject::from(id_attr), (None, Some(node_id_attr), None) => node_id_attr.into(), (None, None, Some(about_attr)) => about_attr.into(), (None, None, None) => BlankNode::default().into(), (Some(_), Some(_), _) => { - return Err(SyntaxError::msg( + return Err(RdfXmlSyntaxError::msg( "Not both rdf:ID and rdf:nodeID could be set at the same time", )) } (_, Some(_), Some(_)) => { - return Err(SyntaxError::msg( + return Err(RdfXmlSyntaxError::msg( "Not both rdf:nodeID and rdf:resource could be set at the same time", )) } (Some(_), _, Some(_)) => { - return Err(SyntaxError::msg( + return Err(RdfXmlSyntaxError::msg( "Not both rdf:ID and rdf:resource could be set at the same time", )) } @@ -1004,7 +1012,7 @@ impl RdfXmlReader { &mut self, state: RdfXmlState, results: &mut Vec, - ) -> Result<(), SyntaxError> { + ) -> Result<(), RdfXmlSyntaxError> { match state { RdfXmlState::PropertyElt { iri, @@ -1059,7 +1067,7 @@ impl RdfXmlReader { if emit { let object = writer.into_inner(); if object.is_empty() { - return Err(SyntaxError::msg(format!( + return Err(RdfXmlSyntaxError::msg(format!( "No value found for rdf:XMLLiteral value of property {iri}" ))); } @@ -1068,7 +1076,9 @@ impl RdfXmlReader { iri, Literal::new_typed_literal( str::from_utf8(&object).map_err(|_| { - SyntaxError::msg("The XML literal is not in valid UTF-8".to_owned()) + RdfXmlSyntaxError::msg( + "The XML literal is not in valid UTF-8".to_owned(), + ) })?, rdf::XML_LITERAL, ), @@ -1141,7 +1151,7 @@ impl RdfXmlReader { } } - fn convert_attribute(&self, attribute: &Attribute<'_>) -> Result { + fn convert_attribute(&self, attribute: &Attribute<'_>) -> Result { Ok(attribute .decode_and_unescape_value_with(&self.reader, |e| self.resolve_entity(e))? .into_owned()) @@ -1151,7 +1161,7 @@ impl RdfXmlReader { &self, base_iri: &Option>, attribute: &Attribute<'_>, - ) -> Result { + ) -> Result { Ok(self.resolve_iri(base_iri, self.convert_attribute(attribute)?)?) } @@ -1159,7 +1169,7 @@ impl RdfXmlReader { &self, base_iri: &Option>, relative_iri: String, - ) -> Result { + ) -> Result { if let Some(base_iri) = base_iri { Ok(NamedNode::new_unchecked( if self.unchecked { @@ -1168,7 +1178,7 @@ impl RdfXmlReader { base_iri.resolve(&relative_iri) } .map_err(|error| { - SyntaxError(SyntaxErrorKind::InvalidIri { + RdfXmlSyntaxError(SyntaxErrorKind::InvalidIri { iri: relative_iri, error, }) @@ -1180,13 +1190,13 @@ impl RdfXmlReader { } } - fn parse_iri(&self, relative_iri: String) -> Result { + fn parse_iri(&self, relative_iri: String) -> Result { Ok(NamedNode::new_unchecked(if self.unchecked { relative_iri } else { Iri::parse(relative_iri.clone()) .map_err(|error| { - SyntaxError(SyntaxErrorKind::InvalidIri { + RdfXmlSyntaxError(SyntaxErrorKind::InvalidIri { iri: relative_iri, error, }) diff --git a/lib/oxttl/Cargo.toml b/lib/oxttl/Cargo.toml index 0a4bc3ab..c86150ed 100644 --- a/lib/oxttl/Cargo.toml +++ b/lib/oxttl/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "oxttl" -version = "0.1.0-alpha.2" +version = "0.1.0-alpha.3-dev" authors.workspace = true license.workspace = true readme = "README.md" diff --git a/lib/oxttl/src/lib.rs b/lib/oxttl/src/lib.rs index 0e04e243..a1dd95f9 100644 --- a/lib/oxttl/src/lib.rs +++ b/lib/oxttl/src/lib.rs @@ -17,7 +17,7 @@ pub mod turtle; pub use crate::n3::N3Parser; pub use crate::nquads::{NQuadsParser, NQuadsSerializer}; pub use crate::ntriples::{NTriplesParser, NTriplesSerializer}; -pub use crate::toolkit::{ParseError, SyntaxError, TextPosition}; +pub use crate::toolkit::{TextPosition, TurtleParseError, TurtleSyntaxError}; pub use crate::trig::{TriGParser, TriGSerializer}; pub use crate::turtle::{TurtleParser, TurtleSerializer}; diff --git a/lib/oxttl/src/n3.rs b/lib/oxttl/src/n3.rs index 8b70a01e..de3e9363 100644 --- a/lib/oxttl/src/n3.rs +++ b/lib/oxttl/src/n3.rs @@ -4,9 +4,9 @@ use crate::lexer::{resolve_local_name, N3Lexer, N3LexerMode, N3LexerOptions, N3T #[cfg(feature = "async-tokio")] use crate::toolkit::FromTokioAsyncReadIterator; use crate::toolkit::{ - FromReadIterator, Lexer, Parser, RuleRecognizer, RuleRecognizerError, SyntaxError, + FromReadIterator, Lexer, Parser, RuleRecognizer, RuleRecognizerError, TurtleSyntaxError, }; -use crate::{ParseError, MAX_BUFFER_SIZE, MIN_BUFFER_SIZE}; +use crate::{TurtleParseError, MAX_BUFFER_SIZE, MIN_BUFFER_SIZE}; use oxiri::{Iri, IriParseError}; use oxrdf::vocab::{rdf, xsd}; #[cfg(feature = "rdf-star")] @@ -291,7 +291,7 @@ impl N3Parser { /// use oxttl::n3::{N3Parser, N3Term}; /// /// # #[tokio::main(flavor = "current_thread")] - /// # async fn main() -> Result<(), oxttl::ParseError> { + /// # async fn main() -> Result<(), oxttl::TurtleParseError> { /// let file = br#"@base . /// @prefix schema: . /// a schema:Person ; @@ -461,7 +461,7 @@ impl FromReadN3Reader { } impl Iterator for FromReadN3Reader { - type Item = Result; + type Item = Result; fn next(&mut self) -> Option { self.inner.next() @@ -477,7 +477,7 @@ impl Iterator for FromReadN3Reader { /// use oxttl::n3::{N3Parser, N3Term}; /// /// # #[tokio::main(flavor = "current_thread")] -/// # async fn main() -> Result<(), oxttl::ParseError> { +/// # async fn main() -> Result<(), oxttl::TurtleParseError> { /// let file = br#"@base . /// @prefix schema: . /// a schema:Person ; @@ -508,7 +508,7 @@ pub struct FromTokioAsyncReadN3Reader { #[cfg(feature = "async-tokio")] impl FromTokioAsyncReadN3Reader { /// Reads the next triple or returns `None` if the file is finished. - pub async fn next(&mut self) -> Option> { + pub async fn next(&mut self) -> Option> { Some(self.inner.next().await?.map(Into::into)) } @@ -522,7 +522,7 @@ impl FromTokioAsyncReadN3Reader { /// use oxttl::N3Parser; /// /// # #[tokio::main(flavor = "current_thread")] - /// # async fn main() -> Result<(), oxttl::ParseError> { + /// # async fn main() -> Result<(), oxttl::TurtleParseError> { /// let file = br#"@base . /// @prefix schema: . /// a schema:Person ; @@ -551,7 +551,7 @@ impl FromTokioAsyncReadN3Reader { /// use oxttl::N3Parser; /// /// # #[tokio::main(flavor = "current_thread")] - /// # async fn main() -> Result<(), oxttl::ParseError> { + /// # async fn main() -> Result<(), oxttl::TurtleParseError> { /// let file = br#"@base . /// @prefix schema: . /// a schema:Person ; @@ -641,7 +641,7 @@ impl LowLevelN3Reader { /// /// Returns [`None`] if the parsing is finished or more data is required. /// If it is the case more data should be fed using [`extend_from_slice`](Self::extend_from_slice). - pub fn read_next(&mut self) -> Option> { + pub fn read_next(&mut self) -> Option> { self.parser.read_next() } diff --git a/lib/oxttl/src/nquads.rs b/lib/oxttl/src/nquads.rs index 0ae22119..9f48bc76 100644 --- a/lib/oxttl/src/nquads.rs +++ b/lib/oxttl/src/nquads.rs @@ -4,7 +4,7 @@ use crate::line_formats::NQuadsRecognizer; #[cfg(feature = "async-tokio")] use crate::toolkit::FromTokioAsyncReadIterator; -use crate::toolkit::{FromReadIterator, ParseError, Parser, SyntaxError}; +use crate::toolkit::{FromReadIterator, Parser, TurtleParseError, TurtleSyntaxError}; use oxrdf::{Quad, QuadRef}; use std::io::{self, Read, Write}; #[cfg(feature = "async-tokio")] @@ -106,7 +106,7 @@ impl NQuadsParser { /// use oxttl::NQuadsParser; /// /// # #[tokio::main(flavor = "current_thread")] - /// # async fn main() -> Result<(), oxttl::ParseError> { + /// # async fn main() -> Result<(), oxttl::TurtleParseError> { /// let file = br#" . /// "Foo" . /// . @@ -213,7 +213,7 @@ pub struct FromReadNQuadsReader { } impl Iterator for FromReadNQuadsReader { - type Item = Result; + type Item = Result; fn next(&mut self) -> Option { self.inner.next() @@ -228,7 +228,7 @@ impl Iterator for FromReadNQuadsReader { /// use oxttl::NQuadsParser; /// /// # #[tokio::main(flavor = "current_thread")] -/// # async fn main() -> Result<(), oxttl::ParseError> { +/// # async fn main() -> Result<(), oxttl::TurtleParseError> { /// let file = br#" . /// "Foo" . /// . @@ -256,7 +256,7 @@ pub struct FromTokioAsyncReadNQuadsReader { #[cfg(feature = "async-tokio")] impl FromTokioAsyncReadNQuadsReader { /// Reads the next triple or returns `None` if the file is finished. - pub async fn next(&mut self) -> Option> { + pub async fn next(&mut self) -> Option> { Some(self.inner.next().await?.map(Into::into)) } } @@ -323,7 +323,7 @@ impl LowLevelNQuadsReader { /// /// Returns [`None`] if the parsing is finished or more data is required. /// If it is the case more data should be fed using [`extend_from_slice`](Self::extend_from_slice). - pub fn read_next(&mut self) -> Option> { + pub fn read_next(&mut self) -> Option> { self.parser.read_next() } } diff --git a/lib/oxttl/src/ntriples.rs b/lib/oxttl/src/ntriples.rs index 686907dc..ca033959 100644 --- a/lib/oxttl/src/ntriples.rs +++ b/lib/oxttl/src/ntriples.rs @@ -4,7 +4,7 @@ use crate::line_formats::NQuadsRecognizer; #[cfg(feature = "async-tokio")] use crate::toolkit::FromTokioAsyncReadIterator; -use crate::toolkit::{FromReadIterator, ParseError, Parser, SyntaxError}; +use crate::toolkit::{FromReadIterator, Parser, TurtleParseError, TurtleSyntaxError}; use oxrdf::{Triple, TripleRef}; use std::io::{self, Read, Write}; #[cfg(feature = "async-tokio")] @@ -106,7 +106,7 @@ impl NTriplesParser { /// use oxttl::NTriplesParser; /// /// # #[tokio::main(flavor = "current_thread")] - /// # async fn main() -> Result<(), oxttl::ParseError> { + /// # async fn main() -> Result<(), oxttl::TurtleParseError> { /// let file = br#" . /// "Foo" . /// . @@ -213,7 +213,7 @@ pub struct FromReadNTriplesReader { } impl Iterator for FromReadNTriplesReader { - type Item = Result; + type Item = Result; fn next(&mut self) -> Option { Some(self.inner.next()?.map(Into::into)) @@ -228,7 +228,7 @@ impl Iterator for FromReadNTriplesReader { /// use oxttl::NTriplesParser; /// /// # #[tokio::main(flavor = "current_thread")] -/// # async fn main() -> Result<(), oxttl::ParseError> { +/// # async fn main() -> Result<(), oxttl::TurtleParseError> { /// let file = br#" . /// "Foo" . /// . @@ -256,7 +256,7 @@ pub struct FromTokioAsyncReadNTriplesReader { #[cfg(feature = "async-tokio")] impl FromTokioAsyncReadNTriplesReader { /// Reads the next triple or returns `None` if the file is finished. - pub async fn next(&mut self) -> Option> { + pub async fn next(&mut self) -> Option> { Some(self.inner.next().await?.map(Into::into)) } } @@ -323,7 +323,7 @@ impl LowLevelNTriplesReader { /// /// Returns [`None`] if the parsing is finished or more data is required. /// If it is the case more data should be fed using [`extend_from_slice`](Self::extend_from_slice). - pub fn read_next(&mut self) -> Option> { + pub fn read_next(&mut self) -> Option> { Some(self.parser.read_next()?.map(Into::into)) } } diff --git a/lib/oxttl/src/toolkit/error.rs b/lib/oxttl/src/toolkit/error.rs index 33f2a916..083adefa 100644 --- a/lib/oxttl/src/toolkit/error.rs +++ b/lib/oxttl/src/toolkit/error.rs @@ -13,12 +13,12 @@ pub struct TextPosition { /// /// It is composed of a message and a byte range in the input. #[derive(Debug, thiserror::Error)] -pub struct SyntaxError { +pub struct TurtleSyntaxError { pub(super) location: Range, pub(super) message: String, } -impl SyntaxError { +impl TurtleSyntaxError { /// The location of the error inside of the file. #[inline] pub fn location(&self) -> Range { @@ -32,7 +32,7 @@ impl SyntaxError { } } -impl fmt::Display for SyntaxError { +impl fmt::Display for TurtleSyntaxError { #[inline] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { if self.location.start.offset + 1 >= self.location.end.offset { @@ -66,32 +66,32 @@ impl fmt::Display for SyntaxError { } } -impl From for io::Error { +impl From for io::Error { #[inline] - fn from(error: SyntaxError) -> Self { + fn from(error: TurtleSyntaxError) -> Self { Self::new(io::ErrorKind::InvalidData, error) } } /// A parsing error. /// -/// It is the union of [`SyntaxError`] and [`io::Error`]. +/// It is the union of [`TurtleSyntaxError`] and [`io::Error`]. #[derive(Debug, thiserror::Error)] -pub enum ParseError { +pub enum TurtleParseError { /// I/O error during parsing (file not found...). #[error(transparent)] Io(#[from] io::Error), /// An error in the file syntax. #[error(transparent)] - Syntax(#[from] SyntaxError), + Syntax(#[from] TurtleSyntaxError), } -impl From for io::Error { +impl From for io::Error { #[inline] - fn from(error: ParseError) -> Self { + fn from(error: TurtleParseError) -> Self { match error { - ParseError::Syntax(e) => e.into(), - ParseError::Io(e) => e, + TurtleParseError::Syntax(e) => e.into(), + TurtleParseError::Io(e) => e, } } } diff --git a/lib/oxttl/src/toolkit/lexer.rs b/lib/oxttl/src/toolkit/lexer.rs index 2406dfb1..4db73fae 100644 --- a/lib/oxttl/src/toolkit/lexer.rs +++ b/lib/oxttl/src/toolkit/lexer.rs @@ -1,4 +1,4 @@ -use crate::toolkit::error::{SyntaxError, TextPosition}; +use crate::toolkit::error::{TextPosition, TurtleSyntaxError}; use memchr::{memchr2, memchr2_iter}; use std::borrow::Cow; use std::cmp::min; @@ -163,7 +163,10 @@ impl Lexer { } #[allow(clippy::unwrap_in_result)] - pub fn read_next(&mut self, options: &R::Options) -> Option, SyntaxError>> { + pub fn read_next( + &mut self, + options: &R::Options, + ) -> Option, TurtleSyntaxError>> { self.skip_whitespaces_and_comments()?; self.previous_position = self.position; let Some((consumed, result)) = self.parser.recognize_next_token( @@ -194,7 +197,7 @@ impl Lexer { ), offset: self.position.global_offset, }; - let error = SyntaxError { + let error = TurtleSyntaxError { location: new_position..new_position, message: "Unexpected end of file".into(), }; @@ -224,7 +227,7 @@ impl Lexer { self.position.buffer_offset += consumed; self.position.global_offset += u64::try_from(consumed).unwrap(); self.position.global_line += new_line_jumps; - Some(result.map_err(|e| SyntaxError { + Some(result.map_err(|e| TurtleSyntaxError { location: self.location_from_buffer_offset_range(e.location), message: e.message, })) diff --git a/lib/oxttl/src/toolkit/mod.rs b/lib/oxttl/src/toolkit/mod.rs index cc8e3624..10c42163 100644 --- a/lib/oxttl/src/toolkit/mod.rs +++ b/lib/oxttl/src/toolkit/mod.rs @@ -6,7 +6,7 @@ mod error; mod lexer; mod parser; -pub use self::error::{ParseError, SyntaxError, TextPosition}; +pub use self::error::{TextPosition, TurtleParseError, TurtleSyntaxError}; pub use self::lexer::{Lexer, TokenRecognizer, TokenRecognizerError}; #[cfg(feature = "async-tokio")] pub use self::parser::FromTokioAsyncReadIterator; diff --git a/lib/oxttl/src/toolkit/parser.rs b/lib/oxttl/src/toolkit/parser.rs index 6314640d..6ac8a1ac 100644 --- a/lib/oxttl/src/toolkit/parser.rs +++ b/lib/oxttl/src/toolkit/parser.rs @@ -1,4 +1,4 @@ -use crate::toolkit::error::{ParseError, SyntaxError}; +use crate::toolkit::error::{TurtleParseError, TurtleSyntaxError}; use crate::toolkit::lexer::{Lexer, TokenRecognizer}; use std::io::Read; #[cfg(feature = "async-tokio")] @@ -77,10 +77,10 @@ impl Parser { self.state.is_none() && self.results.is_empty() && self.errors.is_empty() } - pub fn read_next(&mut self) -> Option> { + pub fn read_next(&mut self) -> Option> { loop { if let Some(error) = self.errors.pop() { - return Some(Err(SyntaxError { + return Some(Err(TurtleSyntaxError { location: self.lexer.last_token_location(), message: error .message @@ -141,12 +141,12 @@ pub struct FromReadIterator { } impl Iterator for FromReadIterator { - type Item = Result; + type Item = Result; fn next(&mut self) -> Option { while !self.parser.is_end() { if let Some(result) = self.parser.read_next() { - return Some(result.map_err(ParseError::Syntax)); + return Some(result.map_err(TurtleParseError::Syntax)); } if let Err(e) = self.parser.lexer.extend_from_read(&mut self.read) { return Some(Err(e.into())); @@ -164,10 +164,10 @@ pub struct FromTokioAsyncReadIterator #[cfg(feature = "async-tokio")] impl FromTokioAsyncReadIterator { - pub async fn next(&mut self) -> Option> { + pub async fn next(&mut self) -> Option> { while !self.parser.is_end() { if let Some(result) = self.parser.read_next() { - return Some(result.map_err(ParseError::Syntax)); + return Some(result.map_err(TurtleParseError::Syntax)); } if let Err(e) = self .parser diff --git a/lib/oxttl/src/trig.rs b/lib/oxttl/src/trig.rs index 21434ed4..e230b9be 100644 --- a/lib/oxttl/src/trig.rs +++ b/lib/oxttl/src/trig.rs @@ -5,7 +5,7 @@ use crate::lexer::N3Lexer; use crate::terse::TriGRecognizer; #[cfg(feature = "async-tokio")] use crate::toolkit::FromTokioAsyncReadIterator; -use crate::toolkit::{FromReadIterator, ParseError, Parser, SyntaxError}; +use crate::toolkit::{FromReadIterator, Parser, TurtleParseError, TurtleSyntaxError}; use oxiri::{Iri, IriParseError}; use oxrdf::vocab::{rdf, xsd}; use oxrdf::{ @@ -140,7 +140,7 @@ impl TriGParser { /// use oxttl::TriGParser; /// /// # #[tokio::main(flavor = "current_thread")] - /// # async fn main() -> Result<(), oxttl::ParseError> { + /// # async fn main() -> Result<(), oxttl::TurtleParseError> { /// let file = br#"@base . /// @prefix schema: . /// a schema:Person ; @@ -314,7 +314,7 @@ impl FromReadTriGReader { } impl Iterator for FromReadTriGReader { - type Item = Result; + type Item = Result; fn next(&mut self) -> Option { self.inner.next() @@ -330,7 +330,7 @@ impl Iterator for FromReadTriGReader { /// use oxttl::TriGParser; /// /// # #[tokio::main(flavor = "current_thread")] -/// # async fn main() -> Result<(), oxttl::ParseError> { +/// # async fn main() -> Result<(), oxttl::TurtleParseError> { /// let file = br#"@base . /// @prefix schema: . /// a schema:Person ; @@ -360,7 +360,7 @@ pub struct FromTokioAsyncReadTriGReader { #[cfg(feature = "async-tokio")] impl FromTokioAsyncReadTriGReader { /// Reads the next triple or returns `None` if the file is finished. - pub async fn next(&mut self) -> Option> { + pub async fn next(&mut self) -> Option> { Some(self.inner.next().await?.map(Into::into)) } @@ -374,7 +374,7 @@ impl FromTokioAsyncReadTriGReader { /// use oxttl::TriGParser; /// /// # #[tokio::main(flavor = "current_thread")] - /// # async fn main() -> Result<(), oxttl::ParseError> { + /// # async fn main() -> Result<(), oxttl::TurtleParseError> { /// let file = br#"@base . /// @prefix schema: . /// a schema:Person ; @@ -403,7 +403,7 @@ impl FromTokioAsyncReadTriGReader { /// use oxttl::TriGParser; /// /// # #[tokio::main(flavor = "current_thread")] - /// # async fn main() -> Result<(), oxttl::ParseError> { + /// # async fn main() -> Result<(), oxttl::TurtleParseError> { /// let file = br#"@base . /// @prefix schema: . /// a schema:Person ; @@ -492,7 +492,7 @@ impl LowLevelTriGReader { /// /// Returns [`None`] if the parsing is finished or more data is required. /// If it is the case more data should be fed using [`extend_from_slice`](Self::extend_from_slice). - pub fn read_next(&mut self) -> Option> { + pub fn read_next(&mut self) -> Option> { self.parser.read_next() } diff --git a/lib/oxttl/src/turtle.rs b/lib/oxttl/src/turtle.rs index 0cc9fd77..eb972224 100644 --- a/lib/oxttl/src/turtle.rs +++ b/lib/oxttl/src/turtle.rs @@ -4,7 +4,7 @@ use crate::terse::TriGRecognizer; #[cfg(feature = "async-tokio")] use crate::toolkit::FromTokioAsyncReadIterator; -use crate::toolkit::{FromReadIterator, ParseError, Parser, SyntaxError}; +use crate::toolkit::{FromReadIterator, Parser, TurtleParseError, TurtleSyntaxError}; #[cfg(feature = "async-tokio")] use crate::trig::ToTokioAsyncWriteTriGWriter; use crate::trig::{LowLevelTriGWriter, ToWriteTriGWriter, TriGSerializer}; @@ -138,7 +138,7 @@ impl TurtleParser { /// use oxttl::TurtleParser; /// /// # #[tokio::main(flavor = "current_thread")] - /// # async fn main() -> Result<(), oxttl::ParseError> { + /// # async fn main() -> Result<(), oxttl::TurtleParseError> { /// let file = br#"@base . /// @prefix schema: . /// a schema:Person ; @@ -312,7 +312,7 @@ impl FromReadTurtleReader { } impl Iterator for FromReadTurtleReader { - type Item = Result; + type Item = Result; fn next(&mut self) -> Option { Some(self.inner.next()?.map(Into::into)) @@ -328,7 +328,7 @@ impl Iterator for FromReadTurtleReader { /// use oxttl::TurtleParser; /// /// # #[tokio::main(flavor = "current_thread")] -/// # async fn main() -> Result<(), oxttl::ParseError> { +/// # async fn main() -> Result<(), oxttl::TurtleParseError> { /// let file = br#"@base . /// @prefix schema: . /// a schema:Person ; @@ -358,7 +358,7 @@ pub struct FromTokioAsyncReadTurtleReader { #[cfg(feature = "async-tokio")] impl FromTokioAsyncReadTurtleReader { /// Reads the next triple or returns `None` if the file is finished. - pub async fn next(&mut self) -> Option> { + pub async fn next(&mut self) -> Option> { Some(self.inner.next().await?.map(Into::into)) } @@ -372,7 +372,7 @@ impl FromTokioAsyncReadTurtleReader { /// use oxttl::TurtleParser; /// /// # #[tokio::main(flavor = "current_thread")] - /// # async fn main() -> Result<(), oxttl::ParseError> { + /// # async fn main() -> Result<(), oxttl::TurtleParseError> { /// let file = br#"@base . /// @prefix schema: . /// a schema:Person ; @@ -401,7 +401,7 @@ impl FromTokioAsyncReadTurtleReader { /// use oxttl::TurtleParser; /// /// # #[tokio::main(flavor = "current_thread")] - /// # async fn main() -> Result<(), oxttl::ParseError> { + /// # async fn main() -> Result<(), oxttl::TurtleParseError> { /// let file = br#"@base . /// @prefix schema: . /// a schema:Person ; @@ -490,7 +490,7 @@ impl LowLevelTurtleReader { /// /// Returns [`None`] if the parsing is finished or more data is required. /// If it is the case more data should be fed using [`extend_from_slice`](Self::extend_from_slice). - pub fn read_next(&mut self) -> Option> { + pub fn read_next(&mut self) -> Option> { Some(self.parser.read_next()?.map(Into::into)) } diff --git a/lib/sparesults/src/csv.rs b/lib/sparesults/src/csv.rs index bd2fe4b1..02f4df9b 100644 --- a/lib/sparesults/src/csv.rs +++ b/lib/sparesults/src/csv.rs @@ -1,6 +1,8 @@ //! Implementation of [SPARQL 1.1 Query Results CSV and TSV Formats](https://www.w3.org/TR/sparql11-results-csv-tsv/) -use crate::error::{ParseError, SyntaxError, SyntaxErrorKind, TextPosition}; +use crate::error::{ + QueryResultsParseError, QueryResultsSyntaxError, SyntaxErrorKind, TextPosition, +}; use memchr::memchr; use oxrdf::vocab::xsd; use oxrdf::*; @@ -432,7 +434,7 @@ pub enum TsvQueryResultsReader { } impl TsvQueryResultsReader { - pub fn read(mut read: R) -> Result { + pub fn read(mut read: R) -> Result { let mut reader = LineReader::new(); let mut buffer = Vec::new(); @@ -451,13 +453,13 @@ impl TsvQueryResultsReader { for v in line.split('\t') { let v = v.trim(); if v.is_empty() { - return Err(SyntaxError::msg("Empty column on the first row. The first row should be a list of variables like ?foo or $bar").into()); + return Err(QueryResultsSyntaxError::msg("Empty column on the first row. The first row should be a list of variables like ?foo or $bar").into()); } let variable = Variable::from_str(v).map_err(|e| { - SyntaxError::msg(format!("Invalid variable declaration '{v}': {e}")) + QueryResultsSyntaxError::msg(format!("Invalid variable declaration '{v}': {e}")) })?; if variables.contains(&variable) { - return Err(SyntaxError::msg(format!( + return Err(QueryResultsSyntaxError::msg(format!( "The variable {variable} is declared twice" )) .into()); @@ -487,7 +489,7 @@ pub struct TsvSolutionsReader { impl TsvSolutionsReader { #[allow(clippy::unwrap_in_result)] - pub fn read_next(&mut self) -> Result>>, ParseError> { + pub fn read_next(&mut self) -> Result>>, QueryResultsParseError> { let line = self.reader.next_line(&mut self.buffer, &mut self.read)?; if line.is_empty() { return Ok(None); // EOF @@ -508,35 +510,33 @@ impl TsvSolutionsReader { .sum::(); let start_position_bytes = line.split('\t').take(i).map(|c| c.len() + 1).sum::(); - SyntaxError { - inner: SyntaxErrorKind::Term { - error: e, - term: v.into(), - location: TextPosition { - line: self.reader.line_count - 1, - column: start_position_char.try_into().unwrap(), - offset: self.reader.last_line_start - + u64::try_from(start_position_bytes).unwrap(), - }..TextPosition { - line: self.reader.line_count - 1, - column: (start_position_char + v.chars().count()) - .try_into() - .unwrap(), - offset: self.reader.last_line_start - + u64::try_from(start_position_bytes + v.len()).unwrap(), - }, + QueryResultsSyntaxError(SyntaxErrorKind::Term { + error: e, + term: v.into(), + location: TextPosition { + line: self.reader.line_count - 1, + column: start_position_char.try_into().unwrap(), + offset: self.reader.last_line_start + + u64::try_from(start_position_bytes).unwrap(), + }..TextPosition { + line: self.reader.line_count - 1, + column: (start_position_char + v.chars().count()) + .try_into() + .unwrap(), + offset: self.reader.last_line_start + + u64::try_from(start_position_bytes + v.len()).unwrap(), }, - } + }) })?)) } }) - .collect::, ParseError>>()?; + .collect::, QueryResultsParseError>>()?; if elements.len() == self.column_len { Ok(Some(elements)) } else if self.column_len == 0 && elements == [None] { Ok(Some(Vec::new())) // Zero columns case } else { - Err(SyntaxError::located_message( + Err(QueryResultsSyntaxError::located_message( format!( "This TSV files has {} columns but we found a row on line {} with {} columns: {}", self.column_len, diff --git a/lib/sparesults/src/error.rs b/lib/sparesults/src/error.rs index cc5a1e4a..f26a3349 100644 --- a/lib/sparesults/src/error.rs +++ b/lib/sparesults/src/error.rs @@ -5,44 +5,44 @@ use std::sync::Arc; /// Error returned during SPARQL result formats format parsing. #[derive(Debug, thiserror::Error)] -pub enum ParseError { +pub enum QueryResultsParseError { /// I/O error during parsing (file not found...). #[error(transparent)] Io(#[from] io::Error), /// An error in the file syntax. #[error(transparent)] - Syntax(#[from] SyntaxError), + Syntax(#[from] QueryResultsSyntaxError), } -impl From for io::Error { +impl From for io::Error { #[inline] - fn from(error: ParseError) -> Self { + fn from(error: QueryResultsParseError) -> Self { match error { - ParseError::Io(error) => error, - ParseError::Syntax(error) => error.into(), + QueryResultsParseError::Io(error) => error, + QueryResultsParseError::Syntax(error) => error.into(), } } } -impl From for ParseError { +impl From for QueryResultsParseError { fn from(error: json_event_parser::ParseError) -> Self { match error { - json_event_parser::ParseError::Syntax(error) => SyntaxError::from(error).into(), + json_event_parser::ParseError::Syntax(error) => { + QueryResultsSyntaxError::from(error).into() + } json_event_parser::ParseError::Io(error) => error.into(), } } } -impl From for ParseError { +impl From for QueryResultsParseError { #[inline] fn from(error: quick_xml::Error) -> Self { match error { quick_xml::Error::Io(error) => { Self::Io(Arc::try_unwrap(error).unwrap_or_else(|e| io::Error::new(e.kind(), e))) } - _ => Self::Syntax(SyntaxError { - inner: SyntaxErrorKind::Xml(error), - }), + _ => Self::Syntax(QueryResultsSyntaxError(SyntaxErrorKind::Xml(error))), } } } @@ -50,10 +50,7 @@ impl From for ParseError { /// An error in the syntax of the parsed file. #[derive(Debug, thiserror::Error)] #[error(transparent)] -pub struct SyntaxError { - #[from] - pub(crate) inner: SyntaxErrorKind, -} +pub struct QueryResultsSyntaxError(#[from] pub(crate) SyntaxErrorKind); #[derive(Debug, thiserror::Error)] pub(crate) enum SyntaxErrorKind { @@ -75,33 +72,29 @@ pub(crate) enum SyntaxErrorKind { }, } -impl SyntaxError { +impl QueryResultsSyntaxError { /// Builds an error from a printable error message. #[inline] pub(crate) fn msg(msg: impl Into) -> Self { - Self { - inner: SyntaxErrorKind::Msg { - msg: msg.into(), - location: None, - }, - } + Self(SyntaxErrorKind::Msg { + msg: msg.into(), + location: None, + }) } /// Builds an error from a printable error message and a location #[inline] pub(crate) fn located_message(msg: impl Into, location: Range) -> Self { - Self { - inner: SyntaxErrorKind::Msg { - msg: msg.into(), - location: Some(location), - }, - } + Self(SyntaxErrorKind::Msg { + msg: msg.into(), + location: Some(location), + }) } /// The location of the error inside of the file. #[inline] pub fn location(&self) -> Option> { - match &self.inner { + match &self.0 { SyntaxErrorKind::Json(e) => { let location = e.location(); Some( @@ -123,10 +116,10 @@ impl SyntaxError { } } -impl From for io::Error { +impl From for io::Error { #[inline] - fn from(error: SyntaxError) -> Self { - match error.inner { + fn from(error: QueryResultsSyntaxError) -> Self { + match error.0 { SyntaxErrorKind::Json(error) => Self::new(io::ErrorKind::InvalidData, error), SyntaxErrorKind::Xml(error) => match error { quick_xml::Error::Io(error) => { @@ -143,11 +136,9 @@ impl From for io::Error { } } -impl From for SyntaxError { +impl From for QueryResultsSyntaxError { fn from(error: json_event_parser::SyntaxError) -> Self { - Self { - inner: SyntaxErrorKind::Json(error), - } + Self(SyntaxErrorKind::Json(error)) } } diff --git a/lib/sparesults/src/json.rs b/lib/sparesults/src/json.rs index 2e63fc81..25da2d64 100644 --- a/lib/sparesults/src/json.rs +++ b/lib/sparesults/src/json.rs @@ -1,6 +1,6 @@ //! Implementation of [SPARQL Query Results JSON Format](https://www.w3.org/TR/sparql11-results-json/) -use crate::error::{ParseError, SyntaxError}; +use crate::error::{QueryResultsParseError, QueryResultsSyntaxError}; #[cfg(feature = "async-tokio")] use json_event_parser::ToTokioAsyncWriteJsonWriter; use json_event_parser::{FromReadJsonReader, JsonEvent, ToWriteJsonWriter}; @@ -233,14 +233,16 @@ pub enum JsonQueryResultsReader { } impl JsonQueryResultsReader { - pub fn read(read: R) -> Result { + pub fn read(read: R) -> Result { let mut reader = FromReadJsonReader::new(read); let mut variables = None; let mut buffered_bindings: Option> = None; let mut output_iter = None; if reader.read_next_event()? != JsonEvent::StartObject { - return Err(SyntaxError::msg("SPARQL JSON results should be an object").into()); + return Err( + QueryResultsSyntaxError::msg("SPARQL JSON results should be an object").into(), + ); } loop { @@ -269,14 +271,17 @@ impl JsonQueryResultsReader { } "results" => { if reader.read_next_event()? != JsonEvent::StartObject { - return Err(SyntaxError::msg("'results' should be an object").into()); + return Err(QueryResultsSyntaxError::msg( + "'results' should be an object", + ) + .into()); } loop { match reader.read_next_event()? { JsonEvent::ObjectKey(k) if k == "bindings" => break, // Found JsonEvent::ObjectKey(_) => ignore_value(&mut reader)?, _ => { - return Err(SyntaxError::msg( + return Err(QueryResultsSyntaxError::msg( "'results' should contain a 'bindings' key", ) .into()) @@ -284,7 +289,10 @@ impl JsonQueryResultsReader { } } if reader.read_next_event()? != JsonEvent::StartArray { - return Err(SyntaxError::msg("'bindings' should be an object").into()); + return Err(QueryResultsSyntaxError::msg( + "'bindings' should be an object", + ) + .into()); } if let Some(variables) = variables { let mut mapping = BTreeMap::default(); @@ -318,9 +326,10 @@ impl JsonQueryResultsReader { values.push(read_value(&mut reader, 0)?); } _ => { - return Err( - SyntaxError::msg("Invalid result serialization").into() + return Err(QueryResultsSyntaxError::msg( + "Invalid result serialization", ) + .into()) } } } @@ -329,11 +338,11 @@ impl JsonQueryResultsReader { return if let JsonEvent::Boolean(v) = reader.read_next_event()? { Ok(Self::Boolean(v)) } else { - Err(SyntaxError::msg("Unexpected boolean value").into()) + Err(QueryResultsSyntaxError::msg("Unexpected boolean value").into()) } } _ => { - return Err(SyntaxError::msg(format!( + return Err(QueryResultsSyntaxError::msg(format!( "Expecting head or result key, found {key}" )) .into()); @@ -344,13 +353,18 @@ impl JsonQueryResultsReader { return if let Some(output_iter) = output_iter { Ok(output_iter) } else { - Err(SyntaxError::msg( + Err(QueryResultsSyntaxError::msg( "Unexpected end of JSON object without 'results' or 'boolean' key", ) .into()) } } - _ => return Err(SyntaxError::msg("Invalid SPARQL results serialization").into()), + _ => { + return Err(QueryResultsSyntaxError::msg( + "Invalid SPARQL results serialization", + ) + .into()) + } } } } @@ -371,7 +385,7 @@ enum JsonSolutionsReaderKind { } impl JsonSolutionsReader { - pub fn read_next(&mut self) -> Result>>, ParseError> { + pub fn read_next(&mut self) -> Result>>, QueryResultsParseError> { match &mut self.kind { JsonSolutionsReaderKind::Streaming { reader } => { let mut new_bindings = vec![None; self.mapping.len()]; @@ -382,13 +396,18 @@ impl JsonSolutionsReader { JsonEvent::EndArray | JsonEvent::Eof => return Ok(None), JsonEvent::ObjectKey(key) => { let k = *self.mapping.get(key.as_ref()).ok_or_else(|| { - SyntaxError::msg(format!( + QueryResultsSyntaxError::msg(format!( "The variable {key} has not been defined in the header" )) })?; new_bindings[k] = Some(read_value(reader, 0)?) } - _ => return Err(SyntaxError::msg("Invalid result serialization").into()), + _ => { + return Err(QueryResultsSyntaxError::msg( + "Invalid result serialization", + ) + .into()) + } } } } @@ -397,7 +416,7 @@ impl JsonSolutionsReader { let mut new_bindings = vec![None; self.mapping.len()]; for (variable, value) in variables.into_iter().zip(values) { let k = *self.mapping.get(&variable).ok_or_else(|| { - SyntaxError::msg(format!( + QueryResultsSyntaxError::msg(format!( "The variable {variable} has not been defined in the header" )) })?; @@ -415,7 +434,7 @@ impl JsonSolutionsReader { fn read_value( reader: &mut FromReadJsonReader, number_of_recursive_calls: usize, -) -> Result { +) -> Result { enum Type { Uri, BNode, @@ -432,7 +451,7 @@ fn read_value( } if number_of_recursive_calls == MAX_NUMBER_OF_NESTED_TRIPLES { - return Err(SyntaxError::msg(format!( + return Err(QueryResultsSyntaxError::msg(format!( "Too many nested triples ({MAX_NUMBER_OF_NESTED_TRIPLES}). The parser fails here to avoid a stack overflow." )) .into()); @@ -449,7 +468,7 @@ fn read_value( #[cfg(feature = "rdf-star")] let mut object = None; if reader.read_next_event()? != JsonEvent::StartObject { - return Err(SyntaxError::msg("Term serializations should be an object").into()); + return Err(QueryResultsSyntaxError::msg("Term serializations should be an object").into()); } loop { #[allow(unsafe_code)] @@ -472,7 +491,7 @@ fn read_value( #[cfg(feature = "rdf-star")] "object" => object = Some(read_value(reader, number_of_recursive_calls + 1)?), _ => { - return Err(SyntaxError::msg(format!( + return Err(QueryResultsSyntaxError::msg(format!( "Unexpected key in term serialization: '{key}'" )) .into()) @@ -480,9 +499,10 @@ fn read_value( }, JsonEvent::StartObject => { if state != Some(State::Value) { - return Err( - SyntaxError::msg("Unexpected nested object in term serialization").into(), - ); + return Err(QueryResultsSyntaxError::msg( + "Unexpected nested object in term serialization", + ) + .into()); } } JsonEvent::String(s) => match state { @@ -494,9 +514,10 @@ fn read_value( #[cfg(feature = "rdf-star")] "triple" => t = Some(Type::Triple), _ => { - return Err( - SyntaxError::msg(format!("Unexpected term type: '{s}'")).into() - ) + return Err(QueryResultsSyntaxError::msg(format!( + "Unexpected term type: '{s}'" + )) + .into()) } }; state = None; @@ -510,10 +531,9 @@ fn read_value( state = None; } Some(State::Datatype) => { - datatype = Some( - NamedNode::new(s) - .map_err(|e| SyntaxError::msg(format!("Invalid datatype IRI: {e}")))?, - ); + datatype = Some(NamedNode::new(s).map_err(|e| { + QueryResultsSyntaxError::msg(format!("Invalid datatype IRI: {e}")) + })?); state = None; } _ => (), // impossible @@ -523,41 +543,52 @@ fn read_value( if s == State::Value { state = None; // End of triple } else { - return Err( - SyntaxError::msg("Term description values should be string").into() - ); + return Err(QueryResultsSyntaxError::msg( + "Term description values should be string", + ) + .into()); } } else { return match t { - None => Err(SyntaxError::msg( + None => Err(QueryResultsSyntaxError::msg( "Term serialization should have a 'type' key", ) .into()), Some(Type::Uri) => Ok(NamedNode::new(value.ok_or_else(|| { - SyntaxError::msg("uri serialization should have a 'value' key") + QueryResultsSyntaxError::msg( + "uri serialization should have a 'value' key", + ) })?) - .map_err(|e| SyntaxError::msg(format!("Invalid uri value: {e}")))? + .map_err(|e| { + QueryResultsSyntaxError::msg(format!("Invalid uri value: {e}")) + })? .into()), Some(Type::BNode) => Ok(BlankNode::new(value.ok_or_else(|| { - SyntaxError::msg("bnode serialization should have a 'value' key") + QueryResultsSyntaxError::msg( + "bnode serialization should have a 'value' key", + ) })?) - .map_err(|e| SyntaxError::msg(format!("Invalid bnode value: {e}")))? + .map_err(|e| { + QueryResultsSyntaxError::msg(format!("Invalid bnode value: {e}")) + })? .into()), Some(Type::Literal) => { let value = value.ok_or_else(|| { - SyntaxError::msg("literal serialization should have a 'value' key") + QueryResultsSyntaxError::msg( + "literal serialization should have a 'value' key", + ) })?; Ok(match lang { Some(lang) => { if let Some(datatype) = datatype { if datatype.as_ref() != rdf::LANG_STRING { - return Err(SyntaxError::msg(format!( + return Err(QueryResultsSyntaxError::msg(format!( "xml:lang value '{lang}' provided with the datatype {datatype}" )).into()) } } Literal::new_language_tagged_literal(value, &*lang).map_err(|e| { - SyntaxError::msg(format!("Invalid xml:lang value '{lang}': {e}")) + QueryResultsSyntaxError::msg(format!("Invalid xml:lang value '{lang}': {e}")) })? } None => if let Some(datatype) = datatype { @@ -571,47 +602,53 @@ fn read_value( #[cfg(feature = "rdf-star")] Some(Type::Triple) => Ok(Triple::new( match subject.ok_or_else(|| { - SyntaxError::msg("triple serialization should have a 'subject' key") + QueryResultsSyntaxError::msg( + "triple serialization should have a 'subject' key", + ) })? { Term::NamedNode(subject) => subject.into(), Term::BlankNode(subject) => subject.into(), Term::Triple(subject) => Subject::Triple(subject), Term::Literal(_) => { - return Err(SyntaxError::msg( + return Err(QueryResultsSyntaxError::msg( "The 'subject' value should not be a literal", ) .into()) } }, match predicate.ok_or_else(|| { - SyntaxError::msg( + QueryResultsSyntaxError::msg( "triple serialization should have a 'predicate' key", ) })? { Term::NamedNode(predicate) => predicate, _ => { - return Err(SyntaxError::msg( + return Err(QueryResultsSyntaxError::msg( "The 'predicate' value should be a uri", ) .into()) } }, object.ok_or_else(|| { - SyntaxError::msg("triple serialization should have a 'object' key") + QueryResultsSyntaxError::msg( + "triple serialization should have a 'object' key", + ) })?, ) .into()), }; } } - _ => return Err(SyntaxError::msg("Invalid term serialization").into()), + _ => return Err(QueryResultsSyntaxError::msg("Invalid term serialization").into()), } } } -fn read_head(reader: &mut FromReadJsonReader) -> Result, ParseError> { +fn read_head( + reader: &mut FromReadJsonReader, +) -> Result, QueryResultsParseError> { if reader.read_next_event()? != JsonEvent::StartObject { - return Err(SyntaxError::msg("head should be an object").into()); + return Err(QueryResultsSyntaxError::msg("head should be an object").into()); } let mut variables = Vec::new(); loop { @@ -619,18 +656,21 @@ fn read_head(reader: &mut FromReadJsonReader) -> Result match key.as_ref() { "vars" => { if reader.read_next_event()? != JsonEvent::StartArray { - return Err(SyntaxError::msg("Variable list should be an array").into()); + return Err(QueryResultsSyntaxError::msg( + "Variable list should be an array", + ) + .into()); } loop { match reader.read_next_event()? { JsonEvent::String(s) => { let new_var = Variable::new(s.as_ref()).map_err(|e| { - SyntaxError::msg(format!( + QueryResultsSyntaxError::msg(format!( "Invalid variable declaration '{s}': {e}" )) })?; if variables.contains(&new_var) { - return Err(SyntaxError::msg(format!( + return Err(QueryResultsSyntaxError::msg(format!( "The variable {new_var} is declared twice" )) .into()); @@ -639,23 +679,30 @@ fn read_head(reader: &mut FromReadJsonReader) -> Result break, _ => { - return Err( - SyntaxError::msg("Variable names should be strings").into() + return Err(QueryResultsSyntaxError::msg( + "Variable names should be strings", ) + .into()) } } } } "link" => { if reader.read_next_event()? != JsonEvent::StartArray { - return Err(SyntaxError::msg("Variable list should be an array").into()); + return Err(QueryResultsSyntaxError::msg( + "Variable list should be an array", + ) + .into()); } loop { match reader.read_next_event()? { JsonEvent::String(_) => (), JsonEvent::EndArray => break, _ => { - return Err(SyntaxError::msg("Link names should be strings").into()) + return Err(QueryResultsSyntaxError::msg( + "Link names should be strings", + ) + .into()) } } } @@ -663,12 +710,12 @@ fn read_head(reader: &mut FromReadJsonReader) -> Result ignore_value(reader)?, }, JsonEvent::EndObject => return Ok(variables), - _ => return Err(SyntaxError::msg("Invalid head serialization").into()), + _ => return Err(QueryResultsSyntaxError::msg("Invalid head serialization").into()), } } } -fn ignore_value(reader: &mut FromReadJsonReader) -> Result<(), ParseError> { +fn ignore_value(reader: &mut FromReadJsonReader) -> Result<(), QueryResultsParseError> { let mut nesting = 0; loop { match reader.read_next_event()? { @@ -688,7 +735,9 @@ fn ignore_value(reader: &mut FromReadJsonReader) -> Result<(), Parse return Ok(()); } } - JsonEvent::Eof => return Err(SyntaxError::msg("Unexpected end of file").into()), + JsonEvent::Eof => { + return Err(QueryResultsSyntaxError::msg("Unexpected end of file").into()) + } } } } diff --git a/lib/sparesults/src/lib.rs b/lib/sparesults/src/lib.rs index 301dc2c8..ea3135c4 100644 --- a/lib/sparesults/src/lib.rs +++ b/lib/sparesults/src/lib.rs @@ -13,7 +13,7 @@ mod serializer; pub mod solution; mod xml; -pub use crate::error::{ParseError, SyntaxError, TextPosition}; +pub use crate::error::{QueryResultsParseError, QueryResultsSyntaxError, TextPosition}; pub use crate::format::QueryResultsFormat; pub use crate::parser::{FromReadQueryResultsReader, FromReadSolutionsReader, QueryResultsParser}; pub use crate::serializer::{QueryResultsSerializer, ToWriteSolutionsWriter}; diff --git a/lib/sparesults/src/parser.rs b/lib/sparesults/src/parser.rs index 3332335b..f95d9355 100644 --- a/lib/sparesults/src/parser.rs +++ b/lib/sparesults/src/parser.rs @@ -1,5 +1,5 @@ use crate::csv::{TsvQueryResultsReader, TsvSolutionsReader}; -use crate::error::{ParseError, SyntaxError}; +use crate::error::{QueryResultsParseError, QueryResultsSyntaxError}; use crate::format::QueryResultsFormat; use crate::json::{JsonQueryResultsReader, JsonSolutionsReader}; use crate::solution::QuerySolution; @@ -32,7 +32,7 @@ use std::sync::Arc; /// assert_eq!(solution?.iter().collect::>(), vec![(&Variable::new_unchecked("foo"), &Literal::from("test").into())]); /// } /// } -/// # Result::<(),sparesults::ParseError>::Ok(()) +/// # Result::<(),sparesults::QueryResultsParseError>::Ok(()) /// ``` pub struct QueryResultsParser { format: QueryResultsFormat, @@ -68,12 +68,12 @@ impl QueryResultsParser { /// assert_eq!(solution?.iter().collect::>(), vec![(&Variable::new_unchecked("foo"), &Literal::from("test").into())]); /// } /// } - /// # Result::<(),sparesults::ParseError>::Ok(()) + /// # Result::<(),sparesults::QueryResultsParseError>::Ok(()) /// ``` pub fn parse_read( &self, reader: R, - ) -> Result, ParseError> { + ) -> Result, QueryResultsParseError> { Ok(match self.format { QueryResultsFormat::Xml => match XmlQueryResultsReader::read(reader)? { XmlQueryResultsReader::Boolean(r) => FromReadQueryResultsReader::Boolean(r), @@ -95,7 +95,7 @@ impl QueryResultsParser { solutions: SolutionsReaderKind::Json(solutions), }), }, - QueryResultsFormat::Csv => return Err(SyntaxError::msg("CSV SPARQL results syntax is lossy and can't be parsed to a proper RDF representation").into()), + QueryResultsFormat::Csv => return Err(QueryResultsSyntaxError::msg("CSV SPARQL results syntax is lossy and can't be parsed to a proper RDF representation").into()), QueryResultsFormat::Tsv => match TsvQueryResultsReader::read(reader)? { TsvQueryResultsReader::Boolean(r) => FromReadQueryResultsReader::Boolean(r), TsvQueryResultsReader::Solutions { @@ -113,7 +113,7 @@ impl QueryResultsParser { pub fn read_results( &self, reader: R, - ) -> Result, ParseError> { + ) -> Result, QueryResultsParseError> { self.parse_read(reader) } } @@ -161,7 +161,7 @@ impl From for QueryResultsParser { /// ); /// } /// } -/// # Result::<(),sparesults::ParseError>::Ok(()) +/// # Result::<(),sparesults::QueryResultsParseError>::Ok(()) /// ``` pub enum FromReadQueryResultsReader { Solutions(FromReadSolutionsReader), @@ -184,7 +184,7 @@ pub enum FromReadQueryResultsReader { /// assert_eq!(solution?.iter().collect::>(), vec![(&Variable::new_unchecked("foo"), &Literal::from("test").into())]); /// } /// } -/// # Result::<(),sparesults::ParseError>::Ok(()) +/// # Result::<(),sparesults::QueryResultsParseError>::Ok(()) /// ``` pub struct FromReadSolutionsReader { variables: Arc<[Variable]>, @@ -217,7 +217,7 @@ impl FromReadSolutionsReader { /// ] /// ); /// } - /// # Result::<(),sparesults::ParseError>::Ok(()) + /// # Result::<(),sparesults::QueryResultsParseError>::Ok(()) /// ``` #[inline] pub fn variables(&self) -> &[Variable] { @@ -226,7 +226,7 @@ impl FromReadSolutionsReader { } impl Iterator for FromReadSolutionsReader { - type Item = Result; + type Item = Result; fn next(&mut self) -> Option { Some( diff --git a/lib/sparesults/src/xml.rs b/lib/sparesults/src/xml.rs index fb038d2d..6eb861e2 100644 --- a/lib/sparesults/src/xml.rs +++ b/lib/sparesults/src/xml.rs @@ -1,6 +1,6 @@ //! Implementation of [SPARQL Query Results XML Format](https://www.w3.org/TR/rdf-sparql-XMLres/) -use crate::error::{ParseError, SyntaxError}; +use crate::error::{QueryResultsParseError, QueryResultsSyntaxError}; use oxrdf::vocab::rdf; use oxrdf::*; use quick_xml::events::{BytesDecl, BytesEnd, BytesStart, BytesText, Event}; @@ -227,7 +227,7 @@ pub enum XmlQueryResultsReader { } impl XmlQueryResultsReader { - pub fn read(source: R) -> Result { + pub fn read(source: R) -> Result { enum State { Start, Sparql, @@ -254,14 +254,14 @@ impl XmlQueryResultsReader { if event.local_name().as_ref() == b"sparql" { state = State::Sparql; } else { - return Err(SyntaxError::msg(format!("Expecting tag, found <{}>", decode(&reader, &event.name())?)).into()); + return Err(QueryResultsSyntaxError::msg(format!("Expecting tag, found <{}>", decode(&reader, &event.name())?)).into()); } } State::Sparql => { if event.local_name().as_ref() == b"head" { state = State::Head; } else { - return Err(SyntaxError::msg(format!("Expecting tag, found <{}>",decode(&reader, &event.name())?)).into()); + return Err(QueryResultsSyntaxError::msg(format!("Expecting tag, found <{}>", decode(&reader, &event.name())?)).into()); } } State::Head => { @@ -269,11 +269,11 @@ impl XmlQueryResultsReader { let name = event.attributes() .filter_map(Result::ok) .find(|attr| attr.key.local_name().as_ref() == b"name") - .ok_or_else(|| SyntaxError::msg("No name attribute found for the tag"))? + .ok_or_else(|| QueryResultsSyntaxError::msg("No name attribute found for the tag"))? .decode_and_unescape_value(&reader)?; - let variable = Variable::new(name).map_err(|e| SyntaxError::msg(format!("Invalid variable name: {e}")))?; + let variable = Variable::new(name).map_err(|e| QueryResultsSyntaxError::msg(format!("Invalid variable name: {e}")))?; if variables.contains(&variable) { - return Err(SyntaxError::msg(format!( + return Err(QueryResultsSyntaxError::msg(format!( "The variable {variable} is declared twice" )) .into()); @@ -282,7 +282,7 @@ impl XmlQueryResultsReader { } else if event.local_name().as_ref() == b"link" { // no op } else { - return Err(SyntaxError::msg(format!("Expecting or tag, found <{}>", decode(&reader, &event.name())?)).into()); + return Err(QueryResultsSyntaxError::msg(format!("Expecting or tag, found <{}>", decode(&reader, &event.name())?)).into()); } } State::AfterHead => { @@ -304,10 +304,10 @@ impl XmlQueryResultsReader { object_stack: Vec::new(), }}); } else if event.local_name().as_ref() != b"link" && event.local_name().as_ref() != b"results" && event.local_name().as_ref() != b"boolean" { - return Err(SyntaxError::msg(format!("Expecting sparql tag, found <{}>", decode(&reader, &event.name())?)).into()); + return Err(QueryResultsSyntaxError::msg(format!("Expecting sparql tag, found <{}>", decode(&reader, &event.name())?)).into()); } } - State::Boolean => return Err(SyntaxError::msg(format!("Unexpected tag inside of tag: <{}>", decode(&reader, &event.name())?)).into()) + State::Boolean => return Err(QueryResultsSyntaxError::msg(format!("Unexpected tag inside of tag: <{}>", decode(&reader, &event.name())?)).into()) }, Event::Text(event) => { let value = event.unescape()?; @@ -318,10 +318,10 @@ impl XmlQueryResultsReader { } else if value == "false" { Ok(Self::Boolean(false)) } else { - Err(SyntaxError::msg(format!("Unexpected boolean value. Found '{value}'")).into()) + Err(QueryResultsSyntaxError::msg(format!("Unexpected boolean value. Found '{value}'")).into()) }; } - _ => Err(SyntaxError::msg(format!("Unexpected textual value found: '{value}'")).into()) + _ => Err(QueryResultsSyntaxError::msg(format!("Unexpected textual value found: '{value}'")).into()) }; }, Event::End(event) => { @@ -330,10 +330,10 @@ impl XmlQueryResultsReader { state = State::AfterHead } } else { - return Err(SyntaxError::msg("Unexpected early file end. All results file should have a and a or tag").into()); + return Err(QueryResultsSyntaxError::msg("Unexpected early file end. All results file should have a and a or tag").into()); } }, - Event::Eof => return Err(SyntaxError::msg("Unexpected early file end. All results file should have a and a or tag").into()), + Event::Eof => return Err(QueryResultsSyntaxError::msg("Unexpected early file end. All results file should have a and a or tag").into()), _ => (), } } @@ -365,7 +365,7 @@ pub struct XmlSolutionsReader { } impl XmlSolutionsReader { - pub fn read_next(&mut self) -> Result>>, ParseError> { + pub fn read_next(&mut self) -> Result>>, QueryResultsParseError> { let mut state = State::Start; let mut new_bindings = vec![None; self.mapping.len()]; @@ -383,7 +383,7 @@ impl XmlSolutionsReader { if event.local_name().as_ref() == b"result" { state = State::Result; } else { - return Err(SyntaxError::msg(format!( + return Err(QueryResultsSyntaxError::msg(format!( "Expecting , found <{}>", decode(&self.reader, &event.name())? )) @@ -403,7 +403,7 @@ impl XmlSolutionsReader { ) } None => { - return Err(SyntaxError::msg( + return Err(QueryResultsSyntaxError::msg( "No name attribute found for the tag", ) .into()); @@ -411,7 +411,7 @@ impl XmlSolutionsReader { } state = State::Binding; } else { - return Err(SyntaxError::msg(format!( + return Err(QueryResultsSyntaxError::msg(format!( "Expecting , found <{}>", decode(&self.reader, &event.name())? )) @@ -420,7 +420,7 @@ impl XmlSolutionsReader { } State::Binding | State::Subject | State::Predicate | State::Object => { if term.is_some() { - return Err(SyntaxError::msg( + return Err(QueryResultsSyntaxError::msg( "There is already a value for the current binding", ) .into()); @@ -441,7 +441,7 @@ impl XmlSolutionsReader { let iri = attr.decode_and_unescape_value(&self.reader)?; datatype = Some(NamedNode::new(iri.to_string()).map_err(|e| { - SyntaxError::msg(format!( + QueryResultsSyntaxError::msg(format!( "Invalid datatype IRI '{iri}': {e}" )) })?); @@ -451,7 +451,7 @@ impl XmlSolutionsReader { } else if event.local_name().as_ref() == b"triple" { state = State::Triple; } else { - return Err(SyntaxError::msg(format!( + return Err(QueryResultsSyntaxError::msg(format!( "Expecting , or found <{}>", decode(&self.reader, &event.name())? )) @@ -466,7 +466,7 @@ impl XmlSolutionsReader { } else if event.local_name().as_ref() == b"object" { state = State::Object } else { - return Err(SyntaxError::msg(format!( + return Err(QueryResultsSyntaxError::msg(format!( "Expecting , or found <{}>", decode(&self.reader, &event.name())? )) @@ -482,7 +482,9 @@ impl XmlSolutionsReader { term = Some( NamedNode::new(data.to_string()) .map_err(|e| { - SyntaxError::msg(format!("Invalid IRI value '{data}': {e}")) + QueryResultsSyntaxError::msg(format!( + "Invalid IRI value '{data}': {e}" + )) })? .into(), ) @@ -491,7 +493,7 @@ impl XmlSolutionsReader { term = Some( BlankNode::new(data.to_string()) .map_err(|e| { - SyntaxError::msg(format!( + QueryResultsSyntaxError::msg(format!( "Invalid blank node value '{data}': {e}" )) })? @@ -502,7 +504,7 @@ impl XmlSolutionsReader { term = Some(build_literal(data, lang.take(), datatype.take())?.into()); } _ => { - return Err(SyntaxError::msg(format!( + return Err(QueryResultsSyntaxError::msg(format!( "Unexpected textual value found: {data}" )) .into()); @@ -518,11 +520,14 @@ impl XmlSolutionsReader { new_bindings[*var] = term.take() } else { return Err( - SyntaxError::msg(format!("The variable '{var}' is used in a binding but not declared in the variables list")).into() + QueryResultsSyntaxError::msg(format!("The variable '{var}' is used in a binding but not declared in the variables list")).into() ); } } else { - return Err(SyntaxError::msg("No name found for tag").into()); + return Err(QueryResultsSyntaxError::msg( + "No name found for tag", + ) + .into()); } state = State::Result; } @@ -548,7 +553,7 @@ impl XmlSolutionsReader { state = self .stack .pop() - .ok_or_else(|| SyntaxError::msg("Empty stack"))? + .ok_or_else(|| QueryResultsSyntaxError::msg("Empty stack"))? } State::BNode => { if term.is_none() { @@ -558,7 +563,7 @@ impl XmlSolutionsReader { state = self .stack .pop() - .ok_or_else(|| SyntaxError::msg("Empty stack"))? + .ok_or_else(|| QueryResultsSyntaxError::msg("Empty stack"))? } State::Literal => { if term.is_none() { @@ -568,7 +573,7 @@ impl XmlSolutionsReader { state = self .stack .pop() - .ok_or_else(|| SyntaxError::msg("Empty stack"))?; + .ok_or_else(|| QueryResultsSyntaxError::msg("Empty stack"))?; } State::Triple => { #[cfg(feature = "rdf-star")] @@ -584,7 +589,7 @@ impl XmlSolutionsReader { Term::BlankNode(subject) => subject.into(), Term::Triple(subject) => Subject::Triple(subject), Term::Literal(_) => { - return Err(SyntaxError::msg( + return Err(QueryResultsSyntaxError::msg( "The value should not be a ", ) .into()) @@ -593,7 +598,7 @@ impl XmlSolutionsReader { match predicate { Term::NamedNode(predicate) => predicate, _ => { - return Err(SyntaxError::msg( + return Err(QueryResultsSyntaxError::msg( "The value should be an ", ) .into()) @@ -606,15 +611,15 @@ impl XmlSolutionsReader { state = self .stack .pop() - .ok_or_else(|| SyntaxError::msg("Empty stack"))?; + .ok_or_else(|| QueryResultsSyntaxError::msg("Empty stack"))?; } else { return Err( - SyntaxError::msg("A should contain a , a and an ").into() + QueryResultsSyntaxError::msg("A should contain a , a and an ").into() ); } #[cfg(not(feature = "rdf-star"))] { - return Err(SyntaxError::msg( + return Err(QueryResultsSyntaxError::msg( "The tag is only supported with RDF-star", ) .into()); @@ -633,19 +638,19 @@ fn build_literal( value: impl Into, lang: Option, datatype: Option, -) -> Result { +) -> Result { match lang { Some(lang) => { if let Some(datatype) = datatype { if datatype.as_ref() != rdf::LANG_STRING { - return Err(SyntaxError::msg(format!( + return Err(QueryResultsSyntaxError::msg(format!( "xml:lang value '{lang}' provided with the datatype {datatype}" )) .into()); } } Literal::new_language_tagged_literal(value, &lang).map_err(|e| { - SyntaxError::msg(format!("Invalid xml:lang value '{lang}': {e}")).into() + QueryResultsSyntaxError::msg(format!("Invalid xml:lang value '{lang}': {e}")).into() }) } None => Ok(if let Some(datatype) = datatype { @@ -659,7 +664,7 @@ fn build_literal( fn decode<'a, T>( reader: &Reader, data: &'a impl AsRef<[u8]>, -) -> Result, ParseError> { +) -> Result, QueryResultsParseError> { Ok(reader.decoder().decode(data.as_ref())?) } diff --git a/lib/spargebra/src/lib.rs b/lib/spargebra/src/lib.rs index e2d093f3..654de4ee 100644 --- a/lib/spargebra/src/lib.rs +++ b/lib/spargebra/src/lib.rs @@ -10,6 +10,6 @@ mod query; pub mod term; mod update; -pub use parser::ParseError; +pub use parser::SparqlSyntaxError; pub use query::*; pub use update::*; diff --git a/lib/spargebra/src/parser.rs b/lib/spargebra/src/parser.rs index a0a8e9d1..8e5d7445 100644 --- a/lib/spargebra/src/parser.rs +++ b/lib/spargebra/src/parser.rs @@ -15,16 +15,16 @@ use std::mem::take; use std::str::FromStr; /// Parses a SPARQL query with an optional base IRI to resolve relative IRIs in the query. -pub fn parse_query(query: &str, base_iri: Option<&str>) -> Result { +pub fn parse_query(query: &str, base_iri: Option<&str>) -> Result { let mut state = ParserState::from_base_iri(base_iri)?; - parser::QueryUnit(query, &mut state).map_err(|e| ParseError(ParseErrorKind::Parser(e))) + parser::QueryUnit(query, &mut state).map_err(|e| SparqlSyntaxError(ParseErrorKind::Syntax(e))) } /// Parses a SPARQL update with an optional base IRI to resolve relative IRIs in the query. -pub fn parse_update(update: &str, base_iri: Option<&str>) -> Result { +pub fn parse_update(update: &str, base_iri: Option<&str>) -> Result { let mut state = ParserState::from_base_iri(base_iri)?; let operations = parser::UpdateInit(update, &mut state) - .map_err(|e| ParseError(ParseErrorKind::Parser(e)))?; + .map_err(|e| SparqlSyntaxError(ParseErrorKind::Syntax(e)))?; Ok(Update { operations, base_iri: state.base_iri, @@ -34,14 +34,14 @@ pub fn parse_update(update: &str, base_iri: Option<&str>) -> Result), + Syntax(#[from] peg::error::ParseError), } struct AnnotatedTerm { @@ -669,12 +669,12 @@ pub struct ParserState { } impl ParserState { - pub(crate) fn from_base_iri(base_iri: Option<&str>) -> Result { + pub(crate) fn from_base_iri(base_iri: Option<&str>) -> Result { Ok(Self { base_iri: if let Some(base_iri) = base_iri { Some( Iri::parse(base_iri.to_owned()) - .map_err(|e| ParseError(ParseErrorKind::InvalidBaseIri(e)))?, + .map_err(|e| SparqlSyntaxError(ParseErrorKind::InvalidBaseIri(e)))?, ) } else { None diff --git a/lib/spargebra/src/query.rs b/lib/spargebra/src/query.rs index 780455f7..108221fa 100644 --- a/lib/spargebra/src/query.rs +++ b/lib/spargebra/src/query.rs @@ -1,5 +1,5 @@ use crate::algebra::*; -use crate::parser::{parse_query, ParseError}; +use crate::parser::{parse_query, SparqlSyntaxError}; use crate::term::*; use oxiri::Iri; use std::fmt; @@ -17,7 +17,7 @@ use std::str::FromStr; /// query.to_sse(), /// "(project (?s ?p ?o) (bgp (triple ?s ?p ?o)))" /// ); -/// # Ok::<_, spargebra::ParseError>(()) +/// # Ok::<_, spargebra::SparqlSyntaxError>(()) /// ``` #[derive(Eq, PartialEq, Debug, Clone, Hash)] pub enum Query { @@ -63,7 +63,7 @@ pub enum Query { impl Query { /// Parses a SPARQL query with an optional base IRI to resolve relative IRIs in the query. - pub fn parse(query: &str, base_iri: Option<&str>) -> Result { + pub fn parse(query: &str, base_iri: Option<&str>) -> Result { parse_query(query, base_iri) } @@ -276,7 +276,7 @@ impl fmt::Display for Query { } impl FromStr for Query { - type Err = ParseError; + type Err = SparqlSyntaxError; fn from_str(query: &str) -> Result { Self::parse(query, None) @@ -284,7 +284,7 @@ impl FromStr for Query { } impl<'a> TryFrom<&'a str> for Query { - type Error = ParseError; + type Error = SparqlSyntaxError; fn try_from(query: &str) -> Result { Self::from_str(query) @@ -292,7 +292,7 @@ impl<'a> TryFrom<&'a str> for Query { } impl<'a> TryFrom<&'a String> for Query { - type Error = ParseError; + type Error = SparqlSyntaxError; fn try_from(query: &String) -> Result { Self::from_str(query) diff --git a/lib/spargebra/src/update.rs b/lib/spargebra/src/update.rs index 76c5c65d..8f57c985 100644 --- a/lib/spargebra/src/update.rs +++ b/lib/spargebra/src/update.rs @@ -1,5 +1,5 @@ use crate::algebra::*; -use crate::parser::{parse_update, ParseError}; +use crate::parser::{parse_update, SparqlSyntaxError}; use crate::term::*; use oxiri::Iri; use std::fmt; @@ -14,7 +14,7 @@ use std::str::FromStr; /// let update = Update::parse(update_str, None)?; /// assert_eq!(update.to_string().trim(), update_str); /// assert_eq!(update.to_sse(), "(update (clear all))"); -/// # Ok::<_, spargebra::ParseError>(()) +/// # Ok::<_, spargebra::SparqlSyntaxError>(()) /// ``` #[derive(Eq, PartialEq, Debug, Clone, Hash)] pub struct Update { @@ -26,7 +26,7 @@ pub struct Update { impl Update { /// Parses a SPARQL update with an optional base IRI to resolve relative IRIs in the query. - pub fn parse(update: &str, base_iri: Option<&str>) -> Result { + pub fn parse(update: &str, base_iri: Option<&str>) -> Result { parse_update(update, base_iri) } @@ -68,7 +68,7 @@ impl fmt::Display for Update { } impl FromStr for Update { - type Err = ParseError; + type Err = SparqlSyntaxError; fn from_str(update: &str) -> Result { Self::parse(update, None) @@ -76,7 +76,7 @@ impl FromStr for Update { } impl<'a> TryFrom<&'a str> for Update { - type Error = ParseError; + type Error = SparqlSyntaxError; fn try_from(update: &str) -> Result { Self::from_str(update) @@ -84,7 +84,7 @@ impl<'a> TryFrom<&'a str> for Update { } impl<'a> TryFrom<&'a String> for Update { - type Error = ParseError; + type Error = SparqlSyntaxError; fn try_from(update: &String) -> Result { Self::from_str(update) diff --git a/python/src/io.rs b/python/src/io.rs index bf3a4383..37d97451 100644 --- a/python/src/io.rs +++ b/python/src/io.rs @@ -1,7 +1,7 @@ #![allow(clippy::needless_option_as_deref)] use crate::model::{hash, PyQuad, PyTriple}; -use oxigraph::io::{FromReadQuadReader, ParseError, RdfFormat, RdfParser, RdfSerializer}; +use oxigraph::io::{FromReadQuadReader, RdfFormat, RdfParseError, RdfParser, RdfSerializer}; use oxigraph::model::QuadRef; use pyo3::exceptions::{PyDeprecationWarning, PySyntaxError, PyValueError}; use pyo3::intern; @@ -556,9 +556,9 @@ pub enum PyRdfFormatInput { MediaType(String), } -pub fn map_parse_error(error: ParseError, file_path: Option) -> PyErr { +pub fn map_parse_error(error: RdfParseError, file_path: Option) -> PyErr { match error { - ParseError::Syntax(error) => { + RdfParseError::Syntax(error) => { // Python 3.9 does not support end line and end column if python_version() >= (3, 10) { let params = if let Some(location) = error.location() { @@ -588,7 +588,7 @@ pub fn map_parse_error(error: ParseError, file_path: Option) -> PyErr { PySyntaxError::new_err((error.to_string(), params)) } } - ParseError::Io(error) => error.into(), + RdfParseError::Io(error) => error.into(), } } diff --git a/python/src/sparql.rs b/python/src/sparql.rs index 383a7413..b1133fa1 100644 --- a/python/src/sparql.rs +++ b/python/src/sparql.rs @@ -4,8 +4,8 @@ use crate::store::map_storage_error; use oxigraph::io::RdfSerializer; use oxigraph::model::Term; use oxigraph::sparql::results::{ - FromReadQueryResultsReader, FromReadSolutionsReader, ParseError, QueryResultsFormat, - QueryResultsParser, QueryResultsSerializer, + FromReadQueryResultsReader, FromReadSolutionsReader, QueryResultsFormat, + QueryResultsParseError, QueryResultsParser, QueryResultsSerializer, }; use oxigraph::sparql::{ EvaluationError, Query, QueryResults, QuerySolution, QuerySolutionIter, QueryTripleIter, @@ -699,9 +699,12 @@ pub fn map_evaluation_error(error: EvaluationError) -> PyErr { } } -pub fn map_query_results_parse_error(error: ParseError, file_path: Option) -> PyErr { +pub fn map_query_results_parse_error( + error: QueryResultsParseError, + file_path: Option, +) -> PyErr { match error { - ParseError::Syntax(error) => { + QueryResultsParseError::Syntax(error) => { // Python 3.9 does not support end line and end column if python_version() >= (3, 10) { let params = if let Some(location) = error.location() { @@ -731,6 +734,6 @@ pub fn map_query_results_parse_error(error: ParseError, file_path: Option error.into(), + QueryResultsParseError::Io(error) => error.into(), } } From a078b12508c54952fc2faff528de0b6ea8b2b2c5 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Sat, 10 Feb 2024 15:05:34 -0500 Subject: [PATCH 212/217] Bump Cargo.lock --- Cargo.lock | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 85d35123..6ce4974e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -531,9 +531,9 @@ checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" [[package]] name = "either" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" +checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a" [[package]] name = "errno" @@ -547,9 +547,9 @@ dependencies = [ [[package]] name = "escargot" -version = "0.5.8" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "768064bd3a0e2bedcba91dc87ace90beea91acc41b6a01a3ca8e9aa8827461bf" +checksum = "704ab670cffff92792405528eb8ec3d9f00be8939d56d947f6bc809f9ae249f8" dependencies = [ "log", "once_cell", @@ -751,12 +751,12 @@ checksum = "1e186cfbae8084e513daff4240b4797e342f988cecda4fb6c939150f96315fd8" [[package]] name = "is-terminal" -version = "0.4.10" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bad00257d07be169d870ab665980b06cdb366d792ad690bf2e76876dc503455" +checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b" dependencies = [ "hermit-abi", - "rustix", + "libc", "windows-sys 0.52.0", ] @@ -1141,7 +1141,7 @@ dependencies = [ [[package]] name = "oxrdfio" -version = "0.1.0-alpha.2" +version = "0.1.0-alpha.3-dev" dependencies = [ "oxrdf", "oxrdfxml", @@ -1152,7 +1152,7 @@ dependencies = [ [[package]] name = "oxrdfxml" -version = "0.1.0-alpha.2" +version = "0.1.0-alpha.3-dev" dependencies = [ "oxilangtag", "oxiri", @@ -1182,7 +1182,7 @@ dependencies = [ [[package]] name = "oxttl" -version = "0.1.0-alpha.2" +version = "0.1.0-alpha.3-dev" dependencies = [ "memchr", "oxilangtag", From ea300e9081cada562e714d1d2a218b88ce9d32a9 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Sat, 10 Feb 2024 12:44:20 -0500 Subject: [PATCH 213/217] Normalize unicode refs Just to keep them a bit more consistent. Note that there are a lot of code duplications here - but I do not know if they are worth consolidating, and/or any perf implications. --- lib/oxigraph/tests/store.rs | 2 +- lib/oxrdf/src/literal.rs | 4 ++-- lib/oxrdf/src/parser.rs | 4 ++-- lib/oxrdf/src/variable.rs | 2 +- lib/oxrdfxml/src/utils.rs | 10 +++++----- lib/spargebra/src/parser.rs | 6 +++--- 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/lib/oxigraph/tests/store.rs b/lib/oxigraph/tests/store.rs index 48ff00f1..a3c5fe03 100644 --- a/lib/oxigraph/tests/store.rs +++ b/lib/oxigraph/tests/store.rs @@ -78,7 +78,7 @@ fn quads(graph_name: impl Into>) -> Vec> QuadRef::new( paris, name, - LiteralRef::new_language_tagged_literal_unchecked("la ville lumi\u{e8}re", "fr"), + LiteralRef::new_language_tagged_literal_unchecked("la ville lumi\u{E8}re", "fr"), graph_name, ), QuadRef::new(paris, country, france, graph_name), diff --git a/lib/oxrdf/src/literal.rs b/lib/oxrdf/src/literal.rs index 0872fab5..13ab07fe 100644 --- a/lib/oxrdf/src/literal.rs +++ b/lib/oxrdf/src/literal.rs @@ -622,11 +622,11 @@ pub fn print_quoted_str(string: &str, f: &mut impl Write) -> fmt::Result { '\u{08}' => f.write_str("\\b"), '\t' => f.write_str("\\t"), '\n' => f.write_str("\\n"), - '\u{0c}' => f.write_str("\\f"), + '\u{0C}' => f.write_str("\\f"), '\r' => f.write_str("\\r"), '"' => f.write_str("\\\""), '\\' => f.write_str("\\\\"), - '\0'..='\u{1f}' | '\u{7f}' => write!(f, "\\u{:04X}", u32::from(c)), + '\0'..='\u{1F}' | '\u{7F}' => write!(f, "\\u{:04X}", u32::from(c)), _ => f.write_char(c), }?; } diff --git a/lib/oxrdf/src/parser.rs b/lib/oxrdf/src/parser.rs index a531a86e..bcadd1ad 100644 --- a/lib/oxrdf/src/parser.rs +++ b/lib/oxrdf/src/parser.rs @@ -256,10 +256,10 @@ fn read_literal(s: &str) -> Result<(Literal, &str), TermParseError> { if let Some(c) = chars.next() { value.push(match c { 't' => '\t', - 'b' => '\u{8}', + 'b' => '\u{08}', 'n' => '\n', 'r' => '\r', - 'f' => '\u{C}', + 'f' => '\u{0C}', '"' => '"', '\'' => '\'', '\\' => '\\', diff --git a/lib/oxrdf/src/variable.rs b/lib/oxrdf/src/variable.rs index 36dcb3e4..c2d89ca9 100644 --- a/lib/oxrdf/src/variable.rs +++ b/lib/oxrdf/src/variable.rs @@ -187,7 +187,7 @@ fn validate_variable_identifier(id: &str) -> Result<(), VariableNameParseError> match c { '0'..='9' | '\u{00B7}' - | '\u{00300}'..='\u{036F}' + | '\u{0300}'..='\u{036F}' | '\u{203F}'..='\u{2040}' | '_' | 'A'..='Z' diff --git a/lib/oxrdfxml/src/utils.rs b/lib/oxrdfxml/src/utils.rs index b8fb2447..0483488d 100644 --- a/lib/oxrdfxml/src/utils.rs +++ b/lib/oxrdfxml/src/utils.rs @@ -5,11 +5,11 @@ pub fn is_name_start_char(c: char) -> bool { | 'A'..='Z' | '_' | 'a'..='z' - | '\u{C0}'..='\u{D6}' - | '\u{D8}'..='\u{F6}' - | '\u{F8}'..='\u{2FF}' - | '\u{370}'..='\u{37D}' - | '\u{37F}'..='\u{1FFF}' + | '\u{00C0}'..='\u{00D6}' + | '\u{00D8}'..='\u{00F6}' + | '\u{00F8}'..='\u{02FF}' + | '\u{0370}'..='\u{037D}' + | '\u{037F}'..='\u{1FFF}' | '\u{200C}'..='\u{200D}' | '\u{2070}'..='\u{218F}' | '\u{2C00}'..='\u{2FEF}' diff --git a/lib/spargebra/src/parser.rs b/lib/spargebra/src/parser.rs index 8e5d7445..38496387 100644 --- a/lib/spargebra/src/parser.rs +++ b/lib/spargebra/src/parser.rs @@ -2019,13 +2019,13 @@ parser! { rule STRING_LITERAL1() -> String = "'" l:$((STRING_LITERAL1_simple_char() / ECHAR() / UCHAR())*) "'" {? unescape_string(l) } - rule STRING_LITERAL1_simple_char() = !['\u{27}' | '\u{5C}' | '\u{A}' | '\u{D}'] [_] + rule STRING_LITERAL1_simple_char() = !['\u{27}' | '\u{5C}' | '\u{0A}' | '\u{0D}'] [_] rule STRING_LITERAL2() -> String = "\"" l:$((STRING_LITERAL2_simple_char() / ECHAR() / UCHAR())*) "\"" {? unescape_string(l) } - rule STRING_LITERAL2_simple_char() = !['\u{22}' | '\u{5C}' | '\u{A}' | '\u{D}'] [_] + rule STRING_LITERAL2_simple_char() = !['\u{22}' | '\u{5C}' | '\u{0A}' | '\u{0D}'] [_] rule STRING_LITERAL_LONG1() -> String = "'''" l:$(STRING_LITERAL_LONG1_inner()*) "'''" {? unescape_string(l) @@ -2045,7 +2045,7 @@ parser! { rule NIL() = "(" WS()* ")" - rule WS() = quiet! { ['\u{20}' | '\u{9}' | '\u{D}' | '\u{A}'] } + rule WS() = quiet! { ['\u{20}' | '\u{09}' | '\u{0D}' | '\u{0A}'] } rule ANON() = "[" WS()* "]" From 269c73a7c211f59b871d5ccf36fc861e66a66004 Mon Sep 17 00:00:00 2001 From: Tpt Date: Sun, 11 Feb 2024 21:48:24 +0100 Subject: [PATCH 214/217] Upgrades to Ruff 0.2 --- python/pyproject.toml | 2 ++ python/requirements.dev.txt | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index 441baffb..98cf75d8 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -30,6 +30,8 @@ Tracker = "https://github.com/oxigraph/oxigraph/issues" [tool.ruff] line-length = 120 + +[tool.ruff.lint] select = [ "ARG", "B", diff --git a/python/requirements.dev.txt b/python/requirements.dev.txt index f260a9e9..a47b2d4c 100644 --- a/python/requirements.dev.txt +++ b/python/requirements.dev.txt @@ -1,6 +1,6 @@ furo maturin~=1.0 mypy~=1.0 -ruff~=0.1.0 +ruff~=0.2.0 sphinx~=7.0 sphinx-lint~=0.9.1 From efae84b5f89123bb1ea8d66dc81c92d2b0f87627 Mon Sep 17 00:00:00 2001 From: Tpt Date: Sun, 11 Feb 2024 08:00:03 +0100 Subject: [PATCH 215/217] Convert from spargebra Update to oxigraph Update Issue #784 --- lib/oxigraph/src/sparql/algebra.rs | 35 +++++++++++++++++------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/lib/oxigraph/src/sparql/algebra.rs b/lib/oxigraph/src/sparql/algebra.rs index 41689904..8b3f385e 100644 --- a/lib/oxigraph/src/sparql/algebra.rs +++ b/lib/oxigraph/src/sparql/algebra.rs @@ -130,21 +130,7 @@ impl Update { update: &str, base_iri: Option<&str>, ) -> Result { - let update = spargebra::Update::parse(update, base_iri)?; - Ok(Self { - using_datasets: update - .operations - .iter() - .map(|operation| { - if let GraphUpdateOperation::DeleteInsert { using, .. } = operation { - Some(QueryDataset::from_algebra(using)) - } else { - None - } - }) - .collect(), - inner: update, - }) + Ok(spargebra::Update::parse(update, base_iri)?.into()) } /// Returns [the query dataset specification](https://www.w3.org/TR/sparql11-query/#specifyingDataset) in [DELETE/INSERT operations](https://www.w3.org/TR/sparql11-update/#deleteInsert). @@ -188,6 +174,25 @@ impl TryFrom<&String> for Update { } } +impl From for Update { + fn from(update: spargebra::Update) -> Self { + Self { + using_datasets: update + .operations + .iter() + .map(|operation| { + if let GraphUpdateOperation::DeleteInsert { using, .. } = operation { + Some(QueryDataset::from_algebra(using)) + } else { + None + } + }) + .collect(), + inner: update, + } + } +} + /// A SPARQL query [dataset specification](https://www.w3.org/TR/sparql11-query/#specifyingDataset) #[derive(Eq, PartialEq, Debug, Clone, Hash)] pub struct QueryDataset { From c277804026180ac1bab695be633857dd3a7bfb8a Mon Sep 17 00:00:00 2001 From: Tpt Date: Thu, 15 Feb 2024 21:57:46 +0100 Subject: [PATCH 216/217] RocksDB: uses multi-columns flush --- lib/oxigraph/src/storage/backend/rocksdb.rs | 17 +++++++++-------- lib/oxigraph/src/storage/mod.rs | 14 ++------------ oxrocksdb-sys/api/c.cc | 10 +++++++--- oxrocksdb-sys/api/c.h | 5 +++-- 4 files changed, 21 insertions(+), 25 deletions(-) diff --git a/lib/oxigraph/src/storage/backend/rocksdb.rs b/lib/oxigraph/src/storage/backend/rocksdb.rs index 14c45d0a..6f5561f4 100644 --- a/lib/oxigraph/src/storage/backend/rocksdb.rs +++ b/lib/oxigraph/src/storage/backend/rocksdb.rs @@ -9,7 +9,7 @@ )] use crate::storage::error::{CorruptionError, StorageError}; -use libc::{self, c_void, free}; +use libc::{self, c_void}; use oxrocksdb_sys::*; use rand::random; use std::borrow::Borrow; @@ -625,7 +625,7 @@ impl Db { ffi_result!(rocksdb_transaction_commit_with_status(transaction)); rocksdb_transaction_destroy(transaction); rocksdb_readoptions_destroy(read_options); - free(snapshot as *mut c_void); + rocksdb_free(snapshot as *mut c_void); r.map_err(StorageError::from)?; // We make sure to also run destructors if the commit fails } return Ok(result); @@ -636,7 +636,7 @@ impl Db { ffi_result!(rocksdb_transaction_rollback_with_status(transaction)); rocksdb_transaction_destroy(transaction); rocksdb_readoptions_destroy(read_options); - free(snapshot as *mut c_void); + rocksdb_free(snapshot as *mut c_void); r.map_err(StorageError::from)?; // We make sure to also run destructors if the commit fails } // We look for the root error @@ -738,13 +738,14 @@ impl Db { } } - pub fn flush(&self, column_family: &ColumnFamily) -> Result<(), StorageError> { + pub fn flush(&self) -> Result<(), StorageError> { if let DbKind::ReadWrite(db) = &self.inner { unsafe { - ffi_result!(rocksdb_transactiondb_flush_cf_with_status( + ffi_result!(rocksdb_transactiondb_flush_cfs_with_status( db.db, db.flush_options, - column_family.0, + db.cf_handles.as_ptr().cast_mut(), + db.cf_handles.len().try_into().unwrap() )) }?; Ok(()) @@ -1190,7 +1191,7 @@ pub struct Buffer { impl Drop for Buffer { fn drop(&mut self) { unsafe { - free(self.base.cast()); + rocksdb_free(self.base.cast()); } } } @@ -1325,7 +1326,7 @@ impl Drop for ErrorStatus { fn drop(&mut self) { if !self.0.string.is_null() { unsafe { - free(self.0.string as *mut c_void); + rocksdb_free(self.0.string as *mut c_void); } } } diff --git a/lib/oxigraph/src/storage/mod.rs b/lib/oxigraph/src/storage/mod.rs index 1f77d1b0..10d26c32 100644 --- a/lib/oxigraph/src/storage/mod.rs +++ b/lib/oxigraph/src/storage/mod.rs @@ -258,7 +258,7 @@ impl Storage { fn update_version(&self, version: u64) -> Result<(), StorageError> { self.db .insert(&self.default_cf, b"oxversion", &version.to_be_bytes())?; - self.db.flush(&self.default_cf) + self.db.flush() } pub fn snapshot(&self) -> StorageReader { @@ -283,17 +283,7 @@ impl Storage { #[cfg(not(target_family = "wasm"))] pub fn flush(&self) -> Result<(), StorageError> { - self.db.flush(&self.default_cf)?; - self.db.flush(&self.gspo_cf)?; - self.db.flush(&self.gpos_cf)?; - self.db.flush(&self.gosp_cf)?; - self.db.flush(&self.spog_cf)?; - self.db.flush(&self.posg_cf)?; - self.db.flush(&self.ospg_cf)?; - self.db.flush(&self.dspo_cf)?; - self.db.flush(&self.dpos_cf)?; - self.db.flush(&self.dosp_cf)?; - self.db.flush(&self.id2str_cf) + self.db.flush() } #[cfg(not(target_family = "wasm"))] diff --git a/oxrocksdb-sys/api/c.cc b/oxrocksdb-sys/api/c.cc index 9a948ea2..b62ee7ff 100644 --- a/oxrocksdb-sys/api/c.cc +++ b/oxrocksdb-sys/api/c.cc @@ -272,11 +272,15 @@ void rocksdb_transactiondb_put_cf_with_status( Slice(key, keylen), Slice(val, vallen))); } -void rocksdb_transactiondb_flush_cf_with_status( +void rocksdb_transactiondb_flush_cfs_with_status( rocksdb_transactiondb_t* db, const rocksdb_flushoptions_t* options, - rocksdb_column_family_handle_t* column_family, + rocksdb_column_family_handle_t** column_families, int num_column_families, rocksdb_status_t* statusptr) { - SaveStatus(statusptr, db->rep->Flush(options->rep, column_family->rep)); + vector column_family_handles(num_column_families); + for (int i = 0; i < num_column_families; i++) { + column_family_handles[i] = column_families[i]->rep; + } + SaveStatus(statusptr, db->rep->Flush(options->rep, column_family_handles)); } void rocksdb_transactiondb_compact_range_cf_opt_with_status( diff --git a/oxrocksdb-sys/api/c.h b/oxrocksdb-sys/api/c.h index 022e8f97..8b7c173e 100644 --- a/oxrocksdb-sys/api/c.h +++ b/oxrocksdb-sys/api/c.h @@ -114,9 +114,10 @@ extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_put_cf_with_status( rocksdb_column_family_handle_t* column_family, const char* key, size_t keylen, const char* val, size_t vallen, rocksdb_status_t* statusptr); -extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_flush_cf_with_status( +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_flush_cfs_with_status( rocksdb_transactiondb_t* db, const rocksdb_flushoptions_t* options, - rocksdb_column_family_handle_t* column_family, rocksdb_status_t* statusptr); + rocksdb_column_family_handle_t** column_families, int num_column_families, + rocksdb_status_t* statusptr); extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_compact_range_cf_opt_with_status( From e48b268fc5affeafe015b7cd365062a1046b16b8 Mon Sep 17 00:00:00 2001 From: Tpt Date: Fri, 5 Jan 2024 09:38:47 +0100 Subject: [PATCH 217/217] Adds an async SPARQL JSON results reader --- lib/sparesults/src/json.rs | 1238 ++++++++++++++++++++++------------ lib/sparesults/src/lib.rs | 4 + lib/sparesults/src/parser.rs | 234 ++++++- 3 files changed, 1013 insertions(+), 463 deletions(-) diff --git a/lib/sparesults/src/json.rs b/lib/sparesults/src/json.rs index 25da2d64..efed124a 100644 --- a/lib/sparesults/src/json.rs +++ b/lib/sparesults/src/json.rs @@ -1,20 +1,16 @@ //! Implementation of [SPARQL Query Results JSON Format](https://www.w3.org/TR/sparql11-results-json/) use crate::error::{QueryResultsParseError, QueryResultsSyntaxError}; -#[cfg(feature = "async-tokio")] -use json_event_parser::ToTokioAsyncWriteJsonWriter; use json_event_parser::{FromReadJsonReader, JsonEvent, ToWriteJsonWriter}; +#[cfg(feature = "async-tokio")] +use json_event_parser::{FromTokioAsyncReadJsonReader, ToTokioAsyncWriteJsonWriter}; use oxrdf::vocab::rdf; use oxrdf::*; use std::collections::BTreeMap; use std::io::{self, Read, Write}; use std::mem::take; #[cfg(feature = "async-tokio")] -use tokio::io::AsyncWrite; - -/// This limit is set in order to avoid stack overflow error when parsing nested triples due to too many recursive calls. -/// The actual limit value is a wet finger compromise between not failing to parse valid files and avoiding to trigger stack overflow errors. -const MAX_NUMBER_OF_NESTED_TRIPLES: usize = 128; +use tokio::io::{AsyncRead, AsyncWrite}; pub fn write_boolean_json_result(write: W, value: bool) -> io::Result { let mut writer = ToWriteJsonWriter::new(write); @@ -224,520 +220,882 @@ fn write_json_term<'a>(output: &mut Vec>, term: TermRef<'a>) { } } -pub enum JsonQueryResultsReader { +pub enum FromReadJsonQueryResultsReader { Solutions { variables: Vec, - solutions: JsonSolutionsReader, + solutions: FromReadJsonSolutionsReader, }, Boolean(bool), } -impl JsonQueryResultsReader { +impl FromReadJsonQueryResultsReader { pub fn read(read: R) -> Result { let mut reader = FromReadJsonReader::new(read); - let mut variables = None; - let mut buffered_bindings: Option> = None; - let mut output_iter = None; - - if reader.read_next_event()? != JsonEvent::StartObject { - return Err( - QueryResultsSyntaxError::msg("SPARQL JSON results should be an object").into(), - ); + let mut inner = JsonInnerReader::new(); + loop { + if let Some(result) = inner.read_event(reader.read_next_event()?)? { + return match result { + JsonInnerQueryResults::Solutions { + variables, + solutions, + } => Ok(Self::Solutions { + variables, + solutions: FromReadJsonSolutionsReader { + inner: solutions, + reader, + }, + }), + JsonInnerQueryResults::Boolean(value) => Ok(Self::Boolean(value)), + }; + } } + } +} + +pub struct FromReadJsonSolutionsReader { + inner: JsonInnerSolutions, + reader: FromReadJsonReader, +} + +impl FromReadJsonSolutionsReader { + pub fn read_next(&mut self) -> Result>>, QueryResultsParseError> { + match &mut self.inner { + JsonInnerSolutions::Reader(reader) => loop { + let event = self.reader.read_next_event()?; + if event == JsonEvent::Eof { + return Ok(None); + } + if let Some(result) = reader.read_event(event)? { + return Ok(Some(result)); + } + }, + JsonInnerSolutions::Iterator(iter) => iter.next(), + } + } +} +#[cfg(feature = "async-tokio")] +pub enum FromTokioAsyncReadJsonQueryResultsReader { + Solutions { + variables: Vec, + solutions: FromTokioAsyncReadJsonSolutionsReader, + }, + Boolean(bool), +} + +#[cfg(feature = "async-tokio")] +impl FromTokioAsyncReadJsonQueryResultsReader { + pub async fn read(read: R) -> Result { + let mut reader = FromTokioAsyncReadJsonReader::new(read); + let mut inner = JsonInnerReader::new(); loop { - let event = reader.read_next_event()?; - match event { + if let Some(result) = inner.read_event(reader.read_next_event().await?)? { + return match result { + JsonInnerQueryResults::Solutions { + variables, + solutions, + } => Ok(Self::Solutions { + variables, + solutions: FromTokioAsyncReadJsonSolutionsReader { + inner: solutions, + reader, + }, + }), + JsonInnerQueryResults::Boolean(value) => Ok(Self::Boolean(value)), + }; + } + } + } +} + +#[cfg(feature = "async-tokio")] +pub struct FromTokioAsyncReadJsonSolutionsReader { + inner: JsonInnerSolutions, + reader: FromTokioAsyncReadJsonReader, +} + +#[cfg(feature = "async-tokio")] +impl FromTokioAsyncReadJsonSolutionsReader { + pub async fn read_next(&mut self) -> Result>>, QueryResultsParseError> { + match &mut self.inner { + JsonInnerSolutions::Reader(reader) => loop { + let event = self.reader.read_next_event().await?; + if event == JsonEvent::Eof { + return Ok(None); + } + if let Some(result) = reader.read_event(event)? { + return Ok(Some(result)); + } + }, + JsonInnerSolutions::Iterator(iter) => iter.next(), + } + } +} + +enum JsonInnerQueryResults { + Solutions { + variables: Vec, + solutions: JsonInnerSolutions, + }, + Boolean(bool), +} + +enum JsonInnerSolutions { + Reader(JsonInnerSolutionsReader), + Iterator(JsonBufferedSolutionsIterator), +} + +struct JsonInnerReader { + state: JsonInnerReaderState, + variables: Vec, + current_solution_variables: Vec, + current_solution_values: Vec, + solutions: Vec<(Vec, Vec)>, + vars_read: bool, + solutions_read: bool, +} + +enum JsonInnerReaderState { + Start, + InRootObject, + BeforeHead, + InHead, + BeforeVars, + InVars, + BeforeLinks, + InLinks, + BeforeResults, + InResults, + BeforeBindings, + BeforeSolution, + BetweenSolutionTerms, + Term { + reader: JsonInnerTermReader, + variable: String, + }, + AfterBindings, + BeforeBoolean, + Ignore { + level: usize, + after: JsonInnerReaderStateAfterIgnore, + }, +} + +#[allow(clippy::enum_variant_names)] +#[derive(Clone, Copy)] +enum JsonInnerReaderStateAfterIgnore { + InRootObject, + InHead, + InResults, + AfterBindings, +} + +impl JsonInnerReader { + fn new() -> Self { + Self { + state: JsonInnerReaderState::Start, + variables: Vec::new(), + current_solution_variables: Vec::new(), + current_solution_values: Vec::new(), + solutions: Vec::new(), + vars_read: false, + solutions_read: false, + } + } + + fn read_event( + &mut self, + event: JsonEvent<'_>, + ) -> Result, QueryResultsSyntaxError> { + match &mut self.state { + JsonInnerReaderState::Start => { + if event == JsonEvent::StartObject { + self.state = JsonInnerReaderState::InRootObject; + Ok(None) + } else { + Err(QueryResultsSyntaxError::msg( + "SPARQL JSON results must be an object", + )) + } + } + JsonInnerReaderState::InRootObject => match event { JsonEvent::ObjectKey(key) => match key.as_ref() { "head" => { - let extracted_variables = read_head(&mut reader)?; - if let Some(buffered_bindings) = buffered_bindings.take() { - let mut mapping = BTreeMap::default(); - for (i, var) in extracted_variables.iter().enumerate() { - mapping.insert(var.as_str().to_owned(), i); - } - output_iter = Some(Self::Solutions { - variables: extracted_variables, - solutions: JsonSolutionsReader { - kind: JsonSolutionsReaderKind::Buffered { - bindings: buffered_bindings.into_iter(), - }, - mapping, - }, - }); - } else { - variables = Some(extracted_variables); - } + self.state = JsonInnerReaderState::BeforeHead; + Ok(None) } "results" => { - if reader.read_next_event()? != JsonEvent::StartObject { - return Err(QueryResultsSyntaxError::msg( - "'results' should be an object", - ) - .into()); - } - loop { - match reader.read_next_event()? { - JsonEvent::ObjectKey(k) if k == "bindings" => break, // Found - JsonEvent::ObjectKey(_) => ignore_value(&mut reader)?, - _ => { - return Err(QueryResultsSyntaxError::msg( - "'results' should contain a 'bindings' key", - ) - .into()) - } - } - } - if reader.read_next_event()? != JsonEvent::StartArray { - return Err(QueryResultsSyntaxError::msg( - "'bindings' should be an object", - ) - .into()); - } - if let Some(variables) = variables { - let mut mapping = BTreeMap::default(); - for (i, var) in variables.iter().enumerate() { - mapping.insert(var.as_str().to_owned(), i); - } - return Ok(Self::Solutions { - variables, - solutions: JsonSolutionsReader { - kind: JsonSolutionsReaderKind::Streaming { reader }, - mapping, - }, - }); - } - // We buffer all results before being able to read the header - let mut bindings = Vec::new(); - let mut variables = Vec::new(); - let mut values = Vec::new(); - loop { - match reader.read_next_event()? { - JsonEvent::StartObject => (), - JsonEvent::EndObject => { - bindings.push((take(&mut variables), take(&mut values))); - } - JsonEvent::EndArray | JsonEvent::Eof => { - buffered_bindings = Some(bindings); - break; - } - JsonEvent::ObjectKey(key) => { - variables.push(key.into_owned()); - values.push(read_value(&mut reader, 0)?); - } - _ => { - return Err(QueryResultsSyntaxError::msg( - "Invalid result serialization", - ) - .into()) - } - } - } + self.state = JsonInnerReaderState::BeforeResults; + Ok(None) } "boolean" => { - return if let JsonEvent::Boolean(v) = reader.read_next_event()? { - Ok(Self::Boolean(v)) - } else { - Err(QueryResultsSyntaxError::msg("Unexpected boolean value").into()) - } + self.state = JsonInnerReaderState::BeforeBoolean; + Ok(None) } _ => { - return Err(QueryResultsSyntaxError::msg(format!( - "Expecting head or result key, found {key}" - )) - .into()); + self.state = JsonInnerReaderState::Ignore { + level: 0, + after: JsonInnerReaderStateAfterIgnore::InRootObject, + }; + Ok(None) + } + }, + JsonEvent::EndObject => Err(QueryResultsSyntaxError::msg( + "SPARQL JSON results must contain a 'boolean' or a 'results' key", + )), + _ => unreachable!(), + }, + JsonInnerReaderState::BeforeHead => { + if event == JsonEvent::StartObject { + self.state = JsonInnerReaderState::InHead; + Ok(None) + } else { + Err(QueryResultsSyntaxError::msg( + "SPARQL JSON results head must be an object", + )) + } + } + JsonInnerReaderState::InHead => match event { + JsonEvent::ObjectKey(key) => match key.as_ref() { + "vars" => { + self.state = JsonInnerReaderState::BeforeVars; + self.vars_read = true; + Ok(None) + } + "links" => { + self.state = JsonInnerReaderState::BeforeLinks; + Ok(None) + } + _ => { + self.state = JsonInnerReaderState::Ignore { + level: 0, + after: JsonInnerReaderStateAfterIgnore::InHead, + }; + Ok(None) + } + }, + JsonEvent::EndObject => { + self.state = JsonInnerReaderState::InRootObject; + Ok(None) + } + _ => unreachable!(), + }, + JsonInnerReaderState::BeforeVars => { + if event == JsonEvent::StartArray { + self.state = JsonInnerReaderState::InVars; + Ok(None) + } else { + Err(QueryResultsSyntaxError::msg( + "SPARQL JSON results vars must be an array", + )) + } + } + JsonInnerReaderState::InVars => match event { + JsonEvent::String(variable) => match Variable::new(variable.clone()) { + Ok(var) => { + if self.variables.contains(&var) { + return Err(QueryResultsSyntaxError::msg(format!( + "The variable {var} is declared twice" + ))); + } + self.variables.push(var); + Ok(None) } + Err(e) => Err(QueryResultsSyntaxError::msg(format!( + "Invalid variable name '{variable}': {e}" + ))), }, - JsonEvent::EndObject => (), - JsonEvent::Eof => { - return if let Some(output_iter) = output_iter { - Ok(output_iter) + JsonEvent::EndArray => { + if self.solutions_read { + let mut mapping = BTreeMap::default(); + for (i, var) in self.variables.iter().enumerate() { + mapping.insert(var.as_str().to_owned(), i); + } + Ok(Some(JsonInnerQueryResults::Solutions { + variables: take(&mut self.variables), + solutions: JsonInnerSolutions::Iterator( + JsonBufferedSolutionsIterator { + mapping, + bindings: take(&mut self.solutions).into_iter(), + }, + ), + })) } else { - Err(QueryResultsSyntaxError::msg( - "Unexpected end of JSON object without 'results' or 'boolean' key", - ) - .into()) + self.state = JsonInnerReaderState::InHead; + Ok(None) } } - _ => { - return Err(QueryResultsSyntaxError::msg( - "Invalid SPARQL results serialization", - ) - .into()) + _ => Err(QueryResultsSyntaxError::msg( + "Variables name in the vars array must be strings", + )), + }, + JsonInnerReaderState::BeforeLinks => { + if event == JsonEvent::StartArray { + self.state = JsonInnerReaderState::InLinks; + Ok(None) + } else { + Err(QueryResultsSyntaxError::msg( + "SPARQL JSON results links must be an array", + )) + } + } + JsonInnerReaderState::InLinks => match event { + JsonEvent::String(_) => Ok(None), + JsonEvent::EndArray => { + self.state = JsonInnerReaderState::InHead; + Ok(None) + } + _ => Err(QueryResultsSyntaxError::msg( + "Links in the links array must be strings", + )), + }, + JsonInnerReaderState::BeforeResults => { + if event == JsonEvent::StartObject { + self.state = JsonInnerReaderState::InResults; + Ok(None) + } else { + Err(QueryResultsSyntaxError::msg( + "SPARQL JSON results result must be an object", + )) + } + } + JsonInnerReaderState::InResults => match event { + JsonEvent::ObjectKey(key) => { + if key == "bindings" { + self.state = JsonInnerReaderState::BeforeBindings; + Ok(None) + } else { + self.state = JsonInnerReaderState::Ignore { + level: 0, + after: JsonInnerReaderStateAfterIgnore::InResults, + }; + Ok(None) + } + } + JsonEvent::EndObject => Err(QueryResultsSyntaxError::msg( + "The results object must contains a 'bindings' key", + )), + _ => unreachable!(), + }, + JsonInnerReaderState::BeforeBindings => { + if event == JsonEvent::StartArray { + self.solutions_read = true; + if self.vars_read { + let mut mapping = BTreeMap::default(); + for (i, var) in self.variables.iter().enumerate() { + mapping.insert(var.as_str().to_owned(), i); + } + Ok(Some(JsonInnerQueryResults::Solutions { + variables: take(&mut self.variables), + solutions: JsonInnerSolutions::Reader(JsonInnerSolutionsReader { + state: JsonInnerSolutionsReaderState::BeforeSolution, + mapping, + new_bindings: Vec::new(), + }), + })) + } else { + self.state = JsonInnerReaderState::BeforeSolution; + Ok(None) + } + } else { + Err(QueryResultsSyntaxError::msg( + "SPARQL JSON results bindings must be an array", + )) + } + } + JsonInnerReaderState::BeforeSolution => match event { + JsonEvent::StartObject => { + self.state = JsonInnerReaderState::BetweenSolutionTerms; + Ok(None) + } + JsonEvent::EndArray => { + self.state = JsonInnerReaderState::AfterBindings; + Ok(None) + } + _ => Err(QueryResultsSyntaxError::msg( + "Expecting a new solution object", + )), + }, + JsonInnerReaderState::BetweenSolutionTerms => match event { + JsonEvent::ObjectKey(key) => { + self.state = JsonInnerReaderState::Term { + reader: JsonInnerTermReader::default(), + variable: key.into(), + }; + Ok(None) + } + JsonEvent::EndObject => { + self.state = JsonInnerReaderState::BeforeSolution; + self.solutions.push(( + take(&mut self.current_solution_variables), + take(&mut self.current_solution_values), + )); + Ok(None) + } + _ => unreachable!(), + }, + JsonInnerReaderState::Term { + ref mut reader, + variable, + } => { + let result = reader.read_event(event); + if let Some(term) = result? { + self.current_solution_variables.push(take(variable)); + self.current_solution_values.push(term); + self.state = JsonInnerReaderState::BetweenSolutionTerms; } + Ok(None) + } + JsonInnerReaderState::AfterBindings => { + if event == JsonEvent::EndObject { + self.state = JsonInnerReaderState::InRootObject; + } else { + self.state = JsonInnerReaderState::Ignore { + level: 0, + after: JsonInnerReaderStateAfterIgnore::AfterBindings, + } + } + Ok(None) + } + JsonInnerReaderState::BeforeBoolean => { + if let JsonEvent::Boolean(v) = event { + Ok(Some(JsonInnerQueryResults::Boolean(v))) + } else { + Err(QueryResultsSyntaxError::msg("Unexpected boolean value")) + } + } + #[allow(clippy::ref_patterns)] + JsonInnerReaderState::Ignore { level, ref after } => { + let level = match event { + JsonEvent::StartArray | JsonEvent::StartObject => *level + 1, + JsonEvent::EndArray | JsonEvent::EndObject => *level - 1, + JsonEvent::String(_) + | JsonEvent::Number(_) + | JsonEvent::Boolean(_) + | JsonEvent::Null + | JsonEvent::ObjectKey(_) + | JsonEvent::Eof => *level, + }; + self.state = if level == 0 { + match after { + JsonInnerReaderStateAfterIgnore::InRootObject => { + JsonInnerReaderState::InRootObject + } + JsonInnerReaderStateAfterIgnore::InHead => JsonInnerReaderState::InHead, + JsonInnerReaderStateAfterIgnore::InResults => { + JsonInnerReaderState::InResults + } + JsonInnerReaderStateAfterIgnore::AfterBindings => { + JsonInnerReaderState::AfterBindings + } + } + } else { + JsonInnerReaderState::Ignore { + level, + after: *after, + } + }; + Ok(None) } } } } -pub struct JsonSolutionsReader { +struct JsonInnerSolutionsReader { + state: JsonInnerSolutionsReaderState, mapping: BTreeMap, - kind: JsonSolutionsReaderKind, + new_bindings: Vec>, } -enum JsonSolutionsReaderKind { - Streaming { - reader: FromReadJsonReader, - }, - Buffered { - bindings: std::vec::IntoIter<(Vec, Vec)>, +enum JsonInnerSolutionsReaderState { + BeforeSolution, + BetweenSolutionTerms, + Term { + reader: JsonInnerTermReader, + key: usize, }, + AfterEnd, } -impl JsonSolutionsReader { - pub fn read_next(&mut self) -> Result>>, QueryResultsParseError> { - match &mut self.kind { - JsonSolutionsReaderKind::Streaming { reader } => { - let mut new_bindings = vec![None; self.mapping.len()]; - loop { - match reader.read_next_event()? { - JsonEvent::StartObject => (), - JsonEvent::EndObject => return Ok(Some(new_bindings)), - JsonEvent::EndArray | JsonEvent::Eof => return Ok(None), - JsonEvent::ObjectKey(key) => { - let k = *self.mapping.get(key.as_ref()).ok_or_else(|| { - QueryResultsSyntaxError::msg(format!( - "The variable {key} has not been defined in the header" - )) - })?; - new_bindings[k] = Some(read_value(reader, 0)?) - } - _ => { - return Err(QueryResultsSyntaxError::msg( - "Invalid result serialization", - ) - .into()) - } - } +impl JsonInnerSolutionsReader { + fn read_event( + &mut self, + event: JsonEvent<'_>, + ) -> Result>>, QueryResultsSyntaxError> { + match &mut self.state { + JsonInnerSolutionsReaderState::BeforeSolution => match event { + JsonEvent::StartObject => { + self.state = JsonInnerSolutionsReaderState::BetweenSolutionTerms; + self.new_bindings = vec![None; self.mapping.len()]; + Ok(None) + } + JsonEvent::EndArray => { + self.state = JsonInnerSolutionsReaderState::AfterEnd; + Ok(None) } + _ => Err(QueryResultsSyntaxError::msg( + "Expecting a new solution object", + )), + }, + JsonInnerSolutionsReaderState::BetweenSolutionTerms => match event { + JsonEvent::ObjectKey(key) => { + let key = *self.mapping.get(key.as_ref()).ok_or_else(|| { + QueryResultsSyntaxError::msg(format!( + "The variable {key} has not been defined in the header" + )) + })?; + self.state = JsonInnerSolutionsReaderState::Term { + reader: JsonInnerTermReader::default(), + key, + }; + Ok(None) + } + JsonEvent::EndObject => { + self.state = JsonInnerSolutionsReaderState::BeforeSolution; + Ok(Some(take(&mut self.new_bindings))) + } + _ => unreachable!(), + }, + JsonInnerSolutionsReaderState::Term { + ref mut reader, + key, + } => { + let result = reader.read_event(event); + if let Some(term) = result? { + self.new_bindings[*key] = Some(term); + self.state = JsonInnerSolutionsReaderState::BetweenSolutionTerms; + } + Ok(None) } - JsonSolutionsReaderKind::Buffered { bindings } => { - Ok(if let Some((variables, values)) = bindings.next() { - let mut new_bindings = vec![None; self.mapping.len()]; - for (variable, value) in variables.into_iter().zip(values) { - let k = *self.mapping.get(&variable).ok_or_else(|| { - QueryResultsSyntaxError::msg(format!( - "The variable {variable} has not been defined in the header" - )) - })?; - new_bindings[k] = Some(value) - } - Some(new_bindings) + JsonInnerSolutionsReaderState::AfterEnd => { + if event == JsonEvent::EndObject { + Ok(None) } else { - None - }) + Err(QueryResultsSyntaxError::msg( + "Unexpected JSON after the end of the bindings array", + )) + } } } } } -fn read_value( - reader: &mut FromReadJsonReader, - number_of_recursive_calls: usize, -) -> Result { - enum Type { - Uri, - BNode, - Literal, - #[cfg(feature = "rdf-star")] - Triple, - } - #[derive(Eq, PartialEq)] - enum State { - Type, - Value, - Lang, - Datatype, - } +#[derive(Default)] +struct JsonInnerTermReader { + state: JsonInnerTermReaderState, + term_type: Option, + value: Option, + lang: Option, + datatype: Option, + #[cfg(feature = "rdf-star")] + subject: Option, + #[cfg(feature = "rdf-star")] + predicate: Option, + #[cfg(feature = "rdf-star")] + object: Option, +} - if number_of_recursive_calls == MAX_NUMBER_OF_NESTED_TRIPLES { - return Err(QueryResultsSyntaxError::msg(format!( - "Too many nested triples ({MAX_NUMBER_OF_NESTED_TRIPLES}). The parser fails here to avoid a stack overflow." - )) - .into()); - } - let mut state = None; - let mut t = None; - let mut value = None; - let mut lang = None; - let mut datatype = None; +#[derive(Default)] +enum JsonInnerTermReaderState { + #[default] + Start, + Middle, + TermType, + Value, + Lang, + Datatype, #[cfg(feature = "rdf-star")] - let mut subject = None; + InValue, #[cfg(feature = "rdf-star")] - let mut predicate = None; + Subject(Box), #[cfg(feature = "rdf-star")] - let mut object = None; - if reader.read_next_event()? != JsonEvent::StartObject { - return Err(QueryResultsSyntaxError::msg("Term serializations should be an object").into()); - } - loop { - #[allow(unsafe_code)] - // SAFETY: Borrow checker workaround https://github.com/rust-lang/rust/issues/70255 - let next_event = unsafe { - let r: *mut FromReadJsonReader = reader; - &mut *r - } - .read_next_event()?; - match next_event { - JsonEvent::ObjectKey(key) => match key.as_ref() { - "type" => state = Some(State::Type), - "value" => state = Some(State::Value), - "xml:lang" => state = Some(State::Lang), - "datatype" => state = Some(State::Datatype), - #[cfg(feature = "rdf-star")] - "subject" => subject = Some(read_value(reader, number_of_recursive_calls + 1)?), - #[cfg(feature = "rdf-star")] - "predicate" => predicate = Some(read_value(reader, number_of_recursive_calls + 1)?), - #[cfg(feature = "rdf-star")] - "object" => object = Some(read_value(reader, number_of_recursive_calls + 1)?), - _ => { - return Err(QueryResultsSyntaxError::msg(format!( - "Unexpected key in term serialization: '{key}'" + Predicate(Box), + #[cfg(feature = "rdf-star")] + Object(Box), +} + +enum TermType { + Uri, + BNode, + Literal, + #[cfg(feature = "rdf-star")] + Triple, +} + +impl JsonInnerTermReader { + fn read_event( + &mut self, + event: JsonEvent<'_>, + ) -> Result, QueryResultsSyntaxError> { + match &mut self.state { + JsonInnerTermReaderState::Start => { + if event == JsonEvent::StartObject { + self.state = JsonInnerTermReaderState::Middle; + Ok(None) + } else { + Err(QueryResultsSyntaxError::msg( + "RDF terms must be encoded using objects", )) - .into()) - } - }, - JsonEvent::StartObject => { - if state != Some(State::Value) { - return Err(QueryResultsSyntaxError::msg( - "Unexpected nested object in term serialization", - ) - .into()); } } - JsonEvent::String(s) => match state { - Some(State::Type) => { - match s.as_ref() { - "uri" => t = Some(Type::Uri), - "bnode" => t = Some(Type::BNode), - "literal" | "typed-literal" => t = Some(Type::Literal), - #[cfg(feature = "rdf-star")] - "triple" => t = Some(Type::Triple), + JsonInnerTermReaderState::Middle => match event { + JsonEvent::ObjectKey(object_key) => { + self.state = match object_key.as_ref() { + "type" => JsonInnerTermReaderState::TermType, + "value" => JsonInnerTermReaderState::Value, + "datatype" => JsonInnerTermReaderState::Datatype, + "xml:lang" => JsonInnerTermReaderState::Lang, _ => { return Err(QueryResultsSyntaxError::msg(format!( - "Unexpected term type: '{s}'" - )) - .into()) + "Unsupported term key: {object_key}" + ))); } }; - state = None; - } - Some(State::Value) => { - value = Some(s.into_owned()); - state = None; + Ok(None) } - Some(State::Lang) => { - lang = Some(s.into_owned()); - state = None; - } - Some(State::Datatype) => { - datatype = Some(NamedNode::new(s).map_err(|e| { - QueryResultsSyntaxError::msg(format!("Invalid datatype IRI: {e}")) - })?); - state = None; - } - _ => (), // impossible - }, - JsonEvent::EndObject => { - if let Some(s) = state { - if s == State::Value { - state = None; // End of triple - } else { - return Err(QueryResultsSyntaxError::msg( - "Term description values should be string", - ) - .into()); - } - } else { - return match t { + JsonEvent::EndObject => { + self.state = JsonInnerTermReaderState::Start; + match self.term_type.take() { None => Err(QueryResultsSyntaxError::msg( "Term serialization should have a 'type' key", - ) - .into()), - Some(Type::Uri) => Ok(NamedNode::new(value.ok_or_else(|| { - QueryResultsSyntaxError::msg( - "uri serialization should have a 'value' key", - ) - })?) - .map_err(|e| { - QueryResultsSyntaxError::msg(format!("Invalid uri value: {e}")) - })? - .into()), - Some(Type::BNode) => Ok(BlankNode::new(value.ok_or_else(|| { - QueryResultsSyntaxError::msg( - "bnode serialization should have a 'value' key", - ) - })?) - .map_err(|e| { - QueryResultsSyntaxError::msg(format!("Invalid bnode value: {e}")) - })? - .into()), - Some(Type::Literal) => { - let value = value.ok_or_else(|| { + )), + Some(TermType::Uri) => Ok(Some( + NamedNode::new(self.value.take().ok_or_else(|| { + QueryResultsSyntaxError::msg( + "uri serialization should have a 'value' key", + ) + })?) + .map_err(|e| { + QueryResultsSyntaxError::msg(format!("Invalid uri value: {e}")) + })? + .into(), + )), + Some(TermType::BNode) => Ok(Some( + BlankNode::new(self.value.take().ok_or_else(|| { + QueryResultsSyntaxError::msg( + "bnode serialization should have a 'value' key", + ) + })?) + .map_err(|e| { + QueryResultsSyntaxError::msg(format!("Invalid bnode value: {e}")) + })? + .into(), + )), + Some(TermType::Literal) => { + let value = self.value.take().ok_or_else(|| { QueryResultsSyntaxError::msg( "literal serialization should have a 'value' key", ) })?; - Ok(match lang { - Some(lang) => { - if let Some(datatype) = datatype { - if datatype.as_ref() != rdf::LANG_STRING { - return Err(QueryResultsSyntaxError::msg(format!( - "xml:lang value '{lang}' provided with the datatype {datatype}" - )).into()) + Ok(Some(match self.lang.take() { + Some(lang) => { + if let Some(datatype) = &self.datatype { + if datatype.as_ref() != rdf::LANG_STRING { + return Err(QueryResultsSyntaxError::msg(format!( + "xml:lang value '{lang}' provided with the datatype {datatype}" + ))); + } } + Literal::new_language_tagged_literal(value, &*lang) + .map_err(|e| { + QueryResultsSyntaxError::msg(format!( + "Invalid xml:lang value '{lang}': {e}" + )) + })? } - Literal::new_language_tagged_literal(value, &*lang).map_err(|e| { - QueryResultsSyntaxError::msg(format!("Invalid xml:lang value '{lang}': {e}")) - })? - } - None => if let Some(datatype) = datatype { - Literal::new_typed_literal(value, datatype) - } else { - Literal::new_simple_literal(value) - } - } - .into()) + None => { + if let Some(datatype) = self.datatype.take() { + Literal::new_typed_literal(value, datatype) + } else { + Literal::new_simple_literal(value) + } + } + }.into())) } #[cfg(feature = "rdf-star")] - Some(Type::Triple) => Ok(Triple::new( - match subject.ok_or_else(|| { - QueryResultsSyntaxError::msg( - "triple serialization should have a 'subject' key", - ) - })? { - Term::NamedNode(subject) => subject.into(), - Term::BlankNode(subject) => subject.into(), - Term::Triple(subject) => Subject::Triple(subject), - Term::Literal(_) => { - return Err(QueryResultsSyntaxError::msg( - "The 'subject' value should not be a literal", + Some(TermType::Triple) => Ok(Some( + Triple::new( + match self.subject.take().ok_or_else(|| { + QueryResultsSyntaxError::msg( + "triple serialization should have a 'subject' key", ) - .into()) - } - }, - match predicate.ok_or_else(|| { - QueryResultsSyntaxError::msg( - "triple serialization should have a 'predicate' key", - ) - })? { - Term::NamedNode(predicate) => predicate, - _ => { - return Err(QueryResultsSyntaxError::msg( - "The 'predicate' value should be a uri", + })? { + Term::NamedNode(subject) => subject.into(), + Term::BlankNode(subject) => subject.into(), + Term::Triple(subject) => Subject::Triple(subject), + Term::Literal(_) => { + return Err(QueryResultsSyntaxError::msg( + "The 'subject' value should not be a literal", + )); + } + }, + match self.predicate.take().ok_or_else(|| { + QueryResultsSyntaxError::msg( + "triple serialization should have a 'predicate' key", ) - .into()) - } - }, - object.ok_or_else(|| { - QueryResultsSyntaxError::msg( - "triple serialization should have a 'object' key", - ) - })?, - ) - .into()), - }; - } - } - _ => return Err(QueryResultsSyntaxError::msg("Invalid term serialization").into()), - } - } -} - -fn read_head( - reader: &mut FromReadJsonReader, -) -> Result, QueryResultsParseError> { - if reader.read_next_event()? != JsonEvent::StartObject { - return Err(QueryResultsSyntaxError::msg("head should be an object").into()); - } - let mut variables = Vec::new(); - loop { - match reader.read_next_event()? { - JsonEvent::ObjectKey(key) => match key.as_ref() { - "vars" => { - if reader.read_next_event()? != JsonEvent::StartArray { - return Err(QueryResultsSyntaxError::msg( - "Variable list should be an array", - ) - .into()); + })? { + Term::NamedNode(predicate) => predicate, + _ => { + return Err(QueryResultsSyntaxError::msg( + "The 'predicate' value should be a uri", + )); + } + }, + self.object.take().ok_or_else(|| { + QueryResultsSyntaxError::msg( + "triple serialization should have a 'object' key", + ) + })?, + ) + .into(), + )), } - loop { - match reader.read_next_event()? { - JsonEvent::String(s) => { - let new_var = Variable::new(s.as_ref()).map_err(|e| { - QueryResultsSyntaxError::msg(format!( - "Invalid variable declaration '{s}': {e}" - )) - })?; - if variables.contains(&new_var) { - return Err(QueryResultsSyntaxError::msg(format!( - "The variable {new_var} is declared twice" - )) - .into()); - } - variables.push(new_var); - } - JsonEvent::EndArray => break, - _ => { - return Err(QueryResultsSyntaxError::msg( - "Variable names should be strings", - ) - .into()) - } + } + _ => unreachable!(), + }, + JsonInnerTermReaderState::TermType => { + self.state = JsonInnerTermReaderState::Middle; + if let JsonEvent::String(value) = event { + match value.as_ref() { + "uri" => { + self.term_type = Some(TermType::Uri); + Ok(None) + } + "bnode" => { + self.term_type = Some(TermType::BNode); + Ok(None) } + "literal" | "typed-literal" => { + self.term_type = Some(TermType::Literal); + Ok(None) + } + #[cfg(feature = "rdf-star")] + "triple" => { + self.term_type = Some(TermType::Triple); + Ok(None) + } + _ => Err(QueryResultsSyntaxError::msg(format!( + "Unexpected term type: '{value}'" + ))), } + } else { + Err(QueryResultsSyntaxError::msg("Term type must be a string")) } - "link" => { - if reader.read_next_event()? != JsonEvent::StartArray { - return Err(QueryResultsSyntaxError::msg( - "Variable list should be an array", - ) - .into()); - } - loop { - match reader.read_next_event()? { - JsonEvent::String(_) => (), - JsonEvent::EndArray => break, - _ => { - return Err(QueryResultsSyntaxError::msg( - "Link names should be strings", - ) - .into()) - } + } + JsonInnerTermReaderState::Value => match event { + JsonEvent::String(value) => { + self.value = Some(value.into_owned()); + self.state = JsonInnerTermReaderState::Middle; + Ok(None) + } + #[cfg(feature = "rdf-star")] + JsonEvent::StartObject => { + self.state = JsonInnerTermReaderState::InValue; + Ok(None) + } + _ => { + self.state = JsonInnerTermReaderState::Middle; + + Err(QueryResultsSyntaxError::msg("Term value must be a string")) + } + }, + JsonInnerTermReaderState::Lang => { + let result = if let JsonEvent::String(value) = event { + self.lang = Some(value.into_owned()); + Ok(None) + } else { + Err(QueryResultsSyntaxError::msg("Term lang must be strings")) + }; + self.state = JsonInnerTermReaderState::Middle; + + result + } + JsonInnerTermReaderState::Datatype => { + let result = if let JsonEvent::String(value) = event { + match NamedNode::new(value) { + Ok(datatype) => { + self.datatype = Some(datatype); + Ok(None) } + Err(e) => Err(QueryResultsSyntaxError::msg(format!( + "Invalid datatype: {e}" + ))), } + } else { + Err(QueryResultsSyntaxError::msg("Term lang must be strings")) + }; + self.state = JsonInnerTermReaderState::Middle; + + result + } + #[cfg(feature = "rdf-star")] + JsonInnerTermReaderState::InValue => match event { + JsonEvent::ObjectKey(object_key) => { + self.state = match object_key.as_ref() { + "subject" => JsonInnerTermReaderState::Subject(Box::default()), + "predicate" => JsonInnerTermReaderState::Predicate(Box::default()), + "object" => JsonInnerTermReaderState::Object(Box::default()), + _ => { + return Err(QueryResultsSyntaxError::msg(format!( + "Unsupported value key: {object_key}" + ))); + } + }; + Ok(None) + } + JsonEvent::EndObject => { + self.state = JsonInnerTermReaderState::Middle; + Ok(None) } - _ => ignore_value(reader)?, + _ => unreachable!(), }, - JsonEvent::EndObject => return Ok(variables), - _ => return Err(QueryResultsSyntaxError::msg("Invalid head serialization").into()), - } - } -} - -fn ignore_value(reader: &mut FromReadJsonReader) -> Result<(), QueryResultsParseError> { - let mut nesting = 0; - loop { - match reader.read_next_event()? { - JsonEvent::Boolean(_) - | JsonEvent::Null - | JsonEvent::Number(_) - | JsonEvent::String(_) => { - if nesting == 0 { - return Ok(()); + #[cfg(feature = "rdf-star")] + JsonInnerTermReaderState::Subject(ref mut inner_state) => { + if let Some(term) = inner_state.read_event(event)? { + self.state = JsonInnerTermReaderState::InValue; + self.subject = Some(term); } + Ok(None) } - JsonEvent::ObjectKey(_) => (), - JsonEvent::StartArray | JsonEvent::StartObject => nesting += 1, - JsonEvent::EndArray | JsonEvent::EndObject => { - nesting -= 1; - if nesting == 0 { - return Ok(()); + #[cfg(feature = "rdf-star")] + JsonInnerTermReaderState::Predicate(ref mut inner_state) => { + if let Some(term) = inner_state.read_event(event)? { + self.state = JsonInnerTermReaderState::InValue; + self.predicate = Some(term); } + Ok(None) } - JsonEvent::Eof => { - return Err(QueryResultsSyntaxError::msg("Unexpected end of file").into()) + #[cfg(feature = "rdf-star")] + JsonInnerTermReaderState::Object(ref mut inner_state) => { + if let Some(term) = inner_state.read_event(event)? { + self.state = JsonInnerTermReaderState::InValue; + self.object = Some(term); + } + Ok(None) } } } } + +pub struct JsonBufferedSolutionsIterator { + mapping: BTreeMap, + bindings: std::vec::IntoIter<(Vec, Vec)>, +} + +impl JsonBufferedSolutionsIterator { + fn next(&mut self) -> Result>>, QueryResultsParseError> { + let Some((variables, values)) = self.bindings.next() else { + return Ok(None); + }; + let mut new_bindings = vec![None; self.mapping.len()]; + for (variable, value) in variables.into_iter().zip(values) { + let k = *self.mapping.get(&variable).ok_or_else(|| { + QueryResultsSyntaxError::msg(format!( + "The variable {variable} has not been defined in the header" + )) + })?; + new_bindings[k] = Some(value); + } + Ok(Some(new_bindings)) + } +} diff --git a/lib/sparesults/src/lib.rs b/lib/sparesults/src/lib.rs index ea3135c4..bfe45051 100644 --- a/lib/sparesults/src/lib.rs +++ b/lib/sparesults/src/lib.rs @@ -16,5 +16,9 @@ mod xml; pub use crate::error::{QueryResultsParseError, QueryResultsSyntaxError, TextPosition}; pub use crate::format::QueryResultsFormat; pub use crate::parser::{FromReadQueryResultsReader, FromReadSolutionsReader, QueryResultsParser}; +#[cfg(feature = "async-tokio")] +pub use crate::parser::{FromTokioAsyncReadQueryResultsReader, FromTokioAsyncReadSolutionsReader}; +#[cfg(feature = "async-tokio")] +pub use crate::serializer::ToTokioAsyncWriteSolutionsWriter; pub use crate::serializer::{QueryResultsSerializer, ToWriteSolutionsWriter}; pub use crate::solution::QuerySolution; diff --git a/lib/sparesults/src/parser.rs b/lib/sparesults/src/parser.rs index f95d9355..602a5e61 100644 --- a/lib/sparesults/src/parser.rs +++ b/lib/sparesults/src/parser.rs @@ -1,12 +1,18 @@ use crate::csv::{TsvQueryResultsReader, TsvSolutionsReader}; use crate::error::{QueryResultsParseError, QueryResultsSyntaxError}; use crate::format::QueryResultsFormat; -use crate::json::{JsonQueryResultsReader, JsonSolutionsReader}; +use crate::json::{FromReadJsonQueryResultsReader, FromReadJsonSolutionsReader}; +#[cfg(feature = "async-tokio")] +use crate::json::{ + FromTokioAsyncReadJsonQueryResultsReader, FromTokioAsyncReadJsonSolutionsReader, +}; use crate::solution::QuerySolution; use crate::xml::{XmlQueryResultsReader, XmlSolutionsReader}; use oxrdf::Variable; use std::io::Read; use std::sync::Arc; +#[cfg(feature = "async-tokio")] +use tokio::io::AsyncRead; /// Parsers for [SPARQL query](https://www.w3.org/TR/sparql11-query/) results serialization formats. /// @@ -45,24 +51,24 @@ impl QueryResultsParser { Self { format } } - /// Reads a result file. + /// Reads a result file from a [`Read`] implementation. /// - /// Reads are buffered. + /// Reads are automatically buffered. /// /// Example in XML (the API is the same for JSON and TSV): /// ``` /// use sparesults::{QueryResultsFormat, QueryResultsParser, FromReadQueryResultsReader}; /// use oxrdf::{Literal, Variable}; /// - /// let json_parser = QueryResultsParser::from_format(QueryResultsFormat::Xml); + /// let xml_parser = QueryResultsParser::from_format(QueryResultsFormat::Xml); /// /// // boolean - /// if let FromReadQueryResultsReader::Boolean(v) = json_parser.parse_read(br#"true"#.as_slice())? { + /// if let FromReadQueryResultsReader::Boolean(v) = xml_parser.parse_read(br#"true"#.as_slice())? { /// assert_eq!(v, true); /// } /// /// // solutions - /// if let FromReadQueryResultsReader::Solutions(solutions) = json_parser.parse_read(br#"test"#.as_slice())? { + /// if let FromReadQueryResultsReader::Solutions(solutions) = xml_parser.parse_read(br#"test"#.as_slice())? { /// assert_eq!(solutions.variables(), &[Variable::new_unchecked("foo"), Variable::new_unchecked("bar")]); /// for solution in solutions { /// assert_eq!(solution?.iter().collect::>(), vec![(&Variable::new_unchecked("foo"), &Literal::from("test").into())]); @@ -82,17 +88,17 @@ impl QueryResultsParser { variables, } => FromReadQueryResultsReader::Solutions(FromReadSolutionsReader { variables: variables.into(), - solutions: SolutionsReaderKind::Xml(solutions), + solutions: FromReadSolutionsReaderKind::Xml(solutions), }), }, - QueryResultsFormat::Json => match JsonQueryResultsReader::read(reader)? { - JsonQueryResultsReader::Boolean(r) => FromReadQueryResultsReader::Boolean(r), - JsonQueryResultsReader::Solutions { + QueryResultsFormat::Json => match FromReadJsonQueryResultsReader::read(reader)? { + FromReadJsonQueryResultsReader::Boolean(r) => FromReadQueryResultsReader::Boolean(r), + FromReadJsonQueryResultsReader::Solutions { solutions, variables, } => FromReadQueryResultsReader::Solutions(FromReadSolutionsReader { variables: variables.into(), - solutions: SolutionsReaderKind::Json(solutions), + solutions: FromReadSolutionsReaderKind::Json(solutions), }), }, QueryResultsFormat::Csv => return Err(QueryResultsSyntaxError::msg("CSV SPARQL results syntax is lossy and can't be parsed to a proper RDF representation").into()), @@ -103,7 +109,7 @@ impl QueryResultsParser { variables, } => FromReadQueryResultsReader::Solutions(FromReadSolutionsReader { variables: variables.into(), - solutions: SolutionsReaderKind::Tsv(solutions), + solutions: FromReadSolutionsReaderKind::Tsv(solutions), }), }, }) @@ -116,6 +122,56 @@ impl QueryResultsParser { ) -> Result, QueryResultsParseError> { self.parse_read(reader) } + + /// Reads a result file from a Tokio [`AsyncRead`] implementation. + /// + /// Reads are automatically buffered. + /// + /// Example in XML (the API is the same for JSON and TSV): + /// ```no_run + /// use sparesults::{QueryResultsFormat, QueryResultsParser, FromTokioAsyncReadQueryResultsReader}; + /// use oxrdf::{Literal, Variable}; + /// + /// # #[tokio::main(flavor = "current_thread")] + /// # async fn main() -> Result<(), sparesults::QueryResultsParseError> { + /// let xml_parser = QueryResultsParser::from_format(QueryResultsFormat::Xml); + /// + /// // boolean + /// if let FromTokioAsyncReadQueryResultsReader::Boolean(v) = xml_parser.parse_tokio_async_read(br#"true"#.as_slice()).await? { + /// assert_eq!(v, true); + /// } + /// + /// // solutions + /// if let FromTokioAsyncReadQueryResultsReader::Solutions(mut solutions) = xml_parser.parse_tokio_async_read(br#"test"#.as_slice()).await? { + /// assert_eq!(solutions.variables(), &[Variable::new_unchecked("foo"), Variable::new_unchecked("bar")]); + /// while let Some(solution) = solutions.next().await { + /// assert_eq!(solution?.iter().collect::>(), vec![(&Variable::new_unchecked("foo"), &Literal::from("test").into())]); + /// } + /// } + /// # Ok(()) + /// # } + /// ``` + #[cfg(feature = "async-tokio")] + pub async fn parse_tokio_async_read( + &self, + reader: R, + ) -> Result, QueryResultsParseError> { + Ok(match self.format { + QueryResultsFormat::Xml => return Err(QueryResultsSyntaxError::msg("The XML query results parser does not support Tokio AsyncRead yet").into()), + QueryResultsFormat::Json => match FromTokioAsyncReadJsonQueryResultsReader::read(reader).await? { + FromTokioAsyncReadJsonQueryResultsReader::Boolean(r) => FromTokioAsyncReadQueryResultsReader::Boolean(r), + FromTokioAsyncReadJsonQueryResultsReader::Solutions { + solutions, + variables, + } => FromTokioAsyncReadQueryResultsReader::Solutions(FromTokioAsyncReadSolutionsReader { + variables: variables.into(), + solutions: FromTokioAsyncReadSolutionsReaderKind::Json(solutions), + }), + }, + QueryResultsFormat::Csv => return Err(QueryResultsSyntaxError::msg("CSV SPARQL results syntax is lossy and can't be parsed to a proper RDF representation").into()), + QueryResultsFormat::Tsv => return Err(QueryResultsSyntaxError::msg("The TSV query results parser does not support Tokio AsyncRead yet").into()), + }) + } } impl From for QueryResultsParser { @@ -133,16 +189,16 @@ impl From for QueryResultsParser { /// use oxrdf::{Literal, Variable}; /// use sparesults::{FromReadQueryResultsReader, QueryResultsFormat, QueryResultsParser}; /// -/// let json_parser = QueryResultsParser::from_format(QueryResultsFormat::Tsv); +/// let tsv_parser = QueryResultsParser::from_format(QueryResultsFormat::Tsv); /// /// // boolean -/// if let FromReadQueryResultsReader::Boolean(v) = json_parser.parse_read(b"true".as_slice())? { +/// if let FromReadQueryResultsReader::Boolean(v) = tsv_parser.parse_read(b"true".as_slice())? { /// assert_eq!(v, true); /// } /// /// // solutions /// if let FromReadQueryResultsReader::Solutions(solutions) = -/// json_parser.parse_read(b"?foo\t?bar\n\"test\"\t".as_slice())? +/// tsv_parser.parse_read(b"?foo\t?bar\n\"test\"\t".as_slice())? /// { /// assert_eq!( /// solutions.variables(), @@ -188,12 +244,12 @@ pub enum FromReadQueryResultsReader { /// ``` pub struct FromReadSolutionsReader { variables: Arc<[Variable]>, - solutions: SolutionsReaderKind, + solutions: FromReadSolutionsReaderKind, } -enum SolutionsReaderKind { +enum FromReadSolutionsReaderKind { Xml(XmlSolutionsReader), - Json(JsonSolutionsReader), + Json(FromReadJsonSolutionsReader), Tsv(TsvSolutionsReader), } @@ -205,9 +261,9 @@ impl FromReadSolutionsReader { /// use oxrdf::Variable; /// use sparesults::{FromReadQueryResultsReader, QueryResultsFormat, QueryResultsParser}; /// - /// let json_parser = QueryResultsParser::from_format(QueryResultsFormat::Tsv); + /// let tsv_parser = QueryResultsParser::from_format(QueryResultsFormat::Tsv); /// if let FromReadQueryResultsReader::Solutions(solutions) = - /// json_parser.parse_read(b"?foo\t?bar\n\"ex1\"\t\"ex2\"".as_slice())? + /// tsv_parser.parse_read(b"?foo\t?bar\n\"ex1\"\t\"ex2\"".as_slice())? /// { /// assert_eq!( /// solutions.variables(), @@ -231,9 +287,141 @@ impl Iterator for FromReadSolutionsReader { fn next(&mut self) -> Option { Some( match &mut self.solutions { - SolutionsReaderKind::Xml(reader) => reader.read_next(), - SolutionsReaderKind::Json(reader) => reader.read_next(), - SolutionsReaderKind::Tsv(reader) => reader.read_next(), + FromReadSolutionsReaderKind::Xml(reader) => reader.read_next(), + FromReadSolutionsReaderKind::Json(reader) => reader.read_next(), + FromReadSolutionsReaderKind::Tsv(reader) => reader.read_next(), + } + .transpose()? + .map(|values| (Arc::clone(&self.variables), values).into()), + ) + } +} + +/// The reader for a given read of a results file. +/// +/// It is either a read boolean ([`bool`]) or a streaming reader of a set of solutions ([`FromReadSolutionsReader`]). +/// +/// Example in TSV (the API is the same for JSON and XML): +/// ```no_run +/// use oxrdf::{Literal, Variable}; +/// use sparesults::{ +/// FromTokioAsyncReadQueryResultsReader, QueryResultsFormat, QueryResultsParser, +/// }; +/// +/// # #[tokio::main(flavor = "current_thread")] +/// # async fn main() -> Result<(), sparesults::QueryResultsParseError> { +/// let tsv_parser = QueryResultsParser::from_format(QueryResultsFormat::Tsv); +/// +/// // boolean +/// if let FromTokioAsyncReadQueryResultsReader::Boolean(v) = tsv_parser +/// .parse_tokio_async_read(b"true".as_slice()) +/// .await? +/// { +/// assert_eq!(v, true); +/// } +/// +/// // solutions +/// if let FromTokioAsyncReadQueryResultsReader::Solutions(mut solutions) = tsv_parser +/// .parse_tokio_async_read(b"?foo\t?bar\n\"test\"\t".as_slice()) +/// .await? +/// { +/// assert_eq!( +/// solutions.variables(), +/// &[ +/// Variable::new_unchecked("foo"), +/// Variable::new_unchecked("bar") +/// ] +/// ); +/// while let Some(solution) = solutions.next().await { +/// assert_eq!( +/// solution?.iter().collect::>(), +/// vec![( +/// &Variable::new_unchecked("foo"), +/// &Literal::from("test").into() +/// )] +/// ); +/// } +/// } +/// # Ok(()) +/// # } +/// ``` +#[cfg(feature = "async-tokio")] +pub enum FromTokioAsyncReadQueryResultsReader { + Solutions(FromTokioAsyncReadSolutionsReader), + Boolean(bool), +} + +/// A streaming reader of a set of [`QuerySolution`] solutions. +/// +/// It implements the [`Iterator`] API to iterate over the solutions. +/// +/// Example in JSON (the API is the same for XML and TSV): +/// ``` +/// use sparesults::{QueryResultsFormat, QueryResultsParser, FromTokioAsyncReadQueryResultsReader}; +/// use oxrdf::{Literal, Variable}; +/// +/// # #[tokio::main(flavor = "current_thread")] +/// # async fn main() -> Result<(), sparesults::QueryResultsParseError> { +/// let json_parser = QueryResultsParser::from_format(QueryResultsFormat::Json); +/// if let FromTokioAsyncReadQueryResultsReader::Solutions(mut solutions) = json_parser.parse_tokio_async_read(br#"{"head":{"vars":["foo","bar"]},"results":{"bindings":[{"foo":{"type":"literal","value":"test"}}]}}"#.as_slice()).await? { +/// assert_eq!(solutions.variables(), &[Variable::new_unchecked("foo"), Variable::new_unchecked("bar")]); +/// while let Some(solution) = solutions.next().await { +/// assert_eq!(solution?.iter().collect::>(), vec![(&Variable::new_unchecked("foo"), &Literal::from("test").into())]); +/// } +/// } +/// # Ok(()) +/// # } +/// ``` +#[cfg(feature = "async-tokio")] +pub struct FromTokioAsyncReadSolutionsReader { + variables: Arc<[Variable]>, + solutions: FromTokioAsyncReadSolutionsReaderKind, +} + +#[cfg(feature = "async-tokio")] +enum FromTokioAsyncReadSolutionsReaderKind { + Json(FromTokioAsyncReadJsonSolutionsReader), +} + +#[cfg(feature = "async-tokio")] +impl FromTokioAsyncReadSolutionsReader { + /// Ordered list of the declared variables at the beginning of the results. + /// + /// Example in TSV (the API is the same for JSON and XML): + /// ```no_run + /// use oxrdf::Variable; + /// use sparesults::{ + /// FromTokioAsyncReadQueryResultsReader, QueryResultsFormat, QueryResultsParser, + /// }; + /// + /// # #[tokio::main(flavor = "current_thread")] + /// # async fn main() -> Result<(), sparesults::QueryResultsParseError> { + /// let tsv_parser = QueryResultsParser::from_format(QueryResultsFormat::Tsv); + /// if let FromTokioAsyncReadQueryResultsReader::Solutions(solutions) = tsv_parser + /// .parse_tokio_async_read(b"?foo\t?bar\n\"ex1\"\t\"ex2\"".as_slice()) + /// .await? + /// { + /// assert_eq!( + /// solutions.variables(), + /// &[ + /// Variable::new_unchecked("foo"), + /// Variable::new_unchecked("bar") + /// ] + /// ); + /// } + /// # Ok(()) + /// # } + /// ``` + #[inline] + pub fn variables(&self) -> &[Variable] { + &self.variables + } + + /// Reads the next solution or returns `None` if the file is finished. + pub async fn next(&mut self) -> Option> { + Some( + match &mut self.solutions { + FromTokioAsyncReadSolutionsReaderKind::Json(reader) => reader.read_next().await, } .transpose()? .map(|values| (Arc::clone(&self.variables), values).into()),