Compare commits

...

5 Commits

Author SHA1 Message Date
Tpt add1dff458 Adds OX_LATERAL OPTIONAL and OX_LATERAL GRAPH 2 years ago
Tpt e922d3293b Fixes LATERAL inside of OPTIONAL behavior 2 years ago
Tpt 2d7eac932f Renames tests 2 years ago
Tpt 15819907af Adds experimental OX_LATERAL operation 2 years ago
Tpt 6262e02edf Initialise v0.4 dev branch 2 years ago
  1. 2
      .github/workflows/tests.yml
  2. 12
      Cargo.lock
  3. 4
      js/Cargo.toml
  4. 4
      lib/Cargo.toml
  5. 3
      lib/spargebra/Cargo.toml
  6. 95
      lib/spargebra/src/algebra.rs
  7. 82
      lib/spargebra/src/parser.rs
  8. 24
      lib/src/sparql/eval.rs
  9. 169
      lib/src/sparql/plan.rs
  10. 78
      lib/src/sparql/plan_builder.rs
  11. 4
      python/Cargo.toml
  12. 4
      server/Cargo.toml
  13. 4
      testsuite/Cargo.toml
  14. 2
      testsuite/oxigraph-tests/sparql-results/manifest.ttl
  15. 5
      testsuite/oxigraph-tests/sparql/lateral/basic_input.ttl
  16. 6
      testsuite/oxigraph-tests/sparql/lateral/graph.rq
  17. 56
      testsuite/oxigraph-tests/sparql/lateral/manifest.ttl
  18. 6
      testsuite/oxigraph-tests/sparql/lateral/optional.rq
  19. 17
      testsuite/oxigraph-tests/sparql/lateral/simple.srx
  20. 6
      testsuite/oxigraph-tests/sparql/lateral/subselect.rq
  21. 41
      testsuite/oxigraph-tests/sparql/lateral/subselect.srx
  22. 6
      testsuite/oxigraph-tests/sparql/lateral/subselect_explicit_aggregate.rq
  23. 25
      testsuite/oxigraph-tests/sparql/lateral/subselect_explicit_aggregate.srx
  24. 6
      testsuite/oxigraph-tests/sparql/lateral/subselect_implicit_aggregate.rq
  25. 30
      testsuite/oxigraph-tests/sparql/lateral/subselect_implicit_aggregate.srx
  26. 6
      testsuite/oxigraph-tests/sparql/lateral/subselect_inside_optional.rq
  27. 57
      testsuite/oxigraph-tests/sparql/lateral/subselect_inside_optional.srx
  28. 1
      testsuite/oxigraph-tests/sparql/manifest.ttl

@ -4,9 +4,11 @@ on:
push:
branches:
- main
- dev
pull_request:
branches:
- main
- dev
schedule:
- cron: "0 0 * * 0"

12
Cargo.lock generated

@ -744,7 +744,7 @@ dependencies = [
[[package]]
name = "oxigraph"
version = "0.3.8"
version = "0.4.0-alpha"
dependencies = [
"criterion",
"digest",
@ -777,7 +777,7 @@ dependencies = [
[[package]]
name = "oxigraph_js"
version = "0.3.8"
version = "0.4.0-alpha"
dependencies = [
"console_error_panic_hook",
"js-sys",
@ -788,7 +788,7 @@ dependencies = [
[[package]]
name = "oxigraph_server"
version = "0.3.8"
version = "0.4.0-alpha"
dependencies = [
"clap 4.0.18",
"flate2",
@ -803,7 +803,7 @@ dependencies = [
[[package]]
name = "oxigraph_testsuite"
version = "0.3.8"
version = "0.4.0-alpha"
dependencies = [
"anyhow",
"clap 4.0.18",
@ -1035,7 +1035,7 @@ dependencies = [
[[package]]
name = "pyoxigraph"
version = "0.3.8"
version = "0.4.0-alpha"
dependencies = [
"oxigraph",
"pyo3",
@ -1374,7 +1374,7 @@ dependencies = [
[[package]]
name = "spargebra"
version = "0.2.2"
version = "0.3.0-alpha"
dependencies = [
"oxilangtag",
"oxiri",

@ -1,6 +1,6 @@
[package]
name = "oxigraph_js"
version = "0.3.8"
version = "0.4.0-alpha"
authors = ["Tpt <thomas@pellissier-tanon.fr>"]
license = "MIT OR Apache-2.0"
readme = "README.md"
@ -14,7 +14,7 @@ crate-type = ["cdylib"]
name = "oxigraph"
[dependencies]
oxigraph = { version = "0.3.8", path="../lib" }
oxigraph = { version = "0.4.0-alpha", path="../lib" }
wasm-bindgen = "0.2"
js-sys = "0.3"
console_error_panic_hook = "0.1"

@ -1,6 +1,6 @@
[package]
name = "oxigraph"
version = "0.3.8"
version = "0.4.0-alpha"
authors = ["Tpt <thomas@pellissier-tanon.fr>"]
license = "MIT OR Apache-2.0"
readme = "README.md"
@ -38,7 +38,7 @@ siphasher = "0.3"
lazy_static = "1"
sysinfo = "0.26"
oxrdf = { version = "0.1.0", path="oxrdf", features = ["rdf-star"] }
spargebra = { version = "0.2.2", path="spargebra", features = ["rdf-star"] }
spargebra = { version = "0.3.0-alpha", path="spargebra", features = ["rdf-star", "ex-lateral"] }
sparesults = { version = "0.1.1", path="sparesults", features = ["rdf-star"] }
[target.'cfg(not(target_arch = "wasm32"))'.dependencies]

@ -1,6 +1,6 @@
[package]
name = "spargebra"
version = "0.2.2"
version = "0.3.0-alpha"
authors = ["Tpt <thomas@pellissier-tanon.fr>"]
license = "MIT OR Apache-2.0"
readme = "README.md"
@ -15,6 +15,7 @@ edition = "2021"
[features]
default = []
rdf-star = ["oxrdf/rdf-star"]
ex-lateral = []
[dependencies]
peg = "0.8"

@ -519,6 +519,8 @@ pub enum GraphPattern {
path: PropertyPathExpression,
object: TermPattern,
},
/// A sequence of operation (lateral join): execute left and for each element execute right
Sequence { left: Box<Self>, right: Box<Self> },
/// [Join](https://www.w3.org/TR/sparql11-query/#defn_algJoin).
Join { left: Box<Self>, right: Box<Self> },
/// [LeftJoin](https://www.w3.org/TR/sparql11-query/#defn_algLeftJoin).
@ -527,6 +529,8 @@ pub enum GraphPattern {
right: Box<Self>,
expression: Option<Expression>,
},
/// Executes the left child and for each result tuple left join with right by injecting tuple to right
LeftSequence { left: Box<Self>, right: Box<Self> },
/// [Filter](https://www.w3.org/TR/sparql11-query/#defn_algFilter).
Filter { expr: Expression, inner: Box<Self> },
/// [Union](https://www.w3.org/TR/sparql11-query/#defn_algUnion).
@ -557,6 +561,8 @@ pub enum GraphPattern {
Project {
inner: Box<Self>,
variables: Vec<Variable>,
#[cfg(feature = "ex-lateral")]
lateral_variables: Vec<Variable>,
},
/// [Distinct](https://www.w3.org/TR/sparql11-query/#defn_algDistinct).
Distinct { inner: Box<Self> },
@ -614,6 +620,20 @@ impl GraphPattern {
right.fmt_sse(f)?;
write!(f, ")")
}
Self::Sequence { left, right } => {
write!(f, "(sequence ")?;
left.fmt_sse(f)?;
write!(f, " ")?;
right.fmt_sse(f)?;
write!(f, ")")
}
Self::LeftSequence { left, right } => {
write!(f, "(leftsequence ")?;
left.fmt_sse(f)?;
write!(f, " ")?;
right.fmt_sse(f)?;
write!(f, ")")
}
Self::LeftJoin {
left,
right,
@ -739,6 +759,34 @@ impl GraphPattern {
inner.fmt_sse(f)?;
write!(f, ")")
}
#[cfg(feature = "ex-lateral")]
Self::Project {
inner,
variables,
lateral_variables,
} => {
write!(f, "(project (")?;
for (i, v) in variables.iter().enumerate() {
if i > 0 {
write!(f, " ")?;
}
write!(f, "{}", v)?;
}
write!(f, ") ")?;
if !lateral_variables.is_empty() {
write!(f, "(lateral (")?;
for (i, v) in lateral_variables.iter().enumerate() {
if i > 0 {
write!(f, " ")?;
}
write!(f, "{}", v)?;
}
write!(f, ")) ")?;
}
inner.fmt_sse(f)?;
write!(f, ")")
}
#[cfg(not(feature = "ex-lateral"))]
Self::Project { inner, variables } => {
write!(f, "(project (")?;
for (i, v) in variables.iter().enumerate() {
@ -806,6 +854,20 @@ impl fmt::Display for GraphPattern {
write!(f, "{} {}", left, right)
}
}
Self::Sequence { left, right } => {
if cfg!(feature = "ex_lateral") && matches!(right.as_ref(), Self::Graph { .. }) {
// The second block might be considered as a modification of the first one.
write!(f, "{} OX_LATERAL {}", left, right)
} else {
write!(f, "{} {}", left, right)
}
}
#[cfg(feature = "ex-lateral")]
Self::LeftSequence { left, right } => {
write!(f, "{} OX_LATERAL OPTIONAL {{ {} }}", left, right)
}
#[cfg(not(feature = "ex-lateral"))]
Self::LeftSequence { .. } => Err(fmt::Error),
Self::LeftJoin {
left,
right,
@ -935,6 +997,8 @@ impl GraphPattern {
}
}
Self::Join { left, right }
| Self::Sequence { left, right }
| Self::LeftSequence { left, right }
| Self::LeftJoin { left, right, .. }
| Self::Union { left, right } => {
left.lookup_in_scope_variables(callback);
@ -1021,6 +1085,8 @@ impl<'a> fmt::Display for SparqlGraphRootPattern<'a> {
let mut start = 0;
let mut length = None;
let mut project: &[Variable] = &[];
#[cfg(feature = "ex-lateral")]
let mut lateral: &[Variable] = &[];
let mut child = self.pattern;
loop {
@ -1029,6 +1095,17 @@ impl<'a> fmt::Display for SparqlGraphRootPattern<'a> {
order = Some(expression);
child = inner;
}
#[cfg(feature = "ex-lateral")]
GraphPattern::Project {
inner,
variables,
lateral_variables,
} if project.is_empty() => {
project = variables;
lateral = lateral_variables;
child = inner;
}
#[cfg(not(feature = "ex-lateral"))]
GraphPattern::Project { inner, variables } if project.is_empty() => {
project = variables;
child = inner;
@ -1051,6 +1128,20 @@ impl<'a> fmt::Display for SparqlGraphRootPattern<'a> {
child = inner;
}
p => {
#[cfg(feature = "ex-lateral")]
let mut closing_brackets = false;
#[cfg(feature = "ex-lateral")]
if !lateral.is_empty() {
write!(f, "OX_LATERAL (")?;
for (i, v) in lateral.iter().enumerate() {
if i > 0 {
write!(f, " ")?;
}
write!(f, "{}", v)?;
}
write!(f, ") {{")?;
closing_brackets = true;
}
write!(f, "SELECT")?;
if distinct {
write!(f, " DISTINCT")?;
@ -1081,6 +1172,10 @@ impl<'a> fmt::Display for SparqlGraphRootPattern<'a> {
if let Some(length) = length {
write!(f, " LIMIT {}", length)?;
}
#[cfg(feature = "ex-lateral")]
if closing_brackets {
write!(f, "}}")?;
}
return Ok(());
}
}

@ -364,9 +364,26 @@ enum PartialGraphPattern {
Bind(Expression, Variable),
Filter(Expression),
Other(GraphPattern),
#[cfg(feature = "ex-lateral")]
LateralOptional(GraphPattern),
#[cfg(feature = "ex-lateral")]
Lateral(GraphPattern),
}
fn new_join(l: GraphPattern, r: GraphPattern) -> GraphPattern {
new_join_like(l, r, |left, right| GraphPattern::Join { left, right })
}
#[cfg(feature = "ex-lateral")]
fn new_sequence(l: GraphPattern, r: GraphPattern) -> GraphPattern {
new_join_like(l, r, |left, right| GraphPattern::Sequence { left, right })
}
fn new_join_like(
l: GraphPattern,
r: GraphPattern,
cons: impl FnOnce(Box<GraphPattern>, Box<GraphPattern>) -> GraphPattern,
) -> GraphPattern {
//Avoid to output empty BGPs
if let GraphPattern::Bgp { patterns: pl } = &l {
if pl.is_empty() {
@ -389,10 +406,7 @@ fn new_join(l: GraphPattern, r: GraphPattern) -> GraphPattern {
{
other
}
(l, r) => GraphPattern::Join {
left: Box::new(l),
right: Box::new(r),
},
(l, r) => cons(Box::new(l), Box::new(r)),
}
}
@ -562,6 +576,8 @@ fn build_select(
m = GraphPattern::Project {
inner: Box::new(m),
variables: pv,
#[cfg(feature = "ex-lateral")]
lateral_variables: Vec::new(),
};
}
match select.option {
@ -581,6 +597,40 @@ fn build_select(
Ok(m)
}
#[cfg(feature = "ex-lateral")]
fn insert_lateral_variables(
pattern: GraphPattern,
new_lateral_variables: Vec<Variable>,
) -> Result<GraphPattern, &'static str> {
match pattern {
GraphPattern::Project {
inner,
variables,
lateral_variables,
} if lateral_variables.is_empty() => Ok(GraphPattern::Project {
inner,
variables,
lateral_variables: new_lateral_variables,
}),
GraphPattern::Distinct { inner } => Ok(GraphPattern::Distinct {
inner: Box::new(insert_lateral_variables(*inner, new_lateral_variables)?),
}),
GraphPattern::Reduced { inner } => Ok(GraphPattern::Reduced {
inner: Box::new(insert_lateral_variables(*inner, new_lateral_variables)?),
}),
GraphPattern::Slice {
inner,
start,
length,
} => Ok(GraphPattern::Slice {
inner: Box::new(insert_lateral_variables(*inner, new_lateral_variables)?),
start,
length,
}),
_ => Err("Not able to parse properly OX_LATERAL"),
}
}
fn are_variables_bound(expression: &Expression, variables: &HashSet<Variable>) -> bool {
match expression {
Expression::NamedNode(_)
@ -1376,6 +1426,11 @@ parser! {
expr
}),
PartialGraphPattern::Other(e) => g = new_join(g, e),
#[cfg(feature = "ex-lateral")]
PartialGraphPattern::LateralOptional(p) =>
g = GraphPattern::LeftSequence { left: Box::new(g), right: Box::new(p) },
#[cfg(feature = "ex-lateral")]
PartialGraphPattern::Lateral(p) => g = new_sequence(g, p),
}
}
@ -1400,7 +1455,24 @@ parser! {
rule TriplesBlock_inner() -> Vec<TripleOrPathPattern> = _ h:TriplesSameSubjectPath() _ { h }
//[56]
rule GraphPatternNotTriples() -> PartialGraphPattern = GroupOrUnionGraphPattern() / OptionalGraphPattern() / MinusGraphPattern() / GraphGraphPattern() / ServiceGraphPattern() / Filter() / Bind() / InlineData()
rule GraphPatternNotTriples() -> PartialGraphPattern = GroupOrUnionGraphPattern() / OptionalGraphPattern() / MinusGraphPattern() / GraphGraphPattern() / ServiceGraphPattern() / Filter() / Bind() / InlineData() / OxLateral()
rule OxLateral() -> PartialGraphPattern =
i("OX_LATERAL") _ i("OPTIONAL") _ p:GroupGraphPattern() {?
#[cfg(feature = "ex-lateral")]{ Ok(PartialGraphPattern::LateralOptional(p)) }
#[cfg(not(feature = "ex-lateral"))]{ Err("The 'OX_LATERAL' syntax is not enabled") }
} /
i("OX_LATERAL") _ i("GRAPH") _ name:VarOrIri() _ p:GroupGraphPattern() {?
#[cfg(feature = "ex-lateral")]{ Ok(PartialGraphPattern::Lateral(GraphPattern::Graph { name, inner: Box::new(p) })) }
#[cfg(not(feature = "ex-lateral"))]{ Err("The 'OX_LATERAL' syntax is not enabled") }
} /
i("OX_LATERAL") _ "(" _ vs:OxLateral_variable()* _ ")" _ "{" _ s:SubSelect() _ "}" {?
#[cfg(feature = "ex-lateral")]{ Ok(PartialGraphPattern::Lateral(insert_lateral_variables(s, vs)?)) }
#[cfg(not(feature = "ex-lateral"))]{ Err("The 'OX_LATERAL' syntax is not enabled") }
}
rule OxLateral_variable() -> Variable = v:Var() _ {
v
}
//[57]
rule OptionalGraphPattern() -> PartialGraphPattern = i("OPTIONAL") _ p:GroupGraphPattern() {

@ -526,21 +526,29 @@ impl SimpleEvaluator {
let count = *count;
Rc::new(move |from| Box::new(child(from).take(count)))
}
PlanNode::Project { child, mapping } => {
PlanNode::Project {
child,
mapping,
lateral_mapping,
} => {
let child = self.plan_evaluator(child);
let mapping = mapping.clone();
let lateral_mapping = lateral_mapping.clone();
Rc::new(move |from| {
let mapping = mapping.clone();
Box::new(
child(EncodedTuple::with_capacity(mapping.len())).filter_map(
move |tuple| {
let mut initial_mapping = EncodedTuple::with_capacity(mapping.len());
for (input_key, output_key) in lateral_mapping.iter() {
if let Some(value) = from.get(*output_key) {
initial_mapping.set(*input_key, value.clone())
}
}
Box::new(child(initial_mapping).filter_map(move |tuple| {
match tuple {
Ok(tuple) => {
let mut output_tuple = from.clone();
for (input_key, output_key) in mapping.iter() {
if let Some(value) = tuple.get(*input_key) {
if let Some(existing_value) =
output_tuple.get(*output_key)
if let Some(existing_value) = output_tuple.get(*output_key)
{
if existing_value != value {
return None; // Conflict
@ -554,9 +562,7 @@ impl SimpleEvaluator {
}
Err(e) => Some(Err(e)),
}
},
),
)
}))
})
}
PlanNode::Aggregate {

@ -86,6 +86,7 @@ pub enum PlanNode {
Project {
child: Box<Self>,
mapping: Rc<Vec<(usize, usize)>>, // pairs of (variable key in child, variable key in output)
lateral_mapping: Rc<Vec<(usize, usize)>>, // pairs of (variable key in child, variable key in output)
},
Aggregate {
// By definition the group by key are the range 0..key_mapping.len()
@ -99,19 +100,17 @@ impl PlanNode {
/// Returns variables that might be bound in the result set
pub fn used_variables(&self) -> BTreeSet<usize> {
let mut set = BTreeSet::default();
self.lookup_used_variables(&mut |v| {
set.insert(v);
});
self.add_used_variables(&mut set);
set
}
pub fn lookup_used_variables(&self, callback: &mut impl FnMut(usize)) {
pub fn add_used_variables(&self, set: &mut BTreeSet<usize>) {
match self {
Self::StaticBindings { tuples } => {
for tuple in tuples {
for (key, value) in tuple.iter().enumerate() {
if value.is_some() {
callback(key);
set.insert(key);
}
}
}
@ -123,16 +122,16 @@ impl PlanNode {
graph_name,
} => {
if let PatternValue::Variable(var) = subject {
callback(*var);
set.insert(*var);
}
if let PatternValue::Variable(var) = predicate {
callback(*var);
set.insert(*var);
}
if let PatternValue::Variable(var) = object {
callback(*var);
set.insert(*var);
}
if let PatternValue::Variable(var) = graph_name {
callback(*var);
set.insert(*var);
}
}
Self::PathPattern {
@ -142,60 +141,69 @@ impl PlanNode {
..
} => {
if let PatternValue::Variable(var) = subject {
callback(*var);
set.insert(*var);
}
if let PatternValue::Variable(var) = object {
callback(*var);
set.insert(*var);
}
if let PatternValue::Variable(var) = graph_name {
callback(*var);
set.insert(*var);
}
}
Self::Filter { child, expression } => {
expression.lookup_used_variables(callback);
child.lookup_used_variables(callback);
expression.add_used_variables(set);
child.add_used_variables(set);
}
Self::Union { children } => {
for child in children.iter() {
child.lookup_used_variables(callback);
child.add_used_variables(set);
}
}
Self::HashJoin { left, right }
| Self::ForLoopJoin { left, right, .. }
| Self::AntiJoin { left, right }
| Self::LeftJoin { left, right, .. } => {
left.lookup_used_variables(callback);
right.lookup_used_variables(callback);
left.add_used_variables(set);
right.add_used_variables(set);
}
Self::Extend {
child,
position,
expression,
} => {
callback(*position);
expression.lookup_used_variables(callback);
child.lookup_used_variables(callback);
set.insert(*position);
expression.add_used_variables(set);
child.add_used_variables(set);
}
Self::Sort { child, .. }
| Self::HashDeduplicate { child }
| Self::Reduced { child }
| Self::Skip { child, .. }
| Self::Limit { child, .. } => child.lookup_used_variables(callback),
| Self::Limit { child, .. } => child.add_used_variables(set),
Self::Service {
child,
service_name,
..
} => {
if let PatternValue::Variable(v) = service_name {
callback(*v);
set.insert(*v);
}
child.add_used_variables(set);
}
Self::Project {
mapping,
child,
lateral_mapping,
} => {
let mut child_bound = child.used_variables();
for (child_i, output_i) in lateral_mapping.iter() {
if set.contains(output_i) {
child_bound.insert(*child_i);
}
child.lookup_used_variables(callback);
}
Self::Project { mapping, child } => {
let child_bound = child.used_variables();
for (child_i, output_i) in mapping.iter() {
if child_bound.contains(child_i) {
callback(*output_i);
set.insert(*output_i);
}
}
}
@ -205,10 +213,10 @@ impl PlanNode {
..
} => {
for var in key_variables.iter() {
callback(*var);
set.insert(*var);
}
for (_, var) in aggregates.iter() {
callback(*var);
set.insert(*var);
}
}
}
@ -219,13 +227,11 @@ impl PlanNode {
/// (subset because this function is not perfect yet)
pub fn always_bound_variables(&self) -> BTreeSet<usize> {
let mut set = BTreeSet::default();
self.lookup_always_bound_variables(&mut |v| {
set.insert(v);
});
self.add_always_bound_variables(&mut set);
set
}
pub fn lookup_always_bound_variables(&self, callback: &mut impl FnMut(usize)) {
pub fn add_always_bound_variables(&self, set: &mut BTreeSet<usize>) {
match self {
Self::StaticBindings { tuples } => {
let mut variables = BTreeMap::default(); // value true iff always bound
@ -246,7 +252,7 @@ impl PlanNode {
}
for (k, v) in variables {
if v {
callback(k);
set.insert(k);
}
}
}
@ -257,16 +263,16 @@ impl PlanNode {
graph_name,
} => {
if let PatternValue::Variable(var) = subject {
callback(*var);
set.insert(*var);
}
if let PatternValue::Variable(var) = predicate {
callback(*var);
set.insert(*var);
}
if let PatternValue::Variable(var) = object {
callback(*var);
set.insert(*var);
}
if let PatternValue::Variable(var) = graph_name {
callback(*var);
set.insert(*var);
}
}
Self::PathPattern {
@ -276,18 +282,18 @@ impl PlanNode {
..
} => {
if let PatternValue::Variable(var) = subject {
callback(*var);
set.insert(*var);
}
if let PatternValue::Variable(var) = object {
callback(*var);
set.insert(*var);
}
if let PatternValue::Variable(var) = graph_name {
callback(*var);
set.insert(*var);
}
}
Self::Filter { child, .. } => {
//TODO: have a look at the expression to know if it filters out unbound variables
child.lookup_always_bound_variables(callback);
child.add_always_bound_variables(set);
}
Self::Union { children } => {
if let Some(vars) = children
@ -296,16 +302,16 @@ impl PlanNode {
.reduce(|a, b| a.intersection(&b).copied().collect())
{
for v in vars {
callback(v);
set.insert(v);
}
}
}
Self::HashJoin { left, right } | Self::ForLoopJoin { left, right, .. } => {
left.lookup_always_bound_variables(callback);
right.lookup_always_bound_variables(callback);
left.add_always_bound_variables(set);
right.add_always_bound_variables(set);
}
Self::AntiJoin { left, .. } | Self::LeftJoin { left, .. } => {
left.lookup_always_bound_variables(callback);
left.add_always_bound_variables(set);
}
Self::Extend {
child,
@ -314,27 +320,37 @@ impl PlanNode {
} => {
if matches!(expression.as_ref(), PlanExpression::Constant(_)) {
// TODO: more cases?
callback(*position);
set.insert(*position);
}
child.lookup_always_bound_variables(callback);
child.add_always_bound_variables(set);
}
Self::Sort { child, .. }
| Self::HashDeduplicate { child }
| Self::Reduced { child }
| Self::Skip { child, .. }
| Self::Limit { child, .. } => child.lookup_always_bound_variables(callback),
| Self::Limit { child, .. } => child.add_always_bound_variables(set),
Self::Service { child, silent, .. } => {
if *silent {
// none, might return a null tuple
} else {
child.lookup_always_bound_variables(callback)
child.add_always_bound_variables(set)
}
}
Self::Project { mapping, child } => {
let child_bound = child.always_bound_variables();
Self::Project {
mapping,
child,
lateral_mapping,
} => {
let mut child_bound = BTreeSet::new();
for (child_i, output_i) in lateral_mapping.iter() {
if set.contains(output_i) {
child_bound.insert(*child_i);
}
}
child.add_always_bound_variables(&mut child_bound);
for (child_i, output_i) in mapping.iter() {
if child_bound.contains(child_i) {
callback(*output_i);
set.insert(*output_i);
}
}
}
@ -344,14 +360,12 @@ impl PlanNode {
}
}
pub fn is_variable_bound(&self, variable: usize) -> bool {
let mut found = false;
self.lookup_always_bound_variables(&mut |v| {
if v == variable {
found = true;
}
});
found
pub fn are_all_variable_bound<'a>(
&self,
variables: impl IntoIterator<Item = &'a usize>,
) -> bool {
let bound = self.always_bound_variables();
variables.into_iter().all(|v| bound.contains(v))
}
}
@ -459,10 +473,17 @@ pub enum PlanExpression {
}
impl PlanExpression {
pub fn lookup_used_variables(&self, callback: &mut impl FnMut(usize)) {
/// Returns variables that are used in the expression
pub fn used_variables(&self) -> BTreeSet<usize> {
let mut set = BTreeSet::default();
self.add_used_variables(&mut set);
set
}
pub fn add_used_variables(&self, set: &mut BTreeSet<usize>) {
match self {
Self::Variable(v) | Self::Bound(v) => {
callback(*v);
set.insert(*v);
}
Self::Constant(_)
| Self::Rand
@ -518,7 +539,7 @@ impl PlanExpression {
| Self::DurationCast(e)
| Self::YearMonthDurationCast(e)
| Self::DayTimeDurationCast(e)
| Self::StringCast(e) => e.lookup_used_variables(callback),
| Self::StringCast(e) => e.add_used_variables(set),
Self::Or(a, b)
| Self::And(a, b)
| Self::Equal(a, b)
@ -541,31 +562,31 @@ impl PlanExpression {
| Self::SameTerm(a, b)
| Self::SubStr(a, b, None)
| Self::Regex(a, b, None) => {
a.lookup_used_variables(callback);
b.lookup_used_variables(callback);
a.add_used_variables(set);
b.add_used_variables(set);
}
Self::If(a, b, c)
| Self::SubStr(a, b, Some(c))
| Self::Regex(a, b, Some(c))
| Self::Replace(a, b, c, None)
| Self::Triple(a, b, c) => {
a.lookup_used_variables(callback);
b.lookup_used_variables(callback);
c.lookup_used_variables(callback);
a.add_used_variables(set);
b.add_used_variables(set);
c.add_used_variables(set);
}
Self::Replace(a, b, c, Some(d)) => {
a.lookup_used_variables(callback);
b.lookup_used_variables(callback);
c.lookup_used_variables(callback);
d.lookup_used_variables(callback);
a.add_used_variables(set);
b.add_used_variables(set);
c.add_used_variables(set);
d.add_used_variables(set);
}
Self::Concat(es) | Self::Coalesce(es) | Self::CustomFunction(_, es) => {
for e in es {
e.lookup_used_variables(callback);
e.add_used_variables(set);
}
}
Self::Exists(e) => {
e.lookup_used_variables(callback);
e.add_used_variables(set);
}
}
}

@ -93,6 +93,15 @@ impl<'a> PlanBuilder<'a> {
self.build_for_graph_pattern(left, variables, graph_name)?,
self.build_for_graph_pattern(right, variables, graph_name)?,
),
GraphPattern::Sequence { left, right } => PlanNode::ForLoopJoin {
left: Box::new(self.build_for_graph_pattern(left, variables, graph_name)?),
right: Box::new(self.build_for_graph_pattern(right, variables, graph_name)?),
},
GraphPattern::LeftSequence { left, right } => PlanNode::LeftJoin {
left: Box::new(self.build_for_graph_pattern(left, variables, graph_name)?),
right: Box::new(self.build_for_graph_pattern(right, variables, graph_name)?),
possible_problem_vars: Rc::new(Vec::new()),
},
GraphPattern::LeftJoin {
left,
right,
@ -220,6 +229,7 @@ impl<'a> PlanBuilder<'a> {
GraphPattern::Project {
inner,
variables: projection,
lateral_variables,
} => {
let mut inner_variables = projection.clone();
let inner_graph_name =
@ -239,6 +249,17 @@ impl<'a> PlanBuilder<'a> {
})
.collect(),
),
lateral_mapping: Rc::new(
lateral_variables
.iter()
.map(|variable| {
(
variable_key(&mut inner_variables, variable),
variable_key(variables, variable),
)
})
.collect(),
),
}
}
GraphPattern::Distinct { inner } => PlanNode::HashDeduplicate {
@ -1054,11 +1075,11 @@ impl<'a> PlanBuilder<'a> {
| PlanNode::PathPattern { .. } => (),
PlanNode::Filter { child, expression } => {
let always_already_bound = child.always_bound_variables();
expression.lookup_used_variables(&mut |v| {
for v in expression.used_variables() {
if !always_already_bound.contains(&v) {
set.insert(v);
}
});
}
Self::add_left_join_problematic_variables(child, set);
}
PlanNode::Union { children } => {
@ -1075,20 +1096,20 @@ impl<'a> PlanBuilder<'a> {
}
PlanNode::LeftJoin { left, right, .. } => {
Self::add_left_join_problematic_variables(left, set);
right.lookup_used_variables(&mut |v| {
set.insert(v);
});
right.add_used_variables(set);
}
PlanNode::Extend {
child, expression, ..
child,
expression,
position,
} => {
let always_already_bound = child.always_bound_variables();
expression.lookup_used_variables(&mut |v| {
for v in expression.used_variables() {
if !always_already_bound.contains(&v) {
set.insert(v);
}
});
Self::add_left_join_problematic_variables(child, set);
}
set.insert(*position); //TODO: too strict
Self::add_left_join_problematic_variables(child, set);
}
PlanNode::Sort { child, .. }
@ -1100,18 +1121,24 @@ impl<'a> PlanBuilder<'a> {
}
PlanNode::Service { child, silent, .. } => {
if *silent {
child.lookup_used_variables(&mut |v| {
set.insert(v);
});
child.add_used_variables(set);
} else {
Self::add_left_join_problematic_variables(child, set)
}
}
PlanNode::Project { mapping, child } => {
let mut child_bound = BTreeSet::new();
Self::add_left_join_problematic_variables(child, &mut child_bound);
PlanNode::Project {
mapping,
child,
lateral_mapping,
} => {
let mut child_problematic_set = BTreeSet::new();
for (child_i, output_i) in lateral_mapping.iter() {
set.insert(*output_i);
child_problematic_set.insert(*child_i);
}
Self::add_left_join_problematic_variables(child, &mut child_problematic_set);
for (child_i, output_i) in mapping.iter() {
if child_bound.contains(child_i) {
if child_problematic_set.contains(child_i) {
set.insert(*output_i);
}
}
@ -1194,14 +1221,11 @@ impl<'a> PlanBuilder<'a> {
if let PlanExpression::And(f1, f2) = *filter {
return Self::push_filter(Box::new(Self::push_filter(node, f1)), f2);
}
let mut filter_variables = BTreeSet::new();
filter.lookup_used_variables(&mut |v| {
filter_variables.insert(v);
});
let filter_variables = filter.used_variables();
match *node {
PlanNode::HashJoin { left, right } => {
if filter_variables.iter().all(|v| left.is_variable_bound(*v)) {
if filter_variables.iter().all(|v| right.is_variable_bound(*v)) {
if left.are_all_variable_bound(&filter_variables) {
if right.are_all_variable_bound(&filter_variables) {
PlanNode::HashJoin {
left: Box::new(Self::push_filter(left, filter.clone())),
right: Box::new(Self::push_filter(right, filter)),
@ -1212,7 +1236,7 @@ impl<'a> PlanBuilder<'a> {
right,
}
}
} else if filter_variables.iter().all(|v| right.is_variable_bound(*v)) {
} else if right.are_all_variable_bound(&filter_variables) {
PlanNode::HashJoin {
left,
right: Box::new(Self::push_filter(right, filter)),
@ -1225,12 +1249,12 @@ impl<'a> PlanBuilder<'a> {
}
}
PlanNode::ForLoopJoin { left, right } => {
if filter_variables.iter().all(|v| left.is_variable_bound(*v)) {
if left.are_all_variable_bound(&filter_variables) {
PlanNode::ForLoopJoin {
left: Box::new(Self::push_filter(left, filter)),
right,
}
} else if filter_variables.iter().all(|v| right.is_variable_bound(*v)) {
} else if right.are_all_variable_bound(&filter_variables) {
PlanNode::ForLoopJoin {
//TODO: should we do that always?
left,
@ -1249,7 +1273,7 @@ impl<'a> PlanBuilder<'a> {
position,
} => {
//TODO: handle the case where the filter generates an expression variable
if filter_variables.iter().all(|v| child.is_variable_bound(*v)) {
if child.are_all_variable_bound(&filter_variables) {
PlanNode::Extend {
child: Box::new(Self::push_filter(child, filter)),
expression,
@ -1267,7 +1291,7 @@ impl<'a> PlanBuilder<'a> {
}
}
PlanNode::Filter { child, expression } => {
if filter_variables.iter().all(|v| child.is_variable_bound(*v)) {
if child.are_all_variable_bound(&filter_variables) {
PlanNode::Filter {
child: Box::new(Self::push_filter(child, filter)),
expression,

@ -1,6 +1,6 @@
[package]
name = "pyoxigraph"
version = "0.3.8"
version = "0.4.0-alpha"
authors = ["Tpt"]
license = "MIT OR Apache-2.0"
readme = "README.md"
@ -16,5 +16,5 @@ name = "pyoxigraph"
doctest = false
[dependencies]
oxigraph = { version = "0.3.8", path="../lib", features = ["http_client"] }
oxigraph = { version = "0.4.0-alpha", path="../lib", features = ["http_client"] }
pyo3 = { version = "0.17", features = ["extension-module", "abi3-py37"] }

@ -1,6 +1,6 @@
[package]
name = "oxigraph_server"
version = "0.3.8"
version = "0.4.0-alpha"
authors = ["Tpt <thomas@pellissier-tanon.fr>"]
license = "MIT OR Apache-2.0"
readme = "README.md"
@ -14,7 +14,7 @@ edition = "2021"
[dependencies]
oxhttp = { version = "0.1", features = ["rayon"] }
clap = { version = "4", features = ["derive"] }
oxigraph = { version = "0.3.8", path = "../lib", features = ["http_client"] }
oxigraph = { version = "0.4.0-alpha", path = "../lib", features = ["http_client"] }
sparesults = { version = "0.1.1", path = "../lib/sparesults", features = ["rdf-star"] }
rand = "0.8"
url = "2"

@ -1,6 +1,6 @@
[package]
name = "oxigraph_testsuite"
version = "0.3.8"
version = "0.4.0-alpha"
authors = ["Tpt <thomas@pellissier-tanon.fr>"]
license = "MIT OR Apache-2.0"
readme = "../README.md"
@ -15,7 +15,7 @@ publish = false
anyhow = "1"
clap = { version = "4", features = ["derive"] }
time = { version = "0.3", features = ["formatting"] }
oxigraph = { version = "0.3.8", path="../lib" }
oxigraph = { version = "0.4.0-alpha", path="../lib" }
text-diff = "0.4"
[dev-dependencies]

@ -5,7 +5,7 @@
@prefix ox: <https://github.com/oxigraph/oxigraph/tests#> .
<> rdf:type mf:Manifest ;
rdfs:label "Oxigraph SPARQL resutls tests" ;
rdfs:label "Oxigraph SPARQL results tests" ;
mf:entries
(
:results_json_duplicated_variables

@ -0,0 +1,5 @@
@prefix ex: <http://example.org/> .
ex:s1 a ex:T ; ex:p 11 , 12 , 13 .
ex:s2 a ex:T ; ex:p 21 , 22 , 23 .
ex:s3 a ex:T .

@ -0,0 +1,6 @@
PREFIX ex: <http://example.org/>
SELECT ?s ?o WHERE {
VALUES ?s { ex:S }
OX_LATERAL GRAPH ex:G { FILTER(BOUND(?s)) . VALUES ?o { ex:O } }
}

@ -0,0 +1,56 @@
@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
@prefix : <https://github.com/oxigraph/oxigraph/tests/sparql/lateral/manifest#> .
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
@prefix mf: <http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#> .
@prefix qt: <http://www.w3.org/2001/sw/DataAccess/tests/test-query#> .
@prefix ut: <http://www.w3.org/2009/sparql/tests/test-update#> .
<> rdf:type mf:Manifest ;
rdfs:label "Oxigraph OX_LATERAL feature SPARQL tests" ;
mf:entries
(
:subselect
:subselect_inside_optional
:subselect_implicit_aggregate
:subselect_explicit_aggregate
:optional
:graph
) .
:subselect rdf:type mf:QueryEvaluationTest ;
mf:name "Basic subselect OX_LATERAL test" ;
mf:action
[ qt:query <subselect.rq> ;
qt:data <basic_input.ttl> ] ;
mf:result <subselect.srx> .
:subselect_inside_optional rdf:type mf:QueryEvaluationTest ;
mf:name "Basic subselect OX_LATERAL test inside optional" ;
mf:action
[ qt:query <subselect_inside_optional.rq> ;
qt:data <basic_input.ttl> ] ;
mf:result <subselect_inside_optional.srx> .
:subselect_implicit_aggregate rdf:type mf:QueryEvaluationTest ;
mf:name "OX_LATERAL test with implicit aggregate" ;
mf:action
[ qt:query <subselect_implicit_aggregate.rq> ;
qt:data <basic_input.ttl> ] ;
mf:result <subselect_implicit_aggregate.srx> .
:subselect_explicit_aggregate rdf:type mf:QueryEvaluationTest ;
mf:name "OX_LATERAL test with explicit aggregate" ;
mf:action
[ qt:query <subselect_explicit_aggregate.rq> ;
qt:data <basic_input.ttl> ] ;
mf:result <subselect_explicit_aggregate.srx> .
:optional rdf:type mf:QueryEvaluationTest ;
mf:name "OX_LATERAL OPTIONAL test" ;
mf:action [ qt:query <optional.rq> ] ;
mf:result <simple.srx> .
:optional rdf:type mf:QueryEvaluationTest ;
mf:name "OX_LATERAL GRAPH test" ;
mf:action [ qt:query <graph.rq> ] ;
mf:result <simple.srx> .

@ -0,0 +1,6 @@
PREFIX ex: <http://example.org/>
SELECT ?s ?o WHERE {
VALUES ?s { ex:S }
OX_LATERAL OPTIONAL { FILTER(BOUND(?s)) . VALUES ?o { ex:O } }
}

@ -0,0 +1,17 @@
<?xml version="1.0"?>
<sparql xmlns="http://www.w3.org/2005/sparql-results#">
<head>
<variable name="s"/>
<variable name="o"/>
</head>
<results>
<result>
<binding name="s">
<uri>http://example.org/S</uri>
</binding>
<binding name="o">
<uri>http://example.org/O</uri>
</binding>
</result>
</results>
</sparql>

@ -0,0 +1,6 @@
PREFIX ex: <http://example.org/>
SELECT ?s ?o WHERE {
?s a ex:T.
OX_LATERAL(?s) {SELECT ?o WHERE { ?s ex:p ?o } ORDER BY ?o LIMIT 2}
}

@ -0,0 +1,41 @@
<?xml version="1.0"?>
<sparql xmlns="http://www.w3.org/2005/sparql-results#">
<head>
<variable name="s"/>
<variable name="o"/>
</head>
<results>
<result>
<binding name="s">
<uri>http://example.org/s1</uri>
</binding>
<binding name="o">
<literal datatype="http://www.w3.org/2001/XMLSchema#integer">11</literal>
</binding>
</result>
<result>
<binding name="s">
<uri>http://example.org/s1</uri>
</binding>
<binding name="o">
<literal datatype="http://www.w3.org/2001/XMLSchema#integer">12</literal>
</binding>
</result>
<result>
<binding name="s">
<uri>http://example.org/s2</uri>
</binding>
<binding name="o">
<literal datatype="http://www.w3.org/2001/XMLSchema#integer">21</literal>
</binding>
</result>
<result>
<binding name="s">
<uri>http://example.org/s2</uri>
</binding>
<binding name="o">
<literal datatype="http://www.w3.org/2001/XMLSchema#integer">22</literal>
</binding>
</result>
</results>
</sparql>

@ -0,0 +1,6 @@
PREFIX ex: <http://example.org/>
SELECT ?s ?c WHERE {
?s a ex:T.
OX_LATERAL(?s) {SELECT (MAX(?o) AS ?c) WHERE { ?s ex:p ?o } GROUP BY ?s}
}

@ -0,0 +1,25 @@
<?xml version="1.0"?>
<sparql xmlns="http://www.w3.org/2005/sparql-results#">
<head>
<variable name="s"/>
<variable name="c"/>
</head>
<results>
<result>
<binding name="s">
<uri>http://example.org/s1</uri>
</binding>
<binding name="c">
<literal datatype="http://www.w3.org/2001/XMLSchema#integer">13</literal>
</binding>
</result>
<result>
<binding name="s">
<uri>http://example.org/s2</uri>
</binding>
<binding name="c">
<literal datatype="http://www.w3.org/2001/XMLSchema#integer">23</literal>
</binding>
</result>
</results>
</sparql>

@ -0,0 +1,6 @@
PREFIX ex: <http://example.org/>
SELECT ?s ?c WHERE {
?s a ex:T.
OX_LATERAL(?s) {SELECT (MAX(?o) AS ?c) WHERE { ?s ex:p ?o }}
}

@ -0,0 +1,30 @@
<?xml version="1.0"?>
<sparql xmlns="http://www.w3.org/2005/sparql-results#">
<head>
<variable name="s"/>
<variable name="c"/>
</head>
<results>
<result>
<binding name="s">
<uri>http://example.org/s1</uri>
</binding>
<binding name="c">
<literal datatype="http://www.w3.org/2001/XMLSchema#integer">13</literal>
</binding>
</result>
<result>
<binding name="s">
<uri>http://example.org/s2</uri>
</binding>
<binding name="c">
<literal datatype="http://www.w3.org/2001/XMLSchema#integer">23</literal>
</binding>
</result>
<result>
<binding name="s">
<uri>http://example.org/s3</uri>
</binding>
</result>
</results>
</sparql>

@ -0,0 +1,6 @@
PREFIX ex: <http://example.org/>
SELECT ?s ?o WHERE {
?s a ex:T.
OPTIONAL { OX_LATERAL(?s) {SELECT ?o WHERE { ?s ex:p ?o } ORDER BY ?o LIMIT 2} }
}

@ -0,0 +1,57 @@
<?xml version="1.0"?>
<sparql xmlns="http://www.w3.org/2005/sparql-results#">
<head>
<variable name="s"/>
<variable name="o"/>
</head>
<results>
<result>
<binding name="s">
<uri>http://example.org/s1</uri>
</binding>
<binding name="o">
<literal datatype="http://www.w3.org/2001/XMLSchema#integer">11</literal>
</binding>
</result>
<result>
<binding name="s">
<uri>http://example.org/s1</uri>
</binding>
<binding name="o">
<literal datatype="http://www.w3.org/2001/XMLSchema#integer">12</literal>
</binding>
</result>
<result>
<binding name="s">
<uri>http://example.org/s2</uri>
</binding>
<binding name="o">
<literal datatype="http://www.w3.org/2001/XMLSchema#integer">11</literal>
</binding>
</result>
<result>
<binding name="s">
<uri>http://example.org/s2</uri>
</binding>
<binding name="o">
<literal datatype="http://www.w3.org/2001/XMLSchema#integer">11</literal>
</binding>
</result>
<result>
<binding name="s">
<uri>http://example.org/s3</uri>
</binding>
<binding name="o">
<literal datatype="http://www.w3.org/2001/XMLSchema#integer">11</literal>
</binding>
</result>
<result>
<binding name="s">
<uri>http://example.org/s3</uri>
</binding>
<binding name="o">
<literal datatype="http://www.w3.org/2001/XMLSchema#integer">11</literal>
</binding>
</result>
</results>
</sparql>

@ -7,6 +7,7 @@
<> rdf:type mf:Manifest ;
rdfs:label "Oxigraph SPARQL tests" ;
mf:include ( <lateral/manifest.ttl> ) ;
mf:entries
(
:small_unicode_escape_with_multibytes_char

Loading…
Cancel
Save