From f817809a2aae65bd8253dd49e0524c4607cb1849 Mon Sep 17 00:00:00 2001 From: Laurin Weger Date: Wed, 15 Oct 2025 12:27:18 +0200 Subject: [PATCH] new backend-update > JSON patch algorithm (WIP) --- engine/verifier/src/orm/mod.rs | 782 ++++++++++++++++---------- engine/verifier/src/orm/validation.rs | 7 +- sdk/rust/src/tests/mod.rs | 7 +- sdk/rust/src/tests/orm_patches.rs | 589 +++++++++---------- 4 files changed, 757 insertions(+), 628 deletions(-) diff --git a/engine/verifier/src/orm/mod.rs b/engine/verifier/src/orm/mod.rs index 65eda13..c3380c0 100644 --- a/engine/verifier/src/orm/mod.rs +++ b/engine/verifier/src/orm/mod.rs @@ -681,18 +681,18 @@ impl Verifier { // Remove old subscriptions subs.retain(|sub| !sub.sender.is_closed()); - if scope.target == NuriTargetV0::UserSite + if !(scope.target == NuriTargetV0::UserSite || scope .overlay .as_ref() .map_or(false, |ol| overlaylink == *ol) - || scope.target == NuriTargetV0::Repo(repo_id) + || scope.target == NuriTargetV0::Repo(repo_id)) { continue; } // prepare to apply updates to tracked subjects and record the changes. - let shapes = subs + let root_shapes = subs .iter() .map(|sub| { sub.shape_type @@ -703,7 +703,7 @@ impl Verifier { }) .collect::>(); - scopes.push((scope.clone(), shapes)); + scopes.push((scope.clone(), root_shapes)); } log_debug!( @@ -713,7 +713,7 @@ impl Verifier { for (scope, shapes) in scopes { let mut orm_changes: OrmChanges = HashMap::new(); - // actually applying updates to tracked subjects and record the changes. + // Apply the changes to tracked subjects. for shape_arc in shapes { let _ = self.process_changes_for_shape_and_session( &scope, @@ -728,330 +728,532 @@ impl Verifier { let subs = self.orm_subscriptions.get(&scope).unwrap(); for sub in subs.iter() { - // TODO: This if-condition is wrong (intended to not re-apply changes coming from the same subscription). - // if sub.session_id == session_id { - // continue; log_debug!( "Applying changes to subscription with nuri {} and shape {}", sub.nuri.repo(), sub.shape_type.shape ); - // } - // Create diff from changes & subscription. - fn create_patches_for_nested_object( - pred_shape: &OrmSchemaPredicate, + // The JSON patches to send to JS land. + let mut patches: Vec = vec![]; + + // Keep track of created objects by path and if they need an id. + // Later we created patches from them to ensure the objects exist. + let mut paths_of_objects_to_create: HashSet<(Vec, Option)> = + HashSet::new(); + + // Function to create diff objects from a given change. + // The function recurses from child to parents down to a root tracked subject. + // If multiple parents exist, it adds separate patches for each. + fn add_diff_ops_for_tracked_subject( + tracked_subject: &OrmTrackedSubject, tracked_subjects: &HashMap< String, HashMap>>, >, - patches: &mut Vec, + root_shape: &String, path: &mut Vec, - object_iri: &String, - orm_changes: &OrmChanges, - sub: &OrmSubscription, + diff_op: ( + OrmDiffOpType, + Option, + Option, // The value added / removed + Option, // The IRI, if change is an added / removed object. + ), + patches: &mut Vec, + paths_of_objects_to_create: &mut HashSet<(Vec, Option)>, ) { - // Object was added. That means, we need to add a basic object with no value, - // Then add further predicates to it in a recursive call. - patches.push(OrmDiffOp { - op: OrmDiffOpType::add, - valType: Some(OrmDiffType::object), - path: format!("/{}", path.join("/")), - value: None, - }); + // If this subject has no parents or its shape matches the root shape, we've reached the root + if tracked_subject.parents.is_empty() + || tracked_subject.shape.iri == *root_shape + { + // Build the final JSON Pointer path + let escaped_path: Vec = + path.iter().map(|seg| escape_json_pointer(seg)).collect(); + let json_pointer = format!("/{}", escaped_path.join("/")); + + // Create the patch + let patch = OrmDiffOp { + op: diff_op.0.clone(), + valType: diff_op.1.clone(), + path: json_pointer, + value: diff_op.2.clone(), + }; + patches.push(patch); + + // If this is an object being added, record it for object creation + if let Some(iri) = &diff_op.3 { + if matches!(diff_op.0, OrmDiffOpType::add) { + paths_of_objects_to_create + .insert((path.clone(), Some(iri.clone()))); + } + } - // Get the shape IRI for a nested object that is valid. - let object_shape_iri = { - // Get the tracked subject for this object IRI - let tracked_subjects_for_obj = tracked_subjects - .get(object_iri) - .expect("Object should be tracked"); - - // Find the first valid shape for this object from the allowed shapes - let allowed_shape_iris: Vec<&String> = pred_shape - .dataTypes - .iter() - .filter_map(|dt| dt.shape.as_ref()) - .collect(); - - allowed_shape_iris - .iter() - .find(|shape_iri| { - tracked_subjects_for_obj - .get(**shape_iri) - .map(|ts| { - ts.read().unwrap().valid == OrmTrackedSubjectValidity::Valid - }) - .unwrap_or(false) - }) - .unwrap() - .to_string() - }; + return; + } - // Apply changes for nested object. - create_patches_for_changed_subj( - orm_changes, - patches, - &object_shape_iri, - &object_iri, - sub, - path, - tracked_subjects, - ); + // Recurse to parents + for (parent_iri, parent_tracked_subject) in tracked_subject.parents.iter() { + // Get predicate schema linking parent with tracked_subject + + // Use predicate schema readable_predicate to add to path. + // If predicate schema is multi, add our own subject iri to path first. + + // If parent is root, we don't need to recurse. + // Instead we add new patches based on the path (we need to escape segments before) + // and the diff_op content + + let parent_ts = parent_tracked_subject.read().unwrap(); + + // Find the predicate schema linking parent to this tracked subject + for pred_arc in &parent_ts.shape.predicates { + // Check if this predicate has our subject as a child + if let Some(tracked_pred) = + parent_ts.tracked_predicates.get(&pred_arc.iri) + { + let tp = tracked_pred.read().unwrap(); + + // Check if this tracked subject is in the children + let is_child = tp.tracked_children.iter().any(|child| { + let child_read = child.read().unwrap(); + child_read.subject_iri == tracked_subject.subject_iri + }); + + if is_child { + // Build the path segment + let mut new_path = path.clone(); + + let is_multi = pred_arc.maxCardinality > 1 + || pred_arc.maxCardinality == -1; + + // For multi-valued predicates, add the object IRI as a key first + if is_multi { + new_path.insert(0, tracked_subject.subject_iri.clone()); + } + + // Add the readable predicate name + new_path.insert(0, pred_arc.readablePredicate.clone()); + + // Recurse to the parent + add_diff_ops_for_tracked_subject( + &parent_ts, + tracked_subjects, + root_shape, + &mut new_path, + diff_op.clone(), + patches, + paths_of_objects_to_create, + ); + + break; + } + } + } + } } - fn create_patches_for_changed_subj( - orm_changes: &OrmChanges, - patches: &mut OrmDiff, - shape_iri: &String, - subject_iri: &String, - sub: &OrmSubscription, - path: &mut Vec, + fn diff_op_from_pred_change( + pred_change: &OrmTrackedPredicateChanges, + ) -> Vec<( + OrmDiffOpType, + Option, + Option, // The value added / removed + Option, // The IRI, if change is an added / removed object. + )> { + let tracked_predicate = pred_change.tracked_predicate.read().unwrap(); + + let is_multi = tracked_predicate.schema.maxCardinality > 1 + || tracked_predicate.schema.maxCardinality == -1; + let is_object = tracked_predicate + .schema + .dataTypes + .iter() + .any(|dt| dt.shape.is_some()); + + if !is_multi && !is_object { + if pred_change.values_added.len() == 1 { + // A value was added. Another one might have been removed + // but the add patch overwrite previous values. + return [( + OrmDiffOpType::add, + None, + Some(json!(pred_change.values_added[0])), + None, + )] + .to_vec(); + } else { + // Since there is only one possible value, removing the path is enough. + return [(OrmDiffOpType::remove, None, None, None)].to_vec(); + } + } else if is_multi && !is_object { + let mut ops = vec![]; + if pred_change.values_added.len() > 0 { + ops.push(( + OrmDiffOpType::add, + Some(OrmDiffType::set), + Some(json!(pred_change.values_added)), + None, + )); + } + if pred_change.values_removed.len() > 0 { + ops.push(( + OrmDiffOpType::remove, + Some(OrmDiffType::set), + Some(json!(pred_change.values_removed)), + None, + )); + } + return ops; + } + // objects are not handled here because objects to create + // are registered during path traversal. + return vec![]; + } + + // Helper function to determine the highest-priority valid shape for a subject + // given the allowed shapes in a predicate's dataTypes. + // Returns (current_valid_shape, previous_valid_shape) + #[allow(dead_code)] + fn get_highest_priority_valid_shapes( + subject_iri: &SubjectIri, + allowed_shapes: &[OrmSchemaDataType], // From predicate.dataTypes (in priority order) tracked_subjects: &HashMap< - SubjectIri, - HashMap>>, + String, + HashMap>>, >, - ) { - let change = orm_changes - .get(shape_iri) - .unwrap() - .get(subject_iri) - .unwrap(); - let subject_shape = sub.shape_type.schema.get(shape_iri).unwrap(); + ) -> (Option, Option) { + let Some(shapes_for_subject) = tracked_subjects.get(subject_iri) else { + return (None, None); + }; - // @Niko, is it safe to do this? - let tracked_subject = tracked_subjects - .get(subject_iri) - .unwrap() - .get(shape_iri) - .unwrap() - .read() - .unwrap(); + // Find current highest-priority valid shape + let current_valid = allowed_shapes + .iter() + .filter_map(|dt| dt.shape.as_ref()) + .find_map(|shape_iri| { + shapes_for_subject.get(shape_iri).and_then(|ts| { + let tracked = ts.read().unwrap(); + if tracked.valid == OrmTrackedSubjectValidity::Valid { + Some(shape_iri.clone()) + } else { + None + } + }) + }); - // Check validity changes - if tracked_subject.prev_valid == OrmTrackedSubjectValidity::Invalid - && tracked_subject.valid == OrmTrackedSubjectValidity::Invalid - { - // Is the subject invalid and was it before? There is nothing we need to inform about. - return; - } else if tracked_subject.prev_valid == OrmTrackedSubjectValidity::Valid - && tracked_subject.valid == OrmTrackedSubjectValidity::Invalid - || tracked_subject.valid == OrmTrackedSubjectValidity::Untracked - { - // Has the subject become invalid or untracked? - // We add a patch, deleting the object at its root. - patches.push(OrmDiffOp { - op: OrmDiffOpType::remove, - valType: Some(OrmDiffType::object), - path: format!("/{}", path.join("/")), - value: None, + // Find previous highest-priority valid shape + let previous_valid = allowed_shapes + .iter() + .filter_map(|dt| dt.shape.as_ref()) + .find_map(|shape_iri| { + shapes_for_subject.get(shape_iri).and_then(|ts| { + let tracked = ts.read().unwrap(); + if tracked.prev_valid == OrmTrackedSubjectValidity::Valid { + Some(shape_iri.clone()) + } else { + None + } + }) }); + + (current_valid, previous_valid) + } + + // Helper function to handle validity changes when highest-priority shape changes + #[allow(dead_code)] + fn handle_shape_priority_change( + subject_iri: &SubjectIri, + shape_iri: &ShapeIri, + tracked_subjects: &HashMap< + String, + HashMap>>, + >, + root_shape: &String, + orm_changes: &OrmChanges, + patches: &mut Vec, + paths_of_objects_to_create: &mut HashSet<(Vec, Option)>, + ) { + // Step 1: Check if this subject has multiple tracked shapes + let Some(shapes_for_subject) = tracked_subjects.get(subject_iri) else { + return; + }; + + if shapes_for_subject.len() <= 1 { + // Only one shape, no priority conflict possible return; - } else { - // The subject is valid or has become valid. - // In both cases, all information necessary to send patches are available in the orm_changes. - - // In case the subject was not valid before, we create the object at the current path as an empty object though. - if tracked_subject.prev_valid != OrmTrackedSubjectValidity::Valid { - patches.push(OrmDiffOp { - op: OrmDiffOpType::add, - valType: Some(OrmDiffType::object), - path: format!("/{}", path.join("/")), - value: None, - }); - // And add the id field. - patches.push(OrmDiffOp { - op: OrmDiffOpType::add, - valType: None, - path: format!("/{}/{}", path.join("/"), subject_iri), - value: None, - }); - } } - // Iterate over every predicate change and create patches - for (pred_iri, pred_change) in change.predicates.iter() { - let pred_shape = subject_shape - .predicates - .iter() - .find(|p| p.iri == *pred_iri) - .unwrap(); - - let is_multi = - pred_shape.maxCardinality > 1 || pred_shape.maxCardinality == -1; - let is_object = pred_shape.dataTypes.iter().any(|dt| !dt.shape.is_none()); - let pred_name = pred_shape.readablePredicate.clone(); - path.push(pred_name); - let path_str = format!("/{}", path.join("/")); - - // Depending on the predicate type (multi / single, object / not object), - // add the respective diff operation. - - // Single primitive value - if !is_multi && !is_object { - if pred_change.values_added.len() > 0 { - patches.push(OrmDiffOp { - op: OrmDiffOpType::add, - valType: None, - path: path_str.clone(), - value: Some(json!(pred_change.values_added[0])), - }); - } - if pred_change.values_removed.len() > 0 { - patches.push(OrmDiffOp { - op: OrmDiffOpType::remove, - valType: None, - path: path_str, - value: Some(json!(pred_change.values_added[0])), - }); - } - } else if is_multi && !is_object { - // Set of primitive values - if pred_change.values_added.len() > 0 { - patches.push(OrmDiffOp { - op: OrmDiffOpType::add, - valType: Some(OrmDiffType::set), - path: path_str.clone(), - value: Some(json!(pred_change.values_added)), - }); - } - if pred_change.values_removed.len() > 0 { - patches.push(OrmDiffOp { - op: OrmDiffOpType::remove, - valType: Some(OrmDiffType::set), - path: path_str, - value: Some(json!(pred_change.values_removed)), + // Step 2: Get the current tracked subject + let Some(tracked_subject_arc) = shapes_for_subject.get(shape_iri) else { + return; + }; + let tracked_subject = tracked_subject_arc.read().unwrap(); + + // Step 3: For each parent, check if the highest-priority valid shape changed + for (parent_iri, parent_tracked_subject_arc) in tracked_subject.parents.iter() { + let parent_ts = parent_tracked_subject_arc.read().unwrap(); + + // Find the predicate linking parent to this subject + for pred_arc in &parent_ts.shape.predicates { + if let Some(tracked_pred) = + parent_ts.tracked_predicates.get(&pred_arc.iri) + { + let tp = tracked_pred.read().unwrap(); + + // Check if this tracked subject is a child of this predicate + let is_child = tp.tracked_children.iter().any(|child| { + let child_read = child.read().unwrap(); + child_read.subject_iri == *subject_iri }); - } - } else if is_object { - // Change in single object property. - if !is_multi { - let object_iri = match &pred_change.values_added[0] { - BasicType::Str(iri) => iri, - _ => panic!("Object no IRI"), - }; - // Single object. - if pred_change.values_added.len() > 0 { - create_patches_for_nested_object( - pred_shape, - tracked_subjects, - patches, - path, - object_iri, - orm_changes, - sub, - ); + + if !is_child { + continue; } - if pred_change.values_removed.len() > 0 { - // Object is removed. - patches.push(OrmDiffOp { - op: OrmDiffOpType::remove, - valType: Some(OrmDiffType::object), - path: path_str, - value: None, - }); + + // Get the allowed shapes for this predicate (in priority order) + let allowed_shapes: Vec<_> = pred_arc + .dataTypes + .iter() + .filter(|dt| dt.shape.is_some()) + .collect(); + + if allowed_shapes.len() <= 1 { + // No priority conflict possible with single shape + continue; } - } else { - // Change(s) in multi object property. - - // Add every new object. - for obj_iri_bt in pred_change.values_added.iter() { - let obj_iri = match obj_iri_bt { - BasicType::Str(iri) => iri, - _ => panic!("Object no IRI"), - }; - - // First, we create a root object (if the object existed before, this has no effect). - patches.push(OrmDiffOp { - op: OrmDiffOpType::add, - valType: Some(OrmDiffType::object), - path: path_str.clone(), - value: None, - }); - - // Add escaped object IRI to path. - path.push(escape_json_pointer(obj_iri)); - - create_patches_for_nested_object( - pred_shape, + + // Determine current and previous highest-priority valid shapes + let (current_valid, previous_valid) = + get_highest_priority_valid_shapes( + subject_iri, + &pred_arc.dataTypes, tracked_subjects, - patches, - path, - obj_iri, - orm_changes, - sub, ); - // Remove object IRI from stack again. - path.pop(); - } - - // Delete objects. - // If there are no more predicates, delete the whole object. - if pred_change - .tracked_predicate - .read() - .unwrap() - .tracked_children - .len() - == 0 - { - // Or the whole thing if no children remain - patches.push(OrmDiffOp { - op: OrmDiffOpType::remove, - valType: Some(OrmDiffType::object), - path: path_str, - value: None, - }); - } else { - for object_iri_removed in pred_change.values_removed.iter() { - patches.push(OrmDiffOp { - op: OrmDiffOpType::remove, - valType: Some(OrmDiffType::object), - path: format!( - "/{}/{}", - path_str, - match object_iri_removed { - BasicType::Str(iri) => iri, - _ => panic!("Object IRI must be string"), - } - ), - value: None, - }); + // Step 4: Create patches based on what changed + if current_valid != previous_valid { + let is_multi = pred_arc.maxCardinality > 1 + || pred_arc.maxCardinality == -1; + + // Case A: Shape switch (ShapeA -> ShapeB) + if let (Some(new_shape), Some(old_shape)) = + (¤t_valid, &previous_valid) + { + // Remove the old object + if let Some(old_ts) = shapes_for_subject.get(old_shape) { + let old_tracked = old_ts.read().unwrap(); + let mut path = vec![]; + let diff_op = ( + OrmDiffOpType::remove, + Some(OrmDiffType::object), + None, + Some(subject_iri.clone()), + ); + + add_diff_ops_for_tracked_subject( + &old_tracked, + tracked_subjects, + root_shape, + &mut path, + diff_op, + patches, + paths_of_objects_to_create, + ); + } + + // Add the new object (need to materialize it) + if let Some(new_ts) = shapes_for_subject.get(new_shape) { + let new_tracked = new_ts.read().unwrap(); + + // TODO: Materialize the object with current triples + // This requires access to the change data or re-querying + // For now, we'll just create an object placeholder patch + let mut path = vec![]; + let diff_op = ( + OrmDiffOpType::add, + Some(OrmDiffType::object), + Some(Value::Null), + Some(subject_iri.clone()), + ); + + add_diff_ops_for_tracked_subject( + &new_tracked, + tracked_subjects, + root_shape, + &mut path, + diff_op, + patches, + paths_of_objects_to_create, + ); + } + } + // Case B: Object became valid (None -> ShapeX) + else if let (Some(new_shape), None) = + (¤t_valid, &previous_valid) + { + if let Some(new_ts) = shapes_for_subject.get(new_shape) { + let new_tracked = new_ts.read().unwrap(); + let mut path = vec![]; + let diff_op = ( + OrmDiffOpType::add, + Some(OrmDiffType::object), + Some(Value::Null), + Some(subject_iri.clone()), + ); + + add_diff_ops_for_tracked_subject( + &new_tracked, + tracked_subjects, + root_shape, + &mut path, + diff_op, + patches, + paths_of_objects_to_create, + ); + } + } + // Case C: Object became invalid (ShapeX -> None) + else if let (None, Some(old_shape)) = + (¤t_valid, &previous_valid) + { + if let Some(old_ts) = shapes_for_subject.get(old_shape) { + let old_tracked = old_ts.read().unwrap(); + let mut path = vec![]; + let diff_op = ( + OrmDiffOpType::remove, + Some(OrmDiffType::object), + None, + Some(subject_iri.clone()), + ); + + add_diff_ops_for_tracked_subject( + &old_tracked, + tracked_subjects, + root_shape, + &mut path, + diff_op, + patches, + paths_of_objects_to_create, + ); + } } } + + break; // Found the predicate, no need to check others } } - - // Remove this predicate name from the path again. - path.pop(); } } - let mut patches: OrmDiff = vec![]; - let mut path: Vec = Vec::with_capacity(4); - - // Iterate over each root subject with the right shape - // For each tracked subject that has the subscription's shape, call fn above - for (subject_iri, tracked_subjects_by_shape) in sub.tracked_subjects.iter() { - for (shape_iri, tracked_subject) in tracked_subjects_by_shape.iter() { - if *shape_iri != sub.shape_type.shape { + // We construct object patches from a change (which is associated with a shape type). {op: add, valType: object, value: Null, path: ...} + // For each change that has a subject tracked in this subscription, + // - Get change operation (calling diff_op_from_pred_change). + // - case not object, single --> either add or remove (must be one of each at max) + // - case not object, multi --> just add and or set patch + // - case object, multi --> create object patch + nested object patch (will be handled when recursing paths to add primitive values) + // - case object, single --> just object patch (will be handled when recursing paths to add primitive values) + // - Add patches for each change operation for the path of the change in the schema. + // We find the path by traversing the schema up to the parents (add_diff_ops_for_tracked_subject). + + // TODO: Special edge case: An object with parents changed and the parents' predicate schema has multiple allowed shapes. + // Now, there are multiple tracked subjects with the same subject IRI but different shapes of which some + // are valid or invalid. The first valid (subject, shape) pair must used for materialization. + // - if a higher-priority shape became invalid but a lower priority shape is valid, delete and new add. + // - if a higher-priority shape became valid, delete and add new valid. + // Problem: We might not have the triples present to materialize the newly valid object so we need to fetch them. + + // Process changes for this subscription + // Iterate through all changes and create patches + for (shape_iri, subject_changes) in &orm_changes { + for (subject_iri, change) in subject_changes { + // Get the tracked subject for this (subject, shape) pair + let tracked_subject_opt = sub + .tracked_subjects + .get(subject_iri) + .and_then(|shapes| shapes.get(shape_iri)) + .map(|ts| ts.read().unwrap()); + + let Some(tracked_subject) = tracked_subject_opt else { continue; + }; + + // Process each predicate change + for (pred_iri, pred_change) in &change.predicates { + let tracked_predicate = pred_change.tracked_predicate.read().unwrap(); + let pred_name = tracked_predicate.schema.readablePredicate.clone(); + // Check validity changes + if tracked_subject.prev_valid == OrmTrackedSubjectValidity::Invalid + && tracked_subject.valid == OrmTrackedSubjectValidity::Invalid + { + // Is the subject invalid and was it before? There is nothing we need to inform about. + return; + } else if tracked_subject.prev_valid == OrmTrackedSubjectValidity::Valid + && tracked_subject.valid == OrmTrackedSubjectValidity::Invalid + || tracked_subject.valid == OrmTrackedSubjectValidity::Untracked + { + // Has the subject become invalid or untracked? + // We add a patch, deleting the object at its root. + let mut path: Vec = vec![pred_name.clone()]; + add_diff_ops_for_tracked_subject( + &tracked_subject, + &sub.tracked_subjects, + &sub.shape_type.shape, + &mut path, + (OrmDiffOpType::remove, Some(OrmDiffType::object), None, None), + &mut patches, + &mut paths_of_objects_to_create, + ); + } else { + // The subject is valid or has become valid. + + // Get the diff operations for this predicate change + let diff_ops = diff_op_from_pred_change(pred_change); + + // For each diff operation, traverse up to the root to build the path + for diff_op in diff_ops { + let mut path = vec![pred_name.clone()]; + + // Start recursion from this tracked subject + add_diff_ops_for_tracked_subject( + &tracked_subject, + &sub.tracked_subjects, + &sub.shape_type.shape, + &mut path, + diff_op, + &mut patches, + &mut paths_of_objects_to_create, + ); + } + } } - // Found a root subject for this shape. - - // Add subject IRI as first part of path pointer. - path.push(escape_json_pointer(subject_iri)); - create_patches_for_changed_subj( - &orm_changes, - &mut patches, - shape_iri, - subject_iri, - sub, - &mut path, - &sub.tracked_subjects, - ); - path.pop(); + } + } + + // Create patches for objects that need to be created + // These are patches with {op: add, valType: object, value: Null, path: ...} + // Sort by path length (shorter first) to ensure parent objects are created before children + let mut sorted_object_paths: Vec<_> = paths_of_objects_to_create.iter().collect(); + sorted_object_paths.sort_by_key(|(path_segments, _)| path_segments.len()); + + for (path_segments, maybe_iri) in sorted_object_paths { + let escaped_path: Vec = path_segments + .iter() + .map(|seg| escape_json_pointer(seg)) + .collect(); + let json_pointer = format!("/{}", escaped_path.join("/")); + + patches.push(OrmDiffOp { + op: OrmDiffOpType::add, + valType: Some(OrmDiffType::object), + path: json_pointer.clone(), + value: None, + }); + if let Some(iri) = maybe_iri { + patches.push(OrmDiffOp { + op: OrmDiffOpType::add, + valType: Some(OrmDiffType::object), + path: format!("{}/id", json_pointer), + value: Some(json!(iri)), + }); } } diff --git a/engine/verifier/src/orm/validation.rs b/engine/verifier/src/orm/validation.rs index fc41cc2..041cf58 100644 --- a/engine/verifier/src/orm/validation.rs +++ b/engine/verifier/src/orm/validation.rs @@ -105,7 +105,10 @@ impl Verifier { // Check 3) Validate subject against each predicate in shape. for p_schema in shape.predicates.iter() { let p_change = s_change.predicates.get(&p_schema.iri); - let tracked_pred = p_change.map(|pc| pc.tracked_predicate.read().unwrap()); + let tracked_pred = tracked_subject + .tracked_predicates + .get(&p_schema.iri) + .map(|tp_write_lock| tp_write_lock.read().unwrap()); let count = tracked_pred .as_ref() @@ -124,6 +127,8 @@ impl Verifier { set_validity(&mut new_validity, OrmTrackedSubjectValidity::Invalid); if count <= 0 { // If cardinality is 0, we can remove the tracked predicate. + // Drop the guard to release the immutable borrow + drop(tracked_pred); tracked_subject.tracked_predicates.remove(&p_schema.iri); } break; diff --git a/sdk/rust/src/tests/mod.rs b/sdk/rust/src/tests/mod.rs index 8c674ea..caac2a7 100644 --- a/sdk/rust/src/tests/mod.rs +++ b/sdk/rust/src/tests/mod.rs @@ -50,7 +50,12 @@ pub(crate) fn assert_json_eq(expected: &mut Value, actual: &mut Value) { let diff = serde_json_diff::values(expected.clone(), actual.clone()); if let Some(diff_) = diff { - log_err!("Expected and actual ORM JSON mismatch.\nDiff: {:?}", diff_); + log_err!( + "Expected and actual ORM JSON mismatch.\nDiff: {:?}\nExpected: {}\nActual: {}", + diff_, + expected, + actual + ); assert!(false); } } diff --git a/sdk/rust/src/tests/orm_patches.rs b/sdk/rust/src/tests/orm_patches.rs index fb14498..bcbb57c 100644 --- a/sdk/rust/src/tests/orm_patches.rs +++ b/sdk/rust/src/tests/orm_patches.rs @@ -23,6 +23,7 @@ use serde_json::json; use serde_json::Value; use std::collections::HashMap; use std::sync::Arc; +use svg2pdf::usvg::tiny_skia_path::SCALAR_NEARLY_ZERO; #[async_std::test] async fn test_orm_path_creation() { @@ -32,7 +33,8 @@ async fn test_orm_path_creation() { // Tests below all in this test, to prevent waiting times through wallet creation. // === - test_orm_root_array(session_id).await; + test_patch_add_array(session_id).await; + test_patch_remove_array(session_id).await; // // === // test_orm_with_optional(session_id).await; @@ -56,7 +58,7 @@ async fn test_orm_path_creation() { // test_orm_nested_4(session_id).await; } -async fn test_orm_root_array(session_id: u64) { +async fn test_patch_add_array(session_id: u64) { let doc_nuri = create_doc_with_data( session_id, r#" @@ -148,10 +150,13 @@ INSERT DATA { ex:arr 4 . - ex:arr 1 . + ex:arr 1, 2 . ex:arr 3 . + + + ex:arr 0 . } "# .to_string(), @@ -160,7 +165,6 @@ INSERT DATA { .await .expect("2nd SPARQL update failed"); - cancel_fn(); while let Some(app_response) = receiver.next().await { let patches = match app_response { AppResponse::V0(v) => match v { @@ -177,133 +181,65 @@ INSERT DATA { let mut expected = json!([ { - "id": "urn:test:numArrayObj1", - "type": "http://example.org/TestObject", - "numArray": [1.0, 2.0, 3.0] + "op": "add", + "valType": "set", + "value": [4.0], + "path": "/urn:test:numArrayObj1/numArray", + }, { - "id": "urn:test:numArrayObj2", - "type": "http://example.org/TestObject", - "numArray": [] + "op": "add", + "valType": "set", + "value": [1.0,2.0], + "path": "/urn:test:numArrayObj2/numArray", }, { - "id": "urn:test:numArrayObj3", - "type": "http://example.org/TestObject", - "numArray": [1.0, 2.0] - } - ]); - - let mut actual_mut = patches.clone(); - // assert_json_eq(&mut expected, actual_mut); - - break; - } -} - -/* - - - - - - -*/ -// -async fn test_orm_with_optional(session_id: u64) { - let doc_nuri = create_doc_with_data( - session_id, - r#" -PREFIX ex: -INSERT DATA { - - ex:opt true ; - ex:str "s1" . - - # Contains no matching data - - ex:str "s2" . -} -"# - .to_string(), - ) - .await; - - // Define the ORM schema - let mut schema = HashMap::new(); - schema.insert( - "http://example.org/OptionShape".to_string(), - OrmSchemaShape { - iri: "http://example.org/OptionShape".to_string(), - predicates: vec![OrmSchemaPredicate { - iri: "http://example.org/opt".to_string(), - extra: Some(false), - maxCardinality: 1, - minCardinality: 1, - readablePredicate: "opt".to_string(), - dataTypes: vec![OrmSchemaDataType { - valType: OrmSchemaLiteralType::boolean, - literals: None, - shape: None, - }], - } - .into()], - } - .into(), - ); - - let shape_type = OrmShapeType { - schema, - shape: "http://example.org/OptionShape".to_string(), - }; - - let nuri = NuriV0::new_from(&doc_nuri).expect("parse nuri"); - let (mut receiver, cancel_fn) = orm_start(nuri, shape_type, session_id) - .await - .expect("orm_start"); - - while let Some(app_response) = receiver.next().await { - let orm_json = match app_response { - AppResponse::V0(v) => match v { - AppResponseV0::OrmInitial(json) => Some(json), - _ => None, + "op": "add", + "valType": "set", + "value": [3.0], + "path": "/urn:test:numArrayObj3/numArray", }, - } - .unwrap(); - - let mut expected = json!([ { - "id": "urn:test:oj1", - "opt": true - } + "op": "add", + "valType": "object", + "path": "/urn:test:numArrayObj4", + "value": Value::Null + }, + { + "op": "add", + "value": "urn:test:numArrayObj4", + "path": "/urn:test:numArrayObj4/id", + "valType": Value::Null, + }, + { + "op": "add", + "valType": "set", + "value": [0.0], + "path": "/urn:test:numArrayObj4/numArray", + }, ]); - let mut actual_mut = orm_json.clone(); - assert_json_eq(&mut expected, &mut actual_mut); + let mut actual = json!(patches); + assert_json_eq(&mut expected, &mut actual); break; } - cancel_fn(); } -async fn test_orm_literal(session_id: u64) { +async fn test_patch_remove_array(session_id: u64) { let doc_nuri = create_doc_with_data( session_id, r#" PREFIX ex: INSERT DATA { - - ex:lit1 "lit 1" ; - ex:lit2 "lit 2" . - - # Valid because ex:lit1 allows extra. - - ex:lit1 "lit 1", "lit 1 extra" ; - ex:lit2 "lit 2" . - - # Invalid because ex:lit2 does not allow extra. - - ex:lit1 "lit 1" ; - ex:lit2 "lit 2", "lit 2 extra" . + a ex:TestObject ; + ex:arr 1, 2, 3 . + + a ex:TestObject . + + a ex:TestObject ; + ex:unrelated ex:TestObject ; + ex:arr 1, 2 . } "# .to_string(), @@ -313,34 +249,36 @@ INSERT DATA { // Define the ORM schema let mut schema = HashMap::new(); schema.insert( - "http://example.org/OptionShape".to_string(), + "http://example.org/TestShape".to_string(), OrmSchemaShape { - iri: "http://example.org/OptionShape".to_string(), + iri: "http://example.org/TestShape".to_string(), predicates: vec![ OrmSchemaPredicate { - iri: "http://example.org/lit1".to_string(), - extra: Some(true), - maxCardinality: -1, + iri: "http://www.w3.org/1999/02/22-rdf-syntax-ns#type".to_string(), + extra: Some(false), + maxCardinality: 1, minCardinality: 1, - readablePredicate: "lit1".to_string(), + readablePredicate: "type".to_string(), dataTypes: vec![OrmSchemaDataType { valType: OrmSchemaLiteralType::literal, - literals: Some(vec![BasicType::Str("lit 1".to_string())]), + literals: Some(vec![BasicType::Str( + "http://example.org/TestObject".to_string(), + )]), shape: None, }], } .into(), OrmSchemaPredicate { - iri: "http://example.org/lit2".to_string(), - extra: Some(false), - maxCardinality: 1, - minCardinality: 1, - readablePredicate: "lit2".to_string(), + iri: "http://example.org/arr".to_string(), dataTypes: vec![OrmSchemaDataType { - valType: OrmSchemaLiteralType::literal, - literals: Some(vec![BasicType::Str("lit 2".to_string())]), + valType: OrmSchemaLiteralType::number, + literals: None, shape: None, }], + extra: Some(false), + maxCardinality: -1, + minCardinality: 0, + readablePredicate: "numArray".to_string(), } .into(), ], @@ -350,7 +288,7 @@ INSERT DATA { let shape_type = OrmShapeType { schema, - shape: "http://example.org/OptionShape".to_string(), + shape: "http://example.org/TestShape".to_string(), }; let nuri = NuriV0::new_from(&doc_nuri).expect("parse nuri"); @@ -359,7 +297,7 @@ INSERT DATA { .expect("orm_start"); while let Some(app_response) = receiver.next().await { - let orm_json = match app_response { + let _ = match app_response { AppResponse::V0(v) => match v { AppResponseV0::OrmInitial(json) => Some(json), _ => None, @@ -367,153 +305,74 @@ INSERT DATA { } .unwrap(); - let mut expected = json!([ - { - "id": "urn:test:oj1", - "lit1": ["lit 1"], - "lit2": "lit 2" - }, - { - "id": "urn:test:obj2", - "lit1": ["lit 1", "lit 1 extra"], - "lit2": "lit 2" - } - ]); - - let mut actual_mut = orm_json.clone(); - assert_json_eq(&mut expected, &mut actual_mut); - break; } - cancel_fn(); -} -async fn test_orm_multi_type(session_id: u64) { - let doc_nuri = create_doc_with_data( + // Add more data, remove some + doc_sparql_update( session_id, r#" PREFIX ex: -INSERT DATA { - - ex:strOrNum "a string" ; - ex:strOrNum "another string" ; - ex:strOrNum 2 . - - # Invalid because false is not string or number. - - ex:strOrNum "a string2" ; - ex:strOrNum 2 ; - ex:strOrNum false . +DELETE DATA { + + ex:arr 1 . } "# .to_string(), + Some(doc_nuri.clone()), ) - .await; - - // Define the ORM schema - let mut schema = HashMap::new(); - schema.insert( - "http://example.org/MultiTypeShape".to_string(), - OrmSchemaShape { - iri: "http://example.org/MultiTypeShape".to_string(), - predicates: vec![OrmSchemaPredicate { - iri: "http://example.org/strOrNum".to_string(), - extra: Some(true), - maxCardinality: -1, - minCardinality: 1, - readablePredicate: "strOrNum".to_string(), - dataTypes: vec![ - OrmSchemaDataType { - valType: OrmSchemaLiteralType::string, - literals: None, - shape: None, - }, - OrmSchemaDataType { - valType: OrmSchemaLiteralType::number, - literals: None, - shape: None, - }, - ], - } - .into()], - } - .into(), - ); - - let shape_type = OrmShapeType { - schema, - shape: "http://example.org/MultiTypeShape".to_string(), - }; - - let nuri = NuriV0::new_from(&doc_nuri).expect("parse nuri"); - let (mut receiver, cancel_fn) = orm_start(nuri, shape_type, session_id) - .await - .expect("orm_start"); + .await + .expect("2nd SPARQL update failed"); while let Some(app_response) = receiver.next().await { - let orm_json = match app_response { + let patches = match app_response { AppResponse::V0(v) => match v { - AppResponseV0::OrmInitial(json) => Some(json), + AppResponseV0::OrmUpdate(json) => Some(json), _ => None, }, } .unwrap(); + log_info!("Diff ops arrived:\n"); + for patch in patches.iter() { + log_info!("{:?}", patch); + } + let mut expected = json!([ { - "id": "urn:test:oj1", - "strOrNum": ["a string", "another string", 2.0] + "op": "remove", + "valType": "set", + "value": [1.0], + "path": "/urn:test:numArrayObj1/numArray", + } ]); - let mut actual_mut = orm_json.clone(); - assert_json_eq(&mut expected, &mut actual_mut); + let mut actual = json!(patches); + assert_json_eq(&mut expected, &mut actual); break; } - cancel_fn(); } -async fn test_orm_nested_1(session_id: u64) { +async fn test_patch_add_nested_1(session_id: u64) { let doc_nuri = create_doc_with_data( session_id, r#" PREFIX ex: INSERT DATA { - # Valid - ex:str "obj1 str" ; - ex:nestedWithExtra , ; - ex:nestedWithoutExtra . + ex:multiNest , ; + ex:singleNest . - - ex:nestedStr "obj1 nested with extra valid" ; - ex:nestedNum 2 . + + ex:multiNest1Str "a multi 1 string" . - - ex:nestedStr "obj1 nested with extra invalid" . + + ex:multiNest2Str "a multi 2 string" . - ex:nestedStr "obj1 nested without extra valid" ; - ex:nestedNum 2 . - - # Invalid because nestedWithoutExtra has an invalid child. - - ex:str "obj2 str" ; - ex:nestedWithExtra ; - ex:nestedWithoutExtra , . - - - ex:nestedStr "obj2: a nested string valid" ; - ex:nestedNum 2 . - - - ex:nestedStr "obj2 nested without extra valid" ; - ex:nestedNum 2 . - - # Invalid because nestedNum is missing. - - ex:nestedStr "obj2 nested without extra invalid" . + ex:singleNestStr "a single nest string" . } "# .to_string(), @@ -528,41 +387,35 @@ INSERT DATA { iri: "http://example.org/RootShape".to_string(), predicates: vec![ OrmSchemaPredicate { - iri: "http://example.org/str".to_string(), + iri: "http://example.org/multiNest".to_string(), extra: None, - maxCardinality: 1, + maxCardinality: 6, minCardinality: 1, - readablePredicate: "str".to_string(), - dataTypes: vec![OrmSchemaDataType { - valType: OrmSchemaLiteralType::string, - literals: None, - shape: None, - }], + readablePredicate: "multiNest".to_string(), + dataTypes: vec![ + OrmSchemaDataType { + valType: OrmSchemaLiteralType::shape, + literals: None, + shape: Some("http://example.org/MultiNestShape1".to_string()), + }, + OrmSchemaDataType { + valType: OrmSchemaLiteralType::shape, + literals: None, + shape: Some("http://example.org/MultiNestShape2".to_string()), + }, + ], } .into(), OrmSchemaPredicate { - iri: "http://example.org/nestedWithExtra".to_string(), + iri: "http://example.org/singleNest".to_string(), extra: Some(true), maxCardinality: 1, minCardinality: 1, - readablePredicate: "nestedWithExtra".to_string(), + readablePredicate: "singleNest".to_string(), dataTypes: vec![OrmSchemaDataType { valType: OrmSchemaLiteralType::shape, literals: None, - shape: Some("http://example.org/NestedShapeWithExtra".to_string()), - }], - } - .into(), - OrmSchemaPredicate { - iri: "http://example.org/nestedWithoutExtra".to_string(), - extra: Some(false), - maxCardinality: 1, - minCardinality: 1, - readablePredicate: "nestedWithoutExtra".to_string(), - dataTypes: vec![OrmSchemaDataType { - valType: OrmSchemaLiteralType::shape, - literals: None, - shape: Some("http://example.org/NestedShapeWithoutExtra".to_string()), + shape: Some("http://example.org/SingleNestShape".to_string()), }], } .into(), @@ -571,72 +424,62 @@ INSERT DATA { .into(), ); schema.insert( - "http://example.org/NestedShapeWithExtra".to_string(), + "http://example.org/SingleNestShape".to_string(), OrmSchemaShape { - iri: "http://example.org/NestedShapeWithExtra".to_string(), - predicates: vec![ - OrmSchemaPredicate { - iri: "http://example.org/nestedStr".to_string(), - extra: None, - readablePredicate: "nestedStr".to_string(), - maxCardinality: 1, - minCardinality: 1, - dataTypes: vec![OrmSchemaDataType { - valType: OrmSchemaLiteralType::string, - literals: None, - shape: None, - }], - } - .into(), - OrmSchemaPredicate { - iri: "http://example.org/nestedNum".to_string(), - extra: None, - readablePredicate: "nestedNum".to_string(), - maxCardinality: 1, - minCardinality: 1, - dataTypes: vec![OrmSchemaDataType { - valType: OrmSchemaLiteralType::number, - literals: None, - shape: None, - }], - } - .into(), - ], + iri: "http://example.org/SingleNestShape".to_string(), + predicates: vec![OrmSchemaPredicate { + iri: "http://example.org/singleNestStr".to_string(), + extra: None, + readablePredicate: "str".to_string(), + maxCardinality: 1, + minCardinality: 1, + dataTypes: vec![OrmSchemaDataType { + valType: OrmSchemaLiteralType::string, + literals: None, + shape: None, + }], + } + .into()], } .into(), ); schema.insert( - "http://example.org/NestedShapeWithoutExtra".to_string(), + "http://example.org/MultiNestShape1".to_string(), OrmSchemaShape { - iri: "http://example.org/NestedShapeWithoutExtra".to_string(), - predicates: vec![ - OrmSchemaPredicate { - iri: "http://example.org/nestedStr".to_string(), - extra: None, - readablePredicate: "nestedStr".to_string(), - maxCardinality: 1, - minCardinality: 1, - dataTypes: vec![OrmSchemaDataType { - valType: OrmSchemaLiteralType::string, - literals: None, - shape: None, - }], - } - .into(), - OrmSchemaPredicate { - iri: "http://example.org/nestedNum".to_string(), - extra: None, - readablePredicate: "nestedNum".to_string(), - maxCardinality: 1, - minCardinality: 1, - dataTypes: vec![OrmSchemaDataType { - valType: OrmSchemaLiteralType::number, - literals: None, - shape: None, - }], - } - .into(), - ], + iri: "http://example.org/MultiNestShape1".to_string(), + predicates: vec![OrmSchemaPredicate { + iri: "http://example.org/multiNest1Str".to_string(), + extra: None, + readablePredicate: "string1".to_string(), + maxCardinality: 1, + minCardinality: 1, + dataTypes: vec![OrmSchemaDataType { + valType: OrmSchemaLiteralType::string, + literals: None, + shape: None, + }], + } + .into()], + } + .into(), + ); + schema.insert( + "http://example.org/MultiNestShape2".to_string(), + OrmSchemaShape { + iri: "http://example.org/MultiNestShape2".to_string(), + predicates: vec![OrmSchemaPredicate { + iri: "http://example.org/multiNest2Str".to_string(), + extra: None, + readablePredicate: "string2".to_string(), + maxCardinality: 1, + minCardinality: 1, + dataTypes: vec![OrmSchemaDataType { + valType: OrmSchemaLiteralType::string, + literals: None, + shape: None, + }], + } + .into()], } .into(), ); @@ -660,29 +503,103 @@ INSERT DATA { } .unwrap(); + break; + } + + // Add more data, remove some + doc_sparql_update( + session_id, + r#" +PREFIX ex: +INSERT DATA { + + ex:multiNest1Str "replacing object shape view" . + + + ex:multiNest2Str "multi 4 added" . + + + ex:singleNestStr "Different nested val" . +} +"# + .to_string(), + Some(doc_nuri.clone()), + ) + .await + .expect("2nd SPARQL update failed"); + + while let Some(app_response) = receiver.next().await { + let patches = match app_response { + AppResponse::V0(v) => match v { + AppResponseV0::OrmUpdate(json) => Some(json), + _ => None, + }, + } + .unwrap(); + + log_info!("Diff ops arrived:\n"); + for patch in patches.iter() { + log_info!("{:?}", patch); + } + let mut expected = json!([ { - "id": "urn:test:oj1", - "str": "obj1 str", - "nestedWithExtra": { - "nestedStr": "obj1 nested with extra valid", - "nestedNum": 2.0 - }, - "nestedWithoutExtra": { - "nestedStr": "obj1 nested without extra valid", - "nestedNum": 2.0 - } - } + "op": "remove", + "path": "/urn:test:oj1/multiNest/urn:test:multiNested2/string2", + // "valType": None, + // "value": None, + }, + { + "op": "add", + // "valType": None, + "value": "replacing object shape view", + "path": "/urn:test:oj1/multiNest/urn:test:multiNested2/string1", + }, + { + "op": "add", + "valType": "object", + // "value": None, + "path": "/urn:test:oj1/multiNest/urn:test:multiNested4", + }, + { + "op": "add", + // "valType": None, + "value": "urn:test:multiNested4", + "path": "/urn:test:oj1/multiNest/urn:test:multiNested4/id", + }, + { + "op": "add", + // "valType": None, + "value": "multi 4 added", + "path": "/urn:test:oj1/multiNest/urn:test:multiNested4/string2", + }, + { + "op": "remove", + // "valType": None, + // "value": None, + "path": "/urn:test:oj1/singleNest/str", + }, + { + "op": "add", + // "valType": None, + "value": "Different nested val", + "path": "/urn:test:oj1/singleNest/str", + }, ]); - let mut actual_mut = orm_json.clone(); - assert_json_eq(&mut expected, &mut actual_mut); + let mut actual = json!(patches); + assert_json_eq(&mut expected, &mut actual); break; } - cancel_fn(); } +/* + + +Old things + +*/ async fn test_orm_nested_2(session_id: u64) { let doc_nuri = create_doc_with_data( session_id,