NextGraph
/
nextgraph-rs
1
6
Fork 2
Code Issues 4 Pull Requests 1 Packages Projects 2 Releases 3 Activity

wip

Browse Source
feat/orm
Laurin Weger 3 weeks ago
parent 27ec9302a6
commit c9d26d87f2
No known key found for this signature in database
GPG Key ID: 9B372BB0B792770F
2 changed files with 255 additions and 208 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 13
      ng-net/src/orm.rs
  2. 450
      ng-verifier/src/orm.rs

13
ng-net/src/orm.rs
Unescape View File

@ -11,6 +11,7 @@
#![allow(non_snake_case)] #![allow(non_snake_case)]
use std::collections::HashSet;
use std::{collections::HashMap, rc::Weak}; use std::{collections::HashMap, rc::Weak};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
@ -100,14 +101,14 @@ pub struct OrmSchemaPredicate {
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
pub struct OrmSubscription<'a> { pub struct OrmSubscription<'a> {
pub sender: Sender<AppResponse>, pub sender: Sender<AppResponse>,
pub tracked_objects: HashMap<String, OrmTrackedSubject<'a>>, pub tracked_objects: HashMap<String, OrmTrackedSubjectAndShape<'a>>,
} }
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
pub struct OrmTrackedSubject<'a> { pub struct OrmTrackedSubjectAndShape<'a> {
pub tracked_predicates: HashMap<String, OrmTrackedPredicate<'a>>, pub tracked_predicates: HashMap<String, OrmTrackedPredicate<'a>>,
// Parents and if they are currently tracking us. // Parents and if they are currently tracking us.
pub parents: HashMap<String, (OrmTrackedSubject<'a>, bool)>, pub parents: HashMap<String, (OrmTrackedSubjectAndShape<'a>, bool)>,
pub valid: OrmTrackedSubjectValidity, pub valid: OrmTrackedSubjectValidity,
pub subj_iri: &'a String, pub subj_iri: &'a String,
pub shape: &'a OrmSchemaShape, pub shape: &'a OrmSchemaShape,
@ -117,14 +118,15 @@ pub struct OrmTrackedSubject<'a> {
pub enum OrmTrackedSubjectValidity { pub enum OrmTrackedSubjectValidity {
Valid, Valid,
Invalid, Invalid,
Unknown, NotEvaluated,
Untracked, Untracked,
NeedsFetch,
} }
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
pub struct OrmTrackedPredicate<'a> { pub struct OrmTrackedPredicate<'a> {
pub schema: &'a OrmSchemaPredicate, pub schema: &'a OrmSchemaPredicate,
pub tracked_children: Vec<Weak<OrmTrackedSubject<'a>>>, pub tracked_children: Vec<Weak<OrmTrackedSubjectAndShape<'a>>>,
pub current_cardinality: i32, pub current_cardinality: i32,
pub current_literals: Option<Vec<BasicType>>, pub current_literals: Option<Vec<BasicType>>,
} }
@ -135,7 +137,6 @@ pub struct OrmTrackedSubjectChange<'a> {
pub subject_iri: String, pub subject_iri: String,
pub predicates: HashMap<String, OrmTrackedPredicateChanges<'a>>, pub predicates: HashMap<String, OrmTrackedPredicateChanges<'a>>,
pub valid: OrmTrackedSubjectValidity, pub valid: OrmTrackedSubjectValidity,
pub tracked_subject: &'a OrmTrackedSubject<'a>,
} }
pub struct OrmTrackedPredicateChanges<'a> { pub struct OrmTrackedPredicateChanges<'a> {
pub tracked_predicate: &'a OrmTrackedPredicate<'a>, pub tracked_predicate: &'a OrmTrackedPredicate<'a>,

450
ng-verifier/src/orm.rs
Unescape View File

@ -25,10 +25,9 @@ use ng_net::orm::OrmShapeTypeRef;
use ng_net::orm::OrmSubscription; use ng_net::orm::OrmSubscription;
use ng_net::orm::OrmTrackedPredicate; use ng_net::orm::OrmTrackedPredicate;
use ng_net::orm::OrmTrackedPredicateChanges; use ng_net::orm::OrmTrackedPredicateChanges;
use ng_net::orm::OrmTrackedSubject; use ng_net::orm::OrmTrackedSubjectAndShape;
use ng_net::orm::OrmTrackedSubjectChange; use ng_net::orm::OrmTrackedSubjectChange;
use ng_net::orm::OrmTrackedSubjectValidity; use ng_net::orm::OrmTrackedSubjectValidity;
use ng_net::orm::Term;
use ng_net::orm::{OrmSchemaDataType, OrmSchemaShape}; use ng_net::orm::{OrmSchemaDataType, OrmSchemaShape};
use ng_net::{app_protocol::*, orm::OrmSchema}; use ng_net::{app_protocol::*, orm::OrmSchema};
use ng_net::{ use ng_net::{
@ -39,12 +38,13 @@ use ng_oxigraph::oxigraph::sparql::{results::*, Query, QueryResults};
use ng_oxigraph::oxrdf::LiteralRef; use ng_oxigraph::oxrdf::LiteralRef;
use ng_oxigraph::oxrdf::NamedNode; use ng_oxigraph::oxrdf::NamedNode;
use ng_oxigraph::oxrdf::Subject; use ng_oxigraph::oxrdf::Subject;
use ng_oxigraph::oxrdf::Term;
use ng_oxigraph::oxrdf::Triple; use ng_oxigraph::oxrdf::Triple;
use ng_repo::errors::NgError; use ng_repo::errors::NgError;
use ng_repo::errors::VerifierError; use ng_repo::errors::VerifierError;
use ng_repo::log::*; use ng_repo::log::*;
use regex::Regex; use regex::Regex;
use serde_json::json;
use serde_json::Value;
use crate::types::*; use crate::types::*;
use crate::verifier::*; use crate::verifier::*;
@ -92,94 +92,60 @@ impl Verifier {
&mut self, &mut self,
scope: &NuriV0, scope: &NuriV0,
schema: &OrmSchema, schema: &OrmSchema,
shape: &String, root_shape: &OrmSchemaShape,
triples_added: &Vec<Triple>, triples_added: &Vec<Triple>,
triples_removed: &Vec<Triple>, triples_removed: &Vec<Triple>,
) { ) -> HashMap<String, HashMap<String, OrmTrackedSubjectChange>> {
let tracked_subjects: HashMap<String, HashMap<String, OrmTrackedSubject>> = let tracked_subjects = self.orm_tracked_subjects;
self.orm_tracked_subjects;
// Structure to store changes in. // === Helper functions ===
let mut subject_changes: HashMap<String, OrmTrackedSubjectChange> = HashMap::new();
fn group_by_subject_for_shape<'a>(
// Group triples by predicate (only keep predicates defined in the shape). Drop others. shape: &'a OrmSchemaShape,
let mut added_triples_by_pred: HashMap<String, Vec<Triple>> = HashMap::new(); triples: &'a Vec<Triple>,
let Some(shape_def) = schema.get(shape) else { allowed_subjects: &Vec<String>,
log_err!( ) -> HashMap<String, Vec<&'a Triple>> {
"Shape {} not found in schema when grouping triples by predicate", let mut triples_by_pred: HashMap<String, Vec<&Triple>> = HashMap::new();
shape let allowed_preds: HashSet<&str> =
); shape.predicates.iter().map(|p| p.iri.as_str()).collect();
return; let allowed_objs: HashSet<&String> = allowed_subjects.iter().collect();
}; for triple in triples {
// triple.subject must be in allowed_subjects (or allowed_subjects empty)
// Collect allowed predicate IRIs for this shape // and triple.predicate must be in allowed_preds.
let allowed: std::collections::HashSet<&str> = shape_def if (allowed_objs.is_empty() || allowed_objs.contains(&triple.subject.to_string()))
.predicates && allowed_preds.contains(triple.predicate.as_str())
.iter() {
.map(|p| p.iri.as_str()) triples_by_pred
.collect(); .entry(triple.predicate.as_str().to_string())
.or_insert_with(|| vec![])
for triple in triples_added { .push(triple);
if allowed.contains(triple.predicate.as_str()) { }
added_triples_by_pred
.entry(triple.predicate.as_str().to_string())
.or_insert_with(|| vec![])
.push(*triple);
} }
} // Based on those triples, group by subject.
let mut triples_by_subject: HashMap<String, Vec<&Triple>> = HashMap::new();
// Based on those triples, group by subject. for triple in triples {
let mut added_triples_by_subject: HashMap<String, Vec<Triple>> = HashMap::new(); let subject_iri = match &triple.subject {
for triple in triples_added { Subject::NamedNode(node) => node.as_str(),
let subject_iri = match &triple.subject { _ => continue, // Won't happen.
Subject::NamedNode(node) => node.as_str(), };
_ => continue, // Won't happen. triples_by_subject
}; .entry(subject_iri.to_string())
added_triples_by_subject
.entry(subject_iri.to_string())
.or_insert_with(|| vec![])
.push(triple.clone());
}
// Do the same for removed ones.
let mut removed_triples_by_pred: HashMap<String, Vec<Triple>> = HashMap::new();
// Collect allowed predicate IRIs for this shape
let allowed: std::collections::HashSet<&str> = shape_def
.predicates
.iter()
.map(|p| p.iri.as_str())
.collect();
for triple in triples_removed {
if allowed.contains(triple.predicate.as_str()) {
removed_triples_by_pred
.entry(triple.predicate.as_str().to_string())
.or_insert_with(|| vec![]) .or_insert_with(|| vec![])
.push(*triple); .push(&triple);
} }
} return triples_by_subject;
let mut removed_triples_by_subject: HashMap<String, Vec<Triple>> = HashMap::new();
for triple in triples_removed {
let subject_iri = match &triple.subject {
Subject::NamedNode(node) => node.as_str(),
_ => continue, // Won't happen.
};
removed_triples_by_subject
.entry(subject_iri.to_string())
.or_insert_with(|| vec![])
.push(triple.clone());
} }
// Assumes all triples have same subject. /// Add all triples to `changes`
fn orm_from_triple_for_level<'a>( /// Returns predicates to nested objects that were touched and need processing.
/// Assumes all triples have same subject.
fn add_remove_triples(
shape: &OrmSchemaShape, shape: &OrmSchemaShape,
subject_iri: &String, subject_iri: &String,
triples_added: &Vec<Triple>, triples_added: &Vec<&Triple>,
triples_removed: &Vec<Triple>, triples_removed: &Vec<&Triple>,
tracked_subjects: &HashMap<String, HashMap<String, OrmTrackedSubject<'a>>>, tracked_subjects: &HashMap<String, HashMap<String, OrmTrackedSubjectAndShape>>,
changes: &HashMap<String, OrmTrackedSubjectChange>, subject_changes: &OrmTrackedSubjectChange,
) -> (
Vec<&'a OrmTrackedPredicateChanges<'a>>,
Vec<&'a OrmTrackedPredicateChanges<'a>>,
) { ) {
let tracked_shapes_for_subject = tracked_subjects let tracked_shapes_for_subject = tracked_subjects
.entry(subject_iri.clone()) .entry(subject_iri.clone())
@ -187,28 +153,15 @@ impl Verifier {
let tracked_subject = tracked_shapes_for_subject let tracked_subject = tracked_shapes_for_subject
.entry(subject_iri.clone()) .entry(subject_iri.clone())
.or_insert_with(|| OrmTrackedSubject { .or_insert_with(|| OrmTrackedSubjectAndShape {
tracked_predicates: HashMap::new(), tracked_predicates: HashMap::new(),
parents: HashMap::new(), parents: HashMap::new(),
valid: ng_net::orm::OrmTrackedSubjectValidity::Unknown, valid: ng_net::orm::OrmTrackedSubjectValidity::NotEvaluated,
subj_iri: subject_iri, subj_iri: subject_iri,
shape, shape,
}); });
let subject_changes = // Process added triples.
changes
.entry(subject_iri.clone())
.or_insert_with(|| OrmTrackedSubjectChange {
subject_iri: subject_iri.clone(),
predicates: HashMap::new(),
tracked_subject,
valid: OrmTrackedSubjectValidity::Unknown,
});
// Keep track of all children that were spotted or removed.
let mut children_removed: Vec<&OrmTrackedPredicateChanges> = vec![];
let mut children_added: Vec<&OrmTrackedPredicateChanges> = vec![];
// For each triple, check matching predicates in shape. // For each triple, check matching predicates in shape.
// keeping track of value count (for later validations). // keeping track of value count (for later validations).
// In parallel, we keep track of the values added (tracked_changes) // In parallel, we keep track of the values added (tracked_changes)
@ -242,7 +195,7 @@ impl Verifier {
tracked_predicate: &tp, tracked_predicate: &tp,
values_added: Vec::new(), values_added: Vec::new(),
values_removed: Vec::new(), values_removed: Vec::new(),
validity: OrmTrackedSubjectValidity::Unknown, validity: OrmTrackedSubjectValidity::NotEvaluated,
}); });
pred_changes.values_added.push(obj_term.clone()); pred_changes.values_added.push(obj_term.clone());
@ -254,24 +207,16 @@ impl Verifier {
.iter() .iter()
.any(|dt| dt.valType == OrmSchemaLiteralType::literal) .any(|dt| dt.valType == OrmSchemaLiteralType::literal)
{ {
if let Some(current_literals) = &mut tp.current_literals { if let Some(current_literals) = tp.current_literals {
current_literals.push(obj_term); current_literals.push(obj_term);
} else { } else {
tp.current_literals = Some(vec![obj_term]); tp.current_literals.insert(vec![obj_term]);
} }
} else if tp
.schema
.dataTypes
.iter()
.any(|dt| dt.valType == OrmSchemaLiteralType::shape)
{
// For nested, add object to tracked predicates and add self as parent.
children_added.push(&pred_changes);
} }
} }
} }
// Removed triples // Process removed triples.
for triple in triples_removed { for triple in triples_removed {
let pred_iri = triple.predicate.as_str(); let pred_iri = triple.predicate.as_str();
@ -309,32 +254,27 @@ impl Verifier {
} else { } else {
tp.current_literals = Some(vec![val_removed]); tp.current_literals = Some(vec![val_removed]);
} }
} else if tp
.schema
.dataTypes
.iter()
.any(|dt| dt.valType == OrmSchemaLiteralType::shape)
{
// For nested, add object to tracked predicates and add self as parent.
children_removed.push(&pred_changes);
} }
} }
return (children_added, children_removed);
} }
/// Check the validity of a subject.
/// Might return nested objects that need to be validated.
/// Assumes all triples to be of same subject.
fn check_subject_validity<'a>( fn check_subject_validity<'a>(
s_change: &'a OrmTrackedSubjectChange<'a>, s_change: &'a OrmTrackedSubjectChange<'a>,
shape: &String, shape: &OrmSchemaShape,
schema: &'a OrmSchema, schema: &'a OrmSchema,
tracked_subjects: &HashMap<String, HashMap<String, OrmTrackedSubjectAndShape<'a>>>,
previous_validity: OrmTrackedSubjectValidity, previous_validity: OrmTrackedSubjectValidity,
) -> ( ) -> (
OrmTrackedSubjectValidity, OrmTrackedSubjectValidity,
HashMap<String, HashMap<String, &'a OrmTrackedSubjectChange<'a>>>, // Vec<subject_iri, shape, needs_refetch>
Vec<(&'a String, &'a OrmSchemaShape, bool)>,
) { ) {
// Check 1) If there are no changes, there is nothing to do.
if s_change.predicates.is_empty() { if s_change.predicates.is_empty() {
// There has not been any changes. There is nothing to do. return (previous_validity, vec![]);
return (previous_validity, HashMap::new());
} }
let previous_validity = s_change.valid; let previous_validity = s_change.valid;
@ -344,24 +284,45 @@ impl Verifier {
if new_val == OrmTrackedSubjectValidity::Invalid { if new_val == OrmTrackedSubjectValidity::Invalid {
new_validity = OrmTrackedSubjectValidity::Invalid; new_validity = OrmTrackedSubjectValidity::Invalid;
// Remove all tracked predicates // Remove all tracked predicates
s_change.tracked_subject.tracked_predicates = HashMap::new(); s_change.tracked_subjects.tracked_predicates = HashMap::new();
} else if new_val == OrmTrackedSubjectValidity::Unknown } else if new_val == OrmTrackedSubjectValidity::NotEvaluated
&& new_validity != OrmTrackedSubjectValidity::Invalid && new_validity != OrmTrackedSubjectValidity::Invalid
{ {
new_validity = OrmTrackedSubjectValidity::Unknown; new_validity = OrmTrackedSubjectValidity::NotEvaluated;
} }
}; };
let tracked_subject = s_change.tracked_subject; let tracked_subject = tracked_subjects
let shape = schema.get(shape).expect("Shape not available"); .get(&s_change.subject_iri)
.unwrap()
.get(&shape.iri)
.unwrap();
// Check 2) If all parents are untracked, return untracked.
if tracked_subject
.parents
.values()
.all(|(parent, tracked)| !tracked)
{
// Remove tracked predicates and set untracked.
tracked_subject.tracked_predicates = HashMap::new();
tracked_subject.valid = OrmTrackedSubjectValidity::Untracked;
return (OrmTrackedSubjectValidity::Untracked, vec![]);
}
// TODO: Check parent validities: // Check 3) If there is an infinite loop of parents pointing back to us, return invalid.
// If no parent is tracking us, we are untracked. // Create a set of visited parents to detect cycles.
// If there is an infinite loop of parents pointing back to use, return invalid. if has_cycle(tracked_subject, &mut HashSet::new()) {
// Remove tracked predicates and set invalid.
tracked_subject.tracked_predicates = HashMap::new();
tracked_subject.valid = OrmTrackedSubjectValidity::Invalid;
return (OrmTrackedSubjectValidity::Invalid, vec![]);
}
// Keep track of objects that need to be validated against a shape to fetch and validate. // Keep track of objects that need to be validated against a shape to fetch and validate.
let mut new_unknowns: Vec<(&String, &OrmSchemaShape)> = vec![]; let mut new_unknowns: Vec<(&String, &OrmSchemaShape, bool)> = vec![];
// Check 4) Validate subject against each predicate in shape.
for p_schema in shape.predicates.iter() { for p_schema in shape.predicates.iter() {
let p_change = s_change.predicates.get(&p_schema.iri); let p_change = s_change.predicates.get(&p_schema.iri);
let tracked_pred = p_change.map(|pc| pc.tracked_predicate); let tracked_pred = p_change.map(|pc| pc.tracked_predicate);
@ -369,12 +330,15 @@ impl Verifier {
let count = tracked_pred let count = tracked_pred
.map_or_else(|| 0, |tp: &OrmTrackedPredicate<'_>| tp.current_cardinality); .map_or_else(|| 0, |tp: &OrmTrackedPredicate<'_>| tp.current_cardinality);
// Check 4.1) Cardinality
if count < p_schema.minCardinality { if count < p_schema.minCardinality {
set_validity(OrmTrackedSubjectValidity::Invalid); set_validity(OrmTrackedSubjectValidity::Invalid);
if count <= 0 { if count <= 0 {
// If no other parent is tracking, remove all tracked predicates. // If cardinality is 0, we can remove the tracked predicate.
tracked_subject.tracked_predicates.remove(&p_schema.iri); tracked_subject.tracked_predicates.remove(&p_schema.iri);
} }
break;
// Check 4.2) Cardinality too high and extra values not allowed.
} else if count > p_schema.maxCardinality } else if count > p_schema.maxCardinality
&& p_schema.maxCardinality != -1 && p_schema.maxCardinality != -1
&& p_schema.extra != Some(true) && p_schema.extra != Some(true)
@ -382,6 +346,7 @@ impl Verifier {
// If cardinality is too high and no extra allowed, invalid. // If cardinality is too high and no extra allowed, invalid.
set_validity(OrmTrackedSubjectValidity::Invalid); set_validity(OrmTrackedSubjectValidity::Invalid);
break; break;
// Check 4.3) Literal present types and valid.
} else if p_schema } else if p_schema
.dataTypes .dataTypes
.iter() .iter()
@ -418,6 +383,7 @@ impl Verifier {
break; break;
} }
} }
// Check 4.4) Nested shape correct.
} else if p_schema } else if p_schema
.dataTypes .dataTypes
.iter() .iter()
@ -435,7 +401,7 @@ impl Verifier {
(1, 0, 0, 0) (1, 0, 0, 0)
} else if tc.valid == OrmTrackedSubjectValidity::Invalid { } else if tc.valid == OrmTrackedSubjectValidity::Invalid {
(0, 1, 0, 0) (0, 1, 0, 0)
} else if tc.valid == OrmTrackedSubjectValidity::Unknown { } else if tc.valid == OrmTrackedSubjectValidity::NotEvaluated {
(0, 0, 1, 0) (0, 0, 1, 0)
} else if tc.valid == OrmTrackedSubjectValidity::Untracked { } else if tc.valid == OrmTrackedSubjectValidity::Untracked {
(0, 0, 0, 1) (0, 0, 0, 1)
@ -459,7 +425,7 @@ impl Verifier {
break; break;
} else if counts.3 > 0 { } else if counts.3 > 0 {
// If we have untracked nested objects, we need to fetch them and validate. // If we have untracked nested objects, we need to fetch them and validate.
set_validity(OrmTrackedSubjectValidity::Unknown); set_validity(OrmTrackedSubjectValidity::NotEvaluated);
// Add them to the list of unknowns to fetch and validate. // Add them to the list of unknowns to fetch and validate.
for o in tracked_pred for o in tracked_pred
.iter() .iter()
@ -467,17 +433,18 @@ impl Verifier {
{ {
if let Some(tc) = o.upgrade() { if let Some(tc) = o.upgrade() {
if tc.valid == OrmTrackedSubjectValidity::Untracked { if tc.valid == OrmTrackedSubjectValidity::Untracked {
new_unknowns.push((tc.subj_iri, tc.shape)); new_unknowns.push((tc.subj_iri, tc.shape, true));
} }
} }
} }
} else if counts.2 > 0 { } else if counts.2 > 0 {
// If we have unknown nested objects, we need to wait for their evaluation. // If we have unknown nested objects, we need to wait for their evaluation.
set_validity(OrmTrackedSubjectValidity::Unknown); set_validity(OrmTrackedSubjectValidity::NotEvaluated);
} else { } else {
// All nested objects are valid and cardinality is correct. // All nested objects are valid and cardinality is correct.
// We are valid with this predicate. // We are valid with this predicate.
} }
// Check 4.5) Data types correct.
} else { } else {
// Check if the data type is correct. // Check if the data type is correct.
let allowed_types: Vec<OrmSchemaLiteralType> = let allowed_types: Vec<OrmSchemaLiteralType> =
@ -508,17 +475,19 @@ impl Verifier {
}; };
} }
// TODO
// If we are invalid, we can discard new unknowns again - they won't be kept in memory. // If we are invalid, we can discard new unknowns again - they won't be kept in memory.
// We need to inform all children (by returning them for later evaluation), to untrack them.
// TODO: Collect info about all children to untrack them.
if new_validity == OrmTrackedSubjectValidity::Invalid { if new_validity == OrmTrackedSubjectValidity::Invalid {
return (OrmTrackedSubjectValidity::Invalid, HashMap::new()); return (OrmTrackedSubjectValidity::Invalid, vec![]);
} else if (new_validity == OrmTrackedSubjectValidity::Valid } else if new_validity == OrmTrackedSubjectValidity::Valid
&& previous_validity != OrmTrackedSubjectValidity::Valid) && previous_validity != OrmTrackedSubjectValidity::Valid
{ {
// If the validity is newly valid, we need to refetch this subject.
// TODO // TODO
// If this subject became valid, we need to refetch this subject.
} }
// If validity changed, inform parents (add to new_unknowns). // TODO...
// If validity changed, parents need to be re-evaluated.
if new_validity != previous_validity { if new_validity != previous_validity {
// TODO // TODO
} }
@ -527,69 +496,130 @@ impl Verifier {
return (new_validity, new_unknowns); return (new_validity, new_unknowns);
} }
let all_subjects: HashSet<&String> = added_triples_by_subject // === Validation ===
.keys()
.chain(removed_triples_by_subject.keys()) // FILO queue: To validate object changes (nested objects first). Strings are object IRIs.
.collect(); let mut shape_validation_queue: Vec<(&OrmSchemaShape, Vec<String>)> = vec![];
// Add root shape for first validation run.
// Process added/removed triples for each subject. shape_validation_queue.push((&root_shape, vec![]));
for subject_iri in all_subjects {
let triples_added_for_subj = added_triples_by_subject // Structure to store changes in. By shape iri > subject iri > OrmTrackedSubjectChange
.get(subject_iri) let mut shape_and_subject_changes: HashMap<
.unwrap_or(&vec![]) String,
.to_vec(); HashMap<String, OrmTrackedSubjectChange>,
let triples_removed_for_subj = removed_triples_by_subject > = HashMap::new();
.get(subject_iri)
.unwrap_or(&vec![]) // Process queue of shapes and subjects to validate.
.to_vec(); while let Some((shape, objects_to_validate)) = shape_validation_queue.pop() {
// For a given shape, we evaluate every subject against that shape.
let _ = orm_from_triple_for_level(
&shape_def, // Collect triples relevant for validation.
&subject_iri, let added_triples_by_subject =
&triples_added_for_subj, group_by_subject_for_shape(shape, triples_added, &objects_to_validate);
&triples_removed_for_subj, let removed_triples_by_subject =
&tracked_subjects, group_by_subject_for_shape(shape, triples_removed, &objects_to_validate);
&subject_changes, let all_modified_subjects: HashSet<&String> = added_triples_by_subject
); .keys()
} .chain(removed_triples_by_subject.keys())
.collect();
// Use to collect nested objects that need validation.
// First string is shape IRI, second are object IRIs.
let nested_objects_to_validate: HashMap<String, Vec<String>> = HashMap::new();
// For each subject, add/remove triples and validate.
for subject_iri in all_modified_subjects {
let triples_added_for_subj = added_triples_by_subject
.get(subject_iri)
.unwrap_or(&vec![])
.to_vec();
let triples_removed_for_subj = removed_triples_by_subject
.get(subject_iri)
.unwrap_or(&vec![])
.to_vec();
// Get or create change object for (shape, subject) pair.
let change = shape_and_subject_changes
.entry(shape.iri.clone())
.or_insert_with(|| HashMap::new())
.entry(subject_iri.clone())
.or_insert_with(|| OrmTrackedSubjectChange {
subject_iri: subject_iri.clone(),
predicates: HashMap::new(),
valid: OrmTrackedSubjectValidity::NeedsFetch,
});
// Apply all triples for that subject to the tracked (shape, subject) pair.
// Record the changes.
add_remove_triples(
shape,
&subject_iri,
&triples_added_for_subj,
&triples_removed_for_subj,
&tracked_subjects,
&change,
);
let tracked_subject = tracked_subjects.get(subject_iri).unwrap();
// Validate the subject.
let (new_validity, new_unknowns) = check_subject_validity(
&change,
shape,
schema,
tracked_subjects,
tracked_subject,
);
// TODO: Add logic to fetch un-fetched objects after validation.
// and return logic to add unprocessed nested objects after validation.
// TODO ==== // We add the new_unknowns to be processed next
for (iri, schema) in new_unknowns {
// To process validation, we collect all subject changes in one of the buckets. // Add to nested_objects_to_validate.
// Subjects for which we did not apply triples. nested_objects_to_validate
let un_processed: HashSet<String> = HashSet::new(); .entry(schema.iri.clone())
// Subjects that are invalid. No further processing needed. .or_insert_with(|| vec![])
let invalids: HashSet<&OrmTrackedSubjectChange> = HashSet::new(); .push(iri.clone());
// Subjects that are valid. Fetch might still be required if newly valid. }
let valids: HashSet<&OrmTrackedSubjectChange> = HashSet::new();
// Will need re-evaluation
let unknowns: HashSet<&OrmTrackedSubjectChange> = HashSet::new();
// Either because it became tracked again or because it's newly valid.
let needs_fetch: HashSet<&OrmTrackedSubjectChange> = HashSet::new();
while !unknowns.is_empty() || !needs_fetch.is_empty() {
// Process buckets by priority and nesting
// First unknown, then needs fetch (the latter could still become invalid).
// Start from from the end because nested objects will less likely need further nested eval.
// Check validity for each modified subject.
for sc in un_processed {
let tracked_subject = tracked_subjects
.get(sc.tracked_subject.subj_iri)
.unwrap()
.get(shape)
.unwrap();
let (is_valid, new_unknowns) =
check_subject_validity(s_change, &shape, schema, tracked_subject.valid);
} }
if !unknowns.is_empty() {
continue; // Now, we add all objects that need re-evaluation to the queue.
for (shape_iri, objects) in nested_objects_to_validate {
shape_validation_queue.push((schema.get(&shape_iri).unwrap(), objects));
} }
for sc in needs_fetch { }
// TODO: fetch and evaluate.
return shape_and_subject_changes;
}
fn create_orm_from_triples(
&mut self,
scope: &NuriV0,
schema: &OrmSchema,
shape: &OrmSchemaShape,
triples: &Vec<Triple>,
) -> Result<Value, NgError> {
let changes = self.apply_changes_from_triples(scope, schema, shape, triples, vec![]);
let root_changes = changes.get(shape.iri).unwrap().values();
let valid_roots = root_changes.filter(|v| v.valid == OrmTrackedSubjectValidity::Valid);
let mut return_vals: Value = Value::Array(vec![]);
for root_change in root_changes {
let new_val = json!({"id": root_change.subject_iri});
for (pred_iri, pred_change) in root_change.predicates {
// Add the readable predicate name
let property_name = pred_change.tracked_predicate.schema.readablePredicate;
// For basic values, add value.
// For arrays of basic values, add array.
// For single nested object, add object.
// For multiple nested objects, create object with iri keys.
} }
} }
// ===
return Ok(return_vals);
//
} }
// Collect result // Collect result
@ -698,6 +728,8 @@ impl Verifier {
} }
} }
/// Heuristic:
/// Consider a string an IRI if it contains alphanumeric characters and then a colon within the first 13 characters
fn is_iri(s: &str) -> bool { fn is_iri(s: &str) -> bool {
lazy_static! { lazy_static! {
static ref IRI_REGEX: Regex = Regex::new(r"^[A-Za-z][A-Za-z0-9+\.\-]{1,12}:").unwrap(); static ref IRI_REGEX: Regex = Regex::new(r"^[A-Za-z][A-Za-z0-9+\.\-]{1,12}:").unwrap();
@ -979,3 +1011,17 @@ fn oxrdf_term_to_orm_basic_type(term: &ng_oxigraph::oxrdf::Term) -> BasicType {
ng_net::orm::Term::Ref(b) => BasicType::Str(b), // Treat IRIs as strings ng_net::orm::Term::Ref(b) => BasicType::Str(b), // Treat IRIs as strings
} }
} }
fn has_cycle(subject: &OrmTrackedSubjectAndShape, visited: &mut HashSet<String>) -> bool {
if visited.contains(subject.subj_iri) {
return true;
}
visited.insert(subject.subj_iri.clone());
for (_parent_iri, (parent_subject, _)) in &subject.parents {
if has_cycle(parent_subject, visited) {
return true;
}
}
visited.remove(subject.subj_iri);
false
}

Write Preview
Loading…
Cancel
Save