diff --git a/Cargo.lock b/Cargo.lock index 524c76a..adf197b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -596,6 +596,19 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "canonical_json" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f89083fd014d71c47a718d7f4ac050864dac8587668dbe90baf9e261064c5710" +dependencies = [ + "hex", + "regex", + "serde", + "serde_json", + "thiserror 1.0.69", +] + [[package]] name = "cast" version = "0.3.0" @@ -2156,6 +2169,7 @@ dependencies = [ "async-std", "async-trait", "base64-url", + "canonical_json", "futures", "lazy_static", "ng-client-ws", @@ -2171,6 +2185,7 @@ dependencies = [ "serde_bare", "serde_bytes", "serde_json", + "serde_json_diff", "svg2pdf", "web-time", "whoami", @@ -3497,6 +3512,16 @@ dependencies = [ "serde_core", ] +[[package]] +name = "serde_json_diff" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eac615f2de9556d78ec9d5924abae441d1764f833fbd6db24bb56d2b6b5200ed" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" diff --git a/nextgraph/Cargo.toml b/nextgraph/Cargo.toml index 1b30f3a..96e0eac 100644 --- a/nextgraph/Cargo.toml +++ b/nextgraph/Cargo.toml @@ -45,6 +45,10 @@ ng-client-ws = { path = "../ng-client-ws", version = "0.1.2" } ng-verifier = { path = "../ng-verifier", version = "0.1.2" } ng-oxigraph = { path = "../ng-oxigraph", version = "0.4.0-alpha.8-ngalpha" } +[dev-dependencies] +serde_json_diff = "0.2.0" +canonical_json = "0.5.0" + [target.'cfg(all(not(target_family = "wasm"),not(docsrs)))'.dependencies] ng-storage-rocksdb = { path = "../ng-storage-rocksdb", version = "0.1.2" } diff --git a/nextgraph/src/tests/orm.rs b/nextgraph/src/tests/orm.rs index 004051a..8881fbd 100644 --- a/nextgraph/src/tests/orm.rs +++ b/nextgraph/src/tests/orm.rs @@ -12,12 +12,14 @@ use crate::tests::create_or_open_wallet::create_or_open_wallet; use async_std::stream::StreamExt; use ng_net::app_protocol::{AppResponse, AppResponseV0, NuriV0}; use ng_net::orm::{ - BasicType, OrmSchema, OrmSchemaDataType, OrmSchemaLiteralType, OrmSchemaPredicate, + self, BasicType, OrmSchema, OrmSchemaDataType, OrmSchemaLiteralType, OrmSchemaPredicate, OrmSchemaShape, OrmShapeType, }; use ng_verifier::orm::utils::shape_type_to_sparql; -use ng_repo::log_info; +use ng_repo::{log_err, log_info}; +use serde_json::json; +use serde_json::Value; use std::collections::HashMap; use std::sync::Arc; @@ -349,7 +351,16 @@ async fn test_orm_creation() { test_orm_multi_type(session_id).await; // === - test_orm_nested(session_id).await; + test_orm_nested_1(session_id).await; + + // // === + // test_orm_nested_2(session_id).await; + + // // === + // test_orm_nested_3(session_id).await; + + // === + test_orm_nested_4(session_id).await; } async fn test_orm_big_object(session_id: u64) { @@ -363,45 +374,50 @@ INSERT DATA { ex:numValue 42 ; ex:boolValue true ; ex:arrayValue 1,2,3 ; - ex:objectValue [ - ex:nestedString "nested" ; - ex:nestedNum 7 ; - ex:nestedArray 5,6 - ] ; - ex:anotherObject [ - ex:prop1 "one" ; - ex:prop2 1 - ], [ - ex:prop1 "two" ; - ex:prop2 2 - ] ; + ex:objectValue ; + ex:anotherObject , ; ex:numOrStr "either" ; ex:lit1Or2 "lit1" ; ex:unrelated "some value" ; ex:anotherUnrelated 4242 . + + ex:nestedString "nested" ; + ex:nestedNum 7 ; + ex:nestedArray 5,6 . + + + ex:prop1 "one" ; + ex:prop2 1 . + + + ex:prop1 "two" ; + ex:prop2 2 . a ex:TestObject ; ex:stringValue "hello world #2" ; ex:numValue 422 ; ex:boolValue false ; ex:arrayValue 4,5,6 ; - ex:objectValue [ - ex:nestedString "nested2" ; - ex:nestedNum 72 ; - ex:nestedArray 7,8,9 - ] ; - ex:anotherObject [ - ex:prop1 "one2" ; - ex:prop2 12 - ], [ - ex:prop1 "two2" ; - ex:prop2 22 - ] ; + ex:objectValue ; + ex:anotherObject , ; ex:numOrStr 4 ; ex:lit1Or2 "lit2" ; ex:unrelated "some value2" ; ex:anotherUnrelated 42422 . + + + ex:nestedString "nested2" ; + ex:nestedNum 72 ; + ex:nestedArray 7,8,9 . + + + ex:prop1 "one2" ; + ex:prop2 12 . + + + ex:prop1 "two2" ; + ex:prop2 22 . } "# .to_string(), @@ -416,24 +432,77 @@ INSERT DATA { shape: "http://example.org/TestObject".to_string(), }; - log_info!("starting orm test"); let nuri = NuriV0::new_from(&doc_nuri).expect("parse nuri"); let (mut receiver, cancel_fn) = orm_start(nuri, shape_type, session_id) .await .expect("orm_start"); - log_info!("orm_start call ended"); - while let Some(app_response) = receiver.next().await { - match app_response { + let orm_json = match app_response { AppResponse::V0(v) => match v { - AppResponseV0::OrmInitial(json) => { - log_info!("ORM JSON arrived\n: {:?}", json); - break; - } - _ => (), + AppResponseV0::OrmInitial(json) => Some(json), + _ => None, }, } + .unwrap(); + + let mut expected = json!([{ + "type":"http://example.org/TestObject", + "id":"urn:test:obj1", + "anotherObject":{ + "urn:test:id1":{ + "prop1":"one", + "prop2":1.0 + }, + "urn:test:id2":{ + "prop1":"two", + "prop2":2.0 + } + }, + "arrayValue":[3.0,2.0,1.0], + "boolValue":true, + "lit1Or2":"lit1", + "numOrStr":"either", + "numValue":42.0, + "objectValue":{ + "id":"urn:test:id3", + "nestedArray":[5.0,6.0], + "nestedNum":7.0, + "nestedString":"nested" + }, + "stringValue": "hello world", + }, + { + "id":"urn:test:obj2", + "type":"http://example.org/TestObject", + "anotherObject": { + "urn:test:id4":{ + "prop1":"one2", + "prop2":12.0 + }, + "urn:test:id5":{ + "prop1":"two2", + "prop2":22.0 + } + }, + "arrayValue":[6.0,5.0,4.0], + "boolValue":false, + "lit1Or2":"lit2", + "numOrStr":4.0, + "numValue":422.0, + "objectValue":{ + "id":"urn:test:id6", + "nestedArray": [7.0,8.0,9.0], + "nestedNum":72.0, + "nestedString":"nested2" + }, + "stringValue":"hello world #2", + }]); + + let mut actual_mut = orm_json.clone(); + assert_json_eq(&mut expected, &mut actual_mut); + + break; } cancel_fn(); } @@ -531,7 +600,26 @@ INSERT DATA { } .unwrap(); - log_info!("ORM JSON arrived\n: {:?}", orm_json); + let mut expected = json!([ + { + "id": "urn:test:numArrayObj1", + "type": "http://example.org/TestObject", + "numArray": [1.0, 2.0, 3.0] + }, + { + "id": "urn:test:numArrayObj2", + "type": "http://example.org/TestObject", + "numArray": [] + }, + { + "id": "urn:test:numArrayObj3", + "type": "http://example.org/TestObject", + "numArray": [1.0, 2.0] + } + ]); + + let mut actual_mut = orm_json.clone(); + assert_json_eq(&mut expected, &mut actual_mut); break; } @@ -548,6 +636,7 @@ INSERT DATA { ex:opt true ; ex:str "s1" . + # Contains no matching data ex:str "s2" . } @@ -598,7 +687,15 @@ INSERT DATA { } .unwrap(); - log_info!("ORM JSON arrived for optional test\n: {:?}", orm_json); + let mut expected = json!([ + { + "id": "urn:test:oj1", + "opt": true + } + ]); + + let mut actual_mut = orm_json.clone(); + assert_json_eq(&mut expected, &mut actual_mut); break; } @@ -687,7 +784,21 @@ INSERT DATA { } .unwrap(); - log_info!("ORM JSON arrived for literal test\n: {:?}", orm_json); + let mut expected = json!([ + { + "id": "urn:test:oj1", + "lit1": ["lit 1"], + "lit2": "lit 2" + }, + { + "id": "urn:test:obj2", + "lit1": ["lit 1", "lit 1 extra"], + "lit2": "lit 2" + } + ]); + + let mut actual_mut = orm_json.clone(); + assert_json_eq(&mut expected, &mut actual_mut); break; } @@ -765,14 +876,22 @@ INSERT DATA { } .unwrap(); - log_info!("ORM JSON arrived for multi type test\n: {:?}", orm_json); + let mut expected = json!([ + { + "id": "urn:test:oj1", + "strOrNum": ["a string", "another string", 2.0] + } + ]); + + let mut actual_mut = orm_json.clone(); + assert_json_eq(&mut expected, &mut actual_mut); break; } cancel_fn(); } -async fn test_orm_nested(session_id: u64) { +async fn test_orm_nested_1(session_id: u64) { let doc_nuri = create_doc_with_data( session_id, r#" @@ -781,33 +900,37 @@ INSERT DATA { # Valid ex:str "obj1 str" ; - ex:nestedWithExtra [ - ex:nestedStr "obj1 nested with extra valid" ; - ex:nestedNum 2 - ] , [ - # Invalid, nestedNum is missing but okay because extra. - ex:nestedStr "obj1 nested with extra invalid" - ] ; - ex:nestedWithoutExtra [ - ex:nestedStr "obj1 nested without extra valid" ; - ex:nestedNum 2 - ] . + ex:nestedWithExtra , ; + ex:nestedWithoutExtra . + + + ex:nestedStr "obj1 nested with extra valid" ; + ex:nestedNum 2 . + + + ex:nestedStr "obj1 nested with extra invalid" . + + + ex:nestedStr "obj1 nested without extra valid" ; + ex:nestedNum 2 . # Invalid because nestedWithoutExtra has an invalid child. ex:str "obj2 str" ; - ex:nestedWithExtra [ - ex:nestedStr "obj2: a nested string valid" ; - ex:nestedNum 2 - ] ; - ex:nestedWithoutExtra [ - ex:nestedStr "obj2 nested without extra valid" ; - ex:nestedNum 2 - ] , - # Invalid because nestedNum is missing. - [ - ex:nestedStr "obj2 nested without extra invalid" - ] . + ex:nestedWithExtra ; + ex:nestedWithoutExtra , . + + + ex:nestedStr "obj2: a nested string valid" ; + ex:nestedNum 2 . + + + ex:nestedStr "obj2 nested without extra valid" ; + ex:nestedNum 2 . + + # Invalid because nestedNum is missing. + + ex:nestedStr "obj2 nested without extra invalid" . } "# .to_string(), @@ -954,7 +1077,483 @@ INSERT DATA { } .unwrap(); - log_info!("ORM JSON arrived for nested test\n: {:?}", orm_json); + let mut expected = json!([ + { + "id": "urn:test:oj1", + "str": "obj1 str", + "nestedWithExtra": { + "nestedStr": "obj1 nested with extra valid", + "nestedNum": 2.0 + }, + "nestedWithoutExtra": { + "nestedStr": "obj1 nested without extra valid", + "nestedNum": 2.0 + } + } + ]); + + let mut actual_mut = orm_json.clone(); + assert_json_eq(&mut expected, &mut actual_mut); + + break; + } + cancel_fn(); +} + +async fn test_orm_nested_2(session_id: u64) { + let doc_nuri = create_doc_with_data( + session_id, + r#" +PREFIX ex: +INSERT DATA { + # Valid + + ex:knows , ; + ex:name "Alice" . + + ex:knows ; + ex:name "Bob" . + + ex:name "Claire" . + + # Invalid because claire2 is invalid + + ex:knows , ; + ex:name "Alice" . + # Invalid because claire2 is invalid + + ex:knows ; + ex:name "Bob" . + # Invalid because name is missing. + + ex:missingName "Claire missing" . +} +"# + .to_string(), + ) + .await; + + // Define the ORM schema + let mut schema = HashMap::new(); + schema.insert( + "http://example.org/PersonShape".to_string(), + OrmSchemaShape { + iri: "http://example.org/PersonShape".to_string(), + predicates: vec![ + OrmSchemaPredicate { + iri: "http://example.org/name".to_string(), + extra: None, + maxCardinality: 1, + minCardinality: 1, + readablePredicate: "name".to_string(), + dataTypes: vec![OrmSchemaDataType { + valType: OrmSchemaLiteralType::string, + literals: None, + shape: None, + }], + } + .into(), + OrmSchemaPredicate { + iri: "http://example.org/knows".to_string(), + extra: Some(false), + maxCardinality: -1, + minCardinality: 0, + readablePredicate: "knows".to_string(), + dataTypes: vec![OrmSchemaDataType { + valType: OrmSchemaLiteralType::shape, + literals: None, + shape: Some("http://example.org/PersonShape".to_string()), + }], + } + .into(), + ], + } + .into(), + ); + + let shape_type = OrmShapeType { + schema, + shape: "http://example.org/PersonShape".to_string(), + }; + + let nuri = NuriV0::new_from(&doc_nuri).expect("parse nuri"); + let (mut receiver, cancel_fn) = orm_start(nuri, shape_type, session_id) + .await + .expect("orm_start"); + + while let Some(app_response) = receiver.next().await { + let orm_json = match app_response { + AppResponse::V0(v) => match v { + AppResponseV0::OrmInitial(json) => Some(json), + _ => None, + }, + } + .unwrap(); + + log_info!( + "ORM JSON arrived for nested2 (person) test\n: {:?}", + orm_json + ); + + // Expected: alice and bob with their nested knows relationships + // claire2 is invalid (missing name), so alice2's knows chain is incomplete + let mut expected = json!([ + { + "id": "urn:test:alice", + "name": "Alice", + "knows": { + "urn:test:bob": { + "name": "Bob", + "knows": { + "urn:test:claire": { + "name": "Claire", + "knows": {} + } + } + }, + "urn:test:claire": { + "name": "Claire", + "knows": {} + } + } + }, + { + "id": "urn:test:bob", + "name": "Bob", + "knows": { + "urn:test:claire": { + "name": "Claire", + "knows": {} + } + } + }, + { + "id": "urn:test:claire", + "name": "Claire", + "knows": {} + } + ]); + + let mut actual_mut = orm_json.clone(); + log_info!( + "JSON for nested2\n{}", + serde_json::to_string(&actual_mut).unwrap() + ); + assert_json_eq(&mut expected, &mut actual_mut); + + break; + } + cancel_fn(); +} + +async fn test_orm_nested_3(session_id: u64) { + let doc_nuri = create_doc_with_data( + session_id, + r#" +PREFIX ex: +INSERT DATA { + # Valid + + a ex:Alice ; + ex:knows , . + + a ex:Bob ; + ex:knows . + + a ex:Claire . + + # Invalid because claire is invalid + + a ex:Alice ; + ex:knows , . + # Invalid because claire is invalid + + a ex:Bob ; + ex:knows . + # Invalid, wrong type. + + a ex:Claire2 . +} +"# + .to_string(), + ) + .await; + + // Define the ORM schema + let mut schema = HashMap::new(); + schema.insert( + "http://example.org/AliceShape".to_string(), + OrmSchemaShape { + iri: "http://example.org/AliceShape".to_string(), + predicates: vec![ + OrmSchemaPredicate { + iri: "http://www.w3.org/1999/02/22-rdf-syntax-ns#type".to_string(), + extra: None, + maxCardinality: 1, + minCardinality: 1, + readablePredicate: "type".to_string(), + dataTypes: vec![OrmSchemaDataType { + valType: OrmSchemaLiteralType::literal, + literals: Some(vec![BasicType::Str( + "http://example.org/Alice".to_string(), + )]), + shape: None, + }], + } + .into(), + OrmSchemaPredicate { + iri: "http://example.org/knows".to_string(), + extra: Some(false), + maxCardinality: -1, + minCardinality: 0, + readablePredicate: "knows".to_string(), + dataTypes: vec![ + OrmSchemaDataType { + valType: OrmSchemaLiteralType::shape, + literals: None, + shape: Some("http://example.org/BobShape".to_string()), + }, + OrmSchemaDataType { + valType: OrmSchemaLiteralType::shape, + literals: None, + shape: Some("http://example.org/ClaireShape".to_string()), + }, + ], + } + .into(), + ], + } + .into(), + ); + schema.insert( + "http://example.org/BobShape".to_string(), + OrmSchemaShape { + iri: "http://example.org/BobShape".to_string(), + predicates: vec![ + OrmSchemaPredicate { + iri: "http://www.w3.org/1999/02/22-rdf-syntax-ns#type".to_string(), + extra: Some(true), + maxCardinality: 1, + minCardinality: 1, + readablePredicate: "type".to_string(), + dataTypes: vec![OrmSchemaDataType { + valType: OrmSchemaLiteralType::literal, + literals: Some(vec![BasicType::Str("http://example.org/Bob".to_string())]), + shape: None, + }], + } + .into(), + OrmSchemaPredicate { + iri: "http://example.org/knows".to_string(), + extra: Some(false), + maxCardinality: -1, + minCardinality: 0, + readablePredicate: "knows".to_string(), + dataTypes: vec![OrmSchemaDataType { + valType: OrmSchemaLiteralType::shape, + literals: None, + shape: Some("http://example.org/ClaireShape".to_string()), + }], + } + .into(), + ], + } + .into(), + ); + schema.insert( + "http://example.org/ClaireShape".to_string(), + OrmSchemaShape { + iri: "http://example.org/ClaireShape".to_string(), + predicates: vec![OrmSchemaPredicate { + iri: "http://www.w3.org/1999/02/22-rdf-syntax-ns#type".to_string(), + extra: None, + maxCardinality: 1, + minCardinality: 1, + readablePredicate: "type".to_string(), + dataTypes: vec![OrmSchemaDataType { + valType: OrmSchemaLiteralType::literal, + literals: Some(vec![BasicType::Str( + "http://example.org/Claire".to_string(), + )]), + shape: None, + }], + } + .into()], + } + .into(), + ); + + let shape_type = OrmShapeType { + schema, + shape: "http://example.org/AliceShape".to_string(), + }; + + let nuri = NuriV0::new_from(&doc_nuri).expect("parse nuri"); + let (mut receiver, cancel_fn) = orm_start(nuri, shape_type, session_id) + .await + .expect("orm_start"); + + while let Some(app_response) = receiver.next().await { + let orm_json = match app_response { + AppResponse::V0(v) => match v { + AppResponseV0::OrmInitial(json) => Some(json), + _ => None, + }, + } + .unwrap(); + + log_info!( + "ORM JSON arrived for nested3 (person) test\n: {:?}", + serde_json::to_string(&orm_json).unwrap() + ); + + // Expected: alice with knows relationships to bob and claire + // alice2 is incomplete because claire2 has wrong type + let mut expected = json!([ + { + "id": "urn:test:alice", + "type": "http://example.org/Alice", + "knows": { + "urn:test:bob": { + "type": "http://example.org/Bob", + "knows": { + "urn:test:claire": { + "type": "http://example.org/Claire" + } + } + }, + "urn:test:claire": { + "type": "http://example.org/Claire" + } + } + } + ]); + + let mut actual_mut = orm_json.clone(); + assert_json_eq(&mut expected, &mut actual_mut); + + break; + } + cancel_fn(); +} + +async fn test_orm_nested_4(session_id: u64) { + let doc_nuri = create_doc_with_data( + session_id, + r#" +PREFIX ex: +INSERT DATA { + # Valid + + a ex:Person ; + ex:hasCat , . + + a ex:Cat . + + a ex:Cat . +} +"# + .to_string(), + ) + .await; + + // Define the ORM schema + let mut schema = HashMap::new(); + schema.insert( + "http://example.org/PersonShape".to_string(), + OrmSchemaShape { + iri: "http://example.org/PersonShape".to_string(), + predicates: vec![ + OrmSchemaPredicate { + iri: "http://www.w3.org/1999/02/22-rdf-syntax-ns#type".to_string(), + extra: None, + maxCardinality: 1, + minCardinality: 1, + readablePredicate: "type".to_string(), + dataTypes: vec![OrmSchemaDataType { + valType: OrmSchemaLiteralType::literal, + literals: Some(vec![BasicType::Str( + "http://example.org/Person".to_string(), + )]), + shape: None, + }], + } + .into(), + OrmSchemaPredicate { + iri: "http://example.org/hasCat".to_string(), + extra: Some(false), + maxCardinality: -1, + minCardinality: 0, + readablePredicate: "cats".to_string(), + dataTypes: vec![OrmSchemaDataType { + valType: OrmSchemaLiteralType::shape, + literals: None, + shape: Some("http://example.org/CatShape".to_string()), + }], + } + .into(), + ], + } + .into(), + ); + schema.insert( + "http://example.org/CatShape".to_string(), + OrmSchemaShape { + iri: "http://example.org/CatShape".to_string(), + predicates: vec![OrmSchemaPredicate { + iri: "http://www.w3.org/1999/02/22-rdf-syntax-ns#type".to_string(), + extra: Some(true), + maxCardinality: 1, + minCardinality: 1, + readablePredicate: "type".to_string(), + dataTypes: vec![OrmSchemaDataType { + valType: OrmSchemaLiteralType::literal, + literals: Some(vec![BasicType::Str("http://example.org/Cat".to_string())]), + shape: None, + }], + } + .into()], + } + .into(), + ); + + let shape_type = OrmShapeType { + schema, + shape: "http://example.org/PersonShape".to_string(), + }; + + let nuri = NuriV0::new_from(&doc_nuri).expect("parse nuri"); + let (mut receiver, cancel_fn) = orm_start(nuri, shape_type, session_id) + .await + .expect("orm_start"); + + while let Some(app_response) = receiver.next().await { + let orm_json = match app_response { + AppResponse::V0(v) => match v { + AppResponseV0::OrmInitial(json) => Some(json), + _ => None, + }, + } + .unwrap(); + + let mut expected = json!([ + { + "id": "urn:test:alice", + "type": "http://example.org/Person", + "cats": { + "urn:test:kitten1": { + "type": "http://example.org/Cat" + }, + "urn:test:kitten2": { + "type": "http://example.org/Cat" + } + }, + } + ]); + + let mut actual_mut = orm_json.clone(); + + assert_json_eq(&mut expected, &mut actual_mut); break; } @@ -1209,3 +1808,67 @@ async fn create_doc_with_data(session_id: u64, sparql_insert: String) -> String return doc_nuri; } + +fn assert_json_eq(expected: &mut Value, actual: &mut Value) { + remove_id_fields(expected); + remove_id_fields(actual); + + sort_arrays(expected); + sort_arrays(actual); + + let diff = serde_json_diff::values(expected.clone(), actual.clone()); + if let Some(diff_) = diff { + log_err!("Expected and actual ORM JSON mismatch.\nDiff: {:?}", diff_); + assert!(false); + } +} + +/// Helper to recursively sort all arrays in nested objects into a stable ordering. +/// Arrays are sorted by their JSON string representation. +fn sort_arrays(value: &mut Value) { + match value { + Value::Object(map) => { + for v in map.values_mut() { + sort_arrays(v); + } + } + Value::Array(arr) => { + // First, recursively sort nested structures + for v in arr.iter_mut() { + sort_arrays(v); + } + // Then sort the array itself by JSON string representation + arr.sort_by(|a, b| { + let a_str = canonical_json::ser::to_string(a).unwrap_or_default(); + let b_str = canonical_json::ser::to_string(b).unwrap_or_default(); + a_str.cmp(&b_str) + }); + } + _ => {} + } +} + +/// Helper to recursively remove nested "id" fields from nested objects, +/// but only if they are not at the root level. +fn remove_id_fields(value: &mut Value) { + fn remove_id_fields_inner(value: &mut Value, is_root: bool) { + match value { + Value::Object(map) => { + if !is_root { + map.remove("id"); + } + for v in map.values_mut() { + remove_id_fields_inner(v, false); + } + } + Value::Array(arr) => { + for v in arr { + remove_id_fields_inner(v, false); + } + } + _ => {} + } + } + + remove_id_fields_inner(value, true); +} diff --git a/ng-verifier/src/orm/add_remove_triples.rs b/ng-verifier/src/orm/add_remove_triples.rs index cfc7c94..a30c8b1 100644 --- a/ng-verifier/src/orm/add_remove_triples.rs +++ b/ng-verifier/src/orm/add_remove_triples.rs @@ -57,15 +57,18 @@ pub fn add_remove_triples( // Process added triples. // For each triple, check if it matches the shape. // In parallel, we record the values added and removed (tracked_changes) - log_debug!("Processing # triples: {}", triples_added.len()); for triple in triples_added { let obj_term = oxrdf_term_to_orm_basic_type(&triple.object); - log_debug!("processing triple {triple}"); + log_debug!(" - processing triple {triple}"); for predicate_schema in &shape.predicates { if predicate_schema.iri != triple.predicate.as_str() { // Triple does not match predicate. continue; } + log_debug!( + " - Matched triple for datatypes {:?}", + predicate_schema.dataTypes + ); // Predicate schema constraint matches this triple. let tracked_subject_lock = get_or_create_tracked_subject(subject_iri, &shape, tracked_subjects); @@ -124,7 +127,7 @@ pub fn add_remove_triples( None } }) { - // log_debug!("dealing with nesting for {shape_iri}"); + log_debug!(" - dealing with nested type {shape_iri}"); if let BasicType::Str(obj_iri) = &obj_term { let tracked_child_arc = { // Get or create object's tracked subject struct. diff --git a/ng-verifier/src/orm/mod.rs b/ng-verifier/src/orm/mod.rs index 8a835d6..336b998 100644 --- a/ng-verifier/src/orm/mod.rs +++ b/ng-verifier/src/orm/mod.rs @@ -75,6 +75,7 @@ impl Verifier { return Err(NgError::SparqlError(e.to_string())); } Ok(triple) => { + log_debug!("Triple fetched: {:?}", triple); result_triples.push(triple); } } @@ -173,9 +174,13 @@ impl Verifier { HashMap::new(); // For each subject, add/remove triples and validate. - log_debug!("all_modified_subjects: {:?}", modified_subject_iris); + log_debug!( + "processing modified subjects: {:?} against shape: {}", + modified_subject_iris, + shape.iri + ); - for subject_iri in modified_subject_iris { + for subject_iri in &modified_subject_iris { let validation_key = (shape.iri.clone(), subject_iri.to_string()); // Cycle detection: Check if this (shape, subject) pair is already being validated @@ -187,7 +192,8 @@ impl Verifier { ); // Mark as invalid due to cycle // TODO: We could handle this by handling nested references as IRIs. - if let Some(tracked_shapes) = orm_subscription.tracked_subjects.get(subject_iri) + if let Some(tracked_shapes) = + orm_subscription.tracked_subjects.get(*subject_iri) { if let Some(tracked_subject) = tracked_shapes.get(&shape.iri) { let mut ts = tracked_subject.write().unwrap(); @@ -201,12 +207,13 @@ impl Verifier { // Mark as currently validating currently_validating.insert(validation_key.clone()); + // Get triples of subject (added & removed). let triples_added_for_subj = added_triples_by_subject - .get(subject_iri) + .get(*subject_iri) .map(|v| v.as_slice()) .unwrap_or(&[]); let triples_removed_for_subj = removed_triples_by_subject - .get(subject_iri) + .get(*subject_iri) .map(|v| v.as_slice()) .unwrap_or(&[]); @@ -214,9 +221,9 @@ impl Verifier { let change = orm_changes .entry(shape.iri.clone()) .or_insert_with(HashMap::new) - .entry(subject_iri.clone()) + .entry((*subject_iri).clone()) .or_insert_with(|| OrmTrackedSubjectChange { - subject_iri: subject_iri.clone(), + subject_iri: (*subject_iri).clone(), predicates: HashMap::new(), data_applied: false, }); @@ -248,7 +255,7 @@ impl Verifier { let validity = { let tracked_subject_opt = orm_subscription .tracked_subjects - .get(subject_iri) + .get(*subject_iri) .and_then(|m| m.get(&shape.iri)); let Some(tracked_subject) = tracked_subject_opt else { continue; @@ -277,14 +284,8 @@ impl Verifier { } } } - - // Remove from validation stack after processing this subject - currently_validating.remove(&validation_key); } - // TODO: Currently, all shape <-> nested subject combinations are queued for re-evaluation. - // Is that okay? - // Now, we queue all non-evaluated objects for (shape_iri, objects_to_eval) in &nested_objects_to_eval { let orm_subscription = self.get_first_orm_subscription_for( @@ -333,6 +334,10 @@ impl Verifier { shape_validation_stack.push((shape_arc, objects_not_to_fetch)); } } + for subject_iri in modified_subject_iris { + let validation_key = (shape.iri.clone(), subject_iri.to_string()); + currently_validating.remove(&validation_key); + } } Ok(()) diff --git a/ng-verifier/src/orm/utils.rs b/ng-verifier/src/orm/utils.rs index 3a1662f..8df48c2 100644 --- a/ng-verifier/src/orm/utils.rs +++ b/ng-verifier/src/orm/utils.rs @@ -80,6 +80,7 @@ pub fn shape_type_to_sparql( let mut visited_shapes: HashSet = HashSet::new(); // Recursive function to call for (nested) shapes. + // Returns nested WHERE statements that should be included with this shape's binding. fn process_shape( schema: &OrmSchema, shape: &OrmSchemaShape, @@ -89,11 +90,11 @@ pub fn shape_type_to_sparql( var_counter: &mut i32, visited_shapes: &mut HashSet, in_recursion: bool, - ) { + ) -> Vec { // Prevent infinite recursion on cyclic schemas. // TODO: We could handle this as IRI string reference. if visited_shapes.contains(&shape.iri) { - return; + return vec![]; } let mut new_where_statements: Vec = vec![]; @@ -102,8 +103,12 @@ pub fn shape_type_to_sparql( visited_shapes.insert(shape.iri.clone()); // Add statements for each predicate. + // If we are in recursion, we want to get all triples. + // That's why we add a " ?p ?o" statement afterwards + // and the extra construct statements are skipped. for predicate in &shape.predicates { let mut union_branches = Vec::new(); + let mut nested_where_statements = Vec::new(); // Predicate constraints might have more than one acceptable nested shape. Traverse each. for datatype in &predicate.dataTypes { @@ -116,10 +121,12 @@ pub fn shape_type_to_sparql( // Each shape option gets its own var. let obj_var_name = get_new_var_name(var_counter); - new_construct_statements.push(format!( - " ?{} <{}> ?{}", - subject_var_name, predicate.iri, obj_var_name - )); + if !in_recursion { + new_construct_statements.push(format!( + " ?{} <{}> ?{}", + subject_var_name, predicate.iri, obj_var_name + )); + } // Those are later added to a UNION, if there is more than one shape. union_branches.push(format!( " ?{} <{}> ?{}", @@ -127,7 +134,8 @@ pub fn shape_type_to_sparql( )); // Recurse to add statements for nested object. - process_shape( + // Collect nested WHERE statements to include within this predicate's scope. + let nested_stmts = process_shape( schema, nested_shape, &obj_var_name, @@ -137,6 +145,7 @@ pub fn shape_type_to_sparql( visited_shapes, true, ); + nested_where_statements.extend(nested_stmts); } } @@ -145,21 +154,32 @@ pub fn shape_type_to_sparql( if !union_branches.is_empty() { // We have nested shape(s) which were already added to CONSTRUCT above. - // Join them with UNION. + // Join them with UNION and include nested WHERE statements. - where_body = union_branches + let union_body = union_branches .into_iter() .map(|b| format!("{{\n{}\n}}", b)) .collect::>() .join(" UNION "); + + // Combine the parent binding with nested statements + if !nested_where_statements.is_empty() { + let nested_joined = nested_where_statements.join(" .\n"); + where_body = format!("{} .\n{}", union_body, nested_joined); + } else { + where_body = union_body; + } } else { // Regular predicate data type. Just add basic CONSTRUCT and WHERE statements. let obj_var_name = get_new_var_name(var_counter); - new_construct_statements.push(format!( - " ?{} <{}> ?{}", - subject_var_name, predicate.iri, obj_var_name - )); + if !in_recursion { + // Only add construct, if we don't have catch-all statement already. + new_construct_statements.push(format!( + " ?{} <{}> ?{}", + subject_var_name, predicate.iri, obj_var_name + )); + } where_body = format!( " ?{} <{}> ?{}", subject_var_name, predicate.iri, obj_var_name @@ -181,7 +201,7 @@ pub fn shape_type_to_sparql( let pred_var_name = get_new_var_name(var_counter); let obj_var_name = get_new_var_name(var_counter); - // The "catch any triple in subject" where statement + // The "catch any triple in subject" construct statement construct_statements.push(format!( " ?{} ?{} ?{}", subject_var_name, pred_var_name, obj_var_name @@ -189,17 +209,20 @@ pub fn shape_type_to_sparql( let joined_where_statements = new_where_statements.join(" .\n"); - // We do a join of the where statements (which will take care of querying further nested objects) - // and the "catch any triple in subject" where statement. - where_statements.push(format!( - " {{?{} ?{} ?{}}}\n UNION {{\n {}\n }}", + // Return nested statements to be included in parent's scope + // Combine catch-all with specific predicates in a UNION + let nested_block = format!( + " {{\n {{?{} ?{} ?{}}}\n UNION {{\n {}\n }}\n }}", subject_var_name, pred_var_name, obj_var_name, joined_where_statements - )); + ); + visited_shapes.remove(&shape.iri); + return vec![nested_block]; } else { where_statements.append(&mut new_where_statements); construct_statements.append(&mut new_construct_statements); } visited_shapes.remove(&shape.iri); + vec![] } let root_shape = schema.get(shape).ok_or(VerifierError::InvalidOrmSchema)?; @@ -239,7 +262,6 @@ pub fn shape_type_to_sparql( construct_body, where_body )) } - /// SPARQL literal escape: backslash, quotes, newlines, tabs. fn escape_literal(lit: &str) -> String { let mut out = String::with_capacity(lit.len() + 4); diff --git a/ng-verifier/src/orm/validation.rs b/ng-verifier/src/orm/validation.rs index be44f18..94754fd 100644 --- a/ng-verifier/src/orm/validation.rs +++ b/ng-verifier/src/orm/validation.rs @@ -36,6 +36,12 @@ impl Verifier { // Keep track of objects that need to be validated against a shape to fetch and validate. let mut need_evaluation: Vec<(String, String, bool)> = vec![]; + log_debug!( + "[Validation] for shape {} and subject {}", + shape.iri, + s_change.subject_iri + ); + // Check 1) Check if this object is untracked and we need to remove children and ourselves. if previous_validity == OrmTrackedSubjectValidity::Untracked { // 1.1) Schedule children for deletion @@ -106,7 +112,7 @@ impl Verifier { // Check 3.1) Cardinality if count < p_schema.minCardinality { log_debug!( - "[VALIDATION] Invalid: minCardinality not met | predicate: {:?} | count: {} | min: {} | schema: {:?} | changed: {:?}", + " - Invalid: minCardinality not met | predicate: {:?} | count: {} | min: {} | schema: {:?} | changed: {:?}", p_schema.iri, count, p_schema.minCardinality, @@ -125,7 +131,7 @@ impl Verifier { && p_schema.extra != Some(true) { log_debug!( - "[VALIDATION] Invalid: maxCardinality exceeded | predicate: {:?} | count: {} | max: {} | schema: {:?} | changed: {:?}", + " - Invalid: maxCardinality exceeded | predicate: {:?} | count: {} | max: {} | schema: {:?} | changed: {:?}", p_schema.iri, count, p_schema.maxCardinality, @@ -171,12 +177,13 @@ impl Verifier { ); if !some_valid { log_debug!( - "[VALIDATION] Invalid: required literals missing | predicate: {:?} | schema: {:?} | changed: {:?}", + " - Invalid: required literals missing | predicate: {:?} | schema: {:?} | changed: {:?}", p_schema.iri, shape.iri, p_change ); set_validity(&mut new_validity, OrmTrackedSubjectValidity::Invalid); + break; } // Check 3.4) Nested shape correct. } else if p_schema @@ -191,6 +198,7 @@ impl Verifier { .map(|tc| tc.read().unwrap()) .collect::>() }); + // First, Count valid, invalid, unknowns, and untracked let counts = tracked_children.as_ref().map_or((0, 0, 0, 0), |children| { children @@ -213,9 +221,11 @@ impl Verifier { }) }); + log_debug!(" - checking nested - Counts: {:?}", counts); + if counts.1 > 0 && p_schema.extra != Some(true) { log_debug!( - "[VALIDATION] Invalid: nested invalid child | predicate: {:?} | schema: {:?} | changed: {:?}", + " - Invalid: nested invalid child | predicate: {:?} | schema: {:?} | changed: {:?}", p_schema.iri, shape.iri, p_change @@ -226,7 +236,7 @@ impl Verifier { break; } else if counts.0 > p_schema.maxCardinality && p_schema.maxCardinality != -1 { log_debug!( - "[VALIDATION] Too many valid children: | predicate: {:?} | schema: {:?} | changed: {:?}", + " - Invalid: Too many valid children: | predicate: {:?} | schema: {:?} | changed: {:?}", p_schema.iri, shape.iri, p_change @@ -236,7 +246,7 @@ impl Verifier { break; } else if counts.0 + counts.2 + counts.3 < p_schema.minCardinality { log_debug!( - "[VALIDATION] Invalid: not enough nested children | predicate: {:?} | valid_count: {} | min: {} | schema: {:?} | changed: {:?}", + " - Invalid: not enough nested children | predicate: {:?} | valid_count: {} | min: {} | schema: {:?} | changed: {:?}", p_schema.iri, counts.0, p_schema.minCardinality, @@ -269,7 +279,7 @@ impl Verifier { } }); } else if counts.2 > 0 { - // If we have pending nested objects, we need to wait for their evaluation. + // If we have pending children, we need to wait for their evaluation. set_validity(&mut new_validity, OrmTrackedSubjectValidity::Pending); // Schedule pending children for re-evaluation without fetch. tracked_children.as_ref().map(|children| { @@ -307,7 +317,7 @@ impl Verifier { }; if !matches { log_debug!( - "[VALIDATION] Invalid: value type mismatch | predicate: {:?} | value: {:?} | allowed_types: {:?} | schema: {:?} | changed: {:?}", + " - Invalid: value type mismatch | predicate: {:?} | value: {:?} | allowed_types: {:?} | schema: {:?} | changed: {:?}", p_schema.iri, val_added, allowed_types, diff --git a/sdk/ng-sdk-js/ng-signals/src/connector/createSignalObjectForShape.ts b/sdk/ng-sdk-js/ng-signals/src/connector/createSignalObjectForShape.ts index 964acf1..761590b 100644 --- a/sdk/ng-sdk-js/ng-signals/src/connector/createSignalObjectForShape.ts +++ b/sdk/ng-sdk-js/ng-signals/src/connector/createSignalObjectForShape.ts @@ -1,6 +1,8 @@ import type { Diff, Scope } from "../types.ts"; import { applyDiff } from "./applyDiff.ts"; +import ng from "@nextgraph-monorepo/ng-sdk-js"; + import { deepSignal, watch, @@ -111,7 +113,7 @@ const setUpConnection = (entry: PoolEntry, wasmMessage: WasmMessage) => { }; // Handler for messages from wasm land. -const onWasmMessage = (event: MessageEvent) => { +const onMessage = (event: MessageEvent) => { console.debug("[JsLand] onWasmMessage", event); const { diff, connectionId, type } = event.data; @@ -121,7 +123,12 @@ const onWasmMessage = (event: MessageEvent) => { // And only process messages that are addressed to js-land. if (type === "FrontendUpdate") return; - if (type === "Request") return; + if (type === "Request") { + // TODO: Handle message from wasm land and js land + // in different functions + + return; + } if (type === "Stop") return; if (type === "InitialResponse") { @@ -137,7 +144,7 @@ const keyToEntry = new Map>(); const connectionIdToEntry = new Map>(); const communicationChannel = new BroadcastChannel("shape-manager"); -communicationChannel.addEventListener("message", onWasmMessage); +communicationChannel.addEventListener("message", onMessage); // FinalizationRegistry to clean up connections when signal objects are GC'd. const cleanupSignalRegistry = @@ -210,16 +217,11 @@ export function createSignalObjectForShape( keyToEntry.set(key, entry); connectionIdToEntry.set(entry.connectionId, entry); - // TODO: Just a hack since the channel is not set up in mock-mode - setTimeout( - () => - communicationChannel.postMessage({ - type: "Request", - connectionId: entry.connectionId, - shapeType, - } as WasmMessage), - 100 - ); + communicationChannel.postMessage({ + type: "Request", + connectionId: entry.connectionId, + shapeType, + } as WasmMessage); function buildReturn(entry: PoolEntry) { return {