diff --git a/packages/cli/src/build.ts b/packages/cli/src/build.ts index 9986bae..e1ae9c7 100644 --- a/packages/cli/src/build.ts +++ b/packages/cli/src/build.ts @@ -1,6 +1,6 @@ import fs from "fs-extra"; import path from "path"; -import type { Schema } from "shexj"; +import type { Schema } from "@ldo/traverser-shexj"; import parser from "@shexjs/parser"; import schemaConverterShex from "@ldo/schema-converter-shex"; import { renderFile } from "ejs"; @@ -30,11 +30,22 @@ export async function build(options: BuildOptions) { } await fs.promises.mkdir(options.output); + const format = options.format || "ldo"; + const fileTemplates: string[] = []; + + if (format === "compact") { + // Pre-annotate schema with readablePredicate to unify naming across outputs + fileTemplates.push("schema.compact", "typings", "shapeTypes.compact"); + } else { + fileTemplates.push("schema", "typings", "shapeTypes", "context"); + } + load.text = "Generating LDO Documents"; await forAllShapes(options.input, async (fileName, shexC) => { // Convert to ShexJ let schema: Schema; try { + // @ts-expect-error ... schema = parser.construct("https://ldo.js.org/").parse(shexC); } catch (err) { const errMessage = @@ -46,29 +57,30 @@ export async function build(options: BuildOptions) { console.error(`Error processing ${fileName}: ${errMessage}`); return; } - // Pre-annotate schema with readablePredicate to unify naming across outputs - annotateReadablePredicates(schema); - // Convert the content to types - const format = options.format || "ldo"; - const [typings, context] = await schemaConverterShex(schema, { format }); - const templates: string[] = ["schema", "typings", "shapeTypes"]; - if (format === "ldo") { - templates.unshift("context"); + // Add readable predicates to schema as the single source of truth. + if (format === "compact") { + // @ts-expect-error ... + annotateReadablePredicates(schema); } + + const [typings, context, compactSchema] = await schemaConverterShex( + schema, + { + format, + }, + ); + await Promise.all( - templates.map(async (templateName) => { - const templateFile = - templateName === "shapeTypes" && format === "compact" - ? "shapeTypes.compact" - : templateName; + fileTemplates.map(async (templateName) => { const finalContent = await renderFile( - path.join(__dirname, "./templates", `${templateFile}.ejs`), + path.join(__dirname, "./templates", `${templateName}.ejs`), { typings: typings.typings, fileName, schema: JSON.stringify(schema, null, 2), context: JSON.stringify(context, null, 2), + compactSchema: JSON.stringify(compactSchema, null, 2), format, }, ); diff --git a/packages/cli/src/templates/schema.compact.ejs b/packages/cli/src/templates/schema.compact.ejs new file mode 100644 index 0000000..e592a31 --- /dev/null +++ b/packages/cli/src/templates/schema.compact.ejs @@ -0,0 +1,8 @@ +import type { CompactSchema } from "@ldo/ldo"; + +/** + * ============================================================================= + * <%- fileName %>Schema: Compact Schema for <%- fileName %> + * ============================================================================= + */ +export const <%- fileName %>Schema: CompactSchema = <%- compactSchema %>; diff --git a/packages/ldo/src/index.ts b/packages/ldo/src/index.ts index 2f12e0a..657c238 100644 --- a/packages/ldo/src/index.ts +++ b/packages/ldo/src/index.ts @@ -9,3 +9,5 @@ export type { LdoBase, LdoCompactBase } from "./util.js"; export * from "./types.js"; export type { LdSet, LdoJsonldContext } from "@ldo/jsonld-dataset-proxy"; export { set } from "@ldo/jsonld-dataset-proxy"; +export type { Schema } from "@ldo/traverser-shexj"; +export type { CompactShape, CompactSchema } from "@ldo/schema-converter-shex"; diff --git a/packages/schema-converter-shex/src/context/shexjToContext.ts b/packages/schema-converter-shex/src/context/shexjToContext.ts index 49adfa9..d2ddae8 100644 --- a/packages/schema-converter-shex/src/context/shexjToContext.ts +++ b/packages/schema-converter-shex/src/context/shexjToContext.ts @@ -1,5 +1,5 @@ import type { ContextDefinition } from "jsonld"; -import type { Schema } from "shexj"; +import type { Schema } from "@ldo/traverser-shexj"; import { JsonLdContextBuilder } from "./JsonLdContextBuilder.js"; import { ShexJNameVisitor } from "./ShexJContextVisitor.js"; import { jsonld2graphobject } from "jsonld2graphobject"; diff --git a/packages/schema-converter-shex/src/index.ts b/packages/schema-converter-shex/src/index.ts index 802b007..f0576b4 100644 --- a/packages/schema-converter-shex/src/index.ts +++ b/packages/schema-converter-shex/src/index.ts @@ -1,4 +1,6 @@ import { shexjToTyping } from "./typing/shexjToTyping.js"; export { annotateReadablePredicates } from "./util/annotateReadablePredicates.js"; - +export type { CompactShape } from "./schema/ShexJSchemaTransformerCompact.js"; +export { ShexJSchemaTransformerCompact } from "./schema/ShexJSchemaTransformerCompact.js"; +export type { CompactSchema } from "./typing/shexjToTypingCompact.js"; export default shexjToTyping; diff --git a/packages/schema-converter-shex/src/schema/ShexJSchemaTransformerCompact.ts b/packages/schema-converter-shex/src/schema/ShexJSchemaTransformerCompact.ts new file mode 100644 index 0000000..20621a5 --- /dev/null +++ b/packages/schema-converter-shex/src/schema/ShexJSchemaTransformerCompact.ts @@ -0,0 +1,209 @@ +import type { ObjectLiteral } from "@ldo/traverser-shexj"; +import ShexJTraverser from "@ldo/traverser-shexj"; + +export interface CompactShape { + schemaUri: string; + predicates: CompactSchemaProperty[]; +} + +type NodeConstraintRet = { + literals?: number[] | string[] | boolean; + type: "number" | "string" | "boolean" | "literal"; +}; + +interface CompactSchemaProperty { + type: "number" | "string" | "boolean" | "nested" | "literal"; + predicateUri: string; + readablePredicate: string; + literalValue?: number | string | boolean | number[] | string[]; + nestedSchema?: string | CompactShape; + maxCardinality: number; + minCardinality: number; +} + +export const ShexJSchemaTransformerCompact = ShexJTraverser.createTransformer< + { + Schema: { return: CompactShape[] }; + ShapeDecl: { return: CompactShape }; + Shape: { return: CompactShape }; + EachOf: { return: CompactShape }; + TripleConstraint: { return: CompactSchemaProperty }; + NodeConstraint: { return: NodeConstraintRet }; + ShapeOr: { return: never }; + ShapeAnd: { return: never }; + ShapeNot: { return: never }; + ShapeExternal: { return: never }; + }, + null +>({ + // Transformer from Schema to interfaces + Schema: { + transformer: async (_schema, getTransformedChildren) => { + const transformedChildren = await getTransformedChildren(); + + return transformedChildren.shapes || []; + }, + }, + + // Transformer from ShapeDecl to interface + ShapeDecl: { + transformer: async (shapeDecl, getTransformedChildren) => { + const schema = await getTransformedChildren(); + + return { ...schema.shapeExpr, schemaUri: shapeDecl.id } as CompactShape; + }, + }, + + // Transformer from Shape to interface + Shape: { + transformer: async (_shape, getTransformedChildren, setReturnPointer) => { + // TODO: We don't handles those + _shape.closed; + _shape.extra; + + const transformedChildren = await getTransformedChildren(); + // Return prelim or expression or assign things? + return transformedChildren.expression as CompactShape; + }, + }, + + // Transformer from EachOf to object type. EachOf contains the `expressions` array of properties (TripleConstraint) + EachOf: { + transformer: async (eachOf, getTransformedChildren, setReturnPointer) => { + const transformedChildren = await getTransformedChildren(); + + return { + schemaUri: "", + predicates: transformedChildren.expressions.map( + // We disregard cases where properties are referenced (strings) + // or where they consist of Unions or Intersections (not supported). + (expr) => expr as CompactSchemaProperty, + ), + }; + }, + }, + + // Transformer from triple constraints to type properties. + TripleConstraint: { + transformer: async ( + tripleConstraint, + getTransformedChildren, + _setReturnPointer, + ) => { + const transformedChildren = await getTransformedChildren(); + + const commonProperties = { + maxCardinality: tripleConstraint.max ?? 1, + minCardinality: tripleConstraint.min ?? 1, + predicateUri: tripleConstraint.predicate, + readablePredicate: tripleConstraint.readablePredicate, + }; + // Make property based on object type which is either a parsed schema, literal or type. + if (typeof transformedChildren.valueExpr === "string") { + // Reference to nested object + return { + type: "nested", + nestedSchema: transformedChildren.valueExpr, + ...commonProperties, + } satisfies CompactSchemaProperty; + } else if ( + transformedChildren.valueExpr && + (transformedChildren.valueExpr as CompactShape).predicates + ) { + // Nested object + return { + type: "nested", + nestedSchema: transformedChildren.valueExpr as CompactShape, + ...commonProperties, + } satisfies CompactSchemaProperty; + } else { + // type or literal + const nodeConstraint = + transformedChildren.valueExpr as NodeConstraintRet; + return { + type: nodeConstraint.type, + literalValue: nodeConstraint.literals, + ...commonProperties, + } satisfies CompactSchemaProperty; + } + }, + }, + + // Transformer from node constraint to type + NodeConstraint: { + transformer: async (nodeConstraint) => { + if (nodeConstraint.datatype) { + switch (nodeConstraint.datatype) { + case "http://www.w3.org/2001/XMLSchema#boolean": + return { type: "boolean" }; + case "http://www.w3.org/2001/XMLSchema#byte": + case "http://www.w3.org/2001/XMLSchema#decimal": + case "http://www.w3.org/2001/XMLSchema#double": + case "http://www.w3.org/2001/XMLSchema#float": + case "http://www.w3.org/2001/XMLSchema#int": + case "http://www.w3.org/2001/XMLSchema#integer": + case "http://www.w3.org/2001/XMLSchema#long": + case "http://www.w3.org/2001/XMLSchema#negativeInteger": + case "http://www.w3.org/2001/XMLSchema#nonNegativeInteger": + case "http://www.w3.org/2001/XMLSchema#nonPositiveInteger": + case "http://www.w3.org/2001/XMLSchema#positiveInteger": + case "http://www.w3.org/2001/XMLSchema#short": + case "http://www.w3.org/2001/XMLSchema#unsignedLong": + case "http://www.w3.org/2001/XMLSchema#unsignedInt": + case "http://www.w3.org/2001/XMLSchema#unsignedShort": + case "http://www.w3.org/2001/XMLSchema#unsignedByte": + return { type: "number" }; + default: + return { type: "string" }; // treat most as string + } + } + if (nodeConstraint.nodeKind) { + // Something reference-like. + return { type: "string" }; + } + if (nodeConstraint.values) { + return { + type: "literal", + literals: nodeConstraint.values.map( + // TODO: We do not convert them to number or boolean or lang tag. + (valueRecord) => (valueRecord as ObjectLiteral).value, + ), + }; + } + + // Maybe we should throw instead... + throw { + error: new Error("Could not parse Node Constraint"), + nodeConstraint, + }; + }, + }, + + // Transformer from ShapeOr to union type + ShapeOr: { + transformer: async () => { + throw new Error("ShapeOr not supported (compact)"); + }, + }, + + // Transformer from ShapeAnd to intersection type + ShapeAnd: { + transformer: async () => { + throw new Error("ShapeAnd not supported (compact)"); + }, + }, + + // Transformer from ShapeNot to type - not supported. + ShapeNot: { + transformer: async () => { + throw new Error("ShapeNot not supported (compact)"); + }, + }, + + // Transformer from ShapeExternal to type - not supported. + ShapeExternal: { + transformer: async () => { + throw new Error("ShapeExternal not supported (compact)"); + }, + }, +}); diff --git a/packages/schema-converter-shex/src/typing/ShexJTypingTransformerCompact.ts b/packages/schema-converter-shex/src/typing/ShexJTypingTransformerCompact.ts index d84e081..e29c446 100644 --- a/packages/schema-converter-shex/src/typing/ShexJTypingTransformerCompact.ts +++ b/packages/schema-converter-shex/src/typing/ShexJTypingTransformerCompact.ts @@ -225,7 +225,7 @@ export const ShexJTypingTransformerCompact = ShexJTraverser.createTransformer< ShapeNot: { return: never }; ShapeExternal: { return: never }; }, - CompactTransformerContext + null >({ // Transformer from Schema to interfaces Schema: { @@ -372,12 +372,9 @@ export const ShexJTypingTransformerCompact = ShexJTraverser.createTransformer< getTransformedChildren, _setReturnPointer, node, - context, ) => { const transformedChildren = await getTransformedChildren(); - const baseName = - ((tripleConstraint as any).readablePredicate as string | undefined) ?? - context.getNameFromIri(tripleConstraint.predicate); + const baseName = (tripleConstraint as any).readablePredicate as string; const max = tripleConstraint.max; const isPlural = max === -1 || (max !== undefined && max !== 1); diff --git a/packages/schema-converter-shex/src/typing/shexjToTyping.ts b/packages/schema-converter-shex/src/typing/shexjToTyping.ts index 4e300d5..99ad12d 100644 --- a/packages/schema-converter-shex/src/typing/shexjToTyping.ts +++ b/packages/schema-converter-shex/src/typing/shexjToTyping.ts @@ -1,7 +1,8 @@ import type { ContextDefinition } from "jsonld"; -import type { Schema } from "shexj"; +import type { Schema } from "@ldo/traverser-shexj"; import type { TypeingReturn } from "./shexjToTypingLdo.js"; import { shexjToTypingLdo } from "./shexjToTypingLdo.js"; +import type { CompactSchema } from "./shexjToTypingCompact.js"; import { shexjToTypingCompact } from "./shexjToTypingCompact.js"; export interface TypingsOptions { @@ -11,7 +12,9 @@ export interface TypingsOptions { export async function shexjToTyping( shexj: Schema, options: TypingsOptions = {}, -): Promise<[TypeingReturn, ContextDefinition | undefined]> { +): Promise< + [TypeingReturn, ContextDefinition] | [TypeingReturn, undefined, CompactSchema] +> { const format = options.format || "ldo"; if (format === "compact") return shexjToTypingCompact(shexj); return shexjToTypingLdo(shexj); diff --git a/packages/schema-converter-shex/src/typing/shexjToTypingCompact.ts b/packages/schema-converter-shex/src/typing/shexjToTypingCompact.ts index 4278381..d136f21 100644 --- a/packages/schema-converter-shex/src/typing/shexjToTypingCompact.ts +++ b/packages/schema-converter-shex/src/typing/shexjToTypingCompact.ts @@ -1,18 +1,20 @@ -import type { Schema } from "shexj"; +import type { Schema } from "@ldo/traverser-shexj"; import { jsonld2graphobject } from "jsonld2graphobject"; -import { ShexJNameVisitor } from "../context/ShexJContextVisitor.js"; -import { JsonLdContextBuilder } from "../context/JsonLdContextBuilder.js"; import { ShexJTypingTransformerCompact, additionalCompactEnumAliases, } from "./ShexJTypingTransformerCompact.js"; import * as dom from "dts-dom"; import type { TypeingReturn } from "./shexjToTypingLdo.js"; -import { annotateReadablePredicates } from "../util/annotateReadablePredicates.js"; +import type { CompactShape } from "../schema/ShexJSchemaTransformerCompact.js"; +import { ShexJSchemaTransformerCompact } from "../schema/ShexJSchemaTransformerCompact.js"; + +type IRI = string; +export type CompactSchema = { [shapeId: IRI]: CompactShape }; export async function shexjToTypingCompact( shexj: Schema, -): Promise<[TypeingReturn, undefined]> { +): Promise<[TypeingReturn, undefined, CompactSchema]> { // Prepare processed schema (names still rely on context visitor) const processedShexj: Schema = (await jsonld2graphobject( { @@ -22,21 +24,22 @@ export async function shexjToTypingCompact( }, "SCHEMA", )) as unknown as Schema; - const nameBuilder = new JsonLdContextBuilder(); - await ShexJNameVisitor.visit(processedShexj, "Schema", nameBuilder); - - // Ensure collisions are pre-resolved and stored in readablePredicate on EachOf TCs - annotateReadablePredicates(processedShexj); additionalCompactEnumAliases.clear(); const declarations = await ShexJTypingTransformerCompact.transform( processedShexj, "Schema", - { - getNameFromIri: nameBuilder.getNameFromIri.bind(nameBuilder), - }, + null, ); + const compactSchemaShapesUnflattened = + await ShexJSchemaTransformerCompact.transform( + processedShexj, + "Schema", + null, + ); + const compactSchema = flattenSchema(compactSchemaShapesUnflattened); + // Append only enum aliases (no interface Id aliases in compact format now) const hasName = (d: unknown): d is { name: string } => typeof (d as { name?: unknown }).name === "string"; @@ -57,5 +60,39 @@ export async function shexjToTypingCompact( const header = `export type IRI = string;\n\n`; const typingsString = header + typings.map((t) => `export ${t.typingString}`).join(""); - return [{ typingsString, typings }, undefined]; + + return [{ typingsString, typings }, undefined, compactSchema]; +} + +/** Shapes may be nested. Put all to their root and give nested ones ids. */ +function flattenSchema(shapes: CompactShape[]): CompactSchema { + let schema: CompactSchema = {}; + + for (const shape of shapes) { + schema[shape.schemaUri] = shape; + + // Find nested, unflattened (i.e. anonymous) schemas in properties. + const nestedSchemaPredicates = shape.predicates.filter( + (pred) => pred.type === "nested" && typeof pred.nestedSchema === "object", + ); + + for (const pred of nestedSchemaPredicates) { + const newId = shape.schemaUri + "::" + pred.predicateUri; + + // Recurse + const flattened = flattenSchema([ + { + ...(pred.nestedSchema as CompactShape), + schemaUri: newId, + }, + ]); + // Replace the nested schema with its new id. + pred.nestedSchema = newId; + + schema = { ...schema, ...flattened }; + } + // Flatten / Recurse + } + + return schema; } diff --git a/packages/schema-converter-shex/src/typing/shexjToTypingLdo.ts b/packages/schema-converter-shex/src/typing/shexjToTypingLdo.ts index 0a6824d..d170eda 100644 --- a/packages/schema-converter-shex/src/typing/shexjToTypingLdo.ts +++ b/packages/schema-converter-shex/src/typing/shexjToTypingLdo.ts @@ -1,5 +1,5 @@ import type { ContextDefinition } from "jsonld"; -import type { Schema } from "shexj"; +import type { Schema } from "@ldo/traverser-shexj"; import { JsonLdContextBuilder } from "../context/JsonLdContextBuilder.js"; import { ShexJNameVisitor } from "../context/ShexJContextVisitor.js"; import { jsonld2graphobject } from "jsonld2graphobject"; diff --git a/packages/schema-converter-shex/src/util/annotateReadablePredicates.ts b/packages/schema-converter-shex/src/util/annotateReadablePredicates.ts index 4ed9d67..272027f 100644 --- a/packages/schema-converter-shex/src/util/annotateReadablePredicates.ts +++ b/packages/schema-converter-shex/src/util/annotateReadablePredicates.ts @@ -11,7 +11,7 @@ type TCwReadable = TripleConstraint & { readablePredicate?: string }; /** * Annotate EachOf-level TripleConstraints with a collision-free readablePredicate. * Rule: for any group that shares the same local token, rename all members using - * prefix-first `${prefix}_${local}` from left to right; fallback to composite. + * prefix-first `${prefix}_${local}` from right to left; fallback to composite. */ export function annotateReadablePredicates(schema: Schema): void { const shapes = schema.shapes ?? []; diff --git a/packages/traverser-shexj/src/ShexJTypes.ts b/packages/traverser-shexj/src/ShexJTypes.ts index 743e913..80fb7cd 100644 --- a/packages/traverser-shexj/src/ShexJTypes.ts +++ b/packages/traverser-shexj/src/ShexJTypes.ts @@ -537,6 +537,10 @@ export interface TripleConstraint extends tripleExprBase { * A {@link shapeExpr} matching a conformant RDF Triples subject or object, depending on the value of {@link inverse}. */ valueExpr?: shapeExprOrRef | undefined; + /** + * A human-readable predicate name used for creating compact ldo objects. + */ + readablePredicate: string; } /**