generate compact schemas

main
Laurin Weger 2 weeks ago
parent 2db70df471
commit 3188733329
No known key found for this signature in database
GPG Key ID: 9B372BB0B792770F
  1. 42
      packages/cli/src/build.ts
  2. 8
      packages/cli/src/templates/schema.compact.ejs
  3. 2
      packages/ldo/src/index.ts
  4. 2
      packages/schema-converter-shex/src/context/shexjToContext.ts
  5. 4
      packages/schema-converter-shex/src/index.ts
  6. 209
      packages/schema-converter-shex/src/schema/ShexJSchemaTransformerCompact.ts
  7. 7
      packages/schema-converter-shex/src/typing/ShexJTypingTransformerCompact.ts
  8. 7
      packages/schema-converter-shex/src/typing/shexjToTyping.ts
  9. 65
      packages/schema-converter-shex/src/typing/shexjToTypingCompact.ts
  10. 2
      packages/schema-converter-shex/src/typing/shexjToTypingLdo.ts
  11. 2
      packages/schema-converter-shex/src/util/annotateReadablePredicates.ts
  12. 4
      packages/traverser-shexj/src/ShexJTypes.ts

@ -1,6 +1,6 @@
import fs from "fs-extra";
import path from "path";
import type { Schema } from "shexj";
import type { Schema } from "@ldo/traverser-shexj";
import parser from "@shexjs/parser";
import schemaConverterShex from "@ldo/schema-converter-shex";
import { renderFile } from "ejs";
@ -30,11 +30,22 @@ export async function build(options: BuildOptions) {
}
await fs.promises.mkdir(options.output);
const format = options.format || "ldo";
const fileTemplates: string[] = [];
if (format === "compact") {
// Pre-annotate schema with readablePredicate to unify naming across outputs
fileTemplates.push("schema.compact", "typings", "shapeTypes.compact");
} else {
fileTemplates.push("schema", "typings", "shapeTypes", "context");
}
load.text = "Generating LDO Documents";
await forAllShapes(options.input, async (fileName, shexC) => {
// Convert to ShexJ
let schema: Schema;
try {
// @ts-expect-error ...
schema = parser.construct("https://ldo.js.org/").parse(shexC);
} catch (err) {
const errMessage =
@ -46,29 +57,30 @@ export async function build(options: BuildOptions) {
console.error(`Error processing ${fileName}: ${errMessage}`);
return;
}
// Pre-annotate schema with readablePredicate to unify naming across outputs
annotateReadablePredicates(schema);
// Convert the content to types
const format = options.format || "ldo";
const [typings, context] = await schemaConverterShex(schema, { format });
const templates: string[] = ["schema", "typings", "shapeTypes"];
if (format === "ldo") {
templates.unshift("context");
// Add readable predicates to schema as the single source of truth.
if (format === "compact") {
// @ts-expect-error ...
annotateReadablePredicates(schema);
}
const [typings, context, compactSchema] = await schemaConverterShex(
schema,
{
format,
},
);
await Promise.all(
templates.map(async (templateName) => {
const templateFile =
templateName === "shapeTypes" && format === "compact"
? "shapeTypes.compact"
: templateName;
fileTemplates.map(async (templateName) => {
const finalContent = await renderFile(
path.join(__dirname, "./templates", `${templateFile}.ejs`),
path.join(__dirname, "./templates", `${templateName}.ejs`),
{
typings: typings.typings,
fileName,
schema: JSON.stringify(schema, null, 2),
context: JSON.stringify(context, null, 2),
compactSchema: JSON.stringify(compactSchema, null, 2),
format,
},
);

@ -0,0 +1,8 @@
import type { CompactSchema } from "@ldo/ldo";
/**
* =============================================================================
* <%- fileName %>Schema: Compact Schema for <%- fileName %>
* =============================================================================
*/
export const <%- fileName %>Schema: CompactSchema = <%- compactSchema %>;

@ -9,3 +9,5 @@ export type { LdoBase, LdoCompactBase } from "./util.js";
export * from "./types.js";
export type { LdSet, LdoJsonldContext } from "@ldo/jsonld-dataset-proxy";
export { set } from "@ldo/jsonld-dataset-proxy";
export type { Schema } from "@ldo/traverser-shexj";
export type { CompactShape, CompactSchema } from "@ldo/schema-converter-shex";

@ -1,5 +1,5 @@
import type { ContextDefinition } from "jsonld";
import type { Schema } from "shexj";
import type { Schema } from "@ldo/traverser-shexj";
import { JsonLdContextBuilder } from "./JsonLdContextBuilder.js";
import { ShexJNameVisitor } from "./ShexJContextVisitor.js";
import { jsonld2graphobject } from "jsonld2graphobject";

@ -1,4 +1,6 @@
import { shexjToTyping } from "./typing/shexjToTyping.js";
export { annotateReadablePredicates } from "./util/annotateReadablePredicates.js";
export type { CompactShape } from "./schema/ShexJSchemaTransformerCompact.js";
export { ShexJSchemaTransformerCompact } from "./schema/ShexJSchemaTransformerCompact.js";
export type { CompactSchema } from "./typing/shexjToTypingCompact.js";
export default shexjToTyping;

@ -0,0 +1,209 @@
import type { ObjectLiteral } from "@ldo/traverser-shexj";
import ShexJTraverser from "@ldo/traverser-shexj";
export interface CompactShape {
schemaUri: string;
predicates: CompactSchemaProperty[];
}
type NodeConstraintRet = {
literals?: number[] | string[] | boolean;
type: "number" | "string" | "boolean" | "literal";
};
interface CompactSchemaProperty {
type: "number" | "string" | "boolean" | "nested" | "literal";
predicateUri: string;
readablePredicate: string;
literalValue?: number | string | boolean | number[] | string[];
nestedSchema?: string | CompactShape;
maxCardinality: number;
minCardinality: number;
}
export const ShexJSchemaTransformerCompact = ShexJTraverser.createTransformer<
{
Schema: { return: CompactShape[] };
ShapeDecl: { return: CompactShape };
Shape: { return: CompactShape };
EachOf: { return: CompactShape };
TripleConstraint: { return: CompactSchemaProperty };
NodeConstraint: { return: NodeConstraintRet };
ShapeOr: { return: never };
ShapeAnd: { return: never };
ShapeNot: { return: never };
ShapeExternal: { return: never };
},
null
>({
// Transformer from Schema to interfaces
Schema: {
transformer: async (_schema, getTransformedChildren) => {
const transformedChildren = await getTransformedChildren();
return transformedChildren.shapes || [];
},
},
// Transformer from ShapeDecl to interface
ShapeDecl: {
transformer: async (shapeDecl, getTransformedChildren) => {
const schema = await getTransformedChildren();
return { ...schema.shapeExpr, schemaUri: shapeDecl.id } as CompactShape;
},
},
// Transformer from Shape to interface
Shape: {
transformer: async (_shape, getTransformedChildren, setReturnPointer) => {
// TODO: We don't handles those
_shape.closed;
_shape.extra;
const transformedChildren = await getTransformedChildren();
// Return prelim or expression or assign things?
return transformedChildren.expression as CompactShape;
},
},
// Transformer from EachOf to object type. EachOf contains the `expressions` array of properties (TripleConstraint)
EachOf: {
transformer: async (eachOf, getTransformedChildren, setReturnPointer) => {
const transformedChildren = await getTransformedChildren();
return {
schemaUri: "",
predicates: transformedChildren.expressions.map(
// We disregard cases where properties are referenced (strings)
// or where they consist of Unions or Intersections (not supported).
(expr) => expr as CompactSchemaProperty,
),
};
},
},
// Transformer from triple constraints to type properties.
TripleConstraint: {
transformer: async (
tripleConstraint,
getTransformedChildren,
_setReturnPointer,
) => {
const transformedChildren = await getTransformedChildren();
const commonProperties = {
maxCardinality: tripleConstraint.max ?? 1,
minCardinality: tripleConstraint.min ?? 1,
predicateUri: tripleConstraint.predicate,
readablePredicate: tripleConstraint.readablePredicate,
};
// Make property based on object type which is either a parsed schema, literal or type.
if (typeof transformedChildren.valueExpr === "string") {
// Reference to nested object
return {
type: "nested",
nestedSchema: transformedChildren.valueExpr,
...commonProperties,
} satisfies CompactSchemaProperty;
} else if (
transformedChildren.valueExpr &&
(transformedChildren.valueExpr as CompactShape).predicates
) {
// Nested object
return {
type: "nested",
nestedSchema: transformedChildren.valueExpr as CompactShape,
...commonProperties,
} satisfies CompactSchemaProperty;
} else {
// type or literal
const nodeConstraint =
transformedChildren.valueExpr as NodeConstraintRet;
return {
type: nodeConstraint.type,
literalValue: nodeConstraint.literals,
...commonProperties,
} satisfies CompactSchemaProperty;
}
},
},
// Transformer from node constraint to type
NodeConstraint: {
transformer: async (nodeConstraint) => {
if (nodeConstraint.datatype) {
switch (nodeConstraint.datatype) {
case "http://www.w3.org/2001/XMLSchema#boolean":
return { type: "boolean" };
case "http://www.w3.org/2001/XMLSchema#byte":
case "http://www.w3.org/2001/XMLSchema#decimal":
case "http://www.w3.org/2001/XMLSchema#double":
case "http://www.w3.org/2001/XMLSchema#float":
case "http://www.w3.org/2001/XMLSchema#int":
case "http://www.w3.org/2001/XMLSchema#integer":
case "http://www.w3.org/2001/XMLSchema#long":
case "http://www.w3.org/2001/XMLSchema#negativeInteger":
case "http://www.w3.org/2001/XMLSchema#nonNegativeInteger":
case "http://www.w3.org/2001/XMLSchema#nonPositiveInteger":
case "http://www.w3.org/2001/XMLSchema#positiveInteger":
case "http://www.w3.org/2001/XMLSchema#short":
case "http://www.w3.org/2001/XMLSchema#unsignedLong":
case "http://www.w3.org/2001/XMLSchema#unsignedInt":
case "http://www.w3.org/2001/XMLSchema#unsignedShort":
case "http://www.w3.org/2001/XMLSchema#unsignedByte":
return { type: "number" };
default:
return { type: "string" }; // treat most as string
}
}
if (nodeConstraint.nodeKind) {
// Something reference-like.
return { type: "string" };
}
if (nodeConstraint.values) {
return {
type: "literal",
literals: nodeConstraint.values.map(
// TODO: We do not convert them to number or boolean or lang tag.
(valueRecord) => (valueRecord as ObjectLiteral).value,
),
};
}
// Maybe we should throw instead...
throw {
error: new Error("Could not parse Node Constraint"),
nodeConstraint,
};
},
},
// Transformer from ShapeOr to union type
ShapeOr: {
transformer: async () => {
throw new Error("ShapeOr not supported (compact)");
},
},
// Transformer from ShapeAnd to intersection type
ShapeAnd: {
transformer: async () => {
throw new Error("ShapeAnd not supported (compact)");
},
},
// Transformer from ShapeNot to type - not supported.
ShapeNot: {
transformer: async () => {
throw new Error("ShapeNot not supported (compact)");
},
},
// Transformer from ShapeExternal to type - not supported.
ShapeExternal: {
transformer: async () => {
throw new Error("ShapeExternal not supported (compact)");
},
},
});

@ -225,7 +225,7 @@ export const ShexJTypingTransformerCompact = ShexJTraverser.createTransformer<
ShapeNot: { return: never };
ShapeExternal: { return: never };
},
CompactTransformerContext
null
>({
// Transformer from Schema to interfaces
Schema: {
@ -372,12 +372,9 @@ export const ShexJTypingTransformerCompact = ShexJTraverser.createTransformer<
getTransformedChildren,
_setReturnPointer,
node,
context,
) => {
const transformedChildren = await getTransformedChildren();
const baseName =
((tripleConstraint as any).readablePredicate as string | undefined) ??
context.getNameFromIri(tripleConstraint.predicate);
const baseName = (tripleConstraint as any).readablePredicate as string;
const max = tripleConstraint.max;
const isPlural = max === -1 || (max !== undefined && max !== 1);

@ -1,7 +1,8 @@
import type { ContextDefinition } from "jsonld";
import type { Schema } from "shexj";
import type { Schema } from "@ldo/traverser-shexj";
import type { TypeingReturn } from "./shexjToTypingLdo.js";
import { shexjToTypingLdo } from "./shexjToTypingLdo.js";
import type { CompactSchema } from "./shexjToTypingCompact.js";
import { shexjToTypingCompact } from "./shexjToTypingCompact.js";
export interface TypingsOptions {
@ -11,7 +12,9 @@ export interface TypingsOptions {
export async function shexjToTyping(
shexj: Schema,
options: TypingsOptions = {},
): Promise<[TypeingReturn, ContextDefinition | undefined]> {
): Promise<
[TypeingReturn, ContextDefinition] | [TypeingReturn, undefined, CompactSchema]
> {
const format = options.format || "ldo";
if (format === "compact") return shexjToTypingCompact(shexj);
return shexjToTypingLdo(shexj);

@ -1,18 +1,20 @@
import type { Schema } from "shexj";
import type { Schema } from "@ldo/traverser-shexj";
import { jsonld2graphobject } from "jsonld2graphobject";
import { ShexJNameVisitor } from "../context/ShexJContextVisitor.js";
import { JsonLdContextBuilder } from "../context/JsonLdContextBuilder.js";
import {
ShexJTypingTransformerCompact,
additionalCompactEnumAliases,
} from "./ShexJTypingTransformerCompact.js";
import * as dom from "dts-dom";
import type { TypeingReturn } from "./shexjToTypingLdo.js";
import { annotateReadablePredicates } from "../util/annotateReadablePredicates.js";
import type { CompactShape } from "../schema/ShexJSchemaTransformerCompact.js";
import { ShexJSchemaTransformerCompact } from "../schema/ShexJSchemaTransformerCompact.js";
type IRI = string;
export type CompactSchema = { [shapeId: IRI]: CompactShape };
export async function shexjToTypingCompact(
shexj: Schema,
): Promise<[TypeingReturn, undefined]> {
): Promise<[TypeingReturn, undefined, CompactSchema]> {
// Prepare processed schema (names still rely on context visitor)
const processedShexj: Schema = (await jsonld2graphobject(
{
@ -22,21 +24,22 @@ export async function shexjToTypingCompact(
},
"SCHEMA",
)) as unknown as Schema;
const nameBuilder = new JsonLdContextBuilder();
await ShexJNameVisitor.visit(processedShexj, "Schema", nameBuilder);
// Ensure collisions are pre-resolved and stored in readablePredicate on EachOf TCs
annotateReadablePredicates(processedShexj);
additionalCompactEnumAliases.clear();
const declarations = await ShexJTypingTransformerCompact.transform(
processedShexj,
"Schema",
{
getNameFromIri: nameBuilder.getNameFromIri.bind(nameBuilder),
},
null,
);
const compactSchemaShapesUnflattened =
await ShexJSchemaTransformerCompact.transform(
processedShexj,
"Schema",
null,
);
const compactSchema = flattenSchema(compactSchemaShapesUnflattened);
// Append only enum aliases (no interface Id aliases in compact format now)
const hasName = (d: unknown): d is { name: string } =>
typeof (d as { name?: unknown }).name === "string";
@ -57,5 +60,39 @@ export async function shexjToTypingCompact(
const header = `export type IRI = string;\n\n`;
const typingsString =
header + typings.map((t) => `export ${t.typingString}`).join("");
return [{ typingsString, typings }, undefined];
return [{ typingsString, typings }, undefined, compactSchema];
}
/** Shapes may be nested. Put all to their root and give nested ones ids. */
function flattenSchema(shapes: CompactShape[]): CompactSchema {
let schema: CompactSchema = {};
for (const shape of shapes) {
schema[shape.schemaUri] = shape;
// Find nested, unflattened (i.e. anonymous) schemas in properties.
const nestedSchemaPredicates = shape.predicates.filter(
(pred) => pred.type === "nested" && typeof pred.nestedSchema === "object",
);
for (const pred of nestedSchemaPredicates) {
const newId = shape.schemaUri + "::" + pred.predicateUri;
// Recurse
const flattened = flattenSchema([
{
...(pred.nestedSchema as CompactShape),
schemaUri: newId,
},
]);
// Replace the nested schema with its new id.
pred.nestedSchema = newId;
schema = { ...schema, ...flattened };
}
// Flatten / Recurse
}
return schema;
}

@ -1,5 +1,5 @@
import type { ContextDefinition } from "jsonld";
import type { Schema } from "shexj";
import type { Schema } from "@ldo/traverser-shexj";
import { JsonLdContextBuilder } from "../context/JsonLdContextBuilder.js";
import { ShexJNameVisitor } from "../context/ShexJContextVisitor.js";
import { jsonld2graphobject } from "jsonld2graphobject";

@ -11,7 +11,7 @@ type TCwReadable = TripleConstraint & { readablePredicate?: string };
/**
* Annotate EachOf-level TripleConstraints with a collision-free readablePredicate.
* Rule: for any group that shares the same local token, rename all members using
* prefix-first `${prefix}_${local}` from left to right; fallback to composite.
* prefix-first `${prefix}_${local}` from right to left; fallback to composite.
*/
export function annotateReadablePredicates(schema: Schema): void {
const shapes = schema.shapes ?? [];

@ -537,6 +537,10 @@ export interface TripleConstraint extends tripleExprBase {
* A {@link shapeExpr} matching a conformant <a href="https://www.w3.org/TR/rdf11-concepts/#dfn-triple">RDF Triple</a>s subject or object, depending on the value of {@link inverse}.
*/
valueExpr?: shapeExprOrRef | undefined;
/**
* A human-readable predicate name used for creating compact ldo objects.
*/
readablePredicate: string;
}
/**

Loading…
Cancel
Save