generate compact schemas

packages/cli/src/build.ts

@@ -1,6 +1,6 @@
 import fs from "fs-extra";
 import path from "path";
-import type { Schema } from "shexj";
+import type { Schema } from "@ldo/traverser-shexj";
 import parser from "@shexjs/parser";
 import schemaConverterShex from "@ldo/schema-converter-shex";
 import { renderFile } from "ejs";
@@ -30,11 +30,22 @@ export async function build(options: BuildOptions) {
   }
   await fs.promises.mkdir(options.output);
 
+  const format = options.format || "ldo";
+  const fileTemplates: string[] = [];
+
+  if (format === "compact") {
+    // Pre-annotate schema with readablePredicate to unify naming across outputs
+    fileTemplates.push("schema.compact", "typings", "shapeTypes.compact");
+  } else {
+    fileTemplates.push("schema", "typings", "shapeTypes", "context");
+  }
+
   load.text = "Generating LDO Documents";
   await forAllShapes(options.input, async (fileName, shexC) => {
     // Convert to ShexJ
     let schema: Schema;
     try {
+      // @ts-expect-error ...
       schema = parser.construct("https://ldo.js.org/").parse(shexC);
     } catch (err) {
       const errMessage =
@@ -46,29 +57,30 @@ export async function build(options: BuildOptions) {
       console.error(`Error processing ${fileName}: ${errMessage}`);
       return;
     }
-    // Pre-annotate schema with readablePredicate to unify naming across outputs
-    annotateReadablePredicates(schema);
 
-    // Convert the content to types
+    // Add readable predicates to schema as the single source of truth.
-    const format = options.format || "ldo";
+    if (format === "compact") {
-    const [typings, context] = await schemaConverterShex(schema, { format });
+      // @ts-expect-error ...
-    const templates: string[] = ["schema", "typings", "shapeTypes"];
+      annotateReadablePredicates(schema);
-    if (format === "ldo") {
-      templates.unshift("context");
     }
+
+    const [typings, context, compactSchema] = await schemaConverterShex(
+      schema,
+      {
+        format,
+      },
+    );
+
     await Promise.all(
-      templates.map(async (templateName) => {
+      fileTemplates.map(async (templateName) => {
-        const templateFile =
-          templateName === "shapeTypes" && format === "compact"
-            ? "shapeTypes.compact"
-            : templateName;
         const finalContent = await renderFile(
-          path.join(__dirname, "./templates", `${templateFile}.ejs`),
+          path.join(__dirname, "./templates", `${templateName}.ejs`),
           {
             typings: typings.typings,
             fileName,
             schema: JSON.stringify(schema, null, 2),
             context: JSON.stringify(context, null, 2),
+            compactSchema: JSON.stringify(compactSchema, null, 2),
             format,
           },
         );

packages/cli/src/templates/schema.compact.ejs

@@ -0,0 +1,8 @@
+import type { CompactSchema } from "@ldo/ldo";
+
+/**
+ * =============================================================================
+ * <%- fileName %>Schema: Compact Schema for <%- fileName %>
+ * =============================================================================
+ */
+export const <%- fileName %>Schema: CompactSchema = <%- compactSchema %>;

packages/ldo/src/index.ts

@@ -9,3 +9,5 @@ export type { LdoBase, LdoCompactBase } from "./util.js";
 export * from "./types.js";
 export type { LdSet, LdoJsonldContext } from "@ldo/jsonld-dataset-proxy";
 export { set } from "@ldo/jsonld-dataset-proxy";
+export type { Schema } from "@ldo/traverser-shexj";
+export type { CompactShape, CompactSchema } from "@ldo/schema-converter-shex";

packages/schema-converter-shex/src/context/shexjToContext.ts

@@ -1,5 +1,5 @@
 import type { ContextDefinition } from "jsonld";
-import type { Schema } from "shexj";
+import type { Schema } from "@ldo/traverser-shexj";
 import { JsonLdContextBuilder } from "./JsonLdContextBuilder.js";
 import { ShexJNameVisitor } from "./ShexJContextVisitor.js";
 import { jsonld2graphobject } from "jsonld2graphobject";

packages/schema-converter-shex/src/index.ts

@@ -1,4 +1,6 @@
 import { shexjToTyping } from "./typing/shexjToTyping.js";
 export { annotateReadablePredicates } from "./util/annotateReadablePredicates.js";
-
+export type { CompactShape } from "./schema/ShexJSchemaTransformerCompact.js";
+export { ShexJSchemaTransformerCompact } from "./schema/ShexJSchemaTransformerCompact.js";
+export type { CompactSchema } from "./typing/shexjToTypingCompact.js";
 export default shexjToTyping;

packages/schema-converter-shex/src/schema/ShexJSchemaTransformerCompact.ts

@@ -0,0 +1,209 @@
+import type { ObjectLiteral } from "@ldo/traverser-shexj";
+import ShexJTraverser from "@ldo/traverser-shexj";
+
+export interface CompactShape {
+  schemaUri: string;
+  predicates: CompactSchemaProperty[];
+}
+
+type NodeConstraintRet = {
+  literals?: number[] | string[] | boolean;
+  type: "number" | "string" | "boolean" | "literal";
+};
+
+interface CompactSchemaProperty {
+  type: "number" | "string" | "boolean" | "nested" | "literal";
+  predicateUri: string;
+  readablePredicate: string;
+  literalValue?: number | string | boolean | number[] | string[];
+  nestedSchema?: string | CompactShape;
+  maxCardinality: number;
+  minCardinality: number;
+}
+
+export const ShexJSchemaTransformerCompact = ShexJTraverser.createTransformer<
+  {
+    Schema: { return: CompactShape[] };
+    ShapeDecl: { return: CompactShape };
+    Shape: { return: CompactShape };
+    EachOf: { return: CompactShape };
+    TripleConstraint: { return: CompactSchemaProperty };
+    NodeConstraint: { return: NodeConstraintRet };
+    ShapeOr: { return: never };
+    ShapeAnd: { return: never };
+    ShapeNot: { return: never };
+    ShapeExternal: { return: never };
+  },
+  null
+>({
+  // Transformer from Schema to interfaces
+  Schema: {
+    transformer: async (_schema, getTransformedChildren) => {
+      const transformedChildren = await getTransformedChildren();
+
+      return transformedChildren.shapes || [];
+    },
+  },
+
+  // Transformer from ShapeDecl to interface
+  ShapeDecl: {
+    transformer: async (shapeDecl, getTransformedChildren) => {
+      const schema = await getTransformedChildren();
+
+      return { ...schema.shapeExpr, schemaUri: shapeDecl.id } as CompactShape;
+    },
+  },
+
+  // Transformer from Shape to interface
+  Shape: {
+    transformer: async (_shape, getTransformedChildren, setReturnPointer) => {
+      // TODO: We don't handles those
+      _shape.closed;
+      _shape.extra;
+
+      const transformedChildren = await getTransformedChildren();
+      // Return prelim or expression or assign things?
+      return transformedChildren.expression as CompactShape;
+    },
+  },
+
+  // Transformer from EachOf to object type. EachOf contains the `expressions` array of properties (TripleConstraint)
+  EachOf: {
+    transformer: async (eachOf, getTransformedChildren, setReturnPointer) => {
+      const transformedChildren = await getTransformedChildren();
+
+      return {
+        schemaUri: "",
+        predicates: transformedChildren.expressions.map(
+          // We disregard cases where properties are referenced (strings)
+          // or where they consist of Unions or Intersections (not supported).
+          (expr) => expr as CompactSchemaProperty,
+        ),
+      };
+    },
+  },
+
+  // Transformer from triple constraints to type properties.
+  TripleConstraint: {
+    transformer: async (
+      tripleConstraint,
+      getTransformedChildren,
+      _setReturnPointer,
+    ) => {
+      const transformedChildren = await getTransformedChildren();
+
+      const commonProperties = {
+        maxCardinality: tripleConstraint.max ?? 1,
+        minCardinality: tripleConstraint.min ?? 1,
+        predicateUri: tripleConstraint.predicate,
+        readablePredicate: tripleConstraint.readablePredicate,
+      };
+      // Make property based on object type which is either a parsed schema, literal or type.
+      if (typeof transformedChildren.valueExpr === "string") {
+        // Reference to nested object
+        return {
+          type: "nested",
+          nestedSchema: transformedChildren.valueExpr,
+          ...commonProperties,
+        } satisfies CompactSchemaProperty;
+      } else if (
+        transformedChildren.valueExpr &&
+        (transformedChildren.valueExpr as CompactShape).predicates
+      ) {
+        // Nested object
+        return {
+          type: "nested",
+          nestedSchema: transformedChildren.valueExpr as CompactShape,
+          ...commonProperties,
+        } satisfies CompactSchemaProperty;
+      } else {
+        // type or literal
+        const nodeConstraint =
+          transformedChildren.valueExpr as NodeConstraintRet;
+        return {
+          type: nodeConstraint.type,
+          literalValue: nodeConstraint.literals,
+          ...commonProperties,
+        } satisfies CompactSchemaProperty;
+      }
+    },
+  },
+
+  // Transformer from node constraint to type
+  NodeConstraint: {
+    transformer: async (nodeConstraint) => {
+      if (nodeConstraint.datatype) {
+        switch (nodeConstraint.datatype) {
+          case "http://www.w3.org/2001/XMLSchema#boolean":
+            return { type: "boolean" };
+          case "http://www.w3.org/2001/XMLSchema#byte":
+          case "http://www.w3.org/2001/XMLSchema#decimal":
+          case "http://www.w3.org/2001/XMLSchema#double":
+          case "http://www.w3.org/2001/XMLSchema#float":
+          case "http://www.w3.org/2001/XMLSchema#int":
+          case "http://www.w3.org/2001/XMLSchema#integer":
+          case "http://www.w3.org/2001/XMLSchema#long":
+          case "http://www.w3.org/2001/XMLSchema#negativeInteger":
+          case "http://www.w3.org/2001/XMLSchema#nonNegativeInteger":
+          case "http://www.w3.org/2001/XMLSchema#nonPositiveInteger":
+          case "http://www.w3.org/2001/XMLSchema#positiveInteger":
+          case "http://www.w3.org/2001/XMLSchema#short":
+          case "http://www.w3.org/2001/XMLSchema#unsignedLong":
+          case "http://www.w3.org/2001/XMLSchema#unsignedInt":
+          case "http://www.w3.org/2001/XMLSchema#unsignedShort":
+          case "http://www.w3.org/2001/XMLSchema#unsignedByte":
+            return { type: "number" };
+          default:
+            return { type: "string" }; // treat most as string
+        }
+      }
+      if (nodeConstraint.nodeKind) {
+        // Something reference-like.
+        return { type: "string" };
+      }
+      if (nodeConstraint.values) {
+        return {
+          type: "literal",
+          literals: nodeConstraint.values.map(
+            // TODO: We do not convert them to number or boolean or lang tag.
+            (valueRecord) => (valueRecord as ObjectLiteral).value,
+          ),
+        };
+      }
+
+      // Maybe we should throw instead...
+      throw {
+        error: new Error("Could not parse Node Constraint"),
+        nodeConstraint,
+      };
+    },
+  },
+
+  // Transformer from ShapeOr to union type
+  ShapeOr: {
+    transformer: async () => {
+      throw new Error("ShapeOr not supported (compact)");
+    },
+  },
+
+  // Transformer from ShapeAnd to intersection type
+  ShapeAnd: {
+    transformer: async () => {
+      throw new Error("ShapeAnd not supported (compact)");
+    },
+  },
+
+  // Transformer from ShapeNot to type - not supported.
+  ShapeNot: {
+    transformer: async () => {
+      throw new Error("ShapeNot not supported (compact)");
+    },
+  },
+
+  // Transformer from ShapeExternal to type - not supported.
+  ShapeExternal: {
+    transformer: async () => {
+      throw new Error("ShapeExternal not supported (compact)");
+    },
+  },
+});

packages/schema-converter-shex/src/typing/ShexJTypingTransformerCompact.ts

@@ -225,7 +225,7 @@ export const ShexJTypingTransformerCompact = ShexJTraverser.createTransformer<
     ShapeNot: { return: never };
     ShapeExternal: { return: never };
   },
-  CompactTransformerContext
+  null
 >({
   // Transformer from Schema to interfaces
   Schema: {
@@ -372,12 +372,9 @@ export const ShexJTypingTransformerCompact = ShexJTraverser.createTransformer<
       getTransformedChildren,
       _setReturnPointer,
       node,
-      context,
     ) => {
       const transformedChildren = await getTransformedChildren();
-      const baseName =
+      const baseName = (tripleConstraint as any).readablePredicate as string;
-        ((tripleConstraint as any).readablePredicate as string | undefined) ??
-        context.getNameFromIri(tripleConstraint.predicate);
 
       const max = tripleConstraint.max;
       const isPlural = max === -1 || (max !== undefined && max !== 1);

packages/schema-converter-shex/src/typing/shexjToTyping.ts

@@ -1,7 +1,8 @@
 import type { ContextDefinition } from "jsonld";
-import type { Schema } from "shexj";
+import type { Schema } from "@ldo/traverser-shexj";
 import type { TypeingReturn } from "./shexjToTypingLdo.js";
 import { shexjToTypingLdo } from "./shexjToTypingLdo.js";
+import type { CompactSchema } from "./shexjToTypingCompact.js";
 import { shexjToTypingCompact } from "./shexjToTypingCompact.js";
 
 export interface TypingsOptions {
@@ -11,7 +12,9 @@ export interface TypingsOptions {
 export async function shexjToTyping(
   shexj: Schema,
   options: TypingsOptions = {},
-): Promise<[TypeingReturn, ContextDefinition | undefined]> {
+): Promise<
+  [TypeingReturn, ContextDefinition] | [TypeingReturn, undefined, CompactSchema]
+> {
   const format = options.format || "ldo";
   if (format === "compact") return shexjToTypingCompact(shexj);
   return shexjToTypingLdo(shexj);

packages/schema-converter-shex/src/typing/shexjToTypingCompact.ts

@@ -1,18 +1,20 @@
-import type { Schema } from "shexj";
+import type { Schema } from "@ldo/traverser-shexj";
 import { jsonld2graphobject } from "jsonld2graphobject";
-import { ShexJNameVisitor } from "../context/ShexJContextVisitor.js";
-import { JsonLdContextBuilder } from "../context/JsonLdContextBuilder.js";
 import {
   ShexJTypingTransformerCompact,
   additionalCompactEnumAliases,
 } from "./ShexJTypingTransformerCompact.js";
 import * as dom from "dts-dom";
 import type { TypeingReturn } from "./shexjToTypingLdo.js";
-import { annotateReadablePredicates } from "../util/annotateReadablePredicates.js";
+import type { CompactShape } from "../schema/ShexJSchemaTransformerCompact.js";
+import { ShexJSchemaTransformerCompact } from "../schema/ShexJSchemaTransformerCompact.js";
+
+type IRI = string;
+export type CompactSchema = { [shapeId: IRI]: CompactShape };
 
 export async function shexjToTypingCompact(
   shexj: Schema,
-): Promise<[TypeingReturn, undefined]> {
+): Promise<[TypeingReturn, undefined, CompactSchema]> {
   // Prepare processed schema (names still rely on context visitor)
   const processedShexj: Schema = (await jsonld2graphobject(
     {
@@ -22,21 +24,22 @@ export async function shexjToTypingCompact(
     },
     "SCHEMA",
   )) as unknown as Schema;
-  const nameBuilder = new JsonLdContextBuilder();
-  await ShexJNameVisitor.visit(processedShexj, "Schema", nameBuilder);
-
-  // Ensure collisions are pre-resolved and stored in readablePredicate on EachOf TCs
-  annotateReadablePredicates(processedShexj);
 
   additionalCompactEnumAliases.clear();
   const declarations = await ShexJTypingTransformerCompact.transform(
     processedShexj,
     "Schema",
-    {
+    null,
-      getNameFromIri: nameBuilder.getNameFromIri.bind(nameBuilder),
-    },
   );
 
+  const compactSchemaShapesUnflattened =
+    await ShexJSchemaTransformerCompact.transform(
+      processedShexj,
+      "Schema",
+      null,
+    );
+  const compactSchema = flattenSchema(compactSchemaShapesUnflattened);
+
   // Append only enum aliases (no interface Id aliases in compact format now)
   const hasName = (d: unknown): d is { name: string } =>
     typeof (d as { name?: unknown }).name === "string";
@@ -57,5 +60,39 @@ export async function shexjToTypingCompact(
   const header = `export type IRI = string;\n\n`;
   const typingsString =
     header + typings.map((t) => `export ${t.typingString}`).join("");
-  return [{ typingsString, typings }, undefined];
+
+  return [{ typingsString, typings }, undefined, compactSchema];
+}
+
+/** Shapes may be nested. Put all to their root and give nested ones ids. */
+function flattenSchema(shapes: CompactShape[]): CompactSchema {
+  let schema: CompactSchema = {};
+
+  for (const shape of shapes) {
+    schema[shape.schemaUri] = shape;
+
+    // Find nested, unflattened (i.e. anonymous) schemas in properties.
+    const nestedSchemaPredicates = shape.predicates.filter(
+      (pred) => pred.type === "nested" && typeof pred.nestedSchema === "object",
+    );
+
+    for (const pred of nestedSchemaPredicates) {
+      const newId = shape.schemaUri + "::" + pred.predicateUri;
+
+      // Recurse
+      const flattened = flattenSchema([
+        {
+          ...(pred.nestedSchema as CompactShape),
+          schemaUri: newId,
+        },
+      ]);
+      // Replace the nested schema with its new id.
+      pred.nestedSchema = newId;
+
+      schema = { ...schema, ...flattened };
+    }
+    // Flatten / Recurse
+  }
+
+  return schema;
 }

packages/schema-converter-shex/src/typing/shexjToTypingLdo.ts

@@ -1,5 +1,5 @@
 import type { ContextDefinition } from "jsonld";
-import type { Schema } from "shexj";
+import type { Schema } from "@ldo/traverser-shexj";
 import { JsonLdContextBuilder } from "../context/JsonLdContextBuilder.js";
 import { ShexJNameVisitor } from "../context/ShexJContextVisitor.js";
 import { jsonld2graphobject } from "jsonld2graphobject";

packages/schema-converter-shex/src/util/annotateReadablePredicates.ts

@@ -11,7 +11,7 @@ type TCwReadable = TripleConstraint & { readablePredicate?: string };
 /**
  * Annotate EachOf-level TripleConstraints with a collision-free readablePredicate.
  * Rule: for any group that shares the same local token, rename all members using
- * prefix-first `${prefix}_${local}` from left to right; fallback to composite.
+ * prefix-first `${prefix}_${local}` from right to left; fallback to composite.
  */
 export function annotateReadablePredicates(schema: Schema): void {
   const shapes = schema.shapes ?? [];

packages/traverser-shexj/src/ShexJTypes.ts

@@ -537,6 +537,10 @@ export interface TripleConstraint extends tripleExprBase {
    * A {@link shapeExpr} matching a conformant <a href="https://www.w3.org/TR/rdf11-concepts/#dfn-triple">RDF Triple</a>s subject or object, depending on the value of {@link inverse}.
    */
   valueExpr?: shapeExprOrRef | undefined;
+  /**
+   * A human-readable predicate name used for creating compact ldo objects.
+   */
+  readablePredicate: string;
 }
 
 /**