[org.json_schema.contrib] Support allOf schemas (#69)

This adds support for JSON Schemas with `allOf`.

packages/k8s.contrib.crd/PklProject

@@ -29,5 +29,5 @@ dependencies {
 }
 
 package {
-  version = "1.0.8"
+  version = "1.0.9"
 }

packages/k8s.contrib.crd/PklProject.deps.json

@@ -15,7 +15,7 @@
     },
     "package://pkg.pkl-lang.org/pkl-pantry/org.json_schema.contrib@1": {
       "type": "local",
-      "uri": "projectpackage://pkg.pkl-lang.org/pkl-pantry/org.json_schema.contrib@1.0.9",
+      "uri": "projectpackage://pkg.pkl-lang.org/pkl-pantry/org.json_schema.contrib@1.1.0",
       "path": "../org.json_schema.contrib"
     },
     "package://pkg.pkl-lang.org/pkl-pantry/pkl.experimental.syntax@1": {

packages/org.json_schema.contrib/PklProject

@@ -25,5 +25,5 @@ dependencies {
 }
 
 package {
-  version = "1.0.9"
+  version = "1.1.0"
 }

packages/org.json_schema.contrib/generate.pkl

@@ -29,9 +29,20 @@
 ///   Otherwise, this falls back to [Dynamic], which is the loosest constraint.
 /// - Cannot generate tuple types (this is missing in Pkl).
 /// - Properties called `default` cannot be generated (currently a limitation of the json parser).
+/// - `allOf` schemas have several limitations
+///   * `default` will only be combined when all subschemas have the same value
+///   * `deprecated` will be [true] if an only if all subschemas have it set to [true], otherwise it will be null
+///   * `readOnly` and `writeOnly` must be identical for all subschemas
+///   * `type` must be identical (or null) for all subschemas
+///   * `const` must be identical (or null) for all subschemas
+///   * if multiple subschemas set `multipleOf`, all must be integers
+///   * `format` must be identical (or null) for all subschemas
+///   * overlapping `properties` and `patternProperties` entries are merged as `allOf` according to these rules
+///   * fields that accept [JsonSchema.Schema] values must all be [JsonSchema] values (or null)
+///   * `items` must be [JsonSchema] or null, not [Listing]<[JsonSchema]>
+///   * there must be precise overlap between elements of `oneOf` and `anyOf`
 ///
 /// TODO:
-/// - Handle usages of `allOf`. We can do this by merging subschemas into a larger schema.
 /// - Copy doc comments from a class or typealias to its usage sites if there isn't a doc comment already.
 /// - Handle if schema root is not an object type (Example: ansible's schema root has `"type": "array"`).
 /// - Handle if schema root should be a mapping (it has `additionalProperties` or `patternProperties` set).

packages/org.json_schema.contrib/internal/ModuleGenerator.pkl

@@ -37,6 +37,9 @@ local jsonRenderer = new JsonRenderer {}
 /// The root schema, used to resolve `$ref` values.
 rootSchema: JsonSchema
 
+// local collatedRootSchema = rootSchema
+local collatedRootSchema = if (rootSchema.allOf != null) TypesGenerator.collateAllOf(rootSchema) else rootSchema
+
 /// The URI representing the root schema, used to resolve `$ref` values.
 baseUri: URI
 
@@ -74,7 +77,7 @@ moduleNode: ModuleNode =
         }
       }
       declaration {
-        docComment = getDocComment(rootSchema, "module")
+        docComment = getDocComment(collatedRootSchema, "module")
         when (moduleName != null) {
           moduleHeader {
             name {
@@ -88,8 +91,8 @@ moduleNode: ModuleNode =
         }
       }
       properties =
-        if (isClassLike(rootSchema))
-          generateClassBody(rootSchema, allTypeNames)
+        if (isClassLike(collatedRootSchema))
+          generateClassBody(collatedRootSchema, allTypeNames)
         else null
     }
 
@@ -142,6 +145,7 @@ local function generateClassBody(
       if (referencedSchema == null) new {}
       else if (referencedSchema is Boolean) let (_ = trace("WARN: `$ref` points to a boolean somehow")) new {}
       else generateClassBody(referencedSchema as JsonSchema, typeNames)
+  else if (schema.allOf != null) generateClassBody(TypesGenerator.collateAllOf(schema), typeNames)
   else
     new {
       for (propName, propSchema in schema.properties!!) {
@@ -220,7 +224,8 @@ local function determineTypeName(path: List<String>, candidateName: String, exis
 /// Classes get rendered for any subschema that has [JsonSchema.properties] defined, and does not have [JsonSchema.`$ref`] defined.
 local classSchemas: Type.TypeNames =
   let (schemas = utils._findMatchingSubSchemas(rootSchema, List(), (elem) -> elem != rootSchema && isClassLike(elem)))
-    schemas
+  let (collatedSchemas = if (collatedRootSchema == rootSchema) Map() else utils._findMatchingSubSchemas(collatedRootSchema, List(), (elem) -> elem != collatedRootSchema && isClassLike(elem)))
+    (schemas + collatedSchemas)
       .entries
       .fold(Map(), (accumulator: Type.TypeNames, pair) ->
         let (path = pair.first)
@@ -238,7 +243,8 @@ local classNames: Set<Type> = classSchemas.values.toSet()
 local typeAliasSchemas: Type.TypeNames =
   // Grab all schemas that have `definitions` or `$defs`
   let (schemasWithDefinitions = utils._findMatchingSubSchemas(rootSchema, List(), (elem) -> (elem.definitions ?? elem.$defs) != null))
-    schemasWithDefinitions
+  let (collatedSchemasWithDefinitions = if (collatedRootSchema == rootSchema) Map() else utils._findMatchingSubSchemas(collatedRootSchema, List(), (elem) -> (elem.definitions ?? elem.$defs) != null))
+    (schemasWithDefinitions + collatedSchemasWithDefinitions)
       .entries
       // For each schema, return the child json schema properties that are not class-like
       .flatMap((pair) ->

packages/org.json_schema.contrib/internal/TypesGenerator.pkl

@@ -60,8 +60,8 @@ function generateTypeNode(
     else
       new TypeNode.BuiltInTypeNode { type = "nothing" }
   // If we have generated a class or typealias definition already, simply use it.
-  else if (typeNames.containsKey(schema))
-    let (type = typeNames[schema as JsonSchema])
+  else if (typeNames.containsKey(schema._inline_?.getOrNull("__ref_orig__") ?? schema))
+    let (type = typeNames[(schema._inline_?.getOrNull("__ref_orig__") ?? schema) as JsonSchema])
       utils.declaredType1(type, type.moduleName != enclosingModuleName)
   // Edge case: if `type` includes `"null"`, treat this as a nullable type.
   else if (schema.type is Listing && schema.type.toList().contains("null"))
@@ -454,7 +454,8 @@ local function generateNumberType(schema: JsonSchema): Pair<JsonSchema, TypeNode
     if (isInt)
       let (minimum = if (schema.exclusiveMinimum != null) schema.exclusiveMinimum!! + 1 else schema.minimum)
       let (maximum = if (schema.exclusiveMaximum != null) schema.exclusiveMaximum!! - 1 else schema.maximum)
-      let (refinedInt = if (minimum == 0 && maximum == math.maxUInt32)
+      let (refinedInt =
+        if (minimum == 0 && maximum == math.maxUInt32)
           utils.declaredType("UInt32")
         else if (minimum == 0 && maximum == math.maxUInt16)
           utils.declaredType("UInt16")
@@ -564,11 +565,11 @@ local function generateUnionType(schema: JsonSchema, typeNames: Type.TypeNames):
       // Force no further constraints to be added by passing on an empty json schema.
       new JsonSchema {},
       new TypeNode.UnionTypeNode {
-        members {
+        members = new Listing<TypeNode> {
           for (_type in (schema.type as Listing)) {
             generateTypeNode((schema) { type = _type }, typeNames)
           }
-        }
+        }.distinct
       }
     )
   else
@@ -578,7 +579,7 @@ local function generateUnionType(schema: JsonSchema, typeNames: Type.TypeNames):
         anyOf = null
       },
       new TypeNode.UnionTypeNode {
-        members {
+        members = new Listing<TypeNode> {
           when (schema.oneOf != null) {
             for (s in schema.oneOf!!) {
               generateTypeNode(s, typeNames)
@@ -591,7 +592,7 @@ local function generateUnionType(schema: JsonSchema, typeNames: Type.TypeNames):
               generateTypeNode(s, typeNames)
             }
           }
-        }
+        }.distinct
       }
     )
 
@@ -609,7 +610,7 @@ local function generateListingType(schema: JsonSchema, typeNames: Type.TypeNames
 /// Used for determining the base type.
 local function constOrEnumsMatch(schema: JsonSchema, predicate: (JsonSchema.JsonSchemaValue?) -> Boolean): Boolean =
   predicate.apply(schema.`const`)
-    || (schema.enum?.toList()?.every(predicate) ?? predicate.apply(schema.enum))
+  || (schema.enum?.toList()?.every(predicate) ?? predicate.apply(schema.enum))
 
 /// Returns the name of the schema's type if there is only one type.
 ///
@@ -630,7 +631,7 @@ local function unwrappedSingularType(schema: JsonSchema): JsonSchema.JsonSchemaT
 /// Tells if the [schema] should be generated as [Number] or [Integer].
 local function isNumberSchema(schema: JsonSchema) =
   let (type = unwrappedSingularType(schema))
-    type == "number" || type == "integer" ||  constOrEnumsMatch(schema, (elem) -> elem is Number)
+    type == "number" || type == "integer" || constOrEnumsMatch(schema, (elem) -> elem is Number)
 
 /// Tells if the [schema] should be generated as a [String].
 local function isStringSchema(schema: JsonSchema) =
@@ -650,7 +651,7 @@ local function isListingSchema(schema: JsonSchema) =
 /// Tells if the [schema] should be generated as a [Mapping].
 local function isMappingSchema(schema: JsonSchema) =
   let (type = unwrappedSingularType(schema))
-    type == "object" ||  schema.properties != null || schema.additionalProperties != null
+    type == "object" || schema.properties != null || schema.additionalProperties != null
 
 /// Generates the basic declared type (Int, Float, etc).
 ///
@@ -669,5 +670,137 @@ local function generateBaseType(schema: JsonSchema, typeNames: Type.TypeNames):
     Pair(schema, generateListingType(schema, typeNames))
   else if (isMappingSchema(schema))
     Pair(schema, generateObjectType(schema, typeNames))
+  else if (schema.allOf != null)
+    let (collatedSchema = collateAllOf(schema))
+      generateBaseType(collatedSchema, typeNames)
   else
     Pair(schema, utils.declaredType("Any"))
+
+local function allOfErr(msg: String) = throw("Unable to combine allOf elements into one schema: \(msg)")
+
+function collateAllOf(schema: JsonSchema(allOf != null)): JsonSchema =
+  schema.allOf.fold(schema, (res, rawElem) ->
+    let (resolvedElem = if (rawElem is JsonSchema && rawElem.$$refUri != null) ref.resolveRef(baseUri, rawElem) else rawElem)
+      let (elem = if (resolvedElem is JsonSchema && resolvedElem.allOf != null) collateAllOf(resolvedElem) else resolvedElem)
+        new JsonSchema {
+          // metadata
+          title = collateInformation(elem.title, res.title)
+          description = collateInformation(elem.description, res.description)
+          default =
+            if (elem.default != null && res.default != null) let (_ = trace("Unable to combine allOf elements into one schema: dropping conflicting default")) res.default
+            else elem.default ?? res.default
+          examples =
+            if (elem.examples != null && res.examples != null)
+              new {
+                when (elem.examples is Listing) { ...elem.examples as Listing } else { elem.examples }
+                when (res.examples is Listing) { ...res.examples as Listing } else { res.examples }
+              }
+            else elem.examples ?? res.examples
+          deprecated =
+            if (elem.deprecated == true && res.deprecated == true) true
+            else null // if we have a mix of null/false/true, this is undeterminable
+          readOnly = collateMetadataBoolean(elem.readOnly, res.readOnly, "readOnly")
+          writeOnly = collateMetadataBoolean(elem.writeOnly, res.writeOnly, "writeOnly")
+
+          // core
+          type =
+            if (res.type != null && elem.type != null && res.type != elem.type) allOfErr("conflicting type")
+            else elem.type ?? res.type
+          `const` =
+            if (res.`const` != null && elem.`const` != null && res.`const` != elem.`const`) allOfErr("conflicting const")
+            else elem.`const` ?? res.`const`
+
+          // number
+          multipleOf =
+            if (elem.multipleOf != null && res.multipleOf != null)
+            if (elem.multipleOf is Int && res.multipleOf is Int) math.lcm(elem.multipleOf!! as Int, res.multipleOf!! as Int)
+            else allOfErr("multiple non-integer multipleOf")
+          else elem.multipleOf ?? res.multipleOf
+          minimum = collateMin(elem.minimum, res.minimum)
+          exclusiveMinimum = collateMin(elem.exclusiveMinimum, res.exclusiveMinimum)
+          maximum = collateMax(elem.maximum, res.maximum)
+          exclusiveMaximum = collateMax(elem.exclusiveMaximum, res.exclusiveMaximum)
+
+          // string
+          pattern =
+            if (elem.pattern != null && res.pattern != null) "(?:\(elem.pattern))|(?:\(res.pattern))"
+            else elem.pattern ?? res.pattern
+          minLength = collateMin(elem.minLength, res.minLength) as UInt?
+          maxLength = collateMax(elem.maxLength, res.maxLength) as UInt?
+          format =
+            if (res.format != null && elem.format != null && res.format != elem.format) allOfErr("conflicting format")
+            else elem.format ?? res.format
+
+          // object
+          properties = collateProperties(elem.properties, res.properties)
+          patternProperties = collateProperties(elem.patternProperties, res.patternProperties)
+          additionalProperties = collateSchema(elem.additionalProperties, res.additionalProperties, "additionalProperties")
+          required =
+            if (elem.required == null && res.required == null) null
+            else ((elem.required?.toSet() ?? Set()) + (res.required?.toSet() ?? Set())).toListing()
+          propertyNames = collateSchema(elem.propertyNames, res.propertyNames, "propertyNames")
+          minProperties = collateMin(elem.minProperties, res.minProperties) as UInt?
+          maxProperties = collateMax(elem.maxProperties, res.maxProperties) as UInt?
+
+          // array
+          items =
+            if (elem.items == null && res.items == null) null
+            else if ((res.items == null) != (elem.items == null)) elem.items ?? res.items
+            else if (res.items is JsonSchema && elem.items is JsonSchema) collateAllOf(new JsonSchema { allOf { res.items; elem.items as JsonSchema } })
+            else allOfErr("conflicting items lhs:'\(elem.items)' rhs:'\(res.items)'")
+          additionalItems = collateSchema(elem.additionalItems, res.additionalItems, "additionalItems")
+          minItems = collateMin(elem.minItems, res.minItems) as UInt?
+          maxItems = collateMax(elem.maxItems, res.maxItems) as UInt?
+          uniqueItems =
+            if (elem.uniqueItems == true || res.uniqueItems == true) true
+            else elem.uniqueItems ?? res.uniqueItems
+
+          // composition
+          oneOf = collateSet(elem.oneOf, res.oneOf)
+          anyOf = collateSet(elem.anyOf, res.anyOf)
+          not = collateSchema(elem.not, res.not, "not")
+        }
+  )
+
+local function collateSet(lhs: Listing<JsonSchema.Schema>(!isEmpty)?, rhs: Listing<JsonSchema.Schema>(!isEmpty)?): Listing<JsonSchema.Schema>(!isEmpty)? =
+  if (lhs != null && rhs != null)
+    let (intersection = lhs.toSet().intersect(rhs.toSet()))
+      if (intersection.isEmpty) allOfErr("conflicting oneOf")
+      else intersection.toListing()
+  else lhs ?? rhs
+
+local function collateProperties(lhs: Mapping<String(isRegex), JsonSchema.Schema>?, rhs: Mapping<String(isRegex), JsonSchema.Schema>?): Mapping<String(isRegex), JsonSchema.Schema>? =
+  if (lhs != null && rhs != null)
+    new {
+      for (key in lhs.keys + rhs.keys) {
+        when (lhs.containsKey(key) && rhs.containsKey(key)) {
+          [key] = collateAllOf(new JsonSchema { allOf { lhs[key]; rhs[key] } })
+        } else {
+          [key] = lhs.getOrNull(key) ?? rhs[key]
+        }
+      }
+    }
+  else lhs ?? rhs
+
+local function collateSchema(lhs: JsonSchema.Schema?, rhs: JsonSchema.Schema?, fieldName: String): JsonSchema.Schema? =
+  if (lhs != null && rhs != null)
+    if (lhs is Boolean && rhs is Boolean) lhs || rhs
+    else if (lhs is JsonSchema && rhs is JsonSchema) collateAllOf(new JsonSchema { allOf { lhs; rhs } })
+    else allOfErr("multiple non-JsonSchema \(fieldName)")
+  else lhs ?? rhs
+
+local function collateMin(lhs: Number?, rhs: Number?): Number? =
+  if (lhs != null && rhs != null) math.max(lhs, rhs)
+  else lhs ?? rhs
+
+local function collateMax(lhs: Number?, rhs: Number?): Number? =
+  if (lhs != null && rhs != null) math.min(lhs, rhs)
+  else lhs ?? rhs
+
+local function collateInformation(lhs: String?, rhs: String?): String? =
+  if (lhs != null && rhs != null) "\(lhs)\n----\n\(rhs)"
+  else lhs ?? rhs
+
+local function collateMetadataBoolean(lhs: Boolean?, rhs: Boolean?, fieldName: String): Boolean? =
+  if (lhs != rhs) allOfErr("conflicting \(fieldName)")
+  else lhs

packages/org.json_schema.contrib/ref.pkl

@@ -85,7 +85,15 @@ function resolveRef(baseUri: URI, schema: JsonSchema): JsonSchema.Schema? =
   let (parts = parseJsonPointer(ref.fragment ?? ""))
     if (isSameDocument(baseUri, ref))
       resolveRefImpl(schema.$$baseSchema, parts.drop(1), ref)
-        .ifNonNull((it) -> utils.mergeSchemas((schema) { $ref = null }, List(it)))
+        .ifNonNull((it) ->
+          (utils.mergeSchemas(
+            (schema) { $ref = null },
+            List(it as JsonSchema.Schema)
+          )) {
+            // store the original content of the referent for subsequent typeNames lookup in generateTypeNode
+            _inline_ { ["__ref_orig__"] = it }
+          }
+        )
     // Otherwise, we need to figure out the correct relative path.
     // This is done by resolving [ref] against the base URI.
     // If the resolved URI is a different document, it is read and parsed first.
@@ -100,4 +108,12 @@ function resolveRef(baseUri: URI, schema: JsonSchema): JsonSchema.Schema? =
             $$baseSchema = this
           })
             resolveRefImpl(parsedJsonSchema, parts.drop(1), ref)
-              .ifNonNull((it) -> utils.mergeSchemas((schema) { $ref = null }, List(it)))
+              .ifNonNull((it) ->
+                (utils.mergeSchemas(
+                  (schema) { $ref = null },
+                  List(it as JsonSchema.Schema)
+                )) {
+                  // store the original content of the referent for subsequent typeNames lookup in generateTypeNode
+                  _inline_ { ["__ref_orig__"] = it }
+                }
+              )

packages/org.json_schema.contrib/tests/ModuleGenerator.pkl

@@ -18,6 +18,7 @@ module org.json_schema.contrib.tests.ModuleGenerator
 amends "pkl:test"
 
 import "@jsonschema/JsonSchema.pkl"
+import "@jsonschema/Parser.pkl"
 import "../internal/ModuleGenerator.pkl"
 
 examples {
@@ -366,4 +367,8 @@ examples {
     }
     new ModuleGenerator { rootSchema = schema; moduleName = "com.apple.Example" }.moduleNode.render("")
   }
+  ["allOf"] {
+    local schema = Parser.parse(read("./test_allOf.json")) as JsonSchema
+    new ModuleGenerator { rootSchema = schema; moduleName = "com.apple.Example" }.moduleNode.render("")
+  }
 }