Skip to content

Commit

Permalink
Rewrite lexer, parser, and evaluator
Browse files Browse the repository at this point in the history 
This rewrites the lexer, parser, and evaluator to have *much* simpler
code, as well as better error reporting and other things generally considered
nice in modern programming language implementations.
  • Loading branch information
@pontaoski
pontaoski committed Aug 23, 2022
1 parent c67a1b0 commit 18147c8
Show file tree
Hide file tree
Showing 29 changed files with 1,980 additions and 3,171 deletions.
82 changes: 3 additions & 79 deletions Sources/LeafKit/LeafAST.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,91 +10,15 @@ public struct LeafAST: Hashable {
// MARK: - Internal/Private Only
let name: String

init(name: String, ast: [Syntax]) {
init(name: String, ast: [Statement]) {
self.name = name
self.ast = ast
self.rawAST = nil
self.flat = false

updateRefs([:])
}

init(from: LeafAST, referencing externals: [String: LeafAST]) {
self.name = from.name
self.ast = from.ast
self.rawAST = from.rawAST
self.externalRefs = from.externalRefs
self.unresolvedRefs = from.unresolvedRefs
self.flat = from.flat

updateRefs(externals)
}
internal private(set) var ast: [Statement]

internal private(set) var ast: [Syntax]
internal private(set) var externalRefs = Set<String>()
internal private(set) var unresolvedRefs = Set<String>()
internal private(set) var flat: Bool

// MARK: - Private Only

private var rawAST: [Syntax]?

mutating private func updateRefs(_ externals: [String: LeafAST]) {
var firstRun = false
if rawAST == nil, flat == false { rawAST = ast; firstRun = true }
unresolvedRefs.removeAll()
var pos = ast.startIndex

// inline provided externals
while pos < ast.endIndex {
// get desired externals for this Syntax - if none, continue
let wantedExts = ast[pos].externals()
if wantedExts.isEmpty {
pos = ast.index(after: pos)
continue
}
// see if we can provide any of them - if not, continue
let providedExts = externals.filter { wantedExts.contains($0.key) }
if providedExts.isEmpty {
unresolvedRefs.formUnion(wantedExts)
pos = ast.index(after: pos)
continue
}

// replace the original Syntax with the results of inlining, potentially 1...n
let replacementSyntax: [Syntax]
if case .extend(let extend) = ast[pos], let context = extend.context {
let inner = ast[pos].inlineRefs(providedExts, [:])
replacementSyntax = [.with(.init(context: context, body: inner))]
} else {
replacementSyntax = ast[pos].inlineRefs(providedExts, [:])
}
ast.replaceSubrange(pos...pos, with: replacementSyntax)
// any returned new inlined syntaxes can't be further resolved at this point
// but we need to add their unresolvable references to the global set
var offset = replacementSyntax.startIndex
while offset < replacementSyntax.endIndex {
unresolvedRefs.formUnion(ast[pos].externals())
offset = replacementSyntax.index(after: offset)
pos = ast.index(after: pos)
}
}

// compress raws
pos = ast.startIndex
while pos < ast.index(before: ast.endIndex) {
if case .raw(var syntax) = ast[pos] {
if case .raw(var add) = ast[ast.index(after: pos)] {
var buffer = ByteBufferAllocator().buffer(capacity: syntax.readableBytes + add.readableBytes)
buffer.writeBuffer(&syntax)
buffer.writeBuffer(&add)
ast[pos] = .raw(buffer)
ast.remove(at: ast.index(after: pos) )
} else { pos = ast.index(after: pos) }
} else { pos = ast.index(after: pos) }
}

flat = unresolvedRefs.isEmpty ? true : false
if firstRun && flat { rawAST = nil }
}
private var rawAST: [Statement]?
}
9 changes: 6 additions & 3 deletions Sources/LeafKit/LeafData/LeafData.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -259,6 +259,9 @@ public struct LeafData: CustomStringConvertible,
/// Try to convert one concrete object to a second type.
internal func convert(to output: NaturalType, _ level: DataConvertible = .castable) -> LeafData {
guard celf != output else { return self }
if self.isNil && output == .bool {
return .bool(false)
}
if case .lazy(let f,_,_) = self.storage { return f().convert(to: output, level) }
guard let input = storage.unwrap,
let conversion = _ConverterMap.symbols.get(input.concreteType!, output),
Expand Down Expand Up @@ -373,7 +376,7 @@ fileprivate enum _ConverterMap {
}),
// String == "true" || "false"
Converter(.string , .bool , is: .castable, via: {
($0 as? String).map { Bool($0) }?.map { .bool($0) } ?? .trueNil
($0 as? String).map { Bool($0) }?.map { .bool($0) } ?? (($0 as? String).map { .bool(!$0.isEmpty) } ?? .trueNil)
}),
// True = 1; False = 0
Converter(.bool , .double , is: .castable, via: {
Expand Down Expand Up @@ -415,9 +418,9 @@ fileprivate enum _ConverterMap {

// MARK: - .coercible (One-direction defined conversion)

// Array.isEmpty == truthiness
// !Array.isEmpty == truthiness
Converter(.array , .bool , is: .coercible, via: {
($0 as? [LeafData]).map { $0.isEmpty }.map { .bool($0) } ?? .trueNil
($0 as? [LeafData]).map { .bool(!$0.isEmpty) } ?? .trueNil
}),
// Data.isEmpty == truthiness
Converter(.data , .bool , is: .coercible, via: {
Expand Down
3 changes: 3 additions & 0 deletions Sources/LeafKit/LeafData/LeafDataRepresentable.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,9 @@ public protocol LeafDataRepresentable {
extension String: LeafDataRepresentable {
public var leafData: LeafData { .string(self) }
}
extension Substring: LeafDataRepresentable {
public var leafData: LeafData { .string(String(self)) }
}

extension FixedWidthInteger {
public var leafData: LeafData {
Expand Down
2 changes: 1 addition & 1 deletion Sources/LeafKit/LeafData/LeafDataStorage.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,7 @@ internal indirect enum LeafDataStorage: Equatable, CustomStringConvertible {
.dictionary(_) : data = try serialize()!.data(using: encoding)
case .data(let d) : data = d
}
guard let validData = data else { throw "Serialization Error" }
guard let validData = data else { throw LeafError(.serializationError) }
buffer.writeBytes(validData)
}

Expand Down
116 changes: 53 additions & 63 deletions Sources/LeafKit/LeafError.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
///
public struct LeafError: Error {
/// Possible cases of a LeafError.Reason, with applicable stored values where useful for the type
public enum Reason {
public enum Reason: Equatable {
// MARK: Errors related to loading raw templates
/// Attempted to access a template blocked for security reasons
case illegalAccess(String)
Expand Down Expand Up @@ -34,13 +34,43 @@ public struct LeafError: Error {

// MARK: Wrapped Errors related to Lexing or Parsing
/// Errors due to malformed template syntax or grammar
case lexerError(LexerError)
case lexerError(LeafScannerError)

// MARK: Errors lacking specificity
/// Errors from protocol adherents that do not support newer features
case unsupportedFeature(String)
/// Errors only when no existing error reason is adequately clear
case unknownError(String)

/// Errors when something goes wrong internally
case internalError(what: String)

/// Errors when an import is not found
case importNotFound(name: String)

/// Errors when a tag is not found
case tagNotFound(name: String)

/// Errors when one type was expected, but another was obtained
case typeError(shouldHaveBeen: LeafData.NaturalType, got: LeafData.NaturalType)

/// A typeError specialised for Double | Int
case expectedNumeric(got: LeafData.NaturalType)

/// A typeError specialised for binary operators of (T, T) -> T
case badOperation(on: LeafData.NaturalType, what: String)

/// Errors when a tag receives a bad parameter count
case badParameterCount(tag: String, expected: Int, got: Int)

/// Errors when a tag receives a body, but doesn't want one
case extraneousBody(tag: String)

/// Errors when a tag doesn't receive a body, but wants one
case missingBody(tag: String)

/// Serialization error
case serializationError
}

/// Source file name causing error
Expand Down Expand Up @@ -84,7 +114,27 @@ public struct LeafError: Error {
return "\(src) - \(key) cyclically referenced in [\(chain.joined(separator: " -> "))]"
case .lexerError(let e):
return "Lexing error - \(e.localizedDescription)"
}
case .importNotFound(let name):
return "Import \(name) was not found"
case .internalError(let what):
return "Something in Leaf broke: \(what)\nPlease report this to https://github.com/vapor/leaf-kit"
case .tagNotFound(let name):
return "Tag \(name) was not found"
case .typeError(let shouldHaveBeen, let got):
return "Type error: I was expecting \(shouldHaveBeen), but I got \(got) instead"
case .badOperation(let on, let what):
return "Type error: \(on) cannot do \(what)"
case .expectedNumeric(let got):
return "Type error: I was expecting a numeric type, but I got \(got) instead"
case .badParameterCount(let tag, let expected, let got):
return "Type error: \(tag) was expecting \(expected) parameters, but got \(got) parameters instead"
case .extraneousBody(let tag):
return "Type error: \(tag) wasn't expecting a body, but got one"
case .missingBody(let tag):
return "Type error: \(tag) was expecting a body, but didn't get one"
case .serializationError:
return "Serialization error"
}
}

/// Create a `LeafError` - only `reason` typically used as source locations are auto-grabbed
Expand All @@ -102,63 +152,3 @@ public struct LeafError: Error {
self.reason = reason
}
}

// MARK: - `LexerError` Summary (Wrapped by LeafError)

/// `LexerError` reports errors during the stage.
public struct LexerError: Error {
// MARK: - Public

public enum Reason {
// MARK: Errors occuring during Lexing
/// A character not usable in parameters is present when Lexer is not expecting it
case invalidParameterToken(Character)
/// A string was opened but never terminated by end of file
case unterminatedStringLiteral
/// Use in place of fatalError to indicate extreme issue
case unknownError(String)
}

/// Template source file line where error occured
public let line: Int
/// Template source column where error occured
public let column: Int
/// Name of template error occured in
public let name: String
/// Stated reason for error
public let reason: Reason

// MARK: - Internal Only

/// State of tokens already processed by Lexer prior to error
internal let lexed: [LeafToken]
/// Flag to true if lexing error is something that may be recoverable during parsing;
/// EG, `"#anhtmlanchor"` may lex as a tag name but fail to tokenize to tag because it isn't
/// followed by a left paren. Parser may be able to recover by decaying it to `.raw`.
internal let recoverable: Bool

/// Create a `LexerError`
/// - Parameters:
/// - reason: The specific reason for the error
/// - src: File being lexed
/// - lexed: `LeafTokens` already lexed prior to error
/// - recoverable: Flag to say whether the error can potentially be recovered during Parse
internal init(
_ reason: Reason,
src: LeafRawTemplate,
lexed: [LeafToken] = [],
recoverable: Bool = false
) {
self.line = src.line
self.column = src.column
self.reason = reason
self.lexed = lexed
self.name = src.name
self.recoverable = recoverable
}

/// Convenience description of source file name, error reason, and location in file of error source
var localizedDescription: String {
return "\"\(name)\": \(reason) - \(line):\(column)"
}
}
Loading

0 comments on commit 18147c8

Please sign in to comment.