Skip to content

Commit 4d7efb9

Browse files
Rewrite lexer, parser, and evaluator
This rewrites the lexer, parser, and evaluator to have *much* simpler code, as well as better error reporting and other things generally considered nice in modern programming language implementations.
1 parent c67a1b0 commit 4d7efb9

28 files changed

+1957
-3148
lines changed

Sources/LeafKit/LeafAST.swift

Lines changed: 3 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -10,91 +10,15 @@ public struct LeafAST: Hashable {
1010
// MARK: - Internal/Private Only
1111
let name: String
1212

13-
init(name: String, ast: [Syntax]) {
13+
init(name: String, ast: [Statement]) {
1414
self.name = name
1515
self.ast = ast
1616
self.rawAST = nil
17-
self.flat = false
18-
19-
updateRefs([:])
2017
}
21-
22-
init(from: LeafAST, referencing externals: [String: LeafAST]) {
23-
self.name = from.name
24-
self.ast = from.ast
25-
self.rawAST = from.rawAST
26-
self.externalRefs = from.externalRefs
27-
self.unresolvedRefs = from.unresolvedRefs
28-
self.flat = from.flat
2918

30-
updateRefs(externals)
31-
}
19+
internal private(set) var ast: [Statement]
3220

33-
internal private(set) var ast: [Syntax]
34-
internal private(set) var externalRefs = Set<String>()
35-
internal private(set) var unresolvedRefs = Set<String>()
36-
internal private(set) var flat: Bool
37-
3821
// MARK: - Private Only
3922

40-
private var rawAST: [Syntax]?
41-
42-
mutating private func updateRefs(_ externals: [String: LeafAST]) {
43-
var firstRun = false
44-
if rawAST == nil, flat == false { rawAST = ast; firstRun = true }
45-
unresolvedRefs.removeAll()
46-
var pos = ast.startIndex
47-
48-
// inline provided externals
49-
while pos < ast.endIndex {
50-
// get desired externals for this Syntax - if none, continue
51-
let wantedExts = ast[pos].externals()
52-
if wantedExts.isEmpty {
53-
pos = ast.index(after: pos)
54-
continue
55-
}
56-
// see if we can provide any of them - if not, continue
57-
let providedExts = externals.filter { wantedExts.contains($0.key) }
58-
if providedExts.isEmpty {
59-
unresolvedRefs.formUnion(wantedExts)
60-
pos = ast.index(after: pos)
61-
continue
62-
}
63-
64-
// replace the original Syntax with the results of inlining, potentially 1...n
65-
let replacementSyntax: [Syntax]
66-
if case .extend(let extend) = ast[pos], let context = extend.context {
67-
let inner = ast[pos].inlineRefs(providedExts, [:])
68-
replacementSyntax = [.with(.init(context: context, body: inner))]
69-
} else {
70-
replacementSyntax = ast[pos].inlineRefs(providedExts, [:])
71-
}
72-
ast.replaceSubrange(pos...pos, with: replacementSyntax)
73-
// any returned new inlined syntaxes can't be further resolved at this point
74-
// but we need to add their unresolvable references to the global set
75-
var offset = replacementSyntax.startIndex
76-
while offset < replacementSyntax.endIndex {
77-
unresolvedRefs.formUnion(ast[pos].externals())
78-
offset = replacementSyntax.index(after: offset)
79-
pos = ast.index(after: pos)
80-
}
81-
}
82-
83-
// compress raws
84-
pos = ast.startIndex
85-
while pos < ast.index(before: ast.endIndex) {
86-
if case .raw(var syntax) = ast[pos] {
87-
if case .raw(var add) = ast[ast.index(after: pos)] {
88-
var buffer = ByteBufferAllocator().buffer(capacity: syntax.readableBytes + add.readableBytes)
89-
buffer.writeBuffer(&syntax)
90-
buffer.writeBuffer(&add)
91-
ast[pos] = .raw(buffer)
92-
ast.remove(at: ast.index(after: pos) )
93-
} else { pos = ast.index(after: pos) }
94-
} else { pos = ast.index(after: pos) }
95-
}
96-
97-
flat = unresolvedRefs.isEmpty ? true : false
98-
if firstRun && flat { rawAST = nil }
99-
}
23+
private var rawAST: [Statement]?
10024
}

Sources/LeafKit/LeafData/LeafData.swift

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -259,6 +259,9 @@ public struct LeafData: CustomStringConvertible,
259259
/// Try to convert one concrete object to a second type.
260260
internal func convert(to output: NaturalType, _ level: DataConvertible = .castable) -> LeafData {
261261
guard celf != output else { return self }
262+
if self.isNil && output == .bool {
263+
return .bool(false)
264+
}
262265
if case .lazy(let f,_,_) = self.storage { return f().convert(to: output, level) }
263266
guard let input = storage.unwrap,
264267
let conversion = _ConverterMap.symbols.get(input.concreteType!, output),
@@ -373,7 +376,7 @@ fileprivate enum _ConverterMap {
373376
}),
374377
// String == "true" || "false"
375378
Converter(.string , .bool , is: .castable, via: {
376-
($0 as? String).map { Bool($0) }?.map { .bool($0) } ?? .trueNil
379+
($0 as? String).map { Bool($0) }?.map { .bool($0) } ?? (($0 as? String).map { .bool(!$0.isEmpty) } ?? .trueNil)
377380
}),
378381
// True = 1; False = 0
379382
Converter(.bool , .double , is: .castable, via: {
@@ -415,9 +418,9 @@ fileprivate enum _ConverterMap {
415418

416419
// MARK: - .coercible (One-direction defined conversion)
417420

418-
// Array.isEmpty == truthiness
421+
// !Array.isEmpty == truthiness
419422
Converter(.array , .bool , is: .coercible, via: {
420-
($0 as? [LeafData]).map { $0.isEmpty }.map { .bool($0) } ?? .trueNil
423+
($0 as? [LeafData]).map { .bool(!$0.isEmpty) } ?? .trueNil
421424
}),
422425
// Data.isEmpty == truthiness
423426
Converter(.data , .bool , is: .coercible, via: {

Sources/LeafKit/LeafData/LeafDataRepresentable.swift

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,9 @@ public protocol LeafDataRepresentable {
1111
extension String: LeafDataRepresentable {
1212
public var leafData: LeafData { .string(self) }
1313
}
14+
extension Substring: LeafDataRepresentable {
15+
public var leafData: LeafData { .string(String(self)) }
16+
}
1417

1518
extension FixedWidthInteger {
1619
public var leafData: LeafData {

Sources/LeafKit/LeafData/LeafDataStorage.swift

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -146,7 +146,7 @@ internal indirect enum LeafDataStorage: Equatable, CustomStringConvertible {
146146
.dictionary(_) : data = try serialize()!.data(using: encoding)
147147
case .data(let d) : data = d
148148
}
149-
guard let validData = data else { throw "Serialization Error" }
149+
guard let validData = data else { throw LeafError(.serializationError) }
150150
buffer.writeBytes(validData)
151151
}
152152

Sources/LeafKit/LeafError.swift

Lines changed: 53 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
///
66
public struct LeafError: Error {
77
/// Possible cases of a LeafError.Reason, with applicable stored values where useful for the type
8-
public enum Reason {
8+
public enum Reason: Equatable {
99
// MARK: Errors related to loading raw templates
1010
/// Attempted to access a template blocked for security reasons
1111
case illegalAccess(String)
@@ -34,13 +34,43 @@ public struct LeafError: Error {
3434

3535
// MARK: Wrapped Errors related to Lexing or Parsing
3636
/// Errors due to malformed template syntax or grammar
37-
case lexerError(LexerError)
37+
case lexerError(LeafScannerError)
3838

3939
// MARK: Errors lacking specificity
4040
/// Errors from protocol adherents that do not support newer features
4141
case unsupportedFeature(String)
4242
/// Errors only when no existing error reason is adequately clear
4343
case unknownError(String)
44+
45+
/// Errors when something goes wrong internally
46+
case internalError(what: String)
47+
48+
/// Errors when an import is not found
49+
case importNotFound(name: String)
50+
51+
/// Errors when a tag is not found
52+
case tagNotFound(name: String)
53+
54+
/// Errors when one type was expected, but another was obtained
55+
case typeError(shouldHaveBeen: LeafData.NaturalType, got: LeafData.NaturalType)
56+
57+
/// A typeError specialised for Double | Int
58+
case expectedNumeric(got: LeafData.NaturalType)
59+
60+
/// A typeError specialised for binary operators of (T, T) -> T
61+
case badOperation(on: LeafData.NaturalType, what: String)
62+
63+
/// Errors when a tag receives a bad parameter count
64+
case badParameterCount(tag: String, expected: Int, got: Int)
65+
66+
/// Errors when a tag receives a body, but doesn't want one
67+
case extraneousBody(tag: String)
68+
69+
/// Errors when a tag doesn't receive a body, but wants one
70+
case missingBody(tag: String)
71+
72+
/// Serialization error
73+
case serializationError
4474
}
4575

4676
/// Source file name causing error
@@ -84,7 +114,27 @@ public struct LeafError: Error {
84114
return "\(src) - \(key) cyclically referenced in [\(chain.joined(separator: " -> "))]"
85115
case .lexerError(let e):
86116
return "Lexing error - \(e.localizedDescription)"
87-
}
117+
case .importNotFound(let name):
118+
return "Import \(name) was not found"
119+
case .internalError(let what):
120+
return "Something in Leaf broke: \(what)\nPlease report this to https://github.com/vapor/leaf-kit"
121+
case .tagNotFound(let name):
122+
return "Tag \(name) was not found"
123+
case .typeError(let shouldHaveBeen, let got):
124+
return "Type error: I was expecting \(shouldHaveBeen), but I got \(got) instead"
125+
case .badOperation(let on, let what):
126+
return "Type error: \(on) cannot do \(what)"
127+
case .expectedNumeric(let got):
128+
return "Type error: I was expecting a numeric type, but I got \(got) instead"
129+
case .badParameterCount(let tag, let expected, let got):
130+
return "Type error: \(tag) was expecting \(expected) parameters, but got \(got) parameters instead"
131+
case .extraneousBody(let tag):
132+
return "Type error: \(tag) wasn't expecting a body, but got one"
133+
case .missingBody(let tag):
134+
return "Type error: \(tag) was expecting a body, but didn't get one"
135+
case .serializationError:
136+
return "Serialization error"
137+
}
88138
}
89139

90140
/// Create a `LeafError` - only `reason` typically used as source locations are auto-grabbed
@@ -102,63 +152,3 @@ public struct LeafError: Error {
102152
self.reason = reason
103153
}
104154
}
105-
106-
// MARK: - `LexerError` Summary (Wrapped by LeafError)
107-
108-
/// `LexerError` reports errors during the stage.
109-
public struct LexerError: Error {
110-
// MARK: - Public
111-
112-
public enum Reason {
113-
// MARK: Errors occuring during Lexing
114-
/// A character not usable in parameters is present when Lexer is not expecting it
115-
case invalidParameterToken(Character)
116-
/// A string was opened but never terminated by end of file
117-
case unterminatedStringLiteral
118-
/// Use in place of fatalError to indicate extreme issue
119-
case unknownError(String)
120-
}
121-
122-
/// Template source file line where error occured
123-
public let line: Int
124-
/// Template source column where error occured
125-
public let column: Int
126-
/// Name of template error occured in
127-
public let name: String
128-
/// Stated reason for error
129-
public let reason: Reason
130-
131-
// MARK: - Internal Only
132-
133-
/// State of tokens already processed by Lexer prior to error
134-
internal let lexed: [LeafToken]
135-
/// Flag to true if lexing error is something that may be recoverable during parsing;
136-
/// EG, `"#anhtmlanchor"` may lex as a tag name but fail to tokenize to tag because it isn't
137-
/// followed by a left paren. Parser may be able to recover by decaying it to `.raw`.
138-
internal let recoverable: Bool
139-
140-
/// Create a `LexerError`
141-
/// - Parameters:
142-
/// - reason: The specific reason for the error
143-
/// - src: File being lexed
144-
/// - lexed: `LeafTokens` already lexed prior to error
145-
/// - recoverable: Flag to say whether the error can potentially be recovered during Parse
146-
internal init(
147-
_ reason: Reason,
148-
src: LeafRawTemplate,
149-
lexed: [LeafToken] = [],
150-
recoverable: Bool = false
151-
) {
152-
self.line = src.line
153-
self.column = src.column
154-
self.reason = reason
155-
self.lexed = lexed
156-
self.name = src.name
157-
self.recoverable = recoverable
158-
}
159-
160-
/// Convenience description of source file name, error reason, and location in file of error source
161-
var localizedDescription: String {
162-
return "\"\(name)\": \(reason) - \(line):\(column)"
163-
}
164-
}

0 commit comments

Comments
 (0)