diff --git a/lex.js b/lex.js index d76f94b..8dbe134 100644 --- a/lex.js +++ b/lex.js @@ -1,3 +1,5 @@ +require('./utils.js'); + const TokenType = Object.freeze({ Word:1, NumberLit:2, StringLit:3, Op:4 }); class TokenStream @@ -18,6 +20,11 @@ this.savedLocations = []; } + getPrettyTokenList() + { + return this.tokens.mapKey("type", v => TokenType.keyName(v)); + } + eof() { return this.index < 0 || this.index >= this.tokens.length; @@ -40,7 +47,7 @@ { if (token.value !== value) { - return { error: { msg: "Expecting " . TokenType.keyName(type) + " '" + value + "', not '" + token.value + "'", ...this.tokenToLinePos(token) } }; + return { error: { msg: "Expecting " + TokenType.keyName(type) + " '" + value + "', not '" + token.value + "'", ...this.tokenToLinePos(token) } }; } } } @@ -51,45 +58,45 @@ consume(type = undefined, value = undefined) { const token = this.peek(type, value); - if (token.error) return error; + if (token.error) return token.error; this.index++; return token; } - beginMatch() + startMatch() { this.savedLocations.push(this.index); } - doneMatch() + finishMatch() { - console.assert(this.savedLocations.length > 0, "Unexpected doneMatch()"); + console.assert(this.savedLocations.length > 0, "Unexpected finishMatch()"); this.savedLocations.pop(); } - failedMatch() + failMatch() { - console.assert(this.savedLocations.length > 0, "Unexpected failedMatch()"); + console.assert(this.savedLocations.length > 0, "Unexpected failMatch()"); this.index = this.savedLocations.pop(); } - stringIndexToLinePos(index) + stringPosToLinePos(pos) { - index = Math.max(0, Math.min(this.inputLength - 1, index)); + pos = Math.max(0, Math.min(this.inputLength - 1, pos)); let line = this.newlines.length + 1; let previousLinesLength = 0; for (let i = 0; i < this.newlines.length; i++) { - if (index <= this.newlines[i]) + if (pos <= this.newlines[i]) { line = i + 1; break; } previousLinesLength += this.newlines[i] + 1; } - return {line:line, pos:index - previousLinesLength + 1}; + return {line:line, pos:pos - previousLinesLength + 1}; } tokenIndexToLinePos(index) @@ -101,7 +108,7 @@ tokenToLinePos(token) { - return this.tokenIndexToLinePos(token.index); + return this.stringPosToLinePos(token.pos); } } @@ -130,7 +137,7 @@ let parts = []; // track where we are in the original string - let index = 0; + let pos = 0; while (input.length > 0) { // match our possible sequences of more than one character (words, digits, or whitespace) @@ -139,19 +146,19 @@ (match = input.match(/^\s+/))) { const matched = match[0]; - parts.push({token:matched, index:index}); + parts.push({token:matched, pos:pos}); // advance the string input = input.substr(matched.length); - index += matched.length; + pos += matched.length; } else { // everything else is a single character (punctuation) - parts.push({token:input[0], index:index}); + parts.push({token:input[0], pos:pos}); input = input.substr(1); - index++; + pos++; } } @@ -161,7 +168,7 @@ { const partsLeft = parts.length - i - 1; - let { token, index } = parts[i]; + let { token, pos } = parts[i]; if (token.trim().length == 0) { @@ -176,7 +183,7 @@ { token += "." + parts[i + 2].token; } - tokens.push({type:TokenType.NumberLit, token:token, index:index}); + tokens.push({type:TokenType.NumberLit, token:token, pos:pos}); } else if (token == '"') { @@ -224,17 +231,17 @@ throw "Unclosed string"; } - tokens.push({type:TokenType.StringLit, token:token, index:index}); + tokens.push({type:TokenType.StringLit, token:token, pos:pos}); } else if (token.match(/\w+/)) { // any keyword or identifier - tokens.push({type:TokenType.Word, token:token, index:index}); + tokens.push({type:TokenType.Word, token:token, pos:pos}); } else { // anything left is an operator - tokens.push({type:TokenType.Op, token:token, index:index}); + tokens.push({type:TokenType.Op, token:token, pos:pos}); } } diff --git a/lex.js b/lex.js index d76f94b..8dbe134 100644 --- a/lex.js +++ b/lex.js @@ -1,3 +1,5 @@ +require('./utils.js'); + const TokenType = Object.freeze({ Word:1, NumberLit:2, StringLit:3, Op:4 }); class TokenStream @@ -18,6 +20,11 @@ this.savedLocations = []; } + getPrettyTokenList() + { + return this.tokens.mapKey("type", v => TokenType.keyName(v)); + } + eof() { return this.index < 0 || this.index >= this.tokens.length; @@ -40,7 +47,7 @@ { if (token.value !== value) { - return { error: { msg: "Expecting " . TokenType.keyName(type) + " '" + value + "', not '" + token.value + "'", ...this.tokenToLinePos(token) } }; + return { error: { msg: "Expecting " + TokenType.keyName(type) + " '" + value + "', not '" + token.value + "'", ...this.tokenToLinePos(token) } }; } } } @@ -51,45 +58,45 @@ consume(type = undefined, value = undefined) { const token = this.peek(type, value); - if (token.error) return error; + if (token.error) return token.error; this.index++; return token; } - beginMatch() + startMatch() { this.savedLocations.push(this.index); } - doneMatch() + finishMatch() { - console.assert(this.savedLocations.length > 0, "Unexpected doneMatch()"); + console.assert(this.savedLocations.length > 0, "Unexpected finishMatch()"); this.savedLocations.pop(); } - failedMatch() + failMatch() { - console.assert(this.savedLocations.length > 0, "Unexpected failedMatch()"); + console.assert(this.savedLocations.length > 0, "Unexpected failMatch()"); this.index = this.savedLocations.pop(); } - stringIndexToLinePos(index) + stringPosToLinePos(pos) { - index = Math.max(0, Math.min(this.inputLength - 1, index)); + pos = Math.max(0, Math.min(this.inputLength - 1, pos)); let line = this.newlines.length + 1; let previousLinesLength = 0; for (let i = 0; i < this.newlines.length; i++) { - if (index <= this.newlines[i]) + if (pos <= this.newlines[i]) { line = i + 1; break; } previousLinesLength += this.newlines[i] + 1; } - return {line:line, pos:index - previousLinesLength + 1}; + return {line:line, pos:pos - previousLinesLength + 1}; } tokenIndexToLinePos(index) @@ -101,7 +108,7 @@ tokenToLinePos(token) { - return this.tokenIndexToLinePos(token.index); + return this.stringPosToLinePos(token.pos); } } @@ -130,7 +137,7 @@ let parts = []; // track where we are in the original string - let index = 0; + let pos = 0; while (input.length > 0) { // match our possible sequences of more than one character (words, digits, or whitespace) @@ -139,19 +146,19 @@ (match = input.match(/^\s+/))) { const matched = match[0]; - parts.push({token:matched, index:index}); + parts.push({token:matched, pos:pos}); // advance the string input = input.substr(matched.length); - index += matched.length; + pos += matched.length; } else { // everything else is a single character (punctuation) - parts.push({token:input[0], index:index}); + parts.push({token:input[0], pos:pos}); input = input.substr(1); - index++; + pos++; } } @@ -161,7 +168,7 @@ { const partsLeft = parts.length - i - 1; - let { token, index } = parts[i]; + let { token, pos } = parts[i]; if (token.trim().length == 0) { @@ -176,7 +183,7 @@ { token += "." + parts[i + 2].token; } - tokens.push({type:TokenType.NumberLit, token:token, index:index}); + tokens.push({type:TokenType.NumberLit, token:token, pos:pos}); } else if (token == '"') { @@ -224,17 +231,17 @@ throw "Unclosed string"; } - tokens.push({type:TokenType.StringLit, token:token, index:index}); + tokens.push({type:TokenType.StringLit, token:token, pos:pos}); } else if (token.match(/\w+/)) { // any keyword or identifier - tokens.push({type:TokenType.Word, token:token, index:index}); + tokens.push({type:TokenType.Word, token:token, pos:pos}); } else { // anything left is an operator - tokens.push({type:TokenType.Op, token:token, index:index}); + tokens.push({type:TokenType.Op, token:token, pos:pos}); } } diff --git a/parse.js b/parse.js index 86ab105..2883b7e 100644 --- a/parse.js +++ b/parse.js @@ -1,5 +1,5 @@ -const lex = require('lex'); -const utils = require('utils'); +const lex = require('./lex.js'); +const utils = require('./utils.js'); /* How powerfully infix operators bind to their left and right arguments @@ -68,7 +68,7 @@ expression(tokenStream) { - return expressionInternal(tokenStream, 0); + return this.expressionInternal(tokenStream, 0); } /* Parses an expression consisting of a set of values, all separated by infix operators, @@ -91,41 +91,45 @@ expressionInternal(minBindingPower) { // grab the type of the next token, and its string representation - const {type, token, index:tokenIndex, error} = this.tokenStream.consume(); + let { type, token, pos, error } = this.tokenStream.consume(); if (error) return this.getError(error); // start parsing the left-hand-side of the next operator at this AST node let lhs = null; if (type == lex.TokenType.Op) { + let power; + // the value of this part of the expression starts with an operator // it will either be prefix operator, or a bracketed expression if (token == "(") { // parse a subexpression - const { expr, error } = this.attemptSubtree(this.expression); + let { expr, error } = this.attemptSubtree(this.expression); if (error) return this.getError(error); lhs = expr; // ensure the next token after the sub expression is a closing paren, and consume it - const { error } = tokens.consume(lex.TokenType.Op, ")"); - if (error) return this.getError(error); + if (error = tokens.consume(lex.TokenType.Op, ")").error) + { + return this.getError(error); + } } else if ((power = PrefixBindingPower[token]) !== undefined) { // this is a prefix operator of the given power // get the expression it applies to... - const { rhs, error } = this.attemptSubtree(expressionInternal, power); + let { rhs, error } = this.attemptSubtree(expressionInternal, power); if (error) return this.getError(error); // ... and generate an AST node for it // this becomes the new LHS of the operator we are parsing - lhs = {node:ASTNode.PrefixOp, rhs:rhs, op:token}; + lhs = { node:ASTNode.PrefixOp, rhs:rhs, op:token }; } else { - return this.getError({ msg: "Invalid Op '" + token + "'", ...this.tokenStream.stringIndexToLinePos(tokenIndex) }); + return this.getError({ msg: "Invalid Op '" + token + "'", ...this.tokenStream.stringPosToLinePos(pos) }); } } else if (type == lex.TokenType.Word) @@ -133,7 +137,7 @@ if (Keywords.includes(token)) { // TODO: handle keywords that can go in expressions - return this.getError({ msg: "Keyword '" + token + "' not expected", ...this.tokenStream.stringIndexToLinePos(tokenIndex) }); + return this.getError({ msg: "Keyword '" + token + "' not expected", ...this.tokenStream.stringPosToLinePos(pos) }); } else if (Literals[token] !== undefined) { @@ -143,20 +147,20 @@ else { // some unresolved identifier (this is the domain of the runtime/interpreter!) - lhs = {node:ASTNode.Identifier, value:token}; + lhs = { node:ASTNode.Identifier, value:token }; } } - else if (type == TokenType.NumberLit) + else if (type == lex.TokenType.NumberLit) { lhs = {node:ASTNode.Literal, value:parseFloat(token)}; } - else if (type == TokenType.StringLit) + else if (type == lex.TokenType.StringLit) { lhs = {node:ASTNode.Literal, value:token}; } else { - return this.getError({ msg: "Unexpected " + lex.TokenType.keyName(type) + " of '" + token + "'", ...this.tokenStream.stringIndexToLinePos(tokenIndex) }); + return this.getError({ msg: "Unexpected " + lex.TokenType.keyName(type) + " of '" + token + "'", ...this.tokenStream.stringPosToLinePos(pos) }); } while (!this.tokenStream.eof()) @@ -168,7 +172,7 @@ * Depending on its precedence, we may not actually get to parse it ourselves, * so don't actually consume the token yet, just peek it (just in case) */ - const {type:opType, token:opToken, index:opIndex, error} = this.tokenStream.peek(lex.TokenType.Op); + let { type:opType, token:opToken, pos:opPos, error } = this.tokenStream.peek(lex.TokenType.Op); if (error) return this.getError(error); if (PostfixBindingPower[opToken] !== undefined) @@ -189,15 +193,17 @@ * We need to parse the index expression. Note we reset the binding power, * this is because we will always end with ], so we don't need to worry about our current precedence */ - const { index, error } = this.attemptSubtree(this.expression); + let { index, error } = this.attemptSubtree(this.expression); if (error) return this.getError(error); // parse the closing ] - const { error } = this.tokenStream.consume(lex.TokenType.Op, "]"); - if (error) return this.getError(error); + if (error = this.tokenStream.consume(lex.TokenType.Op, "]").error) + { + return this.getError(error); + } // our new node is the old lhs indexed with the list of argument expressions - lhs = {node:ASTNode.Index, lhs:lhs, index:index}; + lhs = { node:ASTNode.Index, lhs:lhs, index:index }; } else if (opToken == "(") { @@ -208,13 +214,11 @@ let args = []; while (true) { - if (tokens.length == 0) throw "Unclosed argument list, expected ',' or ')' at " + JSON.stringify(stringIndexToLinePos(inputLength - 1, newlines)); - // peek the next op - const {token:nextToken, type:nextType, index:nextIndex, error} = this.tokenStream.peek(); + let { token:nextToken, type:nextType, pos:nextPos, error } = this.tokenStream.peek(); if (error) return this.getError(error); - if (nextType == TokenType.Op && nextToken == ")") + if (nextType == lex.TokenType.Op && nextToken == ")") { // no more arguments, we should end the call (after consuming the bracket) this.tokenStream.consume(); @@ -223,15 +227,20 @@ if (args.length > 0) { - const { error } = this.tokenStream.consume(lex.TokenType.Op, ","); - if (error) return this.getError(error); + if (error = this.tokenStream.consume(lex.TokenType.Op, ",").error) + { + return this.getError(error); + } } - const { arg, error } = this.attemptSubtree(this.expression); - if (error) return this.getError(error); + { + // avoid re-declaring error... annoying + let { arg, error } = this.attemptSubtree(this.expression); + if (error) return this.getError(error); - // our new node is the old lhs as a function, called with the index expression - args.push(arg); + // our new node is the old lhs as a function, called with the index expression + args.push(arg); + } } // generate the AST node @@ -239,7 +248,7 @@ } else { - return this.getError({ msg: "Unexpected postfix Op '" + opToken + "'", ...this.tokenStream.stringIndexToLinePos(opIndex) }); + return this.getError({ msg: "Unexpected postfix Op '" + opToken + "'", ...this.tokenStream.stringPosToLinePos(opPos) }); } // after parsing a postfix operator, we are expecting an operator again @@ -261,49 +270,33 @@ //... otherwise, this operator is for us, so consume it this.tokenStream.consume(); - // the next expression is the right side of our current operator, but only as much as the - // right-side binding power of our operator can grab - const { rhs, error } = this.attemptSubtree(this.expressionInternal, rightBinding); - if (error) return this.getError(error); + { + // the next expression is the right side of our current operator, but only as much as the + // right-side binding power of our operator can grab + let { rhs, error } = this.attemptSubtree(this.expressionInternal, rightBinding); + if (error) return this.getError(error); - // our new lhs becomes our applied operator, - // leaving us free to match more operators of an appropriate precedence - lhs = {node:ASTNode.InfixOp, lhs:lhs, rhs:rhs, op:opToken}; + // our new lhs becomes our applied operator, + // leaving us free to match more operators of an appropriate precedence + lhs = {node:ASTNode.InfixOp, lhs:lhs, rhs:rhs, op:opToken}; + } } return { ast: lhs, error: null }; } - error(msg, at) - { - let loc = this.tokenStream.tokenIndexToLinePos(this.tokenStream.index); - if (at) - { - if (at.index) - { - loc = this.tokenStream.tokenToLinePos(at); - } - else - { - loc = this.tokenStream.tokenIndexToLinePos(at); - } - } - return { ast:null, error: { msg: msg, ...loc } }; - } - - attemptSubtree(parseFunction, ...args) { - this.tokenStream.attemptMatch(); - const subtree = parseFunction.apply(this, [tokenStream, ...args]); + this.tokenStream.startMatch(); + const subtree = parseFunction.apply(this, [ this.tokenStream, ...args ]); if (subtree.error) { - this.tokenStream.failedMatch(); + this.tokenStream.failMatch(); } else { - this.tokenSTream.doneMatch(); + this.tokenStream.finishMatch(); } return subtree; @@ -316,3 +309,4 @@ } +module.exports = { Parser: Parser }; diff --git a/lex.js b/lex.js index d76f94b..8dbe134 100644 --- a/lex.js +++ b/lex.js @@ -1,3 +1,5 @@ +require('./utils.js'); + const TokenType = Object.freeze({ Word:1, NumberLit:2, StringLit:3, Op:4 }); class TokenStream @@ -18,6 +20,11 @@ this.savedLocations = []; } + getPrettyTokenList() + { + return this.tokens.mapKey("type", v => TokenType.keyName(v)); + } + eof() { return this.index < 0 || this.index >= this.tokens.length; @@ -40,7 +47,7 @@ { if (token.value !== value) { - return { error: { msg: "Expecting " . TokenType.keyName(type) + " '" + value + "', not '" + token.value + "'", ...this.tokenToLinePos(token) } }; + return { error: { msg: "Expecting " + TokenType.keyName(type) + " '" + value + "', not '" + token.value + "'", ...this.tokenToLinePos(token) } }; } } } @@ -51,45 +58,45 @@ consume(type = undefined, value = undefined) { const token = this.peek(type, value); - if (token.error) return error; + if (token.error) return token.error; this.index++; return token; } - beginMatch() + startMatch() { this.savedLocations.push(this.index); } - doneMatch() + finishMatch() { - console.assert(this.savedLocations.length > 0, "Unexpected doneMatch()"); + console.assert(this.savedLocations.length > 0, "Unexpected finishMatch()"); this.savedLocations.pop(); } - failedMatch() + failMatch() { - console.assert(this.savedLocations.length > 0, "Unexpected failedMatch()"); + console.assert(this.savedLocations.length > 0, "Unexpected failMatch()"); this.index = this.savedLocations.pop(); } - stringIndexToLinePos(index) + stringPosToLinePos(pos) { - index = Math.max(0, Math.min(this.inputLength - 1, index)); + pos = Math.max(0, Math.min(this.inputLength - 1, pos)); let line = this.newlines.length + 1; let previousLinesLength = 0; for (let i = 0; i < this.newlines.length; i++) { - if (index <= this.newlines[i]) + if (pos <= this.newlines[i]) { line = i + 1; break; } previousLinesLength += this.newlines[i] + 1; } - return {line:line, pos:index - previousLinesLength + 1}; + return {line:line, pos:pos - previousLinesLength + 1}; } tokenIndexToLinePos(index) @@ -101,7 +108,7 @@ tokenToLinePos(token) { - return this.tokenIndexToLinePos(token.index); + return this.stringPosToLinePos(token.pos); } } @@ -130,7 +137,7 @@ let parts = []; // track where we are in the original string - let index = 0; + let pos = 0; while (input.length > 0) { // match our possible sequences of more than one character (words, digits, or whitespace) @@ -139,19 +146,19 @@ (match = input.match(/^\s+/))) { const matched = match[0]; - parts.push({token:matched, index:index}); + parts.push({token:matched, pos:pos}); // advance the string input = input.substr(matched.length); - index += matched.length; + pos += matched.length; } else { // everything else is a single character (punctuation) - parts.push({token:input[0], index:index}); + parts.push({token:input[0], pos:pos}); input = input.substr(1); - index++; + pos++; } } @@ -161,7 +168,7 @@ { const partsLeft = parts.length - i - 1; - let { token, index } = parts[i]; + let { token, pos } = parts[i]; if (token.trim().length == 0) { @@ -176,7 +183,7 @@ { token += "." + parts[i + 2].token; } - tokens.push({type:TokenType.NumberLit, token:token, index:index}); + tokens.push({type:TokenType.NumberLit, token:token, pos:pos}); } else if (token == '"') { @@ -224,17 +231,17 @@ throw "Unclosed string"; } - tokens.push({type:TokenType.StringLit, token:token, index:index}); + tokens.push({type:TokenType.StringLit, token:token, pos:pos}); } else if (token.match(/\w+/)) { // any keyword or identifier - tokens.push({type:TokenType.Word, token:token, index:index}); + tokens.push({type:TokenType.Word, token:token, pos:pos}); } else { // anything left is an operator - tokens.push({type:TokenType.Op, token:token, index:index}); + tokens.push({type:TokenType.Op, token:token, pos:pos}); } } diff --git a/parse.js b/parse.js index 86ab105..2883b7e 100644 --- a/parse.js +++ b/parse.js @@ -1,5 +1,5 @@ -const lex = require('lex'); -const utils = require('utils'); +const lex = require('./lex.js'); +const utils = require('./utils.js'); /* How powerfully infix operators bind to their left and right arguments @@ -68,7 +68,7 @@ expression(tokenStream) { - return expressionInternal(tokenStream, 0); + return this.expressionInternal(tokenStream, 0); } /* Parses an expression consisting of a set of values, all separated by infix operators, @@ -91,41 +91,45 @@ expressionInternal(minBindingPower) { // grab the type of the next token, and its string representation - const {type, token, index:tokenIndex, error} = this.tokenStream.consume(); + let { type, token, pos, error } = this.tokenStream.consume(); if (error) return this.getError(error); // start parsing the left-hand-side of the next operator at this AST node let lhs = null; if (type == lex.TokenType.Op) { + let power; + // the value of this part of the expression starts with an operator // it will either be prefix operator, or a bracketed expression if (token == "(") { // parse a subexpression - const { expr, error } = this.attemptSubtree(this.expression); + let { expr, error } = this.attemptSubtree(this.expression); if (error) return this.getError(error); lhs = expr; // ensure the next token after the sub expression is a closing paren, and consume it - const { error } = tokens.consume(lex.TokenType.Op, ")"); - if (error) return this.getError(error); + if (error = tokens.consume(lex.TokenType.Op, ")").error) + { + return this.getError(error); + } } else if ((power = PrefixBindingPower[token]) !== undefined) { // this is a prefix operator of the given power // get the expression it applies to... - const { rhs, error } = this.attemptSubtree(expressionInternal, power); + let { rhs, error } = this.attemptSubtree(expressionInternal, power); if (error) return this.getError(error); // ... and generate an AST node for it // this becomes the new LHS of the operator we are parsing - lhs = {node:ASTNode.PrefixOp, rhs:rhs, op:token}; + lhs = { node:ASTNode.PrefixOp, rhs:rhs, op:token }; } else { - return this.getError({ msg: "Invalid Op '" + token + "'", ...this.tokenStream.stringIndexToLinePos(tokenIndex) }); + return this.getError({ msg: "Invalid Op '" + token + "'", ...this.tokenStream.stringPosToLinePos(pos) }); } } else if (type == lex.TokenType.Word) @@ -133,7 +137,7 @@ if (Keywords.includes(token)) { // TODO: handle keywords that can go in expressions - return this.getError({ msg: "Keyword '" + token + "' not expected", ...this.tokenStream.stringIndexToLinePos(tokenIndex) }); + return this.getError({ msg: "Keyword '" + token + "' not expected", ...this.tokenStream.stringPosToLinePos(pos) }); } else if (Literals[token] !== undefined) { @@ -143,20 +147,20 @@ else { // some unresolved identifier (this is the domain of the runtime/interpreter!) - lhs = {node:ASTNode.Identifier, value:token}; + lhs = { node:ASTNode.Identifier, value:token }; } } - else if (type == TokenType.NumberLit) + else if (type == lex.TokenType.NumberLit) { lhs = {node:ASTNode.Literal, value:parseFloat(token)}; } - else if (type == TokenType.StringLit) + else if (type == lex.TokenType.StringLit) { lhs = {node:ASTNode.Literal, value:token}; } else { - return this.getError({ msg: "Unexpected " + lex.TokenType.keyName(type) + " of '" + token + "'", ...this.tokenStream.stringIndexToLinePos(tokenIndex) }); + return this.getError({ msg: "Unexpected " + lex.TokenType.keyName(type) + " of '" + token + "'", ...this.tokenStream.stringPosToLinePos(pos) }); } while (!this.tokenStream.eof()) @@ -168,7 +172,7 @@ * Depending on its precedence, we may not actually get to parse it ourselves, * so don't actually consume the token yet, just peek it (just in case) */ - const {type:opType, token:opToken, index:opIndex, error} = this.tokenStream.peek(lex.TokenType.Op); + let { type:opType, token:opToken, pos:opPos, error } = this.tokenStream.peek(lex.TokenType.Op); if (error) return this.getError(error); if (PostfixBindingPower[opToken] !== undefined) @@ -189,15 +193,17 @@ * We need to parse the index expression. Note we reset the binding power, * this is because we will always end with ], so we don't need to worry about our current precedence */ - const { index, error } = this.attemptSubtree(this.expression); + let { index, error } = this.attemptSubtree(this.expression); if (error) return this.getError(error); // parse the closing ] - const { error } = this.tokenStream.consume(lex.TokenType.Op, "]"); - if (error) return this.getError(error); + if (error = this.tokenStream.consume(lex.TokenType.Op, "]").error) + { + return this.getError(error); + } // our new node is the old lhs indexed with the list of argument expressions - lhs = {node:ASTNode.Index, lhs:lhs, index:index}; + lhs = { node:ASTNode.Index, lhs:lhs, index:index }; } else if (opToken == "(") { @@ -208,13 +214,11 @@ let args = []; while (true) { - if (tokens.length == 0) throw "Unclosed argument list, expected ',' or ')' at " + JSON.stringify(stringIndexToLinePos(inputLength - 1, newlines)); - // peek the next op - const {token:nextToken, type:nextType, index:nextIndex, error} = this.tokenStream.peek(); + let { token:nextToken, type:nextType, pos:nextPos, error } = this.tokenStream.peek(); if (error) return this.getError(error); - if (nextType == TokenType.Op && nextToken == ")") + if (nextType == lex.TokenType.Op && nextToken == ")") { // no more arguments, we should end the call (after consuming the bracket) this.tokenStream.consume(); @@ -223,15 +227,20 @@ if (args.length > 0) { - const { error } = this.tokenStream.consume(lex.TokenType.Op, ","); - if (error) return this.getError(error); + if (error = this.tokenStream.consume(lex.TokenType.Op, ",").error) + { + return this.getError(error); + } } - const { arg, error } = this.attemptSubtree(this.expression); - if (error) return this.getError(error); + { + // avoid re-declaring error... annoying + let { arg, error } = this.attemptSubtree(this.expression); + if (error) return this.getError(error); - // our new node is the old lhs as a function, called with the index expression - args.push(arg); + // our new node is the old lhs as a function, called with the index expression + args.push(arg); + } } // generate the AST node @@ -239,7 +248,7 @@ } else { - return this.getError({ msg: "Unexpected postfix Op '" + opToken + "'", ...this.tokenStream.stringIndexToLinePos(opIndex) }); + return this.getError({ msg: "Unexpected postfix Op '" + opToken + "'", ...this.tokenStream.stringPosToLinePos(opPos) }); } // after parsing a postfix operator, we are expecting an operator again @@ -261,49 +270,33 @@ //... otherwise, this operator is for us, so consume it this.tokenStream.consume(); - // the next expression is the right side of our current operator, but only as much as the - // right-side binding power of our operator can grab - const { rhs, error } = this.attemptSubtree(this.expressionInternal, rightBinding); - if (error) return this.getError(error); + { + // the next expression is the right side of our current operator, but only as much as the + // right-side binding power of our operator can grab + let { rhs, error } = this.attemptSubtree(this.expressionInternal, rightBinding); + if (error) return this.getError(error); - // our new lhs becomes our applied operator, - // leaving us free to match more operators of an appropriate precedence - lhs = {node:ASTNode.InfixOp, lhs:lhs, rhs:rhs, op:opToken}; + // our new lhs becomes our applied operator, + // leaving us free to match more operators of an appropriate precedence + lhs = {node:ASTNode.InfixOp, lhs:lhs, rhs:rhs, op:opToken}; + } } return { ast: lhs, error: null }; } - error(msg, at) - { - let loc = this.tokenStream.tokenIndexToLinePos(this.tokenStream.index); - if (at) - { - if (at.index) - { - loc = this.tokenStream.tokenToLinePos(at); - } - else - { - loc = this.tokenStream.tokenIndexToLinePos(at); - } - } - return { ast:null, error: { msg: msg, ...loc } }; - } - - attemptSubtree(parseFunction, ...args) { - this.tokenStream.attemptMatch(); - const subtree = parseFunction.apply(this, [tokenStream, ...args]); + this.tokenStream.startMatch(); + const subtree = parseFunction.apply(this, [ this.tokenStream, ...args ]); if (subtree.error) { - this.tokenStream.failedMatch(); + this.tokenStream.failMatch(); } else { - this.tokenSTream.doneMatch(); + this.tokenStream.finishMatch(); } return subtree; @@ -316,3 +309,4 @@ } +module.exports = { Parser: Parser }; diff --git a/test.js b/test.js index fa2123d..83521b0 100644 --- a/test.js +++ b/test.js @@ -1,15 +1,17 @@ fs = require('fs'); -parse = require('parse'); +lex = require('./lex.js'); +parse = require('./parse.js'); function debugExpr(expr) { const tokens = new lex.TokenStream(expr); - console.log(tokens.getPrettyTokenList()); //mapKey("type", v => TokenType.keyName(v))); + console.log(tokens.getPrettyTokenList()); - const { ast, error } = parse.attemptExpression(tokens); - if (error) throw error; + const parser = new parse.Parser(tokens); - console.format(parse.getPrettyAST(ast)); //ast.mapKey("node", v => ASTNode.keyName(v))); + const result = parser.expression(tokens); + + console.format(parse.getPrettyAST(result)); //ast.mapKey("node", v => ASTNode.keyName(v))); } debugExpr(fs.readFileSync(process.argv[2], "utf8"));