From f0b8f181f7a2ba38a1567d152552d3b1a73cf2c4 Mon Sep 17 00:00:00 2001 From: ElementG9 Date: Sun, 24 Nov 2019 21:31:20 -0700 Subject: [PATCH] Add package data, member access, pre/postfix operators work now. --- bin/pivot.js | 28 ++++++++++++- package.json | 7 ++-- src/parser.js | 100 +++++++++++++++++++++++++++++++++++++++++------ src/tokenizer.js | 75 +++++++++++++++++++++++++++-------- src/types.js | 15 ++++++- test/test.js | 25 +++++++++++- 6 files changed, 215 insertions(+), 35 deletions(-) diff --git a/bin/pivot.js b/bin/pivot.js index 3d4a31f..cf4bb23 100755 --- a/bin/pivot.js +++ b/bin/pivot.js @@ -1,3 +1,29 @@ #!/usr/bin/env node const args = process.argv.slice(2); -console.log(args); +const tokenizer = require('../src/tokenizer.js'); +const parser = require('../src/parser.js'); + +if (typeof args[0] != 'undefined') { + // Execute from file. +} else { // REPL. + const rl = require('readline-sync'); + const exec = require('child_process').exec; + function repl(prompt, func) { + let answer; + while (answer != 'exit') { + answer = rl.question(prompt); + if (answer == 'exit') + process.exit(0); + if (answer == 'clear') { + console.clear(); + continue; + } + func(answer); + } + } + console.log('Welcome to Pivot v0.1.0 Alpha.'); + console.log('Type \'exit\' to exit.'); + repl('> ', (answer) => { + console.log(require('util').inspect(parser.parse(tokenizer.tokenize(answer)), { depth: null })); + }); +} diff --git a/package.json b/package.json index 1ce2203..183d6db 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { - "name": "pivot", - "version": "1.0.0", + "name": "pivot-lang", + "version": "0.1.0", "description": "Pivot is a new programming language built on JavaScript", "main": "./bin/pivot.js", "directories": { @@ -31,7 +31,8 @@ }, "preferGlobal": true, "dependencies": { - "fs": "0.0.1-security" + "fs": "0.0.1-security", + "readline-sync": "^1.4.10" }, "devDependencies": { "mocha": "^6.2.1" diff --git a/src/parser.js b/src/parser.js index 4f06eea..57a6ce6 100644 --- a/src/parser.js +++ b/src/parser.js @@ -4,9 +4,8 @@ * @author Garen Tyler * @requires module:types */ -const Token = require('./types.js').Token; const Group = require('./types.js').Group; -const tokenizer = require('./tokenizer.js'); +const Operator = require('./types.js').Operator; /** * @function parse @@ -23,10 +22,12 @@ function parse(tokens) { ast = addIndexes(ast); ast = addLevels(ast); - // Combine the groups. - ast = combineGroups(ast); - - // + // Start grouping by precedence + ast = grouping(ast); + ast = memberAccess(ast); + ast = postfixOperators(ast); + ast = prefixOperators(ast); + console.log(ast); return ast; } @@ -84,16 +85,15 @@ function getDeepestLevel(tokens) { } /** - * @function combineGroups + * @function grouping * @desc Combine groups of tokens by delimiter. * @param {Token[]} tokens The tokens. * @returns {Token[]} The grouped tokens, or the basic ast. * @private */ -function combineGroups(tokens) { +function grouping(tokens) { // Get the deepest level. let deepestLevel = getDeepestLevel(tokens); - // Loop through for each level. for (let currentLevel = deepestLevel; currentLevel > 0; currentLevel--) { let groupBuffer = []; @@ -107,8 +107,10 @@ function combineGroups(tokens) { let g = new Group(groupBuffer[0].value, groupBuffer); g.index = g.tokens[0].index; g.level = g.tokens[0].level - 1; // -1 because the group is on the level below. - tokens.splice(g.tokens[0].index, g.tokens.length + 1, g); + tokens.splice(g.tokens[0].index, g.tokens.length, g); j = g.tokens[0].index; + // Remove the delimiters in g.tokens. + g.tokens = g.tokens.splice(1, groupBuffer.length - 2); groupBuffer = []; } } @@ -117,14 +119,88 @@ function combineGroups(tokens) { return tokens; } +/** + * @function memberAccess + * @desc Combine groups of tokens by member access. + * @param {Token[]} tokens The tokens. + * @returns {Token[]} The grouped tokens, or the basic ast. + * @private + */ +function memberAccess(ast) { + for (let i = 0; i < ast.length; i++) { + if (ast[i].type == 'group') + memberAccess(ast[i].tokens); // Recursively order the groups. + else if (ast[i].type == 'operator' && ast[i].value == '.') { // Member access operator. + if (typeof ast[i - 1] == 'undefined' || typeof ast[i + 1] == 'undefined') + throw new SyntaxError('Operator requires two operands.'); + let op = new Operator(ast[i].subtype, ast[i].value, [ast[i - 1], ast[i + 1]]); + op.index = ast[i - 1].index; + op.level = ast[i].level; + ast.splice(i - 1, 3, op); + i--; // Removed 3 tokens, put in 1, skip 1 token. Reduce the counter by 1. + } + } + return ast; +} + +/** + * @function postfixOperators + * @desc Recursively structures the postfix operators. + * @param {Token[]} ast The ast. + * @returns {Token[]} The ast with structured postfix operators. + * @private + */ +function postfixOperators(ast) { + for (let i = 0; i < ast.length; i++) { + // Take care of the tokens in the groups. + if (ast[i].type == 'group') + ast[i].tokens = postfixOperators(ast[i].tokens); + else if (ast[i].type == 'operator' && ast[i].subtype == 'postfix') { // The operand is on the left. + if (typeof ast[i - 1] == 'undefined') + throw new SyntaxError('Postfix operator requires one operand before it.'); + let op = new Operator(ast[i].subtype, ast[i].value, [ast[i - 1]]); + op.index = ast[i].index; + op.level = ast[i].level; + ast.splice(i - 1, 2, op); + // Removing 2 tokens, adding 1, skip 1 token. Don't reduce the counter. + } + } + return ast; +} + +/** + * @function prefixOperators + * @desc Recursively structures the prefix operators. + * @param {Token[]} ast The ast. + * @returns {Token[]} The ast with structured prefix operators. + * @private + */ +function prefixOperators(ast) { + for (let i = 0; i < ast.length; i++) { + // Take care of the tokens in the groups. + if (ast[i].type == 'group') + ast[i].tokens = postfixOperators(ast[i].tokens); + else if (ast[i].type == 'operator' && ast[i].subtype == 'prefix') { // The operand is on the right. + if (typeof ast[i + 1] == 'undefined') + throw new SyntaxError('Prefix operator requires one operand after it.'); + let op = new Operator(ast[i].subtype, ast[i].value, [ast[i + 1]]); + op.index = ast[i].index; + op.level = ast[i].level; + ast.splice(i, 2, op); + // Removing 2 tokens, adding 1, skip 1 token. Don't reduce the counter. + } + } + return ast; +} + module.exports = { parse, util: { addIndexes, addLevels, getDeepestLevel, - combineGroups + grouping, } }; -require('fs').writeFileSync('ast.json', JSON.stringify(parse(tokenizer.tokenize('let x = (5 + (6 * 2));')), null, 2), () => {}); +// require('fs').writeFileSync('ast.json', JSON.stringify(parse(tokenizer.tokenize('let x = (5 + (6 * 2)) - 7;')), null, 2), () => {}); diff --git a/src/tokenizer.js b/src/tokenizer.js index d60c3f3..4a3ecbd 100644 --- a/src/tokenizer.js +++ b/src/tokenizer.js @@ -5,7 +5,6 @@ * @requires module:types */ const Token = require('./types.js').Token; -const Group = require('./types.js').Group; /** * @function tokenize @@ -234,21 +233,64 @@ function getDelimiterToken(delimiter) { /** * @function operatorType - * @desc Determines the type of operator. - * @param {string} operator The operator char. - * @returns {string} The type of operator. + * @desc Turns a delimiter char into a token. + * @param {string} delimiter The delimiter char. + * @returns {Token} The delimiter token. * @private */ -function operatorType(operator) { - // Left operators have parameters on the left. - if (/\+\+|--/.test(operator)) - return 'left'; - else if (false) - return 'right'; - else if (/\;/.test(operator)) - return 'none'; - else - return 'dual'; +function operatorType(op) { + switch (op) { + case '.': + case '+': + case '-': + case '*': + case '/': + case '**': + case '%': + case '<<': + case '>>': + case '<': + case '<=': + case '>': + case '>=': + case '==': + case '!=': + case '&': + case '|': + case '^': + case '&&': + case '||': + case '^^': + case '=': + case '+=': + case '-=': + case '*=': + case '/=': + case '**=': + case '%=': + case '<<=': + case '>>=': + case '&=': + case '|=': + case '^=': + return 'dual'; + break; + case '++': + case '--': + return 'postfix'; + break; + case '!': + case '~': + return 'prefix'; + break; + case ',': + case ';': + return 'none'; + break; + default: + throw new TypeError('Unexpected operator ' + op); + break; + } } /** @@ -261,7 +303,8 @@ function operatorType(operator) { function determineCharType(char) { if (/[A-Za-z]/.test(char)) return 'letter'; - else if (/\+|\-|\*|\/|\=|\=\=|\>|\<|\>\=|\<\=|\=\>|;/.test(char)) + else if (/\.|\+\+|--|!|~|\+|-|\*\*|\*|\/%|<<|>>|<|<=|>|>=|==|!=|&|\^|\||&&|\|\||=|\+=|-=|\*\*=|\*=|\/=|%=|<<=|>>=|&=|\^=|\|=|,|;/.test(char)) + // All the operators in Pivot. return 'operator'; else if (/\(|\)|\[|\]|\{|\}/.test(char)) return 'delimiter'; @@ -273,7 +316,7 @@ function determineCharType(char) { return 'escaped char'; else if (/\s/.test(char)) return 'whitespace'; - else throw new SyntaxError('Unexpected char ' + char); + else throw new TypeError('Unexpected char ' + char); }; /** diff --git a/src/types.js b/src/types.js index 5d2a7a5..1bbc0cc 100644 --- a/src/types.js +++ b/src/types.js @@ -4,7 +4,6 @@ * @author Garen Tyler */ - /** * @class Token * @classdesc Stores the type of token, subtype, and literal char value. @@ -25,7 +24,19 @@ function Group(type, tokens) { this.tokens = tokens; } +/** + * @class Operator + * @classdesc Stores the type of operator, and tokens for an operator. + */ +function Operator(subtype, value, operands) { + this.type = 'operator'; + this.subtype = subtype; + this.value = value; + this.operands = operands; +} + module.exports = { Token, - Group + Group, + Operator }; diff --git a/test/test.js b/test/test.js index a1e103d..3b53bc8 100644 --- a/test/test.js +++ b/test/test.js @@ -13,6 +13,9 @@ describe('types.js', () => { it('Has a Group child.', () => { assert.equal(types.hasOwnProperty('Group'), true); }); + it('Has a Operator child.', () => { + assert.equal(types.hasOwnProperty('Operator'), true); + }); describe('Token', () => { it('Works as a constructor', () => { @@ -54,6 +57,26 @@ describe('types.js', () => { } }); }); + // describe('Operator', () => { + // it('Works as a constructor', () => { + // try { + // let op = new types.Operator('dual', '+', ['3','4']); + // } catch (err) { + // throw err; + // } + // }); + // it('Has values \'type\', \'subtype\', \'value\', and \'operands\'', () => { + // try { + // let op = new types.Operator('dual', '+', ['3','4']); + // if (!op.hasOwnProperty('type') || !op.hasOwnProperty('subtype') || !op.hasOwnProperty('value') || !op.hasOwnProperty('operands')) + // throw new Error('Operator is missing \'type\', \'subtype\', \'value\', or \'operands\' properties.'); + // if (op.type != 'operator' || op.subtype != 'dual' || op.value != '+' || op.operands[0] != '3') + // throw new Error('Operator incorrectly set \'type\', \'subtype\', \'value\', or \'operands\' properties.'); + // } catch (err) { + // throw err; + // } + // }); + // }); }); describe('tokenizer.js', () => { it('combineEscapedChars works', () => { @@ -101,7 +124,7 @@ describe('tokenizer.js', () => { throw new Error('Incorrect value: Expected \'parenthesis\' but got \'' + token.value + '\'') }); it('operatorType works', () => { - assert.equal(tokenizer.util.operatorType('++'), 'left'); + assert.equal(tokenizer.util.operatorType('++'), 'postfix'); assert.equal(tokenizer.util.operatorType(';'), 'none'); assert.equal(tokenizer.util.operatorType('+'), 'dual'); });