From 56438fb268c716bcc0d3e8348f44d2a000c7191e Mon Sep 17 00:00:00 2001 From: ElementG9 Date: Tue, 15 Jan 2019 18:08:42 -0700 Subject: [PATCH 01/15] Update readme.md --- readme.md | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/readme.md b/readme.md index 6677949..6f46894 100644 --- a/readme.md +++ b/readme.md @@ -1,23 +1,15 @@ -# [Pivot][pivot-site] +# [Pivot](https://gthub.us/pivot/) -Pivot is a new programming language built on [JavaScript][javascript-site] by [ElementG9][elementg9-user]. Pivot is currently in the alpha stage of development. +Pivot is a new programming language built on [JavaScript](https://javascript.com/) by [ElementG9](https://github.com/ElementG9/). --- -### Installation -* Download or clone [the GitHub repository][pivot-repo]. -* Download [node.js][nodejs-site]. +## Installation +* Download or clone [the GitHub repository](/pivot/repo). +* Download [node.js](https://nodejs.org/). * Run _pivot.js_ with `node pivot.js`. --- -### Documentation - -The Pivot documentation is at [gthub.us/pivot/docs][pivot-docs]. - -[elementg9-user]: https://github.com/ElementG9/ -[pivot-site]: https://gthub.us/pivot/ -[pivot-docs]: https://gthub.us/pivot/docs/ -[pivot-repo]: https://github.com/ElementG9/Pivot/ -[javascript-site]: https://javascript.com/ -[nodejs-site]: https://nodejs.org/ +## Documentation +The Pivot documentation is [here](https://gthub.us/pivot/docs). From 7f889a79a0077e57f87eb8c04e94987e01572ff9 Mon Sep 17 00:00:00 2001 From: ElementG9 Date: Thu, 14 Mar 2019 19:05:56 -0600 Subject: [PATCH 02/15] Groups work now. --- ast.json | 45 ++++--- classes.js | 29 ++--- parser.js | 173 +++++++++++++++++++-------- pivot.js | 2 +- tokenizer.js | 330 +++++++++++++++++++++++++-------------------------- 5 files changed, 325 insertions(+), 254 deletions(-) diff --git a/ast.json b/ast.json index 4d319a8..ec9f41a 100644 --- a/ast.json +++ b/ast.json @@ -11,44 +11,39 @@ }, { "type": "Group", - "value": "{", - "tokens": [ - { - "type": "Left Delimiter", - "value": "{", - "layer": 0 - } - ] - }, - { - "type": "Group", - "value": "a", + "value": "(", "tokens": [ { "type": "Variable", "value": "a", - "layer": 2 - }, - { - "type": "Operator", - "value": "++", - "layer": 2 + "layer": 1 } ] }, { "type": "Group", - "value": "asdf", + "value": "{", "tokens": [ { - "type": "Variable", - "value": "asdf", - "layer": 0 + "type": "Function Call", + "value": "return", + "layer": 1 }, { - "type": "Operator", - "value": "=", - "layer": 0 + "type": "Group", + "value": "(", + "tokens": [ + { + "type": "Variable", + "value": "a", + "layer": 2 + }, + { + "type": "Operator", + "value": "++", + "layer": 2 + } + ] } ] } diff --git a/classes.js b/classes.js index e5df2f0..4d59265 100644 --- a/classes.js +++ b/classes.js @@ -1,20 +1,21 @@ // This is the most basic type of token. var token = function (type, value) { - this.type = type; - this.value = value; + this.type = type; + this.value = value; }; // This is a group of tokens. -var group = function (type, tokens) { - this.type = "Group"; - this.value = type; - this.tokens; - if(typeof tokens != "undefined") { - this.tokens = tokens; - } else { - this.tokens = []; - } +var group = function (type, tokens, index) { + this.type = "Group"; + this.value = type; + this.index = index; + this.tokens; + if (typeof tokens != "undefined") { + this.tokens = tokens; + } else { + this.tokens = []; + } } module.exports = { - token: token, - group: group -} + token: token, + group: group +} \ No newline at end of file diff --git a/parser.js b/parser.js index e163023..50cba2f 100644 --- a/parser.js +++ b/parser.js @@ -5,55 +5,130 @@ const group = require("./classes.js").group; // parse() takes an array of tokens in, and outputs an // Abstract Syntax Tree (a structured array of tokens). module.exports = tokens => { - // Variables for later. - var layer = 0; - var delimiterCount = 0; - var deepestLayer = 0; - // Give each token a layer number based on delimiters. - for (var i = 0; i < tokens.length; i++) { - if (tokens[i].type == "Left Delimiter") { - layer++; - if(layer > deepestLayer) { - deepestLayer = layer; - } - delimiterCount++; - } - tokens[i].layer = layer; - if (tokens[i].type == "Right Delimiter") { - layer--; - } + // Variables for later. + var layer = 0; + var delimiterCount = 0; + var deepestLayer = 0; + // Give each token a layer number based on delimiters. + for (var i = 0; i < tokens.length; i++) { + if (tokens[i].type == "Left Delimiter") { + layer++; + if (layer > deepestLayer) { + deepestLayer = layer; + } + delimiterCount++; } - // Lower the layer of delimiters. - for (var i = 0; i < tokens.length; i++) { - if ((tokens[i].type == "Left Delimiter") || (tokens[i].type == "Right Delimiter")) { - tokens[i].layer--; - } - } - if (layer > 0) { // Unclosed delimiter. - } else if (layer < 0) { // Overclosed delimiter. - } - // Reset layer for structuring. - layer = 0; - /* - - - DO NOT TOUCH THIS - - - */ - for(var i=deepestLayer;i>=0;i--) { - var temp = []; - var firstIndex; - for(var j=0;j 0) { - var g = new group(tokens[firstIndex].value,temp); - tokens.splice(firstIndex-1,temp.length+2,g); - temp = []; - } - } + tokens[i].layer = layer; + if (tokens[i].type == "Right Delimiter") { + layer--; + } + } + // Lower the layer of delimiters. + for (var i = 0; i < tokens.length; i++) { + if ((tokens[i].type == "Left Delimiter") || (tokens[i].type == "Right Delimiter")) { + tokens[i].layer--; + } + } + if (layer > 0) { // Unclosed delimiter. + } else if (layer < 0) { // Overclosed delimiter. + } + // Give each token an index. + for (let i = 0; i < tokens.length; i++) { + tokens[i].index = i; + } + // Structure the layers. + // Count the rising edges of the layers to determine how many groups should exist. + let structure = function () { + let layer = 0; + let risingFalling = []; // Create an array to store indices of rising/falling edges. + for (let i = 0; i < tokens.length; i++) { + // Add a rising and a falling tag to each token. + tokens[i].rising = false; + tokens[i].falling = false; + if (tokens[i].layer > layer) { // If the token moves up a layer. + // Create a new rising index in risingFalling. + risingFalling.push({ + type: 'rising', + index: i + }); + tokens[i].rising = true; // Note that the token is a rising edge. + layer++; + } else if (tokens[i].layer < layer) { + // Create a new falling index in risingFalling. + risingFalling.push({ + type: 'falling', + index: i + }); + tokens[i].falling = true; // Note that the token is a falling edge. + layer--; + } + } + // Loop through the list of rising/falling edges. + for (let i = 0; i < risingFalling.length; i++) { + if (i != risingFalling.length - 1) { // If not the last edge. + let item = risingFalling[i]; + let nextItem = risingFalling[i + 1]; + // If a falling edge follows a rising edge, classifiy it as a group. + if ((item.type == 'rising') && (nextItem.type == 'falling')) { + // Get the group together as one item. + let selectedItems = tokens.slice(item.index, nextItem.index); + tokens.splice(item.index, selectedItems.length, new group(tokens[item.index - 1].value, selectedItems, item.index)); } + } } - /* - - - OK YOU CAN TOUCH AGAIN - - - */ - // Return the structured tokens. - return tokens; -}; + risingFalling = []; // Reset the list of edges. + // Count the edges again. + for (let i = 0; i < tokens.length; i++) { + if (tokens[i].layer > layer) { + risingFalling.push({ + type: 'rising', + index: i + }); + layer++; + } else if (tokens[i].layer < layer) { + risingFalling.push({ + type: 'falling', + index: i + }); + layer--; + } + } + // If there are still edges, run again. + if (risingFalling.length) { + structure(); + } + }; + // Start the recursion. + structure(); + let trimDelimiters = function (thing) { + // Loop through the tokens of thing. + for (let i = 0; i < thing.length; i++) { + // Delete unnecessary keys. + if (typeof thing[i].rising != 'undefined') { + delete thing[i].rising; + } + if (typeof thing[i].falling != 'undefined') { + delete thing[i].falling; + } + if (typeof thing[i].index != 'undefined') { + delete thing[i].index; + } + if (typeof thing[i].index != 'undefined') { + delete thing[i].index; + } + // Remove delimiters. + if ((thing[i].type == 'Left Delimiter') || (thing[i].type == 'Right Delimiter')) { + thing.splice(i, 1); + i--; + } + // If a token is a group, look at the group's tokens. + if (thing[i].type == 'Group') { + trimDelimiters(thing[i].tokens); + } + } + }; + // Start the recursion. + trimDelimiters(tokens); + // Return the structured tokens. + return tokens; +}; \ No newline at end of file diff --git a/pivot.js b/pivot.js index e9d92cc..2b75014 100644 --- a/pivot.js +++ b/pivot.js @@ -10,4 +10,4 @@ var ast = parse(tokenize(code)); // Write the AST to ast.json. var fs = require("fs"); -fs.writeFileSync("ast.json", JSON.stringify(ast, null, 4)); +fs.writeFileSync("ast.json", JSON.stringify(ast, null, 4)); \ No newline at end of file diff --git a/tokenizer.js b/tokenizer.js index b1ba7f9..f8dab7d 100644 --- a/tokenizer.js +++ b/tokenizer.js @@ -3,168 +3,168 @@ const token = require("./classes.js").token; // Create the tokenizer function. // tokenize() takes Pivot code in, and outputs an array of tokens. -module.exports = exp => { - // Check functions for different token types. - var isDigit = char => { - return /\d/.test(char); - }; - var isLetter = char => { - return /[a-z]/i.test(char); - }; - var isOperator = char => { - return /\+|-|\*|\/|\^|=/.test(char); - }; - var isLeftDelimiter = char => { - return (/\(|\[|\{|"|'|`/.test(char)); - }; - var isRightDelimiter = char => { - return (/\)|\]|\}/.test(char)); - }; - var isComma = char => { - return (char === ","); - }; - var isPeriod = char => { - return (char === "."); - }; - var result = []; // The final array of tokens. - var nb = []; // Number buffer. Allows for multiple digits to be one number. - var lb = []; // Letter buffer. Allows for multiple letters to be one variable / function. - var ob = []; // Operator buffer. Allows for multi-character operators. E.g. ++ or ==. - var sb = []; // String buffer. Allows for multi-character strings. - var inString = false; // Keep track of whether in string or not. - var stringType; // Keep track of what type of string. E.g. "" or ''. - exp = exp.split(""); // Split the expression into an array of characters. - /* - - - DO NOT TOUCH THIS - - - */ - for (var i = 0; i < exp.length; i++) { // Loop through all of the characters. - var char = exp[i]; // Create a quick reference to the current char. - if (i >= 1) { - if (exp[i - 1] == "\\") { - exp.splice(i - 1, 2, `\\${char}`); - i--; - continue; - } - if (exp[i - 1] == "$" && char == "{") { - exp.splice(i - 1, 2, `\${`); - i--; - continue; - } - } - } - /* - - - OK YOU CAN TOUCH AGAIN - - - */ - // Nevermind, just don't mess with any of this file. - for (var i = 0; i < exp.length; i++) { - var char = exp[i]; - if (inString) { - if (char == `'` || char == `"` || char == "`") { - var exitString = () => { - inString = false; - if (sb.length == 0) { - result.push(new token("String", null)); - } else { - var string = sb.join(""); - result.push(new token("String", string)); - } - sb = []; - }; - if (char == `'` && stringType == "single") { - exitString(); - } else if (char == `"` && stringType == "double") { - exitString(); - } else if (char == "`" && stringType == "backtick") { - exitString(); - } else { - if (char == `'`) { - sb.push(`\'`); - } - if (char == `"`) { - sb.push(`\"`); - } - if (char == "`") { - sb.push("\`"); - } - } - } else { - sb.push(char); - } - } else { - if (isDigit(char)) { - result.push(new token("Operator", ob.join(""))); - ob = []; - nb.push(char); - } else if (isLetter(char)) { - result.push(new token("Operator", ob.join(""))); - ob = []; - lb.push(char); - } else if (isOperator(char)) { - result.push(new token("Number", nb.join(""))); - nb = []; - result.push(new token("Variable", lb.join(""))); - lb = []; - ob.push(char); - } else if (isLeftDelimiter(char)) { - result.push(new token("Operator", ob.join(""))); - ob = []; - result.push(new token("Function Call", lb.join(""))); - lb = []; - if (char == `'` || char == `"` || char == "`") { - inString = true; - if (char == `'`) { - stringType = "single"; - } else if (char == `"`) { - stringType = "double"; - } else if (char == "`") { - stringType = "backtick"; - } - } else { - result.push(new token("Left Delimiter", char)); - } - } else if (isRightDelimiter(char)) { - result.push(new token("Operator", ob.join(""))); - ob = []; - result.push(new token("Number", nb.join(""))); - nb = []; - result.push(new token("Variable", lb.join(""))); - lb = []; - result.push(new token("Right Delimiter", char)); - } else if (isComma(char)) { - result.push(new token("Operator", ob.join(""))); - ob = []; - result.push(new token("Number", nb.join(""))); - nb = []; - result.push(new token("Variable", lb.join(""))); - lb = []; - result.push(new token("Comma", char)); - } else if (isPeriod(char)) { - result.push(new token("Operator", ob.join(""))); - ob = []; - nb.push(char); - } - } - } - result.push(new token("Operator", ob.join(""))); - ob = []; - result.push(new token("Number", nb.join(""))); - nb = []; - lb.forEach(item => { - result.push(new token("Variable", item)); - }); - lb = []; - for (var i = 0; i < 3; i++) { - result.forEach((item, index) => { - if (item.value == "") { - result.splice(index, 1); - } - }); - } - result.forEach((item, index) => { - if (item.value == "-" && index != 0) { - if (result[index - 1].type != "Variable" && result[index - 1].type != "Number") { - if (result[index + 1].type == "Number") { - result[index + 1].value = "-" + result[index + 1].value; - result.splice(index, 1); - } - } - } - }); - return result; -}; +module.exports = exp => { + // Check functions for different token types. + var isDigit = char => { + return /\d/.test(char); + }; + var isLetter = char => { + return /[a-z]/i.test(char); + }; + var isOperator = char => { + return /\+|-|\*|\/|\^|=/.test(char); + }; + var isLeftDelimiter = char => { + return (/\(|\[|\{|"|'|`/.test(char)); + }; + var isRightDelimiter = char => { + return (/\)|\]|\}/.test(char)); + }; + var isComma = char => { + return (char === ","); + }; + var isPeriod = char => { + return (char === "."); + }; + var result = []; // The final array of tokens. + var nb = []; // Number buffer. Allows for multiple digits to be one number. + var lb = []; // Letter buffer. Allows for multiple letters to be one variable / function. + var ob = []; // Operator buffer. Allows for multi-character operators. E.g. ++ or ==. + var sb = []; // String buffer. Allows for multi-character strings. + var inString = false; // Keep track of whether in string or not. + var stringType; // Keep track of what type of string. E.g. "" or ''. + exp = exp.split(""); // Split the expression into an array of characters. + /* - - - DO NOT TOUCH THIS - - - */ + for (var i = 0; i < exp.length; i++) { // Loop through all of the characters. + var char = exp[i]; // Create a quick reference to the current char. + if (i >= 1) { + if (exp[i - 1] == "\\") { + exp.splice(i - 1, 2, `\\${char}`); + i--; + continue; + } + if (exp[i - 1] == "$" && char == "{") { + exp.splice(i - 1, 2, `\${`); + i--; + continue; + } + } + } + /* - - - OK YOU CAN TOUCH AGAIN - - - */ + // Nevermind, just don't mess with any of this file. + for (var i = 0; i < exp.length; i++) { + var char = exp[i]; + if (inString) { + if (char == `'` || char == `"` || char == "`") { + var exitString = () => { + inString = false; + if (sb.length == 0) { + result.push(new token("String", null)); + } else { + var string = sb.join(""); + result.push(new token("String", string)); + } + sb = []; + }; + if (char == `'` && stringType == "single") { + exitString(); + } else if (char == `"` && stringType == "double") { + exitString(); + } else if (char == "`" && stringType == "backtick") { + exitString(); + } else { + if (char == `'`) { + sb.push(`\'`); + } + if (char == `"`) { + sb.push(`\"`); + } + if (char == "`") { + sb.push("\`"); + } + } + } else { + sb.push(char); + } + } else { + if (isDigit(char)) { + result.push(new token("Operator", ob.join(""))); + ob = []; + nb.push(char); + } else if (isLetter(char)) { + result.push(new token("Operator", ob.join(""))); + ob = []; + lb.push(char); + } else if (isOperator(char)) { + result.push(new token("Number", nb.join(""))); + nb = []; + result.push(new token("Variable", lb.join(""))); + lb = []; + ob.push(char); + } else if (isLeftDelimiter(char)) { + result.push(new token("Operator", ob.join(""))); + ob = []; + result.push(new token("Function Call", lb.join(""))); + lb = []; + if (char == `'` || char == `"` || char == "`") { + inString = true; + if (char == `'`) { + stringType = "single"; + } else if (char == `"`) { + stringType = "double"; + } else if (char == "`") { + stringType = "backtick"; + } + } else { + result.push(new token("Left Delimiter", char)); + } + } else if (isRightDelimiter(char)) { + result.push(new token("Operator", ob.join(""))); + ob = []; + result.push(new token("Number", nb.join(""))); + nb = []; + result.push(new token("Variable", lb.join(""))); + lb = []; + result.push(new token("Right Delimiter", char)); + } else if (isComma(char)) { + result.push(new token("Operator", ob.join(""))); + ob = []; + result.push(new token("Number", nb.join(""))); + nb = []; + result.push(new token("Variable", lb.join(""))); + lb = []; + result.push(new token("Comma", char)); + } else if (isPeriod(char)) { + result.push(new token("Operator", ob.join(""))); + ob = []; + nb.push(char); + } + } + } + result.push(new token("Operator", ob.join(""))); + ob = []; + result.push(new token("Number", nb.join(""))); + nb = []; + lb.forEach(item => { + result.push(new token("Variable", item)); + }); + lb = []; + for (var i = 0; i < 3; i++) { + result.forEach((item, index) => { + if (item.value == "") { + result.splice(index, 1); + } + }); + } + result.forEach((item, index) => { + if (item.value == "-" && index != 0) { + if (result[index - 1].type != "Variable" && result[index - 1].type != "Number") { + if (result[index + 1].type == "Number") { + result[index + 1].value = "-" + result[index + 1].value; + result.splice(index, 1); + } + } + } + }); + return result; +}; \ No newline at end of file From 5c0a481dbe20cb030e3cf3d39b975c65d5a8e930 Mon Sep 17 00:00:00 2001 From: ElementG9 Date: Thu, 3 Oct 2019 22:29:51 -0600 Subject: [PATCH 03/15] Refactored tokenizer, implemented jsdoc, set up for cli. --- .gitignore | 4 + LICENSE | 4 +- ast.json | 50 -------- bin/pivot.js | 3 + classes.js | 21 ---- package.json | 39 ++++++ parser.js | 134 --------------------- pivot.js | 13 -- readme.md | 15 --- rpn.js | 40 ------- src/parser.js | 44 +++++++ src/tokenizer.js | 303 +++++++++++++++++++++++++++++++++++++++++++++++ src/types.js | 31 +++++ test.pivot | 6 - test/test.js | 126 ++++++++++++++++++++ tokenizer.js | 170 -------------------------- 16 files changed, 552 insertions(+), 451 deletions(-) create mode 100644 .gitignore delete mode 100644 ast.json create mode 100755 bin/pivot.js delete mode 100644 classes.js create mode 100644 package.json delete mode 100644 parser.js delete mode 100644 pivot.js delete mode 100644 rpn.js create mode 100644 src/parser.js create mode 100644 src/tokenizer.js create mode 100644 src/types.js delete mode 100644 test.pivot create mode 100644 test/test.js delete mode 100644 tokenizer.js diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9c4ba90 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +other/ +package-lock.json +node_modules/ +jsdoc/ diff --git a/LICENSE b/LICENSE index 28db51c..ab97016 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2019 ElementG9 +Copyright (c) 2019 Garen Tyler Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -18,4 +18,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. \ No newline at end of file +SOFTWARE. diff --git a/ast.json b/ast.json deleted file mode 100644 index ec9f41a..0000000 --- a/ast.json +++ /dev/null @@ -1,50 +0,0 @@ -[ - { - "type": "Variable", - "value": "asdf", - "layer": 0 - }, - { - "type": "Operator", - "value": "=", - "layer": 0 - }, - { - "type": "Group", - "value": "(", - "tokens": [ - { - "type": "Variable", - "value": "a", - "layer": 1 - } - ] - }, - { - "type": "Group", - "value": "{", - "tokens": [ - { - "type": "Function Call", - "value": "return", - "layer": 1 - }, - { - "type": "Group", - "value": "(", - "tokens": [ - { - "type": "Variable", - "value": "a", - "layer": 2 - }, - { - "type": "Operator", - "value": "++", - "layer": 2 - } - ] - } - ] - } -] \ No newline at end of file diff --git a/bin/pivot.js b/bin/pivot.js new file mode 100755 index 0000000..3d4a31f --- /dev/null +++ b/bin/pivot.js @@ -0,0 +1,3 @@ +#!/usr/bin/env node +const args = process.argv.slice(2); +console.log(args); diff --git a/classes.js b/classes.js deleted file mode 100644 index 4d59265..0000000 --- a/classes.js +++ /dev/null @@ -1,21 +0,0 @@ -// This is the most basic type of token. -var token = function (type, value) { - this.type = type; - this.value = value; -}; -// This is a group of tokens. -var group = function (type, tokens, index) { - this.type = "Group"; - this.value = type; - this.index = index; - this.tokens; - if (typeof tokens != "undefined") { - this.tokens = tokens; - } else { - this.tokens = []; - } -} -module.exports = { - token: token, - group: group -} \ No newline at end of file diff --git a/package.json b/package.json new file mode 100644 index 0000000..1ce2203 --- /dev/null +++ b/package.json @@ -0,0 +1,39 @@ +{ + "name": "pivot", + "version": "1.0.0", + "description": "Pivot is a new programming language built on JavaScript", + "main": "./bin/pivot.js", + "directories": { + "test": "test" + }, + "scripts": { + "test": "mocha", + "build": "npm run test; jsdoc src/ -r -d ./jsdoc/ -R ./readme.md" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/ElementG9/Pivot.git" + }, + "keywords": [ + "pivot", + "javascript", + "programming", + "language" + ], + "author": "Garen Tyler ", + "license": "MIT", + "bugs": { + "url": "https://github.com/ElementG9/Pivot/issues" + }, + "homepage": "https://github.com/ElementG9/Pivot#readme", + "bin": { + "pivot": "./bin/pivot.js" + }, + "preferGlobal": true, + "dependencies": { + "fs": "0.0.1-security" + }, + "devDependencies": { + "mocha": "^6.2.1" + } +} diff --git a/parser.js b/parser.js deleted file mode 100644 index 50cba2f..0000000 --- a/parser.js +++ /dev/null @@ -1,134 +0,0 @@ -// Import the group class. -const group = require("./classes.js").group; - -// Create the parser function. -// parse() takes an array of tokens in, and outputs an -// Abstract Syntax Tree (a structured array of tokens). -module.exports = tokens => { - // Variables for later. - var layer = 0; - var delimiterCount = 0; - var deepestLayer = 0; - // Give each token a layer number based on delimiters. - for (var i = 0; i < tokens.length; i++) { - if (tokens[i].type == "Left Delimiter") { - layer++; - if (layer > deepestLayer) { - deepestLayer = layer; - } - delimiterCount++; - } - tokens[i].layer = layer; - if (tokens[i].type == "Right Delimiter") { - layer--; - } - } - // Lower the layer of delimiters. - for (var i = 0; i < tokens.length; i++) { - if ((tokens[i].type == "Left Delimiter") || (tokens[i].type == "Right Delimiter")) { - tokens[i].layer--; - } - } - if (layer > 0) { // Unclosed delimiter. - } else if (layer < 0) { // Overclosed delimiter. - } - // Give each token an index. - for (let i = 0; i < tokens.length; i++) { - tokens[i].index = i; - } - // Structure the layers. - // Count the rising edges of the layers to determine how many groups should exist. - let structure = function () { - let layer = 0; - let risingFalling = []; // Create an array to store indices of rising/falling edges. - for (let i = 0; i < tokens.length; i++) { - // Add a rising and a falling tag to each token. - tokens[i].rising = false; - tokens[i].falling = false; - if (tokens[i].layer > layer) { // If the token moves up a layer. - // Create a new rising index in risingFalling. - risingFalling.push({ - type: 'rising', - index: i - }); - tokens[i].rising = true; // Note that the token is a rising edge. - layer++; - } else if (tokens[i].layer < layer) { - // Create a new falling index in risingFalling. - risingFalling.push({ - type: 'falling', - index: i - }); - tokens[i].falling = true; // Note that the token is a falling edge. - layer--; - } - } - // Loop through the list of rising/falling edges. - for (let i = 0; i < risingFalling.length; i++) { - if (i != risingFalling.length - 1) { // If not the last edge. - let item = risingFalling[i]; - let nextItem = risingFalling[i + 1]; - // If a falling edge follows a rising edge, classifiy it as a group. - if ((item.type == 'rising') && (nextItem.type == 'falling')) { - // Get the group together as one item. - let selectedItems = tokens.slice(item.index, nextItem.index); - tokens.splice(item.index, selectedItems.length, new group(tokens[item.index - 1].value, selectedItems, item.index)); - } - } - } - risingFalling = []; // Reset the list of edges. - // Count the edges again. - for (let i = 0; i < tokens.length; i++) { - if (tokens[i].layer > layer) { - risingFalling.push({ - type: 'rising', - index: i - }); - layer++; - } else if (tokens[i].layer < layer) { - risingFalling.push({ - type: 'falling', - index: i - }); - layer--; - } - } - // If there are still edges, run again. - if (risingFalling.length) { - structure(); - } - }; - // Start the recursion. - structure(); - let trimDelimiters = function (thing) { - // Loop through the tokens of thing. - for (let i = 0; i < thing.length; i++) { - // Delete unnecessary keys. - if (typeof thing[i].rising != 'undefined') { - delete thing[i].rising; - } - if (typeof thing[i].falling != 'undefined') { - delete thing[i].falling; - } - if (typeof thing[i].index != 'undefined') { - delete thing[i].index; - } - if (typeof thing[i].index != 'undefined') { - delete thing[i].index; - } - // Remove delimiters. - if ((thing[i].type == 'Left Delimiter') || (thing[i].type == 'Right Delimiter')) { - thing.splice(i, 1); - i--; - } - // If a token is a group, look at the group's tokens. - if (thing[i].type == 'Group') { - trimDelimiters(thing[i].tokens); - } - } - }; - // Start the recursion. - trimDelimiters(tokens); - // Return the structured tokens. - return tokens; -}; \ No newline at end of file diff --git a/pivot.js b/pivot.js deleted file mode 100644 index 2b75014..0000000 --- a/pivot.js +++ /dev/null @@ -1,13 +0,0 @@ -// The Pivot code to run. -var code = `asdf = (a) {return(a++)}`; - -// Import the tokenizer and parser. -const tokenize = require("./tokenizer.js"); -const parse = require("./parser.js"); - -// Generate the AST. -var ast = parse(tokenize(code)); - -// Write the AST to ast.json. -var fs = require("fs"); -fs.writeFileSync("ast.json", JSON.stringify(ast, null, 4)); \ No newline at end of file diff --git a/readme.md b/readme.md index 6f46894..e69de29 100644 --- a/readme.md +++ b/readme.md @@ -1,15 +0,0 @@ -# [Pivot](https://gthub.us/pivot/) - -Pivot is a new programming language built on [JavaScript](https://javascript.com/) by [ElementG9](https://github.com/ElementG9/). - ---- - -## Installation -* Download or clone [the GitHub repository](/pivot/repo). -* Download [node.js](https://nodejs.org/). -* Run _pivot.js_ with `node pivot.js`. - ---- - -## Documentation -The Pivot documentation is [here](https://gthub.us/pivot/docs). diff --git a/rpn.js b/rpn.js deleted file mode 100644 index cf84d8d..0000000 --- a/rpn.js +++ /dev/null @@ -1,40 +0,0 @@ -var solve = (exp) => { - var stack = []; - var expression = exp.split(" "); - for(var j=0;j + * @requires module:types + */ +const Token = require('./types.js').Token; +const Group = require('./types.js').Token; + +module.exports = function(tokens) { + let level = 0; + // Add level markers. + tokens.forEach((t, i) => { + if (t.type == 'delimiter' && t.subtype == 'left') { + tokens[i].level = level; + level++; + } else if (t.type == 'delimiter' && t.subtype == 'right') { + level--; + tokens[i].level = level; + } else { + tokens[i].level = level; + } + }); + // Group. + let currentLevel = 0; + let groupStack = [0]; + tokens.forEach((t, i) => { + if (currentLevel < t.level) { + tokens.splice(i, 0, new Group(tokens[i - 1].value, [])); + groupStack.push(i); + currentLevel++; + tokens[i].level = currentLevel; + } + if (t.level != 0) { + tokens[groupStack.slice(-1)].tokens.push(t); + } + if (currentLevel > t.level) { + groupStack.pop(); + currentLevel--; + } + }); + if (currentLevel != 0) {} // Error: missing delimiter. + return tokens; +}; diff --git a/src/tokenizer.js b/src/tokenizer.js new file mode 100644 index 0000000..f61dd43 --- /dev/null +++ b/src/tokenizer.js @@ -0,0 +1,303 @@ +/** + * @module tokenizer + * @file Manages the tokenization phase of Pivot. + * @author Garen Tyler + * @requires module:types + */ +const Token = require('./types.js').Token; +const Group = require('./types.js').Token; + +/** + * @function tokenize + * @desc Takes in raw code, and outputs an array of Tokens. + * @param {string} code The raw input code. + * @returns {Token[]} The code, split into tokens. + * @public + */ +function tokenize(code) { + // Split the string into an array of chars. + let chars = code.split(''); + + // Create buffers. + let letterBuffer = []; + let operatorBuffer = []; + let numberBuffer = []; + let stringBuffer = []; + + // Create the output Token[]. + let tokens = []; + + // Create an object to keep track of string data. + let stringData = { + inString: false, + stringType: null + }; + + // Escape chars and remove comments. + chars = combineEscapedChars(chars); + chars = removeComments(chars); + + + // Actually tokenize the chars. + for (let i = 0; i < chars.length; i++) { + let char = chars[i]; + if (stringData.inString) { // Tokenize differently in a string. + // If a string delimiter and the same as the inital delimiter. + if (determineCharType(char) == 'string delimiter' && char == stringData.stringType) { + stringData.inString = false; // Not in a string any more. + tokens.push(new Token('string', 'n/a', stringBuffer.join(''))); // Push the string. + stringBuffer = []; // Clear the string buffer. + } else stringBuffer.push(char); // Add to the string buffer. + } else { // Tokenize normally. + if (determineCharType(char) == 'string delimiter') { + stringData.inString = true; // In a string now. + stringData.stringType = char; + } else if (determineCharType(char) == 'letter') { + letterBuffer.push(char); // Add to the letter buffer. + // End the other buffers. + if (operatorBuffer.length > 0) { + let operator = operatorBuffer.join(''); + tokens.push(new Token('operator', operatorType(operator), operator)); + operatorBuffer = []; + } + if (numberBuffer.length > 0) { + let number = numberBuffer.join(''); + tokens.push(new Token('number', 'n/a', number)); + numberBuffer = []; + } + } else if (determineCharType(char) == 'operator') { + operatorBuffer.push(char); // Add to the operator buffer. + // End the other buffers. + if (letterBuffer.length > 0) { + let variable = letterBuffer.join(''); + tokens.push(new Token('name', 'variable', variable)); + letterBuffer = []; + } + if (numberBuffer.length > 0) { + let number = numberBuffer.join(''); + tokens.push(new Token('number', 'n/a', number)); + numberBuffer = []; + } + } else if (determineCharType(char) == 'digit') { + numberBuffer.push(char); // Add to the number buffer. + // End the other buffers. + if (letterBuffer.length > 0) { + let variable = letterBuffer.join(''); + tokens.push(new Token('name', 'variable', variable)); + letterBuffer = []; + } + if (operatorBuffer.length > 0) { + let operator = operatorBuffer.join(''); + tokens.push(new Token('operator', operatorType(operator), operator)); + operatorBuffer = []; + } + } else if (determineCharType(char) == 'whitespace') { + // End all buffers. + if (letterBuffer.length > 0) { + let variable = letterBuffer.join(''); + tokens.push(new Token('name', 'variable', variable)); + letterBuffer = []; + } + if (numberBuffer.length > 0) { + let number = numberBuffer.join(''); + tokens.push(new Token('number', 'n/a', number)); + numberBuffer = []; + } + if (operatorBuffer.length > 0) { + let operator = operatorBuffer.join(''); + tokens.push(new Token('operator', operatorType(operator), operator)); + operatorBuffer = []; + } + } else if (determineCharType(char) == 'delimiter') { + // End all buffers. + if (letterBuffer.length > 0) { + let variable = letterBuffer.join(''); + tokens.push(new Token('name', 'variable', variable)); + letterBuffer = []; + } + if (numberBuffer.length > 0) { + let number = numberBuffer.join(''); + tokens.push(new Token('number', 'n/a', number)); + numberBuffer = []; + } + if (operatorBuffer.length > 0) { + let operator = operatorBuffer.join(''); + tokens.push(new Token('operator', operatorType(operator), operator)); + operatorBuffer = []; + } + // Push the delimiter. + tokens.push(getDelimiterToken(char)); + } + } + } + + // Empty all the buffers. + if (letterBuffer.length > 0) { + let variable = letterBuffer.join(''); + tokens.push(new Token('name', 'variable', variable)); + letterBuffer = []; + } + if (numberBuffer.length > 0) { + let number = numberBuffer.join(''); + tokens.push(new Token('number', 'n/a', number)); + numberBuffer = []; + } + if (operatorBuffer.length > 0) { + let operator = operatorBuffer.join(''); + tokens.push(new Token('operator', operatorType(operator), operator)); + operatorBuffer = []; + } + + tokens = changeKeywords(tokens); + + return tokens; +} + +/** + * @function combineEscapedChars + * @desc Combines escaped chars into one char. + * @param {string[]} chars The chars. + * @returns {string[]} The chars with combined escaped chars. + * @private + */ +function combineEscapedChars(chars) { + // Check for characters to be escaped. + for (let i = 0; i < chars.length; i++) { + if (chars[i] == '\\') { + chars.splice(i, 2, chars[i] + chars[i + 1]); + i -= 2; + } + } + return chars; +} + +/** + * @function removeComments + * @desc Removes comments. + * @param {string[]} chars The chars. + * @returns {string[]} The chars without comments. + * @private + */ +function removeComments(chars) { + let inComment = false; // Keep track if in a comment. + for (let i = 0; i < chars.length; i++) { + if (chars[i] == '/') { + if (chars[i + 1] == '/') { + inComment = true; + } + } + if (chars[i] == '\n') { + inComment = false; + chars.splice(i, 1); // Remove the newline at the end of the comment. + i--; + } + if (inComment) { + chars.splice(i, 1); // Remove the char in the comment. + i--; + } + } + return chars; +} + +/** + * @function changeKeywords + * @desc Changes tokens with subtype variable to subtype keyword + * @param {Token[]} tokens The tokens + * @returns {Token[]} The tokens with keywords. + * @private + */ +function changeKeywords(tokens) { + return tokens.map(t => { + if (t.subtype == 'variable' && determineType(t.value) == 'keyword') { + t.subtype = 'keyword'; + } + return t; + }); +} + +/** + * @function getDelimiterToken + * @desc Turns a delimiter char into a token. + * @param {string} delimiter The delimiter char. + * @returns {Token} The delimiter token. + * @private + */ +function getDelimiterToken(delimiter) { + if (/\(|\)/.test(delimiter)) + return new Token('delimiter', delimiter == '(' ? 'left' : 'right', 'parenthesis'); + else if (/\[|\]/.test(delimiter)) + return new Token('delimiter', delimiter == '[' ? 'left' : 'right', 'bracket'); + else if (/\{|\}/.test(delimiter)) + return new Token('delimiter', delimiter == '{' ? 'left' : 'right', 'brace'); + else throw new Error('Expected delimiter but got ' + delimiter); +} + +/** + * @function operatorType + * @desc Determines the type of operator. + * @param {string} operator The operator char. + * @returns {string} The type of operator. + * @private + */ +function operatorType(operator) { + // Left operators have parameters on the left. + if (/\+\+|--/.test(operator)) + return 'left'; + else if (false) + return 'right'; + else if (/\;/.test(operator)) + return 'none'; + else + return 'dual'; +} + +/** + * @function determineCharType + * @desc Detects the type of characters. + * @param {string} char The input character(s). + * @returns {string} The type of char. + * @private + */ +function determineCharType(char) { + if (/[A-Za-z]/.test(char)) + return 'letter'; + else if (/\+|\-|\*|\/|\=|\=\=|\>|\<|\>\=|\<\=|\=\>|;/.test(char)) + return 'operator'; + else if (/\(|\)|\[|\]|\{|\}/.test(char)) + return 'delimiter'; + else if (/'|"|`/.test(char)) + return 'string delimiter'; + else if (/\d/.test(char)) + return 'digit'; + else if (/\\./.test(char)) + return 'escaped char'; + else if (/\s/.test(char)) + return 'whitespace'; + else throw new SyntaxError('Unexpected char ' + char); +}; + +/** + * @function determineType + * @desc Detects the type of a string. + * @param {string} str The input string. + * @returns {string} The type of string. + * @private + */ +function determineType(str) { + if (/let|return/.test(str)) + return 'keyword'; + else return 'unknown'; +}; + +module.exports = { + tokenize, + util: { + combineEscapedChars, + removeComments, + changeKeywords, + getDelimiterToken, + operatorType, + determineCharType, + determineType + } +}; diff --git a/src/types.js b/src/types.js new file mode 100644 index 0000000..5d2a7a5 --- /dev/null +++ b/src/types.js @@ -0,0 +1,31 @@ +/** + * @module types + * @file Provides a consistent source of types for the compiler. + * @author Garen Tyler + */ + + +/** + * @class Token + * @classdesc Stores the type of token, subtype, and literal char value. + */ +function Token(type, subtype, value) { + this.type = type; + this.subtype = subtype; + this.value = value; +} + +/** + * @class Group + * @classdesc Stores the type of group, and the tokens in the group. + */ +function Group(type, tokens) { + this.type = 'group' + this.subtype = type; + this.tokens = tokens; +} + +module.exports = { + Token, + Group +}; diff --git a/test.pivot b/test.pivot deleted file mode 100644 index 98f93a2..0000000 --- a/test.pivot +++ /dev/null @@ -1,6 +0,0 @@ -// Example Pivot code. -// This is a comment. -x = 4 // This creates a variable x with the value of 4. -y = 3 * x // This creates a variable y with the value of the output of (3 * x). -asdf = (a) {return(a++)} // This creates a function asdf that takes a number and returns that number + 1/ -a = asdf(2) // This creates a variable a with the value of the output of asdf(2). diff --git a/test/test.js b/test/test.js new file mode 100644 index 0000000..365ae6d --- /dev/null +++ b/test/test.js @@ -0,0 +1,126 @@ +const assert = require('assert'); + +const types = require('../src/types.js'); +const tokenizer = require('../src/tokenizer.js'); +const parser = require('../src/parser.js'); + +describe('types.js', () => { + it('Has a Token child.', () => { + assert.equal(types.hasOwnProperty('Token'), true); + }); + it('Has a Group child.', () => { + assert.equal(types.hasOwnProperty('Group'), true); + }); + + describe('Token', () => { + it('Works as a constructor', () => { + try { + let token = new types.Token('a', 'b', 'c'); + } catch (err) { + throw err; + } + }); + it('Has values \'type\', \'subtype\', and \'value\'', () => { + try { + let token = new types.Token('a', 'b', 'c'); + if (!token.hasOwnProperty('type') || !token.hasOwnProperty('subtype') || !token.hasOwnProperty('value')) + throw new Error('Token is missing \'type\', \'subtype\', or \'value\' properties.'); + if (token.type != 'a' || token.subtype != 'b' || token.value != 'c') + throw new Error('Token incorrectly set \'type\', \'subtype\', or \'value\' properties.'); + } catch (err) { + throw err; + } + }); + }); + describe('Group', () => { + it('Works as a constructor', () => { + try { + let group = new types.Group('a', 'b'); + } catch (err) { + throw err; + } + }); + it('Has values \'type\', \'subtype\', and \'tokens\'', () => { + try { + let group = new types.Group('a', 'b'); + if (!group.hasOwnProperty('type') || !group.hasOwnProperty('subtype') || !group.hasOwnProperty('tokens')) + throw new Error('Group is missing \'type\', \'subtype\', or \'tokens\' properties.'); + if (group.type != 'group' || group.subtype != 'a' || group.tokens != 'b') + throw new Error('Group incorrectly set \'type\', \'subtype\', or \'tokens\' properties.'); + } catch (err) { + throw err; + } + }); + }); +}); +describe('tokenizer.js', () => { + it('Has a tokenize child', () => { + assert.equal(tokenizer.hasOwnProperty('tokenize'), true); + }); + it('Has a util child', () => { + assert.equal(tokenizer.hasOwnProperty('util'), true); + }); + describe('util', () => { + it('combineEscapedChars works', () => { + assert.equal(tokenizer.util.combineEscapedChars(`let x = 'test\\nnewline';`.split('')).join(''), `let x = 'test\\nnewline';`); + }); + it('removeComments works', () => { + assert.equal(tokenizer.util.removeComments(`// Comment\nlet i = 0;`.split('')).join(''), `let i = 0;`); + }); + it('changeKeywords works', () => { + let tokens = tokenizer.util.changeKeywords([{ + type: 'name', + subtype: 'variable', + value: 'let' + }, { + type: 'name', + subtype: 'variable', + value: 'x' + }]); + let correct = [{ + type: 'name', + subtype: 'keyword', + value: 'let' + }, { + type: 'name', + subtype: 'variable', + value: 'x' + }]; + let isCorrect = true; + tokens.forEach((t, i) => { + if (t.type != correct[i].type) + throw new Error('Changed type: Expected \''+ correct[i].type +'\' but got ' + t.type) + else if (t.subtype != correct[i].subtype) + throw new Error('Incorrectly changed subtype: Expected \''+ correct[i].subtype +'\' but got ' + t.subtype) + else if (t.value != correct[i].value) + throw new Error('Changed value: Expected \''+ correct[i].value +'\' but got ' + t.value) + }); + }); + it('getDelimiterToken works', () => { + let token = tokenizer.util.getDelimiterToken(')'); + if (token.type != 'delimiter') + throw new Error('Incorrect type: Expected \'delimiter\' but got ' + token.type) + else if (token.subtype != 'right') + throw new Error('Incorrect subtype: Expected \'right\' but got ' + token.subtype) + else if (token.value != 'parenthesis') + throw new Error('Incorrect value: Expected \'parenthesis\' but got ' + token.value) + }); + it('operatorType works', () => { + assert.equal(tokenizer.util.operatorType('++'), 'left'); + assert.equal(tokenizer.util.operatorType(';'), 'none'); + assert.equal(tokenizer.util.operatorType('+'), 'dual'); + }); + it('determineCharType works', () => { + assert.equal(tokenizer.util.determineCharType('+'), 'operator'); + assert.equal(tokenizer.util.determineCharType('"'), 'string delimiter'); + assert.equal(tokenizer.util.determineCharType('4'), 'digit'); + }); + it('determineType works', () => { + assert.equal(tokenizer.util.determineType('let'), 'keyword'); + assert.equal(tokenizer.util.determineType('dog'), 'unknown'); + }); + }); +}); +describe('parser.js', () => { + +}); diff --git a/tokenizer.js b/tokenizer.js deleted file mode 100644 index f8dab7d..0000000 --- a/tokenizer.js +++ /dev/null @@ -1,170 +0,0 @@ -// Import the token class. -const token = require("./classes.js").token; - -// Create the tokenizer function. -// tokenize() takes Pivot code in, and outputs an array of tokens. -module.exports = exp => { - // Check functions for different token types. - var isDigit = char => { - return /\d/.test(char); - }; - var isLetter = char => { - return /[a-z]/i.test(char); - }; - var isOperator = char => { - return /\+|-|\*|\/|\^|=/.test(char); - }; - var isLeftDelimiter = char => { - return (/\(|\[|\{|"|'|`/.test(char)); - }; - var isRightDelimiter = char => { - return (/\)|\]|\}/.test(char)); - }; - var isComma = char => { - return (char === ","); - }; - var isPeriod = char => { - return (char === "."); - }; - var result = []; // The final array of tokens. - var nb = []; // Number buffer. Allows for multiple digits to be one number. - var lb = []; // Letter buffer. Allows for multiple letters to be one variable / function. - var ob = []; // Operator buffer. Allows for multi-character operators. E.g. ++ or ==. - var sb = []; // String buffer. Allows for multi-character strings. - var inString = false; // Keep track of whether in string or not. - var stringType; // Keep track of what type of string. E.g. "" or ''. - exp = exp.split(""); // Split the expression into an array of characters. - /* - - - DO NOT TOUCH THIS - - - */ - for (var i = 0; i < exp.length; i++) { // Loop through all of the characters. - var char = exp[i]; // Create a quick reference to the current char. - if (i >= 1) { - if (exp[i - 1] == "\\") { - exp.splice(i - 1, 2, `\\${char}`); - i--; - continue; - } - if (exp[i - 1] == "$" && char == "{") { - exp.splice(i - 1, 2, `\${`); - i--; - continue; - } - } - } - /* - - - OK YOU CAN TOUCH AGAIN - - - */ - // Nevermind, just don't mess with any of this file. - for (var i = 0; i < exp.length; i++) { - var char = exp[i]; - if (inString) { - if (char == `'` || char == `"` || char == "`") { - var exitString = () => { - inString = false; - if (sb.length == 0) { - result.push(new token("String", null)); - } else { - var string = sb.join(""); - result.push(new token("String", string)); - } - sb = []; - }; - if (char == `'` && stringType == "single") { - exitString(); - } else if (char == `"` && stringType == "double") { - exitString(); - } else if (char == "`" && stringType == "backtick") { - exitString(); - } else { - if (char == `'`) { - sb.push(`\'`); - } - if (char == `"`) { - sb.push(`\"`); - } - if (char == "`") { - sb.push("\`"); - } - } - } else { - sb.push(char); - } - } else { - if (isDigit(char)) { - result.push(new token("Operator", ob.join(""))); - ob = []; - nb.push(char); - } else if (isLetter(char)) { - result.push(new token("Operator", ob.join(""))); - ob = []; - lb.push(char); - } else if (isOperator(char)) { - result.push(new token("Number", nb.join(""))); - nb = []; - result.push(new token("Variable", lb.join(""))); - lb = []; - ob.push(char); - } else if (isLeftDelimiter(char)) { - result.push(new token("Operator", ob.join(""))); - ob = []; - result.push(new token("Function Call", lb.join(""))); - lb = []; - if (char == `'` || char == `"` || char == "`") { - inString = true; - if (char == `'`) { - stringType = "single"; - } else if (char == `"`) { - stringType = "double"; - } else if (char == "`") { - stringType = "backtick"; - } - } else { - result.push(new token("Left Delimiter", char)); - } - } else if (isRightDelimiter(char)) { - result.push(new token("Operator", ob.join(""))); - ob = []; - result.push(new token("Number", nb.join(""))); - nb = []; - result.push(new token("Variable", lb.join(""))); - lb = []; - result.push(new token("Right Delimiter", char)); - } else if (isComma(char)) { - result.push(new token("Operator", ob.join(""))); - ob = []; - result.push(new token("Number", nb.join(""))); - nb = []; - result.push(new token("Variable", lb.join(""))); - lb = []; - result.push(new token("Comma", char)); - } else if (isPeriod(char)) { - result.push(new token("Operator", ob.join(""))); - ob = []; - nb.push(char); - } - } - } - result.push(new token("Operator", ob.join(""))); - ob = []; - result.push(new token("Number", nb.join(""))); - nb = []; - lb.forEach(item => { - result.push(new token("Variable", item)); - }); - lb = []; - for (var i = 0; i < 3; i++) { - result.forEach((item, index) => { - if (item.value == "") { - result.splice(index, 1); - } - }); - } - result.forEach((item, index) => { - if (item.value == "-" && index != 0) { - if (result[index - 1].type != "Variable" && result[index - 1].type != "Number") { - if (result[index + 1].type == "Number") { - result[index + 1].value = "-" + result[index + 1].value; - result.splice(index, 1); - } - } - } - }); - return result; -}; \ No newline at end of file From 812ec86acaf0cdd6202b12dfc3bc3af8b3b491d5 Mon Sep 17 00:00:00 2001 From: ElementG9 Date: Sat, 5 Oct 2019 21:07:18 -0600 Subject: [PATCH 04/15] the parsers finally working dont screw it up --- .gitignore | 1 + src/parser.js | 146 +++++++++++++++++----- src/tokenizer.js | 2 +- test/test.js | 318 +++++++++++++++++++++++++++++++++++++---------- 4 files changed, 366 insertions(+), 101 deletions(-) diff --git a/.gitignore b/.gitignore index 9c4ba90..b23e1a4 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ other/ package-lock.json node_modules/ jsdoc/ +ast.json diff --git a/src/parser.js b/src/parser.js index a42a501..7b8e251 100644 --- a/src/parser.js +++ b/src/parser.js @@ -1,44 +1,126 @@ /** * @module parser - * @file Manages the parser phase of Pivot. + * @file Manages the parsing phase of Pivot. * @author Garen Tyler * @requires module:types */ const Token = require('./types.js').Token; -const Group = require('./types.js').Token; +const Group = require('./types.js').Group; +const tokenizer = require('./tokenizer.js'); -module.exports = function(tokens) { +/** + * @function parse + * @desc Takes in an array of tokens, and outputs an AST. + * @param {Token[]} tokens The input tokens. + * @returns {Token[]} The tokens structured in an AST. + * @public + */ +function parse(tokens) { + // Create our output array. + let ast = tokens; + + // Add indexes and levels. + ast = addIndexes(ast); + ast = addLevels(ast); + + // Get the deepest level. + let deepestLevel = getDeepestLevel(ast); + + // Loop through for each level. + for (let currentLevel = deepestLevel; currentLevel > 0; currentLevel--) { + console.log('looping for level ' + currentLevel); + let groupBuffer = []; + for (let j = 0; j < ast.length; j++) { + // Create previousLevel and nextLevel. + // let previousTokenLevel = 0; + // if (typeof ast[j-1] != 'undefined') + // previousTokenLevel = ast[j-1].level; + let nextTokenLevel = 0; + if (typeof ast[j+1] != 'undefined') + nextTokenLevel = ast[j+1].level; + + if (ast[j].level == currentLevel) { + groupBuffer.push(ast[j]); // Add the token to the groupBuffer. + if (ast[j].level > nextTokenLevel) { + let g = new Group(groupBuffer[0].value, groupBuffer); + g.index = g.tokens[0].index; + g.level = g.tokens[0].level - 1; // -1 because the group is on the level below. + ast.splice(g.tokens[0].index, g.tokens.length + 1, g); + j = g.tokens[0].index; + groupBuffer = []; + } + } + + // // Take care of falling edges. + // if (ast[j].level > nextTokenLevel && ast[j].level == currentLevel) { + // // The first item in the group is always a delimiter, steal info from that. + // console.log(groupBuffer[0]); + // } + } + } + + return ast; +} + +/** + * @function addIndexes + * @desc Adds basic indexes to the tokens. + * @param {Token[]} tokens The tokens. + * @returns {Token[]} The tokens with indexes. + * @private + */ +function addIndexes(tokens) { + return tokens.map((t, i) => { + t.index = i; + return t; + }); +} + +/** + * @function addLevels + * @desc Adds basic levels to the tokens. The levels are dependent on the delimiters. + * @param {Token[]} tokens The tokens. + * @returns {Token[]} The tokens with levels. + * @private + */ +function addLevels(tokens) { let level = 0; - // Add level markers. - tokens.forEach((t, i) => { - if (t.type == 'delimiter' && t.subtype == 'left') { - tokens[i].level = level; + tokens = tokens.map((t, i) => { + if (t.type == 'delimiter' && t.subtype == 'left') level++; - } else if (t.type == 'delimiter' && t.subtype == 'right') { + t.level = level; + if (t.type == 'delimiter' && t.subtype == 'right') level--; - tokens[i].level = level; - } else { - tokens[i].level = level; - } + return t; }); - // Group. - let currentLevel = 0; - let groupStack = [0]; - tokens.forEach((t, i) => { - if (currentLevel < t.level) { - tokens.splice(i, 0, new Group(tokens[i - 1].value, [])); - groupStack.push(i); - currentLevel++; - tokens[i].level = currentLevel; - } - if (t.level != 0) { - tokens[groupStack.slice(-1)].tokens.push(t); - } - if (currentLevel > t.level) { - groupStack.pop(); - currentLevel--; - } - }); - if (currentLevel != 0) {} // Error: missing delimiter. - return tokens; + if (level > 0) + throw new SyntaxError('Missing closing delimiter'); + else if (level < 0) + throw new SyntaxError('Missing opening delimiter'); + else + return tokens; +} + +/** + * @function getDeepestLevel + * @desc Finds the deepest level. + * @param {Token[]} tokens The tokens. + * @returns {Number} The deepest level. + * @private + */ +function getDeepestLevel(tokens) { + return tokens.reduce((deepest, t) => { + return t.level > deepest ? t.level : deepest; + }, 0); +} + +module.exports = { + parse, + util: { + addIndexes, + addLevels, + getDeepestLevel + } }; + +require('fs').writeFileSync('ast.json', JSON.stringify(parse(tokenizer.tokenize('let x = (5 + (6 * 2));')), null, 2), () => {}); diff --git a/src/tokenizer.js b/src/tokenizer.js index f61dd43..d60c3f3 100644 --- a/src/tokenizer.js +++ b/src/tokenizer.js @@ -5,7 +5,7 @@ * @requires module:types */ const Token = require('./types.js').Token; -const Group = require('./types.js').Token; +const Group = require('./types.js').Group; /** * @function tokenize diff --git a/test/test.js b/test/test.js index 365ae6d..e1038e4 100644 --- a/test/test.js +++ b/test/test.js @@ -4,6 +4,8 @@ const types = require('../src/types.js'); const tokenizer = require('../src/tokenizer.js'); const parser = require('../src/parser.js'); +console.log(parser.parse(tokenizer.tokenize('let x = (5 + 3);'))); + describe('types.js', () => { it('Has a Token child.', () => { assert.equal(types.hasOwnProperty('Token'), true); @@ -54,73 +56,253 @@ describe('types.js', () => { }); }); describe('tokenizer.js', () => { - it('Has a tokenize child', () => { - assert.equal(tokenizer.hasOwnProperty('tokenize'), true); - }); - it('Has a util child', () => { - assert.equal(tokenizer.hasOwnProperty('util'), true); - }); - describe('util', () => { - it('combineEscapedChars works', () => { - assert.equal(tokenizer.util.combineEscapedChars(`let x = 'test\\nnewline';`.split('')).join(''), `let x = 'test\\nnewline';`); - }); - it('removeComments works', () => { - assert.equal(tokenizer.util.removeComments(`// Comment\nlet i = 0;`.split('')).join(''), `let i = 0;`); - }); - it('changeKeywords works', () => { - let tokens = tokenizer.util.changeKeywords([{ - type: 'name', - subtype: 'variable', - value: 'let' - }, { - type: 'name', - subtype: 'variable', - value: 'x' - }]); - let correct = [{ - type: 'name', - subtype: 'keyword', - value: 'let' - }, { - type: 'name', - subtype: 'variable', - value: 'x' - }]; - let isCorrect = true; - tokens.forEach((t, i) => { - if (t.type != correct[i].type) - throw new Error('Changed type: Expected \''+ correct[i].type +'\' but got ' + t.type) - else if (t.subtype != correct[i].subtype) - throw new Error('Incorrectly changed subtype: Expected \''+ correct[i].subtype +'\' but got ' + t.subtype) - else if (t.value != correct[i].value) - throw new Error('Changed value: Expected \''+ correct[i].value +'\' but got ' + t.value) + it('Has a tokenize child', () => { + assert.equal(tokenizer.hasOwnProperty('tokenize'), true); + }); + it('Has a util child', () => { + assert.equal(tokenizer.hasOwnProperty('util'), true); + }); + describe('util', () => { + it('combineEscapedChars works', () => { + assert.equal(tokenizer.util.combineEscapedChars(`let x = 'test\\nnewline';`.split('')).join(''), `let x = 'test\\nnewline';`); + }); + it('removeComments works', () => { + assert.equal(tokenizer.util.removeComments(`// Comment\nlet i = 0;`.split('')).join(''), `let i = 0;`); + }); + it('changeKeywords works', () => { + let tokens = tokenizer.util.changeKeywords([{ + type: 'name', + subtype: 'variable', + value: 'let' + }, { + type: 'name', + subtype: 'variable', + value: 'x' + }]); + let correct = [{ + type: 'name', + subtype: 'keyword', + value: 'let' + }, { + type: 'name', + subtype: 'variable', + value: 'x' + }]; + let isCorrect = true; + tokens.forEach((t, i) => { + if (t.type != correct[i].type) + throw new Error('Changed type: Expected \'' + correct[i].type + '\' but got \'' + t.type + '\'') + else if (t.subtype != correct[i].subtype) + throw new Error('Incorrectly changed subtype: Expected \'' + correct[i].subtype + '\' but got \'' + t.subtype + '\'') + else if (t.value != correct[i].value) + throw new Error('Changed value: Expected \'' + correct[i].value + '\' but got \'' + t.value + '\'') + }); + }); it('getDelimiterToken works', () => { + let token = tokenizer.util.getDelimiterToken(')'); + if (token.type != 'delimiter') + throw new Error('Incorrect type: Expected \'delimiter\' but got \'' + token.type + '\'') + else if (token.subtype != 'right') + throw new Error('Incorrect subtype: Expected \'right\' but got \'' + token.subtype + '\'') + else if (token.value != 'parenthesis') + throw new Error('Incorrect value: Expected \'parenthesis\' but got \'' + token.value + '\'') + }); it('operatorType works', () => { + assert.equal(tokenizer.util.operatorType('++'), 'left'); + assert.equal(tokenizer.util.operatorType(';'), 'none'); + assert.equal(tokenizer.util.operatorType('+'), 'dual'); + }); it('determineCharType works', () => { + assert.equal(tokenizer.util.determineCharType('+'), 'operator'); + assert.equal(tokenizer.util.determineCharType('"'), 'string delimiter'); + assert.equal(tokenizer.util.determineCharType('4'), 'digit'); + }); it('determineType works', () => { + assert.equal(tokenizer.util.determineType('let'), 'keyword'); + assert.equal(tokenizer.util.determineType('dog'), 'unknown'); + }); + }); + }); describe('parser.js', () => { + it('Has a parse child', () => { + assert.equal(parser.hasOwnProperty('parse'), true); + }); + it('Has a util child', () => { + assert.equal(parser.hasOwnProperty('util'), true); + }); + describe('util', () => { + it('addIndexes works', () => { + let tokens = parser.util.addIndexes([{ + type: 'name', + subtype: 'keyword', + value: 'let' + }, { + type: 'name', + subtype: 'variable', + value: 'x' + }]); + let correct = [{ + type: 'name', + subtype: 'keyword', + value: 'let', + index: 0 + }, { + type: 'name', + subtype: 'variable', + value: 'x', + index: 1 + }]; + let isCorrect = true; + tokens.forEach((t, i) => { + if (t.type != correct[i].type) + throw new Error('Changed type: Expected \'' + correct[i].type + '\' but got ' + t.type) + else if (t.subtype != correct[i].subtype) + throw new Error('Changed subtype: Expected \'' + correct[i].subtype + '\' but got ' + t.subtype) + else if (t.value != correct[i].value) + throw new Error('Changed value: Expected \'' + correct[i].value + '\' but got ' + t.value) + else if (t.index != correct[i].index) + throw new Error('Incorrect index: Expected \'' + correct[i].index + '\' but got ' + t.index) + }); + }); + it('addLevels works', () => { + let tokens = parser.util.addLevels([{ + type: 'name', + subtype: 'keyword', + value: 'let', + index: 0 + }, { + type: 'name', + subtype: 'variable', + value: 'x', + index: 1 + }, { + type: 'operator', + subtype: 'dual', + value: '=', + index: 2 + }, { + type: 'delimiter', + subtype: 'left', + value: 'parenthesis', + index: 3 + }, { + type: 'number', + subtype: 'n/a', + value: '5', + index: 4 + }, { + type: 'delimiter', + subtype: 'right', + value: 'parenthesis', + index: 5 + }]); + let correct = [{ + type: 'name', + subtype: 'keyword', + value: 'let', + index: 0, + level: 0 + }, { + type: 'name', + subtype: 'variable', + value: 'x', + index: 1, + level: 0 + }, { + type: 'operator', + subtype: 'dual', + value: '=', + index: 2, + level: 0 + }, { + type: 'delimiter', + subtype: 'left', + value: 'parenthesis', + index: 3, + level: 1 + }, { + type: 'number', + subtype: 'n/a', + value: '5', + index: 4, + level: 1 + }, { + type: 'delimiter', + subtype: 'right', + value: 'parenthesis', + index: 5, + level: 1 + }]; + let isCorrect = true; + tokens.forEach((t, i) => { + if (t.type != correct[i].type) + throw new Error('Changed type: Expected \'' + correct[i].type + '\' but got ' + t.type) + else if (t.subtype != correct[i].subtype) + throw new Error('Changed subtype: Expected \'' + correct[i].subtype + '\' but got \'' + t.subtype + '\'') + else if (t.value != correct[i].value) + throw new Error('Changed value: Expected \'' + correct[i].value + '\' but got \'' + t.value + '\'') + else if (t.index != correct[i].index) + throw new Error('Incorrect index: Expected \'' + correct[i].index + '\' but got \'' + t.index + '\'') + else if (t.level != correct[i].level) + throw new Error('Incorrect level: Expected \'' + correct[i].level + '\' but got \'' + t.level + '\'') + }); + }); + it('getDeepestLevel works', () => { + let deepestLevel = parser.util.getDeepestLevel([{ + type: 'name', + subtype: 'keyword', + value: 'let', + index: 0, + level: 0 + }, { + type: 'name', + subtype: 'variable', + value: 'x', + index: 1, + level: 0 + + }, { + type: 'operator', + subtype: 'dual', + value: '=', + index: 2, + level: 0 + }, { + type: 'delimiter', + subtype: 'left', + value: 'parenthesis', + index: 3, + level: 1 + }, { + type: 'number', + subtype: 'n/a', + value: '5', + index: 4, + level: 1 + }, { + type: 'operator', + subtype: 'dual', + value: '+', + index: 5, + level: 1 + }, { + type: 'number', + subtype: 'n/a', + value: '3', + index: 6, + level: 1 + }, + { + type: 'delimiter', + subtype: 'right', + value: 'parenthesis', + index: 7, + level: 1 + }, { + type: 'operator', + subtype: 'none', + value: ';', + index: 8, + level: 0 + } + ]); + if (deepestLevel != 1) + throw new Error('Incorrect deepestLevel. Expected \'1\' but got \'' + deepestLevel + '\''); + }); }); }); - it('getDelimiterToken works', () => { - let token = tokenizer.util.getDelimiterToken(')'); - if (token.type != 'delimiter') - throw new Error('Incorrect type: Expected \'delimiter\' but got ' + token.type) - else if (token.subtype != 'right') - throw new Error('Incorrect subtype: Expected \'right\' but got ' + token.subtype) - else if (token.value != 'parenthesis') - throw new Error('Incorrect value: Expected \'parenthesis\' but got ' + token.value) - }); - it('operatorType works', () => { - assert.equal(tokenizer.util.operatorType('++'), 'left'); - assert.equal(tokenizer.util.operatorType(';'), 'none'); - assert.equal(tokenizer.util.operatorType('+'), 'dual'); - }); - it('determineCharType works', () => { - assert.equal(tokenizer.util.determineCharType('+'), 'operator'); - assert.equal(tokenizer.util.determineCharType('"'), 'string delimiter'); - assert.equal(tokenizer.util.determineCharType('4'), 'digit'); - }); - it('determineType works', () => { - assert.equal(tokenizer.util.determineType('let'), 'keyword'); - assert.equal(tokenizer.util.determineType('dog'), 'unknown'); - }); - }); -}); -describe('parser.js', () => { - -}); From f1b73fdaad8aea7de92466afdd12779366eb1512 Mon Sep 17 00:00:00 2001 From: ElementG9 Date: Sat, 5 Oct 2019 21:40:53 -0600 Subject: [PATCH 05/15] Move grouping to a function --- src/parser.js | 76 +++++++++++++++++++++++++++------------------------ 1 file changed, 40 insertions(+), 36 deletions(-) diff --git a/src/parser.js b/src/parser.js index 7b8e251..4f06eea 100644 --- a/src/parser.js +++ b/src/parser.js @@ -23,41 +23,10 @@ function parse(tokens) { ast = addIndexes(ast); ast = addLevels(ast); - // Get the deepest level. - let deepestLevel = getDeepestLevel(ast); + // Combine the groups. + ast = combineGroups(ast); - // Loop through for each level. - for (let currentLevel = deepestLevel; currentLevel > 0; currentLevel--) { - console.log('looping for level ' + currentLevel); - let groupBuffer = []; - for (let j = 0; j < ast.length; j++) { - // Create previousLevel and nextLevel. - // let previousTokenLevel = 0; - // if (typeof ast[j-1] != 'undefined') - // previousTokenLevel = ast[j-1].level; - let nextTokenLevel = 0; - if (typeof ast[j+1] != 'undefined') - nextTokenLevel = ast[j+1].level; - - if (ast[j].level == currentLevel) { - groupBuffer.push(ast[j]); // Add the token to the groupBuffer. - if (ast[j].level > nextTokenLevel) { - let g = new Group(groupBuffer[0].value, groupBuffer); - g.index = g.tokens[0].index; - g.level = g.tokens[0].level - 1; // -1 because the group is on the level below. - ast.splice(g.tokens[0].index, g.tokens.length + 1, g); - j = g.tokens[0].index; - groupBuffer = []; - } - } - - // // Take care of falling edges. - // if (ast[j].level > nextTokenLevel && ast[j].level == currentLevel) { - // // The first item in the group is always a delimiter, steal info from that. - // console.log(groupBuffer[0]); - // } - } - } + // return ast; } @@ -103,7 +72,7 @@ function addLevels(tokens) { /** * @function getDeepestLevel - * @desc Finds the deepest level. + * @desc Finds the deepest level of the ast. * @param {Token[]} tokens The tokens. * @returns {Number} The deepest level. * @private @@ -114,12 +83,47 @@ function getDeepestLevel(tokens) { }, 0); } +/** + * @function combineGroups + * @desc Combine groups of tokens by delimiter. + * @param {Token[]} tokens The tokens. + * @returns {Token[]} The grouped tokens, or the basic ast. + * @private + */ +function combineGroups(tokens) { + // Get the deepest level. + let deepestLevel = getDeepestLevel(tokens); + + // Loop through for each level. + for (let currentLevel = deepestLevel; currentLevel > 0; currentLevel--) { + let groupBuffer = []; + for (let j = 0; j < tokens.length; j++) { + let nextTokenLevel = 0; + if (typeof tokens[j + 1] != 'undefined') + nextTokenLevel = tokens[j + 1].level; + if (tokens[j].level == currentLevel) { + groupBuffer.push(tokens[j]); // Add the token to the groupBuffer. + if (tokens[j].level > nextTokenLevel) { + let g = new Group(groupBuffer[0].value, groupBuffer); + g.index = g.tokens[0].index; + g.level = g.tokens[0].level - 1; // -1 because the group is on the level below. + tokens.splice(g.tokens[0].index, g.tokens.length + 1, g); + j = g.tokens[0].index; + groupBuffer = []; + } + } + } + } + return tokens; +} + module.exports = { parse, util: { addIndexes, addLevels, - getDeepestLevel + getDeepestLevel, + combineGroups } }; From 965d322405333eac0b26cd02d48130de54b2e042 Mon Sep 17 00:00:00 2001 From: ElementG9 Date: Sat, 5 Oct 2019 21:41:08 -0600 Subject: [PATCH 06/15] Add tests for combineGroups --- test/test.js | 595 ++++++++++++++++++++++++++++++--------------------- 1 file changed, 346 insertions(+), 249 deletions(-) diff --git a/test/test.js b/test/test.js index e1038e4..a1e103d 100644 --- a/test/test.js +++ b/test/test.js @@ -56,253 +56,350 @@ describe('types.js', () => { }); }); describe('tokenizer.js', () => { - it('Has a tokenize child', () => { - assert.equal(tokenizer.hasOwnProperty('tokenize'), true); - }); - it('Has a util child', () => { - assert.equal(tokenizer.hasOwnProperty('util'), true); - }); - describe('util', () => { - it('combineEscapedChars works', () => { - assert.equal(tokenizer.util.combineEscapedChars(`let x = 'test\\nnewline';`.split('')).join(''), `let x = 'test\\nnewline';`); - }); - it('removeComments works', () => { - assert.equal(tokenizer.util.removeComments(`// Comment\nlet i = 0;`.split('')).join(''), `let i = 0;`); - }); - it('changeKeywords works', () => { - let tokens = tokenizer.util.changeKeywords([{ - type: 'name', - subtype: 'variable', - value: 'let' - }, { - type: 'name', - subtype: 'variable', - value: 'x' - }]); - let correct = [{ - type: 'name', - subtype: 'keyword', - value: 'let' - }, { - type: 'name', - subtype: 'variable', - value: 'x' - }]; - let isCorrect = true; - tokens.forEach((t, i) => { - if (t.type != correct[i].type) - throw new Error('Changed type: Expected \'' + correct[i].type + '\' but got \'' + t.type + '\'') - else if (t.subtype != correct[i].subtype) - throw new Error('Incorrectly changed subtype: Expected \'' + correct[i].subtype + '\' but got \'' + t.subtype + '\'') - else if (t.value != correct[i].value) - throw new Error('Changed value: Expected \'' + correct[i].value + '\' but got \'' + t.value + '\'') - }); - }); it('getDelimiterToken works', () => { - let token = tokenizer.util.getDelimiterToken(')'); - if (token.type != 'delimiter') - throw new Error('Incorrect type: Expected \'delimiter\' but got \'' + token.type + '\'') - else if (token.subtype != 'right') - throw new Error('Incorrect subtype: Expected \'right\' but got \'' + token.subtype + '\'') - else if (token.value != 'parenthesis') - throw new Error('Incorrect value: Expected \'parenthesis\' but got \'' + token.value + '\'') - }); it('operatorType works', () => { - assert.equal(tokenizer.util.operatorType('++'), 'left'); - assert.equal(tokenizer.util.operatorType(';'), 'none'); - assert.equal(tokenizer.util.operatorType('+'), 'dual'); - }); it('determineCharType works', () => { - assert.equal(tokenizer.util.determineCharType('+'), 'operator'); - assert.equal(tokenizer.util.determineCharType('"'), 'string delimiter'); - assert.equal(tokenizer.util.determineCharType('4'), 'digit'); - }); it('determineType works', () => { - assert.equal(tokenizer.util.determineType('let'), 'keyword'); - assert.equal(tokenizer.util.determineType('dog'), 'unknown'); - }); - }); - }); describe('parser.js', () => { - it('Has a parse child', () => { - assert.equal(parser.hasOwnProperty('parse'), true); - }); - it('Has a util child', () => { - assert.equal(parser.hasOwnProperty('util'), true); - }); - describe('util', () => { - it('addIndexes works', () => { - let tokens = parser.util.addIndexes([{ - type: 'name', - subtype: 'keyword', - value: 'let' - }, { - type: 'name', - subtype: 'variable', - value: 'x' - }]); - let correct = [{ - type: 'name', - subtype: 'keyword', - value: 'let', - index: 0 - }, { - type: 'name', - subtype: 'variable', - value: 'x', - index: 1 - }]; - let isCorrect = true; - tokens.forEach((t, i) => { - if (t.type != correct[i].type) - throw new Error('Changed type: Expected \'' + correct[i].type + '\' but got ' + t.type) - else if (t.subtype != correct[i].subtype) - throw new Error('Changed subtype: Expected \'' + correct[i].subtype + '\' but got ' + t.subtype) - else if (t.value != correct[i].value) - throw new Error('Changed value: Expected \'' + correct[i].value + '\' but got ' + t.value) - else if (t.index != correct[i].index) - throw new Error('Incorrect index: Expected \'' + correct[i].index + '\' but got ' + t.index) - }); - }); - it('addLevels works', () => { - let tokens = parser.util.addLevels([{ - type: 'name', - subtype: 'keyword', - value: 'let', - index: 0 - }, { - type: 'name', - subtype: 'variable', - value: 'x', - index: 1 - }, { - type: 'operator', - subtype: 'dual', - value: '=', - index: 2 - }, { - type: 'delimiter', - subtype: 'left', - value: 'parenthesis', - index: 3 - }, { - type: 'number', - subtype: 'n/a', - value: '5', - index: 4 - }, { - type: 'delimiter', - subtype: 'right', - value: 'parenthesis', - index: 5 - }]); - let correct = [{ - type: 'name', - subtype: 'keyword', - value: 'let', - index: 0, - level: 0 - }, { - type: 'name', - subtype: 'variable', - value: 'x', - index: 1, - level: 0 - }, { - type: 'operator', - subtype: 'dual', - value: '=', - index: 2, - level: 0 - }, { - type: 'delimiter', - subtype: 'left', - value: 'parenthesis', - index: 3, - level: 1 - }, { - type: 'number', - subtype: 'n/a', - value: '5', - index: 4, - level: 1 - }, { - type: 'delimiter', - subtype: 'right', - value: 'parenthesis', - index: 5, - level: 1 - }]; - let isCorrect = true; - tokens.forEach((t, i) => { - if (t.type != correct[i].type) - throw new Error('Changed type: Expected \'' + correct[i].type + '\' but got ' + t.type) - else if (t.subtype != correct[i].subtype) - throw new Error('Changed subtype: Expected \'' + correct[i].subtype + '\' but got \'' + t.subtype + '\'') - else if (t.value != correct[i].value) - throw new Error('Changed value: Expected \'' + correct[i].value + '\' but got \'' + t.value + '\'') - else if (t.index != correct[i].index) - throw new Error('Incorrect index: Expected \'' + correct[i].index + '\' but got \'' + t.index + '\'') - else if (t.level != correct[i].level) - throw new Error('Incorrect level: Expected \'' + correct[i].level + '\' but got \'' + t.level + '\'') - }); - }); - it('getDeepestLevel works', () => { - let deepestLevel = parser.util.getDeepestLevel([{ - type: 'name', - subtype: 'keyword', - value: 'let', - index: 0, - level: 0 - }, { - type: 'name', - subtype: 'variable', - value: 'x', - index: 1, - level: 0 - - }, { - type: 'operator', - subtype: 'dual', - value: '=', - index: 2, - level: 0 - }, { - type: 'delimiter', - subtype: 'left', - value: 'parenthesis', - index: 3, - level: 1 - }, { - type: 'number', - subtype: 'n/a', - value: '5', - index: 4, - level: 1 - }, { - type: 'operator', - subtype: 'dual', - value: '+', - index: 5, - level: 1 - }, { - type: 'number', - subtype: 'n/a', - value: '3', - index: 6, - level: 1 - }, - { - type: 'delimiter', - subtype: 'right', - value: 'parenthesis', - index: 7, - level: 1 - }, { - type: 'operator', - subtype: 'none', - value: ';', - index: 8, - level: 0 - } - ]); - if (deepestLevel != 1) - throw new Error('Incorrect deepestLevel. Expected \'1\' but got \'' + deepestLevel + '\''); - }); - }); + it('combineEscapedChars works', () => { + assert.equal(tokenizer.util.combineEscapedChars(`let x = 'test\\nnewline';`.split('')).join(''), `let x = 'test\\nnewline';`); + }); + it('removeComments works', () => { + assert.equal(tokenizer.util.removeComments(`// Comment\nlet i = 0;`.split('')).join(''), `let i = 0;`); + }); + it('changeKeywords works', () => { + let tokens = tokenizer.util.changeKeywords([{ + type: 'name', + subtype: 'variable', + value: 'let' + }, { + type: 'name', + subtype: 'variable', + value: 'x' + }]); + let correct = [{ + type: 'name', + subtype: 'keyword', + value: 'let' + }, { + type: 'name', + subtype: 'variable', + value: 'x' + }]; + let isCorrect = true; + tokens.forEach((t, i) => { + if (t.type != correct[i].type) + throw new Error('Changed type: Expected \'' + correct[i].type + '\' but got \'' + t.type + '\'') + else if (t.subtype != correct[i].subtype) + throw new Error('Incorrectly changed subtype: Expected \'' + correct[i].subtype + '\' but got \'' + t.subtype + '\'') + else if (t.value != correct[i].value) + throw new Error('Changed value: Expected \'' + correct[i].value + '\' but got \'' + t.value + '\'') }); + }); + it('getDelimiterToken works', () => { + let token = tokenizer.util.getDelimiterToken(')'); + if (token.type != 'delimiter') + throw new Error('Incorrect type: Expected \'delimiter\' but got \'' + token.type + '\'') + else if (token.subtype != 'right') + throw new Error('Incorrect subtype: Expected \'right\' but got \'' + token.subtype + '\'') + else if (token.value != 'parenthesis') + throw new Error('Incorrect value: Expected \'parenthesis\' but got \'' + token.value + '\'') + }); + it('operatorType works', () => { + assert.equal(tokenizer.util.operatorType('++'), 'left'); + assert.equal(tokenizer.util.operatorType(';'), 'none'); + assert.equal(tokenizer.util.operatorType('+'), 'dual'); + }); + it('determineCharType works', () => { + assert.equal(tokenizer.util.determineCharType('+'), 'operator'); + assert.equal(tokenizer.util.determineCharType('"'), 'string delimiter'); + assert.equal(tokenizer.util.determineCharType('4'), 'digit'); + }); + it('determineType works', () => { + assert.equal(tokenizer.util.determineType('let'), 'keyword'); + assert.equal(tokenizer.util.determineType('dog'), 'unknown'); + }); +}); +describe('parser.js', () => { + it('addIndexes works', () => { + let tokens = parser.util.addIndexes([{ + type: 'name', + subtype: 'keyword', + value: 'let' + }, { + type: 'name', + subtype: 'variable', + value: 'x' + }]); + let correct = [{ + type: 'name', + subtype: 'keyword', + value: 'let', + index: 0 + }, { + type: 'name', + subtype: 'variable', + value: 'x', + index: 1 + }]; + let isCorrect = true; + tokens.forEach((t, i) => { + if (t.type != correct[i].type) + throw new Error('Changed type: Expected \'' + correct[i].type + '\' but got ' + t.type) + else if (t.subtype != correct[i].subtype) + throw new Error('Changed subtype: Expected \'' + correct[i].subtype + '\' but got ' + t.subtype) + else if (t.value != correct[i].value) + throw new Error('Changed value: Expected \'' + correct[i].value + '\' but got ' + t.value) + else if (t.index != correct[i].index) + throw new Error('Incorrect index: Expected \'' + correct[i].index + '\' but got ' + t.index) + }); + }); + it('addLevels works', () => { + let tokens = parser.util.addLevels([{ + type: 'name', + subtype: 'keyword', + value: 'let', + index: 0 + }, { + type: 'name', + subtype: 'variable', + value: 'x', + index: 1 + }, { + type: 'operator', + subtype: 'dual', + value: '=', + index: 2 + }, { + type: 'delimiter', + subtype: 'left', + value: 'parenthesis', + index: 3 + }, { + type: 'number', + subtype: 'n/a', + value: '5', + index: 4 + }, { + type: 'delimiter', + subtype: 'right', + value: 'parenthesis', + index: 5 + }]); + let correct = [{ + type: 'name', + subtype: 'keyword', + value: 'let', + index: 0, + level: 0 + }, { + type: 'name', + subtype: 'variable', + value: 'x', + index: 1, + level: 0 + }, { + type: 'operator', + subtype: 'dual', + value: '=', + index: 2, + level: 0 + }, { + type: 'delimiter', + subtype: 'left', + value: 'parenthesis', + index: 3, + level: 1 + }, { + type: 'number', + subtype: 'n/a', + value: '5', + index: 4, + level: 1 + }, { + type: 'delimiter', + subtype: 'right', + value: 'parenthesis', + index: 5, + level: 1 + }]; + let isCorrect = true; + tokens.forEach((t, i) => { + if (t.type != correct[i].type) + throw new Error('Changed type: Expected \'' + correct[i].type + '\' but got ' + t.type) + else if (t.subtype != correct[i].subtype) + throw new Error('Changed subtype: Expected \'' + correct[i].subtype + '\' but got \'' + t.subtype + '\'') + else if (t.value != correct[i].value) + throw new Error('Changed value: Expected \'' + correct[i].value + '\' but got \'' + t.value + '\'') + else if (t.index != correct[i].index) + throw new Error('Incorrect index: Expected \'' + correct[i].index + '\' but got \'' + t.index + '\'') + else if (t.level != correct[i].level) + throw new Error('Incorrect level: Expected \'' + correct[i].level + '\' but got \'' + t.level + '\'') + }); + }); + it('getDeepestLevel works', () => { + let deepestLevel = parser.util.getDeepestLevel([{ + type: 'name', + subtype: 'keyword', + value: 'let', + index: 0, + level: 0 + }, { + type: 'name', + subtype: 'variable', + value: 'x', + index: 1, + level: 0 + + }, { + type: 'operator', + subtype: 'dual', + value: '=', + index: 2, + level: 0 + }, { + type: 'delimiter', + subtype: 'left', + value: 'parenthesis', + index: 3, + level: 1 + }, { + type: 'number', + subtype: 'n/a', + value: '5', + index: 4, + level: 1 + }, { + type: 'operator', + subtype: 'dual', + value: '+', + index: 5, + level: 1 + }, { + type: 'delimiter', + subtype: 'left', + value: 'parenthesis', + index: 6, + level: 2 + }, { + type: 'number', + subtype: 'n/a', + value: '6', + index: 7, + level: 2 + }, { + type: 'operator', + subtype: 'dual', + value: '*', + index: 8, + level: 2 + }, { + type: 'number', + subtype: 'n/a', + value: '2', + index: 9, + level: 2 + }, { + type: 'delimiter', + subtype: 'right', + value: 'parenthesis', + index: 10, + level: 2 + }, { + type: 'delimiter', + subtype: 'right', + value: 'parenthesis', + index: 11, + level: 1 + }, { + type: 'operator', + subtype: 'none', + value: ';', + index: 12, + level: 0 + }]); + if (deepestLevel != 2) + throw new Error('Incorrect deepestLevel. Expected \'2\' but got \'' + deepestLevel + '\''); + }); + it('combineGroups works', () => { + let ast = parser.util.combineGroups([{ + type: 'name', + subtype: 'keyword', + value: 'let', + index: 0, + level: 0 + }, { + type: 'name', + subtype: 'variable', + value: 'x', + index: 1, + level: 0 + + }, { + type: 'operator', + subtype: 'dual', + value: '=', + index: 2, + level: 0 + }, { + type: 'delimiter', + subtype: 'left', + value: 'parenthesis', + index: 3, + level: 1 + }, { + type: 'number', + subtype: 'n/a', + value: '5', + index: 4, + level: 1 + }, { + type: 'operator', + subtype: 'dual', + value: '+', + index: 5, + level: 1 + }, { + type: 'delimiter', + subtype: 'left', + value: 'parenthesis', + index: 6, + level: 2 + }, { + type: 'number', + subtype: 'n/a', + value: '6', + index: 7, + level: 2 + }, { + type: 'operator', + subtype: 'dual', + value: '*', + index: 8, + level: 2 + }, { + type: 'number', + subtype: 'n/a', + value: '2', + index: 9, + level: 2 + }, { + type: 'delimiter', + subtype: 'right', + value: 'parenthesis', + index: 10, + level: 2 + }, { + type: 'delimiter', + subtype: 'right', + value: 'parenthesis', + index: 11, + level: 1 + }, { + type: 'operator', + subtype: 'none', + value: ';', + index: 12, + level: 0 + }]); + if (ast[3].type != 'group') + throw new Error('Incorrectly combined group.'); + if (ast[3].tokens[3].type != 'group') + throw new Error('Incorrectly combined group.'); + }); +}); From f796a43c16103e1fcea13f74e81b74ad0a504028 Mon Sep 17 00:00:00 2001 From: ElementG9 Date: Sun, 6 Oct 2019 17:34:25 -0600 Subject: [PATCH 07/15] First revision of operators --- doc/operators.txt | 81 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 doc/operators.txt diff --git a/doc/operators.txt b/doc/operators.txt new file mode 100644 index 0000000..f436169 --- /dev/null +++ b/doc/operators.txt @@ -0,0 +1,81 @@ +A higher precedence operator becomes an operand for a lower precedence one. +Higher precedence operators get resolved first. + +Operators in Pivot: ++-------------+------------+----------------------------------+--------------------+-----------------+ +| Precedence | Operator | Description | Operands | Associativity | ++-------------+------------+----------------------------------+--------------------+-----------------+ +| 16 | () | Grouping | internal | n/a | ++-------------+------------+----------------------------------+--------------------+-----------------+ +| 15 | . | Member Access | dual | Left to Right | +| 15 | [] | Computed Member Access | before, internal | Left to Right | +| 15 | () | Function Call | before, internal | Left to Right | ++-------------+------------+----------------------------------+--------------------+-----------------+ +| 14 | let | Variable Creation | after | Right to Left | +| 14 | const | Constant Creation | after | Right to Left | +| 14 | new | Object Creation | after | Right to Left | +| 14 | () {} | Function Creation | internal | n/a | ++-------------+------------+----------------------------------+--------------------+-----------------+ +| 13 | ++ | Postfix Increment | before | Left to Right | +| 13 | -- | Postfix Decrement | before | Left to Right | ++-------------+------------+----------------------------------+--------------------+-----------------+ +| 12 | ! | Logical NOT | after | Right to Left | +| 12 | - | Unary Negation | after | Right to Left | ++-------------+------------+----------------------------------+--------------------+-----------------+ +| 11 | ** | Exponentiation | dual | Right to Left | ++-------------+------------+----------------------------------+--------------------+-----------------+ +| 10 | * | Multiplication | dual | Left to Right | +| 10 | / | Division | dual | Left to Right | +| 10 | % | Modulus | dual | Left to Right | ++-------------+------------+----------------------------------+--------------------+-----------------+ +| 9 | + | Addition | dual | Left to Right | +| 9 | - | Subtraction | dual | Left to Right | ++-------------+------------+----------------------------------+--------------------+-----------------+ +| 7 | < | Less Than | dual | Left to Right | +| 7 | <= | Less Than or Equal | dual | Left to Right | +| 7 | > | Greater Than | dual | Left to Right | +| 7 | >= | Greater Than or Equal | dual | Left to Right | ++-------------+------------+----------------------------------+--------------------+-----------------+ +| 6 | == | Equality | dual | Left to Right | +| 6 | != | Inequality | dual | Left to Right | ++-------------+------------+----------------------------------+--------------------+-----------------+ +| 4 | && | Logical AND | dual | Left to Right | +| 4 | ^^ | Logical XOR | dual | Left to Right | +| 4 | || | Logical OR | dual | Left to Right | ++-------------+------------+----------------------------------+--------------------+-----------------+ +| 3 | = | Assignment | dual | Right to Left | +| 3 | += | Add and Assign | dual | Right to Left | +| 3 | -= | Subtract and Assign | dual | Right to Left | +| 3 | **= | Exponentiate and Assign | dual | Right to Left | +| 3 | *= | Multiply and Assign | dual | Right to Left | +| 3 | /= | Divide and Assign | dual | Right to Left | +| 3 | %= | Modulo and Assign | dual | Right to Left | ++-------------+------------+----------------------------------+--------------------+-----------------+ +| 2 | , | Comma | none | Left to Right | ++-------------+------------+----------------------------------+--------------------+-----------------+ +| 1 | ; | Statement End | before | Left to Right | ++-------------+------------+----------------------------------+--------------------+-----------------+ + +Possible Operators in Pivot: ++-------------+------------+----------------------------------+--------------------+-----------------+ +| Precedence | Operator | Description | Operands | Associativity | ++-------------+------------+----------------------------------+--------------------+-----------------+ +| 12 | ~ | Bitwise NOT | after | Right to Left | ++-------------+------------+----------------------------------+--------------------+-----------------+ +| 8 | << | Bitwise Left Shift | dual | Left to Right | +| 8 | >> | Bitwise Right Shift | dual | Left to Right | ++-------------+------------+----------------------------------+--------------------+-----------------+ +| 5 | and | Logical AND | dual | Left to Right | +| 5 | xor | Logical XOR | dual | Left to Right | +| 5 | or | Logical OR | dual | Left to Right | ++-------------+------------+----------------------------------+--------------------+-----------------+ +| 5 | & | Bitwise AND | dual | Left to Right | +| 5 | ^ | Bitwise XOR | dual | Left to Right | +| 5 | | | Bitwise OR | dual | Left to Right | ++-------------+------------+----------------------------------+--------------------+-----------------+ +| 3 | <<= | Bitwise Left Shift and Assign | dual | Right to Left | +| 3 | >>= | Bitwise Right Shift and Assign | dual | Right to Left | +| 3 | &= | Bitwise AND and Assign | dual | Right to Left | +| 3 | ^= | Bitwise XOR and Assign | dual | Right to Left | +| 3 | |= | Bitwise OR and Assign | dual | Right to Left | ++-------------+------------+----------------------------------+--------------------+-----------------+ From f0b8f181f7a2ba38a1567d152552d3b1a73cf2c4 Mon Sep 17 00:00:00 2001 From: ElementG9 Date: Sun, 24 Nov 2019 21:31:20 -0700 Subject: [PATCH 08/15] Add package data, member access, pre/postfix operators work now. --- bin/pivot.js | 28 ++++++++++++- package.json | 7 ++-- src/parser.js | 100 +++++++++++++++++++++++++++++++++++++++++------ src/tokenizer.js | 75 +++++++++++++++++++++++++++-------- src/types.js | 15 ++++++- test/test.js | 25 +++++++++++- 6 files changed, 215 insertions(+), 35 deletions(-) diff --git a/bin/pivot.js b/bin/pivot.js index 3d4a31f..cf4bb23 100755 --- a/bin/pivot.js +++ b/bin/pivot.js @@ -1,3 +1,29 @@ #!/usr/bin/env node const args = process.argv.slice(2); -console.log(args); +const tokenizer = require('../src/tokenizer.js'); +const parser = require('../src/parser.js'); + +if (typeof args[0] != 'undefined') { + // Execute from file. +} else { // REPL. + const rl = require('readline-sync'); + const exec = require('child_process').exec; + function repl(prompt, func) { + let answer; + while (answer != 'exit') { + answer = rl.question(prompt); + if (answer == 'exit') + process.exit(0); + if (answer == 'clear') { + console.clear(); + continue; + } + func(answer); + } + } + console.log('Welcome to Pivot v0.1.0 Alpha.'); + console.log('Type \'exit\' to exit.'); + repl('> ', (answer) => { + console.log(require('util').inspect(parser.parse(tokenizer.tokenize(answer)), { depth: null })); + }); +} diff --git a/package.json b/package.json index 1ce2203..183d6db 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { - "name": "pivot", - "version": "1.0.0", + "name": "pivot-lang", + "version": "0.1.0", "description": "Pivot is a new programming language built on JavaScript", "main": "./bin/pivot.js", "directories": { @@ -31,7 +31,8 @@ }, "preferGlobal": true, "dependencies": { - "fs": "0.0.1-security" + "fs": "0.0.1-security", + "readline-sync": "^1.4.10" }, "devDependencies": { "mocha": "^6.2.1" diff --git a/src/parser.js b/src/parser.js index 4f06eea..57a6ce6 100644 --- a/src/parser.js +++ b/src/parser.js @@ -4,9 +4,8 @@ * @author Garen Tyler * @requires module:types */ -const Token = require('./types.js').Token; const Group = require('./types.js').Group; -const tokenizer = require('./tokenizer.js'); +const Operator = require('./types.js').Operator; /** * @function parse @@ -23,10 +22,12 @@ function parse(tokens) { ast = addIndexes(ast); ast = addLevels(ast); - // Combine the groups. - ast = combineGroups(ast); - - // + // Start grouping by precedence + ast = grouping(ast); + ast = memberAccess(ast); + ast = postfixOperators(ast); + ast = prefixOperators(ast); + console.log(ast); return ast; } @@ -84,16 +85,15 @@ function getDeepestLevel(tokens) { } /** - * @function combineGroups + * @function grouping * @desc Combine groups of tokens by delimiter. * @param {Token[]} tokens The tokens. * @returns {Token[]} The grouped tokens, or the basic ast. * @private */ -function combineGroups(tokens) { +function grouping(tokens) { // Get the deepest level. let deepestLevel = getDeepestLevel(tokens); - // Loop through for each level. for (let currentLevel = deepestLevel; currentLevel > 0; currentLevel--) { let groupBuffer = []; @@ -107,8 +107,10 @@ function combineGroups(tokens) { let g = new Group(groupBuffer[0].value, groupBuffer); g.index = g.tokens[0].index; g.level = g.tokens[0].level - 1; // -1 because the group is on the level below. - tokens.splice(g.tokens[0].index, g.tokens.length + 1, g); + tokens.splice(g.tokens[0].index, g.tokens.length, g); j = g.tokens[0].index; + // Remove the delimiters in g.tokens. + g.tokens = g.tokens.splice(1, groupBuffer.length - 2); groupBuffer = []; } } @@ -117,14 +119,88 @@ function combineGroups(tokens) { return tokens; } +/** + * @function memberAccess + * @desc Combine groups of tokens by member access. + * @param {Token[]} tokens The tokens. + * @returns {Token[]} The grouped tokens, or the basic ast. + * @private + */ +function memberAccess(ast) { + for (let i = 0; i < ast.length; i++) { + if (ast[i].type == 'group') + memberAccess(ast[i].tokens); // Recursively order the groups. + else if (ast[i].type == 'operator' && ast[i].value == '.') { // Member access operator. + if (typeof ast[i - 1] == 'undefined' || typeof ast[i + 1] == 'undefined') + throw new SyntaxError('Operator requires two operands.'); + let op = new Operator(ast[i].subtype, ast[i].value, [ast[i - 1], ast[i + 1]]); + op.index = ast[i - 1].index; + op.level = ast[i].level; + ast.splice(i - 1, 3, op); + i--; // Removed 3 tokens, put in 1, skip 1 token. Reduce the counter by 1. + } + } + return ast; +} + +/** + * @function postfixOperators + * @desc Recursively structures the postfix operators. + * @param {Token[]} ast The ast. + * @returns {Token[]} The ast with structured postfix operators. + * @private + */ +function postfixOperators(ast) { + for (let i = 0; i < ast.length; i++) { + // Take care of the tokens in the groups. + if (ast[i].type == 'group') + ast[i].tokens = postfixOperators(ast[i].tokens); + else if (ast[i].type == 'operator' && ast[i].subtype == 'postfix') { // The operand is on the left. + if (typeof ast[i - 1] == 'undefined') + throw new SyntaxError('Postfix operator requires one operand before it.'); + let op = new Operator(ast[i].subtype, ast[i].value, [ast[i - 1]]); + op.index = ast[i].index; + op.level = ast[i].level; + ast.splice(i - 1, 2, op); + // Removing 2 tokens, adding 1, skip 1 token. Don't reduce the counter. + } + } + return ast; +} + +/** + * @function prefixOperators + * @desc Recursively structures the prefix operators. + * @param {Token[]} ast The ast. + * @returns {Token[]} The ast with structured prefix operators. + * @private + */ +function prefixOperators(ast) { + for (let i = 0; i < ast.length; i++) { + // Take care of the tokens in the groups. + if (ast[i].type == 'group') + ast[i].tokens = postfixOperators(ast[i].tokens); + else if (ast[i].type == 'operator' && ast[i].subtype == 'prefix') { // The operand is on the right. + if (typeof ast[i + 1] == 'undefined') + throw new SyntaxError('Prefix operator requires one operand after it.'); + let op = new Operator(ast[i].subtype, ast[i].value, [ast[i + 1]]); + op.index = ast[i].index; + op.level = ast[i].level; + ast.splice(i, 2, op); + // Removing 2 tokens, adding 1, skip 1 token. Don't reduce the counter. + } + } + return ast; +} + module.exports = { parse, util: { addIndexes, addLevels, getDeepestLevel, - combineGroups + grouping, } }; -require('fs').writeFileSync('ast.json', JSON.stringify(parse(tokenizer.tokenize('let x = (5 + (6 * 2));')), null, 2), () => {}); +// require('fs').writeFileSync('ast.json', JSON.stringify(parse(tokenizer.tokenize('let x = (5 + (6 * 2)) - 7;')), null, 2), () => {}); diff --git a/src/tokenizer.js b/src/tokenizer.js index d60c3f3..4a3ecbd 100644 --- a/src/tokenizer.js +++ b/src/tokenizer.js @@ -5,7 +5,6 @@ * @requires module:types */ const Token = require('./types.js').Token; -const Group = require('./types.js').Group; /** * @function tokenize @@ -234,21 +233,64 @@ function getDelimiterToken(delimiter) { /** * @function operatorType - * @desc Determines the type of operator. - * @param {string} operator The operator char. - * @returns {string} The type of operator. + * @desc Turns a delimiter char into a token. + * @param {string} delimiter The delimiter char. + * @returns {Token} The delimiter token. * @private */ -function operatorType(operator) { - // Left operators have parameters on the left. - if (/\+\+|--/.test(operator)) - return 'left'; - else if (false) - return 'right'; - else if (/\;/.test(operator)) - return 'none'; - else - return 'dual'; +function operatorType(op) { + switch (op) { + case '.': + case '+': + case '-': + case '*': + case '/': + case '**': + case '%': + case '<<': + case '>>': + case '<': + case '<=': + case '>': + case '>=': + case '==': + case '!=': + case '&': + case '|': + case '^': + case '&&': + case '||': + case '^^': + case '=': + case '+=': + case '-=': + case '*=': + case '/=': + case '**=': + case '%=': + case '<<=': + case '>>=': + case '&=': + case '|=': + case '^=': + return 'dual'; + break; + case '++': + case '--': + return 'postfix'; + break; + case '!': + case '~': + return 'prefix'; + break; + case ',': + case ';': + return 'none'; + break; + default: + throw new TypeError('Unexpected operator ' + op); + break; + } } /** @@ -261,7 +303,8 @@ function operatorType(operator) { function determineCharType(char) { if (/[A-Za-z]/.test(char)) return 'letter'; - else if (/\+|\-|\*|\/|\=|\=\=|\>|\<|\>\=|\<\=|\=\>|;/.test(char)) + else if (/\.|\+\+|--|!|~|\+|-|\*\*|\*|\/%|<<|>>|<|<=|>|>=|==|!=|&|\^|\||&&|\|\||=|\+=|-=|\*\*=|\*=|\/=|%=|<<=|>>=|&=|\^=|\|=|,|;/.test(char)) + // All the operators in Pivot. return 'operator'; else if (/\(|\)|\[|\]|\{|\}/.test(char)) return 'delimiter'; @@ -273,7 +316,7 @@ function determineCharType(char) { return 'escaped char'; else if (/\s/.test(char)) return 'whitespace'; - else throw new SyntaxError('Unexpected char ' + char); + else throw new TypeError('Unexpected char ' + char); }; /** diff --git a/src/types.js b/src/types.js index 5d2a7a5..1bbc0cc 100644 --- a/src/types.js +++ b/src/types.js @@ -4,7 +4,6 @@ * @author Garen Tyler */ - /** * @class Token * @classdesc Stores the type of token, subtype, and literal char value. @@ -25,7 +24,19 @@ function Group(type, tokens) { this.tokens = tokens; } +/** + * @class Operator + * @classdesc Stores the type of operator, and tokens for an operator. + */ +function Operator(subtype, value, operands) { + this.type = 'operator'; + this.subtype = subtype; + this.value = value; + this.operands = operands; +} + module.exports = { Token, - Group + Group, + Operator }; diff --git a/test/test.js b/test/test.js index a1e103d..3b53bc8 100644 --- a/test/test.js +++ b/test/test.js @@ -13,6 +13,9 @@ describe('types.js', () => { it('Has a Group child.', () => { assert.equal(types.hasOwnProperty('Group'), true); }); + it('Has a Operator child.', () => { + assert.equal(types.hasOwnProperty('Operator'), true); + }); describe('Token', () => { it('Works as a constructor', () => { @@ -54,6 +57,26 @@ describe('types.js', () => { } }); }); + // describe('Operator', () => { + // it('Works as a constructor', () => { + // try { + // let op = new types.Operator('dual', '+', ['3','4']); + // } catch (err) { + // throw err; + // } + // }); + // it('Has values \'type\', \'subtype\', \'value\', and \'operands\'', () => { + // try { + // let op = new types.Operator('dual', '+', ['3','4']); + // if (!op.hasOwnProperty('type') || !op.hasOwnProperty('subtype') || !op.hasOwnProperty('value') || !op.hasOwnProperty('operands')) + // throw new Error('Operator is missing \'type\', \'subtype\', \'value\', or \'operands\' properties.'); + // if (op.type != 'operator' || op.subtype != 'dual' || op.value != '+' || op.operands[0] != '3') + // throw new Error('Operator incorrectly set \'type\', \'subtype\', \'value\', or \'operands\' properties.'); + // } catch (err) { + // throw err; + // } + // }); + // }); }); describe('tokenizer.js', () => { it('combineEscapedChars works', () => { @@ -101,7 +124,7 @@ describe('tokenizer.js', () => { throw new Error('Incorrect value: Expected \'parenthesis\' but got \'' + token.value + '\'') }); it('operatorType works', () => { - assert.equal(tokenizer.util.operatorType('++'), 'left'); + assert.equal(tokenizer.util.operatorType('++'), 'postfix'); assert.equal(tokenizer.util.operatorType(';'), 'none'); assert.equal(tokenizer.util.operatorType('+'), 'dual'); }); From dbb8a5161e88f4bfa12215fd2a83ea400d42e778 Mon Sep 17 00:00:00 2001 From: ElementG9 Date: Mon, 25 Nov 2019 10:52:15 -0700 Subject: [PATCH 09/15] Save before fixing grouping --- doc/operators.txt | 69 ++++++++++++------------ src/parser.js | 130 ++++++++++++++++++++++++++++++++++++++++++++-- src/tokenizer.js | 2 +- 3 files changed, 161 insertions(+), 40 deletions(-) diff --git a/doc/operators.txt b/doc/operators.txt index f436169..b0c4fb3 100644 --- a/doc/operators.txt +++ b/doc/operators.txt @@ -5,51 +5,52 @@ Operators in Pivot: +-------------+------------+----------------------------------+--------------------+-----------------+ | Precedence | Operator | Description | Operands | Associativity | +-------------+------------+----------------------------------+--------------------+-----------------+ -| 16 | () | Grouping | internal | n/a | +| 16 | () | Grouping | internal | n/a | grouping() +-------------+------------+----------------------------------+--------------------+-----------------+ -| 15 | . | Member Access | dual | Left to Right | -| 15 | [] | Computed Member Access | before, internal | Left to Right | -| 15 | () | Function Call | before, internal | Left to Right | +| 15 | . | Member Access | dual | Left to Right | memberAccess() +| 15 | [] | Computed Member Access | before, internal | Left to Right | computedMemberAccess() +| 15 | () | Function Call | before, internal | Left to Right | functionCall() +-------------+------------+----------------------------------+--------------------+-----------------+ -| 14 | let | Variable Creation | after | Right to Left | -| 14 | const | Constant Creation | after | Right to Left | -| 14 | new | Object Creation | after | Right to Left | -| 14 | () {} | Function Creation | internal | n/a | +| 14 | let | Variable Creation | after | Right to Left | keywords() +| 14 | const | Constant Creation | after | Right to Left | keywords() +| 14 | new | Object Creation | after | Right to Left | keywords() +| 14 | return | Function Return | after | n/a | keywords() +| 14 | () {} | Function Creation | internal | n/a | functionCreation() +-------------+------------+----------------------------------+--------------------+-----------------+ -| 13 | ++ | Postfix Increment | before | Left to Right | -| 13 | -- | Postfix Decrement | before | Left to Right | +| 13 | ++ | Postfix Increment | before | Left to Right | postfixOperators() +| 13 | -- | Postfix Decrement | before | Left to Right | postfixOperators() +-------------+------------+----------------------------------+--------------------+-----------------+ -| 12 | ! | Logical NOT | after | Right to Left | -| 12 | - | Unary Negation | after | Right to Left | +| 12 | ! | Logical NOT | after | Right to Left | prefixOperators() +| 12 | - | Unary Negation | after | Right to Left | prefixOperators() +-------------+------------+----------------------------------+--------------------+-----------------+ -| 11 | ** | Exponentiation | dual | Right to Left | +| 11 | ** | Exponentiation | dual | Right to Left | mathOperators(0) +-------------+------------+----------------------------------+--------------------+-----------------+ -| 10 | * | Multiplication | dual | Left to Right | -| 10 | / | Division | dual | Left to Right | -| 10 | % | Modulus | dual | Left to Right | +| 10 | * | Multiplication | dual | Left to Right | mathOperators(1) +| 10 | / | Division | dual | Left to Right | mathOperators(1) +| 10 | % | Modulus | dual | Left to Right | mathOperators(1) +-------------+------------+----------------------------------+--------------------+-----------------+ -| 9 | + | Addition | dual | Left to Right | -| 9 | - | Subtraction | dual | Left to Right | +| 9 | + | Addition | dual | Left to Right | mathOperators(2) +| 9 | - | Subtraction | dual | Left to Right | mathOperators(2) +-------------+------------+----------------------------------+--------------------+-----------------+ -| 7 | < | Less Than | dual | Left to Right | -| 7 | <= | Less Than or Equal | dual | Left to Right | -| 7 | > | Greater Than | dual | Left to Right | -| 7 | >= | Greater Than or Equal | dual | Left to Right | +| 7 | < | Less Than | dual | Left to Right | comparisonOperators() +| 7 | <= | Less Than or Equal | dual | Left to Right | comparisonOperators() +| 7 | > | Greater Than | dual | Left to Right | comparisonOperators() +| 7 | >= | Greater Than or Equal | dual | Left to Right | comparisonOperators() +-------------+------------+----------------------------------+--------------------+-----------------+ -| 6 | == | Equality | dual | Left to Right | -| 6 | != | Inequality | dual | Left to Right | +| 6 | == | Equality | dual | Left to Right | assign() +| 6 | != | Inequality | dual | Left to Right | assign() +-------------+------------+----------------------------------+--------------------+-----------------+ -| 4 | && | Logical AND | dual | Left to Right | -| 4 | ^^ | Logical XOR | dual | Left to Right | -| 4 | || | Logical OR | dual | Left to Right | +| 4 | && | Logical AND | dual | Left to Right | logicOperators() +| 4 | ^^ | Logical XOR | dual | Left to Right | logicOperators() +| 4 | || | Logical OR | dual | Left to Right | logicOperators() +-------------+------------+----------------------------------+--------------------+-----------------+ -| 3 | = | Assignment | dual | Right to Left | -| 3 | += | Add and Assign | dual | Right to Left | -| 3 | -= | Subtract and Assign | dual | Right to Left | -| 3 | **= | Exponentiate and Assign | dual | Right to Left | -| 3 | *= | Multiply and Assign | dual | Right to Left | -| 3 | /= | Divide and Assign | dual | Right to Left | -| 3 | %= | Modulo and Assign | dual | Right to Left | +| 3 | = | Assignment | dual | Right to Left | opAssign() +| 3 | += | Add and Assign | dual | Right to Left | opAssign() +| 3 | -= | Subtract and Assign | dual | Right to Left | opAssign() +| 3 | **= | Exponentiate and Assign | dual | Right to Left | opAssign() +| 3 | *= | Multiply and Assign | dual | Right to Left | opAssign() +| 3 | /= | Divide and Assign | dual | Right to Left | opAssign() +| 3 | %= | Modulo and Assign | dual | Right to Left | opAssign() +-------------+------------+----------------------------------+--------------------+-----------------+ | 2 | , | Comma | none | Left to Right | +-------------+------------+----------------------------------+--------------------+-----------------+ diff --git a/src/parser.js b/src/parser.js index 57a6ce6..8c22311 100644 --- a/src/parser.js +++ b/src/parser.js @@ -22,12 +22,21 @@ function parse(tokens) { ast = addIndexes(ast); ast = addLevels(ast); - // Start grouping by precedence + // Start grouping by precedence. + + // Precedence 16. ast = grouping(ast); + // Precedence 15. ast = memberAccess(ast); + ast = computedMemberAccess(ast); + ast = functionCall(ast); + // Precedence 14. + ast = keywords(ast); + ast = functionCreation(ast); + // Precedence 13. + // Precedence 12. ast = postfixOperators(ast); ast = prefixOperators(ast); - console.log(ast); return ast; } @@ -123,13 +132,15 @@ function grouping(tokens) { * @function memberAccess * @desc Combine groups of tokens by member access. * @param {Token[]} tokens The tokens. - * @returns {Token[]} The grouped tokens, or the basic ast. + * @returns {Token[]} The ast with grouped member access. * @private */ +// TODO: Member access function memberAccess(ast) { + console.log(ast); for (let i = 0; i < ast.length; i++) { if (ast[i].type == 'group') - memberAccess(ast[i].tokens); // Recursively order the groups. + ast[i].tokens = memberAccess(ast[i].tokens); // Recursively order the groups. else if (ast[i].type == 'operator' && ast[i].value == '.') { // Member access operator. if (typeof ast[i - 1] == 'undefined' || typeof ast[i + 1] == 'undefined') throw new SyntaxError('Operator requires two operands.'); @@ -143,6 +154,115 @@ function memberAccess(ast) { return ast; } +/** + * @function computedMemberAccess + * @desc Combine groups of tokens by computed member access. + * @param {Token[]} tokens The tokens. + * @returns {Token[]} The ast with grouped computed member access. + * @private + */ +function computedMemberAccess(ast) { + // Computed member access is Variable, Bracket Group. + for (let i = 0; i < ast.length; i++) { + if (ast[i].type == 'group') + ast[i].tokens = computedMemberAccess(ast[i].tokens); // Recursively order the groups. + else if (ast[i].type == 'name' && ast[i].subtype == 'variable') { // Member access operator. + if (typeof ast[i + 1] == 'undefined') + continue; // Nothing after the variable; skip this loop. + if (ast[i + 1].type == 'group' && ast[i + 1].subtype == 'bracket') { + ast[i + 1].tokens = computedMemberAccess(ast[i + 1].tokens); // Order the group that we care about before we mess with it. + let op = new Operator('n/a', 'member access', [ast[i], ast[i + 1]]); + op.index = ast[i].index; + op.level = ast[i].level; + ast.splice(i, 2, op); + // Removed 2 tokens, put in 1, skip 1 token. Don't reduce the counter. + } else continue; // Not what we need. + } + } + return ast; +} + +/** + * @function functionCall + * @desc Combine groups of tokens by function calls. + * @param {Token[]} tokens The tokens. + * @returns {Token[]} The ast with grouped function calls. + * @private + */ +function functionCall(ast) { + // Function call is Variable, Parenthesis Group. + for (let i = 0; i < ast.length; i++) { + if (ast[i].type == 'group') + ast[i].tokens = functionCall(ast[i].tokens); // Recursively order the groups. + else if (ast[i].type == 'name' && ast[i].subtype == 'variable') { // Member access operator. + if (typeof ast[i + 1] == 'undefined') + continue; // Nothing after the variable; skip this loop. + if (ast[i + 1].type == 'group' && ast[i + 1].subtype == 'parenthesis') { + ast[i + 1].tokens = functionCall(ast[i + 1].tokens); // Order the group that we care about before we mess with it. + let op = new Operator('function call', ast[i].value, [ast[i], ast[i + 1]]); + op.index = ast[i].index; + op.level = ast[i].level; + ast.splice(i, 2, op); + // Removed 2 tokens, put in 1, skip 1 token. Don't reduce the counter. + } else continue; // Not what we need. + } + } + return ast; +} + +/** + * @function keywords + * @desc Combine groups of tokens by keywords. + * @param {Token[]} tokens The tokens. + * @returns {Token[]} The ast with grouped keywords. + * @private + */ +function keywords(ast) { + for (let i = ast.length - 1; i >= 0; i--) { // Keywords are rtl associative, so loop backwards. + if (ast[i].type == 'group') + ast[i].tokens = keywords(ast[i].tokens); // Recursively order the groups. + else if (ast[i].type == 'name' && ast[i].subtype == 'keyword') { + if (typeof ast[i + 1] == 'undefined') + throw new SyntaxError('Keyword requires one operand after it.'); + let key = new Operator('keyword', ast[i].value, [ast[i + 1]]); + key.level = ast[i].level; + key.index = ast[i].index; + ast.splice(i, 2, key); + // Looping backwards and didn't remove any items before the current one. Don't reduce the counter. + } + } + return ast; +} + +/** + * @function functionCreation + * @desc Combine groups of tokens by function creation. + * @param {Token[]} tokens The tokens. + * @returns {Token[]} The ast with grouped function creation. + * @private + */ +function functionCreation(ast) { + // Function call is Parenthesis Group, Brace Group. + console.log(ast); + for (let i = 0; i < ast.length; i++) { + if (ast[i].type == 'group') + ast[i].tokens = functionCreation(ast[i].tokens); // Recursively order the groups. + else if (ast[i].type == 'group' && ast[i].subtype == 'parenthesis') { + if (typeof ast[i + 1] == 'undefined') + continue; // Nothing after this group, ignore it. + else if (ast[i + 1] == 'group' && ast[i].subtype == 'brace') { + ast[i + 1].tokens = functionCreation(ast[i + 1].tokens); // Order the group that we care about before we mess with it. + let op = new Operator('n/a', 'function creation', [ast[i], ast[i + 1]]); + op.index = ast[i].index; + op.level = ast[i].level; + ast.splice(i, 2, op); + // Removed 2 tokens, put in 1, skip 1 token. Don't reduce the counter. + } else continue; + } + } + return ast; +} + /** * @function postfixOperators * @desc Recursively structures the postfix operators. @@ -181,7 +301,7 @@ function prefixOperators(ast) { if (ast[i].type == 'group') ast[i].tokens = postfixOperators(ast[i].tokens); else if (ast[i].type == 'operator' && ast[i].subtype == 'prefix') { // The operand is on the right. - if (typeof ast[i + 1] == 'undefined') + if (typeof ast[i - 1] == 'undefined') throw new SyntaxError('Prefix operator requires one operand after it.'); let op = new Operator(ast[i].subtype, ast[i].value, [ast[i + 1]]); op.index = ast[i].index; diff --git a/src/tokenizer.js b/src/tokenizer.js index 4a3ecbd..8438ec8 100644 --- a/src/tokenizer.js +++ b/src/tokenizer.js @@ -327,7 +327,7 @@ function determineCharType(char) { * @private */ function determineType(str) { - if (/let|return/.test(str)) + if (/let|const|new|return/.test(str)) return 'keyword'; else return 'unknown'; }; From 432316d640c10e5fff670abe22ebf713b4abe9a8 Mon Sep 17 00:00:00 2001 From: ElementG9 Date: Mon, 25 Nov 2019 11:50:21 -0700 Subject: [PATCH 10/15] Grouping works correctly --- src/parser.js | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/src/parser.js b/src/parser.js index 8c22311..2abc529 100644 --- a/src/parser.js +++ b/src/parser.js @@ -23,7 +23,6 @@ function parse(tokens) { ast = addLevels(ast); // Start grouping by precedence. - // Precedence 16. ast = grouping(ast); // Precedence 15. @@ -103,24 +102,28 @@ function getDeepestLevel(tokens) { function grouping(tokens) { // Get the deepest level. let deepestLevel = getDeepestLevel(tokens); - // Loop through for each level. + let groupBuffer; + let opening; + // Group the deepest levels first. for (let currentLevel = deepestLevel; currentLevel > 0; currentLevel--) { - let groupBuffer = []; - for (let j = 0; j < tokens.length; j++) { - let nextTokenLevel = 0; - if (typeof tokens[j + 1] != 'undefined') - nextTokenLevel = tokens[j + 1].level; - if (tokens[j].level == currentLevel) { - groupBuffer.push(tokens[j]); // Add the token to the groupBuffer. - if (tokens[j].level > nextTokenLevel) { + groupBuffer = []; // Overwrite groupBuffer. + opening = null; // Overwrite opening. + for (let i = 0; i < tokens.length; i++) { + if (tokens[i].level == currentLevel) { + if (groupBuffer.length == 0) + opening = tokens[i]; + groupBuffer.push(tokens[i]); + if (typeof tokens[i + 1] == 'undefined' || (tokens[i].type == 'delimiter' && tokens[i].subtype == 'right' && tokens[i].value == opening.value)) { // The end of the tokens. let g = new Group(groupBuffer[0].value, groupBuffer); g.index = g.tokens[0].index; g.level = g.tokens[0].level - 1; // -1 because the group is on the level below. - tokens.splice(g.tokens[0].index, g.tokens.length, g); - j = g.tokens[0].index; - // Remove the delimiters in g.tokens. - g.tokens = g.tokens.splice(1, groupBuffer.length - 2); + let length = g.tokens.length; // Keep track of how many tokens there are before removing the delimiters. + g.tokens = g.tokens.splice(1, g.tokens.length - 2); // Remove the delimiters in g.tokens. + i -= length - 1; // Reset the counter. + tokens.splice(i, length, g); // Replace the tokens with the new group. + // Reset groupBuffer and opening. groupBuffer = []; + opening = null; } } } @@ -137,7 +140,6 @@ function grouping(tokens) { */ // TODO: Member access function memberAccess(ast) { - console.log(ast); for (let i = 0; i < ast.length; i++) { if (ast[i].type == 'group') ast[i].tokens = memberAccess(ast[i].tokens); // Recursively order the groups. @@ -243,7 +245,6 @@ function keywords(ast) { */ function functionCreation(ast) { // Function call is Parenthesis Group, Brace Group. - console.log(ast); for (let i = 0; i < ast.length; i++) { if (ast[i].type == 'group') ast[i].tokens = functionCreation(ast[i].tokens); // Recursively order the groups. From 6b9c9c8b7fb9cf42bc30592bd4b5eea3c3ed0132 Mon Sep 17 00:00:00 2001 From: ElementG9 Date: Mon, 25 Nov 2019 13:00:32 -0700 Subject: [PATCH 11/15] Refactor postfix and prefix --- src/parser.js | 51 +++++++++++++++++++++++++++++------------------- src/tokenizer.js | 2 +- 2 files changed, 32 insertions(+), 21 deletions(-) diff --git a/src/parser.js b/src/parser.js index 2abc529..c8cb097 100644 --- a/src/parser.js +++ b/src/parser.js @@ -33,8 +33,8 @@ function parse(tokens) { ast = keywords(ast); ast = functionCreation(ast); // Precedence 13. - // Precedence 12. ast = postfixOperators(ast); + // Precedence 12. ast = prefixOperators(ast); return ast; @@ -138,7 +138,6 @@ function grouping(tokens) { * @returns {Token[]} The ast with grouped member access. * @private */ -// TODO: Member access function memberAccess(ast) { for (let i = 0; i < ast.length; i++) { if (ast[i].type == 'group') @@ -248,17 +247,15 @@ function functionCreation(ast) { for (let i = 0; i < ast.length; i++) { if (ast[i].type == 'group') ast[i].tokens = functionCreation(ast[i].tokens); // Recursively order the groups. - else if (ast[i].type == 'group' && ast[i].subtype == 'parenthesis') { - if (typeof ast[i + 1] == 'undefined') - continue; // Nothing after this group, ignore it. - else if (ast[i + 1] == 'group' && ast[i].subtype == 'brace') { - ast[i + 1].tokens = functionCreation(ast[i + 1].tokens); // Order the group that we care about before we mess with it. - let op = new Operator('n/a', 'function creation', [ast[i], ast[i + 1]]); - op.index = ast[i].index; - op.level = ast[i].level; - ast.splice(i, 2, op); - // Removed 2 tokens, put in 1, skip 1 token. Don't reduce the counter. - } else continue; + if (typeof ast[i + 1] == 'undefined') + continue; // Skip this loop. + if ((ast[i].type == 'group' && ast[i].subtype == 'parenthesis') && (ast[i + 1].type == 'group' && ast[i + 1].subtype == 'brace')) { + // Parenthesis group followed by brace group. + ast[i + 1].tokens = functionCreation(ast[i + 1].tokens); // Order the other group before we do anything. + let op = new Operator('function creation', 'n/a', [ast[i], ast[i + 1]]); + op.index = ast[i].index; + op.level = ast[i].level; + ast.splice(i, 2, op); } } return ast; @@ -274,9 +271,16 @@ function functionCreation(ast) { function postfixOperators(ast) { for (let i = 0; i < ast.length; i++) { // Take care of the tokens in the groups. - if (ast[i].type == 'group') - ast[i].tokens = postfixOperators(ast[i].tokens); - else if (ast[i].type == 'operator' && ast[i].subtype == 'postfix') { // The operand is on the left. + if (ast[i].type == 'group') { + if (ast[i].tokens.length > 0) { + ast[i].tokens = postfixOperators(ast[i].tokens); + } + } else if (ast[i].type == 'operator') { + if (typeof ast[i].operands != 'undefined') { + ast[i].operands = postfixOperators(ast[i].operands); + } + } + if (ast[i].type == 'operator' && ast[i].subtype == 'postfix') { // The operand is on the left. if (typeof ast[i - 1] == 'undefined') throw new SyntaxError('Postfix operator requires one operand before it.'); let op = new Operator(ast[i].subtype, ast[i].value, [ast[i - 1]]); @@ -299,10 +303,17 @@ function postfixOperators(ast) { function prefixOperators(ast) { for (let i = 0; i < ast.length; i++) { // Take care of the tokens in the groups. - if (ast[i].type == 'group') - ast[i].tokens = postfixOperators(ast[i].tokens); - else if (ast[i].type == 'operator' && ast[i].subtype == 'prefix') { // The operand is on the right. - if (typeof ast[i - 1] == 'undefined') + if (ast[i].type == 'group') { + if (ast[i].tokens.length > 0) { + ast[i].tokens = prefixOperators(ast[i].tokens); + } + } else if (ast[i].type == 'operator') { + if (typeof ast[i].operands != 'undefined') { + ast[i].operands = prefixOperators(ast[i].operands); + } + } + if (ast[i].type == 'operator' && ast[i].subtype == 'prefix') { // The operand is on the right. + if (typeof ast[i + 1] == 'undefined') throw new SyntaxError('Prefix operator requires one operand after it.'); let op = new Operator(ast[i].subtype, ast[i].value, [ast[i + 1]]); op.index = ast[i].index; diff --git a/src/tokenizer.js b/src/tokenizer.js index 8438ec8..dd581af 100644 --- a/src/tokenizer.js +++ b/src/tokenizer.js @@ -327,7 +327,7 @@ function determineCharType(char) { * @private */ function determineType(str) { - if (/let|const|new|return/.test(str)) + if (/let|const/.test(str)) // TODO: Add more keywords. return 'keyword'; else return 'unknown'; }; From c8195cef5a55b401a6c56cbb4f92cf0e5770de70 Mon Sep 17 00:00:00 2001 From: ElementG9 Date: Mon, 25 Nov 2019 13:23:51 -0700 Subject: [PATCH 12/15] Math operators work now --- src/parser.js | 86 +++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 84 insertions(+), 2 deletions(-) diff --git a/src/parser.js b/src/parser.js index c8cb097..4722745 100644 --- a/src/parser.js +++ b/src/parser.js @@ -36,7 +36,20 @@ function parse(tokens) { ast = postfixOperators(ast); // Precedence 12. ast = prefixOperators(ast); - + // Precedence 11. + ast = mathOperators(ast, 0); // Level 0 math operators: **. + // Precedence 10. + ast = mathOperators(ast, 1); // Level 1 math operators: *, /, %. + // Precedence 9. + ast = mathOperators(ast, 2); // Level 2 math operators: +, -. + // Precedence 7. + // ast = comparisonOperators(ast); + // Precedence 6. + // ast = assign(ast); + // Precedence 4. + // ast = logicOperators(ast); + // Precedence 3. + // ast = opAssign(ast); return ast; } @@ -301,7 +314,7 @@ function postfixOperators(ast) { * @private */ function prefixOperators(ast) { - for (let i = 0; i < ast.length; i++) { + for (let i = ast.length - 1; i >= 0; i--) { // Prefix operators are rtl associative, so loop backwards. // Take care of the tokens in the groups. if (ast[i].type == 'group') { if (ast[i].tokens.length > 0) { @@ -325,6 +338,75 @@ function prefixOperators(ast) { return ast; } +/** + * @function mathOperators + * @desc Recursively structures the math operators. + * @param {Token[]} ast The ast. + * @param {Token[]} level The level of math to do. (Order of operations) + * @returns {Token[]} The ast with structured math operators. + * @private + */ +function mathOperators(ast, level) { + if (level == 0) { // Level 0 operators: ** + for (let i = ast.length - 1; i >= 0; i--) { // Exponentiation is rtl associative, so loop backwards. + // Take care of the tokens in the groups. + if (ast[i].type == 'group') { + if (ast[i].tokens.length > 0) { + ast[i].tokens = mathOperators(ast[i].tokens, level); + } + } else if (ast[i].type == 'operator') { + if (typeof ast[i].operands != 'undefined') { + ast[i].operands = mathOperators(ast[i].operands, level); + } + } + if (ast[i].type == 'operator' && ast[i].value == '**') { + if (typeof ast[i - 1] == 'undefined' || typeof ast[i + 1] == 'undefined') + throw new SyntaxError('Dual operator requires two operands.'); + let op = new Operator('dual', ast[i].value, [ast[i - 1], ast[i + 1]]); + op.index = ast[i].index; + op.level = ast[i].level; + ast.splice(i - 1, 3, op); + i--; + } + } + } else { + for (let i = 0; i < ast.length; i++) { // All other math operators are ltr associative. + // Take care of the tokens in the groups. + if (ast[i].type == 'group') { + if (ast[i].tokens.length > 0) { + ast[i].tokens = mathOperators(ast[i].tokens, level); + } + } else if (ast[i].type == 'operator') { + if (typeof ast[i].operands != 'undefined') { + ast[i].operands = mathOperators(ast[i].operands, level); + } + } + if (level == 1) { + if (ast[i].type == 'operator' && (ast[i].value == '*' || ast[i].value == '/' || ast[i].value == '%')) { + if (typeof ast[i - 1] == 'undefined' || typeof ast[i + 1] == 'undefined') + throw new SyntaxError('Dual operator requires two operands.'); + let op = new Operator('dual', ast[i].value, [ast[i - 1], ast[i + 1]]); + op.index = ast[i].index; + op.level = ast[i].level; + ast.splice(i - 1, 3, op); + i--; + } + } else if (level == 2) { + if (ast[i].type == 'operator' && (ast[i].value == '+' || ast[i].value == '-')) { + if (typeof ast[i - 1] == 'undefined' || typeof ast[i + 1] == 'undefined') + throw new SyntaxError('Dual operator requires two operands.'); + let op = new Operator('dual', ast[i].value, [ast[i - 1], ast[i + 1]]); + op.index = ast[i].index; + op.level = ast[i].level; + ast.splice(i - 1, 3, op); + i--; + } + } + } + } + return ast; +} + module.exports = { parse, util: { From b2a686c27c9aefead1584e2412023458203ab805 Mon Sep 17 00:00:00 2001 From: ElementG9 Date: Fri, 6 Dec 2019 10:02:17 -0700 Subject: [PATCH 13/15] v0.2.0a - Log works with one parameter. Hello world! --- bin/pivot.js | 10 ++++++++-- src/code.js | 38 ++++++++++++++++++++++++++++++++++++++ src/parser.js | 3 ++- 3 files changed, 48 insertions(+), 3 deletions(-) create mode 100644 src/code.js diff --git a/bin/pivot.js b/bin/pivot.js index cf4bb23..b673125 100755 --- a/bin/pivot.js +++ b/bin/pivot.js @@ -2,6 +2,7 @@ const args = process.argv.slice(2); const tokenizer = require('../src/tokenizer.js'); const parser = require('../src/parser.js'); +const code = require('../src/code.js'); if (typeof args[0] != 'undefined') { // Execute from file. @@ -21,9 +22,14 @@ if (typeof args[0] != 'undefined') { func(answer); } } - console.log('Welcome to Pivot v0.1.0 Alpha.'); + console.log('Welcome to Pivot v0.2.0 Alpha.'); console.log('Type \'exit\' to exit.'); + let data = { + log: "console.log" + }; repl('> ', (answer) => { - console.log(require('util').inspect(parser.parse(tokenizer.tokenize(answer)), { depth: null })); + let jsAnswer = code.translate(parser.parse(tokenizer.tokenize(answer)), data); + // console.log(require('util').inspect(jsAnswer, { depth: null })); + eval(jsAnswer); }); } diff --git a/src/code.js b/src/code.js new file mode 100644 index 0000000..f756cf6 --- /dev/null +++ b/src/code.js @@ -0,0 +1,38 @@ +/** + * @module code + * @file Runs the code / transpiles the code to JavaScript + * @author Garen Tyler + */ + + /** + * @function translate + * @desc Translates the code to JS, given an AST + * @param {Token[]} ast The ast. + * @returns {String} The JS code. + * @public + */ +function translate(ast, data) { + let out = ''; + for (let i = 0; i < ast.length; i++) { + if (ast[i].type == 'operator' && ast[i].subtype == 'function call') { + let temp = ''; + if (!(Object.keys(data).indexOf(ast[i].operands[0].value) > -1)) + throw new ReferenceError(`Undefined function ${ast[i].operands[0].value}`); + else temp += data[ast[i].operands[0].value]; + temp += '('; + for (let j = 0; j < ast[i].operands[1].tokens.length; j++) { + if (j != 0) + temp += ', '; + if (ast[i].operands[1].tokens[j].type == 'string') + temp += `"${ast[i].operands[1].tokens[j].value}"`; + } + temp += ');' + out += temp; + } + } + return out; +} + +module.exports = { + translate +}; diff --git a/src/parser.js b/src/parser.js index 4722745..ed260f5 100644 --- a/src/parser.js +++ b/src/parser.js @@ -208,7 +208,8 @@ function functionCall(ast) { for (let i = 0; i < ast.length; i++) { if (ast[i].type == 'group') ast[i].tokens = functionCall(ast[i].tokens); // Recursively order the groups. - else if (ast[i].type == 'name' && ast[i].subtype == 'variable') { // Member access operator. + else if ((ast[i].type == 'name' && ast[i].subtype == 'variable') || // Normal function call + (ast[i].type == 'operator' && (ast[i].value == '.' || ast[i].value == 'member access'))) { // Function call in member access. Example: console.log() if (typeof ast[i + 1] == 'undefined') continue; // Nothing after the variable; skip this loop. if (ast[i + 1].type == 'group' && ast[i + 1].subtype == 'parenthesis') { From 01f6d05038d04cfc2af3c9ab95f2ef5577dcf0f5 Mon Sep 17 00:00:00 2001 From: ElementG9 Date: Fri, 6 Dec 2019 10:08:06 -0700 Subject: [PATCH 14/15] Update package.json version number --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 183d6db..ecf7feb 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "pivot-lang", - "version": "0.1.0", + "version": "0.2.0", "description": "Pivot is a new programming language built on JavaScript", "main": "./bin/pivot.js", "directories": { @@ -35,6 +35,6 @@ "readline-sync": "^1.4.10" }, "devDependencies": { - "mocha": "^6.2.1" + "mocha": "^6.2.2" } } From 243395d319f70972ed8f746f6a23217fcfc7f6ad Mon Sep 17 00:00:00 2001 From: ElementG9 Date: Fri, 6 Dec 2019 10:44:49 -0700 Subject: [PATCH 15/15] Make things able to change data --- bin/pivot.js | 3 ++- src/code.js | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/bin/pivot.js b/bin/pivot.js index b673125..0fd7c3d 100755 --- a/bin/pivot.js +++ b/bin/pivot.js @@ -29,7 +29,8 @@ if (typeof args[0] != 'undefined') { }; repl('> ', (answer) => { let jsAnswer = code.translate(parser.parse(tokenizer.tokenize(answer)), data); + data = jsAnswer.data; // console.log(require('util').inspect(jsAnswer, { depth: null })); - eval(jsAnswer); + eval(jsAnswer.code); }); } diff --git a/src/code.js b/src/code.js index f756cf6..57624a2 100644 --- a/src/code.js +++ b/src/code.js @@ -30,7 +30,10 @@ function translate(ast, data) { out += temp; } } - return out; + return { + data, + code: out + }; } module.exports = {