From 840bf520125c5ca911f594da61c787c8822eb39c Mon Sep 17 00:00:00 2001 From: ElementG9 Date: Sat, 12 Jan 2019 20:28:57 -0700 Subject: [PATCH] Separate pivot.js into separate files: tokenizer.js and pivot.js. --- .gitattributes | 2 - ast.json | 56 ++++++------ classes.js | 18 ++++ parser.js | 47 ++++++++++ pivot.js | 236 ++----------------------------------------------- tokenizer.js | 162 +++++++++++++++++++++++++++++++++ 6 files changed, 265 insertions(+), 256 deletions(-) delete mode 100644 .gitattributes create mode 100644 classes.js create mode 100644 parser.js create mode 100644 tokenizer.js diff --git a/.gitattributes b/.gitattributes deleted file mode 100644 index dfe0770..0000000 --- a/.gitattributes +++ /dev/null @@ -1,2 +0,0 @@ -# Auto detect text files and perform LF normalization -* text=auto diff --git a/ast.json b/ast.json index 880d2a2..4d319a8 100644 --- a/ast.json +++ b/ast.json @@ -11,43 +11,45 @@ }, { "type": "Group", - "value": "a", + "value": "{", "tokens": [ { - "type": "Variable", - "value": "a", - "layer": 1 + "type": "Left Delimiter", + "value": "{", + "layer": 0 } - ], - "layer": 0 - }, - { - "type": "Left Delimiter", - "value": "{", - "layer": 0 + ] }, { "type": "Group", "value": "a", "tokens": [ { - "type": "Group", + "type": "Variable", "value": "a", - "tokens": [ - { - "type": "Variable", - "value": "a", - "layer": 2 - }, - { - "type": "Operator", - "value": "++", - "layer": 2 - } - ], - "layer": 1 + "layer": 2 + }, + { + "type": "Operator", + "value": "++", + "layer": 2 } - ], - "layer": 0 + ] + }, + { + "type": "Group", + "value": "asdf", + "tokens": [ + { + "type": "Variable", + "value": "asdf", + "layer": 0 + }, + { + "type": "Operator", + "value": "=", + "layer": 0 + } + ] } ] \ No newline at end of file diff --git a/classes.js b/classes.js new file mode 100644 index 0000000..f91b1ed --- /dev/null +++ b/classes.js @@ -0,0 +1,18 @@ +var token = function (type, value) { + this.type = type; + this.value = value; +}; +var group = function (type, tokens) { + this.type = "Group"; + this.value = type; + this.tokens; + if(typeof tokens != "undefined") { + this.tokens = tokens; + } else { + this.tokens = []; + } +} +module.exports = { + token: token, + group: group +} diff --git a/parser.js b/parser.js new file mode 100644 index 0000000..3426fd6 --- /dev/null +++ b/parser.js @@ -0,0 +1,47 @@ +const group = require("./classes.js").group; +module.exports = tokens => { + var layer = 0; + var delimiterCount = 0; + var deepestLayer = 0; + for (var i = 0; i < tokens.length; i++) { + if (tokens[i].type == "Left Delimiter") { + layer++; + if(layer > deepestLayer) { + deepestLayer = layer; + } + delimiterCount++; + } + tokens[i].layer = layer; + if (tokens[i].type == "Right Delimiter") { + layer--; + } + } + for (var i = 0; i < tokens.length; i++) { + if ((tokens[i].type == "Left Delimiter") || (tokens[i].type == "Right Delimiter")) { + tokens[i].layer--; + } + } + if (layer > 0) { // Unclosed delimiter. + } else if (layer < 0) { // Overclosed delimiter. + } + layer = 0; + for(var i=deepestLayer;i>=0;i--) { + var temp = []; + var firstIndex; + for(var j=0;j 0) { + var g = new group(tokens[firstIndex].value,temp); + tokens.splice(firstIndex-1,temp.length+2,g); + temp = []; + } + } + } + } + return tokens; +}; diff --git a/pivot.js b/pivot.js index a4bc40e..46f62ad 100644 --- a/pivot.js +++ b/pivot.js @@ -1,228 +1,10 @@ -var token = function (type, value) { - this.type = type; - this.value = value; -}; -var group = function (type, tokens) { - this.type = "Group"; - this.value = type; - this.tokens; - if(typeof tokens != "undefined") { - this.tokens = tokens; - } else { - this.tokens = []; - } -} -var tokenize = exp => { - var isDigit = char => { - return /\d/.test(char); - }; - var isLetter = char => { - return /[a-z]/i.test(char); - }; - var isOperator = char => { - return /\+|-|\*|\/|\^|=/.test(char); - }; - var isLeftDelimiter = char => { - return (/\(|\[|\{|"|'|`/.test(char)); - }; - var isRightDelimiter = char => { - return (/\)|\]|\}/.test(char)); - }; - var isComma = char => { - return (char === ","); - }; - var isPeriod = char => { - return (char === "."); - }; - var result = []; - var nb = []; - var lb = []; - var ob = []; - var sb = []; - var inString = false; - var stringType; - exp = exp.split(""); - for (var i = 0; i < exp.length; i++) { - var char = exp[i]; - if (i >= 1) { - if (exp[i - 1] == "\\") { - exp.splice(i - 1, 2, `\\${char}`); - i--; - continue; - } - if (exp[i - 1] == "$" && char == "{") { - exp.splice(i - 1, 2, `\${`); - i--; - continue; - } - } - } - for (var i = 0; i < exp.length; i++) { - var char = exp[i]; - if (inString) { - if (char == `'` || char == `"` || char == "`") { - var exitString = () => { - inString = false; - if (sb.length == 0) { - result.push(new token("String", null)); - } else { - var string = sb.join(""); - result.push(new token("String", string)); - } - sb = []; - }; - if (char == `'` && stringType == "single") { - exitString(); - } else if (char == `"` && stringType == "double") { - exitString(); - } else if (char == "`" && stringType == "backtick") { - exitString(); - } else { - if (char == `'`) { - sb.push(`\'`); - } - if (char == `"`) { - sb.push(`\"`); - } - if (char == "`") { - sb.push("\`"); - } - } - } else { - sb.push(char); - } - } else { - if (isDigit(char)) { - result.push(new token("Operator", ob.join(""))); - ob = []; - nb.push(char); - } else if (isLetter(char)) { - result.push(new token("Operator", ob.join(""))); - ob = []; - lb.push(char); - } else if (isOperator(char)) { - result.push(new token("Number", nb.join(""))); - nb = []; - result.push(new token("Variable", lb.join(""))); - lb = []; - ob.push(char); - } else if (isLeftDelimiter(char)) { - result.push(new token("Operator", ob.join(""))); - ob = []; - result.push(new token("Function Call", lb.join(""))); - lb = []; - if (char == `'` || char == `"` || char == "`") { - inString = true; - if (char == `'`) { - stringType = "single"; - } else if (char == `"`) { - stringType = "double"; - } else if (char == "`") { - stringType = "backtick"; - } - } else { - result.push(new token("Left Delimiter", char)); - } - } else if (isRightDelimiter(char)) { - result.push(new token("Operator", ob.join(""))); - ob = []; - result.push(new token("Number", nb.join(""))); - nb = []; - result.push(new token("Variable", lb.join(""))); - lb = []; - result.push(new token("Right Delimiter", char)); - } else if (isComma(char)) { - result.push(new token("Operator", ob.join(""))); - ob = []; - result.push(new token("Number", nb.join(""))); - nb = []; - result.push(new token("Variable", lb.join(""))); - lb = []; - result.push(new token("Comma", char)); - } else if (isPeriod(char)) { - result.push(new token("Operator", ob.join(""))); - ob = []; - nb.push(char); - } - } - } - result.push(new token("Operator", ob.join(""))); - ob = []; - result.push(new token("Number", nb.join(""))); - nb = []; - lb.forEach(item => { - result.push(new token("Variable", item)); - }); - lb = []; - for (var i = 0; i < 3; i++) { - result.forEach((item, index) => { - if (item.value == "") { - result.splice(index, 1); - } - }); - } - result.forEach((item, index) => { - if (item.value == "-" && index != 0) { - if (result[index - 1].type != "Variable" && result[index - 1].type != "Number") { - if (result[index + 1].type == "Number") { - result[index + 1].value = "-" + result[index + 1].value; - result.splice(index, 1); - } - } - } - }); - return result; -}; -var parse = tokens => { - var layer = 0; - var delimiterCount = 0; - var deepestLayer = 0; - for (var i = 0; i < tokens.length; i++) { - if (tokens[i].type == "Left Delimiter") { - layer++; - if(layer > deepestLayer) { - deepestLayer = layer; - } - delimiterCount++; - } - tokens[i].layer = layer; - if (tokens[i].type == "Right Delimiter") { - layer--; - } - } - for (var i = 0; i < tokens.length; i++) { - if ((tokens[i].type == "Left Delimiter") || (tokens[i].type == "Right Delimiter")) { - tokens[i].layer--; - } - } - if (layer > 0) { // Unclosed delimiter - } else if (layer < 0) { // Overclosed delimiter - } - layer = 0; - for(var i=deepestLayer;i>=0;i--) { - var temp = []; - var firstIndex; - for(var j=0;j 0) { - var g = new group(tokens[firstIndex].value,temp); - tokens.splice(firstIndex-1,temp.length+2,g); - temp = []; - } - } - } - } - return tokens; -}; -var fs = require("fs"); +// Import the tokenizer and parser. +const tokenize = require("./tokenizer.js"); +const parse = require("./parser.js"); + +// Generate the AST. var ast = parse(tokenize(`asdf = (a) {return(a++)}`)); -fs.writeFileSync("ast.json", JSON.stringify(ast, null, 4)); \ No newline at end of file + +// Write the AST to ast.json. +var fs = require("fs"); +fs.writeFileSync("ast.json", JSON.stringify(ast, null, 4)); diff --git a/tokenizer.js b/tokenizer.js new file mode 100644 index 0000000..9b3edf7 --- /dev/null +++ b/tokenizer.js @@ -0,0 +1,162 @@ +const token = require("./classes.js").token; +module.exports = exp => { + var isDigit = char => { + return /\d/.test(char); + }; + var isLetter = char => { + return /[a-z]/i.test(char); + }; + var isOperator = char => { + return /\+|-|\*|\/|\^|=/.test(char); + }; + var isLeftDelimiter = char => { + return (/\(|\[|\{|"|'|`/.test(char)); + }; + var isRightDelimiter = char => { + return (/\)|\]|\}/.test(char)); + }; + var isComma = char => { + return (char === ","); + }; + var isPeriod = char => { + return (char === "."); + }; + var result = []; + var nb = []; + var lb = []; + var ob = []; + var sb = []; + var inString = false; + var stringType; + exp = exp.split(""); + for (var i = 0; i < exp.length; i++) { + var char = exp[i]; + if (i >= 1) { + if (exp[i - 1] == "\\") { + exp.splice(i - 1, 2, `\\${char}`); + i--; + continue; + } + if (exp[i - 1] == "$" && char == "{") { + exp.splice(i - 1, 2, `\${`); + i--; + continue; + } + } + } + for (var i = 0; i < exp.length; i++) { + var char = exp[i]; + if (inString) { + if (char == `'` || char == `"` || char == "`") { + var exitString = () => { + inString = false; + if (sb.length == 0) { + result.push(new token("String", null)); + } else { + var string = sb.join(""); + result.push(new token("String", string)); + } + sb = []; + }; + if (char == `'` && stringType == "single") { + exitString(); + } else if (char == `"` && stringType == "double") { + exitString(); + } else if (char == "`" && stringType == "backtick") { + exitString(); + } else { + if (char == `'`) { + sb.push(`\'`); + } + if (char == `"`) { + sb.push(`\"`); + } + if (char == "`") { + sb.push("\`"); + } + } + } else { + sb.push(char); + } + } else { + if (isDigit(char)) { + result.push(new token("Operator", ob.join(""))); + ob = []; + nb.push(char); + } else if (isLetter(char)) { + result.push(new token("Operator", ob.join(""))); + ob = []; + lb.push(char); + } else if (isOperator(char)) { + result.push(new token("Number", nb.join(""))); + nb = []; + result.push(new token("Variable", lb.join(""))); + lb = []; + ob.push(char); + } else if (isLeftDelimiter(char)) { + result.push(new token("Operator", ob.join(""))); + ob = []; + result.push(new token("Function Call", lb.join(""))); + lb = []; + if (char == `'` || char == `"` || char == "`") { + inString = true; + if (char == `'`) { + stringType = "single"; + } else if (char == `"`) { + stringType = "double"; + } else if (char == "`") { + stringType = "backtick"; + } + } else { + result.push(new token("Left Delimiter", char)); + } + } else if (isRightDelimiter(char)) { + result.push(new token("Operator", ob.join(""))); + ob = []; + result.push(new token("Number", nb.join(""))); + nb = []; + result.push(new token("Variable", lb.join(""))); + lb = []; + result.push(new token("Right Delimiter", char)); + } else if (isComma(char)) { + result.push(new token("Operator", ob.join(""))); + ob = []; + result.push(new token("Number", nb.join(""))); + nb = []; + result.push(new token("Variable", lb.join(""))); + lb = []; + result.push(new token("Comma", char)); + } else if (isPeriod(char)) { + result.push(new token("Operator", ob.join(""))); + ob = []; + nb.push(char); + } + } + } + result.push(new token("Operator", ob.join(""))); + ob = []; + result.push(new token("Number", nb.join(""))); + nb = []; + lb.forEach(item => { + result.push(new token("Variable", item)); + }); + lb = []; + for (var i = 0; i < 3; i++) { + result.forEach((item, index) => { + if (item.value == "") { + result.splice(index, 1); + } + }); + } + result.forEach((item, index) => { + if (item.value == "-" && index != 0) { + if (result[index - 1].type != "Variable" && result[index - 1].type != "Number") { + if (result[index + 1].type == "Number") { + result[index + 1].value = "-" + result[index + 1].value; + result.splice(index, 1); + } + } + } + }); + return result; +};