From 5c0a481dbe20cb030e3cf3d39b975c65d5a8e930 Mon Sep 17 00:00:00 2001 From: ElementG9 Date: Thu, 3 Oct 2019 22:29:51 -0600 Subject: [PATCH] Refactored tokenizer, implemented jsdoc, set up for cli. --- .gitignore | 4 + LICENSE | 4 +- ast.json | 50 -------- bin/pivot.js | 3 + classes.js | 21 ---- package.json | 39 ++++++ parser.js | 134 --------------------- pivot.js | 13 -- readme.md | 15 --- rpn.js | 40 ------- src/parser.js | 44 +++++++ src/tokenizer.js | 303 +++++++++++++++++++++++++++++++++++++++++++++++ src/types.js | 31 +++++ test.pivot | 6 - test/test.js | 126 ++++++++++++++++++++ tokenizer.js | 170 -------------------------- 16 files changed, 552 insertions(+), 451 deletions(-) create mode 100644 .gitignore delete mode 100644 ast.json create mode 100755 bin/pivot.js delete mode 100644 classes.js create mode 100644 package.json delete mode 100644 parser.js delete mode 100644 pivot.js delete mode 100644 rpn.js create mode 100644 src/parser.js create mode 100644 src/tokenizer.js create mode 100644 src/types.js delete mode 100644 test.pivot create mode 100644 test/test.js delete mode 100644 tokenizer.js diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9c4ba90 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +other/ +package-lock.json +node_modules/ +jsdoc/ diff --git a/LICENSE b/LICENSE index 28db51c..ab97016 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2019 ElementG9 +Copyright (c) 2019 Garen Tyler Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -18,4 +18,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. \ No newline at end of file +SOFTWARE. diff --git a/ast.json b/ast.json deleted file mode 100644 index ec9f41a..0000000 --- a/ast.json +++ /dev/null @@ -1,50 +0,0 @@ -[ - { - "type": "Variable", - "value": "asdf", - "layer": 0 - }, - { - "type": "Operator", - "value": "=", - "layer": 0 - }, - { - "type": "Group", - "value": "(", - "tokens": [ - { - "type": "Variable", - "value": "a", - "layer": 1 - } - ] - }, - { - "type": "Group", - "value": "{", - "tokens": [ - { - "type": "Function Call", - "value": "return", - "layer": 1 - }, - { - "type": "Group", - "value": "(", - "tokens": [ - { - "type": "Variable", - "value": "a", - "layer": 2 - }, - { - "type": "Operator", - "value": "++", - "layer": 2 - } - ] - } - ] - } -] \ No newline at end of file diff --git a/bin/pivot.js b/bin/pivot.js new file mode 100755 index 0000000..3d4a31f --- /dev/null +++ b/bin/pivot.js @@ -0,0 +1,3 @@ +#!/usr/bin/env node +const args = process.argv.slice(2); +console.log(args); diff --git a/classes.js b/classes.js deleted file mode 100644 index 4d59265..0000000 --- a/classes.js +++ /dev/null @@ -1,21 +0,0 @@ -// This is the most basic type of token. -var token = function (type, value) { - this.type = type; - this.value = value; -}; -// This is a group of tokens. -var group = function (type, tokens, index) { - this.type = "Group"; - this.value = type; - this.index = index; - this.tokens; - if (typeof tokens != "undefined") { - this.tokens = tokens; - } else { - this.tokens = []; - } -} -module.exports = { - token: token, - group: group -} \ No newline at end of file diff --git a/package.json b/package.json new file mode 100644 index 0000000..1ce2203 --- /dev/null +++ b/package.json @@ -0,0 +1,39 @@ +{ + "name": "pivot", + "version": "1.0.0", + "description": "Pivot is a new programming language built on JavaScript", + "main": "./bin/pivot.js", + "directories": { + "test": "test" + }, + "scripts": { + "test": "mocha", + "build": "npm run test; jsdoc src/ -r -d ./jsdoc/ -R ./readme.md" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/ElementG9/Pivot.git" + }, + "keywords": [ + "pivot", + "javascript", + "programming", + "language" + ], + "author": "Garen Tyler ", + "license": "MIT", + "bugs": { + "url": "https://github.com/ElementG9/Pivot/issues" + }, + "homepage": "https://github.com/ElementG9/Pivot#readme", + "bin": { + "pivot": "./bin/pivot.js" + }, + "preferGlobal": true, + "dependencies": { + "fs": "0.0.1-security" + }, + "devDependencies": { + "mocha": "^6.2.1" + } +} diff --git a/parser.js b/parser.js deleted file mode 100644 index 50cba2f..0000000 --- a/parser.js +++ /dev/null @@ -1,134 +0,0 @@ -// Import the group class. -const group = require("./classes.js").group; - -// Create the parser function. -// parse() takes an array of tokens in, and outputs an -// Abstract Syntax Tree (a structured array of tokens). -module.exports = tokens => { - // Variables for later. - var layer = 0; - var delimiterCount = 0; - var deepestLayer = 0; - // Give each token a layer number based on delimiters. - for (var i = 0; i < tokens.length; i++) { - if (tokens[i].type == "Left Delimiter") { - layer++; - if (layer > deepestLayer) { - deepestLayer = layer; - } - delimiterCount++; - } - tokens[i].layer = layer; - if (tokens[i].type == "Right Delimiter") { - layer--; - } - } - // Lower the layer of delimiters. - for (var i = 0; i < tokens.length; i++) { - if ((tokens[i].type == "Left Delimiter") || (tokens[i].type == "Right Delimiter")) { - tokens[i].layer--; - } - } - if (layer > 0) { // Unclosed delimiter. - } else if (layer < 0) { // Overclosed delimiter. - } - // Give each token an index. - for (let i = 0; i < tokens.length; i++) { - tokens[i].index = i; - } - // Structure the layers. - // Count the rising edges of the layers to determine how many groups should exist. - let structure = function () { - let layer = 0; - let risingFalling = []; // Create an array to store indices of rising/falling edges. - for (let i = 0; i < tokens.length; i++) { - // Add a rising and a falling tag to each token. - tokens[i].rising = false; - tokens[i].falling = false; - if (tokens[i].layer > layer) { // If the token moves up a layer. - // Create a new rising index in risingFalling. - risingFalling.push({ - type: 'rising', - index: i - }); - tokens[i].rising = true; // Note that the token is a rising edge. - layer++; - } else if (tokens[i].layer < layer) { - // Create a new falling index in risingFalling. - risingFalling.push({ - type: 'falling', - index: i - }); - tokens[i].falling = true; // Note that the token is a falling edge. - layer--; - } - } - // Loop through the list of rising/falling edges. - for (let i = 0; i < risingFalling.length; i++) { - if (i != risingFalling.length - 1) { // If not the last edge. - let item = risingFalling[i]; - let nextItem = risingFalling[i + 1]; - // If a falling edge follows a rising edge, classifiy it as a group. - if ((item.type == 'rising') && (nextItem.type == 'falling')) { - // Get the group together as one item. - let selectedItems = tokens.slice(item.index, nextItem.index); - tokens.splice(item.index, selectedItems.length, new group(tokens[item.index - 1].value, selectedItems, item.index)); - } - } - } - risingFalling = []; // Reset the list of edges. - // Count the edges again. - for (let i = 0; i < tokens.length; i++) { - if (tokens[i].layer > layer) { - risingFalling.push({ - type: 'rising', - index: i - }); - layer++; - } else if (tokens[i].layer < layer) { - risingFalling.push({ - type: 'falling', - index: i - }); - layer--; - } - } - // If there are still edges, run again. - if (risingFalling.length) { - structure(); - } - }; - // Start the recursion. - structure(); - let trimDelimiters = function (thing) { - // Loop through the tokens of thing. - for (let i = 0; i < thing.length; i++) { - // Delete unnecessary keys. - if (typeof thing[i].rising != 'undefined') { - delete thing[i].rising; - } - if (typeof thing[i].falling != 'undefined') { - delete thing[i].falling; - } - if (typeof thing[i].index != 'undefined') { - delete thing[i].index; - } - if (typeof thing[i].index != 'undefined') { - delete thing[i].index; - } - // Remove delimiters. - if ((thing[i].type == 'Left Delimiter') || (thing[i].type == 'Right Delimiter')) { - thing.splice(i, 1); - i--; - } - // If a token is a group, look at the group's tokens. - if (thing[i].type == 'Group') { - trimDelimiters(thing[i].tokens); - } - } - }; - // Start the recursion. - trimDelimiters(tokens); - // Return the structured tokens. - return tokens; -}; \ No newline at end of file diff --git a/pivot.js b/pivot.js deleted file mode 100644 index 2b75014..0000000 --- a/pivot.js +++ /dev/null @@ -1,13 +0,0 @@ -// The Pivot code to run. -var code = `asdf = (a) {return(a++)}`; - -// Import the tokenizer and parser. -const tokenize = require("./tokenizer.js"); -const parse = require("./parser.js"); - -// Generate the AST. -var ast = parse(tokenize(code)); - -// Write the AST to ast.json. -var fs = require("fs"); -fs.writeFileSync("ast.json", JSON.stringify(ast, null, 4)); \ No newline at end of file diff --git a/readme.md b/readme.md index 6f46894..e69de29 100644 --- a/readme.md +++ b/readme.md @@ -1,15 +0,0 @@ -# [Pivot](https://gthub.us/pivot/) - -Pivot is a new programming language built on [JavaScript](https://javascript.com/) by [ElementG9](https://github.com/ElementG9/). - ---- - -## Installation -* Download or clone [the GitHub repository](/pivot/repo). -* Download [node.js](https://nodejs.org/). -* Run _pivot.js_ with `node pivot.js`. - ---- - -## Documentation -The Pivot documentation is [here](https://gthub.us/pivot/docs). diff --git a/rpn.js b/rpn.js deleted file mode 100644 index cf84d8d..0000000 --- a/rpn.js +++ /dev/null @@ -1,40 +0,0 @@ -var solve = (exp) => { - var stack = []; - var expression = exp.split(" "); - for(var j=0;j + * @requires module:types + */ +const Token = require('./types.js').Token; +const Group = require('./types.js').Token; + +module.exports = function(tokens) { + let level = 0; + // Add level markers. + tokens.forEach((t, i) => { + if (t.type == 'delimiter' && t.subtype == 'left') { + tokens[i].level = level; + level++; + } else if (t.type == 'delimiter' && t.subtype == 'right') { + level--; + tokens[i].level = level; + } else { + tokens[i].level = level; + } + }); + // Group. + let currentLevel = 0; + let groupStack = [0]; + tokens.forEach((t, i) => { + if (currentLevel < t.level) { + tokens.splice(i, 0, new Group(tokens[i - 1].value, [])); + groupStack.push(i); + currentLevel++; + tokens[i].level = currentLevel; + } + if (t.level != 0) { + tokens[groupStack.slice(-1)].tokens.push(t); + } + if (currentLevel > t.level) { + groupStack.pop(); + currentLevel--; + } + }); + if (currentLevel != 0) {} // Error: missing delimiter. + return tokens; +}; diff --git a/src/tokenizer.js b/src/tokenizer.js new file mode 100644 index 0000000..f61dd43 --- /dev/null +++ b/src/tokenizer.js @@ -0,0 +1,303 @@ +/** + * @module tokenizer + * @file Manages the tokenization phase of Pivot. + * @author Garen Tyler + * @requires module:types + */ +const Token = require('./types.js').Token; +const Group = require('./types.js').Token; + +/** + * @function tokenize + * @desc Takes in raw code, and outputs an array of Tokens. + * @param {string} code The raw input code. + * @returns {Token[]} The code, split into tokens. + * @public + */ +function tokenize(code) { + // Split the string into an array of chars. + let chars = code.split(''); + + // Create buffers. + let letterBuffer = []; + let operatorBuffer = []; + let numberBuffer = []; + let stringBuffer = []; + + // Create the output Token[]. + let tokens = []; + + // Create an object to keep track of string data. + let stringData = { + inString: false, + stringType: null + }; + + // Escape chars and remove comments. + chars = combineEscapedChars(chars); + chars = removeComments(chars); + + + // Actually tokenize the chars. + for (let i = 0; i < chars.length; i++) { + let char = chars[i]; + if (stringData.inString) { // Tokenize differently in a string. + // If a string delimiter and the same as the inital delimiter. + if (determineCharType(char) == 'string delimiter' && char == stringData.stringType) { + stringData.inString = false; // Not in a string any more. + tokens.push(new Token('string', 'n/a', stringBuffer.join(''))); // Push the string. + stringBuffer = []; // Clear the string buffer. + } else stringBuffer.push(char); // Add to the string buffer. + } else { // Tokenize normally. + if (determineCharType(char) == 'string delimiter') { + stringData.inString = true; // In a string now. + stringData.stringType = char; + } else if (determineCharType(char) == 'letter') { + letterBuffer.push(char); // Add to the letter buffer. + // End the other buffers. + if (operatorBuffer.length > 0) { + let operator = operatorBuffer.join(''); + tokens.push(new Token('operator', operatorType(operator), operator)); + operatorBuffer = []; + } + if (numberBuffer.length > 0) { + let number = numberBuffer.join(''); + tokens.push(new Token('number', 'n/a', number)); + numberBuffer = []; + } + } else if (determineCharType(char) == 'operator') { + operatorBuffer.push(char); // Add to the operator buffer. + // End the other buffers. + if (letterBuffer.length > 0) { + let variable = letterBuffer.join(''); + tokens.push(new Token('name', 'variable', variable)); + letterBuffer = []; + } + if (numberBuffer.length > 0) { + let number = numberBuffer.join(''); + tokens.push(new Token('number', 'n/a', number)); + numberBuffer = []; + } + } else if (determineCharType(char) == 'digit') { + numberBuffer.push(char); // Add to the number buffer. + // End the other buffers. + if (letterBuffer.length > 0) { + let variable = letterBuffer.join(''); + tokens.push(new Token('name', 'variable', variable)); + letterBuffer = []; + } + if (operatorBuffer.length > 0) { + let operator = operatorBuffer.join(''); + tokens.push(new Token('operator', operatorType(operator), operator)); + operatorBuffer = []; + } + } else if (determineCharType(char) == 'whitespace') { + // End all buffers. + if (letterBuffer.length > 0) { + let variable = letterBuffer.join(''); + tokens.push(new Token('name', 'variable', variable)); + letterBuffer = []; + } + if (numberBuffer.length > 0) { + let number = numberBuffer.join(''); + tokens.push(new Token('number', 'n/a', number)); + numberBuffer = []; + } + if (operatorBuffer.length > 0) { + let operator = operatorBuffer.join(''); + tokens.push(new Token('operator', operatorType(operator), operator)); + operatorBuffer = []; + } + } else if (determineCharType(char) == 'delimiter') { + // End all buffers. + if (letterBuffer.length > 0) { + let variable = letterBuffer.join(''); + tokens.push(new Token('name', 'variable', variable)); + letterBuffer = []; + } + if (numberBuffer.length > 0) { + let number = numberBuffer.join(''); + tokens.push(new Token('number', 'n/a', number)); + numberBuffer = []; + } + if (operatorBuffer.length > 0) { + let operator = operatorBuffer.join(''); + tokens.push(new Token('operator', operatorType(operator), operator)); + operatorBuffer = []; + } + // Push the delimiter. + tokens.push(getDelimiterToken(char)); + } + } + } + + // Empty all the buffers. + if (letterBuffer.length > 0) { + let variable = letterBuffer.join(''); + tokens.push(new Token('name', 'variable', variable)); + letterBuffer = []; + } + if (numberBuffer.length > 0) { + let number = numberBuffer.join(''); + tokens.push(new Token('number', 'n/a', number)); + numberBuffer = []; + } + if (operatorBuffer.length > 0) { + let operator = operatorBuffer.join(''); + tokens.push(new Token('operator', operatorType(operator), operator)); + operatorBuffer = []; + } + + tokens = changeKeywords(tokens); + + return tokens; +} + +/** + * @function combineEscapedChars + * @desc Combines escaped chars into one char. + * @param {string[]} chars The chars. + * @returns {string[]} The chars with combined escaped chars. + * @private + */ +function combineEscapedChars(chars) { + // Check for characters to be escaped. + for (let i = 0; i < chars.length; i++) { + if (chars[i] == '\\') { + chars.splice(i, 2, chars[i] + chars[i + 1]); + i -= 2; + } + } + return chars; +} + +/** + * @function removeComments + * @desc Removes comments. + * @param {string[]} chars The chars. + * @returns {string[]} The chars without comments. + * @private + */ +function removeComments(chars) { + let inComment = false; // Keep track if in a comment. + for (let i = 0; i < chars.length; i++) { + if (chars[i] == '/') { + if (chars[i + 1] == '/') { + inComment = true; + } + } + if (chars[i] == '\n') { + inComment = false; + chars.splice(i, 1); // Remove the newline at the end of the comment. + i--; + } + if (inComment) { + chars.splice(i, 1); // Remove the char in the comment. + i--; + } + } + return chars; +} + +/** + * @function changeKeywords + * @desc Changes tokens with subtype variable to subtype keyword + * @param {Token[]} tokens The tokens + * @returns {Token[]} The tokens with keywords. + * @private + */ +function changeKeywords(tokens) { + return tokens.map(t => { + if (t.subtype == 'variable' && determineType(t.value) == 'keyword') { + t.subtype = 'keyword'; + } + return t; + }); +} + +/** + * @function getDelimiterToken + * @desc Turns a delimiter char into a token. + * @param {string} delimiter The delimiter char. + * @returns {Token} The delimiter token. + * @private + */ +function getDelimiterToken(delimiter) { + if (/\(|\)/.test(delimiter)) + return new Token('delimiter', delimiter == '(' ? 'left' : 'right', 'parenthesis'); + else if (/\[|\]/.test(delimiter)) + return new Token('delimiter', delimiter == '[' ? 'left' : 'right', 'bracket'); + else if (/\{|\}/.test(delimiter)) + return new Token('delimiter', delimiter == '{' ? 'left' : 'right', 'brace'); + else throw new Error('Expected delimiter but got ' + delimiter); +} + +/** + * @function operatorType + * @desc Determines the type of operator. + * @param {string} operator The operator char. + * @returns {string} The type of operator. + * @private + */ +function operatorType(operator) { + // Left operators have parameters on the left. + if (/\+\+|--/.test(operator)) + return 'left'; + else if (false) + return 'right'; + else if (/\;/.test(operator)) + return 'none'; + else + return 'dual'; +} + +/** + * @function determineCharType + * @desc Detects the type of characters. + * @param {string} char The input character(s). + * @returns {string} The type of char. + * @private + */ +function determineCharType(char) { + if (/[A-Za-z]/.test(char)) + return 'letter'; + else if (/\+|\-|\*|\/|\=|\=\=|\>|\<|\>\=|\<\=|\=\>|;/.test(char)) + return 'operator'; + else if (/\(|\)|\[|\]|\{|\}/.test(char)) + return 'delimiter'; + else if (/'|"|`/.test(char)) + return 'string delimiter'; + else if (/\d/.test(char)) + return 'digit'; + else if (/\\./.test(char)) + return 'escaped char'; + else if (/\s/.test(char)) + return 'whitespace'; + else throw new SyntaxError('Unexpected char ' + char); +}; + +/** + * @function determineType + * @desc Detects the type of a string. + * @param {string} str The input string. + * @returns {string} The type of string. + * @private + */ +function determineType(str) { + if (/let|return/.test(str)) + return 'keyword'; + else return 'unknown'; +}; + +module.exports = { + tokenize, + util: { + combineEscapedChars, + removeComments, + changeKeywords, + getDelimiterToken, + operatorType, + determineCharType, + determineType + } +}; diff --git a/src/types.js b/src/types.js new file mode 100644 index 0000000..5d2a7a5 --- /dev/null +++ b/src/types.js @@ -0,0 +1,31 @@ +/** + * @module types + * @file Provides a consistent source of types for the compiler. + * @author Garen Tyler + */ + + +/** + * @class Token + * @classdesc Stores the type of token, subtype, and literal char value. + */ +function Token(type, subtype, value) { + this.type = type; + this.subtype = subtype; + this.value = value; +} + +/** + * @class Group + * @classdesc Stores the type of group, and the tokens in the group. + */ +function Group(type, tokens) { + this.type = 'group' + this.subtype = type; + this.tokens = tokens; +} + +module.exports = { + Token, + Group +}; diff --git a/test.pivot b/test.pivot deleted file mode 100644 index 98f93a2..0000000 --- a/test.pivot +++ /dev/null @@ -1,6 +0,0 @@ -// Example Pivot code. -// This is a comment. -x = 4 // This creates a variable x with the value of 4. -y = 3 * x // This creates a variable y with the value of the output of (3 * x). -asdf = (a) {return(a++)} // This creates a function asdf that takes a number and returns that number + 1/ -a = asdf(2) // This creates a variable a with the value of the output of asdf(2). diff --git a/test/test.js b/test/test.js new file mode 100644 index 0000000..365ae6d --- /dev/null +++ b/test/test.js @@ -0,0 +1,126 @@ +const assert = require('assert'); + +const types = require('../src/types.js'); +const tokenizer = require('../src/tokenizer.js'); +const parser = require('../src/parser.js'); + +describe('types.js', () => { + it('Has a Token child.', () => { + assert.equal(types.hasOwnProperty('Token'), true); + }); + it('Has a Group child.', () => { + assert.equal(types.hasOwnProperty('Group'), true); + }); + + describe('Token', () => { + it('Works as a constructor', () => { + try { + let token = new types.Token('a', 'b', 'c'); + } catch (err) { + throw err; + } + }); + it('Has values \'type\', \'subtype\', and \'value\'', () => { + try { + let token = new types.Token('a', 'b', 'c'); + if (!token.hasOwnProperty('type') || !token.hasOwnProperty('subtype') || !token.hasOwnProperty('value')) + throw new Error('Token is missing \'type\', \'subtype\', or \'value\' properties.'); + if (token.type != 'a' || token.subtype != 'b' || token.value != 'c') + throw new Error('Token incorrectly set \'type\', \'subtype\', or \'value\' properties.'); + } catch (err) { + throw err; + } + }); + }); + describe('Group', () => { + it('Works as a constructor', () => { + try { + let group = new types.Group('a', 'b'); + } catch (err) { + throw err; + } + }); + it('Has values \'type\', \'subtype\', and \'tokens\'', () => { + try { + let group = new types.Group('a', 'b'); + if (!group.hasOwnProperty('type') || !group.hasOwnProperty('subtype') || !group.hasOwnProperty('tokens')) + throw new Error('Group is missing \'type\', \'subtype\', or \'tokens\' properties.'); + if (group.type != 'group' || group.subtype != 'a' || group.tokens != 'b') + throw new Error('Group incorrectly set \'type\', \'subtype\', or \'tokens\' properties.'); + } catch (err) { + throw err; + } + }); + }); +}); +describe('tokenizer.js', () => { + it('Has a tokenize child', () => { + assert.equal(tokenizer.hasOwnProperty('tokenize'), true); + }); + it('Has a util child', () => { + assert.equal(tokenizer.hasOwnProperty('util'), true); + }); + describe('util', () => { + it('combineEscapedChars works', () => { + assert.equal(tokenizer.util.combineEscapedChars(`let x = 'test\\nnewline';`.split('')).join(''), `let x = 'test\\nnewline';`); + }); + it('removeComments works', () => { + assert.equal(tokenizer.util.removeComments(`// Comment\nlet i = 0;`.split('')).join(''), `let i = 0;`); + }); + it('changeKeywords works', () => { + let tokens = tokenizer.util.changeKeywords([{ + type: 'name', + subtype: 'variable', + value: 'let' + }, { + type: 'name', + subtype: 'variable', + value: 'x' + }]); + let correct = [{ + type: 'name', + subtype: 'keyword', + value: 'let' + }, { + type: 'name', + subtype: 'variable', + value: 'x' + }]; + let isCorrect = true; + tokens.forEach((t, i) => { + if (t.type != correct[i].type) + throw new Error('Changed type: Expected \''+ correct[i].type +'\' but got ' + t.type) + else if (t.subtype != correct[i].subtype) + throw new Error('Incorrectly changed subtype: Expected \''+ correct[i].subtype +'\' but got ' + t.subtype) + else if (t.value != correct[i].value) + throw new Error('Changed value: Expected \''+ correct[i].value +'\' but got ' + t.value) + }); + }); + it('getDelimiterToken works', () => { + let token = tokenizer.util.getDelimiterToken(')'); + if (token.type != 'delimiter') + throw new Error('Incorrect type: Expected \'delimiter\' but got ' + token.type) + else if (token.subtype != 'right') + throw new Error('Incorrect subtype: Expected \'right\' but got ' + token.subtype) + else if (token.value != 'parenthesis') + throw new Error('Incorrect value: Expected \'parenthesis\' but got ' + token.value) + }); + it('operatorType works', () => { + assert.equal(tokenizer.util.operatorType('++'), 'left'); + assert.equal(tokenizer.util.operatorType(';'), 'none'); + assert.equal(tokenizer.util.operatorType('+'), 'dual'); + }); + it('determineCharType works', () => { + assert.equal(tokenizer.util.determineCharType('+'), 'operator'); + assert.equal(tokenizer.util.determineCharType('"'), 'string delimiter'); + assert.equal(tokenizer.util.determineCharType('4'), 'digit'); + }); + it('determineType works', () => { + assert.equal(tokenizer.util.determineType('let'), 'keyword'); + assert.equal(tokenizer.util.determineType('dog'), 'unknown'); + }); + }); +}); +describe('parser.js', () => { + +}); diff --git a/tokenizer.js b/tokenizer.js deleted file mode 100644 index f8dab7d..0000000 --- a/tokenizer.js +++ /dev/null @@ -1,170 +0,0 @@ -// Import the token class. -const token = require("./classes.js").token; - -// Create the tokenizer function. -// tokenize() takes Pivot code in, and outputs an array of tokens. -module.exports = exp => { - // Check functions for different token types. - var isDigit = char => { - return /\d/.test(char); - }; - var isLetter = char => { - return /[a-z]/i.test(char); - }; - var isOperator = char => { - return /\+|-|\*|\/|\^|=/.test(char); - }; - var isLeftDelimiter = char => { - return (/\(|\[|\{|"|'|`/.test(char)); - }; - var isRightDelimiter = char => { - return (/\)|\]|\}/.test(char)); - }; - var isComma = char => { - return (char === ","); - }; - var isPeriod = char => { - return (char === "."); - }; - var result = []; // The final array of tokens. - var nb = []; // Number buffer. Allows for multiple digits to be one number. - var lb = []; // Letter buffer. Allows for multiple letters to be one variable / function. - var ob = []; // Operator buffer. Allows for multi-character operators. E.g. ++ or ==. - var sb = []; // String buffer. Allows for multi-character strings. - var inString = false; // Keep track of whether in string or not. - var stringType; // Keep track of what type of string. E.g. "" or ''. - exp = exp.split(""); // Split the expression into an array of characters. - /* - - - DO NOT TOUCH THIS - - - */ - for (var i = 0; i < exp.length; i++) { // Loop through all of the characters. - var char = exp[i]; // Create a quick reference to the current char. - if (i >= 1) { - if (exp[i - 1] == "\\") { - exp.splice(i - 1, 2, `\\${char}`); - i--; - continue; - } - if (exp[i - 1] == "$" && char == "{") { - exp.splice(i - 1, 2, `\${`); - i--; - continue; - } - } - } - /* - - - OK YOU CAN TOUCH AGAIN - - - */ - // Nevermind, just don't mess with any of this file. - for (var i = 0; i < exp.length; i++) { - var char = exp[i]; - if (inString) { - if (char == `'` || char == `"` || char == "`") { - var exitString = () => { - inString = false; - if (sb.length == 0) { - result.push(new token("String", null)); - } else { - var string = sb.join(""); - result.push(new token("String", string)); - } - sb = []; - }; - if (char == `'` && stringType == "single") { - exitString(); - } else if (char == `"` && stringType == "double") { - exitString(); - } else if (char == "`" && stringType == "backtick") { - exitString(); - } else { - if (char == `'`) { - sb.push(`\'`); - } - if (char == `"`) { - sb.push(`\"`); - } - if (char == "`") { - sb.push("\`"); - } - } - } else { - sb.push(char); - } - } else { - if (isDigit(char)) { - result.push(new token("Operator", ob.join(""))); - ob = []; - nb.push(char); - } else if (isLetter(char)) { - result.push(new token("Operator", ob.join(""))); - ob = []; - lb.push(char); - } else if (isOperator(char)) { - result.push(new token("Number", nb.join(""))); - nb = []; - result.push(new token("Variable", lb.join(""))); - lb = []; - ob.push(char); - } else if (isLeftDelimiter(char)) { - result.push(new token("Operator", ob.join(""))); - ob = []; - result.push(new token("Function Call", lb.join(""))); - lb = []; - if (char == `'` || char == `"` || char == "`") { - inString = true; - if (char == `'`) { - stringType = "single"; - } else if (char == `"`) { - stringType = "double"; - } else if (char == "`") { - stringType = "backtick"; - } - } else { - result.push(new token("Left Delimiter", char)); - } - } else if (isRightDelimiter(char)) { - result.push(new token("Operator", ob.join(""))); - ob = []; - result.push(new token("Number", nb.join(""))); - nb = []; - result.push(new token("Variable", lb.join(""))); - lb = []; - result.push(new token("Right Delimiter", char)); - } else if (isComma(char)) { - result.push(new token("Operator", ob.join(""))); - ob = []; - result.push(new token("Number", nb.join(""))); - nb = []; - result.push(new token("Variable", lb.join(""))); - lb = []; - result.push(new token("Comma", char)); - } else if (isPeriod(char)) { - result.push(new token("Operator", ob.join(""))); - ob = []; - nb.push(char); - } - } - } - result.push(new token("Operator", ob.join(""))); - ob = []; - result.push(new token("Number", nb.join(""))); - nb = []; - lb.forEach(item => { - result.push(new token("Variable", item)); - }); - lb = []; - for (var i = 0; i < 3; i++) { - result.forEach((item, index) => { - if (item.value == "") { - result.splice(index, 1); - } - }); - } - result.forEach((item, index) => { - if (item.value == "-" && index != 0) { - if (result[index - 1].type != "Variable" && result[index - 1].type != "Number") { - if (result[index + 1].type == "Number") { - result[index + 1].value = "-" + result[index + 1].value; - result.splice(index, 1); - } - } - } - }); - return result; -}; \ No newline at end of file