Refactored tokenizer, implemented jsdoc, set up for cli.

.gitignore

@@ -0,0 +1,4 @@
+other/
+package-lock.json
+node_modules/
+jsdoc/

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2019 ElementG9
+Copyright (c) 2019 Garen Tyler
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

ast.json

@@ -1,50 +0,0 @@
-[
-    {
-        "type": "Variable",
-        "value": "asdf",
-        "layer": 0
-    },
-    {
-        "type": "Operator",
-        "value": "=",
-        "layer": 0
-    },
-    {
-        "type": "Group",
-        "value": "(",
-        "tokens": [
-            {
-                "type": "Variable",
-                "value": "a",
-                "layer": 1
-            }
-        ]
-    },
-    {
-        "type": "Group",
-        "value": "{",
-        "tokens": [
-            {
-                "type": "Function Call",
-                "value": "return",
-                "layer": 1
-            },
-            {
-                "type": "Group",
-                "value": "(",
-                "tokens": [
-                    {
-                        "type": "Variable",
-                        "value": "a",
-                        "layer": 2
-                    },
-                    {
-                        "type": "Operator",
-                        "value": "++",
-                        "layer": 2
-                    }
-                ]
-            }
-        ]
-    }
-]

bin/pivot.js

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+const args = process.argv.slice(2);
+console.log(args);

classes.js

@@ -1,21 +0,0 @@
-// This is the most basic type of token.
-var token = function (type, value) {
-  this.type = type;
-  this.value = value;
-};
-// This is a group of tokens.
-var group = function (type, tokens, index) {
-  this.type = "Group";
-  this.value = type;
-  this.index = index;
-  this.tokens;
-  if (typeof tokens != "undefined") {
-    this.tokens = tokens;
-  } else {
-    this.tokens = [];
-  }
-}
-module.exports = {
-  token: token,
-  group: group
-}

package.json

@@ -0,0 +1,39 @@
+{
+  "name": "pivot",
+  "version": "1.0.0",
+  "description": "Pivot is a new programming language built on JavaScript",
+  "main": "./bin/pivot.js",
+  "directories": {
+    "test": "test"
+  },
+  "scripts": {
+    "test": "mocha",
+    "build": "npm run test; jsdoc src/ -r -d ./jsdoc/ -R ./readme.md"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/ElementG9/Pivot.git"
+  },
+  "keywords": [
+    "pivot",
+    "javascript",
+    "programming",
+    "language"
+  ],
+  "author": "Garen Tyler <garentyler@gmail.com>",
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/ElementG9/Pivot/issues"
+  },
+  "homepage": "https://github.com/ElementG9/Pivot#readme",
+  "bin": {
+    "pivot": "./bin/pivot.js"
+  },
+  "preferGlobal": true,
+  "dependencies": {
+    "fs": "0.0.1-security"
+  },
+  "devDependencies": {
+    "mocha": "^6.2.1"
+  }
+}

parser.js

@@ -1,134 +0,0 @@
-// Import the group class.
-const group = require("./classes.js").group;
-
-// Create the parser function.
-// parse() takes an array of tokens in, and outputs an
-// Abstract Syntax Tree (a structured array of tokens).
-module.exports = tokens => {
-  // Variables for later.
-  var layer = 0;
-  var delimiterCount = 0;
-  var deepestLayer = 0;
-  // Give each token a layer number based on delimiters.
-  for (var i = 0; i < tokens.length; i++) {
-    if (tokens[i].type == "Left Delimiter") {
-      layer++;
-      if (layer > deepestLayer) {
-        deepestLayer = layer;
-      }
-      delimiterCount++;
-    }
-    tokens[i].layer = layer;
-    if (tokens[i].type == "Right Delimiter") {
-      layer--;
-    }
-  }
-  // Lower the layer of delimiters.
-  for (var i = 0; i < tokens.length; i++) {
-    if ((tokens[i].type == "Left Delimiter") || (tokens[i].type == "Right Delimiter")) {
-      tokens[i].layer--;
-    }
-  }
-  if (layer > 0) { // Unclosed delimiter.
-  } else if (layer < 0) { // Overclosed delimiter.
-  }
-  // Give each token an index.
-  for (let i = 0; i < tokens.length; i++) {
-    tokens[i].index = i;
-  }
-  // Structure the layers.
-  // Count the rising edges of the layers to determine how many groups should exist.
-  let structure = function () {
-    let layer = 0;
-    let risingFalling = []; // Create an array to store indices of rising/falling edges.
-    for (let i = 0; i < tokens.length; i++) {
-      // Add a rising and a falling tag to each token.
-      tokens[i].rising = false;
-      tokens[i].falling = false;
-      if (tokens[i].layer > layer) { // If the token moves up a layer.
-        // Create a new rising index in risingFalling.
-        risingFalling.push({
-          type: 'rising',
-          index: i
-        });
-        tokens[i].rising = true; // Note that the token is a rising edge.
-        layer++;
-      } else if (tokens[i].layer < layer) {
-        // Create a new falling index in risingFalling.
-        risingFalling.push({
-          type: 'falling',
-          index: i
-        });
-        tokens[i].falling = true; // Note that the token is a falling edge.
-        layer--;
-      }
-    }
-    // Loop through the list of rising/falling edges.
-    for (let i = 0; i < risingFalling.length; i++) {
-      if (i != risingFalling.length - 1) { // If not the last edge.
-        let item = risingFalling[i];
-        let nextItem = risingFalling[i + 1];
-        // If a falling edge follows a rising edge, classifiy it as a group.
-        if ((item.type == 'rising') && (nextItem.type == 'falling')) {
-          // Get the group together as one item.
-          let selectedItems = tokens.slice(item.index, nextItem.index);
-          tokens.splice(item.index, selectedItems.length, new group(tokens[item.index - 1].value, selectedItems, item.index));
-        }
-      }
-    }
-    risingFalling = []; // Reset the list of edges.
-    // Count the edges again.
-    for (let i = 0; i < tokens.length; i++) {
-      if (tokens[i].layer > layer) {
-        risingFalling.push({
-          type: 'rising',
-          index: i
-        });
-        layer++;
-      } else if (tokens[i].layer < layer) {
-        risingFalling.push({
-          type: 'falling',
-          index: i
-        });
-        layer--;
-      }
-    }
-    // If there are still edges, run again.
-    if (risingFalling.length) {
-      structure();
-    }
-  };
-  // Start the recursion.
-  structure();
-  let trimDelimiters = function (thing) {
-    // Loop through the tokens of thing.
-    for (let i = 0; i < thing.length; i++) {
-      // Delete unnecessary keys.
-      if (typeof thing[i].rising != 'undefined') {
-        delete thing[i].rising;
-      }
-      if (typeof thing[i].falling != 'undefined') {
-        delete thing[i].falling;
-      }
-      if (typeof thing[i].index != 'undefined') {
-        delete thing[i].index;
-      }
-      if (typeof thing[i].index != 'undefined') {
-        delete thing[i].index;
-      }
-      // Remove delimiters.
-      if ((thing[i].type == 'Left Delimiter') || (thing[i].type == 'Right Delimiter')) {
-        thing.splice(i, 1);
-        i--;
-      }
-      // If a token is a group, look at the group's tokens.
-      if (thing[i].type == 'Group') {
-        trimDelimiters(thing[i].tokens);
-      }
-    }
-  };
-  // Start the recursion.
-  trimDelimiters(tokens);
-  // Return the structured tokens.
-  return tokens;
-};

pivot.js

@@ -1,13 +0,0 @@
-// The Pivot code to run.
-var code = `asdf = (a) {return(a++)}`;
-
-// Import the tokenizer and parser.
-const tokenize = require("./tokenizer.js");
-const parse = require("./parser.js");
-
-// Generate the AST.
-var ast = parse(tokenize(code));
-
-// Write the AST to ast.json.
-var fs = require("fs");
-fs.writeFileSync("ast.json", JSON.stringify(ast, null, 4));

readme.md

@@ -1,15 +0,0 @@
-# [Pivot](https://gthub.us/pivot/)
-
-Pivot is a new programming language built on [JavaScript](https://javascript.com/) by [ElementG9](https://github.com/ElementG9/).
-
----
-
-## Installation
-* Download or clone [the GitHub repository](/pivot/repo).
-* Download [node.js](https://nodejs.org/).
-* Run _pivot.js_ with `node pivot.js`.
-
----
-
-## Documentation
-The Pivot documentation is [here](https://gthub.us/pivot/docs).

rpn.js

@@ -1,40 +0,0 @@
-var solve = (exp) => {
-    var stack = [];
-    var expression = exp.split(" ");
-    for(var j=0;j<expression.length;j++) {
-        var key = expression[j];
-        if(key.match(/\d/)) {
-            stack.push(parseInt(key));
-        } else if(key.match(/\w/)) {
-            if(Object.keys(progData).includes(key)) {
-                stack.push(progData[key]);
-            } else {
-                stack.push(key);
-            }
-        }
-        switch(key) {
-            case "+": // add
-                var opItems = stack.splice(stack.length-2,2);
-                var result = opItems[0]+opItems[1];
-                stack.push(result);
-            break;
-            case "-": // subtract
-                var opItems = stack.splice(stack.length-2,2);
-                var result = opItems[0]-opItems[1];
-                stack.push(result);
-            break;
-            case "*": // multiply
-                var opItems = stack.splice(stack.length-2,2);
-                var result = opItems[0]*opItems[1];
-                stack.push(result);
-            break;
-            case "/": // divide
-                var opItems = stack.splice(stack.length-2,2);
-                var result = opItems[0]/opItems[1];
-                stack.push(result);
-            break;
-        }
-    }
-    return stack;
-};
-module.exports = solve;

src/parser.js

@@ -0,0 +1,44 @@
+/**
+ * @module parser
+ * @file Manages the parser phase of Pivot.
+ * @author Garen Tyler <garentyler@gmail.com>
+ * @requires module:types
+ */
+const Token = require('./types.js').Token;
+const Group = require('./types.js').Token;
+
+module.exports = function(tokens) {
+  let level = 0;
+  // Add level markers.
+  tokens.forEach((t, i) => {
+    if (t.type == 'delimiter' && t.subtype == 'left') {
+      tokens[i].level = level;
+      level++;
+    } else if (t.type == 'delimiter' && t.subtype == 'right') {
+      level--;
+      tokens[i].level = level;
+    } else {
+      tokens[i].level = level;
+    }
+  });
+  // Group.
+  let currentLevel = 0;
+  let groupStack = [0];
+  tokens.forEach((t, i) => {
+    if (currentLevel < t.level) {
+      tokens.splice(i, 0, new Group(tokens[i - 1].value, []));
+      groupStack.push(i);
+      currentLevel++;
+      tokens[i].level = currentLevel;
+    }
+    if (t.level != 0) {
+      tokens[groupStack.slice(-1)].tokens.push(t);
+    }
+    if (currentLevel > t.level) {
+      groupStack.pop();
+      currentLevel--;
+    }
+  });
+  if (currentLevel != 0) {} // Error: missing delimiter.
+  return tokens;
+};

src/tokenizer.js

@@ -0,0 +1,303 @@
+/**
+ * @module tokenizer
+ * @file Manages the tokenization phase of Pivot.
+ * @author Garen Tyler <garentyler@gmail.com>
+ * @requires module:types
+ */
+const Token = require('./types.js').Token;
+const Group = require('./types.js').Token;
+
+/**
+ * @function tokenize
+ * @desc Takes in raw code, and outputs an array of Tokens.
+ * @param {string} code The raw input code.
+ * @returns {Token[]} The code, split into tokens.
+ * @public
+ */
+function tokenize(code) {
+  // Split the string into an array of chars.
+  let chars = code.split('');
+
+  // Create buffers.
+  let letterBuffer = [];
+  let operatorBuffer = [];
+  let numberBuffer = [];
+  let stringBuffer = [];
+
+  // Create the output Token[].
+  let tokens = [];
+
+  // Create an object to keep track of string data.
+  let stringData = {
+    inString: false,
+    stringType: null
+  };
+
+  // Escape chars and remove comments.
+  chars = combineEscapedChars(chars);
+  chars = removeComments(chars);
+
+
+  // Actually tokenize the chars.
+  for (let i = 0; i < chars.length; i++) {
+    let char = chars[i];
+    if (stringData.inString) { // Tokenize differently in a string.
+      // If a string delimiter and the same as the inital delimiter.
+      if (determineCharType(char) == 'string delimiter' && char == stringData.stringType) {
+        stringData.inString = false; // Not in a string any more.
+        tokens.push(new Token('string', 'n/a', stringBuffer.join(''))); // Push the string.
+        stringBuffer = []; // Clear the string buffer.
+      } else stringBuffer.push(char); // Add to the string buffer.
+    } else { // Tokenize normally.
+      if (determineCharType(char) == 'string delimiter') {
+        stringData.inString = true; // In a string now.
+        stringData.stringType = char;
+      } else if (determineCharType(char) == 'letter') {
+        letterBuffer.push(char); // Add to the letter buffer.
+        // End the other buffers.
+        if (operatorBuffer.length > 0) {
+          let operator = operatorBuffer.join('');
+          tokens.push(new Token('operator', operatorType(operator), operator));
+          operatorBuffer = [];
+        }
+        if (numberBuffer.length > 0) {
+          let number = numberBuffer.join('');
+          tokens.push(new Token('number', 'n/a', number));
+          numberBuffer = [];
+        }
+      } else if (determineCharType(char) == 'operator') {
+        operatorBuffer.push(char); // Add to the operator buffer.
+        // End the other buffers.
+        if (letterBuffer.length > 0) {
+          let variable = letterBuffer.join('');
+          tokens.push(new Token('name', 'variable', variable));
+          letterBuffer = [];
+        }
+        if (numberBuffer.length > 0) {
+          let number = numberBuffer.join('');
+          tokens.push(new Token('number', 'n/a', number));
+          numberBuffer = [];
+        }
+      } else if (determineCharType(char) == 'digit') {
+        numberBuffer.push(char); // Add to the number buffer.
+        // End the other buffers.
+        if (letterBuffer.length > 0) {
+          let variable = letterBuffer.join('');
+          tokens.push(new Token('name', 'variable', variable));
+          letterBuffer = [];
+        }
+        if (operatorBuffer.length > 0) {
+          let operator = operatorBuffer.join('');
+          tokens.push(new Token('operator', operatorType(operator), operator));
+          operatorBuffer = [];
+        }
+      } else if (determineCharType(char) == 'whitespace') {
+        // End all buffers.
+        if (letterBuffer.length > 0) {
+          let variable = letterBuffer.join('');
+          tokens.push(new Token('name', 'variable', variable));
+          letterBuffer = [];
+        }
+        if (numberBuffer.length > 0) {
+          let number = numberBuffer.join('');
+          tokens.push(new Token('number', 'n/a', number));
+          numberBuffer = [];
+        }
+        if (operatorBuffer.length > 0) {
+          let operator = operatorBuffer.join('');
+          tokens.push(new Token('operator', operatorType(operator), operator));
+          operatorBuffer = [];
+        }
+      } else if (determineCharType(char) == 'delimiter') {
+        // End all buffers.
+        if (letterBuffer.length > 0) {
+          let variable = letterBuffer.join('');
+          tokens.push(new Token('name', 'variable', variable));
+          letterBuffer = [];
+        }
+        if (numberBuffer.length > 0) {
+          let number = numberBuffer.join('');
+          tokens.push(new Token('number', 'n/a', number));
+          numberBuffer = [];
+        }
+        if (operatorBuffer.length > 0) {
+          let operator = operatorBuffer.join('');
+          tokens.push(new Token('operator', operatorType(operator), operator));
+          operatorBuffer = [];
+        }
+        // Push the delimiter.
+        tokens.push(getDelimiterToken(char));
+      }
+    }
+  }
+
+  // Empty all the buffers.
+  if (letterBuffer.length > 0) {
+    let variable = letterBuffer.join('');
+    tokens.push(new Token('name', 'variable', variable));
+    letterBuffer = [];
+  }
+  if (numberBuffer.length > 0) {
+    let number = numberBuffer.join('');
+    tokens.push(new Token('number', 'n/a', number));
+    numberBuffer = [];
+  }
+  if (operatorBuffer.length > 0) {
+    let operator = operatorBuffer.join('');
+    tokens.push(new Token('operator', operatorType(operator), operator));
+    operatorBuffer = [];
+  }
+
+  tokens = changeKeywords(tokens);
+
+  return tokens;
+}
+
+/**
+ * @function combineEscapedChars
+ * @desc Combines escaped chars into one char.
+ * @param {string[]} chars The chars.
+ * @returns {string[]} The chars with combined escaped chars.
+ * @private
+ */
+function combineEscapedChars(chars) {
+  // Check for characters to be escaped.
+  for (let i = 0; i < chars.length; i++) {
+    if (chars[i] == '\\') {
+      chars.splice(i, 2, chars[i] + chars[i + 1]);
+      i -= 2;
+    }
+  }
+  return chars;
+}
+
+/**
+ * @function removeComments
+ * @desc Removes comments.
+ * @param {string[]} chars The chars.
+ * @returns {string[]} The chars without comments.
+ * @private
+ */
+function removeComments(chars) {
+  let inComment = false; // Keep track if in a comment.
+  for (let i = 0; i < chars.length; i++) {
+    if (chars[i] == '/') {
+      if (chars[i + 1] == '/') {
+        inComment = true;
+      }
+    }
+    if (chars[i] == '\n') {
+      inComment = false;
+      chars.splice(i, 1); // Remove the newline at the end of the comment.
+      i--;
+    }
+    if (inComment) {
+      chars.splice(i, 1); // Remove the char in the comment.
+      i--;
+    }
+  }
+  return chars;
+}
+
+/**
+ * @function changeKeywords
+ * @desc Changes tokens with subtype variable to subtype keyword
+ * @param {Token[]} tokens The tokens
+ * @returns {Token[]} The tokens with keywords.
+ * @private
+ */
+function changeKeywords(tokens) {
+  return tokens.map(t => {
+    if (t.subtype == 'variable' && determineType(t.value) == 'keyword') {
+      t.subtype = 'keyword';
+    }
+    return t;
+  });
+}
+
+/**
+ * @function getDelimiterToken
+ * @desc Turns a delimiter char into a token.
+ * @param {string} delimiter The delimiter char.
+ * @returns {Token} The delimiter token.
+ * @private
+ */
+function getDelimiterToken(delimiter) {
+  if (/\(|\)/.test(delimiter))
+    return new Token('delimiter', delimiter == '(' ? 'left' : 'right', 'parenthesis');
+  else if (/\[|\]/.test(delimiter))
+    return new Token('delimiter', delimiter == '[' ? 'left' : 'right', 'bracket');
+  else if (/\{|\}/.test(delimiter))
+    return new Token('delimiter', delimiter == '{' ? 'left' : 'right', 'brace');
+  else throw new Error('Expected delimiter but got ' + delimiter);
+}
+
+/**
+ * @function operatorType
+ * @desc Determines the type of operator.
+ * @param {string} operator The operator char.
+ * @returns {string} The type of operator.
+ * @private
+ */
+function operatorType(operator) {
+  // Left operators have parameters on the left.
+  if (/\+\+|--/.test(operator))
+    return 'left';
+  else if (false)
+    return 'right';
+  else if (/\;/.test(operator))
+    return 'none';
+  else
+    return 'dual';
+}
+
+/**
+ * @function determineCharType
+ * @desc Detects the type of characters.
+ * @param {string} char The input character(s).
+ * @returns {string} The type of char.
+ * @private
+ */
+function determineCharType(char) {
+  if (/[A-Za-z]/.test(char))
+    return 'letter';
+  else if (/\+|\-|\*|\/|\=|\=\=|\>|\<|\>\=|\<\=|\=\>|;/.test(char))
+    return 'operator';
+  else if (/\(|\)|\[|\]|\{|\}/.test(char))
+    return 'delimiter';
+  else if (/'|"|`/.test(char))
+    return 'string delimiter';
+  else if (/\d/.test(char))
+    return 'digit';
+  else if (/\\./.test(char))
+    return 'escaped char';
+  else if (/\s/.test(char))
+    return 'whitespace';
+  else throw new SyntaxError('Unexpected char ' + char);
+};
+
+/**
+ * @function determineType
+ * @desc Detects the type of a string.
+ * @param {string} str The input string.
+ * @returns {string} The type of string.
+ * @private
+ */
+function determineType(str) {
+  if (/let|return/.test(str))
+    return 'keyword';
+  else return 'unknown';
+};
+
+module.exports = {
+  tokenize,
+  util: {
+    combineEscapedChars,
+    removeComments,
+    changeKeywords,
+    getDelimiterToken,
+    operatorType,
+    determineCharType,
+    determineType
+  }
+};

src/types.js

@@ -0,0 +1,31 @@
+/**
+ * @module types
+ * @file Provides a consistent source of types for the compiler.
+ * @author Garen Tyler <garentyler@gmail.com>
+ */
+
+
+/**
+ * @class Token
+ * @classdesc Stores the type of token, subtype, and literal char value.
+ */
+function Token(type, subtype, value) {
+  this.type = type;
+  this.subtype = subtype;
+  this.value = value;
+}
+
+/**
+ * @class Group
+ * @classdesc Stores the type of group, and the tokens in the group.
+ */
+function Group(type, tokens) {
+  this.type = 'group'
+  this.subtype = type;
+  this.tokens = tokens;
+}
+
+module.exports = {
+  Token,
+  Group
+};

test.pivot

@@ -1,6 +0,0 @@
-// Example Pivot code.
-// This is a comment.
-x = 4 // This creates a variable x with the value of 4.
-y = 3 * x // This creates a variable y with the value of the output of (3 * x).
-asdf = (a) {return(a++)} // This creates a function asdf that takes a number and returns that number + 1/
-a = asdf(2) // This creates a variable a with the value of the output of asdf(2).

test/test.js

@@ -0,0 +1,126 @@
+const assert = require('assert');
+
+const types = require('../src/types.js');
+const tokenizer = require('../src/tokenizer.js');
+const parser = require('../src/parser.js');
+
+describe('types.js', () => {
+  it('Has a Token child.', () => {
+    assert.equal(types.hasOwnProperty('Token'), true);
+  });
+  it('Has a Group child.', () => {
+    assert.equal(types.hasOwnProperty('Group'), true);
+  });
+
+  describe('Token', () => {
+    it('Works as a constructor', () => {
+      try {
+        let token = new types.Token('a', 'b', 'c');
+      } catch (err) {
+        throw err;
+      }
+    });
+    it('Has values \'type\', \'subtype\', and \'value\'', () => {
+      try {
+        let token = new types.Token('a', 'b', 'c');
+        if (!token.hasOwnProperty('type') || !token.hasOwnProperty('subtype') || !token.hasOwnProperty('value'))
+          throw new Error('Token is missing \'type\', \'subtype\', or \'value\' properties.');
+        if (token.type != 'a' || token.subtype != 'b' || token.value != 'c')
+          throw new Error('Token incorrectly set \'type\', \'subtype\', or \'value\' properties.');
+      } catch (err) {
+        throw err;
+      }
+    });
+  });
+  describe('Group', () => {
+    it('Works as a constructor', () => {
+      try {
+        let group = new types.Group('a', 'b');
+      } catch (err) {
+        throw err;
+      }
+    });
+    it('Has values \'type\', \'subtype\', and \'tokens\'', () => {
+      try {
+        let group = new types.Group('a', 'b');
+        if (!group.hasOwnProperty('type') || !group.hasOwnProperty('subtype') || !group.hasOwnProperty('tokens'))
+          throw new Error('Group is missing \'type\', \'subtype\', or \'tokens\' properties.');
+        if (group.type != 'group' || group.subtype != 'a' || group.tokens != 'b')
+          throw new Error('Group incorrectly set \'type\', \'subtype\', or \'tokens\' properties.');
+      } catch (err) {
+        throw err;
+      }
+    });
+  });
+});
+describe('tokenizer.js', () => {
+  it('Has a tokenize child', () => {
+    assert.equal(tokenizer.hasOwnProperty('tokenize'), true);
+  });
+  it('Has a util child', () => {
+    assert.equal(tokenizer.hasOwnProperty('util'), true);
+  });
+  describe('util', () => {
+    it('combineEscapedChars works', () => {
+      assert.equal(tokenizer.util.combineEscapedChars(`let x = 'test\\nnewline';`.split('')).join(''), `let x = 'test\\nnewline';`);
+    });
+    it('removeComments works', () => {
+      assert.equal(tokenizer.util.removeComments(`// Comment\nlet i = 0;`.split('')).join(''), `let i = 0;`);
+    });
+    it('changeKeywords works', () => {
+      let tokens = tokenizer.util.changeKeywords([{
+        type: 'name',
+        subtype: 'variable',
+        value: 'let'
+      }, {
+        type: 'name',
+        subtype: 'variable',
+        value: 'x'
+      }]);
+      let correct = [{
+        type: 'name',
+        subtype: 'keyword',
+        value: 'let'
+      }, {
+        type: 'name',
+        subtype: 'variable',
+        value: 'x'
+      }];
+      let isCorrect = true;
+      tokens.forEach((t, i) => {
+        if (t.type != correct[i].type)
+          throw new Error('Changed type: Expected \''+ correct[i].type +'\' but got ' + t.type)
+        else if (t.subtype != correct[i].subtype)
+          throw new Error('Incorrectly changed subtype: Expected \''+ correct[i].subtype +'\' but got ' + t.subtype)
+        else if (t.value != correct[i].value)
+          throw new Error('Changed value: Expected \''+ correct[i].value +'\' but got ' + t.value)
+      });
+    });
+    it('getDelimiterToken works', () => {
+      let token = tokenizer.util.getDelimiterToken(')');
+      if (token.type != 'delimiter')
+        throw new Error('Incorrect type: Expected \'delimiter\' but got ' + token.type)
+      else if (token.subtype != 'right')
+        throw new Error('Incorrect subtype: Expected \'right\' but got ' + token.subtype)
+      else if (token.value != 'parenthesis')
+        throw new Error('Incorrect value: Expected \'parenthesis\' but got ' + token.value)
+    });
+    it('operatorType works', () => {
+      assert.equal(tokenizer.util.operatorType('++'), 'left');
+      assert.equal(tokenizer.util.operatorType(';'), 'none');
+      assert.equal(tokenizer.util.operatorType('+'), 'dual');
+    });
+    it('determineCharType works', () => {
+      assert.equal(tokenizer.util.determineCharType('+'), 'operator');
+      assert.equal(tokenizer.util.determineCharType('"'), 'string delimiter');
+      assert.equal(tokenizer.util.determineCharType('4'), 'digit');
+    });
+    it('determineType works', () => {
+      assert.equal(tokenizer.util.determineType('let'), 'keyword');
+      assert.equal(tokenizer.util.determineType('dog'), 'unknown');
+    });
+  });
+});
+describe('parser.js', () => {
+  
+});

tokenizer.js

@@ -1,170 +0,0 @@
-// Import the token class.
-const token = require("./classes.js").token;
-
-// Create the tokenizer function.
-// tokenize() takes Pivot code in, and outputs an array of tokens.
-module.exports = exp => {
-  // Check functions for different token types.
-  var isDigit = char => {
-    return /\d/.test(char);
-  };
-  var isLetter = char => {
-    return /[a-z]/i.test(char);
-  };
-  var isOperator = char => {
-    return /\+|-|\*|\/|\^|=/.test(char);
-  };
-  var isLeftDelimiter = char => {
-    return (/\(|\[|\{|"|'|`/.test(char));
-  };
-  var isRightDelimiter = char => {
-    return (/\)|\]|\}/.test(char));
-  };
-  var isComma = char => {
-    return (char === ",");
-  };
-  var isPeriod = char => {
-    return (char === ".");
-  };
-  var result = []; // The final array of tokens.
-  var nb = []; // Number buffer. Allows for multiple digits to be one number.
-  var lb = []; // Letter buffer. Allows for multiple letters to be one variable / function.
-  var ob = []; // Operator buffer. Allows for multi-character operators. E.g. ++ or ==.
-  var sb = []; // String buffer. Allows for multi-character strings.
-  var inString = false; // Keep track of whether in string or not.
-  var stringType; // Keep track of what type of string. E.g. "" or ''.
-  exp = exp.split(""); // Split the expression into an array of characters.
-  /* - - - DO NOT TOUCH THIS - - - */
-  for (var i = 0; i < exp.length; i++) { // Loop through all of the characters.
-    var char = exp[i]; // Create a quick reference to the current char.
-    if (i >= 1) {
-      if (exp[i - 1] == "\\") {
-        exp.splice(i - 1, 2, `\\${char}`);
-        i--;
-        continue;
-      }
-      if (exp[i - 1] == "$" && char == "{") {
-        exp.splice(i - 1, 2, `\${`);
-        i--;
-        continue;
-      }
-    }
-  }
-  /* - - - OK YOU CAN TOUCH AGAIN - - - */
-  // Nevermind, just don't mess with any of this file.
-  for (var i = 0; i < exp.length; i++) {
-    var char = exp[i];
-    if (inString) {
-      if (char == `'` || char == `"` || char == "`") {
-        var exitString = () => {
-          inString = false;
-          if (sb.length == 0) {
-            result.push(new token("String", null));
-          } else {
-            var string = sb.join("");
-            result.push(new token("String", string));
-          }
-          sb = [];
-        };
-        if (char == `'` && stringType == "single") {
-          exitString();
-        } else if (char == `"` && stringType == "double") {
-          exitString();
-        } else if (char == "`" && stringType == "backtick") {
-          exitString();
-        } else {
-          if (char == `'`) {
-            sb.push(`\'`);
-          }
-          if (char == `"`) {
-            sb.push(`\"`);
-          }
-          if (char == "`") {
-            sb.push("\`");
-          }
-        }
-      } else {
-        sb.push(char);
-      }
-    } else {
-      if (isDigit(char)) {
-        result.push(new token("Operator", ob.join("")));
-        ob = [];
-        nb.push(char);
-      } else if (isLetter(char)) {
-        result.push(new token("Operator", ob.join("")));
-        ob = [];
-        lb.push(char);
-      } else if (isOperator(char)) {
-        result.push(new token("Number", nb.join("")));
-        nb = [];
-        result.push(new token("Variable", lb.join("")));
-        lb = [];
-        ob.push(char);
-      } else if (isLeftDelimiter(char)) {
-        result.push(new token("Operator", ob.join("")));
-        ob = [];
-        result.push(new token("Function Call", lb.join("")));
-        lb = [];
-        if (char == `'` || char == `"` || char == "`") {
-          inString = true;
-          if (char == `'`) {
-            stringType = "single";
-          } else if (char == `"`) {
-            stringType = "double";
-          } else if (char == "`") {
-            stringType = "backtick";
-          }
-        } else {
-          result.push(new token("Left Delimiter", char));
-        }
-      } else if (isRightDelimiter(char)) {
-        result.push(new token("Operator", ob.join("")));
-        ob = [];
-        result.push(new token("Number", nb.join("")));
-        nb = [];
-        result.push(new token("Variable", lb.join("")));
-        lb = [];
-        result.push(new token("Right Delimiter", char));
-      } else if (isComma(char)) {
-        result.push(new token("Operator", ob.join("")));
-        ob = [];
-        result.push(new token("Number", nb.join("")));
-        nb = [];
-        result.push(new token("Variable", lb.join("")));
-        lb = [];
-        result.push(new token("Comma", char));
-      } else if (isPeriod(char)) {
-        result.push(new token("Operator", ob.join("")));
-        ob = [];
-        nb.push(char);
-      }
-    }
-  }
-  result.push(new token("Operator", ob.join("")));
-  ob = [];
-  result.push(new token("Number", nb.join("")));
-  nb = [];
-  lb.forEach(item => {
-    result.push(new token("Variable", item));
-  });
-  lb = [];
-  for (var i = 0; i < 3; i++) {
-    result.forEach((item, index) => {
-      if (item.value == "") {
-        result.splice(index, 1);
-      }
-    });
-  }
-  result.forEach((item, index) => {
-    if (item.value == "-" && index != 0) {
-      if (result[index - 1].type != "Variable" && result[index - 1].type != "Number") {
-        if (result[index + 1].type == "Number") {
-          result[index + 1].value = "-" + result[index + 1].value;
-          result.splice(index, 1);
-        }
-      }
-    }
-  });
-  return result;
-};