pivot/tokenizer.js

// Import the token class.
const token = require("./classes.js").token;

// Create the tokenizer function.
// tokenize() takes Pivot code in, and outputs an array of tokens.
module.exports = exp => {
  // Check functions for different token types.
  var isDigit = char => {
    return /\d/.test(char);
  };
  var isLetter = char => {
    return /[a-z]/i.test(char);
  };
  var isOperator = char => {
    return /\+|-|\*|\/|\^|=/.test(char);
  };
  var isLeftDelimiter = char => {
    return (/\(|\[|\{|"|'|`/.test(char));
  };
  var isRightDelimiter = char => {
    return (/\)|\]|\}/.test(char));
  };
  var isComma = char => {
    return (char === ",");
  };
  var isPeriod = char => {
    return (char === ".");
  };
  var result = []; // The final array of tokens.
  var nb = []; // Number buffer. Allows for multiple digits to be one number.
  var lb = []; // Letter buffer. Allows for multiple letters to be one variable / function.
  var ob = []; // Operator buffer. Allows for multi-character operators. E.g. ++ or ==.
  var sb = []; // String buffer. Allows for multi-character strings.
  var inString = false; // Keep track of whether in string or not.
  var stringType; // Keep track of what type of string. E.g. "" or ''.
  exp = exp.split(""); // Split the expression into an array of characters.
  /* - - - DO NOT TOUCH THIS - - - */
  for (var i = 0; i < exp.length; i++) { // Loop through all of the characters.
    var char = exp[i]; // Create a quick reference to the current char.
    if (i >= 1) {
      if (exp[i - 1] == "\\") {
        exp.splice(i - 1, 2, `\\${char}`);
        i--;
        continue;
      }
      if (exp[i - 1] == "$" && char == "{") {
        exp.splice(i - 1, 2, `\${`);
        i--;
        continue;
      }
    }
  }
  /* - - - OK YOU CAN TOUCH AGAIN - - - */
  // Nevermind, just don't mess with any of this file.
  for (var i = 0; i < exp.length; i++) {
    var char = exp[i];
    if (inString) {
      if (char == `'` || char == `"` || char == "`") {
        var exitString = () => {
          inString = false;
          if (sb.length == 0) {
            result.push(new token("String", null));
          } else {
            var string = sb.join("");
            result.push(new token("String", string));
          }
          sb = [];
        };
        if (char == `'` && stringType == "single") {
          exitString();
        } else if (char == `"` && stringType == "double") {
          exitString();
        } else if (char == "`" && stringType == "backtick") {
          exitString();
        } else {
          if (char == `'`) {
            sb.push(`\'`);
          }
          if (char == `"`) {
            sb.push(`\"`);
          }
          if (char == "`") {
            sb.push("\`");
          }
        }
      } else {
        sb.push(char);
      }
    } else {
      if (isDigit(char)) {
        result.push(new token("Operator", ob.join("")));
        ob = [];
        nb.push(char);
      } else if (isLetter(char)) {
        result.push(new token("Operator", ob.join("")));
        ob = [];
        lb.push(char);
      } else if (isOperator(char)) {
        result.push(new token("Number", nb.join("")));
        nb = [];
        result.push(new token("Variable", lb.join("")));
        lb = [];
        ob.push(char);
      } else if (isLeftDelimiter(char)) {
        result.push(new token("Operator", ob.join("")));
        ob = [];
        result.push(new token("Function Call", lb.join("")));
        lb = [];
        if (char == `'` || char == `"` || char == "`") {
          inString = true;
          if (char == `'`) {
            stringType = "single";
          } else if (char == `"`) {
            stringType = "double";
          } else if (char == "`") {
            stringType = "backtick";
          }
        } else {
          result.push(new token("Left Delimiter", char));
        }
      } else if (isRightDelimiter(char)) {
        result.push(new token("Operator", ob.join("")));
        ob = [];
        result.push(new token("Number", nb.join("")));
        nb = [];
        result.push(new token("Variable", lb.join("")));
        lb = [];
        result.push(new token("Right Delimiter", char));
      } else if (isComma(char)) {
        result.push(new token("Operator", ob.join("")));
        ob = [];
        result.push(new token("Number", nb.join("")));
        nb = [];
        result.push(new token("Variable", lb.join("")));
        lb = [];
        result.push(new token("Comma", char));
      } else if (isPeriod(char)) {
        result.push(new token("Operator", ob.join("")));
        ob = [];
        nb.push(char);
      }
    }
  }
  result.push(new token("Operator", ob.join("")));
  ob = [];
  result.push(new token("Number", nb.join("")));
  nb = [];
  lb.forEach(item => {
    result.push(new token("Variable", item));
  });
  lb = [];
  for (var i = 0; i < 3; i++) {
    result.forEach((item, index) => {
      if (item.value == "") {
        result.splice(index, 1);
      }
    });
  }
  result.forEach((item, index) => {
    if (item.value == "-" && index != 0) {
      if (result[index - 1].type != "Variable" && result[index - 1].type != "Number") {
        if (result[index + 1].type == "Number") {
          result[index + 1].value = "-" + result[index + 1].value;
          result.splice(index, 1);
        }
      }
    }
  });
  return result;
};