Remove wasm

.gitattributes

@@ -0,0 +1,2 @@
+# Auto detect text files and perform LF normalization
+* text=auto

.gitignore

@@ -1,5 +1,3 @@
-other/
+/target
-package-lock.json
+node_modules
-node_modules/
+*.wasm
-jsdoc/
-ast.json

Cargo.lock

@@ -0,0 +1,7 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 3
+
+[[package]]
+name = "pivot"
+version = "0.1.0"

Cargo.toml

@@ -0,0 +1,8 @@
+[package]
+name = "pivot"
+version = "0.1.0"
+authors = ["Garen Tyler <garentyler@garen.dev>"]
+edition = "2018"
+description = "Pivot is a new programming language built with Rust. Pivot is currently in the alpha stage of development."
+license = "MIT"
+repository = "https://github.com/garentyler/pivot"

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2019 Garen Tyler
+Copyright (c) 2020 Garen Tyler
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

bin/pivot.js

@@ -1,35 +0,0 @@
-#!/usr/bin/env node
-const args = process.argv.slice(2);
-const tokenizer = require('../src/tokenizer.js');
-const parser = require('../src/parser.js');
-const code = require('../src/code.js');
-
-if (typeof args[0] != 'undefined') {
-  // Execute from file.
-} else { // REPL.
-  const rl = require('readline-sync');
-  const exec = require('child_process').exec;
-  function repl(prompt, func) {
-    let answer;
-    while (answer != 'exit') {
-      answer = rl.question(prompt);
-      if (answer == 'exit')
-        process.exit(0);
-      if (answer == 'clear') {
-        console.clear();
-        continue;
-      }
-      func(answer);
-    }
-  }
-  console.log('Welcome to Pivot v0.2.0 Alpha.');
-  console.log('Type \'exit\' to exit.');
-  let data = {
-    log: "console.log"
-  };
-  repl('> ', (answer) => {
-    let jsAnswer = code.translate(parser.parse(tokenizer.tokenize(answer)), data);
-    // console.log(require('util').inspect(jsAnswer, { depth: null }));
-    eval(jsAnswer);
-  });
-}

doc/operators.txt

@@ -1,82 +0,0 @@
-A higher precedence operator becomes an operand for a lower precedence one.
-Higher precedence operators get resolved first.
-
-Operators in Pivot:
-+-------------+------------+----------------------------------+--------------------+-----------------+
-| Precedence  |  Operator  |  Description                     |  Operands          |  Associativity  |
-+-------------+------------+----------------------------------+--------------------+-----------------+
-|  16         |  ()        |  Grouping                        |  internal          |  n/a            | grouping()
-+-------------+------------+----------------------------------+--------------------+-----------------+
-|  15         |  .         |  Member Access                   |  dual              |  Left to Right  | memberAccess()
-|  15         |  []        |  Computed Member Access          |  before, internal  |  Left to Right  | computedMemberAccess()
-|  15         |  ()        |  Function Call                   |  before, internal  |  Left to Right  | functionCall()
-+-------------+------------+----------------------------------+--------------------+-----------------+
-|  14         |  let       |  Variable Creation               |  after             |  Right to Left  | keywords()
-|  14         |  const     |  Constant Creation               |  after             |  Right to Left  | keywords()
-|  14         |  new       |  Object Creation                 |  after             |  Right to Left  | keywords()
-|  14         |  return    |  Function Return                 |  after             |  n/a            | keywords()
-|  14         |  () {}     |  Function Creation               |  internal          |  n/a            | functionCreation()
-+-------------+------------+----------------------------------+--------------------+-----------------+
-|  13         |  ++        |  Postfix Increment               |  before            |  Left to Right  | postfixOperators()
-|  13         |  --        |  Postfix Decrement               |  before            |  Left to Right  | postfixOperators()
-+-------------+------------+----------------------------------+--------------------+-----------------+
-|  12         |  !         |  Logical NOT                     |  after             |  Right to Left  | prefixOperators()
-|  12         |  -         |  Unary Negation                  |  after             |  Right to Left  | prefixOperators()
-+-------------+------------+----------------------------------+--------------------+-----------------+
-|  11         |  **        |  Exponentiation                  |  dual              |  Right to Left  | mathOperators(0)
-+-------------+------------+----------------------------------+--------------------+-----------------+
-|  10         |  *         |  Multiplication                  |  dual              |  Left to Right  | mathOperators(1)
-|  10         |  /         |  Division                        |  dual              |  Left to Right  | mathOperators(1)
-|  10         |  %         |  Modulus                         |  dual              |  Left to Right  | mathOperators(1)
-+-------------+------------+----------------------------------+--------------------+-----------------+
-|   9         |  +         |  Addition                        |  dual              |  Left to Right  | mathOperators(2)
-|   9         |  -         |  Subtraction                     |  dual              |  Left to Right  | mathOperators(2)
-+-------------+------------+----------------------------------+--------------------+-----------------+
-|   7         |  <         |  Less Than                       |  dual              |  Left to Right  | comparisonOperators()
-|   7         |  <=        |  Less Than or Equal              |  dual              |  Left to Right  | comparisonOperators()
-|   7         |  >         |  Greater Than                    |  dual              |  Left to Right  | comparisonOperators()
-|   7         |  >=        |  Greater Than or Equal           |  dual              |  Left to Right  | comparisonOperators()
-+-------------+------------+----------------------------------+--------------------+-----------------+
-|   6         |  ==        |  Equality                        |  dual              |  Left to Right  | assign()
-|   6         |  !=        |  Inequality                      |  dual              |  Left to Right  | assign()
-+-------------+------------+----------------------------------+--------------------+-----------------+
-|   4         |  &&        |  Logical AND                     |  dual              |  Left to Right  | logicOperators()
-|   4         |  ^^        |  Logical XOR                     |  dual              |  Left to Right  | logicOperators()
-|   4         |  ||        |  Logical OR                      |  dual              |  Left to Right  | logicOperators()
-+-------------+------------+----------------------------------+--------------------+-----------------+
-|   3         |  =         |  Assignment                      |  dual              |  Right to Left  | opAssign()
-|   3         |  +=        |  Add and Assign                  |  dual              |  Right to Left  | opAssign()
-|   3         |  -=        |  Subtract and Assign             |  dual              |  Right to Left  | opAssign()
-|   3         |  **=       |  Exponentiate and Assign         |  dual              |  Right to Left  | opAssign()
-|   3         |  *=        |  Multiply and Assign             |  dual              |  Right to Left  | opAssign()
-|   3         |  /=        |  Divide and Assign               |  dual              |  Right to Left  | opAssign()
-|   3         |  %=        |  Modulo and Assign               |  dual              |  Right to Left  | opAssign()
-+-------------+------------+----------------------------------+--------------------+-----------------+
-|   2         |  ,         |  Comma                           |  none              |  Left to Right  |
-+-------------+------------+----------------------------------+--------------------+-----------------+
-|   1         |  ;         |  Statement End                   |  before            |  Left to Right  |
-+-------------+------------+----------------------------------+--------------------+-----------------+
-
-Possible Operators in Pivot:
-+-------------+------------+----------------------------------+--------------------+-----------------+
-| Precedence  |  Operator  |  Description                     |  Operands          |  Associativity  |
-+-------------+------------+----------------------------------+--------------------+-----------------+
-|  12         |  ~         |  Bitwise NOT                     |  after             |  Right to Left  |
-+-------------+------------+----------------------------------+--------------------+-----------------+
-|   8         |  <<        |  Bitwise Left Shift              |  dual              |  Left to Right  |
-|   8         |  >>        |  Bitwise Right Shift             |  dual              |  Left to Right  |
-+-------------+------------+----------------------------------+--------------------+-----------------+
-|   5         |  and       |  Logical AND                     |  dual              |  Left to Right  |
-|   5         |  xor       |  Logical XOR                     |  dual              |  Left to Right  |
-|   5         |  or        |  Logical OR                      |  dual              |  Left to Right  |
-+-------------+------------+----------------------------------+--------------------+-----------------+
-|   5         |  &         |  Bitwise AND                     |  dual              |  Left to Right  |
-|   5         |  ^         |  Bitwise XOR                     |  dual              |  Left to Right  |
-|   5         |  |         |  Bitwise OR                      |  dual              |  Left to Right  |
-+-------------+------------+----------------------------------+--------------------+-----------------+
-|   3         |  <<=       |  Bitwise Left Shift and Assign   |  dual              |  Right to Left  |
-|   3         |  >>=       |  Bitwise Right Shift and Assign  |  dual              |  Right to Left  |
-|   3         |  &=        |  Bitwise AND and Assign          |  dual              |  Right to Left  |
-|   3         |  ^=        |  Bitwise XOR and Assign          |  dual              |  Right to Left  |
-|   3         |  |=        |  Bitwise OR and Assign           |  dual              |  Right to Left  |
-+-------------+------------+----------------------------------+--------------------+-----------------+

package.json

@@ -1,40 +0,0 @@
-{
-  "name": "pivot-lang",
-  "version": "0.1.0",
-  "description": "Pivot is a new programming language built on JavaScript",
-  "main": "./bin/pivot.js",
-  "directories": {
-    "test": "test"
-  },
-  "scripts": {
-    "test": "mocha",
-    "build": "npm run test; jsdoc src/ -r -d ./jsdoc/ -R ./readme.md"
-  },
-  "repository": {
-    "type": "git",
-    "url": "git+https://github.com/ElementG9/Pivot.git"
-  },
-  "keywords": [
-    "pivot",
-    "javascript",
-    "programming",
-    "language"
-  ],
-  "author": "Garen Tyler <garentyler@gmail.com>",
-  "license": "MIT",
-  "bugs": {
-    "url": "https://github.com/ElementG9/Pivot/issues"
-  },
-  "homepage": "https://github.com/ElementG9/Pivot#readme",
-  "bin": {
-    "pivot": "./bin/pivot.js"
-  },
-  "preferGlobal": true,
-  "dependencies": {
-    "fs": "0.0.1-security",
-    "readline-sync": "^1.4.10"
-  },
-  "devDependencies": {
-    "mocha": "^6.2.1"
-  }
-}

readme.md

@@ -0,0 +1,15 @@
+# Pivot
+## Version
+Pivot is a new programming language built with Rust by Garen Tyler. Pivot is currently in the alpha stage of development.
+
+---
+
+### Development
+* Download the repo with `git clone https://github.com/garentyler/pivot`.
+* Make sure you have [Rust](https://www.rust-lang.org/), [NodeJS](https://nodejs.org/en/), and [NPM](https://www.npmjs.com/) installed.
+* Run `npm install` to install the JavaScript dependencies.
+* Scripts:
+    * `npm run clean` to delete dependencies and output files.
+    * `npm run build` to build the project.
+    * `npm run run` to run the example.
+    * `npm run test` to run the tests.

src/code.js

@@ -1,38 +0,0 @@
-/**
- * @module code
- * @file Runs the code / transpiles the code to JavaScript
- * @author Garen Tyler <garentyler@gmail.com>
- */
-
- /**
-  * @function translate
-  * @desc Translates the code to JS, given an AST
-  * @param {Token[]} ast The ast.
-  * @returns {String} The JS code.
-  * @public
-  */
-function translate(ast, data) {
-  let out = '';
-  for (let i = 0; i < ast.length; i++) {
-    if (ast[i].type == 'operator' && ast[i].subtype == 'function call') {
-      let temp = '';
-      if (!(Object.keys(data).indexOf(ast[i].operands[0].value) > -1))
-        throw new ReferenceError(`Undefined function ${ast[i].operands[0].value}`);
-      else temp += data[ast[i].operands[0].value];
-      temp += '(';
-      for (let j = 0; j < ast[i].operands[1].tokens.length; j++) {
-        if (j != 0)
-          temp += ', ';
-        if (ast[i].operands[1].tokens[j].type == 'string')
-          temp += `"${ast[i].operands[1].tokens[j].value}"`;
-      }
-      temp += ');'
-      out += temp;
-    }
-  }
-  return out;
-}
-
-module.exports = {
-  translate
-};

src/codegen.rs

@@ -0,0 +1,5 @@
+use crate::{parse::AstNode, InterpreterError};
+
+pub fn codegen(_ast: &AstNode) -> Result<String, InterpreterError> {
+    Ok(String::new())
+}

src/lib.rs

@@ -0,0 +1,32 @@
+pub mod codegen;
+pub mod parse;
+pub mod tokenize;
+
+#[derive(Debug)]
+pub enum InterpreterError {
+    /// Error parsing source
+    ParseError(String),
+    /// Unexpected token
+    UnexpectedToken,
+    /// Mismatched types
+    MismatchedTypes,
+    /// Type error
+    TypeError,
+    /// Unexpected EOF
+    UnexpectedEOF,
+    /// Expected value
+    ExpectedValue,
+    /// Unimplemented
+    Unimplemented,
+}
+impl<T> From<Option<T>> for InterpreterError {
+    fn from(_value: Option<T>) -> InterpreterError {
+        InterpreterError::ExpectedValue
+    }
+}
+
+pub fn compile(source: &str) -> Result<String, InterpreterError> {
+    let tokens = tokenize::tokenize(source);
+    let ast = parse::parse(&tokens?);
+    codegen::codegen(&ast?)
+}

src/main.rs

@@ -0,0 +1,10 @@
+use std::{fs::File, io::Write};
+
+fn main() -> std::io::Result<()> {
+    // Read the source from a file.
+    let source = std::fs::read_to_string("test.pvt").unwrap();
+    let code = pivot::compile(&source).unwrap();
+    // Write it to a file.
+    File::create("out.bf")?.write_all(code.as_bytes())?;
+    Ok(())
+}

src/parse.rs

@@ -0,0 +1,291 @@
+use crate::{tokenize::Token, InterpreterError};
+
+#[derive(Clone, PartialEq, Debug)]
+pub enum AstPrimitive {
+    Integer(i64),
+    Float(f64),
+    String(String),
+    Boolean(bool),
+    Null,
+}
+#[allow(clippy::derive_hash_xor_eq)]
+impl std::hash::Hash for AstPrimitive {
+    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
+        match self {
+            AstPrimitive::Float(f) => format!("{}", f).hash(state),
+            _ => self.hash(state),
+        }
+    }
+}
+impl std::fmt::Display for AstPrimitive {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> {
+        use AstPrimitive::*;
+        match self {
+            Integer(n) => write!(f, "{}", n),
+            Float(n) => write!(f, "{}", n),
+            String(s) => write!(f, "{}", s),
+            Boolean(b) => write!(f, "{}", b),
+            Null => write!(f, "null"),
+        }
+    }
+}
+
+#[derive(Clone, PartialEq, Debug)]
+pub enum AstNode {
+    Primitive(AstPrimitive),
+    Identifier(String),
+    Negate {
+        body: Box<AstNode>,
+    },
+    Add {
+        left: Box<AstNode>,
+        right: Box<AstNode>,
+    },
+    Subtract {
+        left: Box<AstNode>,
+        right: Box<AstNode>,
+    },
+    Multiply {
+        left: Box<AstNode>,
+        right: Box<AstNode>,
+    },
+    Divide {
+        left: Box<AstNode>,
+        right: Box<AstNode>,
+    },
+    Declare {
+        identifier: String,
+    },
+    Assign {
+        left: Box<AstNode>,
+        right: Box<AstNode>,
+    },
+    FunctionCall {
+        identifier: String,
+        arguments: Vec<AstNode>,
+    },
+    Program {
+        statements: Vec<AstNode>,
+    },
+    Null,
+}
+
+pub fn parse(tokens: &[Token]) -> Result<AstNode, InterpreterError> {
+    fn parse_function_call(tokens: &[Token]) -> Result<(AstNode, usize), InterpreterError> {
+        let mut index = 0;
+        let identifier;
+        if let Token::Identifier(id) = tokens[index].clone() {
+            identifier = id;
+            index += 1;
+        } else {
+            return Err(InterpreterError::UnexpectedToken);
+        }
+        if !matches!(tokens[index], Token::Parenthesis { closing: false }) {
+            return Ok((AstNode::Identifier(identifier), 1));
+        } else {
+            index += 1;
+        }
+        // Check if it closes right away.
+        if matches!(tokens[index], Token::Parenthesis { closing: true }) {
+            index += 1;
+            return Ok((
+                AstNode::FunctionCall {
+                    identifier,
+                    arguments: vec![],
+                },
+                index,
+            ));
+        }
+
+        let mut arguments = vec![];
+        if let Ok((argument, tokens_consumed)) = parse_expression(&tokens[index..]) {
+            arguments.push(argument);
+            index += tokens_consumed;
+        }
+        while index + 2 < tokens.len() {
+            if tokens[index] != Token::Comma {
+                break;
+            }
+            index += 1;
+            if let Ok((argument, tokens_consumed)) = parse_expression(&tokens[index..]) {
+                arguments.push(argument);
+                index += tokens_consumed;
+            } else {
+                break;
+            }
+        }
+        if matches!(tokens[index], Token::Parenthesis { closing: true }) {
+            index += 1;
+            Ok((
+                AstNode::FunctionCall {
+                    identifier,
+                    arguments,
+                },
+                index,
+            ))
+        } else {
+            Err(InterpreterError::UnexpectedToken)
+        }
+    }
+    fn parse_primary_expression(tokens: &[Token]) -> Result<(AstNode, usize), InterpreterError> {
+        if tokens.is_empty() {
+            Err(InterpreterError::UnexpectedEOF)
+        } else if let Token::Integer(n) = &tokens[0] {
+            Ok((AstNode::Primitive(AstPrimitive::Integer(*n)), 1))
+        } else if let Token::Float(n) = &tokens[0] {
+            Ok((AstNode::Primitive(AstPrimitive::Float(*n)), 1))
+        } else if let Token::Boolean(n) = &tokens[0] {
+            Ok((AstNode::Primitive(AstPrimitive::Boolean(*n)), 1))
+        } else if let Token::String(s) = &tokens[0] {
+            Ok((AstNode::Primitive(AstPrimitive::String(s.clone())), 1))
+        } else if let Token::Identifier(_) = &tokens[0] {
+            parse_function_call(tokens)
+        } else if tokens[0] == Token::Keyword("let".to_owned()) {
+            if tokens.len() < 2 {
+                Err(InterpreterError::UnexpectedEOF)
+            } else if let Token::Identifier(s) = &tokens[1] {
+                Ok((
+                    AstNode::Declare {
+                        identifier: s.clone(),
+                    },
+                    2,
+                ))
+            } else {
+                Err(InterpreterError::UnexpectedToken)
+            }
+        } else {
+            Err(InterpreterError::UnexpectedToken)
+        }
+    }
+    fn parse_grouped_expression(tokens: &[Token]) -> Result<(AstNode, usize), InterpreterError> {
+        let mut index = 0;
+        // '('
+        if !matches!(tokens[index], Token::Parenthesis { closing: false }) {
+            return parse_primary_expression(tokens);
+        } else {
+            index += 1;
+        }
+        // expression of any kind
+        let (value, tokens_consumed) = parse_expression(&tokens[index..])?;
+        index += tokens_consumed;
+        // ')'
+        if !matches!(tokens[index], Token::Parenthesis { closing: true }) {
+            return Err(InterpreterError::ParseError(
+                "No closing parenthesis".to_owned(),
+            ));
+        } else {
+            index += 1;
+        }
+        Ok((value, index))
+    }
+    fn parse_unary_expression(tokens: &[Token]) -> Result<(AstNode, usize), InterpreterError> {
+        let mut index = 0;
+        if tokens[index] != Token::Minus && tokens[index] != Token::Bang {
+            parse_grouped_expression(&tokens[index..])
+        } else {
+            let operation = tokens[index].clone();
+            index += 1;
+            let (body, tokens_consumed) = parse_unary_expression(&tokens[index..])?;
+            index += tokens_consumed;
+            Ok((
+                match operation {
+                    Token::Minus => AstNode::Negate {
+                        body: Box::new(body),
+                    },
+                    Token::Bang => AstNode::Negate {
+                        body: Box::new(body),
+                    },
+                    _ => return Err(InterpreterError::ParseError("Impossible".to_owned())),
+                },
+                index,
+            ))
+        }
+    }
+    fn parse_multiplication_expression(
+        tokens: &[Token],
+    ) -> Result<(AstNode, usize), InterpreterError> {
+        let mut index = 0;
+        let (mut value, tokens_consumed) = parse_unary_expression(&tokens[index..])?;
+        index += tokens_consumed;
+        while index < tokens.len() {
+            if tokens[index] != Token::Star && tokens[index] != Token::Slash {
+                break;
+            }
+            let operation = tokens[index].clone();
+            index += 1;
+            let (right, tokens_consumed) = parse_unary_expression(&tokens[index..])?;
+            index += tokens_consumed;
+            value = match operation {
+                Token::Star => AstNode::Multiply {
+                    left: Box::new(value),
+                    right: Box::new(right),
+                },
+                Token::Slash => AstNode::Divide {
+                    left: Box::new(value),
+                    right: Box::new(right),
+                },
+                _ => return Err(InterpreterError::ParseError("Impossible".to_owned())),
+            };
+        }
+        Ok((value, index))
+    }
+    fn parse_addition_expression(tokens: &[Token]) -> Result<(AstNode, usize), InterpreterError> {
+        let mut index = 0;
+        let (mut value, tokens_consumed) = parse_multiplication_expression(&tokens[index..])?;
+        index += tokens_consumed;
+        while index < tokens.len() {
+            if tokens[index] != Token::Plus && tokens[index] != Token::Minus {
+                break;
+            }
+            let operation = tokens[index].clone();
+            index += 1;
+            let (right, tokens_consumed) = parse_multiplication_expression(&tokens[index..])?;
+            index += tokens_consumed;
+            value = match operation {
+                Token::Plus => AstNode::Add {
+                    left: Box::new(value),
+                    right: Box::new(right),
+                },
+                Token::Minus => AstNode::Subtract {
+                    left: Box::new(value),
+                    right: Box::new(right),
+                },
+                _ => return Err(InterpreterError::ParseError("Impossible".to_owned())),
+            };
+        }
+        Ok((value, index))
+    }
+    fn parse_assign_expression(tokens: &[Token]) -> Result<(AstNode, usize), InterpreterError> {
+        let mut index = 0;
+        let (identifier, tokens_consumed) = parse_addition_expression(&tokens[index..])?;
+        index += tokens_consumed;
+        if index < tokens.len() && tokens[index] == Token::Equals {
+            index += 1;
+        } else {
+            return Ok((identifier, index));
+        }
+        let (value, tokens_consumed) = parse_addition_expression(&tokens[index..])?;
+        index += tokens_consumed;
+        Ok((
+            AstNode::Assign {
+                left: Box::new(identifier),
+                right: Box::new(value),
+            },
+            index,
+        ))
+    }
+    fn parse_expression(tokens: &[Token]) -> Result<(AstNode, usize), InterpreterError> {
+        parse_assign_expression(tokens)
+    }
+    let mut statements = vec![];
+    let mut index = 0;
+    loop {
+        if index >= tokens.len() {
+            break;
+        }
+        let (statement, tokens_consumed) = parse_expression(&tokens[index..])?;
+        statements.push(statement);
+        index += tokens_consumed;
+    }
+    Ok(AstNode::Program { statements })
+}

src/parser.js

@@ -1,421 +0,0 @@
-/**
- * @module parser
- * @file Manages the parsing phase of Pivot.
- * @author Garen Tyler <garentyler@gmail.com>
- * @requires module:types
- */
-const Group = require('./types.js').Group;
-const Operator = require('./types.js').Operator;
-
-/**
- * @function parse
- * @desc Takes in an array of tokens, and outputs an AST.
- * @param {Token[]} tokens The input tokens.
- * @returns {Token[]} The tokens structured in an AST.
- * @public
- */
-function parse(tokens) {
-  // Create our output array.
-  let ast = tokens;
-
-  // Add indexes and levels.
-  ast = addIndexes(ast);
-  ast = addLevels(ast);
-
-  // Start grouping by precedence.
-  // Precedence 16.
-  ast = grouping(ast);
-  // Precedence 15.
-  ast = memberAccess(ast);
-  ast = computedMemberAccess(ast);
-  ast = functionCall(ast);
-  // Precedence 14.
-  ast = keywords(ast);
-  ast = functionCreation(ast);
-  // Precedence 13.
-  ast = postfixOperators(ast);
-  // Precedence 12.
-  ast = prefixOperators(ast);
-  // Precedence 11.
-  ast = mathOperators(ast, 0); // Level 0 math operators: **.
-  // Precedence 10.
-  ast = mathOperators(ast, 1); // Level 1 math operators: *, /, %.
-  // Precedence 9.
-  ast = mathOperators(ast, 2); // Level 2 math operators: +, -.
-  // Precedence 7.
-  // ast = comparisonOperators(ast);
-  // Precedence 6.
-  // ast = assign(ast);
-  // Precedence 4.
-  // ast = logicOperators(ast);
-  // Precedence 3.
-  // ast = opAssign(ast);
-  return ast;
-}
-
-/**
- * @function addIndexes
- * @desc Adds basic indexes to the tokens.
- * @param {Token[]} tokens The tokens.
- * @returns {Token[]} The tokens with indexes.
- * @private
- */
-function addIndexes(tokens) {
-  return tokens.map((t, i) => {
-    t.index = i;
-    return t;
-  });
-}
-
-/**
- * @function addLevels
- * @desc Adds basic levels to the tokens. The levels are dependent on the delimiters.
- * @param {Token[]} tokens The tokens.
- * @returns {Token[]} The tokens with levels.
- * @private
- */
-function addLevels(tokens) {
-  let level = 0;
-  tokens = tokens.map((t, i) => {
-    if (t.type == 'delimiter' && t.subtype == 'left')
-      level++;
-    t.level = level;
-    if (t.type == 'delimiter' && t.subtype == 'right')
-      level--;
-    return t;
-  });
-  if (level > 0)
-    throw new SyntaxError('Missing closing delimiter');
-  else if (level < 0)
-    throw new SyntaxError('Missing opening delimiter');
-  else
-    return tokens;
-}
-
-/**
- * @function getDeepestLevel
- * @desc Finds the deepest level of the ast.
- * @param {Token[]} tokens The tokens.
- * @returns {Number} The deepest level.
- * @private
- */
-function getDeepestLevel(tokens) {
-  return tokens.reduce((deepest, t) => {
-    return t.level > deepest ? t.level : deepest;
-  }, 0);
-}
-
-/**
- * @function grouping
- * @desc Combine groups of tokens by delimiter.
- * @param {Token[]} tokens The tokens.
- * @returns {Token[]} The grouped tokens, or the basic ast.
- * @private
- */
-function grouping(tokens) {
-  // Get the deepest level.
-  let deepestLevel = getDeepestLevel(tokens);
-  let groupBuffer;
-  let opening;
-  // Group the deepest levels first.
-  for (let currentLevel = deepestLevel; currentLevel > 0; currentLevel--) {
-    groupBuffer = []; // Overwrite groupBuffer.
-    opening = null; // Overwrite opening.
-    for (let i = 0; i < tokens.length; i++) {
-      if (tokens[i].level == currentLevel) {
-        if (groupBuffer.length == 0)
-          opening = tokens[i];
-        groupBuffer.push(tokens[i]);
-        if (typeof tokens[i + 1] == 'undefined' || (tokens[i].type == 'delimiter' && tokens[i].subtype == 'right' && tokens[i].value == opening.value)) { // The end of the tokens.
-          let g = new Group(groupBuffer[0].value, groupBuffer);
-          g.index = g.tokens[0].index;
-          g.level = g.tokens[0].level - 1; // -1 because the group is on the level below.
-          let length = g.tokens.length; // Keep track of how many tokens there are before removing the delimiters.
-          g.tokens = g.tokens.splice(1, g.tokens.length - 2); // Remove the delimiters in g.tokens.
-          i -= length - 1; // Reset the counter.
-          tokens.splice(i, length, g); // Replace the tokens with the new group.
-          // Reset groupBuffer and opening.
-          groupBuffer = [];
-          opening = null;
-        }
-      }
-    }
-  }
-  return tokens;
-}
-
-/**
- * @function memberAccess
- * @desc Combine groups of tokens by member access.
- * @param {Token[]} tokens The tokens.
- * @returns {Token[]} The ast with grouped member access.
- * @private
- */
-function memberAccess(ast) {
-  for (let i = 0; i < ast.length; i++) {
-    if (ast[i].type == 'group')
-      ast[i].tokens = memberAccess(ast[i].tokens); // Recursively order the groups.
-    else if (ast[i].type == 'operator' && ast[i].value == '.') { // Member access operator.
-      if (typeof ast[i - 1] == 'undefined' || typeof ast[i + 1] == 'undefined')
-        throw new SyntaxError('Operator requires two operands.');
-      let op = new Operator(ast[i].subtype, ast[i].value, [ast[i - 1], ast[i + 1]]);
-      op.index = ast[i - 1].index;
-      op.level = ast[i].level;
-      ast.splice(i - 1, 3, op);
-      i--; // Removed 3 tokens, put in 1, skip 1 token. Reduce the counter by 1.
-    }
-  }
-  return ast;
-}
-
-/**
- * @function computedMemberAccess
- * @desc Combine groups of tokens by computed member access.
- * @param {Token[]} tokens The tokens.
- * @returns {Token[]} The ast with grouped computed member access.
- * @private
- */
-function computedMemberAccess(ast) {
-  // Computed member access is Variable, Bracket Group.
-  for (let i = 0; i < ast.length; i++) {
-    if (ast[i].type == 'group')
-      ast[i].tokens = computedMemberAccess(ast[i].tokens); // Recursively order the groups.
-    else if (ast[i].type == 'name' && ast[i].subtype == 'variable') { // Member access operator.
-      if (typeof ast[i + 1] == 'undefined')
-        continue; // Nothing after the variable; skip this loop.
-      if (ast[i + 1].type == 'group' && ast[i + 1].subtype == 'bracket') {
-        ast[i + 1].tokens = computedMemberAccess(ast[i + 1].tokens); // Order the group that we care about before we mess with it.
-        let op = new Operator('n/a', 'member access', [ast[i], ast[i + 1]]);
-        op.index = ast[i].index;
-        op.level = ast[i].level;
-        ast.splice(i, 2, op);
-        // Removed 2 tokens, put in 1, skip 1 token. Don't reduce the counter.
-      } else continue; // Not what we need.
-    }
-  }
-  return ast;
-}
-
-/**
- * @function functionCall
- * @desc Combine groups of tokens by function calls.
- * @param {Token[]} tokens The tokens.
- * @returns {Token[]} The ast with grouped function calls.
- * @private
- */
-function functionCall(ast) {
-  // Function call is Variable, Parenthesis Group.
-  for (let i = 0; i < ast.length; i++) {
-    if (ast[i].type == 'group')
-      ast[i].tokens = functionCall(ast[i].tokens); // Recursively order the groups.
-    else if ((ast[i].type == 'name' && ast[i].subtype == 'variable') || // Normal function call
-      (ast[i].type == 'operator' && (ast[i].value == '.' || ast[i].value == 'member access'))) { // Function call in member access. Example: console.log()
-      if (typeof ast[i + 1] == 'undefined')
-        continue; // Nothing after the variable; skip this loop.
-      if (ast[i + 1].type == 'group' && ast[i + 1].subtype == 'parenthesis') {
-        ast[i + 1].tokens = functionCall(ast[i + 1].tokens); // Order the group that we care about before we mess with it.
-        let op = new Operator('function call', ast[i].value, [ast[i], ast[i + 1]]);
-        op.index = ast[i].index;
-        op.level = ast[i].level;
-        ast.splice(i, 2, op);
-        // Removed 2 tokens, put in 1, skip 1 token. Don't reduce the counter.
-      } else continue; // Not what we need.
-    }
-  }
-  return ast;
-}
-
-/**
- * @function keywords
- * @desc Combine groups of tokens by keywords.
- * @param {Token[]} tokens The tokens.
- * @returns {Token[]} The ast with grouped keywords.
- * @private
- */
-function keywords(ast) {
-  for (let i = ast.length - 1; i >= 0; i--) { // Keywords are rtl associative, so loop backwards.
-    if (ast[i].type == 'group')
-      ast[i].tokens = keywords(ast[i].tokens); // Recursively order the groups.
-    else if (ast[i].type == 'name' && ast[i].subtype == 'keyword') {
-      if (typeof ast[i + 1] == 'undefined')
-        throw new SyntaxError('Keyword requires one operand after it.');
-      let key = new Operator('keyword', ast[i].value, [ast[i + 1]]);
-      key.level = ast[i].level;
-      key.index = ast[i].index;
-      ast.splice(i, 2, key);
-      // Looping backwards and didn't remove any items before the current one. Don't reduce the counter.
-    }
-  }
-  return ast;
-}
-
-/**
- * @function functionCreation
- * @desc Combine groups of tokens by function creation.
- * @param {Token[]} tokens The tokens.
- * @returns {Token[]} The ast with grouped function creation.
- * @private
- */
-function functionCreation(ast) {
-  // Function call is Parenthesis Group, Brace Group.
-  for (let i = 0; i < ast.length; i++) {
-    if (ast[i].type == 'group')
-      ast[i].tokens = functionCreation(ast[i].tokens); // Recursively order the groups.
-    if (typeof ast[i + 1] == 'undefined')
-      continue; // Skip this loop.
-    if ((ast[i].type == 'group' && ast[i].subtype == 'parenthesis') && (ast[i + 1].type == 'group' && ast[i + 1].subtype == 'brace')) {
-      // Parenthesis group followed by brace group.
-      ast[i + 1].tokens = functionCreation(ast[i + 1].tokens); // Order the other group before we do anything.
-      let op = new Operator('function creation', 'n/a', [ast[i], ast[i + 1]]);
-      op.index = ast[i].index;
-      op.level = ast[i].level;
-      ast.splice(i, 2, op);
-    }
-  }
-  return ast;
-}
-
-/**
- * @function postfixOperators
- * @desc Recursively structures the postfix operators.
- * @param {Token[]} ast The ast.
- * @returns {Token[]} The ast with structured postfix operators.
- * @private
- */
-function postfixOperators(ast) {
-  for (let i = 0; i < ast.length; i++) {
-    // Take care of the tokens in the groups.
-    if (ast[i].type == 'group') {
-      if (ast[i].tokens.length > 0) {
-        ast[i].tokens = postfixOperators(ast[i].tokens);
-      }
-    } else if (ast[i].type == 'operator') {
-      if (typeof ast[i].operands != 'undefined') {
-        ast[i].operands = postfixOperators(ast[i].operands);
-      }
-    }
-    if (ast[i].type == 'operator' && ast[i].subtype == 'postfix') { // The operand is on the left.
-      if (typeof ast[i - 1] == 'undefined')
-        throw new SyntaxError('Postfix operator requires one operand before it.');
-      let op = new Operator(ast[i].subtype, ast[i].value, [ast[i - 1]]);
-      op.index = ast[i].index;
-      op.level = ast[i].level;
-      ast.splice(i - 1, 2, op);
-      // Removing 2 tokens, adding 1, skip 1 token. Don't reduce the counter.
-    }
-  }
-  return ast;
-}
-
-/**
- * @function prefixOperators
- * @desc Recursively structures the prefix operators.
- * @param {Token[]} ast The ast.
- * @returns {Token[]} The ast with structured prefix operators.
- * @private
- */
-function prefixOperators(ast) {
-  for (let i = ast.length - 1; i >= 0; i--) { // Prefix operators are rtl associative, so loop backwards.
-    // Take care of the tokens in the groups.
-    if (ast[i].type == 'group') {
-      if (ast[i].tokens.length > 0) {
-        ast[i].tokens = prefixOperators(ast[i].tokens);
-      }
-    } else if (ast[i].type == 'operator') {
-      if (typeof ast[i].operands != 'undefined') {
-        ast[i].operands = prefixOperators(ast[i].operands);
-      }
-    }
-    if (ast[i].type == 'operator' && ast[i].subtype == 'prefix') { // The operand is on the right.
-      if (typeof ast[i + 1] == 'undefined')
-        throw new SyntaxError('Prefix operator requires one operand after it.');
-      let op = new Operator(ast[i].subtype, ast[i].value, [ast[i + 1]]);
-      op.index = ast[i].index;
-      op.level = ast[i].level;
-      ast.splice(i, 2, op);
-      // Removing 2 tokens, adding 1, skip 1 token. Don't reduce the counter.
-    }
-  }
-  return ast;
-}
-
-/**
- * @function mathOperators
- * @desc Recursively structures the math operators.
- * @param {Token[]} ast The ast.
- * @param {Token[]} level The level of math to do. (Order of operations)
- * @returns {Token[]} The ast with structured math operators.
- * @private
- */
-function mathOperators(ast, level) {
-  if (level == 0) { // Level 0 operators: **
-    for (let i = ast.length - 1; i >= 0; i--) { // Exponentiation is rtl associative, so loop backwards.
-      // Take care of the tokens in the groups.
-      if (ast[i].type == 'group') {
-        if (ast[i].tokens.length > 0) {
-          ast[i].tokens = mathOperators(ast[i].tokens, level);
-        }
-      } else if (ast[i].type == 'operator') {
-        if (typeof ast[i].operands != 'undefined') {
-          ast[i].operands = mathOperators(ast[i].operands, level);
-        }
-      }
-      if (ast[i].type == 'operator' && ast[i].value == '**') {
-        if (typeof ast[i - 1] == 'undefined' || typeof ast[i + 1] == 'undefined')
-          throw new SyntaxError('Dual operator requires two operands.');
-        let op = new Operator('dual', ast[i].value, [ast[i - 1], ast[i + 1]]);
-        op.index = ast[i].index;
-        op.level = ast[i].level;
-        ast.splice(i - 1, 3, op);
-        i--;
-      }
-    }
-  } else {
-    for (let i = 0; i < ast.length; i++) { // All other math operators are ltr associative.
-      // Take care of the tokens in the groups.
-      if (ast[i].type == 'group') {
-        if (ast[i].tokens.length > 0) {
-          ast[i].tokens = mathOperators(ast[i].tokens, level);
-        }
-      } else if (ast[i].type == 'operator') {
-        if (typeof ast[i].operands != 'undefined') {
-          ast[i].operands = mathOperators(ast[i].operands, level);
-        }
-      }
-      if (level == 1) {
-        if (ast[i].type == 'operator' && (ast[i].value == '*' || ast[i].value == '/' || ast[i].value == '%')) {
-          if (typeof ast[i - 1] == 'undefined' || typeof ast[i + 1] == 'undefined')
-            throw new SyntaxError('Dual operator requires two operands.');
-          let op = new Operator('dual', ast[i].value, [ast[i - 1], ast[i + 1]]);
-          op.index = ast[i].index;
-          op.level = ast[i].level;
-          ast.splice(i - 1, 3, op);
-          i--;
-        }
-      } else if (level == 2) {
-        if (ast[i].type == 'operator' && (ast[i].value == '+' || ast[i].value == '-')) {
-          if (typeof ast[i - 1] == 'undefined' || typeof ast[i + 1] == 'undefined')
-            throw new SyntaxError('Dual operator requires two operands.');
-          let op = new Operator('dual', ast[i].value, [ast[i - 1], ast[i + 1]]);
-          op.index = ast[i].index;
-          op.level = ast[i].level;
-          ast.splice(i - 1, 3, op);
-          i--;
-        }
-      }
-    }
-  }
-  return ast;
-}
-
-module.exports = {
-  parse,
-  util: {
-    addIndexes,
-    addLevels,
-    getDeepestLevel,
-    grouping,
-  }
-};
-
-// require('fs').writeFileSync('ast.json', JSON.stringify(parse(tokenizer.tokenize('let x = (5 + (6 * 2)) - 7;')), null, 2), () => {});

src/tokenize.rs

@@ -0,0 +1,164 @@
+use crate::InterpreterError;
+
+#[derive(Clone, PartialEq, Debug)]
+pub enum Token {
+    Integer(i64),
+    Float(f64),
+    String(String),
+    Boolean(bool),
+    Identifier(String),
+    Keyword(String),
+    Comma,
+    Plus,
+    Minus,
+    Star,
+    Slash,
+    Bang,
+    Equals,
+    Semicolon,
+    Quote(char),
+    Parenthesis { closing: bool },
+    Whitespace(String),
+    Unknown,
+}
+
+pub fn tokenize(source: &str) -> Result<Vec<Token>, InterpreterError> {
+    fn tokenize_number(chars: &[char]) -> Result<(Token, usize), ()> {
+        let mut current_value = String::new();
+        let mut chars_consumed = 0;
+        for c in chars {
+            if !c.is_digit(10) && *c != '.' {
+                break;
+            }
+            current_value.push(*c);
+            chars_consumed += 1;
+        }
+        if chars_consumed == 0 {
+            Err(())
+        } else if current_value.contains('.') {
+            Ok((
+                Token::Float(current_value.parse::<f64>().unwrap()),
+                chars_consumed,
+            ))
+        } else {
+            Ok((
+                Token::Integer(current_value.parse::<i64>().unwrap()),
+                chars_consumed,
+            ))
+        }
+    }
+    fn tokenize_identifier(chars: &[char]) -> Result<(Token, usize), ()> {
+        let mut current_value = String::new();
+        let mut chars_consumed = 0;
+        if chars[chars_consumed].is_alphabetic() {
+            current_value.push(chars[chars_consumed]);
+        } else {
+            return Err(());
+        }
+        chars_consumed += 1;
+        while chars_consumed < chars.len()
+            && (chars[chars_consumed].is_alphanumeric() || chars[chars_consumed] == '_')
+        {
+            current_value.push(chars[chars_consumed]);
+            chars_consumed += 1;
+        }
+        match &current_value[..] {
+            "true" => Ok((Token::Boolean(true), 4)),
+            "false" => Ok((Token::Boolean(false), 5)),
+            "let" => Ok((Token::Keyword(current_value), chars_consumed)),
+            _ => Ok((Token::Identifier(current_value), chars_consumed)),
+        }
+    }
+    fn tokenize_string(chars: &[char]) -> Result<(Token, usize), ()> {
+        let mut current_value = String::new();
+        let mut chars_consumed = 0;
+        fn is_quote(c: char) -> bool {
+            matches!(c, '\'' | '"' | '`')
+        }
+        let start_quote = if is_quote(chars[chars_consumed]) {
+            chars[chars_consumed]
+        } else {
+            return Err(());
+        };
+        chars_consumed += 1;
+        while chars_consumed < chars.len() && chars[chars_consumed] != start_quote {
+            current_value.push(chars[chars_consumed]);
+            chars_consumed += 1;
+        }
+        chars_consumed += 1;
+        Ok((Token::String(current_value), chars_consumed))
+    }
+    fn tokenize_whitespace(chars: &[char]) -> Result<(Token, usize), ()> {
+        let mut current_value = String::new();
+        let mut chars_consumed = 0;
+        for c in chars {
+            if !c.is_whitespace() {
+                break;
+            }
+            chars_consumed += 1;
+            current_value.push(*c);
+        }
+        if chars_consumed == 0 {
+            Err(())
+        } else {
+            Ok((Token::Whitespace(current_value), chars_consumed))
+        }
+    }
+    fn tokenize_operator(chars: &[char]) -> Result<(Token, usize), ()> {
+        if chars.is_empty() {
+            Err(())
+        } else if chars[0] == '+' {
+            Ok((Token::Plus, 1))
+        } else if chars[0] == '-' {
+            Ok((Token::Minus, 1))
+        } else if chars[0] == '*' {
+            Ok((Token::Star, 1))
+        } else if chars[0] == '/' {
+            Ok((Token::Slash, 1))
+        } else if chars[0] == '!' {
+            Ok((Token::Bang, 1))
+        } else if chars[0] == '=' {
+            Ok((Token::Equals, 1))
+        } else {
+            Err(())
+        }
+    }
+
+    let source = source.chars().collect::<Vec<char>>();
+    let mut tokens = vec![];
+    let mut index = 0;
+    while index < source.len() {
+        if let Ok((_whitespace, chars_consumed)) = tokenize_whitespace(&source[index..]) {
+            // Ignore whitespace
+            index += chars_consumed;
+        } else if let Ok((num, chars_consumed)) = tokenize_number(&source[index..]) {
+            tokens.push(num);
+            index += chars_consumed;
+        } else if let Ok((num, chars_consumed)) = tokenize_string(&source[index..]) {
+            tokens.push(num);
+            index += chars_consumed;
+        } else if let Ok((num, chars_consumed)) = tokenize_identifier(&source[index..]) {
+            tokens.push(num);
+            index += chars_consumed;
+        } else if let Ok((operator, chars_consumed)) = tokenize_operator(&source[index..]) {
+            tokens.push(operator);
+            index += chars_consumed;
+        } else if source[index] == ',' {
+            tokens.push(Token::Comma);
+            index += 1;
+        } else if source[index] == ';' {
+            tokens.push(Token::Semicolon);
+            index += 1;
+        } else if source[index] == '(' {
+            tokens.push(Token::Parenthesis { closing: false });
+            index += 1;
+        } else if source[index] == ')' {
+            tokens.push(Token::Parenthesis { closing: true });
+            index += 1;
+        } else {
+            // Skip if things fail
+            index += 1;
+        }
+    }
+    Ok(tokens)
+}

src/tokenizer.js

@@ -1,346 +0,0 @@
-/**
- * @module tokenizer
- * @file Manages the tokenization phase of Pivot.
- * @author Garen Tyler <garentyler@gmail.com>
- * @requires module:types
- */
-const Token = require('./types.js').Token;
-
-/**
- * @function tokenize
- * @desc Takes in raw code, and outputs an array of Tokens.
- * @param {string} code The raw input code.
- * @returns {Token[]} The code, split into tokens.
- * @public
- */
-function tokenize(code) {
-  // Split the string into an array of chars.
-  let chars = code.split('');
-
-  // Create buffers.
-  let letterBuffer = [];
-  let operatorBuffer = [];
-  let numberBuffer = [];
-  let stringBuffer = [];
-
-  // Create the output Token[].
-  let tokens = [];
-
-  // Create an object to keep track of string data.
-  let stringData = {
-    inString: false,
-    stringType: null
-  };
-
-  // Escape chars and remove comments.
-  chars = combineEscapedChars(chars);
-  chars = removeComments(chars);
-
-
-  // Actually tokenize the chars.
-  for (let i = 0; i < chars.length; i++) {
-    let char = chars[i];
-    if (stringData.inString) { // Tokenize differently in a string.
-      // If a string delimiter and the same as the inital delimiter.
-      if (determineCharType(char) == 'string delimiter' && char == stringData.stringType) {
-        stringData.inString = false; // Not in a string any more.
-        tokens.push(new Token('string', 'n/a', stringBuffer.join(''))); // Push the string.
-        stringBuffer = []; // Clear the string buffer.
-      } else stringBuffer.push(char); // Add to the string buffer.
-    } else { // Tokenize normally.
-      if (determineCharType(char) == 'string delimiter') {
-        stringData.inString = true; // In a string now.
-        stringData.stringType = char;
-      } else if (determineCharType(char) == 'letter') {
-        letterBuffer.push(char); // Add to the letter buffer.
-        // End the other buffers.
-        if (operatorBuffer.length > 0) {
-          let operator = operatorBuffer.join('');
-          tokens.push(new Token('operator', operatorType(operator), operator));
-          operatorBuffer = [];
-        }
-        if (numberBuffer.length > 0) {
-          let number = numberBuffer.join('');
-          tokens.push(new Token('number', 'n/a', number));
-          numberBuffer = [];
-        }
-      } else if (determineCharType(char) == 'operator') {
-        operatorBuffer.push(char); // Add to the operator buffer.
-        // End the other buffers.
-        if (letterBuffer.length > 0) {
-          let variable = letterBuffer.join('');
-          tokens.push(new Token('name', 'variable', variable));
-          letterBuffer = [];
-        }
-        if (numberBuffer.length > 0) {
-          let number = numberBuffer.join('');
-          tokens.push(new Token('number', 'n/a', number));
-          numberBuffer = [];
-        }
-      } else if (determineCharType(char) == 'digit') {
-        numberBuffer.push(char); // Add to the number buffer.
-        // End the other buffers.
-        if (letterBuffer.length > 0) {
-          let variable = letterBuffer.join('');
-          tokens.push(new Token('name', 'variable', variable));
-          letterBuffer = [];
-        }
-        if (operatorBuffer.length > 0) {
-          let operator = operatorBuffer.join('');
-          tokens.push(new Token('operator', operatorType(operator), operator));
-          operatorBuffer = [];
-        }
-      } else if (determineCharType(char) == 'whitespace') {
-        // End all buffers.
-        if (letterBuffer.length > 0) {
-          let variable = letterBuffer.join('');
-          tokens.push(new Token('name', 'variable', variable));
-          letterBuffer = [];
-        }
-        if (numberBuffer.length > 0) {
-          let number = numberBuffer.join('');
-          tokens.push(new Token('number', 'n/a', number));
-          numberBuffer = [];
-        }
-        if (operatorBuffer.length > 0) {
-          let operator = operatorBuffer.join('');
-          tokens.push(new Token('operator', operatorType(operator), operator));
-          operatorBuffer = [];
-        }
-      } else if (determineCharType(char) == 'delimiter') {
-        // End all buffers.
-        if (letterBuffer.length > 0) {
-          let variable = letterBuffer.join('');
-          tokens.push(new Token('name', 'variable', variable));
-          letterBuffer = [];
-        }
-        if (numberBuffer.length > 0) {
-          let number = numberBuffer.join('');
-          tokens.push(new Token('number', 'n/a', number));
-          numberBuffer = [];
-        }
-        if (operatorBuffer.length > 0) {
-          let operator = operatorBuffer.join('');
-          tokens.push(new Token('operator', operatorType(operator), operator));
-          operatorBuffer = [];
-        }
-        // Push the delimiter.
-        tokens.push(getDelimiterToken(char));
-      }
-    }
-  }
-
-  // Empty all the buffers.
-  if (letterBuffer.length > 0) {
-    let variable = letterBuffer.join('');
-    tokens.push(new Token('name', 'variable', variable));
-    letterBuffer = [];
-  }
-  if (numberBuffer.length > 0) {
-    let number = numberBuffer.join('');
-    tokens.push(new Token('number', 'n/a', number));
-    numberBuffer = [];
-  }
-  if (operatorBuffer.length > 0) {
-    let operator = operatorBuffer.join('');
-    tokens.push(new Token('operator', operatorType(operator), operator));
-    operatorBuffer = [];
-  }
-
-  tokens = changeKeywords(tokens);
-
-  return tokens;
-}
-
-/**
- * @function combineEscapedChars
- * @desc Combines escaped chars into one char.
- * @param {string[]} chars The chars.
- * @returns {string[]} The chars with combined escaped chars.
- * @private
- */
-function combineEscapedChars(chars) {
-  // Check for characters to be escaped.
-  for (let i = 0; i < chars.length; i++) {
-    if (chars[i] == '\\') {
-      chars.splice(i, 2, chars[i] + chars[i + 1]);
-      i -= 2;
-    }
-  }
-  return chars;
-}
-
-/**
- * @function removeComments
- * @desc Removes comments.
- * @param {string[]} chars The chars.
- * @returns {string[]} The chars without comments.
- * @private
- */
-function removeComments(chars) {
-  let inComment = false; // Keep track if in a comment.
-  for (let i = 0; i < chars.length; i++) {
-    if (chars[i] == '/') {
-      if (chars[i + 1] == '/') {
-        inComment = true;
-      }
-    }
-    if (chars[i] == '\n') {
-      inComment = false;
-      chars.splice(i, 1); // Remove the newline at the end of the comment.
-      i--;
-    }
-    if (inComment) {
-      chars.splice(i, 1); // Remove the char in the comment.
-      i--;
-    }
-  }
-  return chars;
-}
-
-/**
- * @function changeKeywords
- * @desc Changes tokens with subtype variable to subtype keyword
- * @param {Token[]} tokens The tokens
- * @returns {Token[]} The tokens with keywords.
- * @private
- */
-function changeKeywords(tokens) {
-  return tokens.map(t => {
-    if (t.subtype == 'variable' && determineType(t.value) == 'keyword') {
-      t.subtype = 'keyword';
-    }
-    return t;
-  });
-}
-
-/**
- * @function getDelimiterToken
- * @desc Turns a delimiter char into a token.
- * @param {string} delimiter The delimiter char.
- * @returns {Token} The delimiter token.
- * @private
- */
-function getDelimiterToken(delimiter) {
-  if (/\(|\)/.test(delimiter))
-    return new Token('delimiter', delimiter == '(' ? 'left' : 'right', 'parenthesis');
-  else if (/\[|\]/.test(delimiter))
-    return new Token('delimiter', delimiter == '[' ? 'left' : 'right', 'bracket');
-  else if (/\{|\}/.test(delimiter))
-    return new Token('delimiter', delimiter == '{' ? 'left' : 'right', 'brace');
-  else throw new Error('Expected delimiter but got ' + delimiter);
-}
-
-/**
- * @function operatorType
- * @desc Turns a delimiter char into a token.
- * @param {string} delimiter The delimiter char.
- * @returns {Token} The delimiter token.
- * @private
- */
-function operatorType(op) {
-  switch (op) {
-    case '.':
-    case '+':
-    case '-':
-    case '*':
-    case '/':
-    case '**':
-    case '%':
-    case '<<':
-    case '>>':
-    case '<':
-    case '<=':
-    case '>':
-    case '>=':
-    case '==':
-    case '!=':
-    case '&':
-    case '|':
-    case '^':
-    case '&&':
-    case '||':
-    case '^^':
-    case '=':
-    case '+=':
-    case '-=':
-    case '*=':
-    case '/=':
-    case '**=':
-    case '%=':
-    case '<<=':
-    case '>>=':
-    case '&=':
-    case '|=':
-    case '^=':
-      return 'dual';
-      break;
-    case '++':
-    case '--':
-      return 'postfix';
-      break;
-    case '!':
-    case '~':
-      return 'prefix';
-      break;
-    case ',':
-    case ';':
-      return 'none';
-      break;
-    default:
-      throw new TypeError('Unexpected operator ' + op);
-      break;
-  }
-}
-
-/**
- * @function determineCharType
- * @desc Detects the type of characters.
- * @param {string} char The input character(s).
- * @returns {string} The type of char.
- * @private
- */
-function determineCharType(char) {
-  if (/[A-Za-z]/.test(char))
-    return 'letter';
-  else if (/\.|\+\+|--|!|~|\+|-|\*\*|\*|\/%|<<|>>|<|<=|>|>=|==|!=|&|\^|\||&&|\|\||=|\+=|-=|\*\*=|\*=|\/=|%=|<<=|>>=|&=|\^=|\|=|,|;/.test(char))
-    // All the operators in Pivot.
-    return 'operator';
-  else if (/\(|\)|\[|\]|\{|\}/.test(char))
-    return 'delimiter';
-  else if (/'|"|`/.test(char))
-    return 'string delimiter';
-  else if (/\d/.test(char))
-    return 'digit';
-  else if (/\\./.test(char))
-    return 'escaped char';
-  else if (/\s/.test(char))
-    return 'whitespace';
-  else throw new TypeError('Unexpected char ' + char);
-};
-
-/**
- * @function determineType
- * @desc Detects the type of a string.
- * @param {string} str The input string.
- * @returns {string} The type of string.
- * @private
- */
-function determineType(str) {
-  if (/let|const/.test(str)) // TODO: Add more keywords.
-    return 'keyword';
-  else return 'unknown';
-};
-
-module.exports = {
-  tokenize,
-  util: {
-    combineEscapedChars,
-    removeComments,
-    changeKeywords,
-    getDelimiterToken,
-    operatorType,
-    determineCharType,
-    determineType
-  }
-};

src/types.js

@@ -1,42 +0,0 @@
-/**
- * @module types
- * @file Provides a consistent source of types for the compiler.
- * @author Garen Tyler <garentyler@gmail.com>
- */
-
-/**
- * @class Token
- * @classdesc Stores the type of token, subtype, and literal char value.
- */
-function Token(type, subtype, value) {
-  this.type = type;
-  this.subtype = subtype;
-  this.value = value;
-}
-
-/**
- * @class Group
- * @classdesc Stores the type of group, and the tokens in the group.
- */
-function Group(type, tokens) {
-  this.type = 'group'
-  this.subtype = type;
-  this.tokens = tokens;
-}
-
-/**
- * @class Operator
- * @classdesc Stores the type of operator, and tokens for an operator.
- */
-function Operator(subtype, value, operands) {
-  this.type = 'operator';
-  this.subtype = subtype;
-  this.value = value;
-  this.operands = operands;
-}
-
-module.exports = {
-  Token,
-  Group,
-  Operator
-};

test.pvt

@@ -0,0 +1 @@
+log("Hello world!")

test/test.js

@@ -1,428 +0,0 @@
-const assert = require('assert');
-
-const types = require('../src/types.js');
-const tokenizer = require('../src/tokenizer.js');
-const parser = require('../src/parser.js');
-
-console.log(parser.parse(tokenizer.tokenize('let x = (5 + 3);')));
-
-describe('types.js', () => {
-  it('Has a Token child.', () => {
-    assert.equal(types.hasOwnProperty('Token'), true);
-  });
-  it('Has a Group child.', () => {
-    assert.equal(types.hasOwnProperty('Group'), true);
-  });
-  it('Has a Operator child.', () => {
-    assert.equal(types.hasOwnProperty('Operator'), true);
-  });
-
-  describe('Token', () => {
-    it('Works as a constructor', () => {
-      try {
-        let token = new types.Token('a', 'b', 'c');
-      } catch (err) {
-        throw err;
-      }
-    });
-    it('Has values \'type\', \'subtype\', and \'value\'', () => {
-      try {
-        let token = new types.Token('a', 'b', 'c');
-        if (!token.hasOwnProperty('type') || !token.hasOwnProperty('subtype') || !token.hasOwnProperty('value'))
-          throw new Error('Token is missing \'type\', \'subtype\', or \'value\' properties.');
-        if (token.type != 'a' || token.subtype != 'b' || token.value != 'c')
-          throw new Error('Token incorrectly set \'type\', \'subtype\', or \'value\' properties.');
-      } catch (err) {
-        throw err;
-      }
-    });
-  });
-  describe('Group', () => {
-    it('Works as a constructor', () => {
-      try {
-        let group = new types.Group('a', 'b');
-      } catch (err) {
-        throw err;
-      }
-    });
-    it('Has values \'type\', \'subtype\', and \'tokens\'', () => {
-      try {
-        let group = new types.Group('a', 'b');
-        if (!group.hasOwnProperty('type') || !group.hasOwnProperty('subtype') || !group.hasOwnProperty('tokens'))
-          throw new Error('Group is missing \'type\', \'subtype\', or \'tokens\' properties.');
-        if (group.type != 'group' || group.subtype != 'a' || group.tokens != 'b')
-          throw new Error('Group incorrectly set \'type\', \'subtype\', or \'tokens\' properties.');
-      } catch (err) {
-        throw err;
-      }
-    });
-  });
-  // describe('Operator', () => {
-  //   it('Works as a constructor', () => {
-  //     try {
-  //       let op = new types.Operator('dual', '+', ['3','4']);
-  //     } catch (err) {
-  //       throw err;
-  //     }
-  //   });
-  //   it('Has values \'type\', \'subtype\', \'value\', and \'operands\'', () => {
-  //     try {
-  //       let op = new types.Operator('dual', '+', ['3','4']);
-  //       if (!op.hasOwnProperty('type') || !op.hasOwnProperty('subtype') || !op.hasOwnProperty('value') || !op.hasOwnProperty('operands'))
-  //         throw new Error('Operator is missing \'type\', \'subtype\', \'value\', or \'operands\' properties.');
-  //       if (op.type != 'operator' || op.subtype != 'dual' || op.value != '+' || op.operands[0] != '3')
-  //         throw new Error('Operator incorrectly set \'type\', \'subtype\', \'value\', or \'operands\' properties.');
-  //     } catch (err) {
-  //       throw err;
-  //     }
-  //   });
-  // });
-});
-describe('tokenizer.js', () => {
-  it('combineEscapedChars works', () => {
-    assert.equal(tokenizer.util.combineEscapedChars(`let x = 'test\\nnewline';`.split('')).join(''), `let x = 'test\\nnewline';`);
-  });
-  it('removeComments works', () => {
-    assert.equal(tokenizer.util.removeComments(`// Comment\nlet i = 0;`.split('')).join(''), `let i = 0;`);
-  });
-  it('changeKeywords works', () => {
-    let tokens = tokenizer.util.changeKeywords([{
-      type: 'name',
-      subtype: 'variable',
-      value: 'let'
-    }, {
-      type: 'name',
-      subtype: 'variable',
-      value: 'x'
-    }]);
-    let correct = [{
-      type: 'name',
-      subtype: 'keyword',
-      value: 'let'
-    }, {
-      type: 'name',
-      subtype: 'variable',
-      value: 'x'
-    }];
-    let isCorrect = true;
-    tokens.forEach((t, i) => {
-      if (t.type != correct[i].type)
-        throw new Error('Changed type: Expected \'' + correct[i].type + '\' but got \'' + t.type + '\'')
-      else if (t.subtype != correct[i].subtype)
-        throw new Error('Incorrectly changed subtype: Expected \'' + correct[i].subtype + '\' but got \'' + t.subtype + '\'')
-      else if (t.value != correct[i].value)
-        throw new Error('Changed value: Expected \'' + correct[i].value + '\' but got \'' + t.value + '\'')
-    });
-  });
-  it('getDelimiterToken works', () => {
-    let token = tokenizer.util.getDelimiterToken(')');
-    if (token.type != 'delimiter')
-      throw new Error('Incorrect type: Expected \'delimiter\' but got \'' + token.type + '\'')
-    else if (token.subtype != 'right')
-      throw new Error('Incorrect subtype: Expected \'right\' but got \'' + token.subtype + '\'')
-    else if (token.value != 'parenthesis')
-      throw new Error('Incorrect value: Expected \'parenthesis\' but got \'' + token.value + '\'')
-  });
-  it('operatorType works', () => {
-    assert.equal(tokenizer.util.operatorType('++'), 'postfix');
-    assert.equal(tokenizer.util.operatorType(';'), 'none');
-    assert.equal(tokenizer.util.operatorType('+'), 'dual');
-  });
-  it('determineCharType works', () => {
-    assert.equal(tokenizer.util.determineCharType('+'), 'operator');
-    assert.equal(tokenizer.util.determineCharType('"'), 'string delimiter');
-    assert.equal(tokenizer.util.determineCharType('4'), 'digit');
-  });
-  it('determineType works', () => {
-    assert.equal(tokenizer.util.determineType('let'), 'keyword');
-    assert.equal(tokenizer.util.determineType('dog'), 'unknown');
-  });
-});
-describe('parser.js', () => {
-  it('addIndexes works', () => {
-    let tokens = parser.util.addIndexes([{
-      type: 'name',
-      subtype: 'keyword',
-      value: 'let'
-    }, {
-      type: 'name',
-      subtype: 'variable',
-      value: 'x'
-    }]);
-    let correct = [{
-      type: 'name',
-      subtype: 'keyword',
-      value: 'let',
-      index: 0
-    }, {
-      type: 'name',
-      subtype: 'variable',
-      value: 'x',
-      index: 1
-    }];
-    let isCorrect = true;
-    tokens.forEach((t, i) => {
-      if (t.type != correct[i].type)
-        throw new Error('Changed type: Expected \'' + correct[i].type + '\' but got ' + t.type)
-      else if (t.subtype != correct[i].subtype)
-        throw new Error('Changed subtype: Expected \'' + correct[i].subtype + '\' but got ' + t.subtype)
-      else if (t.value != correct[i].value)
-        throw new Error('Changed value: Expected \'' + correct[i].value + '\' but got ' + t.value)
-      else if (t.index != correct[i].index)
-        throw new Error('Incorrect index: Expected \'' + correct[i].index + '\' but got ' + t.index)
-    });
-  });
-  it('addLevels works', () => {
-    let tokens = parser.util.addLevels([{
-      type: 'name',
-      subtype: 'keyword',
-      value: 'let',
-      index: 0
-    }, {
-      type: 'name',
-      subtype: 'variable',
-      value: 'x',
-      index: 1
-    }, {
-      type: 'operator',
-      subtype: 'dual',
-      value: '=',
-      index: 2
-    }, {
-      type: 'delimiter',
-      subtype: 'left',
-      value: 'parenthesis',
-      index: 3
-    }, {
-      type: 'number',
-      subtype: 'n/a',
-      value: '5',
-      index: 4
-    }, {
-      type: 'delimiter',
-      subtype: 'right',
-      value: 'parenthesis',
-      index: 5
-    }]);
-    let correct = [{
-      type: 'name',
-      subtype: 'keyword',
-      value: 'let',
-      index: 0,
-      level: 0
-    }, {
-      type: 'name',
-      subtype: 'variable',
-      value: 'x',
-      index: 1,
-      level: 0
-    }, {
-      type: 'operator',
-      subtype: 'dual',
-      value: '=',
-      index: 2,
-      level: 0
-    }, {
-      type: 'delimiter',
-      subtype: 'left',
-      value: 'parenthesis',
-      index: 3,
-      level: 1
-    }, {
-      type: 'number',
-      subtype: 'n/a',
-      value: '5',
-      index: 4,
-      level: 1
-    }, {
-      type: 'delimiter',
-      subtype: 'right',
-      value: 'parenthesis',
-      index: 5,
-      level: 1
-    }];
-    let isCorrect = true;
-    tokens.forEach((t, i) => {
-      if (t.type != correct[i].type)
-        throw new Error('Changed type: Expected \'' + correct[i].type + '\' but got ' + t.type)
-      else if (t.subtype != correct[i].subtype)
-        throw new Error('Changed subtype: Expected \'' + correct[i].subtype + '\' but got \'' + t.subtype + '\'')
-      else if (t.value != correct[i].value)
-        throw new Error('Changed value: Expected \'' + correct[i].value + '\' but got \'' + t.value + '\'')
-      else if (t.index != correct[i].index)
-        throw new Error('Incorrect index: Expected \'' + correct[i].index + '\' but got \'' + t.index + '\'')
-      else if (t.level != correct[i].level)
-        throw new Error('Incorrect level: Expected \'' + correct[i].level + '\' but got \'' + t.level + '\'')
-    });
-  });
-  it('getDeepestLevel works', () => {
-    let deepestLevel = parser.util.getDeepestLevel([{
-      type: 'name',
-      subtype: 'keyword',
-      value: 'let',
-      index: 0,
-      level: 0
-    }, {
-      type: 'name',
-      subtype: 'variable',
-      value: 'x',
-      index: 1,
-      level: 0
-
-    }, {
-      type: 'operator',
-      subtype: 'dual',
-      value: '=',
-      index: 2,
-      level: 0
-    }, {
-      type: 'delimiter',
-      subtype: 'left',
-      value: 'parenthesis',
-      index: 3,
-      level: 1
-    }, {
-      type: 'number',
-      subtype: 'n/a',
-      value: '5',
-      index: 4,
-      level: 1
-    }, {
-      type: 'operator',
-      subtype: 'dual',
-      value: '+',
-      index: 5,
-      level: 1
-    }, {
-      type: 'delimiter',
-      subtype: 'left',
-      value: 'parenthesis',
-      index: 6,
-      level: 2
-    }, {
-      type: 'number',
-      subtype: 'n/a',
-      value: '6',
-      index: 7,
-      level: 2
-    }, {
-      type: 'operator',
-      subtype: 'dual',
-      value: '*',
-      index: 8,
-      level: 2
-    }, {
-      type: 'number',
-      subtype: 'n/a',
-      value: '2',
-      index: 9,
-      level: 2
-    }, {
-      type: 'delimiter',
-      subtype: 'right',
-      value: 'parenthesis',
-      index: 10,
-      level: 2
-    }, {
-      type: 'delimiter',
-      subtype: 'right',
-      value: 'parenthesis',
-      index: 11,
-      level: 1
-    }, {
-      type: 'operator',
-      subtype: 'none',
-      value: ';',
-      index: 12,
-      level: 0
-    }]);
-    if (deepestLevel != 2)
-      throw new Error('Incorrect deepestLevel. Expected \'2\' but got \'' + deepestLevel + '\'');
-  });
-  it('combineGroups works', () => {
-    let ast = parser.util.combineGroups([{
-      type: 'name',
-      subtype: 'keyword',
-      value: 'let',
-      index: 0,
-      level: 0
-    }, {
-      type: 'name',
-      subtype: 'variable',
-      value: 'x',
-      index: 1,
-      level: 0
-
-    }, {
-      type: 'operator',
-      subtype: 'dual',
-      value: '=',
-      index: 2,
-      level: 0
-    }, {
-      type: 'delimiter',
-      subtype: 'left',
-      value: 'parenthesis',
-      index: 3,
-      level: 1
-    }, {
-      type: 'number',
-      subtype: 'n/a',
-      value: '5',
-      index: 4,
-      level: 1
-    }, {
-      type: 'operator',
-      subtype: 'dual',
-      value: '+',
-      index: 5,
-      level: 1
-    }, {
-      type: 'delimiter',
-      subtype: 'left',
-      value: 'parenthesis',
-      index: 6,
-      level: 2
-    }, {
-      type: 'number',
-      subtype: 'n/a',
-      value: '6',
-      index: 7,
-      level: 2
-    }, {
-      type: 'operator',
-      subtype: 'dual',
-      value: '*',
-      index: 8,
-      level: 2
-    }, {
-      type: 'number',
-      subtype: 'n/a',
-      value: '2',
-      index: 9,
-      level: 2
-    }, {
-      type: 'delimiter',
-      subtype: 'right',
-      value: 'parenthesis',
-      index: 10,
-      level: 2
-    }, {
-      type: 'delimiter',
-      subtype: 'right',
-      value: 'parenthesis',
-      index: 11,
-      level: 1
-    }, {
-      type: 'operator',
-      subtype: 'none',
-      value: ';',
-      index: 12,
-      level: 0
-    }]);
-    if (ast[3].type != 'group')
-      throw new Error('Incorrectly combined group.');
-    if (ast[3].tokens[3].type != 'group')
-      throw new Error('Incorrectly combined group.');
-  });
-});