Remove wasm

.gitignore

@@ -1 +1,3 @@
 /target
+node_modules
+*.wasm

Cargo.lock

@@ -0,0 +1,7 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 3
+
+[[package]]
+name = "pivot"
+version = "0.1.0"

Cargo.toml

@@ -0,0 +1,8 @@
+[package]
+name = "pivot"
+version = "0.1.0"
+authors = ["Garen Tyler <garentyler@garen.dev>"]
+edition = "2018"
+description = "Pivot is a new programming language built with Rust. Pivot is currently in the alpha stage of development."
+license = "MIT"
+repository = "https://github.com/garentyler/pivot"

output.s

@@ -1,10 +0,0 @@
-.global main
-main:
-  push {fp, lr}
-  ldr r0, =1
-  cmp r0, #1
-  moveq r0, #'.'
-  movne r0, #'F'
-  bl putchar
-  mov r0, #0
-  pop {fp, pc}

package.json

@@ -1,20 +0,0 @@
-{
-  "name": "pivot",
-  "version": "1.0.0",
-  "description": "A programming language",
-  "main": "index.ts",
-  "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1",
-    "run": "deno run -c tsconfig.json --allow-read --allow-write --unstable src/index.ts"
-  },
-  "repository": {
-    "type": "git",
-    "url": "git+https://github.com/ElementG9/pivot.git"
-  },
-  "author": "Garen Tyler",
-  "license": "MIT",
-  "bugs": {
-    "url": "https://github.com/ElementG9/pivot/issues"
-  },
-  "homepage": "https://github.com/ElementG9/pivot#readme"
-}

readme.md

@@ -1,10 +1,15 @@
 # Pivot
-
+## Version
 Pivot is a new programming language built with Rust by Garen Tyler. Pivot is currently in the alpha stage of development.
 
 ---
 
-### Installation
+### Development
-* Download the repo with `git clone https://github.com/ElementG9/pivot`.
+* Download the repo with `git clone https://github.com/garentyler/pivot`.
-* Install [Rust](https://www.rust-lang.org/).
+* Make sure you have [Rust](https://www.rust-lang.org/), [NodeJS](https://nodejs.org/en/), and [NPM](https://www.npmjs.com/) installed.
-* Run with `cargo run <filename>`.
+* Run `npm install` to install the JavaScript dependencies.
+* Scripts:
+    * `npm run clean` to delete dependencies and output files.
+    * `npm run build` to build the project.
+    * `npm run run` to run the example.
+    * `npm run test` to run the tests.

src/ast.ts

@@ -1,273 +0,0 @@
-import { emit } from './codegen.ts';
-
-export interface AST {
-  emit(): void;
-  equals(other: AST): boolean;
-}
-
-// Number node.
-export class Num implements AST {
-  constructor(public value: number) {}
-
-  emit() {
-    emit(`  ldr r0, =${this.value}`);
-  }
-  equals(other: AST): boolean {
-    return other instanceof Num
-      && this.value === other.value;
-  }
-}
-// Identifier node.
-export class Id implements AST {
-  constructor(public value: string) {}
-
-  emit() {
-    throw Error('Not implemented yet');
-  }
-  equals(other: AST): boolean {
-    return other instanceof Id
-      && this.value === other.value;
-  }
-}
-// Operator nodes.
-export class Not implements AST {
-  constructor(public operand: AST) {}
-
-  emit() {
-    this.operand.emit();
-    emit('  cmp r0, #0');
-    emit('  moveq r0, #1');
-    emit('  movne r0, #0');
-  }
-  equals(other: AST): boolean {
-    return other instanceof Not
-      && this.operand.equals(other.operand);
-  }
-}
-export abstract class Infix {
-  constructor(public left: AST, public right: AST) {}
-  emit() {
-    this.left.emit();
-    emit('  push {r0, ip}');
-    this.right.emit();
-    emit('  pop {r1, ip}');
-  }
-}
-export class Equal extends Infix implements AST {
-  constructor(public left: AST, public right: AST) {
-    super(left, right);
-  }
-
-  emit() {
-    super.emit();
-    emit('  cmp r0, r1');
-    emit('  moveq r0, #1');
-    emit('  movne r0, #0');
-  }
-  equals(other: AST): boolean {
-    return other instanceof Equal
-      && this.left.equals(other.left)
-      && this.right.equals(other.right);
-  }
-}
-export class NotEqual extends Infix implements AST {
-  constructor(public left: AST, public right: AST) {
-    super(left, right);
-  }
-
-  emit() {
-    super.emit();
-    emit('  cmp r0, r1');
-    emit('  moveq r0, #0');
-    emit('  movne r0, #1');
-  }
-  equals(other: AST): boolean {
-    return other instanceof NotEqual
-      && this.left.equals(other.left)
-      && this.right.equals(other.right);
-  }
-}
-export class Add extends Infix implements AST {
-  constructor(public left: AST, public right: AST) {
-    super(left, right);
-  }
-
-  emit() {
-    super.emit();
-    emit('  add r0, r0, r1');
-  }
-  equals(other: AST): boolean {
-    return other instanceof Add
-      && this.left.equals(other.left)
-      && this.right.equals(other.right);
-  }
-}
-export class Subtract extends Infix implements AST {
-  constructor(public left: AST, public right: AST) {
-    super(left, right);
-  }
-
-  emit() {
-    super.emit();
-    emit('  sub r0, r0, r1');
-  }
-  equals(other: AST): boolean {
-    return other instanceof Subtract
-      && this.left.equals(other.left)
-      && this.right.equals(other.right);
-  }
-}
-export class Multiply extends Infix implements AST {
-  constructor(public left: AST, public right: AST) {
-    super(left, right);
-  }
-
-  emit() {
-    super.emit();
-    emit('  mul r0, r0, r1');
-  }
-  equals(other: AST): boolean {
-    return other instanceof Multiply
-      && this.left.equals(other.left)
-      && this.right.equals(other.right);
-  }
-}
-export class Divide extends Infix implements AST {
-  constructor(public left: AST, public right: AST) {
-    super(left, right);
-  }
-
-  emit() {
-    super.emit();
-    emit('  udiv r0, r0, r1');
-  }
-  equals(other: AST): boolean {
-    return other instanceof Divide
-      && this.left.equals(other.left)
-      && this.right.equals(other.right);
-  }
-}
-// Function call node.
-export class FunctionCall implements AST {
-  constructor(public callee: string, public args: Array<AST>) {}
-
-  emit() {
-    throw Error('Not implemented yet');
-  }
-  equals(other: AST): boolean {
-    return other instanceof FunctionCall
-      && this.callee === other.callee
-      && this.args.length === other.args.length
-      && this.args.every((arg: AST, i: number) => arg.equals(other.args[i]));
-  }
-}
-// Return node.
-export class Return implements AST {
-  constructor(public operand: AST) {}
-
-  emit() {
-    throw Error('Not implemented yet');
-  }
-  equals(other: AST): boolean {
-    return other instanceof Return
-      && this.operand.equals(other.operand);
-  }
-}
-// Block node.
-export class Block implements AST {
-  constructor(public statements: Array<AST>) {}
-
-  emit() {
-    this.statements.forEach((statement: any) => statement.emit());
-  }
-  equals(other: AST): boolean {
-    return other instanceof Block
-      && this.statements.length === other.statements.length
-      && this.statements.every((arg: AST, i: number) => arg.equals(other.statements[i]));
-  }
-}
-// If node.
-export class If implements AST {
-  constructor(public conditional: AST,
-              public consequence: AST,
-              public alternative: AST) {}
-
-  emit() {
-    throw Error('Not implemented yet');
-  }
-  equals(other: AST): boolean {
-    return other instanceof If
-      && this.conditional.equals(other.conditional)
-      && this.consequence.equals(other.consequence)
-      && this.alternative.equals(other.alternative);
-  }
-}
-// Function definition node.
-export class FunctionDefinition implements AST {
-  constructor(public name: string,
-              public parameters: Array<string>,
-              public body: AST) {}
-
-  emit() {
-    throw Error(`Not implemented yet (${this.name})`);
-  }
-  equals(other: AST): boolean {
-    return other instanceof FunctionDefinition
-      && this.name === other.name
-      && this.parameters.length === other.parameters.length
-      && this.parameters.every((arg: string, i: number) => arg === other.parameters[i])
-      && this.body.equals(other.body);
-  }
-}
-// Variable declaration node.
-export class VariableDeclaration implements AST {
-  constructor(public name: string, public value: AST) {}
-
-  emit() {
-    throw Error('Not implemented yet');
-  }
-  equals(other: AST): boolean {
-    return other instanceof VariableDeclaration
-      && this.name === other.name
-      && this.value.equals(other.value);
-  }
-}
-// Assignment node.
-export class Assign implements AST {
-  constructor(public name: string, public value: AST) {}
-
-  emit() {
-    throw Error('Not implemented yet');
-  }
-  equals(other: AST): boolean {
-    return other instanceof Assign
-      && this.name === other.name
-      && this.value.equals(other.value);
-  }
-}
-// While loop node.
-export class While implements AST {
-  constructor(public conditional: AST, public body: AST) {}
-
-  emit() {
-    throw Error('Not implemented yet');
-  }
-  equals(other: AST): boolean {
-    return other instanceof While
-      && this.conditional.equals(other.conditional)
-      && this.body.equals(other.body);
-  }
-}
-// Variable node.
-export class Var implements AST {
-  constructor(public name: string, public value: AST) {}
-
-  emit() {
-    throw Error('Not implemented yet');
-  }
-  equals(other: AST): boolean {
-    return other instanceof Var
-      && this.name === other.name
-      && this.value.equals(other.value);
-  }
-}

src/codegen.rs

@@ -0,0 +1,5 @@
+use crate::{parse::AstNode, InterpreterError};
+
+pub fn codegen(_ast: &AstNode) -> Result<String, InterpreterError> {
+    Ok(String::new())
+}

src/codegen.ts

@@ -1,55 +0,0 @@
-import { AST } from './ast.ts';
-import { existsSync } from 'https://deno.land/std/fs/mod.ts';
-
-const outputFileName = '../output.s';
-
-const outputToFile = false;
-export async function emit(input: string) {
-  if (outputToFile) {
-    if (existsSync(outputFileName)) {
-      await Deno.remove(outputFileName);
-    }
-    await Deno.create(outputFileName);
-    const outputFile = await Deno.open(outputFileName, {
-      write: true,
-      append: true,
-    });
-    await Deno.write(outputFile.rid, new TextEncoder().encode(input));
-  } else {
-    console.log(input);
-  }
-}
-
-export class Main implements AST {
-  constructor(public statements: Array<AST>) {}
-
-  emit() {
-    emit('.global main');
-    emit('main:');
-    emit('  push {fp, lr}');
-    this.statements.forEach((statement: any) => statement.emit());
-    emit('  mov r0, #0');
-    emit('  pop {fp, pc}');
-  }
-  equals(other: AST): boolean {
-    return other instanceof Main
-      && this.statements.length === other.statements.length
-      && this.statements.every((arg: AST, i: number) => arg.equals(other.statements[i]));
-  }
-}
-
-export class Assert implements AST {
-  constructor(public condition: AST) {}
-
-  emit() {
-    this.condition.emit();
-    emit('  cmp r0, #1');
-    emit(`  moveq r0, #'.'`);
-    emit(`  movne r0, #'F'`);
-    emit('  bl putchar');
-  }
-  equals(other: AST): boolean {
-    return other instanceof Assert
-      && this.condition.equals(other.condition);
-  }
-}

src/index.ts

@@ -1,10 +0,0 @@
-import { parse } from './parser.ts';
-
-let result = parse(`
-function main() {
-  assert(1);
-  assert(!0);
-}
-`);
-// console.log(JSON.stringify(result, null, 2));
-result.emit();

src/lib.rs

@@ -0,0 +1,32 @@
+pub mod codegen;
+pub mod parse;
+pub mod tokenize;
+
+#[derive(Debug)]
+pub enum InterpreterError {
+    /// Error parsing source
+    ParseError(String),
+    /// Unexpected token
+    UnexpectedToken,
+    /// Mismatched types
+    MismatchedTypes,
+    /// Type error
+    TypeError,
+    /// Unexpected EOF
+    UnexpectedEOF,
+    /// Expected value
+    ExpectedValue,
+    /// Unimplemented
+    Unimplemented,
+}
+impl<T> From<Option<T>> for InterpreterError {
+    fn from(_value: Option<T>) -> InterpreterError {
+        InterpreterError::ExpectedValue
+    }
+}
+
+pub fn compile(source: &str) -> Result<String, InterpreterError> {
+    let tokens = tokenize::tokenize(source);
+    let ast = parse::parse(&tokens?);
+    codegen::codegen(&ast?)
+}

src/main.rs

@@ -0,0 +1,10 @@
+use std::{fs::File, io::Write};
+
+fn main() -> std::io::Result<()> {
+    // Read the source from a file.
+    let source = std::fs::read_to_string("test.pvt").unwrap();
+    let code = pivot::compile(&source).unwrap();
+    // Write it to a file.
+    File::create("out.bf")?.write_all(code.as_bytes())?;
+    Ok(())
+}

src/parse.rs

@@ -0,0 +1,291 @@
+use crate::{tokenize::Token, InterpreterError};
+
+#[derive(Clone, PartialEq, Debug)]
+pub enum AstPrimitive {
+    Integer(i64),
+    Float(f64),
+    String(String),
+    Boolean(bool),
+    Null,
+}
+#[allow(clippy::derive_hash_xor_eq)]
+impl std::hash::Hash for AstPrimitive {
+    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
+        match self {
+            AstPrimitive::Float(f) => format!("{}", f).hash(state),
+            _ => self.hash(state),
+        }
+    }
+}
+impl std::fmt::Display for AstPrimitive {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> {
+        use AstPrimitive::*;
+        match self {
+            Integer(n) => write!(f, "{}", n),
+            Float(n) => write!(f, "{}", n),
+            String(s) => write!(f, "{}", s),
+            Boolean(b) => write!(f, "{}", b),
+            Null => write!(f, "null"),
+        }
+    }
+}
+
+#[derive(Clone, PartialEq, Debug)]
+pub enum AstNode {
+    Primitive(AstPrimitive),
+    Identifier(String),
+    Negate {
+        body: Box<AstNode>,
+    },
+    Add {
+        left: Box<AstNode>,
+        right: Box<AstNode>,
+    },
+    Subtract {
+        left: Box<AstNode>,
+        right: Box<AstNode>,
+    },
+    Multiply {
+        left: Box<AstNode>,
+        right: Box<AstNode>,
+    },
+    Divide {
+        left: Box<AstNode>,
+        right: Box<AstNode>,
+    },
+    Declare {
+        identifier: String,
+    },
+    Assign {
+        left: Box<AstNode>,
+        right: Box<AstNode>,
+    },
+    FunctionCall {
+        identifier: String,
+        arguments: Vec<AstNode>,
+    },
+    Program {
+        statements: Vec<AstNode>,
+    },
+    Null,
+}
+
+pub fn parse(tokens: &[Token]) -> Result<AstNode, InterpreterError> {
+    fn parse_function_call(tokens: &[Token]) -> Result<(AstNode, usize), InterpreterError> {
+        let mut index = 0;
+        let identifier;
+        if let Token::Identifier(id) = tokens[index].clone() {
+            identifier = id;
+            index += 1;
+        } else {
+            return Err(InterpreterError::UnexpectedToken);
+        }
+        if !matches!(tokens[index], Token::Parenthesis { closing: false }) {
+            return Ok((AstNode::Identifier(identifier), 1));
+        } else {
+            index += 1;
+        }
+        // Check if it closes right away.
+        if matches!(tokens[index], Token::Parenthesis { closing: true }) {
+            index += 1;
+            return Ok((
+                AstNode::FunctionCall {
+                    identifier,
+                    arguments: vec![],
+                },
+                index,
+            ));
+        }
+
+        let mut arguments = vec![];
+        if let Ok((argument, tokens_consumed)) = parse_expression(&tokens[index..]) {
+            arguments.push(argument);
+            index += tokens_consumed;
+        }
+        while index + 2 < tokens.len() {
+            if tokens[index] != Token::Comma {
+                break;
+            }
+            index += 1;
+            if let Ok((argument, tokens_consumed)) = parse_expression(&tokens[index..]) {
+                arguments.push(argument);
+                index += tokens_consumed;
+            } else {
+                break;
+            }
+        }
+        if matches!(tokens[index], Token::Parenthesis { closing: true }) {
+            index += 1;
+            Ok((
+                AstNode::FunctionCall {
+                    identifier,
+                    arguments,
+                },
+                index,
+            ))
+        } else {
+            Err(InterpreterError::UnexpectedToken)
+        }
+    }
+    fn parse_primary_expression(tokens: &[Token]) -> Result<(AstNode, usize), InterpreterError> {
+        if tokens.is_empty() {
+            Err(InterpreterError::UnexpectedEOF)
+        } else if let Token::Integer(n) = &tokens[0] {
+            Ok((AstNode::Primitive(AstPrimitive::Integer(*n)), 1))
+        } else if let Token::Float(n) = &tokens[0] {
+            Ok((AstNode::Primitive(AstPrimitive::Float(*n)), 1))
+        } else if let Token::Boolean(n) = &tokens[0] {
+            Ok((AstNode::Primitive(AstPrimitive::Boolean(*n)), 1))
+        } else if let Token::String(s) = &tokens[0] {
+            Ok((AstNode::Primitive(AstPrimitive::String(s.clone())), 1))
+        } else if let Token::Identifier(_) = &tokens[0] {
+            parse_function_call(tokens)
+        } else if tokens[0] == Token::Keyword("let".to_owned()) {
+            if tokens.len() < 2 {
+                Err(InterpreterError::UnexpectedEOF)
+            } else if let Token::Identifier(s) = &tokens[1] {
+                Ok((
+                    AstNode::Declare {
+                        identifier: s.clone(),
+                    },
+                    2,
+                ))
+            } else {
+                Err(InterpreterError::UnexpectedToken)
+            }
+        } else {
+            Err(InterpreterError::UnexpectedToken)
+        }
+    }
+    fn parse_grouped_expression(tokens: &[Token]) -> Result<(AstNode, usize), InterpreterError> {
+        let mut index = 0;
+        // '('
+        if !matches!(tokens[index], Token::Parenthesis { closing: false }) {
+            return parse_primary_expression(tokens);
+        } else {
+            index += 1;
+        }
+        // expression of any kind
+        let (value, tokens_consumed) = parse_expression(&tokens[index..])?;
+        index += tokens_consumed;
+        // ')'
+        if !matches!(tokens[index], Token::Parenthesis { closing: true }) {
+            return Err(InterpreterError::ParseError(
+                "No closing parenthesis".to_owned(),
+            ));
+        } else {
+            index += 1;
+        }
+        Ok((value, index))
+    }
+    fn parse_unary_expression(tokens: &[Token]) -> Result<(AstNode, usize), InterpreterError> {
+        let mut index = 0;
+        if tokens[index] != Token::Minus && tokens[index] != Token::Bang {
+            parse_grouped_expression(&tokens[index..])
+        } else {
+            let operation = tokens[index].clone();
+            index += 1;
+            let (body, tokens_consumed) = parse_unary_expression(&tokens[index..])?;
+            index += tokens_consumed;
+            Ok((
+                match operation {
+                    Token::Minus => AstNode::Negate {
+                        body: Box::new(body),
+                    },
+                    Token::Bang => AstNode::Negate {
+                        body: Box::new(body),
+                    },
+                    _ => return Err(InterpreterError::ParseError("Impossible".to_owned())),
+                },
+                index,
+            ))
+        }
+    }
+    fn parse_multiplication_expression(
+        tokens: &[Token],
+    ) -> Result<(AstNode, usize), InterpreterError> {
+        let mut index = 0;
+        let (mut value, tokens_consumed) = parse_unary_expression(&tokens[index..])?;
+        index += tokens_consumed;
+        while index < tokens.len() {
+            if tokens[index] != Token::Star && tokens[index] != Token::Slash {
+                break;
+            }
+            let operation = tokens[index].clone();
+            index += 1;
+            let (right, tokens_consumed) = parse_unary_expression(&tokens[index..])?;
+            index += tokens_consumed;
+            value = match operation {
+                Token::Star => AstNode::Multiply {
+                    left: Box::new(value),
+                    right: Box::new(right),
+                },
+                Token::Slash => AstNode::Divide {
+                    left: Box::new(value),
+                    right: Box::new(right),
+                },
+                _ => return Err(InterpreterError::ParseError("Impossible".to_owned())),
+            };
+        }
+        Ok((value, index))
+    }
+    fn parse_addition_expression(tokens: &[Token]) -> Result<(AstNode, usize), InterpreterError> {
+        let mut index = 0;
+        let (mut value, tokens_consumed) = parse_multiplication_expression(&tokens[index..])?;
+        index += tokens_consumed;
+        while index < tokens.len() {
+            if tokens[index] != Token::Plus && tokens[index] != Token::Minus {
+                break;
+            }
+            let operation = tokens[index].clone();
+            index += 1;
+            let (right, tokens_consumed) = parse_multiplication_expression(&tokens[index..])?;
+            index += tokens_consumed;
+            value = match operation {
+                Token::Plus => AstNode::Add {
+                    left: Box::new(value),
+                    right: Box::new(right),
+                },
+                Token::Minus => AstNode::Subtract {
+                    left: Box::new(value),
+                    right: Box::new(right),
+                },
+                _ => return Err(InterpreterError::ParseError("Impossible".to_owned())),
+            };
+        }
+        Ok((value, index))
+    }
+    fn parse_assign_expression(tokens: &[Token]) -> Result<(AstNode, usize), InterpreterError> {
+        let mut index = 0;
+        let (identifier, tokens_consumed) = parse_addition_expression(&tokens[index..])?;
+        index += tokens_consumed;
+        if index < tokens.len() && tokens[index] == Token::Equals {
+            index += 1;
+        } else {
+            return Ok((identifier, index));
+        }
+        let (value, tokens_consumed) = parse_addition_expression(&tokens[index..])?;
+        index += tokens_consumed;
+        Ok((
+            AstNode::Assign {
+                left: Box::new(identifier),
+                right: Box::new(value),
+            },
+            index,
+        ))
+    }
+    fn parse_expression(tokens: &[Token]) -> Result<(AstNode, usize), InterpreterError> {
+        parse_assign_expression(tokens)
+    }
+    let mut statements = vec![];
+    let mut index = 0;
+    loop {
+        if index >= tokens.len() {
+            break;
+        }
+        let (statement, tokens_consumed) = parse_expression(&tokens[index..])?;
+        statements.push(statement);
+        index += tokens_consumed;
+    }
+    Ok(AstNode::Program { statements })
+}

src/parser.ts

@@ -1,200 +0,0 @@
-import { AST, Num, Id, Not, Equal, NotEqual, Add, Subtract, Multiply, Divide, FunctionCall, Return, If, While, Var, Assign, Block, FunctionDefinition } from './ast.ts';
-import { Main, Assert } from './codegen.ts';
-
-export function parse(source: string): AST {
-  // Whitespace and comments.
-  let whitespace = Parser.regex(/[ \n\r\t]+/y);
-  let comments = Parser.regex(/[/][/].*/y).or(
-    Parser.regex(/[/][*].*[*][/]/sy)
-  );
-  let ignored = Parser.zeroOrMore(whitespace.or(comments));
-  // Tokens
-  let token = (pattern: RegExp) => Parser.regex(pattern).bind((value: any) => ignored.and(Parser.constant(value)));
-  let FUNCTION = token(/function\b/y);
-  let IF = token(/if\b/y);
-  let ELSE = token(/else\b/y);
-  let RETURN = token(/return\b/y);
-  let ASSIGN = token(/=/y).map(_ => Assign);
-  let VAR = token(/var\b/y);
-  let WHILE = token(/while\b/y);
-  let COMMA = token(/[,]/y);
-  let SEMICOLON = token(/[;]/y);
-  let LEFT_PAREN = token(/[(]/y);
-  let RIGHT_PAREN = token(/[)]/y);
-  let LEFT_BRACE = token(/[{]/y);
-  let RIGHT_BRACE = token(/[}]/y);
-  let NUMBER = token(/[0-9]/y).map((digits: any) => new Num(parseInt(digits)));
-  let ID = token(/[a-zA-Z_][a-zA-Z0-9_]*/y).map((x: any) => new Id(x));
-  let NOT = token(/!/y).map(_ => Not);
-  let EQUAL = token(/==/y).map(_ => Equal);
-  let NOT_EQUAL = token(/!=/y).map(_ => NotEqual);
-  let PLUS = token(/[+]/y).map(_ => Add);
-  let MINUS = token(/[-]/y).map(_ => Subtract);
-  let STAR = token(/[*]/y).map(_ => Multiply);
-  let SLASH = token(/[/]/y).map(_ => Divide);
-  // Expression parser
-  let expression: Parser<AST> = Parser.error('expression parser used before definition');
-  // Call parser
-  let args: Parser<Array<AST>> = expression.bind((arg: any) => Parser.zeroOrMore(COMMA.and(expression)).bind((args: any) => Parser.constant([arg, ...args]))).or(Parser.constant([]));
-  let functionCall: Parser<AST> = ID.bind((callee: any) => LEFT_PAREN.and(args.bind((args: any) => RIGHT_PAREN.and(Parser.constant(callee.equals(new Id('assert')) ? new Assert(args[0]) : new FunctionCall(callee, args))))));
-  // Atom
-  let atom: Parser<AST> = functionCall.or(ID).or(NUMBER).or(LEFT_PAREN.and(expression).bind((e: any) => RIGHT_PAREN.and(Parser.constant(e))));
-  // Unary operators
-  let unary: Parser<AST> = Parser.optional(NOT).bind((not: any) => atom.map((operand: any) => not ? new Not(operand) : operand));
-  // Infix operators
-  let infix = (operatorParser: any, operandParser: any) =>
-    operandParser.bind((operand: any) =>
-      Parser.zeroOrMore(
-        operatorParser.bind((operator: any) =>
-          operandParser.bind((operand: any) =>
-            Parser.constant({ operator, operand })
-          )
-        )
-      ).map((operatorTerms: any) =>
-        operatorTerms.reduce((left: any, { operator, operand }: { operator: any, operand: any }) =>
-         new operator(left, operand), operand)
-      )
-    );
-  let product = infix(STAR.or(SLASH), unary);
-  let sum = infix(PLUS.or(MINUS), product);
-  let comparison = infix(EQUAL.or(NOT_EQUAL), sum);
-  // Associativity
-  // Closing the loop: expression
-  expression.parse = comparison.parse;
-  // Statement
-  let statement: Parser<AST> = Parser.error('statement parser used before definition');
-  let returnStatement: Parser<AST> = RETURN.and(expression).bind((operand: any) => SEMICOLON.and(Parser.constant(new Return(operand))));
-  let expressionStatement: Parser<AST> = expression.bind((operand: any) => SEMICOLON.and(Parser.constant(operand)));
-  let ifStatement: Parser<AST> = IF.and(LEFT_PAREN).and(expression).bind((conditional: any) =>
-    RIGHT_PAREN.and(statement).bind((consequence: any) =>
-      ELSE.and(statement).bind((alternative: any) =>
-        Parser.constant(new If(conditional, consequence, alternative))
-      )
-    )
-  );
-  let whileStatement: Parser<AST> = WHILE.and(LEFT_PAREN).and(expression).bind((conditional: any) =>
-    RIGHT_PAREN.and(statement).bind((body: any) =>
-      Parser.constant(new While(conditional, body))
-    )
-  );
-  let varStatement: Parser<AST> = VAR.and(ID).bind((name: any) =>
-    ASSIGN.and(expression).bind((value: any) =>
-      SEMICOLON.and(Parser.constant(new Var(name, value)))
-    )
-  );
-  let assignmentStatement: Parser<AST> = ID.bind((name: any) =>
-    ASSIGN.and(expression).bind((value: any) =>
-      SEMICOLON.and(Parser.constant(new Assign(name, value)))
-    )
-  );
-  let blockStatement: Parser<AST> = LEFT_BRACE.and(Parser.zeroOrMore(statement)).bind((statements: any) =>
-    RIGHT_BRACE.and(Parser.constant(new Block(statements)))
-  );
-  let parameters: Parser<Array<string>> = ID.bind((param: any) =>
-    Parser.zeroOrMore(COMMA.and(ID)).bind((params: any) =>
-      Parser.constant([param, ...params])
-    )
-  ).or(Parser.constant([]));
-  let functionStatement: Parser<AST> = FUNCTION.and(ID).bind((name: any) =>
-    LEFT_PAREN.and(parameters).bind((parameters: any) =>
-      RIGHT_PAREN.and(blockStatement).bind((block: any) =>
-        Parser.constant(name.equals(new Id('main')) ? new Main(block.statements) : new FunctionDefinition(name, parameters, block))
-      )
-    )
-  );
-  let statementParser: Parser<AST> =
-    returnStatement
-      .or(functionStatement)
-      .or(ifStatement)
-      .or(whileStatement)
-      .or(varStatement)
-      .or(assignmentStatement)
-      .or(blockStatement)
-      .or(expressionStatement);
-  statement.parse = statementParser.parse;
-  let parser: Parser<AST> = ignored.and(Parser.zeroOrMore(statement)).map((statements: any) => new Block(statements));
-
-  return parser.parseStringToCompletion(source);
-}
-
-export class Parser<T> {
-  constructor(public parse: (src: Source) => (ParseResult<T> | null)) {}
-
-  parseStringToCompletion(string: string): T {
-    let source = new Source(string, 0);
-    let result = this.parse(source);
-    if (!result)
-      throw Error('Parse error at index 0');
-    let index = result.source.index;
-    if (index != result.source.string.length)
-      throw Error(`Parse error at index ${index}`);
-    return result.value;
-  }
-
-  static regex(regex: RegExp): Parser<string> {
-    return new Parser(source => source.match(regex));
-  }
-  static constant<U>(value: U): Parser<U> {
-    return new Parser(source => new ParseResult(value, source));
-  }
-  static error<U>(message: string): Parser<U> {
-    return new Parser(source => {
-      throw Error(message);
-    })
-  }
-  static zeroOrMore<U>(parser: Parser<U>): Parser<Array<U>> {
-    return new Parser(source => {
-      let results = [];
-      let item: any;
-      while (item = parser.parse(source)) {
-        source = item.source;
-        results.push(item.value);
-      }
-      return new ParseResult(results, source);
-    });
-  }
-  static optional<U>(parser: Parser<U>): Parser<U | null> {
-    return parser.or(Parser.constant(null));
-  }
-
-  or(parser: Parser<T>): Parser<T> {
-    return new Parser(source => {
-      let result = this.parse(source);
-      return result ? result : parser.parse(source);
-    });
-  }
-  bind<U>(callback: (value: T) => Parser<U>): Parser<U> {
-    return new Parser(source => {
-      let result = this.parse(source);
-      if (result) {
-        let value = result.value;
-        let source = result.source;
-        return callback(value).parse(source);
-      } else return null;
-    });
-  }
-  and<U>(parser: Parser<U>): Parser<U> {
-    return this.bind(_ => parser);
-  }
-  map<U>(callback: (t: T) => U): Parser<U> {
-    return this.bind(value => Parser.constant(callback(value)))
-  }
-}
-export class Source {
-  constructor(public string: string, public index: number) {}
-
-  match(regex: RegExp): (ParseResult<string> | null) {
-    regex.lastIndex = this.index;
-    let match = this.string.match(regex);
-    if (match) {
-      let value = match[0];
-      let newIndex = this.index + value.length;
-      let source = new Source(this.string, newIndex);
-      return new ParseResult(value, source);
-    }
-    return null;
-  }
-}
-export class ParseResult<T> {
-  constructor(public value: T, public source: Source) {}
-}

src/tokenize.rs

@@ -0,0 +1,164 @@
+use crate::InterpreterError;
+
+#[derive(Clone, PartialEq, Debug)]
+pub enum Token {
+    Integer(i64),
+    Float(f64),
+    String(String),
+    Boolean(bool),
+    Identifier(String),
+    Keyword(String),
+    Comma,
+    Plus,
+    Minus,
+    Star,
+    Slash,
+    Bang,
+    Equals,
+    Semicolon,
+    Quote(char),
+    Parenthesis { closing: bool },
+    Whitespace(String),
+    Unknown,
+}
+
+pub fn tokenize(source: &str) -> Result<Vec<Token>, InterpreterError> {
+    fn tokenize_number(chars: &[char]) -> Result<(Token, usize), ()> {
+        let mut current_value = String::new();
+        let mut chars_consumed = 0;
+        for c in chars {
+            if !c.is_digit(10) && *c != '.' {
+                break;
+            }
+            current_value.push(*c);
+            chars_consumed += 1;
+        }
+        if chars_consumed == 0 {
+            Err(())
+        } else if current_value.contains('.') {
+            Ok((
+                Token::Float(current_value.parse::<f64>().unwrap()),
+                chars_consumed,
+            ))
+        } else {
+            Ok((
+                Token::Integer(current_value.parse::<i64>().unwrap()),
+                chars_consumed,
+            ))
+        }
+    }
+    fn tokenize_identifier(chars: &[char]) -> Result<(Token, usize), ()> {
+        let mut current_value = String::new();
+        let mut chars_consumed = 0;
+        if chars[chars_consumed].is_alphabetic() {
+            current_value.push(chars[chars_consumed]);
+        } else {
+            return Err(());
+        }
+        chars_consumed += 1;
+        while chars_consumed < chars.len()
+            && (chars[chars_consumed].is_alphanumeric() || chars[chars_consumed] == '_')
+        {
+            current_value.push(chars[chars_consumed]);
+            chars_consumed += 1;
+        }
+        match &current_value[..] {
+            "true" => Ok((Token::Boolean(true), 4)),
+            "false" => Ok((Token::Boolean(false), 5)),
+            "let" => Ok((Token::Keyword(current_value), chars_consumed)),
+            _ => Ok((Token::Identifier(current_value), chars_consumed)),
+        }
+    }
+    fn tokenize_string(chars: &[char]) -> Result<(Token, usize), ()> {
+        let mut current_value = String::new();
+        let mut chars_consumed = 0;
+        fn is_quote(c: char) -> bool {
+            matches!(c, '\'' | '"' | '`')
+        }
+        let start_quote = if is_quote(chars[chars_consumed]) {
+            chars[chars_consumed]
+        } else {
+            return Err(());
+        };
+        chars_consumed += 1;
+        while chars_consumed < chars.len() && chars[chars_consumed] != start_quote {
+            current_value.push(chars[chars_consumed]);
+            chars_consumed += 1;
+        }
+        chars_consumed += 1;
+        Ok((Token::String(current_value), chars_consumed))
+    }
+    fn tokenize_whitespace(chars: &[char]) -> Result<(Token, usize), ()> {
+        let mut current_value = String::new();
+        let mut chars_consumed = 0;
+        for c in chars {
+            if !c.is_whitespace() {
+                break;
+            }
+            chars_consumed += 1;
+            current_value.push(*c);
+        }
+        if chars_consumed == 0 {
+            Err(())
+        } else {
+            Ok((Token::Whitespace(current_value), chars_consumed))
+        }
+    }
+    fn tokenize_operator(chars: &[char]) -> Result<(Token, usize), ()> {
+        if chars.is_empty() {
+            Err(())
+        } else if chars[0] == '+' {
+            Ok((Token::Plus, 1))
+        } else if chars[0] == '-' {
+            Ok((Token::Minus, 1))
+        } else if chars[0] == '*' {
+            Ok((Token::Star, 1))
+        } else if chars[0] == '/' {
+            Ok((Token::Slash, 1))
+        } else if chars[0] == '!' {
+            Ok((Token::Bang, 1))
+        } else if chars[0] == '=' {
+            Ok((Token::Equals, 1))
+        } else {
+            Err(())
+        }
+    }
+
+    let source = source.chars().collect::<Vec<char>>();
+    let mut tokens = vec![];
+    let mut index = 0;
+    while index < source.len() {
+        if let Ok((_whitespace, chars_consumed)) = tokenize_whitespace(&source[index..]) {
+            // Ignore whitespace
+            index += chars_consumed;
+        } else if let Ok((num, chars_consumed)) = tokenize_number(&source[index..]) {
+            tokens.push(num);
+            index += chars_consumed;
+        } else if let Ok((num, chars_consumed)) = tokenize_string(&source[index..]) {
+            tokens.push(num);
+            index += chars_consumed;
+        } else if let Ok((num, chars_consumed)) = tokenize_identifier(&source[index..]) {
+            tokens.push(num);
+            index += chars_consumed;
+        } else if let Ok((operator, chars_consumed)) = tokenize_operator(&source[index..]) {
+            tokens.push(operator);
+            index += chars_consumed;
+        } else if source[index] == ',' {
+            tokens.push(Token::Comma);
+            index += 1;
+        } else if source[index] == ';' {
+            tokens.push(Token::Semicolon);
+            index += 1;
+        } else if source[index] == '(' {
+            tokens.push(Token::Parenthesis { closing: false });
+            index += 1;
+        } else if source[index] == ')' {
+            tokens.push(Token::Parenthesis { closing: true });
+            index += 1;
+        } else {
+            // Skip if things fail
+            index += 1;
+        }
+    }
+    Ok(tokens)
+}

test.pvt

@@ -1 +1 @@
-let x = {2 + 2};
+log("Hello world!")

tsconfig.json

@@ -1,7 +0,0 @@
-{
-  "compilerOptions": {
-    "strictNullChecks": false
-  },
-  "include": ["src/**/*"],
-  "exclude": ["node_modules"]
-}