Add package data, member access, pre/postfix operators work now.

bin/pivot.js

@@ -1,3 +1,29 @@
 #!/usr/bin/env node
 const args = process.argv.slice(2);
-console.log(args);
+const tokenizer = require('../src/tokenizer.js');
+const parser = require('../src/parser.js');
+
+if (typeof args[0] != 'undefined') {
+  // Execute from file.
+} else { // REPL.
+  const rl = require('readline-sync');
+  const exec = require('child_process').exec;
+  function repl(prompt, func) {
+    let answer;
+    while (answer != 'exit') {
+      answer = rl.question(prompt);
+      if (answer == 'exit')
+        process.exit(0);
+      if (answer == 'clear') {
+        console.clear();
+        continue;
+      }
+      func(answer);
+    }
+  }
+  console.log('Welcome to Pivot v0.1.0 Alpha.');
+  console.log('Type \'exit\' to exit.');
+  repl('> ', (answer) => {
+    console.log(require('util').inspect(parser.parse(tokenizer.tokenize(answer)), { depth: null }));
+  });
+}

package.json

@@ -1,6 +1,6 @@
 {
-  "name": "pivot",
+  "name": "pivot-lang",
-  "version": "1.0.0",
+  "version": "0.1.0",
   "description": "Pivot is a new programming language built on JavaScript",
   "main": "./bin/pivot.js",
   "directories": {
@@ -31,7 +31,8 @@
   },
   "preferGlobal": true,
   "dependencies": {
-    "fs": "0.0.1-security"
+    "fs": "0.0.1-security",
+    "readline-sync": "^1.4.10"
   },
   "devDependencies": {
     "mocha": "^6.2.1"

src/parser.js

@@ -4,9 +4,8 @@
  * @author Garen Tyler <garentyler@gmail.com>
  * @requires module:types
  */
-const Token = require('./types.js').Token;
 const Group = require('./types.js').Group;
-const tokenizer = require('./tokenizer.js');
+const Operator = require('./types.js').Operator;
 
 /**
  * @function parse
@@ -23,10 +22,12 @@ function parse(tokens) {
   ast = addIndexes(ast);
   ast = addLevels(ast);
 
-  // Combine the groups.
+  // Start grouping by precedence
-  ast = combineGroups(ast);
+  ast = grouping(ast);
-
+  ast = memberAccess(ast);
-  //
+  ast = postfixOperators(ast);
+  ast = prefixOperators(ast);
+  console.log(ast);
 
   return ast;
 }
@@ -84,16 +85,15 @@ function getDeepestLevel(tokens) {
 }
 
 /**
- * @function combineGroups
+ * @function grouping
  * @desc Combine groups of tokens by delimiter.
  * @param {Token[]} tokens The tokens.
  * @returns {Token[]} The grouped tokens, or the basic ast.
  * @private
  */
-function combineGroups(tokens) {
+function grouping(tokens) {
   // Get the deepest level.
   let deepestLevel = getDeepestLevel(tokens);
-
   // Loop through for each level.
   for (let currentLevel = deepestLevel; currentLevel > 0; currentLevel--) {
     let groupBuffer = [];
@@ -107,8 +107,10 @@ function combineGroups(tokens) {
           let g = new Group(groupBuffer[0].value, groupBuffer);
           g.index = g.tokens[0].index;
           g.level = g.tokens[0].level - 1; // -1 because the group is on the level below.
-          tokens.splice(g.tokens[0].index, g.tokens.length + 1, g);
+          tokens.splice(g.tokens[0].index, g.tokens.length, g);
           j = g.tokens[0].index;
+          // Remove the delimiters in g.tokens.
+          g.tokens = g.tokens.splice(1, groupBuffer.length - 2);
           groupBuffer = [];
         }
       }
@@ -117,14 +119,88 @@ function combineGroups(tokens) {
   return tokens;
 }
 
+/**
+ * @function memberAccess
+ * @desc Combine groups of tokens by member access.
+ * @param {Token[]} tokens The tokens.
+ * @returns {Token[]} The grouped tokens, or the basic ast.
+ * @private
+ */
+function memberAccess(ast) {
+  for (let i = 0; i < ast.length; i++) {
+    if (ast[i].type == 'group')
+      memberAccess(ast[i].tokens); // Recursively order the groups.
+    else if (ast[i].type == 'operator' && ast[i].value == '.') { // Member access operator.
+      if (typeof ast[i - 1] == 'undefined' || typeof ast[i + 1] == 'undefined')
+        throw new SyntaxError('Operator requires two operands.');
+      let op = new Operator(ast[i].subtype, ast[i].value, [ast[i - 1], ast[i + 1]]);
+      op.index = ast[i - 1].index;
+      op.level = ast[i].level;
+      ast.splice(i - 1, 3, op);
+      i--; // Removed 3 tokens, put in 1, skip 1 token. Reduce the counter by 1.
+    }
+  }
+  return ast;
+}
+
+/**
+ * @function postfixOperators
+ * @desc Recursively structures the postfix operators.
+ * @param {Token[]} ast The ast.
+ * @returns {Token[]} The ast with structured postfix operators.
+ * @private
+ */
+function postfixOperators(ast) {
+  for (let i = 0; i < ast.length; i++) {
+    // Take care of the tokens in the groups.
+    if (ast[i].type == 'group')
+      ast[i].tokens = postfixOperators(ast[i].tokens);
+    else if (ast[i].type == 'operator' && ast[i].subtype == 'postfix') { // The operand is on the left.
+      if (typeof ast[i - 1] == 'undefined')
+        throw new SyntaxError('Postfix operator requires one operand before it.');
+      let op = new Operator(ast[i].subtype, ast[i].value, [ast[i - 1]]);
+      op.index = ast[i].index;
+      op.level = ast[i].level;
+      ast.splice(i - 1, 2, op);
+      // Removing 2 tokens, adding 1, skip 1 token. Don't reduce the counter.
+    }
+  }
+  return ast;
+}
+
+/**
+ * @function prefixOperators
+ * @desc Recursively structures the prefix operators.
+ * @param {Token[]} ast The ast.
+ * @returns {Token[]} The ast with structured prefix operators.
+ * @private
+ */
+function prefixOperators(ast) {
+  for (let i = 0; i < ast.length; i++) {
+    // Take care of the tokens in the groups.
+    if (ast[i].type == 'group')
+      ast[i].tokens = postfixOperators(ast[i].tokens);
+    else if (ast[i].type == 'operator' && ast[i].subtype == 'prefix') { // The operand is on the right.
+      if (typeof ast[i + 1] == 'undefined')
+        throw new SyntaxError('Prefix operator requires one operand after it.');
+      let op = new Operator(ast[i].subtype, ast[i].value, [ast[i + 1]]);
+      op.index = ast[i].index;
+      op.level = ast[i].level;
+      ast.splice(i, 2, op);
+      // Removing 2 tokens, adding 1, skip 1 token. Don't reduce the counter.
+    }
+  }
+  return ast;
+}
+
 module.exports = {
   parse,
   util: {
     addIndexes,
     addLevels,
     getDeepestLevel,
-    combineGroups
+    grouping,
   }
 };
 
-require('fs').writeFileSync('ast.json', JSON.stringify(parse(tokenizer.tokenize('let x = (5 + (6 * 2));')), null, 2), () => {});
+// require('fs').writeFileSync('ast.json', JSON.stringify(parse(tokenizer.tokenize('let x = (5 + (6 * 2)) - 7;')), null, 2), () => {});

src/tokenizer.js

@@ -5,7 +5,6 @@
  * @requires module:types
  */
 const Token = require('./types.js').Token;
-const Group = require('./types.js').Group;
 
 /**
  * @function tokenize
@@ -234,21 +233,64 @@ function getDelimiterToken(delimiter) {
 
 /**
  * @function operatorType
- * @desc Determines the type of operator.
+ * @desc Turns a delimiter char into a token.
- * @param {string} operator The operator char.
+ * @param {string} delimiter The delimiter char.
- * @returns {string} The type of operator.
+ * @returns {Token} The delimiter token.
  * @private
  */
-function operatorType(operator) {
+function operatorType(op) {
-  // Left operators have parameters on the left.
+  switch (op) {
-  if (/\+\+|--/.test(operator))
+    case '.':
-    return 'left';
+    case '+':
-  else if (false)
+    case '-':
-    return 'right';
+    case '*':
-  else if (/\;/.test(operator))
+    case '/':
-    return 'none';
+    case '**':
-  else
+    case '%':
+    case '<<':
+    case '>>':
+    case '<':
+    case '<=':
+    case '>':
+    case '>=':
+    case '==':
+    case '!=':
+    case '&':
+    case '|':
+    case '^':
+    case '&&':
+    case '||':
+    case '^^':
+    case '=':
+    case '+=':
+    case '-=':
+    case '*=':
+    case '/=':
+    case '**=':
+    case '%=':
+    case '<<=':
+    case '>>=':
+    case '&=':
+    case '|=':
+    case '^=':
       return 'dual';
+      break;
+    case '++':
+    case '--':
+      return 'postfix';
+      break;
+    case '!':
+    case '~':
+      return 'prefix';
+      break;
+    case ',':
+    case ';':
+      return 'none';
+      break;
+    default:
+      throw new TypeError('Unexpected operator ' + op);
+      break;
+  }
 }
 
 /**
@@ -261,7 +303,8 @@ function operatorType(operator) {
 function determineCharType(char) {
   if (/[A-Za-z]/.test(char))
     return 'letter';
-  else if (/\+|\-|\*|\/|\=|\=\=|\>|\<|\>\=|\<\=|\=\>|;/.test(char))
+  else if (/\.|\+\+|--|!|~|\+|-|\*\*|\*|\/%|<<|>>|<|<=|>|>=|==|!=|&|\^|\||&&|\|\||=|\+=|-=|\*\*=|\*=|\/=|%=|<<=|>>=|&=|\^=|\|=|,|;/.test(char))
+    // All the operators in Pivot.
     return 'operator';
   else if (/\(|\)|\[|\]|\{|\}/.test(char))
     return 'delimiter';
@@ -273,7 +316,7 @@ function determineCharType(char) {
     return 'escaped char';
   else if (/\s/.test(char))
     return 'whitespace';
-  else throw new SyntaxError('Unexpected char ' + char);
+  else throw new TypeError('Unexpected char ' + char);
 };
 
 /**

src/types.js

@@ -4,7 +4,6 @@
  * @author Garen Tyler <garentyler@gmail.com>
  */
 
-
 /**
  * @class Token
  * @classdesc Stores the type of token, subtype, and literal char value.
@@ -25,7 +24,19 @@ function Group(type, tokens) {
   this.tokens = tokens;
 }
 
+/**
+ * @class Operator
+ * @classdesc Stores the type of operator, and tokens for an operator.
+ */
+function Operator(subtype, value, operands) {
+  this.type = 'operator';
+  this.subtype = subtype;
+  this.value = value;
+  this.operands = operands;
+}
+
 module.exports = {
   Token,
-  Group
+  Group,
+  Operator
 };

test/test.js

@@ -13,6 +13,9 @@ describe('types.js', () => {
   it('Has a Group child.', () => {
     assert.equal(types.hasOwnProperty('Group'), true);
   });
+  it('Has a Operator child.', () => {
+    assert.equal(types.hasOwnProperty('Operator'), true);
+  });
 
   describe('Token', () => {
     it('Works as a constructor', () => {
@@ -54,6 +57,26 @@ describe('types.js', () => {
       }
     });
   });
+  // describe('Operator', () => {
+  //   it('Works as a constructor', () => {
+  //     try {
+  //       let op = new types.Operator('dual', '+', ['3','4']);
+  //     } catch (err) {
+  //       throw err;
+  //     }
+  //   });
+  //   it('Has values \'type\', \'subtype\', \'value\', and \'operands\'', () => {
+  //     try {
+  //       let op = new types.Operator('dual', '+', ['3','4']);
+  //       if (!op.hasOwnProperty('type') || !op.hasOwnProperty('subtype') || !op.hasOwnProperty('value') || !op.hasOwnProperty('operands'))
+  //         throw new Error('Operator is missing \'type\', \'subtype\', \'value\', or \'operands\' properties.');
+  //       if (op.type != 'operator' || op.subtype != 'dual' || op.value != '+' || op.operands[0] != '3')
+  //         throw new Error('Operator incorrectly set \'type\', \'subtype\', \'value\', or \'operands\' properties.');
+  //     } catch (err) {
+  //       throw err;
+  //     }
+  //   });
+  // });
 });
 describe('tokenizer.js', () => {
   it('combineEscapedChars works', () => {
@@ -101,7 +124,7 @@ describe('tokenizer.js', () => {
       throw new Error('Incorrect value: Expected \'parenthesis\' but got \'' + token.value + '\'')
   });
   it('operatorType works', () => {
-    assert.equal(tokenizer.util.operatorType('++'), 'left');
+    assert.equal(tokenizer.util.operatorType('++'), 'postfix');
     assert.equal(tokenizer.util.operatorType(';'), 'none');
     assert.equal(tokenizer.util.operatorType('+'), 'dual');
   });