garentyler/pivot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'bleeding-edge' into stable

Browse Source
This commit is contained in:
ElementG9 2020-06-12 17:06:38 -06:00
commit be6f0b1b93
17 changed files with 1443 additions and 365 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.gitignore vendored Normal file
View File

@ -0,0 +1,5 @@
other/
package-lock.json
node_modules/
jsdoc/
ast.json

4
LICENSE
View File

@ -1,6 +1,6 @@
MIT License MIT License
Copyright (c) 2019 ElementG9 Copyright (c) 2019 Garen Tyler
Permission is hereby granted, free of charge, to any person obtaining a copy Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal of this software and associated documentation files (the "Software"), to deal
@ -18,4 +18,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE. SOFTWARE.

55
ast.json
View File

@ -1,55 +0,0 @@
[
{
"type": "Variable",
"value": "asdf",
"layer": 0
},
{
"type": "Operator",
"value": "=",
"layer": 0
},
{
"type": "Group",
"value": "{",
"tokens": [
{
"type": "Left Delimiter",
"value": "{",
"layer": 0
}
]
},
{
"type": "Group",
"value": "a",
"tokens": [
{
"type": "Variable",
"value": "a",
"layer": 2
},
{
"type": "Operator",
"value": "++",
"layer": 2
}
]
},
{
"type": "Group",
"value": "asdf",
"tokens": [
{
"type": "Variable",
"value": "asdf",
"layer": 0
},
{
"type": "Operator",
"value": "=",
"layer": 0
}
]
}
]

36
bin/pivot.js Executable file
View File

@ -0,0 +1,36 @@
#!/usr/bin/env node
const args = process.argv.slice(2);
const tokenizer = require('../src/tokenizer.js');
const parser = require('../src/parser.js');
const code = require('../src/code.js');
if (typeof args[0] != 'undefined') {
// Execute from file.
} else { // REPL.
const rl = require('readline-sync');
const exec = require('child_process').exec;
function repl(prompt, func) {
let answer;
while (answer != 'exit') {
answer = rl.question(prompt);
if (answer == 'exit')
process.exit(0);
if (answer == 'clear') {
console.clear();
continue;
}
func(answer);
}
}
console.log('Welcome to Pivot v0.2.0 Alpha.');
console.log('Type \'exit\' to exit.');
let data = {
log: "console.log"
};
repl('> ', (answer) => {
let jsAnswer = code.translate(parser.parse(tokenizer.tokenize(answer)), data);
data = jsAnswer.data;
// console.log(require('util').inspect(jsAnswer, { depth: null }));
eval(jsAnswer.code);
});
}

20
classes.js
View File

@ -1,20 +0,0 @@
// This is the most basic type of token.
var token = function (type, value) {
this.type = type;
this.value = value;
};
// This is a group of tokens.
var group = function (type, tokens) {
this.type = "Group";
this.value = type;
this.tokens;
if(typeof tokens != "undefined") {
this.tokens = tokens;
} else {
this.tokens = [];
}
}
module.exports = {
token: token,
group: group
}

82
doc/operators.txt Normal file
View File

@ -0,0 +1,82 @@
A higher precedence operator becomes an operand for a lower precedence one.
Higher precedence operators get resolved first.
Operators in Pivot:
+-------------+------------+----------------------------------+--------------------+-----------------+
| Precedence | Operator | Description | Operands | Associativity |
+-------------+------------+----------------------------------+--------------------+-----------------+
| 16 | () | Grouping | internal | n/a | grouping()
+-------------+------------+----------------------------------+--------------------+-----------------+
| 15 | . | Member Access | dual | Left to Right | memberAccess()
| 15 | [] | Computed Member Access | before, internal | Left to Right | computedMemberAccess()
| 15 | () | Function Call | before, internal | Left to Right | functionCall()
+-------------+------------+----------------------------------+--------------------+-----------------+
| 14 | let | Variable Creation | after | Right to Left | keywords()
| 14 | const | Constant Creation | after | Right to Left | keywords()
| 14 | new | Object Creation | after | Right to Left | keywords()
| 14 | return | Function Return | after | n/a | keywords()
| 14 | () {} | Function Creation | internal | n/a | functionCreation()
+-------------+------------+----------------------------------+--------------------+-----------------+
| 13 | ++ | Postfix Increment | before | Left to Right | postfixOperators()
| 13 | -- | Postfix Decrement | before | Left to Right | postfixOperators()
+-------------+------------+----------------------------------+--------------------+-----------------+
| 12 | ! | Logical NOT | after | Right to Left | prefixOperators()
| 12 | - | Unary Negation | after | Right to Left | prefixOperators()
+-------------+------------+----------------------------------+--------------------+-----------------+
| 11 | ** | Exponentiation | dual | Right to Left | mathOperators(0)
+-------------+------------+----------------------------------+--------------------+-----------------+
| 10 | * | Multiplication | dual | Left to Right | mathOperators(1)
| 10 | / | Division | dual | Left to Right | mathOperators(1)
| 10 | % | Modulus | dual | Left to Right | mathOperators(1)
+-------------+------------+----------------------------------+--------------------+-----------------+
| 9 | + | Addition | dual | Left to Right | mathOperators(2)
| 9 | - | Subtraction | dual | Left to Right | mathOperators(2)
+-------------+------------+----------------------------------+--------------------+-----------------+
| 7 | < | Less Than | dual | Left to Right | comparisonOperators()
| 7 | <= | Less Than or Equal | dual | Left to Right | comparisonOperators()
| 7 | > | Greater Than | dual | Left to Right | comparisonOperators()
| 7 | >= | Greater Than or Equal | dual | Left to Right | comparisonOperators()
+-------------+------------+----------------------------------+--------------------+-----------------+
| 6 | == | Equality | dual | Left to Right | assign()
| 6 | != | Inequality | dual | Left to Right | assign()
+-------------+------------+----------------------------------+--------------------+-----------------+
| 4 | && | Logical AND | dual | Left to Right | logicOperators()
| 4 | ^^ | Logical XOR | dual | Left to Right | logicOperators()
| 4 | || | Logical OR | dual | Left to Right | logicOperators()
+-------------+------------+----------------------------------+--------------------+-----------------+
| 3 | = | Assignment | dual | Right to Left | opAssign()
| 3 | += | Add and Assign | dual | Right to Left | opAssign()
| 3 | -= | Subtract and Assign | dual | Right to Left | opAssign()
| 3 | **= | Exponentiate and Assign | dual | Right to Left | opAssign()
| 3 | *= | Multiply and Assign | dual | Right to Left | opAssign()
| 3 | /= | Divide and Assign | dual | Right to Left | opAssign()
| 3 | %= | Modulo and Assign | dual | Right to Left | opAssign()
+-------------+------------+----------------------------------+--------------------+-----------------+
| 2 | , | Comma | none | Left to Right |
+-------------+------------+----------------------------------+--------------------+-----------------+
| 1 | ; | Statement End | before | Left to Right |
+-------------+------------+----------------------------------+--------------------+-----------------+
Possible Operators in Pivot:
+-------------+------------+----------------------------------+--------------------+-----------------+
| Precedence | Operator | Description | Operands | Associativity |
+-------------+------------+----------------------------------+--------------------+-----------------+
| 12 | ~ | Bitwise NOT | after | Right to Left |
+-------------+------------+----------------------------------+--------------------+-----------------+
| 8 | << | Bitwise Left Shift | dual | Left to Right |
| 8 | >> | Bitwise Right Shift | dual | Left to Right |
+-------------+------------+----------------------------------+--------------------+-----------------+
| 5 | and | Logical AND | dual | Left to Right |
| 5 | xor | Logical XOR | dual | Left to Right |
| 5 | or | Logical OR | dual | Left to Right |
+-------------+------------+----------------------------------+--------------------+-----------------+
| 5 | & | Bitwise AND | dual | Left to Right |
| 5 | ^ | Bitwise XOR | dual | Left to Right |
| 5 | | | Bitwise OR | dual | Left to Right |
+-------------+------------+----------------------------------+--------------------+-----------------+
| 3 | <<= | Bitwise Left Shift and Assign | dual | Right to Left |
| 3 | >>= | Bitwise Right Shift and Assign | dual | Right to Left |
| 3 | &= | Bitwise AND and Assign | dual | Right to Left |
| 3 | ^= | Bitwise XOR and Assign | dual | Right to Left |
| 3 | |= | Bitwise OR and Assign | dual | Right to Left |
+-------------+------------+----------------------------------+--------------------+-----------------+

40
package.json Normal file
View File

@ -0,0 +1,40 @@
{
"name": "pivot-lang",
"version": "0.2.0",
"description": "Pivot is a new programming language built on JavaScript",
"main": "./bin/pivot.js",
"directories": {
"test": "test"
},
"scripts": {
"test": "mocha",
"build": "npm run test; jsdoc src/ -r -d ./jsdoc/ -R ./readme.md"
},
"repository": {
"type": "git",
"url": "git+https://github.com/ElementG9/Pivot.git"
},
"keywords": [
"pivot",
"javascript",
"programming",
"language"
],
"author": "Garen Tyler <garentyler@gmail.com>",
"license": "MIT",
"bugs": {
"url": "https://github.com/ElementG9/Pivot/issues"
},
"homepage": "https://github.com/ElementG9/Pivot#readme",
"bin": {
"pivot": "./bin/pivot.js"
},
"preferGlobal": true,
"dependencies": {
"fs": "0.0.1-security",
"readline-sync": "^1.4.10"
},
"devDependencies": {
"mocha": "^6.2.2"
}
}

59
parser.js
View File

@ -1,59 +0,0 @@
// Import the group class.
const group = require("./classes.js").group;
// Create the parser function.
// parse() takes an array of tokens in, and outputs an
// Abstract Syntax Tree (a structured array of tokens).
module.exports = tokens => {
// Variables for later.
var layer = 0;
var delimiterCount = 0;
var deepestLayer = 0;
// Give each token a layer number based on delimiters.
for (var i = 0; i < tokens.length; i++) {
if (tokens[i].type == "Left Delimiter") {
layer++;
if(layer > deepestLayer) {
deepestLayer = layer;
}
delimiterCount++;
}
tokens[i].layer = layer;
if (tokens[i].type == "Right Delimiter") {
layer--;
}
}
// Lower the layer of delimiters.
for (var i = 0; i < tokens.length; i++) {
if ((tokens[i].type == "Left Delimiter") || (tokens[i].type == "Right Delimiter")) {
tokens[i].layer--;
}
}
if (layer > 0) { // Unclosed delimiter.
} else if (layer < 0) { // Overclosed delimiter.
}
// Reset layer for structuring.
layer = 0;
/* - - - DO NOT TOUCH THIS - - - */
for(var i=deepestLayer;i>=0;i--) {
var temp = [];
var firstIndex;
for(var j=0;j<tokens.length;j++) {
if(tokens[j].layer == i) {
if(temp.length <= 0) {
firstIndex = j;
}
temp.push(tokens[j]);
} else {
if(temp.length > 0) {
var g = new group(tokens[firstIndex].value,temp);
tokens.splice(firstIndex-1,temp.length+2,g);
temp = [];
}
}
}
}
/* - - - OK YOU CAN TOUCH AGAIN - - - */
// Return the structured tokens.
return tokens;
};

13
pivot.js
View File

@ -1,13 +0,0 @@
// The Pivot code to run.
var code = `asdf = (a) {return(a++)}`;
// Import the tokenizer and parser.
const tokenize = require("./tokenizer.js");
const parse = require("./parser.js");
// Generate the AST.
var ast = parse(tokenize(code));
// Write the AST to ast.json.
var fs = require("fs");
fs.writeFileSync("ast.json", JSON.stringify(ast, null, 4));

40
rpn.js
View File

@ -1,40 +0,0 @@
var solve = (exp) => {
var stack = [];
var expression = exp.split(" ");
for(var j=0;j<expression.length;j++) {
var key = expression[j];
if(key.match(/\d/)) {
stack.push(parseInt(key));
} else if(key.match(/\w/)) {
if(Object.keys(progData).includes(key)) {
stack.push(progData[key]);
} else {
stack.push(key);
}
}
switch(key) {
case "+": // add
var opItems = stack.splice(stack.length-2,2);
var result = opItems[0]+opItems[1];
stack.push(result);
break;
case "-": // subtract
var opItems = stack.splice(stack.length-2,2);
var result = opItems[0]-opItems[1];
stack.push(result);
break;
case "*": // multiply
var opItems = stack.splice(stack.length-2,2);
var result = opItems[0]*opItems[1];
stack.push(result);
break;
case "/": // divide
var opItems = stack.splice(stack.length-2,2);
var result = opItems[0]/opItems[1];
stack.push(result);
break;
}
}
return stack;
};
module.exports = solve;

41
src/code.js Normal file
View File

@ -0,0 +1,41 @@
/**
* @module code
* @file Runs the code / transpiles the code to JavaScript
* @author Garen Tyler <garentyler@gmail.com>
*/
/**
* @function translate
* @desc Translates the code to JS, given an AST
* @param {Token[]} ast The ast.
* @returns {String} The JS code.
* @public
*/
function translate(ast, data) {
let out = '';
for (let i = 0; i < ast.length; i++) {
if (ast[i].type == 'operator' && ast[i].subtype == 'function call') {
let temp = '';
if (!(Object.keys(data).indexOf(ast[i].operands[0].value) > -1))
throw new ReferenceError(`Undefined function ${ast[i].operands[0].value}`);
else temp += data[ast[i].operands[0].value];
temp += '(';
for (let j = 0; j < ast[i].operands[1].tokens.length; j++) {
if (j != 0)
temp += ', ';
if (ast[i].operands[1].tokens[j].type == 'string')
temp += `"${ast[i].operands[1].tokens[j].value}"`;
}
temp += ');'
out += temp;
}
}
return {
data,
code: out
};
}
module.exports = {
translate
};

421
src/parser.js Normal file
View File

@ -0,0 +1,421 @@
/**
* @module parser
* @file Manages the parsing phase of Pivot.
* @author Garen Tyler <garentyler@gmail.com>
* @requires module:types
*/
const Group = require('./types.js').Group;
const Operator = require('./types.js').Operator;
/**
* @function parse
* @desc Takes in an array of tokens, and outputs an AST.
* @param {Token[]} tokens The input tokens.
* @returns {Token[]} The tokens structured in an AST.
* @public
*/
function parse(tokens) {
// Create our output array.
let ast = tokens;
// Add indexes and levels.
ast = addIndexes(ast);
ast = addLevels(ast);
// Start grouping by precedence.
// Precedence 16.
ast = grouping(ast);
// Precedence 15.
ast = memberAccess(ast);
ast = computedMemberAccess(ast);
ast = functionCall(ast);
// Precedence 14.
ast = keywords(ast);
ast = functionCreation(ast);
// Precedence 13.
ast = postfixOperators(ast);
// Precedence 12.
ast = prefixOperators(ast);
// Precedence 11.
ast = mathOperators(ast, 0); // Level 0 math operators: **.
// Precedence 10.
ast = mathOperators(ast, 1); // Level 1 math operators: *, /, %.
// Precedence 9.
ast = mathOperators(ast, 2); // Level 2 math operators: +, -.
// Precedence 7.
// ast = comparisonOperators(ast);
// Precedence 6.
// ast = assign(ast);
// Precedence 4.
// ast = logicOperators(ast);
// Precedence 3.
// ast = opAssign(ast);
return ast;
}
/**
* @function addIndexes
* @desc Adds basic indexes to the tokens.
* @param {Token[]} tokens The tokens.
* @returns {Token[]} The tokens with indexes.
* @private
*/
function addIndexes(tokens) {
return tokens.map((t, i) => {
t.index = i;
return t;
});
}
/**
* @function addLevels
* @desc Adds basic levels to the tokens. The levels are dependent on the delimiters.
* @param {Token[]} tokens The tokens.
* @returns {Token[]} The tokens with levels.
* @private
*/
function addLevels(tokens) {
let level = 0;
tokens = tokens.map((t, i) => {
if (t.type == 'delimiter' && t.subtype == 'left')
level++;
t.level = level;
if (t.type == 'delimiter' && t.subtype == 'right')
level--;
return t;
});
if (level > 0)
throw new SyntaxError('Missing closing delimiter');
else if (level < 0)
throw new SyntaxError('Missing opening delimiter');
else
return tokens;
}
/**
* @function getDeepestLevel
* @desc Finds the deepest level of the ast.
* @param {Token[]} tokens The tokens.
* @returns {Number} The deepest level.
* @private
*/
function getDeepestLevel(tokens) {
return tokens.reduce((deepest, t) => {
return t.level > deepest ? t.level : deepest;
}, 0);
}
/**
* @function grouping
* @desc Combine groups of tokens by delimiter.
* @param {Token[]} tokens The tokens.
* @returns {Token[]} The grouped tokens, or the basic ast.
* @private
*/
function grouping(tokens) {
// Get the deepest level.
let deepestLevel = getDeepestLevel(tokens);
let groupBuffer;
let opening;
// Group the deepest levels first.
for (let currentLevel = deepestLevel; currentLevel > 0; currentLevel--) {
groupBuffer = []; // Overwrite groupBuffer.
opening = null; // Overwrite opening.
for (let i = 0; i < tokens.length; i++) {
if (tokens[i].level == currentLevel) {
if (groupBuffer.length == 0)
opening = tokens[i];
groupBuffer.push(tokens[i]);
if (typeof tokens[i + 1] == 'undefined' || (tokens[i].type == 'delimiter' && tokens[i].subtype == 'right' && tokens[i].value == opening.value)) { // The end of the tokens.
let g = new Group(groupBuffer[0].value, groupBuffer);
g.index = g.tokens[0].index;
g.level = g.tokens[0].level - 1; // -1 because the group is on the level below.
let length = g.tokens.length; // Keep track of how many tokens there are before removing the delimiters.
g.tokens = g.tokens.splice(1, g.tokens.length - 2); // Remove the delimiters in g.tokens.
i -= length - 1; // Reset the counter.
tokens.splice(i, length, g); // Replace the tokens with the new group.
// Reset groupBuffer and opening.
groupBuffer = [];
opening = null;
}
}
}
}
return tokens;
}
/**
* @function memberAccess
* @desc Combine groups of tokens by member access.
* @param {Token[]} tokens The tokens.
* @returns {Token[]} The ast with grouped member access.
* @private
*/
function memberAccess(ast) {
for (let i = 0; i < ast.length; i++) {
if (ast[i].type == 'group')
ast[i].tokens = memberAccess(ast[i].tokens); // Recursively order the groups.
else if (ast[i].type == 'operator' && ast[i].value == '.') { // Member access operator.
if (typeof ast[i - 1] == 'undefined' || typeof ast[i + 1] == 'undefined')
throw new SyntaxError('Operator requires two operands.');
let op = new Operator(ast[i].subtype, ast[i].value, [ast[i - 1], ast[i + 1]]);
op.index = ast[i - 1].index;
op.level = ast[i].level;
ast.splice(i - 1, 3, op);
i--; // Removed 3 tokens, put in 1, skip 1 token. Reduce the counter by 1.
}
}
return ast;
}
/**
* @function computedMemberAccess
* @desc Combine groups of tokens by computed member access.
* @param {Token[]} tokens The tokens.
* @returns {Token[]} The ast with grouped computed member access.
* @private
*/
function computedMemberAccess(ast) {
// Computed member access is Variable, Bracket Group.
for (let i = 0; i < ast.length; i++) {
if (ast[i].type == 'group')
ast[i].tokens = computedMemberAccess(ast[i].tokens); // Recursively order the groups.
else if (ast[i].type == 'name' && ast[i].subtype == 'variable') { // Member access operator.
if (typeof ast[i + 1] == 'undefined')
continue; // Nothing after the variable; skip this loop.
if (ast[i + 1].type == 'group' && ast[i + 1].subtype == 'bracket') {
ast[i + 1].tokens = computedMemberAccess(ast[i + 1].tokens); // Order the group that we care about before we mess with it.
let op = new Operator('n/a', 'member access', [ast[i], ast[i + 1]]);
op.index = ast[i].index;
op.level = ast[i].level;
ast.splice(i, 2, op);
// Removed 2 tokens, put in 1, skip 1 token. Don't reduce the counter.
} else continue; // Not what we need.
}
}
return ast;
}
/**
* @function functionCall
* @desc Combine groups of tokens by function calls.
* @param {Token[]} tokens The tokens.
* @returns {Token[]} The ast with grouped function calls.
* @private
*/
function functionCall(ast) {
// Function call is Variable, Parenthesis Group.
for (let i = 0; i < ast.length; i++) {
if (ast[i].type == 'group')
ast[i].tokens = functionCall(ast[i].tokens); // Recursively order the groups.
else if ((ast[i].type == 'name' && ast[i].subtype == 'variable') || // Normal function call
(ast[i].type == 'operator' && (ast[i].value == '.' || ast[i].value == 'member access'))) { // Function call in member access. Example: console.log()
if (typeof ast[i + 1] == 'undefined')
continue; // Nothing after the variable; skip this loop.
if (ast[i + 1].type == 'group' && ast[i + 1].subtype == 'parenthesis') {
ast[i + 1].tokens = functionCall(ast[i + 1].tokens); // Order the group that we care about before we mess with it.
let op = new Operator('function call', ast[i].value, [ast[i], ast[i + 1]]);
op.index = ast[i].index;
op.level = ast[i].level;
ast.splice(i, 2, op);
// Removed 2 tokens, put in 1, skip 1 token. Don't reduce the counter.
} else continue; // Not what we need.
}
}
return ast;
}
/**
* @function keywords
* @desc Combine groups of tokens by keywords.
* @param {Token[]} tokens The tokens.
* @returns {Token[]} The ast with grouped keywords.
* @private
*/
function keywords(ast) {
for (let i = ast.length - 1; i >= 0; i--) { // Keywords are rtl associative, so loop backwards.
if (ast[i].type == 'group')
ast[i].tokens = keywords(ast[i].tokens); // Recursively order the groups.
else if (ast[i].type == 'name' && ast[i].subtype == 'keyword') {
if (typeof ast[i + 1] == 'undefined')
throw new SyntaxError('Keyword requires one operand after it.');
let key = new Operator('keyword', ast[i].value, [ast[i + 1]]);
key.level = ast[i].level;
key.index = ast[i].index;
ast.splice(i, 2, key);
// Looping backwards and didn't remove any items before the current one. Don't reduce the counter.
}
}
return ast;
}
/**
* @function functionCreation
* @desc Combine groups of tokens by function creation.
* @param {Token[]} tokens The tokens.
* @returns {Token[]} The ast with grouped function creation.
* @private
*/
function functionCreation(ast) {
// Function call is Parenthesis Group, Brace Group.
for (let i = 0; i < ast.length; i++) {
if (ast[i].type == 'group')
ast[i].tokens = functionCreation(ast[i].tokens); // Recursively order the groups.
if (typeof ast[i + 1] == 'undefined')
continue; // Skip this loop.
if ((ast[i].type == 'group' && ast[i].subtype == 'parenthesis') && (ast[i + 1].type == 'group' && ast[i + 1].subtype == 'brace')) {
// Parenthesis group followed by brace group.
ast[i + 1].tokens = functionCreation(ast[i + 1].tokens); // Order the other group before we do anything.
let op = new Operator('function creation', 'n/a', [ast[i], ast[i + 1]]);
op.index = ast[i].index;
op.level = ast[i].level;
ast.splice(i, 2, op);
}
}
return ast;
}
/**
* @function postfixOperators
* @desc Recursively structures the postfix operators.
* @param {Token[]} ast The ast.
* @returns {Token[]} The ast with structured postfix operators.
* @private
*/
function postfixOperators(ast) {
for (let i = 0; i < ast.length; i++) {
// Take care of the tokens in the groups.
if (ast[i].type == 'group') {
if (ast[i].tokens.length > 0) {
ast[i].tokens = postfixOperators(ast[i].tokens);
}
} else if (ast[i].type == 'operator') {
if (typeof ast[i].operands != 'undefined') {
ast[i].operands = postfixOperators(ast[i].operands);
}
}
if (ast[i].type == 'operator' && ast[i].subtype == 'postfix') { // The operand is on the left.
if (typeof ast[i - 1] == 'undefined')
throw new SyntaxError('Postfix operator requires one operand before it.');
let op = new Operator(ast[i].subtype, ast[i].value, [ast[i - 1]]);
op.index = ast[i].index;
op.level = ast[i].level;
ast.splice(i - 1, 2, op);
// Removing 2 tokens, adding 1, skip 1 token. Don't reduce the counter.
}
}
return ast;
}
/**
* @function prefixOperators
* @desc Recursively structures the prefix operators.
* @param {Token[]} ast The ast.
* @returns {Token[]} The ast with structured prefix operators.
* @private
*/
function prefixOperators(ast) {
for (let i = ast.length - 1; i >= 0; i--) { // Prefix operators are rtl associative, so loop backwards.
// Take care of the tokens in the groups.
if (ast[i].type == 'group') {
if (ast[i].tokens.length > 0) {
ast[i].tokens = prefixOperators(ast[i].tokens);
}
} else if (ast[i].type == 'operator') {
if (typeof ast[i].operands != 'undefined') {
ast[i].operands = prefixOperators(ast[i].operands);
}
}
if (ast[i].type == 'operator' && ast[i].subtype == 'prefix') { // The operand is on the right.
if (typeof ast[i + 1] == 'undefined')
throw new SyntaxError('Prefix operator requires one operand after it.');
let op = new Operator(ast[i].subtype, ast[i].value, [ast[i + 1]]);
op.index = ast[i].index;
op.level = ast[i].level;
ast.splice(i, 2, op);
// Removing 2 tokens, adding 1, skip 1 token. Don't reduce the counter.
}
}
return ast;
}
/**
* @function mathOperators
* @desc Recursively structures the math operators.
* @param {Token[]} ast The ast.
* @param {Token[]} level The level of math to do. (Order of operations)
* @returns {Token[]} The ast with structured math operators.
* @private
*/
function mathOperators(ast, level) {
if (level == 0) { // Level 0 operators: **
for (let i = ast.length - 1; i >= 0; i--) { // Exponentiation is rtl associative, so loop backwards.
// Take care of the tokens in the groups.
if (ast[i].type == 'group') {
if (ast[i].tokens.length > 0) {
ast[i].tokens = mathOperators(ast[i].tokens, level);
}
} else if (ast[i].type == 'operator') {
if (typeof ast[i].operands != 'undefined') {
ast[i].operands = mathOperators(ast[i].operands, level);
}
}
if (ast[i].type == 'operator' && ast[i].value == '**') {
if (typeof ast[i - 1] == 'undefined' || typeof ast[i + 1] == 'undefined')
throw new SyntaxError('Dual operator requires two operands.');
let op = new Operator('dual', ast[i].value, [ast[i - 1], ast[i + 1]]);
op.index = ast[i].index;
op.level = ast[i].level;
ast.splice(i - 1, 3, op);
i--;
}
}
} else {
for (let i = 0; i < ast.length; i++) { // All other math operators are ltr associative.
// Take care of the tokens in the groups.
if (ast[i].type == 'group') {
if (ast[i].tokens.length > 0) {
ast[i].tokens = mathOperators(ast[i].tokens, level);
}
} else if (ast[i].type == 'operator') {
if (typeof ast[i].operands != 'undefined') {
ast[i].operands = mathOperators(ast[i].operands, level);
}
}
if (level == 1) {
if (ast[i].type == 'operator' && (ast[i].value == '*' || ast[i].value == '/' || ast[i].value == '%')) {
if (typeof ast[i - 1] == 'undefined' || typeof ast[i + 1] == 'undefined')
throw new SyntaxError('Dual operator requires two operands.');
let op = new Operator('dual', ast[i].value, [ast[i - 1], ast[i + 1]]);
op.index = ast[i].index;
op.level = ast[i].level;
ast.splice(i - 1, 3, op);
i--;
}
} else if (level == 2) {
if (ast[i].type == 'operator' && (ast[i].value == '+' || ast[i].value == '-')) {
if (typeof ast[i - 1] == 'undefined' || typeof ast[i + 1] == 'undefined')
throw new SyntaxError('Dual operator requires two operands.');
let op = new Operator('dual', ast[i].value, [ast[i - 1], ast[i + 1]]);
op.index = ast[i].index;
op.level = ast[i].level;
ast.splice(i - 1, 3, op);
i--;
}
}
}
}
return ast;
}
module.exports = {
parse,
util: {
addIndexes,
addLevels,
getDeepestLevel,
grouping,
}
};
// require('fs').writeFileSync('ast.json', JSON.stringify(parse(tokenizer.tokenize('let x = (5 + (6 * 2)) - 7;')), null, 2), () => {});

346
src/tokenizer.js Normal file
View File

@ -0,0 +1,346 @@
/**
* @module tokenizer
* @file Manages the tokenization phase of Pivot.
* @author Garen Tyler <garentyler@gmail.com>
* @requires module:types
*/
const Token = require('./types.js').Token;
/**
* @function tokenize
* @desc Takes in raw code, and outputs an array of Tokens.
* @param {string} code The raw input code.
* @returns {Token[]} The code, split into tokens.
* @public
*/
function tokenize(code) {
// Split the string into an array of chars.
let chars = code.split('');
// Create buffers.
let letterBuffer = [];
let operatorBuffer = [];
let numberBuffer = [];
let stringBuffer = [];
// Create the output Token[].
let tokens = [];
// Create an object to keep track of string data.
let stringData = {
inString: false,
stringType: null
};
// Escape chars and remove comments.
chars = combineEscapedChars(chars);
chars = removeComments(chars);
// Actually tokenize the chars.
for (let i = 0; i < chars.length; i++) {
let char = chars[i];
if (stringData.inString) { // Tokenize differently in a string.
// If a string delimiter and the same as the inital delimiter.
if (determineCharType(char) == 'string delimiter' && char == stringData.stringType) {
stringData.inString = false; // Not in a string any more.
tokens.push(new Token('string', 'n/a', stringBuffer.join(''))); // Push the string.
stringBuffer = []; // Clear the string buffer.
} else stringBuffer.push(char); // Add to the string buffer.
} else { // Tokenize normally.
if (determineCharType(char) == 'string delimiter') {
stringData.inString = true; // In a string now.
stringData.stringType = char;
} else if (determineCharType(char) == 'letter') {
letterBuffer.push(char); // Add to the letter buffer.
// End the other buffers.
if (operatorBuffer.length > 0) {
let operator = operatorBuffer.join('');
tokens.push(new Token('operator', operatorType(operator), operator));
operatorBuffer = [];
}
if (numberBuffer.length > 0) {
let number = numberBuffer.join('');
tokens.push(new Token('number', 'n/a', number));
numberBuffer = [];
}
} else if (determineCharType(char) == 'operator') {
operatorBuffer.push(char); // Add to the operator buffer.
// End the other buffers.
if (letterBuffer.length > 0) {
let variable = letterBuffer.join('');
tokens.push(new Token('name', 'variable', variable));
letterBuffer = [];
}
if (numberBuffer.length > 0) {
let number = numberBuffer.join('');
tokens.push(new Token('number', 'n/a', number));
numberBuffer = [];
}
} else if (determineCharType(char) == 'digit') {
numberBuffer.push(char); // Add to the number buffer.
// End the other buffers.
if (letterBuffer.length > 0) {
let variable = letterBuffer.join('');
tokens.push(new Token('name', 'variable', variable));
letterBuffer = [];
}
if (operatorBuffer.length > 0) {
let operator = operatorBuffer.join('');
tokens.push(new Token('operator', operatorType(operator), operator));
operatorBuffer = [];
}
} else if (determineCharType(char) == 'whitespace') {
// End all buffers.
if (letterBuffer.length > 0) {
let variable = letterBuffer.join('');
tokens.push(new Token('name', 'variable', variable));
letterBuffer = [];
}
if (numberBuffer.length > 0) {
let number = numberBuffer.join('');
tokens.push(new Token('number', 'n/a', number));
numberBuffer = [];
}
if (operatorBuffer.length > 0) {
let operator = operatorBuffer.join('');
tokens.push(new Token('operator', operatorType(operator), operator));
operatorBuffer = [];
}
} else if (determineCharType(char) == 'delimiter') {
// End all buffers.
if (letterBuffer.length > 0) {
let variable = letterBuffer.join('');
tokens.push(new Token('name', 'variable', variable));
letterBuffer = [];
}
if (numberBuffer.length > 0) {
let number = numberBuffer.join('');
tokens.push(new Token('number', 'n/a', number));
numberBuffer = [];
}
if (operatorBuffer.length > 0) {
let operator = operatorBuffer.join('');
tokens.push(new Token('operator', operatorType(operator), operator));
operatorBuffer = [];
}
// Push the delimiter.
tokens.push(getDelimiterToken(char));
}
}
}
// Empty all the buffers.
if (letterBuffer.length > 0) {
let variable = letterBuffer.join('');
tokens.push(new Token('name', 'variable', variable));
letterBuffer = [];
}
if (numberBuffer.length > 0) {
let number = numberBuffer.join('');
tokens.push(new Token('number', 'n/a', number));
numberBuffer = [];
}
if (operatorBuffer.length > 0) {
let operator = operatorBuffer.join('');
tokens.push(new Token('operator', operatorType(operator), operator));
operatorBuffer = [];
}
tokens = changeKeywords(tokens);
return tokens;
}
/**
* @function combineEscapedChars
* @desc Combines escaped chars into one char.
* @param {string[]} chars The chars.
* @returns {string[]} The chars with combined escaped chars.
* @private
*/
function combineEscapedChars(chars) {
// Check for characters to be escaped.
for (let i = 0; i < chars.length; i++) {
if (chars[i] == '\\') {
chars.splice(i, 2, chars[i] + chars[i + 1]);
i -= 2;
}
}
return chars;
}
/**
* @function removeComments
* @desc Removes comments.
* @param {string[]} chars The chars.
* @returns {string[]} The chars without comments.
* @private
*/
function removeComments(chars) {
let inComment = false; // Keep track if in a comment.
for (let i = 0; i < chars.length; i++) {
if (chars[i] == '/') {
if (chars[i + 1] == '/') {
inComment = true;
}
}
if (chars[i] == '\n') {
inComment = false;
chars.splice(i, 1); // Remove the newline at the end of the comment.
i--;
}
if (inComment) {
chars.splice(i, 1); // Remove the char in the comment.
i--;
}
}
return chars;
}
/**
* @function changeKeywords
* @desc Changes tokens with subtype variable to subtype keyword
* @param {Token[]} tokens The tokens
* @returns {Token[]} The tokens with keywords.
* @private
*/
function changeKeywords(tokens) {
return tokens.map(t => {
if (t.subtype == 'variable' && determineType(t.value) == 'keyword') {
t.subtype = 'keyword';
}
return t;
});
}
/**
* @function getDelimiterToken
* @desc Turns a delimiter char into a token.
* @param {string} delimiter The delimiter char.
* @returns {Token} The delimiter token.
* @private
*/
function getDelimiterToken(delimiter) {
if (/\(|\)/.test(delimiter))
return new Token('delimiter', delimiter == '(' ? 'left' : 'right', 'parenthesis');
else if (/\[|\]/.test(delimiter))
return new Token('delimiter', delimiter == '[' ? 'left' : 'right', 'bracket');
else if (/\{|\}/.test(delimiter))
return new Token('delimiter', delimiter == '{' ? 'left' : 'right', 'brace');
else throw new Error('Expected delimiter but got ' + delimiter);
}
/**
* @function operatorType
* @desc Turns a delimiter char into a token.
* @param {string} delimiter The delimiter char.
* @returns {Token} The delimiter token.
* @private
*/
function operatorType(op) {
switch (op) {
case '.':
case '+':
case '-':
case '*':
case '/':
case '**':
case '%':
case '<<':
case '>>':
case '<':
case '<=':
case '>':
case '>=':
case '==':
case '!=':
case '&':
case '|':
case '^':
case '&&':
case '||':
case '^^':
case '=':
case '+=':
case '-=':
case '*=':
case '/=':
case '**=':
case '%=':
case '<<=':
case '>>=':
case '&=':
case '|=':
case '^=':
return 'dual';
break;
case '++':
case '--':
return 'postfix';
break;
case '!':
case '~':
return 'prefix';
break;
case ',':
case ';':
return 'none';
break;
default:
throw new TypeError('Unexpected operator ' + op);
break;
}
}
/**
* @function determineCharType
* @desc Detects the type of characters.
* @param {string} char The input character(s).
* @returns {string} The type of char.
* @private
*/
function determineCharType(char) {
if (/[A-Za-z]/.test(char))
return 'letter';
else if (/\.|\+\+|--|!|~|\+|-|\*\*|\*|\/%|<<|>>|<|<=|>|>=|==|!=|&|\^|\||&&|\|\||=|\+=|-=|\*\*=|\*=|\/=|%=|<<=|>>=|&=|\^=|\|=|,|;/.test(char))
// All the operators in Pivot.
return 'operator';
else if (/\(|\)|\[|\]|\{|\}/.test(char))
return 'delimiter';
else if (/'|"|`/.test(char))
return 'string delimiter';
else if (/\d/.test(char))
return 'digit';
else if (/\\./.test(char))
return 'escaped char';
else if (/\s/.test(char))
return 'whitespace';
else throw new TypeError('Unexpected char ' + char);
};
/**
* @function determineType
* @desc Detects the type of a string.
* @param {string} str The input string.
* @returns {string} The type of string.
* @private
*/
function determineType(str) {
if (/let|const/.test(str)) // TODO: Add more keywords.
return 'keyword';
else return 'unknown';
};
module.exports = {
tokenize,
util: {
combineEscapedChars,
removeComments,
changeKeywords,
getDelimiterToken,
operatorType,
determineCharType,
determineType
}
};

42
src/types.js Normal file
View File

@ -0,0 +1,42 @@
/**
* @module types
* @file Provides a consistent source of types for the compiler.
* @author Garen Tyler <garentyler@gmail.com>
*/
/**
* @class Token
* @classdesc Stores the type of token, subtype, and literal char value.
*/
function Token(type, subtype, value) {
this.type = type;
this.subtype = subtype;
this.value = value;
}
/**
* @class Group
* @classdesc Stores the type of group, and the tokens in the group.
*/
function Group(type, tokens) {
this.type = 'group'
this.subtype = type;
this.tokens = tokens;
}
/**
* @class Operator
* @classdesc Stores the type of operator, and tokens for an operator.
*/
function Operator(subtype, value, operands) {
this.type = 'operator';
this.subtype = subtype;
this.value = value;
this.operands = operands;
}
module.exports = {
Token,
Group,
Operator
};

6
test.pivot
View File

@ -1,6 +0,0 @@
// Example Pivot code.
// This is a comment.
x = 4 // This creates a variable x with the value of 4.
y = 3 * x // This creates a variable y with the value of the output of (3 * x).
asdf = (a) {return(a++)} // This creates a function asdf that takes a number and returns that number + 1/
a = asdf(2) // This creates a variable a with the value of the output of asdf(2).

428
test/test.js Normal file
View File

@ -0,0 +1,428 @@
const assert = require('assert');
const types = require('../src/types.js');
const tokenizer = require('../src/tokenizer.js');
const parser = require('../src/parser.js');
console.log(parser.parse(tokenizer.tokenize('let x = (5 + 3);')));
describe('types.js', () => {
it('Has a Token child.', () => {
assert.equal(types.hasOwnProperty('Token'), true);
});
it('Has a Group child.', () => {
assert.equal(types.hasOwnProperty('Group'), true);
});
it('Has a Operator child.', () => {
assert.equal(types.hasOwnProperty('Operator'), true);
});
describe('Token', () => {
it('Works as a constructor', () => {
try {
let token = new types.Token('a', 'b', 'c');
} catch (err) {
throw err;
}
});
it('Has values \'type\', \'subtype\', and \'value\'', () => {
try {
let token = new types.Token('a', 'b', 'c');
if (!token.hasOwnProperty('type') || !token.hasOwnProperty('subtype') || !token.hasOwnProperty('value'))
throw new Error('Token is missing \'type\', \'subtype\', or \'value\' properties.');
if (token.type != 'a' || token.subtype != 'b' || token.value != 'c')
throw new Error('Token incorrectly set \'type\', \'subtype\', or \'value\' properties.');
} catch (err) {
throw err;
}
});
});
describe('Group', () => {
it('Works as a constructor', () => {
try {
let group = new types.Group('a', 'b');
} catch (err) {
throw err;
}
});
it('Has values \'type\', \'subtype\', and \'tokens\'', () => {
try {
let group = new types.Group('a', 'b');
if (!group.hasOwnProperty('type') || !group.hasOwnProperty('subtype') || !group.hasOwnProperty('tokens'))
throw new Error('Group is missing \'type\', \'subtype\', or \'tokens\' properties.');
if (group.type != 'group' || group.subtype != 'a' || group.tokens != 'b')
throw new Error('Group incorrectly set \'type\', \'subtype\', or \'tokens\' properties.');
} catch (err) {
throw err;
}
});
});
// describe('Operator', () => {
// it('Works as a constructor', () => {
// try {
// let op = new types.Operator('dual', '+', ['3','4']);
// } catch (err) {
// throw err;
// }
// });
// it('Has values \'type\', \'subtype\', \'value\', and \'operands\'', () => {
// try {
// let op = new types.Operator('dual', '+', ['3','4']);
// if (!op.hasOwnProperty('type') || !op.hasOwnProperty('subtype') || !op.hasOwnProperty('value') || !op.hasOwnProperty('operands'))
// throw new Error('Operator is missing \'type\', \'subtype\', \'value\', or \'operands\' properties.');
// if (op.type != 'operator' || op.subtype != 'dual' || op.value != '+' || op.operands[0] != '3')
// throw new Error('Operator incorrectly set \'type\', \'subtype\', \'value\', or \'operands\' properties.');
// } catch (err) {
// throw err;
// }
// });
// });
});
describe('tokenizer.js', () => {
it('combineEscapedChars works', () => {
assert.equal(tokenizer.util.combineEscapedChars(`let x = 'test\\nnewline';`.split('')).join(''), `let x = 'test\\nnewline';`);
});
it('removeComments works', () => {
assert.equal(tokenizer.util.removeComments(`// Comment\nlet i = 0;`.split('')).join(''), `let i = 0;`);
});
it('changeKeywords works', () => {
let tokens = tokenizer.util.changeKeywords([{
type: 'name',
subtype: 'variable',
value: 'let'
}, {
type: 'name',
subtype: 'variable',
value: 'x'
}]);
let correct = [{
type: 'name',
subtype: 'keyword',
value: 'let'
}, {
type: 'name',
subtype: 'variable',
value: 'x'
}];
let isCorrect = true;
tokens.forEach((t, i) => {
if (t.type != correct[i].type)
throw new Error('Changed type: Expected \'' + correct[i].type + '\' but got \'' + t.type + '\'')
else if (t.subtype != correct[i].subtype)
throw new Error('Incorrectly changed subtype: Expected \'' + correct[i].subtype + '\' but got \'' + t.subtype + '\'')
else if (t.value != correct[i].value)
throw new Error('Changed value: Expected \'' + correct[i].value + '\' but got \'' + t.value + '\'')
});
});
it('getDelimiterToken works', () => {
let token = tokenizer.util.getDelimiterToken(')');
if (token.type != 'delimiter')
throw new Error('Incorrect type: Expected \'delimiter\' but got \'' + token.type + '\'')
else if (token.subtype != 'right')
throw new Error('Incorrect subtype: Expected \'right\' but got \'' + token.subtype + '\'')
else if (token.value != 'parenthesis')
throw new Error('Incorrect value: Expected \'parenthesis\' but got \'' + token.value + '\'')
});
it('operatorType works', () => {
assert.equal(tokenizer.util.operatorType('++'), 'postfix');
assert.equal(tokenizer.util.operatorType(';'), 'none');
assert.equal(tokenizer.util.operatorType('+'), 'dual');
});
it('determineCharType works', () => {
assert.equal(tokenizer.util.determineCharType('+'), 'operator');
assert.equal(tokenizer.util.determineCharType('"'), 'string delimiter');
assert.equal(tokenizer.util.determineCharType('4'), 'digit');
});
it('determineType works', () => {
assert.equal(tokenizer.util.determineType('let'), 'keyword');
assert.equal(tokenizer.util.determineType('dog'), 'unknown');
});
});
describe('parser.js', () => {
it('addIndexes works', () => {
let tokens = parser.util.addIndexes([{
type: 'name',
subtype: 'keyword',
value: 'let'
}, {
type: 'name',
subtype: 'variable',
value: 'x'
}]);
let correct = [{
type: 'name',
subtype: 'keyword',
value: 'let',
index: 0
}, {
type: 'name',
subtype: 'variable',
value: 'x',
index: 1
}];
let isCorrect = true;
tokens.forEach((t, i) => {
if (t.type != correct[i].type)
throw new Error('Changed type: Expected \'' + correct[i].type + '\' but got ' + t.type)
else if (t.subtype != correct[i].subtype)
throw new Error('Changed subtype: Expected \'' + correct[i].subtype + '\' but got ' + t.subtype)
else if (t.value != correct[i].value)
throw new Error('Changed value: Expected \'' + correct[i].value + '\' but got ' + t.value)
else if (t.index != correct[i].index)
throw new Error('Incorrect index: Expected \'' + correct[i].index + '\' but got ' + t.index)
});
});
it('addLevels works', () => {
let tokens = parser.util.addLevels([{
type: 'name',
subtype: 'keyword',
value: 'let',
index: 0
}, {
type: 'name',
subtype: 'variable',
value: 'x',
index: 1
}, {
type: 'operator',
subtype: 'dual',
value: '=',
index: 2
}, {
type: 'delimiter',
subtype: 'left',
value: 'parenthesis',
index: 3
}, {
type: 'number',
subtype: 'n/a',
value: '5',
index: 4
}, {
type: 'delimiter',
subtype: 'right',
value: 'parenthesis',
index: 5
}]);
let correct = [{
type: 'name',
subtype: 'keyword',
value: 'let',
index: 0,
level: 0
}, {
type: 'name',
subtype: 'variable',
value: 'x',
index: 1,
level: 0
}, {
type: 'operator',
subtype: 'dual',
value: '=',
index: 2,
level: 0
}, {
type: 'delimiter',
subtype: 'left',
value: 'parenthesis',
index: 3,
level: 1
}, {
type: 'number',
subtype: 'n/a',
value: '5',
index: 4,
level: 1
}, {
type: 'delimiter',
subtype: 'right',
value: 'parenthesis',
index: 5,
level: 1
}];
let isCorrect = true;
tokens.forEach((t, i) => {
if (t.type != correct[i].type)
throw new Error('Changed type: Expected \'' + correct[i].type + '\' but got ' + t.type)
else if (t.subtype != correct[i].subtype)
throw new Error('Changed subtype: Expected \'' + correct[i].subtype + '\' but got \'' + t.subtype + '\'')
else if (t.value != correct[i].value)
throw new Error('Changed value: Expected \'' + correct[i].value + '\' but got \'' + t.value + '\'')
else if (t.index != correct[i].index)
throw new Error('Incorrect index: Expected \'' + correct[i].index + '\' but got \'' + t.index + '\'')
else if (t.level != correct[i].level)
throw new Error('Incorrect level: Expected \'' + correct[i].level + '\' but got \'' + t.level + '\'')
});
});
it('getDeepestLevel works', () => {
let deepestLevel = parser.util.getDeepestLevel([{
type: 'name',
subtype: 'keyword',
value: 'let',
index: 0,
level: 0
}, {
type: 'name',
subtype: 'variable',
value: 'x',
index: 1,
level: 0
}, {
type: 'operator',
subtype: 'dual',
value: '=',
index: 2,
level: 0
}, {
type: 'delimiter',
subtype: 'left',
value: 'parenthesis',
index: 3,
level: 1
}, {
type: 'number',
subtype: 'n/a',
value: '5',
index: 4,
level: 1
}, {
type: 'operator',
subtype: 'dual',
value: '+',
index: 5,
level: 1
}, {
type: 'delimiter',
subtype: 'left',
value: 'parenthesis',
index: 6,
level: 2
}, {
type: 'number',
subtype: 'n/a',
value: '6',
index: 7,
level: 2
}, {
type: 'operator',
subtype: 'dual',
value: '*',
index: 8,
level: 2
}, {
type: 'number',
subtype: 'n/a',
value: '2',
index: 9,
level: 2
}, {
type: 'delimiter',
subtype: 'right',
value: 'parenthesis',
index: 10,
level: 2
}, {
type: 'delimiter',
subtype: 'right',
value: 'parenthesis',
index: 11,
level: 1
}, {
type: 'operator',
subtype: 'none',
value: ';',
index: 12,
level: 0
}]);
if (deepestLevel != 2)
throw new Error('Incorrect deepestLevel. Expected \'2\' but got \'' + deepestLevel + '\'');
});
it('combineGroups works', () => {
let ast = parser.util.combineGroups([{
type: 'name',
subtype: 'keyword',
value: 'let',
index: 0,
level: 0
}, {
type: 'name',
subtype: 'variable',
value: 'x',
index: 1,
level: 0
}, {
type: 'operator',
subtype: 'dual',
value: '=',
index: 2,
level: 0
}, {
type: 'delimiter',
subtype: 'left',
value: 'parenthesis',
index: 3,
level: 1
}, {
type: 'number',
subtype: 'n/a',
value: '5',
index: 4,
level: 1
}, {
type: 'operator',
subtype: 'dual',
value: '+',
index: 5,
level: 1
}, {
type: 'delimiter',
subtype: 'left',
value: 'parenthesis',
index: 6,
level: 2
}, {
type: 'number',
subtype: 'n/a',
value: '6',
index: 7,
level: 2
}, {
type: 'operator',
subtype: 'dual',
value: '*',
index: 8,
level: 2
}, {
type: 'number',
subtype: 'n/a',
value: '2',
index: 9,
level: 2
}, {
type: 'delimiter',
subtype: 'right',
value: 'parenthesis',
index: 10,
level: 2
}, {
type: 'delimiter',
subtype: 'right',
value: 'parenthesis',
index: 11,
level: 1
}, {
type: 'operator',
subtype: 'none',
value: ';',
index: 12,
level: 0
}]);
if (ast[3].type != 'group')
throw new Error('Incorrectly combined group.');
if (ast[3].tokens[3].type != 'group')
throw new Error('Incorrectly combined group.');
});
});

170
tokenizer.js
View File

@ -1,170 +0,0 @@
// Import the token class.
const token = require("./classes.js").token;
// Create the tokenizer function.
// tokenize() takes Pivot code in, and outputs an array of tokens.
module.exports = exp => {
// Check functions for different token types.
var isDigit = char => {
return /\d/.test(char);
};
var isLetter = char => {
return /[a-z]/i.test(char);
};
var isOperator = char => {
return /\+|-|\*|\/|\^|=/.test(char);
};
var isLeftDelimiter = char => {
return (/\(|\[|\{|"|'|`/.test(char));
};
var isRightDelimiter = char => {
return (/\)|\]|\}/.test(char));
};
var isComma = char => {
return (char === ",");
};
var isPeriod = char => {
return (char === ".");
};
var result = []; // The final array of tokens.
var nb = []; // Number buffer. Allows for multiple digits to be one number.
var lb = []; // Letter buffer. Allows for multiple letters to be one variable / function.
var ob = []; // Operator buffer. Allows for multi-character operators. E.g. ++ or ==.
var sb = []; // String buffer. Allows for multi-character strings.
var inString = false; // Keep track of whether in string or not.
var stringType; // Keep track of what type of string. E.g. "" or ''.
exp = exp.split(""); // Split the expression into an array of characters.
/* - - - DO NOT TOUCH THIS - - - */
for (var i = 0; i < exp.length; i++) { // Loop through all of the characters.
var char = exp[i]; // Create a quick reference to the current char.
if (i >= 1) {
if (exp[i - 1] == "\\") {
exp.splice(i - 1, 2, `\\${char}`);
i--;
continue;
}
if (exp[i - 1] == "$" && char == "{") {
exp.splice(i - 1, 2, `\${`);
i--;
continue;
}
}
}
/* - - - OK YOU CAN TOUCH AGAIN - - - */
// Nevermind, just don't mess with any of this file.
for (var i = 0; i < exp.length; i++) {
var char = exp[i];
if (inString) {
if (char == `'` || char == `"` || char == "`") {
var exitString = () => {
inString = false;
if (sb.length == 0) {
result.push(new token("String", null));
} else {
var string = sb.join("");
result.push(new token("String", string));
}
sb = [];
};
if (char == `'` && stringType == "single") {
exitString();
} else if (char == `"` && stringType == "double") {
exitString();
} else if (char == "`" && stringType == "backtick") {
exitString();
} else {
if (char == `'`) {
sb.push(`\'`);
}
if (char == `"`) {
sb.push(`\"`);
}
if (char == "`") {
sb.push("\`");
}
}
} else {
sb.push(char);
}
} else {
if (isDigit(char)) {
result.push(new token("Operator", ob.join("")));
ob = [];
nb.push(char);
} else if (isLetter(char)) {
result.push(new token("Operator", ob.join("")));
ob = [];
lb.push(char);
} else if (isOperator(char)) {
result.push(new token("Number", nb.join("")));
nb = [];
result.push(new token("Variable", lb.join("")));
lb = [];
ob.push(char);
} else if (isLeftDelimiter(char)) {
result.push(new token("Operator", ob.join("")));
ob = [];
result.push(new token("Function Call", lb.join("")));
lb = [];
if (char == `'` || char == `"` || char == "`") {
inString = true;
if (char == `'`) {
stringType = "single";
} else if (char == `"`) {
stringType = "double";
} else if (char == "`") {
stringType = "backtick";
}
} else {
result.push(new token("Left Delimiter", char));
}
} else if (isRightDelimiter(char)) {
result.push(new token("Operator", ob.join("")));
ob = [];
result.push(new token("Number", nb.join("")));
nb = [];
result.push(new token("Variable", lb.join("")));
lb = [];
result.push(new token("Right Delimiter", char));
} else if (isComma(char)) {
result.push(new token("Operator", ob.join("")));
ob = [];
result.push(new token("Number", nb.join("")));
nb = [];
result.push(new token("Variable", lb.join("")));
lb = [];
result.push(new token("Comma", char));
} else if (isPeriod(char)) {
result.push(new token("Operator", ob.join("")));
ob = [];
nb.push(char);
}
}
}
result.push(new token("Operator", ob.join("")));
ob = [];
result.push(new token("Number", nb.join("")));
nb = [];
lb.forEach(item => {
result.push(new token("Variable", item));
});
lb = [];
for (var i = 0; i < 3; i++) {
result.forEach((item, index) => {
if (item.value == "") {
result.splice(index, 1);
}
});
}
result.forEach((item, index) => {
if (item.value == "-" && index != 0) {
if (result[index - 1].type != "Variable" && result[index - 1].type != "Number") {
if (result[index + 1].type == "Number") {
result[index + 1].value = "-" + result[index + 1].value;
result.splice(index, 1);
}
}
}
});
return result;
};