garentyler/pivot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add package data, member access, pre/postfix operators work now.

Browse Source
This commit is contained in:
ElementG9 2019-11-24 21:31:20 -07:00
parent f796a43c16
commit f0b8f181f7
6 changed files with 215 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
bin/pivot.js
View File

@ -1,3 +1,29 @@
#!/usr/bin/env node
const args = process.argv.slice(2);
console.log(args);
const tokenizer = require('../src/tokenizer.js');
const parser = require('../src/parser.js');
if (typeof args[0] != 'undefined') {
// Execute from file.
} else { // REPL.
const rl = require('readline-sync');
const exec = require('child_process').exec;
function repl(prompt, func) {
let answer;
while (answer != 'exit') {
answer = rl.question(prompt);
if (answer == 'exit')
process.exit(0);
if (answer == 'clear') {
console.clear();
continue;
}
func(answer);
}
}
console.log('Welcome to Pivot v0.1.0 Alpha.');
console.log('Type \'exit\' to exit.');
repl('> ', (answer) => {
console.log(require('util').inspect(parser.parse(tokenizer.tokenize(answer)), { depth: null }));
});
}

7
package.json
View File

@ -1,6 +1,6 @@
{
"name": "pivot",
"version": "1.0.0",
"name": "pivot-lang",
"version": "0.1.0",
"description": "Pivot is a new programming language built on JavaScript",
"main": "./bin/pivot.js",
"directories": {
@ -31,7 +31,8 @@
},
"preferGlobal": true,
"dependencies": {
"fs": "0.0.1-security"
"fs": "0.0.1-security",
"readline-sync": "^1.4.10"
},
"devDependencies": {
"mocha": "^6.2.1"

100
src/parser.js
View File

@ -4,9 +4,8 @@
* @author Garen Tyler <garentyler@gmail.com>
* @requires module:types
*/
const Token = require('./types.js').Token;
const Group = require('./types.js').Group;
const tokenizer = require('./tokenizer.js');
const Operator = require('./types.js').Operator;
/**
* @function parse
@ -23,10 +22,12 @@ function parse(tokens) {
ast = addIndexes(ast);
ast = addLevels(ast);
// Combine the groups.
ast = combineGroups(ast);
//
// Start grouping by precedence
ast = grouping(ast);
ast = memberAccess(ast);
ast = postfixOperators(ast);
ast = prefixOperators(ast);
console.log(ast);
return ast;
}
@ -84,16 +85,15 @@ function getDeepestLevel(tokens) {
}
/**
* @function combineGroups
* @function grouping
* @desc Combine groups of tokens by delimiter.
* @param {Token[]} tokens The tokens.
* @returns {Token[]} The grouped tokens, or the basic ast.
* @private
*/
function combineGroups(tokens) {
function grouping(tokens) {
// Get the deepest level.
let deepestLevel = getDeepestLevel(tokens);
// Loop through for each level.
for (let currentLevel = deepestLevel; currentLevel > 0; currentLevel--) {
let groupBuffer = [];
@ -107,8 +107,10 @@ function combineGroups(tokens) {
let g = new Group(groupBuffer[0].value, groupBuffer);
g.index = g.tokens[0].index;
g.level = g.tokens[0].level - 1; // -1 because the group is on the level below.
tokens.splice(g.tokens[0].index, g.tokens.length + 1, g);
tokens.splice(g.tokens[0].index, g.tokens.length, g);
j = g.tokens[0].index;
// Remove the delimiters in g.tokens.
g.tokens = g.tokens.splice(1, groupBuffer.length - 2);
groupBuffer = [];
}
}
@ -117,14 +119,88 @@ function combineGroups(tokens) {
return tokens;
}
/**
* @function memberAccess
* @desc Combine groups of tokens by member access.
* @param {Token[]} tokens The tokens.
* @returns {Token[]} The grouped tokens, or the basic ast.
* @private
*/
function memberAccess(ast) {
for (let i = 0; i < ast.length; i++) {
if (ast[i].type == 'group')
memberAccess(ast[i].tokens); // Recursively order the groups.
else if (ast[i].type == 'operator' && ast[i].value == '.') { // Member access operator.
if (typeof ast[i - 1] == 'undefined' || typeof ast[i + 1] == 'undefined')
throw new SyntaxError('Operator requires two operands.');
let op = new Operator(ast[i].subtype, ast[i].value, [ast[i - 1], ast[i + 1]]);
op.index = ast[i - 1].index;
op.level = ast[i].level;
ast.splice(i - 1, 3, op);
i--; // Removed 3 tokens, put in 1, skip 1 token. Reduce the counter by 1.
}
}
return ast;
}
/**
* @function postfixOperators
* @desc Recursively structures the postfix operators.
* @param {Token[]} ast The ast.
* @returns {Token[]} The ast with structured postfix operators.
* @private
*/
function postfixOperators(ast) {
for (let i = 0; i < ast.length; i++) {
// Take care of the tokens in the groups.
if (ast[i].type == 'group')
ast[i].tokens = postfixOperators(ast[i].tokens);
else if (ast[i].type == 'operator' && ast[i].subtype == 'postfix') { // The operand is on the left.
if (typeof ast[i - 1] == 'undefined')
throw new SyntaxError('Postfix operator requires one operand before it.');
let op = new Operator(ast[i].subtype, ast[i].value, [ast[i - 1]]);
op.index = ast[i].index;
op.level = ast[i].level;
ast.splice(i - 1, 2, op);
// Removing 2 tokens, adding 1, skip 1 token. Don't reduce the counter.
}
}
return ast;
}
/**
* @function prefixOperators
* @desc Recursively structures the prefix operators.
* @param {Token[]} ast The ast.
* @returns {Token[]} The ast with structured prefix operators.
* @private
*/
function prefixOperators(ast) {
for (let i = 0; i < ast.length; i++) {
// Take care of the tokens in the groups.
if (ast[i].type == 'group')
ast[i].tokens = postfixOperators(ast[i].tokens);
else if (ast[i].type == 'operator' && ast[i].subtype == 'prefix') { // The operand is on the right.
if (typeof ast[i + 1] == 'undefined')
throw new SyntaxError('Prefix operator requires one operand after it.');
let op = new Operator(ast[i].subtype, ast[i].value, [ast[i + 1]]);
op.index = ast[i].index;
op.level = ast[i].level;
ast.splice(i, 2, op);
// Removing 2 tokens, adding 1, skip 1 token. Don't reduce the counter.
}
}
return ast;
}
module.exports = {
parse,
util: {
addIndexes,
addLevels,
getDeepestLevel,
combineGroups
grouping,
}
};
require('fs').writeFileSync('ast.json', JSON.stringify(parse(tokenizer.tokenize('let x = (5 + (6 * 2));')), null, 2), () => {});
// require('fs').writeFileSync('ast.json', JSON.stringify(parse(tokenizer.tokenize('let x = (5 + (6 * 2)) - 7;')), null, 2), () => {});

73
src/tokenizer.js
View File

@ -5,7 +5,6 @@
* @requires module:types
*/
const Token = require('./types.js').Token;
const Group = require('./types.js').Group;
/**
* @function tokenize
@ -234,21 +233,64 @@ function getDelimiterToken(delimiter) {
/**
* @function operatorType
* @desc Determines the type of operator.
* @param {string} operator The operator char.
* @returns {string} The type of operator.
* @desc Turns a delimiter char into a token.
* @param {string} delimiter The delimiter char.
* @returns {Token} The delimiter token.
* @private
*/
function operatorType(operator) {
// Left operators have parameters on the left.
if (/\+\+|--/.test(operator))
return 'left';
else if (false)
return 'right';
else if (/\;/.test(operator))
return 'none';
else
function operatorType(op) {
switch (op) {
case '.':
case '+':
case '-':
case '*':
case '/':
case '**':
case '%':
case '<<':
case '>>':
case '<':
case '<=':
case '>':
case '>=':
case '==':
case '!=':
case '&':
case '|':
case '^':
case '&&':
case '||':
case '^^':
case '=':
case '+=':
case '-=':
case '*=':
case '/=':
case '**=':
case '%=':
case '<<=':
case '>>=':
case '&=':
case '|=':
case '^=':
return 'dual';
break;
case '++':
case '--':
return 'postfix';
break;
case '!':
case '~':
return 'prefix';
break;
case ',':
case ';':
return 'none';
break;
default:
throw new TypeError('Unexpected operator ' + op);
break;
}
}
/**
@ -261,7 +303,8 @@ function operatorType(operator) {
function determineCharType(char) {
if (/[A-Za-z]/.test(char))
return 'letter';
else if (/\+|\-|\*|\/|\=|\=\=|\>|\<|\>\=|\<\=|\=\>|;/.test(char))
else if (/\.|\+\+|--|!|~|\+|-|\*\*|\*|\/%|<<|>>|<|<=|>|>=|==|!=|&|\^|\||&&|\|\||=|\+=|-=|\*\*=|\*=|\/=|%=|<<=|>>=|&=|\^=|\|=|,|;/.test(char))
// All the operators in Pivot.
return 'operator';
else if (/\(|\)|\[|\]|\{|\}/.test(char))
return 'delimiter';
@ -273,7 +316,7 @@ function determineCharType(char) {
return 'escaped char';
else if (/\s/.test(char))
return 'whitespace';
else throw new SyntaxError('Unexpected char ' + char);
else throw new TypeError('Unexpected char ' + char);
};
/**

15
src/types.js
View File

@ -4,7 +4,6 @@
* @author Garen Tyler <garentyler@gmail.com>
*/
/**
* @class Token
* @classdesc Stores the type of token, subtype, and literal char value.
@ -25,7 +24,19 @@ function Group(type, tokens) {
this.tokens = tokens;
}
/**
* @class Operator
* @classdesc Stores the type of operator, and tokens for an operator.
*/
function Operator(subtype, value, operands) {
this.type = 'operator';
this.subtype = subtype;
this.value = value;
this.operands = operands;
}
module.exports = {
Token,
Group
Group,
Operator
};

25
test/test.js
View File

@ -13,6 +13,9 @@ describe('types.js', () => {
it('Has a Group child.', () => {
assert.equal(types.hasOwnProperty('Group'), true);
});
it('Has a Operator child.', () => {
assert.equal(types.hasOwnProperty('Operator'), true);
});
describe('Token', () => {
it('Works as a constructor', () => {
@ -54,6 +57,26 @@ describe('types.js', () => {
}
});
});
// describe('Operator', () => {
// it('Works as a constructor', () => {
// try {
// let op = new types.Operator('dual', '+', ['3','4']);
// } catch (err) {
// throw err;
// }
// });
// it('Has values \'type\', \'subtype\', \'value\', and \'operands\'', () => {
// try {
// let op = new types.Operator('dual', '+', ['3','4']);
// if (!op.hasOwnProperty('type') || !op.hasOwnProperty('subtype') || !op.hasOwnProperty('value') || !op.hasOwnProperty('operands'))
// throw new Error('Operator is missing \'type\', \'subtype\', \'value\', or \'operands\' properties.');
// if (op.type != 'operator' || op.subtype != 'dual' || op.value != '+' || op.operands[0] != '3')
// throw new Error('Operator incorrectly set \'type\', \'subtype\', \'value\', or \'operands\' properties.');
// } catch (err) {
// throw err;
// }
// });
// });
});
describe('tokenizer.js', () => {
it('combineEscapedChars works', () => {
@ -101,7 +124,7 @@ describe('tokenizer.js', () => {
throw new Error('Incorrect value: Expected \'parenthesis\' but got \'' + token.value + '\'')
});
it('operatorType works', () => {
assert.equal(tokenizer.util.operatorType('++'), 'left');
assert.equal(tokenizer.util.operatorType('++'), 'postfix');
assert.equal(tokenizer.util.operatorType(';'), 'none');
assert.equal(tokenizer.util.operatorType('+'), 'dual');
});