Refactored tokenizer, implemented jsdoc, set up for cli.
This commit is contained in:
ElementG9
parent
7f889a79a0
commit
5c0a481dbe
4
.gitignore
vendored
Normal file
4
.gitignore
vendored
Normal file
@ -0,0 +1,4 @@
|
||||
other/
|
||||
package-lock.json
|
||||
node_modules/
|
||||
jsdoc/
|
||||
2
LICENSE
2
LICENSE
@ -1,6 +1,6 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2019 ElementG9
|
||||
Copyright (c) 2019 Garen Tyler
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
|
||||
50
ast.json
50
ast.json
@ -1,50 +0,0 @@
|
||||
[
|
||||
{
|
||||
"type": "Variable",
|
||||
"value": "asdf",
|
||||
"layer": 0
|
||||
},
|
||||
{
|
||||
"type": "Operator",
|
||||
"value": "=",
|
||||
"layer": 0
|
||||
},
|
||||
{
|
||||
"type": "Group",
|
||||
"value": "(",
|
||||
"tokens": [
|
||||
{
|
||||
"type": "Variable",
|
||||
"value": "a",
|
||||
"layer": 1
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "Group",
|
||||
"value": "{",
|
||||
"tokens": [
|
||||
{
|
||||
"type": "Function Call",
|
||||
"value": "return",
|
||||
"layer": 1
|
||||
},
|
||||
{
|
||||
"type": "Group",
|
||||
"value": "(",
|
||||
"tokens": [
|
||||
{
|
||||
"type": "Variable",
|
||||
"value": "a",
|
||||
"layer": 2
|
||||
},
|
||||
{
|
||||
"type": "Operator",
|
||||
"value": "++",
|
||||
"layer": 2
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
3
bin/pivot.js
Executable file
3
bin/pivot.js
Executable file
@ -0,0 +1,3 @@
|
||||
#!/usr/bin/env node
|
||||
const args = process.argv.slice(2);
|
||||
console.log(args);
|
||||
21
classes.js
21
classes.js
@ -1,21 +0,0 @@
|
||||
// This is the most basic type of token.
|
||||
var token = function (type, value) {
|
||||
this.type = type;
|
||||
this.value = value;
|
||||
};
|
||||
// This is a group of tokens.
|
||||
var group = function (type, tokens, index) {
|
||||
this.type = "Group";
|
||||
this.value = type;
|
||||
this.index = index;
|
||||
this.tokens;
|
||||
if (typeof tokens != "undefined") {
|
||||
this.tokens = tokens;
|
||||
} else {
|
||||
this.tokens = [];
|
||||
}
|
||||
}
|
||||
module.exports = {
|
||||
token: token,
|
||||
group: group
|
||||
}
|
||||
39
package.json
Normal file
39
package.json
Normal file
@ -0,0 +1,39 @@
|
||||
{
|
||||
"name": "pivot",
|
||||
"version": "1.0.0",
|
||||
"description": "Pivot is a new programming language built on JavaScript",
|
||||
"main": "./bin/pivot.js",
|
||||
"directories": {
|
||||
"test": "test"
|
||||
},
|
||||
"scripts": {
|
||||
"test": "mocha",
|
||||
"build": "npm run test; jsdoc src/ -r -d ./jsdoc/ -R ./readme.md"
|
||||
},
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "git+https://github.com/ElementG9/Pivot.git"
|
||||
},
|
||||
"keywords": [
|
||||
"pivot",
|
||||
"javascript",
|
||||
"programming",
|
||||
"language"
|
||||
],
|
||||
"author": "Garen Tyler <garentyler@gmail.com>",
|
||||
"license": "MIT",
|
||||
"bugs": {
|
||||
"url": "https://github.com/ElementG9/Pivot/issues"
|
||||
},
|
||||
"homepage": "https://github.com/ElementG9/Pivot#readme",
|
||||
"bin": {
|
||||
"pivot": "./bin/pivot.js"
|
||||
},
|
||||
"preferGlobal": true,
|
||||
"dependencies": {
|
||||
"fs": "0.0.1-security"
|
||||
},
|
||||
"devDependencies": {
|
||||
"mocha": "^6.2.1"
|
||||
}
|
||||
}
|
||||
134
parser.js
134
parser.js
@ -1,134 +0,0 @@
|
||||
// Import the group class.
|
||||
const group = require("./classes.js").group;
|
||||
|
||||
// Create the parser function.
|
||||
// parse() takes an array of tokens in, and outputs an
|
||||
// Abstract Syntax Tree (a structured array of tokens).
|
||||
module.exports = tokens => {
|
||||
// Variables for later.
|
||||
var layer = 0;
|
||||
var delimiterCount = 0;
|
||||
var deepestLayer = 0;
|
||||
// Give each token a layer number based on delimiters.
|
||||
for (var i = 0; i < tokens.length; i++) {
|
||||
if (tokens[i].type == "Left Delimiter") {
|
||||
layer++;
|
||||
if (layer > deepestLayer) {
|
||||
deepestLayer = layer;
|
||||
}
|
||||
delimiterCount++;
|
||||
}
|
||||
tokens[i].layer = layer;
|
||||
if (tokens[i].type == "Right Delimiter") {
|
||||
layer--;
|
||||
}
|
||||
}
|
||||
// Lower the layer of delimiters.
|
||||
for (var i = 0; i < tokens.length; i++) {
|
||||
if ((tokens[i].type == "Left Delimiter") || (tokens[i].type == "Right Delimiter")) {
|
||||
tokens[i].layer--;
|
||||
}
|
||||
}
|
||||
if (layer > 0) { // Unclosed delimiter.
|
||||
} else if (layer < 0) { // Overclosed delimiter.
|
||||
}
|
||||
// Give each token an index.
|
||||
for (let i = 0; i < tokens.length; i++) {
|
||||
tokens[i].index = i;
|
||||
}
|
||||
// Structure the layers.
|
||||
// Count the rising edges of the layers to determine how many groups should exist.
|
||||
let structure = function () {
|
||||
let layer = 0;
|
||||
let risingFalling = []; // Create an array to store indices of rising/falling edges.
|
||||
for (let i = 0; i < tokens.length; i++) {
|
||||
// Add a rising and a falling tag to each token.
|
||||
tokens[i].rising = false;
|
||||
tokens[i].falling = false;
|
||||
if (tokens[i].layer > layer) { // If the token moves up a layer.
|
||||
// Create a new rising index in risingFalling.
|
||||
risingFalling.push({
|
||||
type: 'rising',
|
||||
index: i
|
||||
});
|
||||
tokens[i].rising = true; // Note that the token is a rising edge.
|
||||
layer++;
|
||||
} else if (tokens[i].layer < layer) {
|
||||
// Create a new falling index in risingFalling.
|
||||
risingFalling.push({
|
||||
type: 'falling',
|
||||
index: i
|
||||
});
|
||||
tokens[i].falling = true; // Note that the token is a falling edge.
|
||||
layer--;
|
||||
}
|
||||
}
|
||||
// Loop through the list of rising/falling edges.
|
||||
for (let i = 0; i < risingFalling.length; i++) {
|
||||
if (i != risingFalling.length - 1) { // If not the last edge.
|
||||
let item = risingFalling[i];
|
||||
let nextItem = risingFalling[i + 1];
|
||||
// If a falling edge follows a rising edge, classifiy it as a group.
|
||||
if ((item.type == 'rising') && (nextItem.type == 'falling')) {
|
||||
// Get the group together as one item.
|
||||
let selectedItems = tokens.slice(item.index, nextItem.index);
|
||||
tokens.splice(item.index, selectedItems.length, new group(tokens[item.index - 1].value, selectedItems, item.index));
|
||||
}
|
||||
}
|
||||
}
|
||||
risingFalling = []; // Reset the list of edges.
|
||||
// Count the edges again.
|
||||
for (let i = 0; i < tokens.length; i++) {
|
||||
if (tokens[i].layer > layer) {
|
||||
risingFalling.push({
|
||||
type: 'rising',
|
||||
index: i
|
||||
});
|
||||
layer++;
|
||||
} else if (tokens[i].layer < layer) {
|
||||
risingFalling.push({
|
||||
type: 'falling',
|
||||
index: i
|
||||
});
|
||||
layer--;
|
||||
}
|
||||
}
|
||||
// If there are still edges, run again.
|
||||
if (risingFalling.length) {
|
||||
structure();
|
||||
}
|
||||
};
|
||||
// Start the recursion.
|
||||
structure();
|
||||
let trimDelimiters = function (thing) {
|
||||
// Loop through the tokens of thing.
|
||||
for (let i = 0; i < thing.length; i++) {
|
||||
// Delete unnecessary keys.
|
||||
if (typeof thing[i].rising != 'undefined') {
|
||||
delete thing[i].rising;
|
||||
}
|
||||
if (typeof thing[i].falling != 'undefined') {
|
||||
delete thing[i].falling;
|
||||
}
|
||||
if (typeof thing[i].index != 'undefined') {
|
||||
delete thing[i].index;
|
||||
}
|
||||
if (typeof thing[i].index != 'undefined') {
|
||||
delete thing[i].index;
|
||||
}
|
||||
// Remove delimiters.
|
||||
if ((thing[i].type == 'Left Delimiter') || (thing[i].type == 'Right Delimiter')) {
|
||||
thing.splice(i, 1);
|
||||
i--;
|
||||
}
|
||||
// If a token is a group, look at the group's tokens.
|
||||
if (thing[i].type == 'Group') {
|
||||
trimDelimiters(thing[i].tokens);
|
||||
}
|
||||
}
|
||||
};
|
||||
// Start the recursion.
|
||||
trimDelimiters(tokens);
|
||||
// Return the structured tokens.
|
||||
return tokens;
|
||||
};
|
||||
13
pivot.js
13
pivot.js
@ -1,13 +0,0 @@
|
||||
// The Pivot code to run.
|
||||
var code = `asdf = (a) {return(a++)}`;
|
||||
|
||||
// Import the tokenizer and parser.
|
||||
const tokenize = require("./tokenizer.js");
|
||||
const parse = require("./parser.js");
|
||||
|
||||
// Generate the AST.
|
||||
var ast = parse(tokenize(code));
|
||||
|
||||
// Write the AST to ast.json.
|
||||
var fs = require("fs");
|
||||
fs.writeFileSync("ast.json", JSON.stringify(ast, null, 4));
|
||||
15
readme.md
15
readme.md
@ -1,15 +0,0 @@
|
||||
# [Pivot](https://gthub.us/pivot/)
|
||||
|
||||
Pivot is a new programming language built on [JavaScript](https://javascript.com/) by [ElementG9](https://github.com/ElementG9/).
|
||||
|
||||
---
|
||||
|
||||
## Installation
|
||||
* Download or clone [the GitHub repository](/pivot/repo).
|
||||
* Download [node.js](https://nodejs.org/).
|
||||
* Run _pivot.js_ with `node pivot.js`.
|
||||
|
||||
---
|
||||
|
||||
## Documentation
|
||||
The Pivot documentation is [here](https://gthub.us/pivot/docs).
|
||||
40
rpn.js
40
rpn.js
@ -1,40 +0,0 @@
|
||||
var solve = (exp) => {
|
||||
var stack = [];
|
||||
var expression = exp.split(" ");
|
||||
for(var j=0;j<expression.length;j++) {
|
||||
var key = expression[j];
|
||||
if(key.match(/\d/)) {
|
||||
stack.push(parseInt(key));
|
||||
} else if(key.match(/\w/)) {
|
||||
if(Object.keys(progData).includes(key)) {
|
||||
stack.push(progData[key]);
|
||||
} else {
|
||||
stack.push(key);
|
||||
}
|
||||
}
|
||||
switch(key) {
|
||||
case "+": // add
|
||||
var opItems = stack.splice(stack.length-2,2);
|
||||
var result = opItems[0]+opItems[1];
|
||||
stack.push(result);
|
||||
break;
|
||||
case "-": // subtract
|
||||
var opItems = stack.splice(stack.length-2,2);
|
||||
var result = opItems[0]-opItems[1];
|
||||
stack.push(result);
|
||||
break;
|
||||
case "*": // multiply
|
||||
var opItems = stack.splice(stack.length-2,2);
|
||||
var result = opItems[0]*opItems[1];
|
||||
stack.push(result);
|
||||
break;
|
||||
case "/": // divide
|
||||
var opItems = stack.splice(stack.length-2,2);
|
||||
var result = opItems[0]/opItems[1];
|
||||
stack.push(result);
|
||||
break;
|
||||
}
|
||||
}
|
||||
return stack;
|
||||
};
|
||||
module.exports = solve;
|
||||
44
src/parser.js
Normal file
44
src/parser.js
Normal file
@ -0,0 +1,44 @@
|
||||
/**
|
||||
* @module parser
|
||||
* @file Manages the parser phase of Pivot.
|
||||
* @author Garen Tyler <garentyler@gmail.com>
|
||||
* @requires module:types
|
||||
*/
|
||||
const Token = require('./types.js').Token;
|
||||
const Group = require('./types.js').Token;
|
||||
|
||||
module.exports = function(tokens) {
|
||||
let level = 0;
|
||||
// Add level markers.
|
||||
tokens.forEach((t, i) => {
|
||||
if (t.type == 'delimiter' && t.subtype == 'left') {
|
||||
tokens[i].level = level;
|
||||
level++;
|
||||
} else if (t.type == 'delimiter' && t.subtype == 'right') {
|
||||
level--;
|
||||
tokens[i].level = level;
|
||||
} else {
|
||||
tokens[i].level = level;
|
||||
}
|
||||
});
|
||||
// Group.
|
||||
let currentLevel = 0;
|
||||
let groupStack = [0];
|
||||
tokens.forEach((t, i) => {
|
||||
if (currentLevel < t.level) {
|
||||
tokens.splice(i, 0, new Group(tokens[i - 1].value, []));
|
||||
groupStack.push(i);
|
||||
currentLevel++;
|
||||
tokens[i].level = currentLevel;
|
||||
}
|
||||
if (t.level != 0) {
|
||||
tokens[groupStack.slice(-1)].tokens.push(t);
|
||||
}
|
||||
if (currentLevel > t.level) {
|
||||
groupStack.pop();
|
||||
currentLevel--;
|
||||
}
|
||||
});
|
||||
if (currentLevel != 0) {} // Error: missing delimiter.
|
||||
return tokens;
|
||||
};
|
||||
303
src/tokenizer.js
Normal file
303
src/tokenizer.js
Normal file
@ -0,0 +1,303 @@
|
||||
/**
|
||||
* @module tokenizer
|
||||
* @file Manages the tokenization phase of Pivot.
|
||||
* @author Garen Tyler <garentyler@gmail.com>
|
||||
* @requires module:types
|
||||
*/
|
||||
const Token = require('./types.js').Token;
|
||||
const Group = require('./types.js').Token;
|
||||
|
||||
/**
|
||||
* @function tokenize
|
||||
* @desc Takes in raw code, and outputs an array of Tokens.
|
||||
* @param {string} code The raw input code.
|
||||
* @returns {Token[]} The code, split into tokens.
|
||||
* @public
|
||||
*/
|
||||
function tokenize(code) {
|
||||
// Split the string into an array of chars.
|
||||
let chars = code.split('');
|
||||
|
||||
// Create buffers.
|
||||
let letterBuffer = [];
|
||||
let operatorBuffer = [];
|
||||
let numberBuffer = [];
|
||||
let stringBuffer = [];
|
||||
|
||||
// Create the output Token[].
|
||||
let tokens = [];
|
||||
|
||||
// Create an object to keep track of string data.
|
||||
let stringData = {
|
||||
inString: false,
|
||||
stringType: null
|
||||
};
|
||||
|
||||
// Escape chars and remove comments.
|
||||
chars = combineEscapedChars(chars);
|
||||
chars = removeComments(chars);
|
||||
|
||||
|
||||
// Actually tokenize the chars.
|
||||
for (let i = 0; i < chars.length; i++) {
|
||||
let char = chars[i];
|
||||
if (stringData.inString) { // Tokenize differently in a string.
|
||||
// If a string delimiter and the same as the inital delimiter.
|
||||
if (determineCharType(char) == 'string delimiter' && char == stringData.stringType) {
|
||||
stringData.inString = false; // Not in a string any more.
|
||||
tokens.push(new Token('string', 'n/a', stringBuffer.join(''))); // Push the string.
|
||||
stringBuffer = []; // Clear the string buffer.
|
||||
} else stringBuffer.push(char); // Add to the string buffer.
|
||||
} else { // Tokenize normally.
|
||||
if (determineCharType(char) == 'string delimiter') {
|
||||
stringData.inString = true; // In a string now.
|
||||
stringData.stringType = char;
|
||||
} else if (determineCharType(char) == 'letter') {
|
||||
letterBuffer.push(char); // Add to the letter buffer.
|
||||
// End the other buffers.
|
||||
if (operatorBuffer.length > 0) {
|
||||
let operator = operatorBuffer.join('');
|
||||
tokens.push(new Token('operator', operatorType(operator), operator));
|
||||
operatorBuffer = [];
|
||||
}
|
||||
if (numberBuffer.length > 0) {
|
||||
let number = numberBuffer.join('');
|
||||
tokens.push(new Token('number', 'n/a', number));
|
||||
numberBuffer = [];
|
||||
}
|
||||
} else if (determineCharType(char) == 'operator') {
|
||||
operatorBuffer.push(char); // Add to the operator buffer.
|
||||
// End the other buffers.
|
||||
if (letterBuffer.length > 0) {
|
||||
let variable = letterBuffer.join('');
|
||||
tokens.push(new Token('name', 'variable', variable));
|
||||
letterBuffer = [];
|
||||
}
|
||||
if (numberBuffer.length > 0) {
|
||||
let number = numberBuffer.join('');
|
||||
tokens.push(new Token('number', 'n/a', number));
|
||||
numberBuffer = [];
|
||||
}
|
||||
} else if (determineCharType(char) == 'digit') {
|
||||
numberBuffer.push(char); // Add to the number buffer.
|
||||
// End the other buffers.
|
||||
if (letterBuffer.length > 0) {
|
||||
let variable = letterBuffer.join('');
|
||||
tokens.push(new Token('name', 'variable', variable));
|
||||
letterBuffer = [];
|
||||
}
|
||||
if (operatorBuffer.length > 0) {
|
||||
let operator = operatorBuffer.join('');
|
||||
tokens.push(new Token('operator', operatorType(operator), operator));
|
||||
operatorBuffer = [];
|
||||
}
|
||||
} else if (determineCharType(char) == 'whitespace') {
|
||||
// End all buffers.
|
||||
if (letterBuffer.length > 0) {
|
||||
let variable = letterBuffer.join('');
|
||||
tokens.push(new Token('name', 'variable', variable));
|
||||
letterBuffer = [];
|
||||
}
|
||||
if (numberBuffer.length > 0) {
|
||||
let number = numberBuffer.join('');
|
||||
tokens.push(new Token('number', 'n/a', number));
|
||||
numberBuffer = [];
|
||||
}
|
||||
if (operatorBuffer.length > 0) {
|
||||
let operator = operatorBuffer.join('');
|
||||
tokens.push(new Token('operator', operatorType(operator), operator));
|
||||
operatorBuffer = [];
|
||||
}
|
||||
} else if (determineCharType(char) == 'delimiter') {
|
||||
// End all buffers.
|
||||
if (letterBuffer.length > 0) {
|
||||
let variable = letterBuffer.join('');
|
||||
tokens.push(new Token('name', 'variable', variable));
|
||||
letterBuffer = [];
|
||||
}
|
||||
if (numberBuffer.length > 0) {
|
||||
let number = numberBuffer.join('');
|
||||
tokens.push(new Token('number', 'n/a', number));
|
||||
numberBuffer = [];
|
||||
}
|
||||
if (operatorBuffer.length > 0) {
|
||||
let operator = operatorBuffer.join('');
|
||||
tokens.push(new Token('operator', operatorType(operator), operator));
|
||||
operatorBuffer = [];
|
||||
}
|
||||
// Push the delimiter.
|
||||
tokens.push(getDelimiterToken(char));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Empty all the buffers.
|
||||
if (letterBuffer.length > 0) {
|
||||
let variable = letterBuffer.join('');
|
||||
tokens.push(new Token('name', 'variable', variable));
|
||||
letterBuffer = [];
|
||||
}
|
||||
if (numberBuffer.length > 0) {
|
||||
let number = numberBuffer.join('');
|
||||
tokens.push(new Token('number', 'n/a', number));
|
||||
numberBuffer = [];
|
||||
}
|
||||
if (operatorBuffer.length > 0) {
|
||||
let operator = operatorBuffer.join('');
|
||||
tokens.push(new Token('operator', operatorType(operator), operator));
|
||||
operatorBuffer = [];
|
||||
}
|
||||
|
||||
tokens = changeKeywords(tokens);
|
||||
|
||||
return tokens;
|
||||
}
|
||||
|
||||
/**
|
||||
* @function combineEscapedChars
|
||||
* @desc Combines escaped chars into one char.
|
||||
* @param {string[]} chars The chars.
|
||||
* @returns {string[]} The chars with combined escaped chars.
|
||||
* @private
|
||||
*/
|
||||
function combineEscapedChars(chars) {
|
||||
// Check for characters to be escaped.
|
||||
for (let i = 0; i < chars.length; i++) {
|
||||
if (chars[i] == '\\') {
|
||||
chars.splice(i, 2, chars[i] + chars[i + 1]);
|
||||
i -= 2;
|
||||
}
|
||||
}
|
||||
return chars;
|
||||
}
|
||||
|
||||
/**
|
||||
* @function removeComments
|
||||
* @desc Removes comments.
|
||||
* @param {string[]} chars The chars.
|
||||
* @returns {string[]} The chars without comments.
|
||||
* @private
|
||||
*/
|
||||
function removeComments(chars) {
|
||||
let inComment = false; // Keep track if in a comment.
|
||||
for (let i = 0; i < chars.length; i++) {
|
||||
if (chars[i] == '/') {
|
||||
if (chars[i + 1] == '/') {
|
||||
inComment = true;
|
||||
}
|
||||
}
|
||||
if (chars[i] == '\n') {
|
||||
inComment = false;
|
||||
chars.splice(i, 1); // Remove the newline at the end of the comment.
|
||||
i--;
|
||||
}
|
||||
if (inComment) {
|
||||
chars.splice(i, 1); // Remove the char in the comment.
|
||||
i--;
|
||||
}
|
||||
}
|
||||
return chars;
|
||||
}
|
||||
|
||||
/**
|
||||
* @function changeKeywords
|
||||
* @desc Changes tokens with subtype variable to subtype keyword
|
||||
* @param {Token[]} tokens The tokens
|
||||
* @returns {Token[]} The tokens with keywords.
|
||||
* @private
|
||||
*/
|
||||
function changeKeywords(tokens) {
|
||||
return tokens.map(t => {
|
||||
if (t.subtype == 'variable' && determineType(t.value) == 'keyword') {
|
||||
t.subtype = 'keyword';
|
||||
}
|
||||
return t;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* @function getDelimiterToken
|
||||
* @desc Turns a delimiter char into a token.
|
||||
* @param {string} delimiter The delimiter char.
|
||||
* @returns {Token} The delimiter token.
|
||||
* @private
|
||||
*/
|
||||
function getDelimiterToken(delimiter) {
|
||||
if (/\(|\)/.test(delimiter))
|
||||
return new Token('delimiter', delimiter == '(' ? 'left' : 'right', 'parenthesis');
|
||||
else if (/\[|\]/.test(delimiter))
|
||||
return new Token('delimiter', delimiter == '[' ? 'left' : 'right', 'bracket');
|
||||
else if (/\{|\}/.test(delimiter))
|
||||
return new Token('delimiter', delimiter == '{' ? 'left' : 'right', 'brace');
|
||||
else throw new Error('Expected delimiter but got ' + delimiter);
|
||||
}
|
||||
|
||||
/**
|
||||
* @function operatorType
|
||||
* @desc Determines the type of operator.
|
||||
* @param {string} operator The operator char.
|
||||
* @returns {string} The type of operator.
|
||||
* @private
|
||||
*/
|
||||
function operatorType(operator) {
|
||||
// Left operators have parameters on the left.
|
||||
if (/\+\+|--/.test(operator))
|
||||
return 'left';
|
||||
else if (false)
|
||||
return 'right';
|
||||
else if (/\;/.test(operator))
|
||||
return 'none';
|
||||
else
|
||||
return 'dual';
|
||||
}
|
||||
|
||||
/**
|
||||
* @function determineCharType
|
||||
* @desc Detects the type of characters.
|
||||
* @param {string} char The input character(s).
|
||||
* @returns {string} The type of char.
|
||||
* @private
|
||||
*/
|
||||
function determineCharType(char) {
|
||||
if (/[A-Za-z]/.test(char))
|
||||
return 'letter';
|
||||
else if (/\+|\-|\*|\/|\=|\=\=|\>|\<|\>\=|\<\=|\=\>|;/.test(char))
|
||||
return 'operator';
|
||||
else if (/\(|\)|\[|\]|\{|\}/.test(char))
|
||||
return 'delimiter';
|
||||
else if (/'|"|`/.test(char))
|
||||
return 'string delimiter';
|
||||
else if (/\d/.test(char))
|
||||
return 'digit';
|
||||
else if (/\\./.test(char))
|
||||
return 'escaped char';
|
||||
else if (/\s/.test(char))
|
||||
return 'whitespace';
|
||||
else throw new SyntaxError('Unexpected char ' + char);
|
||||
};
|
||||
|
||||
/**
|
||||
* @function determineType
|
||||
* @desc Detects the type of a string.
|
||||
* @param {string} str The input string.
|
||||
* @returns {string} The type of string.
|
||||
* @private
|
||||
*/
|
||||
function determineType(str) {
|
||||
if (/let|return/.test(str))
|
||||
return 'keyword';
|
||||
else return 'unknown';
|
||||
};
|
||||
|
||||
module.exports = {
|
||||
tokenize,
|
||||
util: {
|
||||
combineEscapedChars,
|
||||
removeComments,
|
||||
changeKeywords,
|
||||
getDelimiterToken,
|
||||
operatorType,
|
||||
determineCharType,
|
||||
determineType
|
||||
}
|
||||
};
|
||||
31
src/types.js
Normal file
31
src/types.js
Normal file
@ -0,0 +1,31 @@
|
||||
/**
|
||||
* @module types
|
||||
* @file Provides a consistent source of types for the compiler.
|
||||
* @author Garen Tyler <garentyler@gmail.com>
|
||||
*/
|
||||
|
||||
|
||||
/**
|
||||
* @class Token
|
||||
* @classdesc Stores the type of token, subtype, and literal char value.
|
||||
*/
|
||||
function Token(type, subtype, value) {
|
||||
this.type = type;
|
||||
this.subtype = subtype;
|
||||
this.value = value;
|
||||
}
|
||||
|
||||
/**
|
||||
* @class Group
|
||||
* @classdesc Stores the type of group, and the tokens in the group.
|
||||
*/
|
||||
function Group(type, tokens) {
|
||||
this.type = 'group'
|
||||
this.subtype = type;
|
||||
this.tokens = tokens;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
Token,
|
||||
Group
|
||||
};
|
||||
@ -1,6 +0,0 @@
|
||||
// Example Pivot code.
|
||||
// This is a comment.
|
||||
x = 4 // This creates a variable x with the value of 4.
|
||||
y = 3 * x // This creates a variable y with the value of the output of (3 * x).
|
||||
asdf = (a) {return(a++)} // This creates a function asdf that takes a number and returns that number + 1/
|
||||
a = asdf(2) // This creates a variable a with the value of the output of asdf(2).
|
||||
126
test/test.js
Normal file
126
test/test.js
Normal file
@ -0,0 +1,126 @@
|
||||
const assert = require('assert');
|
||||
|
||||
const types = require('../src/types.js');
|
||||
const tokenizer = require('../src/tokenizer.js');
|
||||
const parser = require('../src/parser.js');
|
||||
|
||||
describe('types.js', () => {
|
||||
it('Has a Token child.', () => {
|
||||
assert.equal(types.hasOwnProperty('Token'), true);
|
||||
});
|
||||
it('Has a Group child.', () => {
|
||||
assert.equal(types.hasOwnProperty('Group'), true);
|
||||
});
|
||||
|
||||
describe('Token', () => {
|
||||
it('Works as a constructor', () => {
|
||||
try {
|
||||
let token = new types.Token('a', 'b', 'c');
|
||||
} catch (err) {
|
||||
throw err;
|
||||
}
|
||||
});
|
||||
it('Has values \'type\', \'subtype\', and \'value\'', () => {
|
||||
try {
|
||||
let token = new types.Token('a', 'b', 'c');
|
||||
if (!token.hasOwnProperty('type') || !token.hasOwnProperty('subtype') || !token.hasOwnProperty('value'))
|
||||
throw new Error('Token is missing \'type\', \'subtype\', or \'value\' properties.');
|
||||
if (token.type != 'a' || token.subtype != 'b' || token.value != 'c')
|
||||
throw new Error('Token incorrectly set \'type\', \'subtype\', or \'value\' properties.');
|
||||
} catch (err) {
|
||||
throw err;
|
||||
}
|
||||
});
|
||||
});
|
||||
describe('Group', () => {
|
||||
it('Works as a constructor', () => {
|
||||
try {
|
||||
let group = new types.Group('a', 'b');
|
||||
} catch (err) {
|
||||
throw err;
|
||||
}
|
||||
});
|
||||
it('Has values \'type\', \'subtype\', and \'tokens\'', () => {
|
||||
try {
|
||||
let group = new types.Group('a', 'b');
|
||||
if (!group.hasOwnProperty('type') || !group.hasOwnProperty('subtype') || !group.hasOwnProperty('tokens'))
|
||||
throw new Error('Group is missing \'type\', \'subtype\', or \'tokens\' properties.');
|
||||
if (group.type != 'group' || group.subtype != 'a' || group.tokens != 'b')
|
||||
throw new Error('Group incorrectly set \'type\', \'subtype\', or \'tokens\' properties.');
|
||||
} catch (err) {
|
||||
throw err;
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
describe('tokenizer.js', () => {
|
||||
it('Has a tokenize child', () => {
|
||||
assert.equal(tokenizer.hasOwnProperty('tokenize'), true);
|
||||
});
|
||||
it('Has a util child', () => {
|
||||
assert.equal(tokenizer.hasOwnProperty('util'), true);
|
||||
});
|
||||
describe('util', () => {
|
||||
it('combineEscapedChars works', () => {
|
||||
assert.equal(tokenizer.util.combineEscapedChars(`let x = 'test\\nnewline';`.split('')).join(''), `let x = 'test\\nnewline';`);
|
||||
});
|
||||
it('removeComments works', () => {
|
||||
assert.equal(tokenizer.util.removeComments(`// Comment\nlet i = 0;`.split('')).join(''), `let i = 0;`);
|
||||
});
|
||||
it('changeKeywords works', () => {
|
||||
let tokens = tokenizer.util.changeKeywords([{
|
||||
type: 'name',
|
||||
subtype: 'variable',
|
||||
value: 'let'
|
||||
}, {
|
||||
type: 'name',
|
||||
subtype: 'variable',
|
||||
value: 'x'
|
||||
}]);
|
||||
let correct = [{
|
||||
type: 'name',
|
||||
subtype: 'keyword',
|
||||
value: 'let'
|
||||
}, {
|
||||
type: 'name',
|
||||
subtype: 'variable',
|
||||
value: 'x'
|
||||
}];
|
||||
let isCorrect = true;
|
||||
tokens.forEach((t, i) => {
|
||||
if (t.type != correct[i].type)
|
||||
throw new Error('Changed type: Expected \''+ correct[i].type +'\' but got ' + t.type)
|
||||
else if (t.subtype != correct[i].subtype)
|
||||
throw new Error('Incorrectly changed subtype: Expected \''+ correct[i].subtype +'\' but got ' + t.subtype)
|
||||
else if (t.value != correct[i].value)
|
||||
throw new Error('Changed value: Expected \''+ correct[i].value +'\' but got ' + t.value)
|
||||
});
|
||||
});
|
||||
it('getDelimiterToken works', () => {
|
||||
let token = tokenizer.util.getDelimiterToken(')');
|
||||
if (token.type != 'delimiter')
|
||||
throw new Error('Incorrect type: Expected \'delimiter\' but got ' + token.type)
|
||||
else if (token.subtype != 'right')
|
||||
throw new Error('Incorrect subtype: Expected \'right\' but got ' + token.subtype)
|
||||
else if (token.value != 'parenthesis')
|
||||
throw new Error('Incorrect value: Expected \'parenthesis\' but got ' + token.value)
|
||||
});
|
||||
it('operatorType works', () => {
|
||||
assert.equal(tokenizer.util.operatorType('++'), 'left');
|
||||
assert.equal(tokenizer.util.operatorType(';'), 'none');
|
||||
assert.equal(tokenizer.util.operatorType('+'), 'dual');
|
||||
});
|
||||
it('determineCharType works', () => {
|
||||
assert.equal(tokenizer.util.determineCharType('+'), 'operator');
|
||||
assert.equal(tokenizer.util.determineCharType('"'), 'string delimiter');
|
||||
assert.equal(tokenizer.util.determineCharType('4'), 'digit');
|
||||
});
|
||||
it('determineType works', () => {
|
||||
assert.equal(tokenizer.util.determineType('let'), 'keyword');
|
||||
assert.equal(tokenizer.util.determineType('dog'), 'unknown');
|
||||
});
|
||||
});
|
||||
});
|
||||
describe('parser.js', () => {
|
||||
|
||||
});
|
||||
170
tokenizer.js
170
tokenizer.js
@ -1,170 +0,0 @@
|
||||
// Import the token class.
|
||||
const token = require("./classes.js").token;
|
||||
|
||||
// Create the tokenizer function.
|
||||
// tokenize() takes Pivot code in, and outputs an array of tokens.
|
||||
module.exports = exp => {
|
||||
// Check functions for different token types.
|
||||
var isDigit = char => {
|
||||
return /\d/.test(char);
|
||||
};
|
||||
var isLetter = char => {
|
||||
return /[a-z]/i.test(char);
|
||||
};
|
||||
var isOperator = char => {
|
||||
return /\+|-|\*|\/|\^|=/.test(char);
|
||||
};
|
||||
var isLeftDelimiter = char => {
|
||||
return (/\(|\[|\{|"|'|`/.test(char));
|
||||
};
|
||||
var isRightDelimiter = char => {
|
||||
return (/\)|\]|\}/.test(char));
|
||||
};
|
||||
var isComma = char => {
|
||||
return (char === ",");
|
||||
};
|
||||
var isPeriod = char => {
|
||||
return (char === ".");
|
||||
};
|
||||
var result = []; // The final array of tokens.
|
||||
var nb = []; // Number buffer. Allows for multiple digits to be one number.
|
||||
var lb = []; // Letter buffer. Allows for multiple letters to be one variable / function.
|
||||
var ob = []; // Operator buffer. Allows for multi-character operators. E.g. ++ or ==.
|
||||
var sb = []; // String buffer. Allows for multi-character strings.
|
||||
var inString = false; // Keep track of whether in string or not.
|
||||
var stringType; // Keep track of what type of string. E.g. "" or ''.
|
||||
exp = exp.split(""); // Split the expression into an array of characters.
|
||||
/* - - - DO NOT TOUCH THIS - - - */
|
||||
for (var i = 0; i < exp.length; i++) { // Loop through all of the characters.
|
||||
var char = exp[i]; // Create a quick reference to the current char.
|
||||
if (i >= 1) {
|
||||
if (exp[i - 1] == "\\") {
|
||||
exp.splice(i - 1, 2, `\\${char}`);
|
||||
i--;
|
||||
continue;
|
||||
}
|
||||
if (exp[i - 1] == "$" && char == "{") {
|
||||
exp.splice(i - 1, 2, `\${`);
|
||||
i--;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
/* - - - OK YOU CAN TOUCH AGAIN - - - */
|
||||
// Nevermind, just don't mess with any of this file.
|
||||
for (var i = 0; i < exp.length; i++) {
|
||||
var char = exp[i];
|
||||
if (inString) {
|
||||
if (char == `'` || char == `"` || char == "`") {
|
||||
var exitString = () => {
|
||||
inString = false;
|
||||
if (sb.length == 0) {
|
||||
result.push(new token("String", null));
|
||||
} else {
|
||||
var string = sb.join("");
|
||||
result.push(new token("String", string));
|
||||
}
|
||||
sb = [];
|
||||
};
|
||||
if (char == `'` && stringType == "single") {
|
||||
exitString();
|
||||
} else if (char == `"` && stringType == "double") {
|
||||
exitString();
|
||||
} else if (char == "`" && stringType == "backtick") {
|
||||
exitString();
|
||||
} else {
|
||||
if (char == `'`) {
|
||||
sb.push(`\'`);
|
||||
}
|
||||
if (char == `"`) {
|
||||
sb.push(`\"`);
|
||||
}
|
||||
if (char == "`") {
|
||||
sb.push("\`");
|
||||
}
|
||||
}
|
||||
} else {
|
||||
sb.push(char);
|
||||
}
|
||||
} else {
|
||||
if (isDigit(char)) {
|
||||
result.push(new token("Operator", ob.join("")));
|
||||
ob = [];
|
||||
nb.push(char);
|
||||
} else if (isLetter(char)) {
|
||||
result.push(new token("Operator", ob.join("")));
|
||||
ob = [];
|
||||
lb.push(char);
|
||||
} else if (isOperator(char)) {
|
||||
result.push(new token("Number", nb.join("")));
|
||||
nb = [];
|
||||
result.push(new token("Variable", lb.join("")));
|
||||
lb = [];
|
||||
ob.push(char);
|
||||
} else if (isLeftDelimiter(char)) {
|
||||
result.push(new token("Operator", ob.join("")));
|
||||
ob = [];
|
||||
result.push(new token("Function Call", lb.join("")));
|
||||
lb = [];
|
||||
if (char == `'` || char == `"` || char == "`") {
|
||||
inString = true;
|
||||
if (char == `'`) {
|
||||
stringType = "single";
|
||||
} else if (char == `"`) {
|
||||
stringType = "double";
|
||||
} else if (char == "`") {
|
||||
stringType = "backtick";
|
||||
}
|
||||
} else {
|
||||
result.push(new token("Left Delimiter", char));
|
||||
}
|
||||
} else if (isRightDelimiter(char)) {
|
||||
result.push(new token("Operator", ob.join("")));
|
||||
ob = [];
|
||||
result.push(new token("Number", nb.join("")));
|
||||
nb = [];
|
||||
result.push(new token("Variable", lb.join("")));
|
||||
lb = [];
|
||||
result.push(new token("Right Delimiter", char));
|
||||
} else if (isComma(char)) {
|
||||
result.push(new token("Operator", ob.join("")));
|
||||
ob = [];
|
||||
result.push(new token("Number", nb.join("")));
|
||||
nb = [];
|
||||
result.push(new token("Variable", lb.join("")));
|
||||
lb = [];
|
||||
result.push(new token("Comma", char));
|
||||
} else if (isPeriod(char)) {
|
||||
result.push(new token("Operator", ob.join("")));
|
||||
ob = [];
|
||||
nb.push(char);
|
||||
}
|
||||
}
|
||||
}
|
||||
result.push(new token("Operator", ob.join("")));
|
||||
ob = [];
|
||||
result.push(new token("Number", nb.join("")));
|
||||
nb = [];
|
||||
lb.forEach(item => {
|
||||
result.push(new token("Variable", item));
|
||||
});
|
||||
lb = [];
|
||||
for (var i = 0; i < 3; i++) {
|
||||
result.forEach((item, index) => {
|
||||
if (item.value == "") {
|
||||
result.splice(index, 1);
|
||||
}
|
||||
});
|
||||
}
|
||||
result.forEach((item, index) => {
|
||||
if (item.value == "-" && index != 0) {
|
||||
if (result[index - 1].type != "Variable" && result[index - 1].type != "Number") {
|
||||
if (result[index + 1].type == "Number") {
|
||||
result[index + 1].value = "-" + result[index + 1].value;
|
||||
result.splice(index, 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
return result;
|
||||
};
|
||||
Loading…
x
Reference in New Issue
Block a user