Spaces:
Sleeping
Sleeping
; | |
Object.defineProperty(exports, "__esModule", { | |
value: true | |
}); | |
exports.tokenize = tokenize; | |
exports.tokens = exports.keywords = void 0; | |
var _helperFsm = require("@webassemblyjs/helper-fsm"); | |
var _helperCodeFrame = require("@webassemblyjs/helper-code-frame"); | |
// eslint-disable-next-line | |
function getCodeFrame(source, line, column) { | |
var loc = { | |
start: { | |
line: line, | |
column: column | |
} | |
}; | |
return "\n" + (0, _helperCodeFrame.codeFrameFromSource)(source, loc) + "\n"; | |
} | |
var WHITESPACE = /\s/; | |
var PARENS = /\(|\)/; | |
var LETTERS = /[a-z0-9_/]/i; | |
var idchar = /[a-z0-9!#$%&*+./:<=>?@\\[\]^_`|~-]/i; | |
var valtypes = ["i32", "i64", "f32", "f64"]; | |
var NUMBERS = /[0-9|.|_]/; | |
var NUMBER_KEYWORDS = /nan|inf/; | |
function isNewLine(char) { | |
return char.charCodeAt(0) === 10 || char.charCodeAt(0) === 13; | |
} | |
function Token(type, value, start, end) { | |
var opts = arguments.length > 4 && arguments[4] !== undefined ? arguments[4] : {}; | |
var token = { | |
type: type, | |
value: value, | |
loc: { | |
start: start, | |
end: end | |
} | |
}; | |
if (Object.keys(opts).length > 0) { | |
// $FlowIgnore | |
token["opts"] = opts; | |
} | |
return token; | |
} | |
var tokenTypes = { | |
openParen: "openParen", | |
closeParen: "closeParen", | |
number: "number", | |
string: "string", | |
name: "name", | |
identifier: "identifier", | |
valtype: "valtype", | |
dot: "dot", | |
comment: "comment", | |
equal: "equal", | |
keyword: "keyword" | |
}; | |
var keywords = { | |
module: "module", | |
func: "func", | |
param: "param", | |
result: "result", | |
export: "export", | |
loop: "loop", | |
block: "block", | |
if: "if", | |
then: "then", | |
else: "else", | |
call: "call", | |
call_indirect: "call_indirect", | |
import: "import", | |
memory: "memory", | |
table: "table", | |
global: "global", | |
anyfunc: "anyfunc", | |
mut: "mut", | |
data: "data", | |
type: "type", | |
elem: "elem", | |
start: "start", | |
offset: "offset" | |
}; | |
exports.keywords = keywords; | |
var NUMERIC_SEPARATOR = "_"; | |
/** | |
* Build the FSM for number literals | |
*/ | |
var numberLiteralFSM = new _helperFsm.FSM({ | |
START: [(0, _helperFsm.makeTransition)(/-|\+/, "AFTER_SIGN"), (0, _helperFsm.makeTransition)(/nan:0x/, "NAN_HEX", { | |
n: 6 | |
}), (0, _helperFsm.makeTransition)(/nan|inf/, "STOP", { | |
n: 3 | |
}), (0, _helperFsm.makeTransition)(/0x/, "HEX", { | |
n: 2 | |
}), (0, _helperFsm.makeTransition)(/[0-9]/, "DEC"), (0, _helperFsm.makeTransition)(/\./, "DEC_FRAC")], | |
AFTER_SIGN: [(0, _helperFsm.makeTransition)(/nan:0x/, "NAN_HEX", { | |
n: 6 | |
}), (0, _helperFsm.makeTransition)(/nan|inf/, "STOP", { | |
n: 3 | |
}), (0, _helperFsm.makeTransition)(/0x/, "HEX", { | |
n: 2 | |
}), (0, _helperFsm.makeTransition)(/[0-9]/, "DEC"), (0, _helperFsm.makeTransition)(/\./, "DEC_FRAC")], | |
DEC_FRAC: [(0, _helperFsm.makeTransition)(/[0-9]/, "DEC_FRAC", { | |
allowedSeparator: NUMERIC_SEPARATOR | |
}), (0, _helperFsm.makeTransition)(/e|E/, "DEC_SIGNED_EXP")], | |
DEC: [(0, _helperFsm.makeTransition)(/[0-9]/, "DEC", { | |
allowedSeparator: NUMERIC_SEPARATOR | |
}), (0, _helperFsm.makeTransition)(/\./, "DEC_FRAC"), (0, _helperFsm.makeTransition)(/e|E/, "DEC_SIGNED_EXP")], | |
DEC_SIGNED_EXP: [(0, _helperFsm.makeTransition)(/\+|-/, "DEC_EXP"), (0, _helperFsm.makeTransition)(/[0-9]/, "DEC_EXP")], | |
DEC_EXP: [(0, _helperFsm.makeTransition)(/[0-9]/, "DEC_EXP", { | |
allowedSeparator: NUMERIC_SEPARATOR | |
})], | |
HEX: [(0, _helperFsm.makeTransition)(/[0-9|A-F|a-f]/, "HEX", { | |
allowedSeparator: NUMERIC_SEPARATOR | |
}), (0, _helperFsm.makeTransition)(/\./, "HEX_FRAC"), (0, _helperFsm.makeTransition)(/p|P/, "HEX_SIGNED_EXP")], | |
HEX_FRAC: [(0, _helperFsm.makeTransition)(/[0-9|A-F|a-f]/, "HEX_FRAC", { | |
allowedSeparator: NUMERIC_SEPARATOR | |
}), (0, _helperFsm.makeTransition)(/p|P|/, "HEX_SIGNED_EXP")], | |
HEX_SIGNED_EXP: [(0, _helperFsm.makeTransition)(/[0-9|+|-]/, "HEX_EXP")], | |
HEX_EXP: [(0, _helperFsm.makeTransition)(/[0-9]/, "HEX_EXP", { | |
allowedSeparator: NUMERIC_SEPARATOR | |
})], | |
NAN_HEX: [(0, _helperFsm.makeTransition)(/[0-9|A-F|a-f]/, "NAN_HEX", { | |
allowedSeparator: NUMERIC_SEPARATOR | |
})], | |
STOP: [] | |
}, "START", "STOP"); | |
function tokenize(input) { | |
var current = 0; | |
var char = input[current]; // Used by SourceLocation | |
var column = 1; | |
var line = 1; | |
var tokens = []; | |
/** | |
* Creates a pushToken function for a given type | |
*/ | |
function pushToken(type) { | |
return function (v) { | |
var opts = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : {}; | |
var startColumn = opts.startColumn || column - String(v).length; | |
delete opts.startColumn; | |
var endColumn = opts.endColumn || startColumn + String(v).length - 1; | |
delete opts.endColumn; | |
var start = { | |
line: line, | |
column: startColumn | |
}; | |
var end = { | |
line: line, | |
column: endColumn | |
}; | |
tokens.push(Token(type, v, start, end, opts)); | |
}; | |
} | |
/** | |
* Functions to save newly encountered tokens | |
*/ | |
var pushCloseParenToken = pushToken(tokenTypes.closeParen); | |
var pushOpenParenToken = pushToken(tokenTypes.openParen); | |
var pushNumberToken = pushToken(tokenTypes.number); | |
var pushValtypeToken = pushToken(tokenTypes.valtype); | |
var pushNameToken = pushToken(tokenTypes.name); | |
var pushIdentifierToken = pushToken(tokenTypes.identifier); | |
var pushKeywordToken = pushToken(tokenTypes.keyword); | |
var pushDotToken = pushToken(tokenTypes.dot); | |
var pushStringToken = pushToken(tokenTypes.string); | |
var pushCommentToken = pushToken(tokenTypes.comment); | |
var pushEqualToken = pushToken(tokenTypes.equal); | |
/** | |
* Can be used to look at the next character(s). | |
* | |
* The default behavior `lookahead()` simply returns the next character without consuming it. | |
* Letters are always returned in lowercase. | |
* | |
* @param {number} length How many characters to query. Default = 1 | |
* @param {number} offset How many characters to skip forward from current one. Default = 1 | |
* | |
*/ | |
function lookahead() { | |
var length = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : 1; | |
var offset = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : 1; | |
return input.substring(current + offset, current + offset + length).toLowerCase(); | |
} | |
/** | |
* Advances the cursor in the input by a certain amount | |
* | |
* @param {number} amount How many characters to consume. Default = 1 | |
*/ | |
function eatCharacter() { | |
var amount = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : 1; | |
column += amount; | |
current += amount; | |
char = input[current]; | |
} | |
while (current < input.length) { | |
// ;; | |
if (char === ";" && lookahead() === ";") { | |
var startColumn = column; | |
eatCharacter(2); | |
var text = ""; | |
while (!isNewLine(char)) { | |
text += char; | |
eatCharacter(); | |
if (char === undefined) { | |
break; | |
} | |
} | |
var endColumn = column; | |
pushCommentToken(text, { | |
type: "leading", | |
startColumn: startColumn, | |
endColumn: endColumn | |
}); | |
continue; | |
} // (; | |
if (char === "(" && lookahead() === ";") { | |
var _startColumn = column; | |
eatCharacter(2); | |
var _text = ""; // ;) | |
while (true) { | |
char = input[current]; | |
if (char === ";" && lookahead() === ")") { | |
eatCharacter(2); | |
break; | |
} | |
_text += char; | |
eatCharacter(); | |
if (isNewLine(char)) { | |
line++; | |
column = 0; | |
} | |
} | |
var _endColumn = column; | |
pushCommentToken(_text, { | |
type: "block", | |
startColumn: _startColumn, | |
endColumn: _endColumn | |
}); | |
continue; | |
} | |
if (char === "(") { | |
pushOpenParenToken(char); | |
eatCharacter(); | |
continue; | |
} | |
if (char === "=") { | |
pushEqualToken(char); | |
eatCharacter(); | |
continue; | |
} | |
if (char === ")") { | |
pushCloseParenToken(char); | |
eatCharacter(); | |
continue; | |
} | |
if (isNewLine(char)) { | |
line++; | |
eatCharacter(); | |
column = 0; | |
continue; | |
} | |
if (WHITESPACE.test(char)) { | |
eatCharacter(); | |
continue; | |
} | |
if (char === "$") { | |
var _startColumn2 = column; | |
eatCharacter(); | |
var value = ""; | |
while (idchar.test(char)) { | |
value += char; | |
eatCharacter(); | |
} | |
var _endColumn2 = column; | |
pushIdentifierToken(value, { | |
startColumn: _startColumn2, | |
endColumn: _endColumn2 | |
}); | |
continue; | |
} | |
if (NUMBERS.test(char) || NUMBER_KEYWORDS.test(lookahead(3, 0)) || char === "-" || char === "+") { | |
var _startColumn3 = column; | |
var _value = numberLiteralFSM.run(input.slice(current)); | |
if (_value === "") { | |
throw new Error(getCodeFrame(input, line, column) + "Unexpected character " + JSON.stringify(char)); | |
} | |
pushNumberToken(_value, { | |
startColumn: _startColumn3 | |
}); | |
eatCharacter(_value.length); | |
if (char && !PARENS.test(char) && !WHITESPACE.test(char)) { | |
throw new Error(getCodeFrame(input, line, column) + "Unexpected character " + JSON.stringify(char)); | |
} | |
continue; | |
} | |
if (char === '"') { | |
var _startColumn4 = column; | |
var _value2 = ""; | |
eatCharacter(); // " | |
while (char !== '"') { | |
if (isNewLine(char)) { | |
throw new Error(getCodeFrame(input, line, column) + "Unexpected character " + JSON.stringify(char)); | |
} | |
_value2 += char; | |
eatCharacter(); // char | |
} | |
eatCharacter(); // " | |
var _endColumn3 = column; | |
pushStringToken(_value2, { | |
startColumn: _startColumn4, | |
endColumn: _endColumn3 | |
}); | |
continue; | |
} | |
if (LETTERS.test(char)) { | |
var _value3 = ""; | |
var _startColumn5 = column; | |
while (char && LETTERS.test(char)) { | |
_value3 += char; | |
eatCharacter(); | |
} | |
/* | |
* Handle MemberAccess | |
*/ | |
if (char === ".") { | |
var dotStartColumn = column; | |
if (valtypes.indexOf(_value3) !== -1) { | |
pushValtypeToken(_value3, { | |
startColumn: _startColumn5 | |
}); | |
} else { | |
pushNameToken(_value3); | |
} | |
eatCharacter(); | |
_value3 = ""; | |
var nameStartColumn = column; | |
while (LETTERS.test(char)) { | |
_value3 += char; | |
eatCharacter(); | |
} | |
pushDotToken(".", { | |
startColumn: dotStartColumn | |
}); | |
pushNameToken(_value3, { | |
startColumn: nameStartColumn | |
}); | |
continue; | |
} | |
/* | |
* Handle keywords | |
*/ | |
// $FlowIgnore | |
if (typeof keywords[_value3] === "string") { | |
pushKeywordToken(_value3, { | |
startColumn: _startColumn5 | |
}); | |
continue; | |
} | |
/* | |
* Handle types | |
*/ | |
if (valtypes.indexOf(_value3) !== -1) { | |
pushValtypeToken(_value3, { | |
startColumn: _startColumn5 | |
}); | |
continue; | |
} | |
/* | |
* Handle literals | |
*/ | |
pushNameToken(_value3, { | |
startColumn: _startColumn5 | |
}); | |
continue; | |
} | |
throw new Error(getCodeFrame(input, line, column) + "Unexpected character " + JSON.stringify(char)); | |
} | |
return tokens; | |
} | |
var tokens = tokenTypes; | |
exports.tokens = tokens; |