This commit is contained in:
Tutur33
2023-11-24 22:35:41 +01:00
parent 3c0b507a93
commit 7644b2a0f7
45165 changed files with 4803356 additions and 3 deletions
+3
View File
@@ -0,0 +1,3 @@
export { Tokenizer } from './src/tokenizer';
export * as utils from './src/utils';
export * from './src/types.js';
+33
View File
@@ -0,0 +1,33 @@
"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
__setModuleDefault(result, mod);
return result;
};
var __exportStar = (this && this.__exportStar) || function(m, exports) {
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.utils = exports.Tokenizer = void 0;
var tokenizer_1 = require("./src/tokenizer");
Object.defineProperty(exports, "Tokenizer", { enumerable: true, get: function () { return tokenizer_1.Tokenizer; } });
exports.utils = __importStar(require("./src/utils"));
__exportStar(require("./src/types.js"), exports);
+3
View File
@@ -0,0 +1,3 @@
import type { Tags, RuntimeTag, RuntimeComment, RuntimeMustache, LexerTagDefinitionContract } from './types';
export declare function getTag(content: string, filename: string, line: number, col: number, tags: Tags, claimTag?: (name: string) => LexerTagDefinitionContract | null): RuntimeTag | null;
export declare function getMustache(content: string, filename: string, line: number, col: number): RuntimeMustache | RuntimeComment | null;
+72
View File
@@ -0,0 +1,72 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.getMustache = exports.getTag = void 0;
const TAG_REGEX = /^(\s*)(@{1,2})(!)?([a-zA-Z._]+)(\s{0,2})/;
function getTag(content, filename, line, col, tags, claimTag) {
const match = TAG_REGEX.exec(content);
if (!match) {
return null;
}
const name = match[4];
let tag = tags[name];
if (!tag && claimTag) {
tag = claimTag(name);
}
if (!tag) {
return null;
}
const escaped = match[2] === '@@';
const selfclosed = !!match[3];
const whitespaceLeft = match[1].length;
const whitespaceRight = match[5].length;
const seekable = tag.seekable;
const block = tag.block;
const noNewLine = !!tag.noNewLine;
col += whitespaceLeft + match[2].length + name.length + whitespaceRight;
if (selfclosed) {
col++;
}
const hasBrace = seekable && content[col] === '(';
return {
name,
filename,
seekable,
selfclosed,
block,
line,
col,
escaped,
hasBrace,
noNewLine,
};
}
exports.getTag = getTag;
function getMustache(content, filename, line, col) {
const mustacheIndex = content.indexOf('{{');
if (mustacheIndex === -1) {
return null;
}
const realCol = mustacheIndex;
const isComment = content[mustacheIndex + 2] === '-' && content[mustacheIndex + 3] === '-';
if (isComment) {
return {
isComment,
filename,
line,
col: col + realCol,
realCol,
};
}
const safe = content[mustacheIndex + 2] === '{';
const escaped = content[mustacheIndex - 1] === '@';
return {
isComment,
safe,
filename,
escaped,
line,
col: col + realCol,
realCol,
};
}
exports.getMustache = getMustache;
+21
View File
@@ -0,0 +1,21 @@
import { EdgeError } from 'edge-error';
export declare function cannotSeekStatement(chars: string, pos: {
line: number;
col: number;
}, filename: string): EdgeError;
export declare function unclosedParen(pos: {
line: number;
col: number;
}, filename: string): EdgeError;
export declare function unopenedParen(pos: {
line: number;
col: number;
}, filename: string): EdgeError;
export declare function unclosedCurlyBrace(pos: {
line: number;
col: number;
}, filename: string): EdgeError;
export declare function unclosedTag(tag: string, pos: {
line: number;
col: number;
}, filename: string): EdgeError;
+44
View File
@@ -0,0 +1,44 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.unclosedTag = exports.unclosedCurlyBrace = exports.unopenedParen = exports.unclosedParen = exports.cannotSeekStatement = void 0;
const edge_error_1 = require("edge-error");
function cannotSeekStatement(chars, pos, filename) {
return new edge_error_1.EdgeError(`Unexpected token "${chars}"`, 'E_CANNOT_SEEK_STATEMENT', {
line: pos.line,
col: pos.col,
filename: filename,
});
}
exports.cannotSeekStatement = cannotSeekStatement;
function unclosedParen(pos, filename) {
return new edge_error_1.EdgeError('Missing token ")"', 'E_UNCLOSED_PAREN', {
line: pos.line,
col: pos.col,
filename: filename,
});
}
exports.unclosedParen = unclosedParen;
function unopenedParen(pos, filename) {
return new edge_error_1.EdgeError('Missing token "("', 'E_UNOPENED_PAREN', {
line: pos.line,
col: pos.col,
filename: filename,
});
}
exports.unopenedParen = unopenedParen;
function unclosedCurlyBrace(pos, filename) {
return new edge_error_1.EdgeError('Missing token "}"', 'E_UNCLOSED_CURLY_BRACE', {
line: pos.line,
col: pos.col,
filename: filename,
});
}
exports.unclosedCurlyBrace = unclosedCurlyBrace;
function unclosedTag(tag, pos, filename) {
return new edge_error_1.EdgeError(`Unclosed tag ${tag}`, 'E_UNCLOSED_TAG', {
line: pos.line,
col: pos.col,
filename: filename,
});
}
exports.unclosedTag = unclosedTag;
+19
View File
@@ -0,0 +1,19 @@
export declare class Scanner {
private pattern;
private line;
private col;
private tolaretionCounts;
private tolerateLhs;
private tolerateRhs;
private patternLength;
closed: boolean;
match: string;
leftOver: string;
loc: {
line: number;
col: number;
};
constructor(pattern: string, toleratePair: [string, string], line: number, col: number);
private matchesPattern;
scan(chunk: string): void;
}
+70
View File
@@ -0,0 +1,70 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.Scanner = void 0;
class Scanner {
pattern;
line;
col;
tolaretionCounts = 0;
tolerateLhs = '';
tolerateRhs = '';
patternLength = 0;
closed = false;
match = '';
leftOver = '';
loc;
constructor(pattern, toleratePair, line, col) {
this.pattern = pattern;
this.line = line;
this.col = col;
this.tolerateLhs = toleratePair[0];
this.tolerateRhs = toleratePair[1];
this.patternLength = this.pattern.length;
this.loc = {
line: this.line,
col: this.col,
};
}
matchesPattern(chars, iterationCount) {
for (let i = 0; i < this.patternLength; i++) {
if (this.pattern[i] !== chars[iterationCount + i]) {
return false;
}
}
return true;
}
scan(chunk) {
if (chunk === '\n') {
this.loc.line++;
this.loc.col = 0;
this.match += '\n';
return;
}
if (!chunk.trim()) {
return;
}
const chunkLength = chunk.length;
let iterations = 0;
while (iterations < chunkLength) {
const char = chunk[iterations];
if (this.tolaretionCounts === 0 && this.matchesPattern(chunk, iterations)) {
iterations += this.patternLength;
this.closed = true;
break;
}
if (char === this.tolerateLhs) {
this.tolaretionCounts++;
}
if (char === this.tolerateRhs) {
this.tolaretionCounts--;
}
this.match += char;
iterations++;
}
if (this.closed) {
this.loc.col += iterations;
this.leftOver = chunk.slice(iterations);
}
}
}
exports.Scanner = Scanner;
+42
View File
@@ -0,0 +1,42 @@
import { Scanner } from './scanner';
import { Tags, Token, RuntimeTag, RuntimeComment, RuntimeMustache, LexerTagDefinitionContract } from './types';
export declare class Tokenizer {
private template;
private tagsDef;
private options;
tokens: Token[];
tagStatement: null | {
scanner: Scanner;
tag: RuntimeTag;
};
mustacheStatement: null | {
scanner: Scanner;
mustache: RuntimeMustache | RuntimeComment;
};
private line;
private isLastLineATag;
private dropNewLine;
private openedTags;
constructor(template: string, tagsDef: Tags, options: {
filename: string;
onLine?: (line: string) => string;
claimTag?: (name: string) => LexerTagDefinitionContract | null;
});
private getRawNode;
private getNewLineNode;
private getTagNode;
private consumeTag;
private handleTagOpening;
private feedCharsToCurrentTag;
private getMustacheType;
private getMustacheNode;
private getCommentNode;
private handleMustacheOpening;
private feedCharsToCurrentMustache;
private isClosingTag;
private consumeNode;
private pushNewLine;
private processText;
private checkForErrors;
parse(): void;
}
+267
View File
@@ -0,0 +1,267 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.Tokenizer = void 0;
const scanner_1 = require("./scanner");
const detector_1 = require("./detector");
const exceptions_1 = require("./exceptions");
const types_1 = require("./types");
class Tokenizer {
template;
tagsDef;
options;
tokens = [];
tagStatement = null;
mustacheStatement = null;
line = 0;
isLastLineATag = false;
dropNewLine = false;
openedTags = [];
constructor(template, tagsDef, options) {
this.template = template;
this.tagsDef = tagsDef;
this.options = options;
}
getRawNode(text) {
return {
type: 'raw',
value: text,
filename: this.options.filename,
line: this.line,
};
}
getNewLineNode(line) {
return {
type: 'newline',
filename: this.options.filename,
line: (line || this.line) - 1,
};
}
getTagNode(tag, jsArg, closingLoc) {
return {
type: tag.escaped ? types_1.TagTypes.ETAG : types_1.TagTypes.TAG,
filename: tag.filename,
properties: {
name: tag.name,
jsArg: jsArg,
selfclosed: tag.selfclosed,
},
loc: {
start: {
line: tag.line,
col: tag.col,
},
end: closingLoc,
},
children: [],
};
}
consumeTag(tag, jsArg, loc) {
if (tag.block && !tag.selfclosed) {
this.openedTags.push(this.getTagNode(tag, jsArg, loc));
}
else {
this.consumeNode(this.getTagNode(tag, jsArg, loc));
}
}
handleTagOpening(line, tag) {
if (tag.seekable && !tag.hasBrace) {
throw (0, exceptions_1.unopenedParen)({ line: tag.line, col: tag.col }, tag.filename);
}
if (!tag.seekable) {
this.consumeTag(tag, '', { line: tag.line, col: tag.col });
if (tag.noNewLine || line.endsWith('~')) {
this.dropNewLine = true;
}
return;
}
tag.col += 1;
this.tagStatement = {
tag: tag,
scanner: new scanner_1.Scanner(')', ['(', ')'], this.line, tag.col),
};
this.feedCharsToCurrentTag(line.slice(tag.col));
}
feedCharsToCurrentTag(content) {
const { tag, scanner } = this.tagStatement;
scanner.scan(content);
if (!scanner.closed) {
return;
}
this.consumeTag(tag, scanner.match, scanner.loc);
if (scanner.leftOver.trim() === '~') {
this.tagStatement = null;
this.dropNewLine = true;
return;
}
if (scanner.leftOver.trim()) {
throw (0, exceptions_1.cannotSeekStatement)(scanner.leftOver, scanner.loc, tag.filename);
}
if (tag.noNewLine) {
this.dropNewLine = true;
}
this.tagStatement = null;
}
getMustacheType(mustache) {
if (mustache.safe) {
return mustache.escaped ? types_1.MustacheTypes.ESMUSTACHE : types_1.MustacheTypes.SMUSTACHE;
}
return mustache.escaped ? types_1.MustacheTypes.EMUSTACHE : types_1.MustacheTypes.MUSTACHE;
}
getMustacheNode(mustache, jsArg, closingLoc) {
return {
type: this.getMustacheType(mustache),
filename: mustache.filename,
properties: {
jsArg: jsArg,
},
loc: {
start: {
line: mustache.line,
col: mustache.col,
},
end: closingLoc,
},
};
}
getCommentNode(comment, value, closingLoc) {
return {
type: 'comment',
filename: comment.filename,
value: value,
loc: {
start: {
line: comment.line,
col: comment.col,
},
end: closingLoc,
},
};
}
handleMustacheOpening(line, mustache) {
const pattern = mustache.isComment ? '--}}' : mustache.safe ? '}}}' : '}}';
const textLeftIndex = mustache.isComment || !mustache.escaped ? mustache.realCol : mustache.realCol - 1;
if (textLeftIndex > 0) {
this.consumeNode(this.getRawNode(line.slice(0, textLeftIndex)));
}
mustache.col += pattern.length;
mustache.realCol += pattern.length;
this.mustacheStatement = {
mustache,
scanner: new scanner_1.Scanner(pattern, ['{', '}'], mustache.line, mustache.col),
};
this.feedCharsToCurrentMustache(line.slice(mustache.realCol));
}
feedCharsToCurrentMustache(content) {
const { mustache, scanner } = this.mustacheStatement;
scanner.scan(content);
if (!scanner.closed) {
return;
}
if (mustache.isComment) {
this.consumeNode(this.getCommentNode(mustache, scanner.match, scanner.loc));
}
else {
this.consumeNode(this.getMustacheNode(mustache, scanner.match, scanner.loc));
}
if (scanner.leftOver.trim()) {
const anotherMustache = (0, detector_1.getMustache)(scanner.leftOver, this.options.filename, scanner.loc.line, scanner.loc.col);
if (anotherMustache) {
this.handleMustacheOpening(scanner.leftOver, anotherMustache);
return;
}
this.consumeNode(this.getRawNode(scanner.leftOver));
}
this.mustacheStatement = null;
}
isClosingTag(line) {
if (!this.openedTags.length) {
return false;
}
line = line.trim();
const recentTag = this.openedTags[this.openedTags.length - 1];
const endStatement = `@end${recentTag.properties.name}`;
return (line === endStatement || line === `${endStatement}~` || line === '@end' || line === '@end~');
}
consumeNode(tag) {
if (this.openedTags.length) {
this.openedTags[this.openedTags.length - 1].children.push(tag);
return;
}
this.tokens.push(tag);
}
pushNewLine(line) {
if ((line || this.line) === 1) {
return;
}
if (this.dropNewLine) {
this.dropNewLine = false;
return;
}
this.consumeNode(this.getNewLineNode(line));
}
processText(line) {
if (typeof this.options.onLine === 'function') {
line = this.options.onLine(line);
}
if (this.tagStatement) {
this.feedCharsToCurrentTag('\n');
this.feedCharsToCurrentTag(line);
return;
}
if (this.mustacheStatement) {
this.feedCharsToCurrentMustache('\n');
this.feedCharsToCurrentMustache(line);
return;
}
if (this.isClosingTag(line)) {
this.consumeNode(this.openedTags.pop());
if (line.endsWith('~')) {
this.dropNewLine = true;
}
return;
}
const tag = (0, detector_1.getTag)(line, this.options.filename, this.line, 0, this.tagsDef, this.options.claimTag);
if (tag) {
if (this.isLastLineATag) {
this.pushNewLine();
}
this.isLastLineATag = true;
this.handleTagOpening(line, tag);
return;
}
this.isLastLineATag = false;
const mustache = (0, detector_1.getMustache)(line, this.options.filename, this.line, 0);
if (mustache) {
this.pushNewLine();
this.handleMustacheOpening(line, mustache);
return;
}
this.pushNewLine();
this.consumeNode(this.getRawNode(line));
}
checkForErrors() {
if (this.tagStatement) {
const { tag } = this.tagStatement;
throw (0, exceptions_1.unclosedParen)({ line: tag.line, col: tag.col }, tag.filename);
}
if (this.mustacheStatement) {
const { mustache } = this.mustacheStatement;
throw (0, exceptions_1.unclosedCurlyBrace)({ line: mustache.line, col: mustache.col }, mustache.filename);
}
if (this.openedTags.length) {
const openedTag = this.openedTags[this.openedTags.length - 1];
throw (0, exceptions_1.unclosedTag)(openedTag.properties.name, openedTag.loc.start, openedTag.filename);
}
}
parse() {
const lines = this.template.split(/\r\n|\r|\n/g);
const linesLength = lines.length;
while (this.line < linesLength) {
const line = lines[this.line];
this.line++;
this.processText(line);
}
this.checkForErrors();
}
}
exports.Tokenizer = Tokenizer;
+92
View File
@@ -0,0 +1,92 @@
export declare enum MustacheTypes {
SMUSTACHE = "s__mustache",
ESMUSTACHE = "es__mustache",
MUSTACHE = "mustache",
EMUSTACHE = "e__mustache"
}
export declare enum TagTypes {
TAG = "tag",
ETAG = "e__tag"
}
export type TagProps = {
name: string;
jsArg: string;
selfclosed: boolean;
};
export type MustacheProps = {
jsArg: string;
};
export type LexerLoc = {
start: {
line: number;
col: number;
};
end: {
line: number;
col: number;
};
};
export interface LexerTagDefinitionContract {
block: boolean;
seekable: boolean;
noNewLine?: boolean;
}
export type RawToken = {
type: 'raw';
value: string;
line: number;
filename: string;
};
export type NewLineToken = {
type: 'newline';
line: number;
filename: string;
};
export type CommentToken = {
type: 'comment';
value: string;
loc: LexerLoc;
filename: string;
};
export type MustacheToken = {
type: MustacheTypes;
properties: MustacheProps;
loc: LexerLoc;
filename: string;
};
export type TagToken = {
type: TagTypes;
properties: TagProps;
loc: LexerLoc;
children: Token[];
filename: string;
};
export type Token = RawToken | NewLineToken | TagToken | MustacheToken | CommentToken;
export type RuntimeTag = LexerTagDefinitionContract & {
name: string;
filename: string;
selfclosed: boolean;
col: number;
line: number;
escaped: boolean;
hasBrace: boolean;
};
export type RuntimeMustache = {
isComment: false;
escaped: boolean;
filename: string;
safe: boolean;
line: number;
col: number;
realCol: number;
};
export type RuntimeComment = {
isComment: true;
filename: string;
line: number;
col: number;
realCol: number;
};
export interface Tags {
[name: string]: LexerTagDefinitionContract;
}
+15
View File
@@ -0,0 +1,15 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.TagTypes = exports.MustacheTypes = void 0;
var MustacheTypes;
(function (MustacheTypes) {
MustacheTypes["SMUSTACHE"] = "s__mustache";
MustacheTypes["ESMUSTACHE"] = "es__mustache";
MustacheTypes["MUSTACHE"] = "mustache";
MustacheTypes["EMUSTACHE"] = "e__mustache";
})(MustacheTypes = exports.MustacheTypes || (exports.MustacheTypes = {}));
var TagTypes;
(function (TagTypes) {
TagTypes["TAG"] = "tag";
TagTypes["ETAG"] = "e__tag";
})(TagTypes = exports.TagTypes || (exports.TagTypes = {}));
+7
View File
@@ -0,0 +1,7 @@
import { Token, TagToken, MustacheToken } from './types';
export declare function isTag(token: Token, name?: string): token is TagToken;
export declare function isEscapedTag(token: Token, name?: string): token is TagToken;
export declare function isMustache(token: Token): token is MustacheToken;
export declare function isSafeMustache(token: Token): token is MustacheToken;
export declare function isEscapedMustache(token: Token): token is MustacheToken;
export declare function getLineAndColumn(token: Token): [number, number];
+40
View File
@@ -0,0 +1,40 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.getLineAndColumn = exports.isEscapedMustache = exports.isSafeMustache = exports.isMustache = exports.isEscapedTag = exports.isTag = void 0;
const types_1 = require("./types");
function isTag(token, name) {
if (token.type === types_1.TagTypes.TAG || token.type === types_1.TagTypes.ETAG) {
return name ? token.properties.name === name : true;
}
return false;
}
exports.isTag = isTag;
function isEscapedTag(token, name) {
if (token.type === types_1.TagTypes.ETAG) {
return name ? token.properties.name === name : true;
}
return false;
}
exports.isEscapedTag = isEscapedTag;
function isMustache(token) {
return (token.type === types_1.MustacheTypes.EMUSTACHE ||
token.type === types_1.MustacheTypes.ESMUSTACHE ||
token.type === types_1.MustacheTypes.MUSTACHE ||
token.type === types_1.MustacheTypes.SMUSTACHE);
}
exports.isMustache = isMustache;
function isSafeMustache(token) {
return token.type === types_1.MustacheTypes.ESMUSTACHE || token.type === types_1.MustacheTypes.SMUSTACHE;
}
exports.isSafeMustache = isSafeMustache;
function isEscapedMustache(token) {
return token.type === types_1.MustacheTypes.EMUSTACHE || token.type === types_1.MustacheTypes.ESMUSTACHE;
}
exports.isEscapedMustache = isEscapedMustache;
function getLineAndColumn(token) {
if (token.type === 'newline' || token.type === 'raw') {
return [token.line, 0];
}
return [token.loc.start.line, token.loc.start.col];
}
exports.getLineAndColumn = getLineAndColumn;