Interface Tokenizer

interface Tokenizer {
    createSplitterRegexp(src: string, flag?: string): RegExp;
    getLineTokens(line: string, startState: string | string[]): Token[];
    removeCapturingGroups(src: string): string;
}