diff --git a/acorn/acorn-tests.ts b/acorn/acorn-tests.ts index fb79d23718..bda6844c84 100644 --- a/acorn/acorn-tests.ts +++ b/acorn/acorn-tests.ts @@ -67,3 +67,6 @@ acorn.getLineInfo('string', 56); acorn.plugins['test'] = function (p: acorn.Parser, config: any) { } + +acorn.tokenizer('console.log("hello world)', {locations: true}).getToken(); +acorn.tokenizer('console.log("hello world)', {locations: true})[Symbol.iterator]().next(); diff --git a/acorn/index.d.ts b/acorn/index.d.ts index 1944abc257..441deb29c3 100644 --- a/acorn/index.d.ts +++ b/acorn/index.d.ts @@ -238,9 +238,12 @@ declare namespace acorn { function parseExpressionAt(input: string, pos?: number, options?: Options): ESTree.Expression; - // todo: here the tokenizer function returns a Parser instance, that is targeting the detail of - // Parser prototype. Someone need this please reade README.md first. - // function tokenizer(options: Options, input: string): Parser; + interface ITokenizer { + getToken() : Token, + [Symbol.iterator](): Iterator + } + + function tokenizer(input: string, options: Options): ITokenizer; let parse_dammit: IParse | undefined; let LooseParser: ILooseParserClass | undefined;