A tokenizer written in JavaScript.

npm install node-tokenizer
3 downloads in the last week
21 downloads in the last month


A simple tokenizer written in javascript for Node.JS


var fs = require('fs');
var tokenizer = require('./tokenizer');

tokenizer.debug = true;
tokenizer.rule('newline', /^\n/);
tokenizer.rule('whitespace', /^\s+/);
tokenizer.rule('word', /^[^\s]+/);
var tokens = tokenizer.tokenize(fs.readFileSync('./nick.n', 'utf8'));

console.log('Parsed ' + tokens.length + ' tokens');

where the source file is:

fn hello_world
  puts 'hello world'

will output:

``` Starting tokenizer -- fn hello_world puts 'hello world'

word token: fn whitespace token:
word token: hello_world newline token:

whitespace token:
word token: puts whitespace token:
word token: 'hello whitespace token:
word token: world' newline token:

-- Tokenizing complete -- [ 'fn', ' ', 'hello_world', '\n', ' ', 'puts', ' ', '\'hello', ' ', 'world\'', '\n' ]

Parsed 11 tokens

npm loves you