md-tokenizer

Streaming markdown tokenizer.

Example

README.md:

# Hello World
Some text

var fs = require("fs")
var mdtok = require("md-tokenizer")
 
var t = mdtok(function (src, token) {
  JSON.stringify([token.type, src])
})
 
fs.createReadStream("README.md").pipe(t)

Output:

["heading 1","# "]
["text","Hello"]
["whitespace"," "]
["text","World"]
["new line","\n"]
["text","Some"]
["whitespace"," "]
["text","text"]
["new line","\n"]