waff

simple crawler with cheerio

DEPENDENCIES

https://github.com/mooz/node-icu-charset-detector#installing-icu

HOW TO USE

var waff = require('waff');

var crawler = waff({
    concurrency: 20,
    timeout: 10 * 1000,
    callback: function(err, response, $) {
        if (err) return console.log(err);
        console.log($('title').text());
    }
});

// console.log(crawler);

crawler.queue('http://www.baidu.com/');
crawler.queue(['http://www.baidu.com/', 'http://baidu.com/']);