Easy Crawler
A simple webcrawler for Node.js supporting Concurrent Connections and Queues.
Installation
npm install easycrawler
Usage
var Crawler = ; var crawler = thread: 5 logs: true depth: 2 headers : 'user-agent' : 'foobar' onlyCrawl : 'reddit' 'reddit.com' //will only crawl urls containing these strings reject : 'rutube' //will reject links containing rutube { //console.log(data.url); //console.log(data.body); } { //console.log(data.url); //console.log(data.status); } { //console.log(urls.crawled);//urls.crawled for visited urls; //console.log(urls.discovered);//urls.discovered for discovered urls }; crawler;