node package manager

spiderweb

Crawl multiple domains using one or more entry URLs.

Spiderweb

Crawl multiple domains using one or more entry URLs.

npm install spiderweb
var Spiderweb = require('spiderweb'),
urls, options, spiderweb;
 
urls = ['bar.com', 'foo.bar.com'];
 
options = {
strictSSL: false,
images: true,
excludedUrls: ['*biz.foo.com*', '*/admin/*']
};
 
spiderweb = new Spiderweb(urls, options);
 
spider.pageHandler = function(errrespbodyentry) {
console.log(entry.url);
Spiderweb.prototype.pageHandler.apply(this, arguments);
}
 
spiderweb.start(function(errlog) {
if (err) {
console.log('ERROR: ', err)
}
 
if (log.length) {
console.log('DONE: ' + log.length + ' page errors');
}
else {
console.log('DONE: no errors');
}
});