Neurotic Pantaloon Maker

    spiderweb

    0.0.14 • Public • Published

    Spiderweb

    Crawl multiple domains using one or more entry URLs.

    Installation

    npm install spiderweb
    var Spiderweb = require('spiderweb'),
    	urls, options, spiderweb;
    
    urls = ['bar.com', 'foo.bar.com'];
    
    options = {
    	strictSSL: false,
    	images: true,
    	excludedUrls: ['*biz.foo.com*', '*/admin/*']
    };
    
    spiderweb = new Spiderweb(urls, options);
    
    spider.pageHandler = function(err, resp, body, entry) {
    	console.log(entry.url);
    	Spiderweb.prototype.pageHandler.apply(this, arguments);
    }
    
    spiderweb.start(function(err, log) {
    	if (err) {
    		console.log('ERROR: ', err)
    	}
    
    	if (log.length) {
    		console.log('DONE: ' + log.length + ' page errors');
    	}
    	else {
    		console.log('DONE: no errors');
    	}
    });

    Keywords

    none

    Install

    npm i spiderweb

    DownloadsWeekly Downloads

    2

    Version

    0.0.14

    License

    none

    Last publish

    Collaborators

    • jmerrick