bauer-crawler
Multi-process task runner.
Installation
npm install bauer-crawler
Usage
var Crawler = ; var myCrawler = // plugins to be loaded plugins: "bauer-plugin-fetch" "bauer-plugin-scrape" // configurations for each plugin config: fetch: workers: 2 cache: dir: "./http-cache" expires: "1d" scrape: workers: 1 cache: dir: "./scraper-output" expires: "1d" ; // starts the cluster applicationmyCrawlerstart { // when returned promise is resolved process exits return promise ;};
API Summary
-
Crawler
new Crawler(options Object) :Crawler
.require(module String) :Object
.setupConfig(config Object) :void
.setupConfig(file String) :void
.setupCluster() :void
.setupCluster(cluster Cluster) :void
.setupCrawler(options Object) :void
.loadPlugin(plugins Array) :void
.loadPlugin(plugin String) :void
.loadPlugin(plugin Object) :void
.wrapCallback(module String) :Function
.wrapCallback(callback Function) :Function
.getWorkerRoleMap() :Object
.createWorkers() :void
.requestWorker(role String, request Object) :Promise
.wrapPromise(module String) :void
.wrapPromise(callback Function) :void
.wrapPromise(promise Promise, callback Function) :void
.start(main Function) :void
.setRepeat(repeat Boolean) :void
.getRepeat() :Boolean
-
Response
.sendOk() :void
.sendOk(data Object) :void
.sendError(error Error) :void
-
Worker
.sendReady() :void
-
Promise
.requestWorker(role String, request Object) :Promise
.repeat() :Promise
.repeat(repeat Boolean) :Promise
.time(callback Function) :Promise
.print() :Promise
.print(text String) :Promise
.print(array Array) :Promise
.print(object Object) :Promise
.print(anything) :Promise
.exit() :Promise