bauer-crawler

0.2.9 • Public • Published

bauer-crawler

Multi-process task runner.

Installation

npm install bauer-crawler

Usage

var Crawler = require("bauer-crawler");
 
var myCrawler = new Crawler({
  
  // plugins to be loaded
  plugins: [
    "bauer-plugin-fetch",
    "bauer-plugin-scrape"
  ],
  
  // configurations for each plugin
  config: {
    fetch: {
      workers: 2,
      cache: {
        dir: "./http-cache",
        expires: "1d"
      }
    },
    scrape: {
      workers: 1,
      cache: {
        dir: "./scraper-output",
        expires: "1d"
      }
    }
  }
});
 
// starts the cluster application
myCrawler.start(function(promise) {
  
  // when returned promise is resolved process exits
  return promise
    .fetch("http://http-bin.org")
    .scrape({
      "a[href]": {
        urls: "attr:href"
      }
    });
});

API Summary

  • Crawler

    • new Crawler(options Object) :Crawler
    • .require(module String) :Object
    • .setupConfig(config Object) :void
    • .setupConfig(file String) :void
    • .setupCluster() :void
    • .setupCluster(cluster Cluster) :void
    • .setupCrawler(options Object) :void
    • .loadPlugin(plugins Array) :void
    • .loadPlugin(plugin String) :void
    • .loadPlugin(plugin Object) :void
    • .wrapCallback(module String) :Function
    • .wrapCallback(callback Function) :Function
    • .getWorkerRoleMap() :Object
    • .createWorkers() :void
    • .requestWorker(role String, request Object) :Promise
    • .wrapPromise(module String) :void
    • .wrapPromise(callback Function) :void
    • .wrapPromise(promise Promise, callback Function) :void
    • .start(main Function) :void
    • .setRepeat(repeat Boolean) :void
    • .getRepeat() :Boolean
  • Response

    • .sendOk() :void
    • .sendOk(data Object) :void
    • .sendError(error Error) :void
  • Worker

    • .sendReady() :void
  • Promise

    • .requestWorker(role String, request Object) :Promise
    • .repeat() :Promise
    • .repeat(repeat Boolean) :Promise
    • .time(callback Function) :Promise
    • .print() :Promise
    • .print(text String) :Promise
    • .print(array Array) :Promise
    • .print(object Object) :Promise
    • .print(anything) :Promise
    • .exit() :Promise

License

MIT

Package Sidebar

Install

npm i bauer-crawler

Weekly Downloads

11

Version

0.2.9

License

MIT

Last publish

Collaborators

  • yneves