bauer-crawler

    0.2.9 • Public • Published

    bauer-crawler

    Multi-process task runner.

    Installation

    npm install bauer-crawler
    

    Usage

    var Crawler = require("bauer-crawler");
     
    var myCrawler = new Crawler({
      
      // plugins to be loaded
      plugins: [
        "bauer-plugin-fetch",
        "bauer-plugin-scrape"
      ],
      
      // configurations for each plugin
      config: {
        fetch: {
          workers: 2,
          cache: {
            dir: "./http-cache",
            expires: "1d"
          }
        },
        scrape: {
          workers: 1,
          cache: {
            dir: "./scraper-output",
            expires: "1d"
          }
        }
      }
    });
     
    // starts the cluster application
    myCrawler.start(function(promise) {
      
      // when returned promise is resolved process exits
      return promise
        .fetch("http://http-bin.org")
        .scrape({
          "a[href]": {
            urls: "attr:href"
          }
        });
    });

    API Summary

    • Crawler

      • new Crawler(options Object) :Crawler
      • .require(module String) :Object
      • .setupConfig(config Object) :void
      • .setupConfig(file String) :void
      • .setupCluster() :void
      • .setupCluster(cluster Cluster) :void
      • .setupCrawler(options Object) :void
      • .loadPlugin(plugins Array) :void
      • .loadPlugin(plugin String) :void
      • .loadPlugin(plugin Object) :void
      • .wrapCallback(module String) :Function
      • .wrapCallback(callback Function) :Function
      • .getWorkerRoleMap() :Object
      • .createWorkers() :void
      • .requestWorker(role String, request Object) :Promise
      • .wrapPromise(module String) :void
      • .wrapPromise(callback Function) :void
      • .wrapPromise(promise Promise, callback Function) :void
      • .start(main Function) :void
      • .setRepeat(repeat Boolean) :void
      • .getRepeat() :Boolean
    • Response

      • .sendOk() :void
      • .sendOk(data Object) :void
      • .sendError(error Error) :void
    • Worker

      • .sendReady() :void
    • Promise

      • .requestWorker(role String, request Object) :Promise
      • .repeat() :Promise
      • .repeat(repeat Boolean) :Promise
      • .time(callback Function) :Promise
      • .print() :Promise
      • .print(text String) :Promise
      • .print(array Array) :Promise
      • .print(object Object) :Promise
      • .print(anything) :Promise
      • .exit() :Promise

    License

    MIT

    Install

    npm i bauer-crawler

    DownloadsWeekly Downloads

    88

    Version

    0.2.9

    License

    MIT

    Last publish

    Collaborators

    • yneves