proxy-balancer

Proxy Load Balancer

Table of Contents

• Install
• Usage
• License

Install

npm install proxy-balancer

Usage

const Balancer = require('proxy-balancer');
const fetch = require('node-fetch');

const balancer = new Balancer({
  // Required function to populate proxy list, in this case we use a simple web request using node-fetch.
  // Proxies should be in this format:
  // [ http://0.0.0.0:8080, https://0.0.0.0:8081, socks4://0.0.0.0:8000 ]
  fetchProxies: () => {
    return fetch('https://www.cool-proxy.net/proxies.json')
      .then(res => res.json())
      .then(proxies => {
        return proxies
          .filter(proxy => proxy.working_average > 70)
          .map(proxy => `http://${proxy.ip}:${proxy.port}`)
      })
  },

  /*
   * * * Optional Configs * * *
   * all times in milli-seconds
   */

  // Time in milli-seconds, until the proxy list will be updated.
  poolExpired: 1 * 60 * 1000,

  // Max concurrent requests at once. Set to null for infinite
  maxConcurrent: 15,

  // Minimum time between each request in milli-seconds.
  minTime: 100,

  // Time in milli-seconds, to wait for request response.
  timeout: 3 * 1000,

  // Time in milli-seconds, to wait for proxy connection to establish.
  proxyTimeout: 2 * 1000,

  // optionally specify a request agent of your choosing, default is node-fetch
  requestor: axios,

  // Shuffle fetched proxies, based on Fisher-Yates algorithm.
  shuffle: false,

  // optional agent function to use other proxy agents (i.e. tunnel)
  // or you can add proxy agent auth settings or
  // return a unique agent object (supports async/await)
  agentFn: ({ url, timeout }) => new ProxyAgent(url, {
    timeout
  }),

  // optional configs for bottleneck package
  bottleneck: {},

  // optional retry function logic (supports async/await)
  // es6: import Balancer, { retryOptions } from 'proxy-balancer'
  // es5: const retryOptions = Balancer.retryOptions
  retryFn: ({ error, retryCount, timesThisIpRetried, ipsTried }, { retrySameIp, retryNextIp, abort }) => {
    if (retryCount >= 3) {
      return abort();
    }

    if (error.name && (error.name === "FetchError" || error.name === "AbortError")) {
      return retryNextIp();
    }

    return abort();
  },

  // optional limiter to fine tune timeouts
  limiter: {
    callsPerDuration: 5, // required
    duration: 60 * 1000, // required
    postDurationWait: 5 * 60 * 1000, // required
  },

  // optionally handle no available proxies, i.e. request more proxies
  handleNoAvailableProxies: () => {
  },

  // optional proxy formatting function if you're using unique proxy objects
  formatProxy: (proxy) => {
    if (proxy.url) {
      // default will return proxy.url
      return proxy.url
    } else {
      // if you use unique objects, it expects it to resemble a URL object
      // Url is the node-url package https://www.npmjs.com/package/url
      return Url.format(proxy)
    }
  }
});

// Each request will use a fresh proxy, using round robin.
// If a proxy fails or times out, next available proxy in list will be used.
balancer.request('https://www.cool-proxy.net')
  .then(res => res.text())
  .then(body => console.log(body))
  .catch(err => console.error(err))

balancer.request('https://www.cool-proxy.net/proxies.json')
  .then(res => res.json())
  .then(json => console.log(json))
  .catch(err => console.error(err))

License

MIT