crawlee-proxyport

Proxy Port provider for Crawlee

crawlee-proxyport package provides easy way to use rotating proxy with Crawlee. Using Proxy Port API as source of proxy.

Prerequisites

To use this package you will need a free API key. Get your AIP key here. Detailed instructions here.

Installation

Install via npm:

$ npm i crawlee-proxyport

Getting Started

Before you get your first proxy, you need to assign an API key.

import { CheerioCrawler, ProxyConfiguration } from 'crawlee';
import { ProxyProvider } from 'crawlee-proxyport';

const startUrls = ['https://example.com'];

const crawler: CheerioCrawler = new CheerioCrawler({
    useSessionPool: true,
    persistCookiesPerSession: true,
    proxyConfiguration: new ProxyConfiguration({
        newUrlFunction: (sId) => proxyProvider.newUrlFunction(sId)
    }),
    maxRequestRetries: 20,
    sessionPoolOptions:{
        sessionOptions: {
            // you may want to play with this number, we recomend to use value between 10 and 50
            maxUsageCount: 20,
        },
    },
    async requestHandler({ request, $, log }) {
        const title = $('title').text();
        log.info(`Title of ${request.loadedUrl} is '${title}'`);
    },
});

const proxyProvider = new ProxyProvider(<API_KEY>, crawler);

await crawler.run(startUrls);