egg-crawler
Crawler tool for egg.
Install
$ npm i egg-crawler --save
or
$ yarn add egg-crawler
Usage
// {app_root}/config/plugin.js
exports.crawler = {
enable: true,
package: 'egg-crawler',
};
Example
Create a crawler
// {app_root}/crawler/profile.js
import { Crawler } from 'egg-crawler';
class ProfileCrawler extends Crawler {
// class instantiation name
name: string = 'profile';
async run() {
// this.scraper -> is scrape-it instantiation
// const { app, ctx } = this -> that can use every ctx module
const data = await this.scraper('https://ionicabizau.net', {
title: '.header h1',
desc: '.header h2',
avatar: {
selector: '.header img',
attr: 'src',
},
});
return data;
}
}
Run it
// {app_root}/controller/profile.js
import { Controller } from 'egg';
class ProfileController extends Controller {
async info() {
const { ctx, app } = this;
const data = await app.crawler.profile.run();
ctx.body = data;
}
}