Build and maintain simple webscrapers easily.
$ npm i sgrape
const Sgrape = require('sgrape');
const schema = {
startPage: 'https://www.website.com/page',
selector: '.element-selector',
nextPageSelector: '.page-link[rel=next]', // optional
title: { selector: 'h1', attr: 'text' },
link: { selector: 'a.title', attr: 'href' },
desc: { selector: 'p.desc', attr: 'text' },
};
// per field helpers (optional)
const helpers = {
title: title => title.replace('/', ' '),
desc : desc => desc.substr(0,30) + '...'
};
const scraperConfig = { schema, helpers };
// executes once job finished.
scraperConfig.done = res => {
// insert res to db or whatever
console.log(res);
};
const scraper = new Sgrape(scraperConfig);
Sgrape uses SemVer for versioning.
This project is licensed under the MIT License