jsmaestro-skrap-plus

Web site crawling package with ES2015

Usage

const Crawler = require('jsmaestro-skrap-plus').Crawler;

const pageInfo = {
  url: `http://m.daum.net/`,
  collections: [{
    name: 'ranking',
    query: "$('#channel_news1_realtime .list_realtime')",
    fields: {
      1: "find('li .txt_issue').eq(0).text()",
      2: "find('li .txt_issue').eq(1).text()",
      3: "find('li .txt_issue').eq(2).text()"
    },
  }],
};

const promise = Crawler(pageInfo, {});

promise.then(function(res) {
  // do something
  const page = value.ranking[0]; //  { '1': 'whatever1', '2': 'whatever2', '3': 'whatever3' }
}).catch(function(error) {
  // do error handling
});

For Testing with Mocha and Chai

npm test