node package manager

kickstarter-crawler

kickstarter-crawler

Build Status

NPM

Returns 30 + 8n data points - where n is the number of pledges
Analyze 61,356 kickstarter projects using crapi

Installation

npm install kickstarter-crawler -g

Test

npm test

Getting Started

Examples
API
The crawler as a service

Examples

The following example crawls a project collecting general, funding, and location
related data

'use strict';
 
 
var kickstarterCrawler = require('kickstarter-crawler');
 
 
var config, project;
 
 
// Project configurations 
config = {
  url: 'https://www.kickstarter.com/projects/maxtemkin/philosophy-posters',
  fields: ['general', 'funding', 'location']
};
 
 
// Initialize the crawler 
project = new kickstarterCrawler.project(config);
 
 
// Make request (crawl) 
project.request(function onRequest (err, data) {
 
  // Something broke 
  if (err) {
    console.log(err);
    return;
  }
 
  // Log crawled data 
  console.log(data);
 
});
 

** 🍕 MORE EXAMPLES COMING 🍕 **

API

kickstarterCrawler.project(config)

  • {Object} project constructor
  • Initializes the crawler and exposes its interface

config

  • {Object} project configurations
  • Configurations necessary to instantiate the project constructor

config.url

  • {String} project profile url

config.fields

  • {Array} array of project data-fields, which indicate what data points will be crawled
  • If undefined, data from all fields will be returned.

VALID FIELD VALUES

  • general
  • time
  • funding
  • location
  • other
  • media
  • pledges

project.request(callback)

  • Makes a HTTP request to the respective project url
  • @param callback(err, data) {Function}

project.parse(HTML)

  • Parses the HTML corresponding to the respective project profile page
  • Returns a nested JSON of parsed data
  • @param {String} HTML
  • @return {Object}

project.getTitle(callback)

  • @param {Function} callback(err, data)
  • data {Object} will consist of {String} generalTitle

project.getCreator(callback)

  • @param {Function} callback(err, data)
  • data {Object} will consist of {String} generalCreator

project.getCategory(callback)

  • @param {Function} callback(err, data)
  • data {Object} will consist of {String} generalCategory

project.getSubCategory(callback)

  • @param {Function} callback(err, data)
  • data {Object} will consist of {String} generalSubCategory

project.getAvatar(callback)

  • @param {Function} callback(err, data)
  • data {Object} will consist of {String} generalProjectAvatarURL

project.getProjectUrl(callback)

  • @param {Function} callback(err, data)
  • data {Object} will consist of {String} generalProjectURL

project.getCreatorUrl(callback)

  • @param {Function} callback(err, data)
  • data {Object} will consist of {String} generalCreatorURL

project.getProjectVideo(callback)

  • @param {Function} callback(err, data)
  • data {Object} will consist of {String} generalProjectVideoURL

project.getNumDays(callback)

  • @param {Function} callback(err, data)
  • data {Object} will consist of {Number} timeNumDays

project.getStartTime(callback)

  • @param {Function} callback(err, data)
  • data {Object} will consist of {Number} timeStart

project.getEndTime(callback)

  • @param {Function} callback(err, data)
  • data {Object} will consist of {Number} timeEnd

project.getDollarsRaised(callback)

  • @param {Function} callback(err, data)
  • data {Object} will consist of {Number} fundingDollarsRaised

project.getFundingGoal(callback)

  • @param {Function} callback(err, data)
  • data {Object} will consist of {Number} fundingGoal

project.getPercentRaised(callback)

  • @param {Function} callback(err, data)
  • data {Object} will consist of {Number} fundingPercentRaised

project.getCurrency(callback)

  • @param {Function} callback(err, data)
  • data {Object} will consist of {String} fundingCurrency

project.getSuccess(callback)

  • @param {Function} callback(err, data)
  • data {Object} will consist of {Boolean} fundingSuccess

project.getBackers(callback)

  • @param {Function} callback(err, data)
  • data {Object} will consist of {Number} fundingNumBackers

project.getCity(callback)

  • @param {Function} callback(err, data)
  • data {Object} will consist of {String} locationCity

project.getState(callback)

  • @param {Function} callback(err, data)
  • data {Object} will consist of {String} locationState

project.getCountry(callback)

  • @param {Function} callback(err, data)
  • data {Object} will consist of {String} locationCountry

project.getUpdates(callback)

  • @param {Function} callback(err, data)
  • data {Object} will consist of {Number} otherUpdates

project.getComments(callback)

  • @param {Function} callback(err, data)
  • data {Object} will consist of {Number} otherComments

project.getProjectsCreated(callback)

  • @param {Function} callback(err, data)
  • data {Object} will consist of {Number} otherProjectsCreated

project.getProjectsBacked(callback)

  • @param {Function} callback(err, data)
  • data {Object} will consist of {Number} otherProjectsBacked

project.getWebsiteUrl(callback)

  • @param {Function} callback(err, data)
  • data {Object} will consist of {String} otherWebsiteURL

project.getNumImages(callback)

  • @param {Function} callback(err, data)
  • data {Object} will consist of {Number} mediaNumImages

project.getImages(callback)

  • @param {Function} callback(err, data)
  • data {Object} will consist of {Array} mediaImages

project.getNumPledges(callback)

  • @param {Function} callback(err, data)
  • data {Object} will consist of {Number} pledgesNumPledges

project.getNumLimitedPledges(callback)

  • @param {Function} callback(err, data)
  • data {Object} will consist of {Number} pledgesNumLimitedPledges

project.getPledgeAmounts(callback)

  • @param {Function} callback(err, data)
  • data {Object} will consist of {Array} pledgesAmounts

project.getPledgesData(callback)

  • @param {Function} callback(err, data)
  • data {Object} will consist of {Object} pledgesData