node package manager
We need your input. Help make JavaScript better: Take the 2017 JavaScript Ecosystem survey »



build status dependency status

incremental map reduce function for mongodb


To run an incremental map reduce you have to have a raw collection that you want to map reduce and a target collection where you reduce the data into.

The raw collection must have some kind of timestamp so that we can query it to only reduce values since the last timestamp.

The output into the reduced collection must also have some kind of timestamp which we can use to query the reduced collection for the latest document that was reduced.

As long as those two are upheld we can run an incremental map reduce which means only re-reducing new data which allows for efficient map reduces on large append only collections.

var incrementalMapReduce = require("incremental-map-reduce")
var passback = require("callback-reduce/passback")
var mongo = require("mongodb")
// Get a mongo db in whatever way you prefer 
var db = someMongoDb
/* A collection containing
    id: someId
    , count: numberOfItemsOrSomething
    , timestamp: lastUpdatedTime
var rawCollection = db.collection("raw-data")
var reducedCollection = db.collection("reduced.some-data")
var result = incrementalMapReduce(rawCollection, {
    reducedCollection: reducedCollection
    , map: map
    , reduce: reduce
    , options: {
        out: {
            reduce: "reduced.some-data"
        , finalize: finalize
passback(result, function (err, collection) {
    /* result of rawCollection.mapReduce(...) */
function map() {
    /*global emit*/
    emit(, this)
function reduce(key, values) {
    var result = {
        id: values[0].id
        , count: 0
        , timestamp: 0
    values.forEach(function (value) {
        result.count += value.count
        if (value.timestamp > result.timestamp) {
            result.timestamp = value.timestamp
    return result
function finalize(key, value) {
    return {
        , count: value.count
        , timestamp: value.timestamp


npm install incremental-map-reduce


  • Raynos

MIT Licenced