mongoose-divide-and-conquer

divide and conquer mongoose collections

Mongoose Divide and Conquer

Allows you to divide and conquer an entire mongo collection using mongoose.

We can give divide a model, and it will find id ranges for the given batch size. These can be pushed onto a queue for processing.

var waitress   = require('waitress')
  , PikaQueue  = require('pika-queue')
  , daq        = require('mongoose-divide-and-conquer')
  , Model      = require('./some/model')
  , queue      = new PikaQueue()
  ;
 
daq.divide({
  model: Model,
  batchSize: 1000
}, function(errbatches) {
  if (err) throw err;
 
  var done = waitress(batches.length, function(err) {
    if (err) throw err;
    console.log('all done!');
    process.exit();
  });
 
  console.log('queueing %d batches', batches.length);
  batches.forEach(function(batchi) {
    queue.queueJob('divide-and-conquer', batch, function(err) {
      console.log('batch %s finished', i);
      done(err);
    });
  });
});

We can pop batches off the queue, and give them to conquer with two methods.

The first method is used to process one document. The second method is called when all documents have been processed. All documents from the batch are processed in parallel.

var waitress   = require('waitress')
  , PikaQueue  = require('pika-queue')
  , daq        = require('mongoose-divide-and-conquer')
  , Model      = require('./some/model')
  , queue      = new PikaQueue()
  ;
 
queue.monitorJobQueue('divide-and-conquer', function(batchcb) {
  daq.conquer(
    {
      batch: batch,
      model: Model
    },
    function(docdone) {
      // do something with the document 
      process.stdout.write('.');
      done();
    },
    function(errcount) {
      if (err) throw err;
      console.log();
      console.log('processed %d docs', count);
      cb(null, count);
    }
  );
});