Non-Partisan Magicians

    catstreams

    1.0.0 • Public • Published

    catstreams: Concatenate data from multiple streams fetched concurrently

    catstreams provides a Readable stream interface that concatenates the results of N other streams.

    Example

    $ cat examples/catstreams.js
    /*
     * examples/catstreams.js: basic CatStreams example
     */
    
    var mod_restify = require('restify');
    var mod_bunyan = require('bunyan');
    var HttpStream = require('httpstream');
    var CatStreams = require('../lib/catstreams');
    
    var log = new mod_bunyan({
        'name': 'example',
        'level': 'warn',
        'serializers': mod_restify.bunyan.serializers
    });
    
    var client = mod_restify.createClient({
        'log': log,
        'url': 'https://us-east.manta.joyent.com'
    });
    
    var stream = new CatStreams({
        'log': log,
        'perRequestBuffer': 10 * 1024 * 1024,
        'maxConcurrency': 2
    });
    
    stream.cat(function (options) {
    	return (new HttpStream({
    	    'client': client,
    	    'path': '/manta/public/sdks/node-manta.tar.gz',
    	    'log': log,
    	    'highWaterMark': options['highWaterMark']
    	}));
    });
    
    stream.cat(function (options) {
    	return (new HttpStream({
    	    'client': client,
    	    'path': '/manta/public/sdks/node-manta.tar.gz',
    	    'log': log,
    	    'highWaterMark': options['highWaterMark']
    	}));
    });
    
    stream.cat(null);
    
    var sum = 0;
    console.log('fetching ... ');
    stream.on('data', function (c) { sum += c.length; });
    stream.on('end', function () {
    	console.log('fetched %d bytes', sum);
    	client.close();
    });
    
    $ mls -l /manta/public/sdks/node-manta.tar.gz
    -rwxr-xr-x 1 manta       2697998 Jul 22 15:52 node-manta.tar.gz
    
    $ node examples/catstream.js
    fetching ...
    fetched 5395996 bytes
    

    Contributions

    Pull requests should be "make prepush" clean.

    Keywords

    none

    Install

    npm i catstreams

    DownloadsWeekly Downloads

    0

    Version

    1.0.0

    License

    MIT

    Last publish

    Collaborators

    • bahamat
    • todd.whiteman
    • kusor
    • michael.hicks
    • trentm
    • dap
    • jclulow
    • arekinath
    • melloc
    • kellymclaughlin