split-file-stream

2.0.0 • Public • Published

split-file-stream

npm version

Partition your readable streams into multiple files or combine files into one merged readable stream.

You should use this module if:

  • You have a readable stream and want to save to multiple files
    • Other solutions require you to supply a path as the source.
      • This means you'd have to write your readable stream first, before partitioning the data.
    • Faster solution as disk writes are slow
  • Vice Versa: You want to pipe the merge output as a stream
    • Pipe the merge output to the response of a web request
    • Pipe encrypted partitioned files merge output to a decryption stream

Installation

npm install --save split-file-stream

Usage

To split a read stream into multiple files:

var splitFileStream = require("split-file-stream");
let maxFileSize = 1024; // 1024 bytes per file
let outputPath = __dirname + "/outputFiles"; // file path partition's prefix
 
splitFileStream.split(readStream, maxFileSize, outputPath, (error, filePaths) => {
    /* If an error occured, filePaths will still contain all files that were written */
    if (error) throw error; // Alternatively you could just log the error instead of throwing: if (error) console.error(error)
 
    console.log("This is an array of my new files:", filePaths);
    /* stream will be saved to files in the path ∈ { ./outputFiles.split-x | x ∈ N } */
});

To merge a set of files together into one output stream:

var splitFileStream = require("split-file-stream");
let filePaths = filePaths; // take this filePaths array from the output of the split method
 
splitFileStream.mergeFilesToStream(filePaths, (outStream) => {
    outStream.on("data", (chunk) => {
        console.log(`Received chunk of ${chunk.length} bytes`);
    });
 
    outStream.on("end", () => {
        console.log("Out stream closed. All files have been merged")
    });
});

To merge a set of files to write to disk:

// Note: You can also do this with the mergeFilesToStream method and piping the stream to a fs writeStream.
var splitFileStream = require("split-file-stream");
let filePaths = filePaths; // take this filePaths array from the output of the split method
let outputPath = __dirname + "/outputFile";
 
splitFileStream.mergeFilesToDisk(filePaths, outputPath, () => {
    console.log("Finished merging files");
});

Example usage of the mergeFilesToDisk method using the mergeFilesToStream method:

var fs = require("fs");
var splitFileStream = require("split-file-stream");
let filePaths = filePaths; // take this filePaths array from the output of the split method
let outputPath = __dirname + "/outputFile";
 
splitFileStream.mergeFilesToStream(filePaths, (outStream) => {
    let writeStream = fs.createWriteStream(outputPath);
    outStream.pipe(writeStream);
});

To split a read stream with a custom function that determines the file name:

var splitFileStream = require("split-file-stream");
let maxFileSize = 1024; // 1024 bytes per file
let outputPath = __dirname + "/outputFiles"; // file path partition's prefix
var customSplit = splitFileStream.getSplitWithGenFilePath((n) => `${outputPath}-${(+ 1)}`)
 
customSplit(readStream, maxFileSize, (error, filePaths) => {
    /* If an error occured, filePaths will still contain all files that were written */
    if (error) throw error; // Alternatively you could just log the error instead of throwing: if (error) console.error(error)
 
    console.log("This is an array of my new files:", filePaths);
});

Alternatively, if you'd like a lower level API for splitting a stream, you can use _splitToStream. This function will split your readable stream into multiple streams. This function is what is used to implement the split function.

var stream = require("stream");
var splitFileStream = require("split-file-stream");
let partitionStreamSize = 1024; // 1024 bytes per partition
const outStreamCreate = (partitionNum) => {
    return stream.passThrough();
};
 
splitFileStream._splitToStream(outStreamCreate, readStream, partitionStreamSize, (error, outStreams) => {
    /* If an error occured, filePaths will still contain all files that were written */
    if (error) throw error; // Alternatively you could just log the error instead of throwing: if (error) console.error(error)
 
    console.log("This is an array of the created output streams:", outStreams);
});

Dependents (0)

Package Sidebar

Install

npm i split-file-stream

Weekly Downloads

47

Version

2.0.0

License

ISC

Unpacked Size

18.5 kB

Total Files

7

Last publish

Collaborators

  • dannycho7