pg-large-object
    DefinitelyTyped icon, indicating that this package has TypeScript declarations provided by the separate @types/pg-large-object package

    2.0.0 • Public • Published

    node-pg-large-object

    Large object support for PostgreSQL clients using the node-postgres library.

    The API of this library resembles the JDBC library for PostgreSQL.

    Installation

    npm install --save pg-large-object
    

    You will also need to install either the pg library, or the pg-promise library:

    npm install --save pg
    # or
    npm install --save pg-promise
    

    Some of the methods in this library require PostgreSQL 9.3 (server) and up:

    • LargeObject.seek()
    • LargeObject.tell()
    • LargeObject.size()
    • LargeObject.truncate()

    All other methods should work on PostgreSQL 8.4 and up.

    Large Objects

    Large Objects in PostgreSQL lets you store files/objects up to 4 TiB in size. The main benefit of using Large Objects instead of a simple column is that the data can be read and written in chunks (e.g. as a stream), instead of having to load the entire column into memory.

    Examples

    The easiest way to use this library is in combination with promises and pg-promise. This library exposes a callback style interface (for backwards compatibility) and a promise style interface (see API Documentation). All functions that end with "Async" will return a promise

    Reading a large object using a stream and pg-promise:

    const pgp = require('pg-promise')();
    const {LargeObjectManager} = require('pg-large-object');
    const {createWriteStream} = require('fs');
     
    const db = pgp('postgres://postgres:1234@localhost/postgres');
     
    // When working with Large Objects, always use a transaction
    db.tx(tx => {
      const man = new LargeObjectManager({pgPromise: tx});
     
      // A LargeObject oid, probably stored somewhere in one of your own tables.
      const oid = 123;
     
      // If you are on a high latency connection and working with
      // large LargeObjects, you should increase the buffer size.
      // The buffer should be divisible by 2048 for best performance
      // (2048 is the default page size in PostgreSQL, see LOBLKSIZE)
      const bufferSize = 16384;
     
      return man.openAndReadableStreamAsync(oid, bufferSize)
      .then(([size, stream]) => {
        console.log('Streaming a large object with a total size of', size);
     
        // Store it as an image
        const fileStream = createWriteStream('my-file.png');
        stream.pipe(fileStream);
     
        return new Promise((resolve, reject) => {
          stream.on('end', resolve);
          stream.on('error', reject);
        });
      });
    })
    .then(() => {
      console.log('Done!');
    })
    .catch(error => {
      console.log('Something went horribly wrong!', error);
    });

    Creating a new large object using a stream and pg-promise:

    const pgp = require('pg-promise')();
    const {LargeObjectManager} = require('pg-large-object');
    const {createReadStream} = require('fs');
     
    const db = pgp('postgres://postgres:1234@localhost/postgres');
     
    // When working with Large Objects, always use a transaction
    db.tx(tx => {
      const man = new LargeObjectManager({pgPromise: tx});
     
      // If you are on a high latency connection and working with
      // large LargeObjects, you should increase the buffer size.
      // The buffer should be divisible by 2048 for best performance
      // (2048 is the default page size in PostgreSQL, see LOBLKSIZE)
      const bufferSize = 16384;
     
      return man.createAndWritableStreamAsync(bufferSize)
      .then(([oid, stream]) => {
        // The server has generated an oid
        console.log('Creating a large object with the oid', oid);
     
        const fileStream = createReadStream('upload-my-file.png');
        fileStream.pipe(stream);
     
        return new Promise((resolve, reject) => {
          stream.on('finish', resolve);
          stream.on('error', reject);
        });
      });
    })
    .then(() => {
      console.log('Done!');
    })
    .catch(error => {
      console.log('Something went horribly wrong!', error);
    });

    Reading a large object using a stream and callbacks:

    var pg = require('pg');
    var LargeObjectManager = require('pg-large-object').LargeObjectManager;
    var conString = "postgres://postgres:1234@localhost/postgres";
     
    pg.connect(conString, function(err, client, done)
    {
      if (err)
      {
        return console.error('could not connect to postgres', err);
      }
     
      var man = new LargeObjectManager({pg: client});
     
      // When working with Large Objects, always use a transaction
      client.query('BEGIN', function(err, result)
      {
        if (err)
        {
          done(err);
          return client.emit('error', err);
        }
     
        // A LargeObject oid, probably stored somewhere in one of your own tables.
        var oid = 123;
     
        // If you are on a high latency connection and working with
        // large LargeObjects, you should increase the buffer size.
        // The buffer should be divisible by 2048 for best performance
        // (2048 is the default page size in PostgreSQL, see LOBLKSIZE)
        var bufferSize = 16384;
        man.openAndReadableStream(oid, bufferSize, function(err, size, stream)
        {
          if (err)
          {
            done(err);
            return console.error('Unable to read the given large object', err);
          }
     
          console.log('Streaming a large object with a total size of', size);
          stream.on('end', function()
          {
            client.query('COMMIT', done);
          });
     
          // Store it as an image
          var fileStream = require('fs').createWriteStream('my-file.png');
          stream.pipe(fileStream);
        });
      });
    });

    Creating a new large object using a stream:

    var pg = require('pg');
    var LargeObjectManager = require('pg-large-object').LargeObjectManager;
    var conString = "postgres://postgres:1234@localhost/postgres";
     
    pg.connect(conString, function(err, client, done)
    {
      if (err)
      {
        return console.error('could not connect to postgres', err);
      }
     
      var man = new LargeObjectManager({pg: client});
     
      // When working with Large Objects, always use a transaction
      client.query('BEGIN', function(err, result)
      {
        if (err)
        {
          done(err);
          return client.emit('error', err);
        }
     
        // If you are on a high latency connection and working with
        // large LargeObjects, you should increase the buffer size.
        // The buffer should be divisible by 2048 for best performance
        // (2048 is the default page size in PostgreSQL, see LOBLKSIZE)
        var bufferSize = 16384;
        man.createAndWritableStream(bufferSize, function(err, oid, stream)
        {
          if (err)
          {
            done(err);
            return console.error('Unable to create a new large object', err);
          }
     
          // The server has generated an oid
          console.log('Creating a large object with the oid', oid);
          stream.on('finish', function()
          {
            // Actual writing of the large object in DB may
            // take some time, so one should provide a
            // callback to client.query.
            client.query('COMMIT', done);
          });
     
          // Upload an image
          var fileStream = require('fs').createReadStream('upload-my-file.png');
          fileStream.pipe(stream);
        });
      });
    });

    Using low level LargeObject functions:

    var pg = require('pg');
    var LargeObjectManager = require('pg-large-object').LargeObjectManager;
    var LargeObject = require('pg-large-object').LargeObject;
    var conString = "postgres://postgres:1234@localhost/postgres";
     
    pg.connect(conString, function(err, client, done)
    {
      if (err)
      {
        return console.error('could not connect to postgres', err);
      }
     
      var man = new LargeObjectManager({pg: client});
     
      // When working with Large Objects, always use a transaction
      client.query('BEGIN', function(err, result)
      {
        if (err)
        {
          done(err);
          return client.emit('error');
        }
     
        // A LargeObject oid, probably stored somewhere in one of your own tables.
        var oid = 123;
     
        // Open with READWRITE if you would like to use
        // write() and truncate()
        man.open(oid, LargeObjectManager.READ, function(err, obj)
        {
          if (err)
          {
            done(err);
            return console.error(
              'Unable to open the given large object',
              oid,
              err);
          }
     
          // Read the first 50 bytes
          obj.read(50, function(err, buf)
          {
            // buf is a standard node.js Buffer
            console.log(buf.toString('hex'));
          });
     
          // pg uses a query queue, this guarantees the LargeObject
          // will be executed in the order you call them, even if you do not
          // wait on the callbacks.
          // In this library the callback for methods that only return an error
          // is optional (such as for seek below). If you do not give a callback
          // and an error occurs, this error will me emit()ted on the client object.
     
          // Set the position to byte 5000
          obj.seek(5000, LargeObject.SEEK_SET);
          obj.tell(function(err, position)
          {
            console.log(err, position); // 5000
          });
          obj.size(function(err, size)
          {
            console.log(err, size); // The size of the entire LargeObject
          });
     
          // Done with the object, close it
          obj.close();
          client.query('COMMIT', done);
        });
      });
    });

    Testing

    You can test this library by running:

    npm install pg-large-object
    npm test
    

    The test assumes that postgres://nodetest:nodetest@localhost/nodetest is a valid database. You also need to place a large file named "test.jpg" in the test folder.

    API Documentation

    Modules

    pg-large-object
    pg-large-object/lib/LargeObject

    Represents an opened large object.

    pg-large-object/lib/LargeObjectManager

    This class lets you use the Large Object functionality of PostgreSQL. All usage of Large Object should take place within a transaction block! (BEGIN ... COMMIT)

    pg-large-object/lib/promiseFromCallbackPromise
    pg-large-object/lib/ReadStreamstream.Readable
    pg-large-object/lib/WriteStreamstream.Writable

    pg-large-object

    pg-large-object.LargeObjectManager : function

    pg-large-object/lib/LargeObjectManager

    Kind: static constant of pg-large-object

    pg-large-object.LargeObject : function

    pg-large-object/lib/LargeObject

    Kind: static constant of pg-large-object

    pg-large-object.ReadStream : function

    pg-large-object/lib/ReadStream

    Kind: static constant of pg-large-object

    pg-large-object.WriteStream : function

    pg-large-object/lib/WriteStream

    Kind: static constant of pg-large-object

    pg-large-object/lib/LargeObject

    Represents an opened large object.

    pg-large-object/lib/LargeObject.close([callback])

    Closes this large object. You should no longer call any methods on this object.

    Kind: instance method of pg-large-object/lib/LargeObject

    Param Type
    [callback] closeCallback

    pg-large-object/lib/LargeObject.closeAsync() ⇒ Promise

    Closes this large object. You should no longer call any methods on this object.

    Kind: instance method of pg-large-object/lib/LargeObject

    pg-large-object/lib/LargeObject.read(length, callback)

    Reads some data from the large object.

    Kind: instance method of pg-large-object/lib/LargeObject

    Param Type Description
    length Number How many bytes to read
    callback readCallback

    pg-large-object/lib/LargeObject.readAsync(length) ⇒ Promise.<Buffer>

    Reads some data from the large object.

    Kind: instance method of pg-large-object/lib/LargeObject
    Returns: Promise.<Buffer> - The binary data that was read. If the length of this buffer is less than the supplied length param, there is no more data to be read.

    Param Type Description
    length Number How many bytes to read

    pg-large-object/lib/LargeObject.write(buffer, [callback])

    Writes some data to the large object.

    Kind: instance method of pg-large-object/lib/LargeObject

    Param Type Description
    buffer Buffer data to write
    [callback] writeCallback

    pg-large-object/lib/LargeObject.writeAsync(buffer) ⇒ Promise

    Writes some data to the large object.

    Kind: instance method of pg-large-object/lib/LargeObject

    Param Type Description
    buffer Buffer data to write

    pg-large-object/lib/LargeObject.seek(position, ref, [callback])

    Sets the position within the large object. Beware floating point rounding with values greater than 2^53 (8192 TiB)

    Kind: instance method of pg-large-object/lib/LargeObject

    Param Type Description
    position Number
    ref Number One of SEEK_SET, SEEK_CUR, SEEK_END
    [callback] seekCallback

    pg-large-object/lib/LargeObject.seekAsync(position, ref) ⇒ Promise.<number>

    Sets the position within the large object. Beware floating point rounding with values greater than 2^53 (8192 TiB)

    Kind: instance method of pg-large-object/lib/LargeObject
    Returns: Promise.<number> - The new position

    Param Type Description
    position Number
    ref Number One of SEEK_SET, SEEK_CUR, SEEK_END

    pg-large-object/lib/LargeObject.tell(callback)

    Retrieves the current position within the large object. Beware floating point rounding with values greater than 2^53 (8192 TiB)

    Kind: instance method of pg-large-object/lib/LargeObject

    Param Type
    callback tellCallback

    pg-large-object/lib/LargeObject.tellAsync() ⇒ Promise.<number>

    Retrieves the current position within the large object. Beware floating point rounding with values greater than 2^53 (8192 TiB)

    Kind: instance method of pg-large-object/lib/LargeObject

    pg-large-object/lib/LargeObject.size(callback)

    Find the total size of the large object.

    Kind: instance method of pg-large-object/lib/LargeObject

    Param Type
    callback sizeCallback

    pg-large-object/lib/LargeObject.sizeAsync() ⇒ Promise.<number>

    Find the total size of the large object.

    Kind: instance method of pg-large-object/lib/LargeObject

    pg-large-object/lib/LargeObject.truncate(length, [callback])

    Truncates the large object to the given length in bytes. If the number of bytes is larger than the current large object length, the large object will be filled with zero bytes. This method does not modify the current file offset.

    Kind: instance method of pg-large-object/lib/LargeObject

    Param Type
    length Number
    [callback] truncateCallback

    pg-large-object/lib/LargeObject.truncateAsync(length) ⇒ Promise

    Truncates the large object to the given length in bytes. If the number of bytes is larger than the current large object length, the large object will be filled with zero bytes. This method does not modify the current file offset.

    Kind: instance method of pg-large-object/lib/LargeObject

    Param Type
    length Number

    pg-large-object/lib/LargeObject.getReadableStream([bufferSize]) ⇒ pg-large-object/lib/ReadStream

    Return a stream to read this large object. Call this within a transaction block.

    Kind: instance method of pg-large-object/lib/LargeObject

    Param Type Default Description
    [bufferSize] Number 16384 A larger buffer size will require more memory on both the server and client, however it will make transfers faster because there is less overhead (less read calls to the server). his overhead is most noticeable on high latency connections because each ransfered chunk will incur at least RTT of additional transfer time.

    pg-large-object/lib/LargeObject.getWritableStream([bufferSize]) ⇒ pg-large-object/lib/WriteStream

    Return a stream to write to this large object. Call this within a transaction block.

    Kind: instance method of pg-large-object/lib/LargeObject

    Param Type Default Description
    [bufferSize] Number 16384 A larger buffer size will require more memory on both the server and client, however it will make transfers faster because there is less overhead (less read calls to the server). his overhead is most noticeable on high latency connections because each ransfered chunk will incur at least RTT of additional transfer time.

    pg-large-object/lib/LargeObject.SEEK_SET : Number

    A seek from the beginning of a object

    Kind: static constant of pg-large-object/lib/LargeObject

    pg-large-object/lib/LargeObject.SEEK_CUR : Number

    A seek from the current position

    Kind: static constant of pg-large-object/lib/LargeObject

    pg-large-object/lib/LargeObject.SEEK_END : Number

    A seek from the end of a object

    Kind: static constant of pg-large-object/lib/LargeObject

    pg-large-object/lib/LargeObject~closeCallback : function

    Kind: inner typedef of pg-large-object/lib/LargeObject

    Param Type Description
    error Error If set, an error occurred.

    pg-large-object/lib/LargeObject~readCallback : function

    Kind: inner typedef of pg-large-object/lib/LargeObject

    Param Type Description
    error Error If set, an error occurred.
    data Buffer The binary data that was read. If the length of this buffer is less than the supplied length param, there is no more data to be read.

    pg-large-object/lib/LargeObject~writeCallback : function

    Kind: inner typedef of pg-large-object/lib/LargeObject

    Param Type Description
    error Error If set, an error occurred.

    pg-large-object/lib/LargeObject~seekCallback : function

    Kind: inner typedef of pg-large-object/lib/LargeObject

    Param Type Description
    error Error If set, an error occurred.
    position Number The new position

    pg-large-object/lib/LargeObject~tellCallback : function

    Kind: inner typedef of pg-large-object/lib/LargeObject

    Param Type Description
    error Error If set, an error occurred.
    position Number The position

    pg-large-object/lib/LargeObject~sizeCallback : function

    Kind: inner typedef of pg-large-object/lib/LargeObject

    Param Type Description
    error Error If set, an error occurred.
    size Number Object size in bytes

    pg-large-object/lib/LargeObject~truncateCallback : function

    Kind: inner typedef of pg-large-object/lib/LargeObject

    Param Type Description
    error Error If set, an error occurred.

    pg-large-object/lib/LargeObjectManager

    This class lets you use the Large Object functionality of PostgreSQL. All usage of Large Object should take place within a transaction block! (BEGIN ... COMMIT)

    Param Type Description
    options object Either pg or pgPromise must be given
    options.pg module:pg/Client A pg (https://www.npmjs.com/package/pg) Client object
    options.pgPromise module:pg-promise/Task A pg-promise (https://www.npmjs.com/package/pg-promise) transaction object as given by db.tx()

    Example

    new LargeObjectManager(client)

    pg-large-object/lib/LargeObjectManager.open(oid, mode, callback)

    Open an existing large object, based on its OID. In mode READ, the data read from it will reflect the contents of the large object at the time of the transaction snapshot that was active when open was executed, regardless of later writes by this or other transactions. If opened using WRITE (or READWRITE), data read will reflect all writes of other committed transactions as well as writes of the current transaction.

    Kind: instance method of pg-large-object/lib/LargeObjectManager

    Param Type Description
    oid Number
    mode Number One of WRITE, READ, or READWRITE
    callback openCallback

    pg-large-object/lib/LargeObjectManager.openAsync(oid, mode) ⇒ Promise.<pg-large-object/lib/LargeObject>

    Open an existing large object, based on its OID. In mode READ, the data read from it will reflect the contents of the large object at the time of the transaction snapshot that was active when open was executed, regardless of later writes by this or other transactions. If opened using WRITE (or READWRITE), data read will reflect all writes of other committed transactions as well as writes of the current transaction.

    Kind: instance method of pg-large-object/lib/LargeObjectManager

    Param Type Description
    oid Number
    mode Number One of WRITE, READ, or READWRITE

    pg-large-object/lib/LargeObjectManager.create(callback)

    Creates a large object, returning its OID. After which you can open() it.

    Kind: instance method of pg-large-object/lib/LargeObjectManager

    Param Type
    callback createCallback

    pg-large-object/lib/LargeObjectManager.createAsync() ⇒ Promise.<number>

    Creates a large object, returning its OID. After which you can open() it.

    Kind: instance method of pg-large-object/lib/LargeObjectManager
    Returns: Promise.<number> - oid

    pg-large-object/lib/LargeObjectManager.unlink(oid, [callback])

    Unlinks (deletes) a large object

    Kind: instance method of pg-large-object/lib/LargeObjectManager

    Param Type
    oid number
    [callback] unlinkCallback

    pg-large-object/lib/LargeObjectManager.unlinkAsync(oid) ⇒ Promise

    Unlinks (deletes) a large object

    Kind: instance method of pg-large-object/lib/LargeObjectManager

    Param Type
    oid number

    pg-large-object/lib/LargeObjectManager.openAndReadableStream(oid, [bufferSize], callback)

    Open a large object, return a stream and close the object when done streaming. Only call this within a transaction block.

    Kind: instance method of pg-large-object/lib/LargeObjectManager

    Param Type Default
    oid Number
    [bufferSize] Number 16384
    callback openAndReadableStreamCallback

    pg-large-object/lib/LargeObjectManager.openAndReadableStreamAsync(oid, [bufferSize]) ⇒ Promise.<Array>

    Open a large object, return a stream and close the object when done streaming. Only call this within a transaction block.

    Kind: instance method of pg-large-object/lib/LargeObjectManager
    Returns: Promise.<Array> - The total size and a ReadStream

    Param Type Default
    oid Number
    [bufferSize] Number 16384

    pg-large-object/lib/LargeObjectManager.createAndWritableStream([bufferSize], [callback])

    Create and open a large object, return a stream and close the object when done streaming. Only call this within a transaction block.

    Kind: instance method of pg-large-object/lib/LargeObjectManager

    Param Type Default
    [bufferSize] Number 16384
    [callback] createAndWritableStreamCallback

    pg-large-object/lib/LargeObjectManager.createAndWritableStreamAsync([bufferSize]) ⇒ promise.<Array>

    Create and open a large object, return a stream and close the object when done streaming. Only call this within a transaction block.

    Kind: instance method of pg-large-object/lib/LargeObjectManager
    Returns: promise.<Array> - The oid and a WriteStream

    Param Type Default
    [bufferSize] Number 16384

    pg-large-object/lib/LargeObjectManager.WRITE : Number

    Kind: static constant of pg-large-object/lib/LargeObjectManager

    pg-large-object/lib/LargeObjectManager.READ : Number

    Kind: static constant of pg-large-object/lib/LargeObjectManager

    pg-large-object/lib/LargeObjectManager.READWRITE : Number

    Kind: static constant of pg-large-object/lib/LargeObjectManager

    pg-large-object/lib/LargeObjectManager~openCallback : function

    Kind: inner typedef of pg-large-object/lib/LargeObjectManager

    Param Type Description
    error Error If set, an error occurred.
    result pg-large-object/lib/LargeObject

    pg-large-object/lib/LargeObjectManager~createCallback : function

    Kind: inner typedef of pg-large-object/lib/LargeObjectManager

    Param Type Description
    error Error If set, an error occurred.
    oid Number

    pg-large-object/lib/LargeObjectManager~unlinkCallback : function

    Kind: inner typedef of pg-large-object/lib/LargeObjectManager

    Param Type Description
    error Error If set, an error occurred.

    pg-large-object/lib/LargeObjectManager~openAndReadableStreamCallback : function

    Kind: inner typedef of pg-large-object/lib/LargeObjectManager

    Param Type Description
    error Error If set, an error occurred.
    size Number The total size of the large object
    stream pg-large-object/lib/ReadStream

    pg-large-object/lib/LargeObjectManager~createAndWritableStreamCallback : function

    Kind: inner typedef of pg-large-object/lib/LargeObjectManager

    Param Type Description
    error Error If set, an error occurred.
    oid Number
    stream pg-large-object/lib/WriteStream

    pg-large-object/lib/promiseFromCallback ⇒ Promise

    Param Type
    fn function
    self object
    [options] object

    pg-large-object/lib/ReadStream ⇐ stream.Readable

    Extends: stream.Readable

    pg-large-object/lib/WriteStream ⇐ stream.Writable

    Extends: stream.Writable

    Install

    npm i pg-large-object

    DownloadsWeekly Downloads

    3,654

    Version

    2.0.0

    License

    MIT

    Last publish

    Collaborators

    • joris-van-der-wel