image-augment

Augment images (geometric, noise, ...) for visual machine learning data augmentation.

This library has been freely inspired from imgaug

This library is intend to work

• with hasard for randomness
• with 2 different backends :
  • opencv4nodejs
  • tensorflowjs

Installation

npm install image-augment

Simple example

// First you need a backend for image processing
// this can be one of the following :
// * @tensorflow/tfjs
// * @tensorflow/tfjs-node
// * @tensorflow/tfjs-node-gpu
// * opencv4nodejs
 
const tf = require('@tensorflow/tfjs-node');
 
// Then initialize with the backend
 
const ia = require('image-augment')(tf);
 
// Create an augmentation pipeline
const basicAugmentation = ia.sequential([
    // Add a noise with a standard deviation of 15
    ia.additiveNoise(15),
    // Rotate 30°
    ia.affine({rotate: 30}),
    // Add a blur kernel of 3 pixel
    ia.blur(3)
]);
 
// tensorflow backend needs Tensor4d <-> filename function
// see test/examples/simple-example.js for full implementation of those helpers (fileToTensor and tensorToFile)
 
fileToTensor('test/data/tfjs/lenna.png')
    .then(({images}) => {
        return basicAugmentation.read({images});
    })
    .then(({images}) => {
        return tensorToFile('test/data/tfjs/lenna-example.png', {images});
    })
    .then(() => {
        console.log('done');
    });
 

Output is :

Grid example with opencv4nodejs

const h = require('hasard');
const cv = require('opencv4nodejs');
const ia = require('image-augment')(cv);
 
// Random example images
const sometimes = (aug => h.value([aug, ia.identity()]));
 
const seq = ia.sequential({
    steps: [
        ia.fliplr(0.5),
        ia.flipud(0.5),
        ia.pad({
            percent: h.array({size: 2, value: h.number(0, 0.1)}),
            borderType: ia.RD_BORDER_TYPE,
            borderValue: h.integer(0, 255)
        }),
        sometimes(ia.crop({
            percent: h.array({size: 2, value: h.number(0, 0.1)})
        })),
        sometimes(ia.affine({
            // Scale images to 80-120% of their size, individually per axis
            scale: h.array([h.number(0.6, 1.2), h.number(0.6, 1.2)]),
            // Translate by -20 to +20 percent (per axis)
            translatePercent: h.array([h.number(-0.2, 0.2), h.number(-0.2, 0.2)]),
            // Rotate by -45 to +45 degrees
            rotate: h.number(-45, 45),
            // Shear by -16 to +16 degrees
            shear: h.number(-16, 16),
            // If borderType is constant, use a random rgba value between 0 and 255
            borderValue: h.array({value: h.integer(0, 255), size: 4}),
            borderType: ia.RD_BORDER_TYPE
        }))
    ],
    randomOrder: true
});
const image = cv.imread('test/data/opencv4nodejs/lenna.png');
 
seq.toGrid({images: [image, image, image, image, image, image, image, image]}, {
    filename: 'test/data/opencv4nodejs/lenna-grid.png',
    imageShape: [300, 300],
    gridShape: [4, 2]
});

Output :

Grid Example with tensorflowjs

const h = require('hasard');
const tf = require('@tensorflow/tfjs-node');
const ia = require('image-augment')(tf);
 
 
// Random example images
const sometimes = (aug => h.value([aug, ia.identity()]));
 
const seq = ia.sequential({
    steps: [
        ia.fliplr(0.5),
        ia.flipud(0.5),
        ia.pad({
            percent: h.array({size: 2, value: h.number(0, 0.1)}),
            borderType: ia.RD_BORDER_TYPE,
            borderValue: h.integer(0, 255)
        }),
        sometimes(ia.crop({
            percent: h.array({size: 2, value: h.number(0, 0.1)})
        })),
        sometimes(ia.affine({
            // Scale images to 80-120% of their size, individually per axis
            scale: h.array([h.number(0.6, 1.2), h.number(0.6, 1.2)]),
            // Translate by -20 to +20 percent (per axis)
            translatePercent: h.array([h.number(-0.2, 0.2), h.number(-0.2, 0.2)]),
            // Rotate by -45 to +45 degrees
            rotate: h.number(-45, 45),
            // Shear by -16 to +16 degrees
            shear: h.number(-16, 16),
            // If borderType is constant, use a random rgba value between 0 and 255
            borderValue: h.array({value: h.integer(0, 255), size: 4}),
            borderType: ia.RD_BORDER_TYPE
        }))
    ],
    randomOrder: true
});
 
// tensorflow backend needs Tensor4d <-> filename function
// see test/helpers/files-to-images for full implementation of those helpers (fileToTensor and tensorToFile)
const filenames = new Array(8).fill('test/data/opencv4nodejs/lenna.png');
 
filesToImages(filenames, seq.backend).then(images => {
    seq.toGrid({images}, {
        filename: 'test/data/tfjs/lenna-grid.png',
        imageShape: [300, 300],
        gridShape: [4, 2]
    });
})
 

Output :

API documentation

See documentation

Discussion

Opencv4nodejs vs Tensorflowjs

Both librairies have advantages, this is what you need to know

Why opencv4nodejs :

• easier to manipulate files in node.js (cv.imread ...)
• Using different image sizes with no impact on performance

Why tensorflowjs :

• Browser support
• integrate with DL training
• Fast Noise image generation (truncatedNormal)

See benchmark for more info about performance

Todo list

Help appreciated, please open an issue if you have any question.

• Add benchmark test to measure the speed
• Faster random generator using tensorflow js truncated normal
• Get affine transform to work with tensorflow backend
• add examples/explanations/benchmark in the README.md
• Run all unit tests on Travis
• Implement perspective Transform using tensorflowjs backend
• Put documentation on github pages
• Remove jimp deps
• create a demo app running in the browser with tfjs + webgl
• Speed up all non-batch implemented tensorflow augmenters
• Add more augmenters
• Add unit test and examples for cropToBox and DrawBoxes
• Stream API
• Faster gaussian and poisson noise generators