adnn

adnn provides Javascript-native neural networks on top of general scalar/tensor reverse-mode automatic differentiation. You can use just the AD code, or the NN layer built on top of it. This architecture makes it easy to define big, complex numerical computations and compute derivatives w.r.t. their inputs/parameters. adnn also includes utilities for optimizing/training the parameters of such computations.

Examples

Scalar code

The simplest use case for adnn:

var ad = require('adnn/ad')
 
function dist(x1, y1, x2, y2) {
  var xdiff = ad.scalar.sub(x1, x2);
  var ydiff = ad.scalar.sub(y1, y2);
  return ad.scalar.sqrt(ad.scalar.add(
    ad.scalar.mul(xdiff, xdiff),
    ad.scalar.mul(ydiff, ydiff)
  ));
}
 
// Can use normal scalar inputs 
var out = dist(0, 1, 1, 4);
console.log(out);   // 3.162... 
 
// Use 'lifted' inputs to track derivatives 
var x1 = ad.lift(0);
var y1 = ad.lift(1);
var x2 = ad.lift(1);
var y2 = ad.lift(4);
var out = dist(x1, y1, x2, y2);
console.log(ad.value(out));   // still 3.162... 
out.backprop();   // Compute derivatives of inputs 
console.log(ad.derivative(x1)); // -0.316...

Tensor code

adnn also supports computations involving tensors, or a mixture of scalars and tensors:

var ad = require('adnn/ad');
var Tensor = require('adnn/tensor');
 
function dot(vec) {
  var sq = ad.tensor.mul(vec, vec);
  return ad.tensor.sumreduce(sq);
}
 
function dist(vec1, vec2) {
  return ad.scalar.sqrt(dot(ad.tensor.sub(vec1, vec2)));
}
 
var vec1 = ad.lift(new Tensor([3]).fromFlatArray([0, 1, 1]));
var vec2 = ad.lift(new Tensor([3]).fromFlatArray([2, 0, 3]));
var out = dist(vec1, vec2);
console.log(ad.value(out));   // 3 
out.backprop();
console.log(ad.derivative(vec1).toFlatArray());  // [-0.66, 0.33, -0.66]

Simple neural network

adnn makes it easy to define simple, feedforward neural networks. Here's a basic multilayer perceptron that takes a feature vector as input and outputs class probabilities:

var Tensor = require('adnn/tensor');
var ad = require('adnn/ad');
var nn = require('adnn/nn');
var opt = require('adnn/opt');
 
var nInputs = 20;
var nHidden = 10;
var nClasses = 5;
 
// Definition using basic layers 
var net = nn.sequence([
  nn.linear(nInputs, nHidden),
  nn.tanh,
  nn.linear(nHidden, nClasses),
  nn.softmax
]);
 
// Alternate definition using 'nn.mlp' utility 
net = nn.sequence([
  nn.mlp(nInputs, [
    {nOut: nHidden, activation: nn.tanh},
    {nOut: nClasses}
  ]),
  nn.softmax
]);
 
// Train the parameters of the network from some dataset 
// 'loadData' is a stand-in for a user-provided function that 
//    loads in an array of {input: , output: } objects 
// Here, 'input' is a feature vector, and 'output' is a class label 
var trainingData = loadData(...);
opt.nnTrain(net, trainingData, opt.classificationLoss, {
  batchSize: 10,
  iterations: 100,
  method: opt.adagrad()
});
 
// Predict class probabilities for new, unseen features 
var features = new Tensor([nInputs]).fillRandom();
var classProbs = net.eval(features);

Convolutional neural network

adnn includes the building blocks necessary to create convolutional networks. Here is a simple example, adapted from a ConvNetJS example:

var nn = require('adnn/nn');
 
var net = nn.sequence([
  // Assumes inputs are 32x32 RGB images (i.e. 3x32x32 Tensors) 
  nn.convolution({inDepth: 3, outDepth: 16, filterSize: 5}),
  nn.relu,
  nn.maxpool({filterSize: 2}),
  // Data now has size 16x16x16 
  nn.convolution({inDepth: 16, outDepth: 20, filterSize: 5}),
  nn.relu,
  nn.maxpool({filterSize: 2}),
  // Data now has size 20x8x8 
  nn.convolution({inDepth: 20, outDepth: 20, filterSize: 5}),
  nn.relu,
  nn.maxpool({filterSize: 2}),
  // Data now has size 20x4x4 = 320 
  nn.linear(320, 10),
  nn.softmax
  // Output is 10 class probabilities 
]);

Recurrent neural network

adnn is also flexible enough to support recurrent neural networks. Here's an example of a rudimentary RNN:

var ad = require('adnn/ad');
var nn = require('adnn/nn');
 
var inputSize = 10;
var outputSize = 5;
var stateSize = 20;
 
// Component neural networks used by the RNN 
var inputNet = nn.linear(inputSize, stateSize);
var stateNet = nn.linear(stateSize, stateSize);
var outputNet = nn.linear(stateSize, outputSize);
var initialStateNet = nn.constantparams([stateSize]);
 
function processSequence(seq) {
  // Initialize hidden state 
  var state = initialStateNet.eval();
  // Process input sequence in order 
  var outputs = [];
  for (var i = 0; i < seq.length; i++) {
    // Update hidden state 
    state = ad.tensor.tanh(ad.tensor.add(inputNet.eval(seq[i]), stateNet.eval(state)))
    // Generate output 
    outputs.push(outputNet.eval(state));
  }
  return outputs;
}

The `ad` module

The ad module has its own documentation here

The `nn` module

The nn module has its own documentation here

The `opt` module

The opt module has its own documentation here

Tensors

adnn provides a Tensor type for representing multidimensional arrays of numbers and various operations on them. This is the core datatype underlying neural net computations.

var Tensor = require('adnn/tensor');
 
// Create a rank-1 tensor (i.e. a vector) 
var vec = new Tensor([3]);  // vec is a 3-D vector 
// Fill vec with the contents of an array 
vec.fromArray([1, 2, 3]);
// Return the contents of vec as an array 
vec.toArray();  // returns [1, 2, 3] 
// Fill vec with a given value 
vec.fill(1);    // vec is now [1, 1, 1] 
// Fill vec with random values 
vec.fillRandom();
// Create a copy of vec 
var dupvec = vec.clone();
 
// Create a rank-2 tensor (i.e. a matrix) 
var mat = new Tensor([2, 2]);  // mat is a 2x2 matrix 
// Fill mat with the contents of an array 
mat.fromArray([[1, 2], [3, 4]]);
// Can also use a flattened array 
mat.fromFlatArray([1, 2, 3, 4]);
// Retrieve an individual element of mat 
var elem = mat.get([0, 1]);   // elem = 2 
// Set an individual element of mat 
mat.set([0, 1], 5);   // mat is now [[1, 5], [3, 4]]

The Tensor type also provides a large number of mathematical functions--unary operators, binary operators, reductions, matrix operations, etc. See tensor.js for a complete listing.