radagaisus/adnn

Javascript neural networks on top of general scalar/tensor reverse-mode automatic differentiation.

adnn

adnn provides Javascript-native neural networks on top of general scalar/tensor reverse-mode automatic differentiation. You can use just the AD code, or the NN layer built on top of it. This architecture makes it easy to define big, complex numerical computations and compute derivatives w.r.t. their inputs/parameters. adnn also includes utilities for optimizing/training the parameters of such computations.

Examples

Scalar code

The simplest use case for adnn:

var ad = require('adnn/ad')

function dist(x1, y1, x2, y2) {
  var xdiff = ad.scalar.sub(x1, x2);
  var ydiff = ad.scalar.sub(y1, y2);
  return ad.scalar.sqrt(ad.scalar.add(
    ad.scalar.mul(xdiff, xdiff),
    ad.scalar.mul(ydiff, ydiff)
  ));
}

// Can use normal scalar inputs
var out = dist(0, 1, 1, 4);
console.log(out);   // 3.162...

// Use 'lifted' inputs to track derivatives
var x1 = ad.lift(0);
var y1 = ad.lift(1);
var x2 = ad.lift(1);
var y2 = ad.lift(4);
var out = dist(x1, y1, x2, y2);
console.log(ad.value(out));   // still 3.162...
out.backprop();   // Compute derivatives of inputs
console.log(ad.derivative(x1)); // -0.316...

Tensor code

adnn also supports computations involving tensors, or a mixture of scalars and tensors:

var ad = require('adnn/ad');
var Tensor = require('adnn/tensor');

function dot(vec) {
  var sq = ad.tensor.mul(vec, vec);
  return ad.tensor.sumreduce(sq);
}

function dist(vec1, vec2) {
  return ad.scalar.sqrt(dot(ad.tensor.sub(vec1, vec2)));
}

var vec1 = ad.lift(new Tensor([3]).fromFlatArray([0, 1, 1]));
var vec2 = ad.lift(new Tensor([3]).fromFlatArray([2, 0, 3]));
var out = dist(vec1, vec2);
console.log(ad.value(out));   // 3
out.backprop();
console.log(ad.derivative(vec1).toFlatArray());  // [-0.66, 0.33, -0.66]

Simple neural network

adnn makes it easy to define simple, feedforward neural networks. Here's a basic multilayer perceptron that takes a feature vector as input and outputs class probabilities:

var Tensor = require('adnn/tensor');
var ad = require('adnn/ad');
var nn = require('adnn/nn');
var opt = require('adnn/opt');

var nInputs = 20;
var nHidden = 10;
var nClasses = 5;

// Definition using basic layers
var net = nn.sequence([
  nn.linear(nInputs, nHidden),
  nn.tanh,
  nn.linear(nHidden, nClasses),
  nn.softmax
]);

// Alternate definition using 'nn.mlp' utility
net = nn.sequence([
  nn.mlp(nInputs, [
    {nOut: nHidden, activation: nn.tanh},
    {nOut: nClasses}
  ]),
  nn.softmax
]);

// Train the parameters of the network from some dataset
// 'loadData' is a stand-in for a user-provided function that
//    loads in an array of {input: , output: } objects
// Here, 'input' is a feature vector, and 'output' is a class label
var trainingData = loadData(...);
opt.nnTrain(net, trainingData, opt.classificationLoss, {
  batchSize: 10,
  iterations: 100,
  method: opt.adagrad()
});

// Predict class probabilities for new, unseen features
var features = new Tensor([nInputs]).fillRandom();
var classProbs = net.eval(features);

Convolutional neural network

adnn includes the building blocks necessary to create convolutional networks. Here is a simple example, adapted from a ConvNetJS example:

var nn = require('adnn/nn');

var net = nn.sequence([
  // Assumes inputs are 32x32 RGB images (i.e. 3x32x32 Tensors)
  nn.convolution({inDepth: 3, outDepth: 16, filterSize: 5}),
  nn.relu,
  nn.maxpool({filterSize: 2}),
  // Data now has size 16x16x16
  nn.convolution({inDepth: 16, outDepth: 20, filterSize: 5}),
  nn.relu,
  nn.maxpool({filterSize: 2}),
  // Data now has size 20x8x8
  nn.convolution({inDepth: 20, outDepth: 20, filterSize: 5}),
  nn.relu,
  nn.maxpool({filterSize: 2}),
  // Data now has size 20x4x4 = 320
  nn.linear(320, 10),
  nn.softmax
  // Output is 10 class probabilities
]);

Recurrent neural network

adnn is also flexible enough to support recurrent neural networks. Here's an example of a rudimentary RNN:

var ad = require('adnn/ad');
var nn = require('adnn/nn');

var inputSize = 10;
var outputSize = 5;
var stateSize = 20;

// Component neural networks used by the RNN
var inputNet = nn.linear(inputSize, stateSize);
var stateNet = nn.linear(stateSize, stateSize);
var outputNet = nn.linear(stateSize, outputSize);
var initialStateNet = nn.constantparams([stateSize]);

function processSequence(seq) {
  // Initialize hidden state
  var state = initialStateNet.eval();
  // Process input sequence in order
  var outputs = [];
  for (var i = 0; i < seq.length; i++) {
    // Update hidden state
    state = ad.tensor.tanh(ad.tensor.add(inputNet.eval(seq[i]), stateNet.eval(state)))
    // Generate output
    outputs.push(outputNet.eval(state));
  }
  return outputs;
}

The ad module

The ad module has its own documentation here

The nn module

The nn module has its own documentation here

The opt module

The opt module has its own documentation here

Tensors

adnn provides a Tensor type for representing multidimensional arrays of numbers and various operations on them. This is the core datatype underlying neural net computations.

var Tensor = require('adnn/tensor');

// Create a rank-1 tensor (i.e. a vector)
var vec = new Tensor([3]);  // vec is a 3-D vector
// Fill vec with the contents of an array
vec.fromArray([1, 2, 3]);
// Return the contents of vec as an array
vec.toArray();  // returns [1, 2, 3]
// Fill vec with a given value
vec.fill(1);    // vec is now [1, 1, 1]
// Fill vec with random values
vec.fillRandom();
// Create a copy of vec
var dupvec = vec.clone();

// Create a rank-2 tensor (i.e. a matrix)
var mat = new Tensor([2, 2]);  // mat is a 2x2 matrix
// Fill mat with the contents of an array
mat.fromArray([[1, 2], [3, 4]]);
// Can also use a flattened array
mat.fromFlatArray([1, 2, 3, 4]);
// Retrieve an individual element of mat
var elem = mat.get([0, 1]);   // elem = 2
// Set an individual element of mat
mat.set([0, 1], 5);   // mat is now [[1, 5], [3, 4]]

The Tensor type also provides a large number of mathematical functions--unary operators, binary operators, reductions, matrix operations, etc. See tensor.js for a complete listing.

Projects using adnn

If you use adnn for anything, let us know and we'll list it here! Send email to [email protected]

• Neurally-Guided Procedural Models: Learning to Guide Procedural Models with Deep Neural Networks
• WebPPL uses adnn as part of its variational inference implementation.