diff --git a/.gitattributes b/.gitattributes
new file mode 100644
index 00000000..2f21222a
--- /dev/null
+++ b/.gitattributes
@@ -0,0 +1,2 @@
+build/convnet-min.js -diff
+build/convnet.js -diff
\ No newline at end of file
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 00000000..39904dbc
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,2 @@
+node_modules/
+demo/mnist/
diff --git a/build/convnet-min.js b/build/convnet-min.js
index a97c95f9..8c762ee0 100644
--- a/build/convnet-min.js
+++ b/build/convnet-min.js
@@ -1 +1 @@
-var convnetjs=convnetjs||{REVISION:"ALPHA"};(function(c){var h=false;var d=0;var i=function(){if(h){h=false;return d}var k=2*Math.random()-1;var j=2*Math.random()-1;var l=k*k+j*j;if(l==0||l>1){return i()}var m=Math.sqrt(-2*Math.log(l)/l);d=j*m;h=true;return k*m};var g=function(k,j){return Math.random()*(j-k)+k};var f=function(k,j){return Math.floor(Math.random()*(j-k)+k)};var b=function(k,j){return k+i()*j};var e=function(l){if(typeof(l)==="undefined"||isNaN(l)){return[]}if(typeof ArrayBuffer==="undefined"){var j=new Array(l);for(var k=0;k<l;k++){j[k]=0}return j}else{return new Float64Array(l)}};var a=function(k){if(k.length===0){return{}}var j=k[0];var m=k[0];var l=0;var p=0;var q=k.length;for(var o=1;o<q;o++){if(k[o]>j){j=k[o];l=o}if(k[o]<m){m=k[o];p=o}}return{maxi:l,maxv:j,mini:p,minv:m,dv:j-m}};c.randf=g;c.randi=f;c.randn=b;c.zeros=e;c.maxmin=a})(convnetjs);(function(b){var a=function(k,g,f,j){this.sx=k;this.sy=g;this.depth=f;var h=k*g*f;this.w=b.zeros(h);this.dw=b.zeros(h);if(typeof j==="undefined"){var e=Math.sqrt(1/(k*g*f));for(var d=0;d<h;d++){this.w[d]=b.randn(0,e)}}else{for(var d=0;d<h;d++){this.w[d]=j}}};a.prototype={get:function(c,g,f){var e=((this.sx*g)+c)*this.depth+f;return this.w[e]},set:function(c,h,g,f){var e=((this.sx*h)+c)*this.depth+g;this.w[e]=f},add:function(c,h,g,f){var e=((this.sx*h)+c)*this.depth+g;this.w[e]+=f},get_grad:function(c,g,f){var e=((this.sx*g)+c)*this.depth+f;return this.dw[e]},set_grad:function(c,h,g,f){var e=((this.sx*h)+c)*this.depth+g;this.dw[e]=f},add_grad:function(c,h,g,f){var e=((this.sx*h)+c)*this.depth+g;this.dw[e]+=f},cloneAndZero:function(){return new a(this.sx,this.sy,this.depth,0)},clone:function(){var c=new a(this.sx,this.sy,this.depth,0);var e=this.w.length;for(var d=0;d<e;d++){c.w[d]=this.w[d]}return c},addFrom:function(c){for(var d=0;d<this.w.length;d++){this.w[d]+=c.w[d]}},addFromScaled:function(d,c){for(var e=0;e<this.w.length;e++){this.w[e]+=c*d.w[e]}},setConst:function(c){for(var d=0;d<this.w.length;d++){this.w[d]=c}},toJSON:function(){var c={};c.sx=this.sx;c.sy=this.sy;c.depth=this.depth;c.w=this.w;return c},fromJSON:function(d){this.sx=d.sx;this.sy=d.sy;this.depth=d.depth;var e=this.sx*this.sy*this.depth;this.w=b.zeros(e);this.dw=b.zeros(e);for(var c=0;c<e;c++){this.w[c]=d.w[c]}}};b.Vol=a})(convnetjs);(function(c){var a=c.Vol;var b=function(f,h,n,m,g){if(typeof(g)==="undefined"){var g=false}if(typeof(n)==="undefined"){var n=c.randi(0,f.sx-h)}if(typeof(m)==="undefined"){var m=c.randi(0,f.sy-h)}var e;if(h!==f.sx||n!==0||m!==0){e=new a(h,h,f.depth,0);for(var l=0;l<h;l++){for(var k=0;k<h;k++){if(l+n<0||l+n>=f.sx||k+m<0||k+m>=f.sy){continue}for(var j=0;j<f.depth;j++){e.set(l,k,j,f.get(l+n,k+m,j))}}}}else{e=f}if(g){var i=e.cloneAndZero();for(var l=0;l<e.sx;l++){for(var k=0;k<e.sy;k++){for(var j=0;j<e.depth;j++){i.set(l,k,j,e.get(e.sx-l-1,k,j))}}}e=i}return e};var d=function(o,n){if(typeof(n)==="undefined"){var n=false}var h=document.createElement("canvas");h.width=o.width;h.height=o.height;var u=h.getContext("2d");try{u.drawImage(o,0,0)}catch(q){if(q.name==="NS_ERROR_NOT_AVAILABLE"){return false}else{throw q}}try{var v=u.getImageData(0,0,h.width,h.height)}catch(q){if(q.name==="IndexSizeError"){return false}else{throw q}}var g=v.data;var k=o.width;var s=o.height;var t=[];for(var m=0;m<g.length;m++){t.push(g[m]/255-0.5)}var r=new a(k,s,4,0);r.w=t;if(n){var f=new a(k,s,1,0);for(var m=0;m<k;m++){for(var l=0;l<s;l++){f.set(m,l,0,r.get(m,l,0))}}r=f}return r};c.augment=b;c.img_to_vol=d})(convnetjs);(function(c){var a=c.Vol;var d=function(g){var g=g||{};this.out_depth=g.filters;this.sx=g.sx;this.in_depth=g.in_depth;this.in_sx=g.in_sx;this.in_sy=g.in_sy;this.sy=typeof g.sy!=="undefined"?g.sy:this.sx;this.stride=typeof g.stride!=="undefined"?g.stride:1;this.pad=typeof g.pad!=="undefined"?g.pad:0;this.l1_decay_mul=typeof g.l1_decay_mul!=="undefined"?g.l1_decay_mul:0;this.l2_decay_mul=typeof g.l2_decay_mul!=="undefined"?g.l2_decay_mul:1;this.out_sx=Math.floor((this.in_sx+this.pad*2-this.sx)/this.stride+1);this.out_sy=Math.floor((this.in_sy+this.pad*2-this.sy)/this.stride+1);this.layer_type="conv";var e=typeof g.bias_pref!=="undefined"?g.bias_pref:0;this.filters=[];for(var f=0;f<this.out_depth;f++){this.filters.push(new a(this.sx,this.sy,this.in_depth))}this.biases=new a(1,1,this.out_depth,e)};d.prototype={forward:function(l,s){this.in_act=l;var j=new a(this.out_sx,this.out_sy,this.out_depth,0);for(var o=0;o<this.out_depth;o++){var n=this.filters[o];var r=-this.pad;var p=-this.pad;for(var e=0;e<this.out_sx;r+=this.stride,e++){p=-this.pad;for(var t=0;t<this.out_sy;p+=this.stride,t++){var q=0;for(var m=0;m<n.sx;m++){for(var k=0;k<n.sy;k++){for(var i=0;i<n.depth;i++){var g=p+k;var h=r+m;if(g>=0&&g<l.sy&&h>=0&&h<l.sx){q+=n.w[((n.sx*k)+m)*n.depth+i]*l.w[((l.sx*g)+h)*l.depth+i]}}}}q+=this.biases.w[o];j.set(e,t,o,q)}}}this.out_act=j;return this.out_act},backward:function(){var k=this.in_act;k.dw=c.zeros(k.w.length);for(var n=0;n<this.out_depth;n++){var m=this.filters[n];var q=-this.pad;var p=-this.pad;for(var e=0;e<this.out_sx;q+=this.stride,e++){p=-this.pad;for(var t=0;t<this.out_sy;p+=this.stride,t++){var o=this.out_act.get_grad(e,t,n);for(var l=0;l<m.sx;l++){for(var j=0;j<m.sy;j++){for(var i=0;i<m.depth;i++){var g=p+j;var h=q+l;if(g>=0&&g<k.sy&&h>=0&&h<k.sx){var s=((k.sx*g)+h)*k.depth+i;var r=((m.sx*j)+l)*m.depth+i;m.dw[r]+=k.w[s]*o;k.dw[s]+=m.w[r]*o}}}}this.biases.dw[n]+=o}}}},getParamsAndGrads:function(){var e=[];for(var f=0;f<this.out_depth;f++){e.push({params:this.filters[f].w,grads:this.filters[f].dw,l2_decay_mul:this.l2_decay_mul,l1_decay_mul:this.l1_decay_mul})}e.push({params:this.biases.w,grads:this.biases.dw,l1_decay_mul:0,l2_decay_mul:0});return e},toJSON:function(){var f={};f.sx=this.sx;f.sy=this.sy;f.stride=this.stride;f.in_depth=this.in_depth;f.out_depth=this.out_depth;f.out_sx=this.out_sx;f.out_sy=this.out_sy;f.layer_type=this.layer_type;f.l1_decay_mul=this.l1_decay_mul;f.l2_decay_mul=this.l2_decay_mul;f.pad=this.pad;f.filters=[];for(var e=0;e<this.filters.length;e++){f.filters.push(this.filters[e].toJSON())}f.biases=this.biases.toJSON();return f},fromJSON:function(g){this.out_depth=g.out_depth;this.out_sx=g.out_sx;this.out_sy=g.out_sy;this.layer_type=g.layer_type;this.sx=g.sx;this.sy=g.sy;this.stride=g.stride;this.in_depth=g.in_depth;this.filters=[];this.l1_decay_mul=typeof g.l1_decay_mul!=="undefined"?g.l1_decay_mul:1;this.l2_decay_mul=typeof g.l2_decay_mul!=="undefined"?g.l2_decay_mul:1;this.pad=typeof g.pad!=="undefined"?g.pad:0;for(var f=0;f<g.filters.length;f++){var e=new a(0,0,0,0);e.fromJSON(g.filters[f]);this.filters.push(e)}this.biases=new a(0,0,0,0);this.biases.fromJSON(g.biases)}};var b=function(g){var g=g||{};this.out_depth=typeof g.num_neurons!=="undefined"?g.num_neurons:g.filters;this.l1_decay_mul=typeof g.l1_decay_mul!=="undefined"?g.l1_decay_mul:0;this.l2_decay_mul=typeof g.l2_decay_mul!=="undefined"?g.l2_decay_mul:1;this.num_inputs=g.in_sx*g.in_sy*g.in_depth;this.out_sx=1;this.out_sy=1;this.layer_type="fc";var e=typeof g.bias_pref!=="undefined"?g.bias_pref:0;this.filters=[];for(var f=0;f<this.out_depth;f++){this.filters.push(new a(1,1,this.num_inputs))}this.biases=new a(1,1,this.out_depth,e)};b.prototype={forward:function(h,l){this.in_act=h;var f=new a(1,1,this.out_depth,0);var k=h.w;for(var j=0;j<this.out_depth;j++){var g=0;var e=this.filters[j].w;for(var m=0;m<this.num_inputs;m++){g+=k[m]*e[m]}g+=this.biases.w[j];f.w[j]=g}this.out_act=f;return this.out_act},backward:function(){var e=this.in_act;e.dw=c.zeros(e.w.length);for(var f=0;f<this.out_depth;f++){var h=this.filters[f];var g=this.out_act.dw[f];for(var j=0;j<this.num_inputs;j++){e.dw[j]+=h.w[j]*g;h.dw[j]+=e.w[j]*g}this.biases.dw[f]+=g}},getParamsAndGrads:function(){var e=[];for(var f=0;f<this.out_depth;f++){e.push({params:this.filters[f].w,grads:this.filters[f].dw,l1_decay_mul:this.l1_decay_mul,l2_decay_mul:this.l2_decay_mul})}e.push({params:this.biases.w,grads:this.biases.dw,l1_decay_mul:0,l2_decay_mul:0});return e},toJSON:function(){var f={};f.out_depth=this.out_depth;f.out_sx=this.out_sx;f.out_sy=this.out_sy;f.layer_type=this.layer_type;f.num_inputs=this.num_inputs;f.l1_decay_mul=this.l1_decay_mul;f.l2_decay_mul=this.l2_decay_mul;f.filters=[];for(var e=0;e<this.filters.length;e++){f.filters.push(this.filters[e].toJSON())}f.biases=this.biases.toJSON();return f},fromJSON:function(g){this.out_depth=g.out_depth;this.out_sx=g.out_sx;this.out_sy=g.out_sy;this.layer_type=g.layer_type;this.num_inputs=g.num_inputs;this.l1_decay_mul=typeof g.l1_decay_mul!=="undefined"?g.l1_decay_mul:1;this.l2_decay_mul=typeof g.l2_decay_mul!=="undefined"?g.l2_decay_mul:1;this.filters=[];for(var f=0;f<g.filters.length;f++){var e=new a(0,0,0,0);e.fromJSON(g.filters[f]);this.filters.push(e)}this.biases=new a(0,0,0,0);this.biases.fromJSON(g.biases)}};c.ConvLayer=d;c.FullyConnLayer=b})(convnetjs);(function(c){var a=c.Vol;var b=function(d){var d=d||{};this.sx=d.sx;this.in_depth=d.in_depth;this.in_sx=d.in_sx;this.in_sy=d.in_sy;this.sy=typeof d.sy!=="undefined"?d.sy:this.sx;this.stride=typeof d.stride!=="undefined"?d.stride:2;this.pad=typeof d.pad!=="undefined"?d.pad:0;this.out_depth=this.in_depth;this.out_sx=Math.floor((this.in_sx+this.pad*2-this.sx)/this.stride+1);this.out_sy=Math.floor((this.in_sy+this.pad*2-this.sy)/this.stride+1);this.layer_type="pool";this.switchx=c.zeros(this.out_sx*this.out_sy*this.out_depth);this.switchy=c.zeros(this.out_sx*this.out_sy*this.out_depth)};b.prototype={forward:function(l,u){this.in_act=l;var h=new a(this.out_sx,this.out_sy,this.out_depth,0);var i=0;for(var p=0;p<this.out_depth;p++){var s=-this.pad;var q=-this.pad;for(var e=0;e<this.out_sx;s+=this.stride,e++){q=-this.pad;for(var w=0;w<this.out_sy;q+=this.stride,w++){var r=-99999;var o=-1,k=-1;for(var m=0;m<this.sx;m++){for(var j=0;j<this.sy;j++){var f=q+j;var g=s+m;if(f>=0&&f<l.sy&&g>=0&&g<l.sx){var t=l.get(g,f,p);if(t>r){r=t;o=g;k=f}}}}this.switchx[i]=o;this.switchy[i]=k;i++;h.set(e,w,p,r)}}}this.out_act=h;return this.out_act},backward:function(){var h=this.in_act;h.dw=c.zeros(h.w.length);var f=this.out_act;var g=0;for(var j=0;j<this.out_depth;j++){var l=-this.pad;var k=-this.pad;for(var e=0;e<this.out_sx;l+=this.stride,e++){k=-this.pad;for(var m=0;m<this.out_sy;k+=this.stride,m++){var i=this.out_act.get_grad(e,m,j);h.add_grad(this.switchx[g],this.switchy[g],j,i);g++}}}},getParamsAndGrads:function(){return[]},toJSON:function(){var d={};d.sx=this.sx;d.sy=this.sy;d.stride=this.stride;d.in_depth=this.in_depth;d.out_depth=this.out_depth;d.out_sx=this.out_sx;d.out_sy=this.out_sy;d.layer_type=this.layer_type;d.pad=this.pad;return d},fromJSON:function(d){this.out_depth=d.out_depth;this.out_sx=d.out_sx;this.out_sy=d.out_sy;this.layer_type=d.layer_type;this.sx=d.sx;this.sy=d.sy;this.stride=d.stride;this.in_depth=d.in_depth;this.pad=typeof d.pad!=="undefined"?d.pad:0;this.switchx=c.zeros(this.out_sx*this.out_sy*this.out_depth);this.switchy=c.zeros(this.out_sx*this.out_sy*this.out_depth)}};c.PoolLayer=b})(convnetjs);(function(c){var a=c.Vol;var b=function(d){var d=d||{};this.out_sx=typeof d.out_sx!=="undefined"?d.out_sx:d.in_sx;this.out_sy=typeof d.out_sy!=="undefined"?d.out_sy:d.in_sy;this.out_depth=typeof d.out_depth!=="undefined"?d.out_depth:d.in_depth;this.layer_type="input"};b.prototype={forward:function(d,e){this.in_act=d;this.out_act=d;return this.out_act},backward:function(){},getParamsAndGrads:function(){return[]},toJSON:function(){var d={};d.out_depth=this.out_depth;d.out_sx=this.out_sx;d.out_sy=this.out_sy;d.layer_type=this.layer_type;return d},fromJSON:function(d){this.out_depth=d.out_depth;this.out_sx=d.out_sx;this.out_sy=d.out_sy;this.layer_type=d.layer_type}};c.InputLayer=b})(convnetjs);(function(e){var a=e.Vol;var c=function(f){var f=f||{};this.num_inputs=f.in_sx*f.in_sy*f.in_depth;this.out_depth=this.num_inputs;this.out_sx=1;this.out_sy=1;this.layer_type="softmax"};c.prototype={forward:function(h,o){this.in_act=h;var f=new a(1,1,this.out_depth,0);var j=h.w;var k=h.w[0];for(var l=1;l<this.out_depth;l++){if(j[l]>k){k=j[l]}}var n=e.zeros(this.out_depth);var g=0;for(var l=0;l<this.out_depth;l++){var m=Math.exp(j[l]-k);g+=m;n[l]=m}for(var l=0;l<this.out_depth;l++){n[l]/=g;f.w[l]=n[l]}this.es=n;this.out_act=f;return this.out_act},backward:function(k){var f=this.in_act;f.dw=e.zeros(f.w.length);for(var h=0;h<this.out_depth;h++){var g=h===k?1:0;var j=-(g-this.es[h]);f.dw[h]=j}return -Math.log(this.es[k])},getParamsAndGrads:function(){return[]},toJSON:function(){var f={};f.out_depth=this.out_depth;f.out_sx=this.out_sx;f.out_sy=this.out_sy;f.layer_type=this.layer_type;f.num_inputs=this.num_inputs;return f},fromJSON:function(f){this.out_depth=f.out_depth;this.out_sx=f.out_sx;this.out_sy=f.out_sy;this.layer_type=f.layer_type;this.num_inputs=f.num_inputs}};var d=function(f){var f=f||{};this.num_inputs=f.in_sx*f.in_sy*f.in_depth;this.out_depth=this.num_inputs;this.out_sx=1;this.out_sy=1;this.layer_type="regression"};d.prototype={forward:function(f,g){this.in_act=f;this.out_act=f;return f},backward:function(l){var f=this.in_act;f.dw=e.zeros(f.w.length);var k=0;if(l instanceof Array||l instanceof Float64Array){for(var j=0;j<this.out_depth;j++){var g=f.w[j]-l[j];f.dw[j]=g;k+=2*g*g}}else{var j=l.dim;var h=l.val;var g=f.w[j]-h;f.dw[j]=g;k+=2*g*g}return k},getParamsAndGrads:function(){return[]},toJSON:function(){var f={};f.out_depth=this.out_depth;f.out_sx=this.out_sx;f.out_sy=this.out_sy;f.layer_type=this.layer_type;f.num_inputs=this.num_inputs;return f},fromJSON:function(f){this.out_depth=f.out_depth;this.out_sx=f.out_sx;this.out_sy=f.out_sy;this.layer_type=f.layer_type;this.num_inputs=f.num_inputs}};var b=function(f){var f=f||{};this.num_inputs=f.in_sx*f.in_sy*f.in_depth;this.out_depth=this.num_inputs;this.out_sx=1;this.out_sy=1;this.layer_type="svm"};b.prototype={forward:function(f,g){this.in_act=f;this.out_act=f;return f},backward:function(l){var g=this.in_act;g.dw=e.zeros(g.w.length);var f=g.w[l];var k=1;var j=0;for(var h=0;h<this.out_depth;h++){if(-f+g.w[h]+k>0){g.dw[h]+=1;g.dw[l]-=1;j+=-f+g.w[h]+k}}return j},getParamsAndGrads:function(){return[]},toJSON:function(){var f={};f.out_depth=this.out_depth;f.out_sx=this.out_sx;f.out_sy=this.out_sy;f.layer_type=this.layer_type;f.num_inputs=this.num_inputs;return f},fromJSON:function(f){this.out_depth=f.out_depth;this.out_sx=f.out_sx;this.out_sy=f.out_sy;this.layer_type=f.layer_type;this.num_inputs=f.num_inputs}};e.RegressionLayer=d;e.SoftmaxLayer=c;e.SVMLayer=b})(convnetjs);(function(d){var a=d.Vol;var e=function(h){var h=h||{};this.out_sx=h.in_sx;this.out_sy=h.in_sy;this.out_depth=h.in_depth;this.layer_type="relu"};e.prototype={forward:function(j,l){this.in_act=j;var h=j.clone();var m=j.w.length;var n=h.w;for(var k=0;k<m;k++){if(n[k]<0){n[k]=0}}this.out_act=h;return this.out_act},backward:function(){var j=this.in_act;var h=this.out_act;var l=j.w.length;j.dw=d.zeros(l);for(var k=0;k<l;k++){if(h.w[k]<=0){j.dw[k]=0}else{j.dw[k]=h.dw[k]}}},getParamsAndGrads:function(){return[]},toJSON:function(){var h={};h.out_depth=this.out_depth;h.out_sx=this.out_sx;h.out_sy=this.out_sy;h.layer_type=this.layer_type;return h},fromJSON:function(h){this.out_depth=h.out_depth;this.out_sx=h.out_sx;this.out_sy=h.out_sy;this.layer_type=h.layer_type}};var g=function(h){var h=h||{};this.out_sx=h.in_sx;this.out_sy=h.in_sy;this.out_depth=h.in_depth;this.layer_type="sigmoid"};g.prototype={forward:function(j,m){this.in_act=j;var h=j.cloneAndZero();var n=j.w.length;var o=h.w;var l=j.w;for(var k=0;k<n;k++){o[k]=1/(1+Math.exp(-l[k]))}this.out_act=h;return this.out_act},backward:function(){var j=this.in_act;var h=this.out_act;var m=j.w.length;j.dw=d.zeros(m);for(var k=0;k<m;k++){var l=h.w[k];j.dw[k]=l*(1-l)*h.dw[k]}},getParamsAndGrads:function(){return[]},toJSON:function(){var h={};h.out_depth=this.out_depth;h.out_sx=this.out_sx;h.out_sy=this.out_sy;h.layer_type=this.layer_type;return h},fromJSON:function(h){this.out_depth=h.out_depth;this.out_sx=h.out_sx;this.out_sy=h.out_sy;this.layer_type=h.layer_type}};var f=function(h){var h=h||{};this.group_size=typeof h.group_size!=="undefined"?h.group_size:2;this.out_sx=h.in_sx;this.out_sy=h.in_sy;this.out_depth=Math.floor(h.in_depth/this.group_size);this.layer_type="maxout";this.switches=d.zeros(this.out_sx*this.out_sy*this.out_depth)};f.prototype={forward:function(l,w){this.in_act=l;var q=this.out_depth;var v=new a(this.out_sx,this.out_sy,this.out_depth,0);if(this.out_sx===1&&this.out_sy===1){for(var p=0;p<q;p++){var m=p*this.group_size;var u=l.w[m];var r=0;for(var o=1;o<this.group_size;o++){var h=l.w[m+o];if(h>u){u=h;r=o}}v.w[p]=u;this.switches[p]=m+r}}else{var k=0;for(var t=0;t<l.sx;t++){for(var s=0;s<l.sy;s++){for(var p=0;p<q;p++){var m=p*this.group_size;var u=l.get(t,s,m);var r=0;for(var o=1;o<this.group_size;o++){var h=l.get(t,s,m+o);if(h>u){u=h;r=o}}v.set(t,s,p,u);this.switches[k]=m+r;k++}}}}this.out_act=v;return this.out_act},backward:function(){var k=this.in_act;var j=this.out_act;var o=this.out_depth;k.dw=d.zeros(k.w.length);if(this.out_sx===1&&this.out_sy===1){for(var l=0;l<o;l++){var m=j.dw[l];k.dw[this.switches[l]]=m}}else{var q=0;for(var h=0;h<j.sx;h++){for(var p=0;p<j.sy;p++){for(var l=0;l<o;l++){var m=j.get_grad(h,p,l);k.set_grad(h,p,this.switches[q],m);q++}}}}},getParamsAndGrads:function(){return[]},toJSON:function(){var h={};h.out_depth=this.out_depth;h.out_sx=this.out_sx;h.out_sy=this.out_sy;h.layer_type=this.layer_type;h.group_size=this.group_size;return h},fromJSON:function(h){this.out_depth=h.out_depth;this.out_sx=h.out_sx;this.out_sy=h.out_sy;this.layer_type=h.layer_type;this.group_size=h.group_size;this.switches=d.zeros(this.group_size)}};function c(h){var i=Math.exp(2*h);return(i-1)/(i+1)}var b=function(h){var h=h||{};this.out_sx=h.in_sx;this.out_sy=h.in_sy;this.out_depth=h.in_depth;this.layer_type="tanh"};b.prototype={forward:function(j,l){this.in_act=j;var h=j.cloneAndZero();var m=j.w.length;for(var k=0;k<m;k++){h.w[k]=c(j.w[k])}this.out_act=h;return this.out_act},backward:function(){var j=this.in_act;var h=this.out_act;var m=j.w.length;j.dw=d.zeros(m);for(var k=0;k<m;k++){var l=h.w[k];j.dw[k]=(1-l*l)*h.dw[k]}},getParamsAndGrads:function(){return[]},toJSON:function(){var h={};h.out_depth=this.out_depth;h.out_sx=this.out_sx;h.out_sy=this.out_sy;h.layer_type=this.layer_type;return h},fromJSON:function(h){this.out_depth=h.out_depth;this.out_sx=h.out_sx;this.out_sy=h.out_sy;this.layer_type=h.layer_type}};d.TanhLayer=b;d.MaxoutLayer=f;d.ReluLayer=e;d.SigmoidLayer=g})(convnetjs);(function(c){var a=c.Vol;var b=function(d){var d=d||{};this.out_sx=d.in_sx;this.out_sy=d.in_sy;this.out_depth=d.in_depth;this.layer_type="dropout";this.drop_prob=typeof d.drop_prob!=="undefined"?d.drop_prob:0.5;this.dropped=c.zeros(this.out_sx*this.out_sy*this.out_depth)};b.prototype={forward:function(e,g){this.in_act=e;if(typeof(g)==="undefined"){g=false}var d=e.clone();var h=e.w.length;if(g){for(var f=0;f<h;f++){if(Math.random()<this.drop_prob){d.w[f]=0;this.dropped[f]=true}else{this.dropped[f]=false}}}else{for(var f=0;f<h;f++){d.w[f]*=this.drop_prob}}this.out_act=d;return this.out_act},backward:function(){var d=this.in_act;var f=this.out_act;var g=d.w.length;d.dw=c.zeros(g);for(var e=0;e<g;e++){if(!(this.dropped[e])){d.dw[e]=f.dw[e]}}},getParamsAndGrads:function(){return[]},toJSON:function(){var d={};d.out_depth=this.out_depth;d.out_sx=this.out_sx;d.out_sy=this.out_sy;d.layer_type=this.layer_type;d.drop_prob=this.drop_prob;return d},fromJSON:function(d){this.out_depth=d.out_depth;this.out_sx=d.out_sx;this.out_sy=d.out_sy;this.layer_type=d.layer_type;this.drop_prob=d.drop_prob}};c.DropoutLayer=b})(convnetjs);(function(c){var a=c.Vol;var b=function(d){var d=d||{};this.k=d.k;this.n=d.n;this.alpha=d.alpha;this.beta=d.beta;this.out_sx=d.in_sx;this.out_sy=d.in_sy;this.out_depth=d.in_depth;this.layer_type="lrn";if(this.n%2===0){console.log("WARNING n should be odd for LRN layer")}};b.prototype={forward:function(f,p){this.in_act=f;var e=f.cloneAndZero();this.S_cache_=f.cloneAndZero();var k=Math.floor(this.n/2);for(var n=0;n<f.sx;n++){for(var m=0;m<f.sy;m++){for(var h=0;h<f.depth;h++){var l=f.get(n,m,h);var o=0;for(var g=Math.max(0,h-k);g<=Math.min(h+k,f.depth-1);g++){var d=f.get(n,m,g);o+=d*d}o*=this.alpha/this.n;o+=this.k;this.S_cache_.set(n,m,h,o);o=Math.pow(o,this.beta);e.set(n,m,h,l/o)}}}this.out_act=e;return this.out_act},backward:function(){var f=this.in_act;f.dw=c.zeros(f.w.length);var d=this.out_act;var n=Math.floor(this.n/2);for(var r=0;r<f.sx;r++){for(var q=0;q<f.sy;q++){for(var l=0;l<f.depth;l++){var p=this.out_act.get_grad(r,q,l);var k=this.S_cache_.get(r,q,l);var e=Math.pow(k,this.beta);var s=e*e;for(var h=Math.max(0,l-n);h<=Math.min(l+n,f.depth-1);h++){var o=f.get(r,q,h);var m=-o*this.beta*Math.pow(k,this.beta-1)*this.alpha/this.n*2*o;if(h===l){m+=e}m/=s;m*=p;f.add_grad(r,q,h,m)}}}}},getParamsAndGrads:function(){return[]},toJSON:function(){var d={};d.k=this.k;d.n=this.n;d.alpha=this.alpha;d.beta=this.beta;d.out_sx=this.out_sx;d.out_sy=this.out_sy;d.out_depth=this.out_depth;d.layer_type=this.layer_type;return d},fromJSON:function(d){this.k=d.k;this.n=d.n;this.alpha=d.alpha;this.beta=d.beta;this.out_sx=d.out_sx;this.out_sy=d.out_sy;this.out_depth=d.out_depth;this.layer_type=d.layer_type}};c.LocalResponseNormalizationLayer=b})(convnetjs);(function(c){var a=c.Vol;var b=function(d){var d=d||{};this.out_sx=d.in_sx;this.out_sy=d.in_sy;this.out_depth=d.in_depth+d.in_depth*d.in_depth;this.layer_type="quadtransform"};b.prototype={forward:function(d,n){this.in_act=d;var j=this.out_depth;var e=d.depth;var m=new a(this.out_sx,this.out_sy,this.out_depth,0);for(var l=0;l<d.sx;l++){for(var k=0;k<d.sy;k++){for(var h=0;h<j;h++){if(h<e){m.set(l,k,h,d.get(l,k,h))}else{var g=Math.floor((h-e)/e);var f=(h-e)-g*e;m.set(l,k,h,d.get(l,k,g)*d.get(l,k,f))}}}}this.out_act=m;return this.out_act},backward:function(){var d=this.in_act;d.dw=c.zeros(d.w.length);var n=this.out_act;var j=this.out_depth;var e=d.depth;for(var m=0;m<d.sx;m++){for(var l=0;l<d.sy;l++){for(var h=0;h<j;h++){var k=n.get_grad(m,l,h);if(h<e){d.add_grad(m,l,h,k)}else{var g=Math.floor((h-e)/e);var f=(h-e)-g*e;d.add_grad(m,l,g,d.get(m,l,f)*k);d.add_grad(m,l,f,d.get(m,l,g)*k)}}}}},getParamsAndGrads:function(){return[]},toJSON:function(){var d={};d.out_depth=this.out_depth;d.out_sx=this.out_sx;d.out_sy=this.out_sy;d.layer_type=this.layer_type;return d},fromJSON:function(d){this.out_depth=d.out_depth;this.out_sx=d.out_sx;this.out_sy=d.out_sy;this.layer_type=d.layer_type}};c.QuadTransformLayer=b})(convnetjs);(function(c){var a=c.Vol;var b=function(d){this.layers=[]};b.prototype={makeLayers:function(d){if(d.length<2){console.log("ERROR! For now at least have input and softmax layers.")}if(d[0].type!=="input"){console.log("ERROR! For now first layer should be input.")}var e=function(){var l=[];for(var k=0;k<d.length;k++){var m=d[k];if(m.type==="softmax"||m.type==="svm"){l.push({type:"fc",num_neurons:m.num_classes})}if(m.type==="regression"){l.push({type:"fc",num_neurons:m.num_neurons})}if((m.type==="fc"||m.type==="conv")&&typeof(m.bias_pref)==="undefined"){m.bias_pref=0;if(typeof m.activation!=="undefined"&&m.activation==="relu"){m.bias_pref=0.1}}if(typeof m.tensor!=="undefined"){if(m.tensor){l.push({type:"quadtransform"})}}l.push(m);if(typeof m.activation!=="undefined"){if(m.activation==="relu"){l.push({type:"relu"})}else{if(m.activation==="sigmoid"){l.push({type:"sigmoid"})}else{if(m.activation==="tanh"){l.push({type:"tanh"})}else{if(m.activation==="maxout"){var j=m.group_size!=="undefined"?m.group_size:2;l.push({type:"maxout",group_size:j})}else{console.log("ERROR unsupported activation "+m.activation)}}}}}if(typeof m.drop_prob!=="undefined"&&m.type!=="dropout"){l.push({type:"dropout",drop_prob:m.drop_prob})}}return l};d=e(d);this.layers=[];for(var f=0;f<d.length;f++){var h=d[f];if(f>0){var g=this.layers[f-1];h.in_sx=g.out_sx;h.in_sy=g.out_sy;h.in_depth=g.out_depth}switch(h.type){case"fc":this.layers.push(new c.FullyConnLayer(h));break;case"lrn":this.layers.push(new c.LocalResponseNormalizationLayer(h));break;case"dropout":this.layers.push(new c.DropoutLayer(h));break;case"input":this.layers.push(new c.InputLayer(h));break;case"softmax":this.layers.push(new c.SoftmaxLayer(h));break;case"regression":this.layers.push(new c.RegressionLayer(h));break;case"conv":this.layers.push(new c.ConvLayer(h));break;case"pool":this.layers.push(new c.PoolLayer(h));break;case"relu":this.layers.push(new c.ReluLayer(h));break;case"sigmoid":this.layers.push(new c.SigmoidLayer(h));break;case"tanh":this.layers.push(new c.TanhLayer(h));break;case"maxout":this.layers.push(new c.MaxoutLayer(h));break;case"quadtransform":this.layers.push(new c.QuadTransformLayer(h));break;case"svm":this.layers.push(new c.SVMLayer(h));break;default:console.log("ERROR: UNRECOGNIZED LAYER TYPE!")}}},forward:function(e,g){if(typeof(g)==="undefined"){g=false}var d=this.layers[0].forward(e,g);for(var f=1;f<this.layers.length;f++){d=this.layers[f].forward(d,g)}return d},backward:function(g){var f=this.layers.length;var e=this.layers[f-1].backward(g);for(var d=f-2;d>=0;d--){this.layers[d].backward()}return e},getParamsAndGrads:function(){var d=[];for(var f=0;f<this.layers.length;f++){var g=this.layers[f].getParamsAndGrads();for(var e=0;e<g.length;e++){d.push(g[e])}}return d},getPrediction:function(){var g=this.layers[this.layers.length-1];var h=g.out_act.w;var d=h[0];var e=0;for(var f=1;f<h.length;f++){if(h[f]>d){d=h[f];e=f}}return e},toJSON:function(){var e={};e.layers=[];for(var d=0;d<this.layers.length;d++){e.layers.push(this.layers[d].toJSON())}return e},fromJSON:function(h){this.layers=[];for(var g=0;g<h.layers.length;g++){var e=h.layers[g];var f=e.layer_type;var d;if(f==="input"){d=new c.InputLayer()}if(f==="relu"){d=new c.ReluLayer()}if(f==="sigmoid"){d=new c.SigmoidLayer()}if(f==="tanh"){d=new c.TanhLayer()}if(f==="dropout"){d=new c.DropoutLayer()}if(f==="conv"){d=new c.ConvLayer()}if(f==="pool"){d=new c.PoolLayer()}if(f==="lrn"){d=new c.LocalResponseNormalizationLayer()}if(f==="softmax"){d=new c.SoftmaxLayer()}if(f==="regression"){d=new c.RegressionLayer()}if(f==="fc"){d=new c.FullyConnLayer()}if(f==="maxout"){d=new c.MaxoutLayer()}if(f==="quadtransform"){d=new c.QuadTransformLayer()}if(f==="svm"){d=new c.SVMLayer()}d.fromJSON(e);this.layers.push(d)}}};c.Net=b})(convnetjs);(function(b){var a=b.Vol;var c=function(e,d){this.net=e;var d=d||{};this.learning_rate=typeof d.learning_rate!=="undefined"?d.learning_rate:0.01;this.l1_decay=typeof d.l1_decay!=="undefined"?d.l1_decay:0;this.l2_decay=typeof d.l2_decay!=="undefined"?d.l2_decay:0;this.batch_size=typeof d.batch_size!=="undefined"?d.batch_size:1;this.method=typeof d.method!=="undefined"?d.method:"sgd";this.momentum=typeof d.momentum!=="undefined"?d.momentum:0.9;this.ro=typeof d.ro!=="undefined"?d.ro:0.95;this.eps=typeof d.eps!=="undefined"?d.eps:0.000001;this.k=0;this.gsum=[];this.xsum=[]};c.prototype={train:function(s,r){var h=new Date().getTime();this.net.forward(s,true);var f=new Date().getTime();var q=f-h;var h=new Date().getTime();var A=this.net.backward(r);var k=0;var d=0;var f=new Date().getTime();var G=f-h;this.k++;if(this.k%this.batch_size===0){var e=this.net.getParamsAndGrads();if(this.gsum.length===0&&(this.method!=="sgd"||this.momentum>0)){for(var E=0;E<e.length;E++){this.gsum.push(b.zeros(e[E].params.length));if(this.method==="adadelta"){this.xsum.push(b.zeros(e[E].params.length))}else{this.xsum.push([])}}}for(var E=0;E<e.length;E++){var H=e[E];var w=H.params;var F=H.grads;var z=typeof H.l2_decay_mul!=="undefined"?H.l2_decay_mul:1;var I=typeof H.l1_decay_mul!=="undefined"?H.l1_decay_mul:1;var l=this.l2_decay*z;var n=this.l1_decay*I;var u=w.length;for(var B=0;B<u;B++){k+=l*w[B]*w[B]/2;d+=n*Math.abs(w[B]);var D=n*(w[B]>0?1:-1);var o=l*(w[B]);var t=(o+D+F[B])/this.batch_size;var m=this.gsum[E];var C=this.xsum[E];if(this.method==="adagrad"){m[B]=m[B]+t*t;var v=-this.learning_rate/Math.sqrt(m[B]+this.eps)*t;w[B]+=v}else{if(this.method==="windowgrad"){m[B]=this.ro*m[B]+(1-this.ro)*t*t;var v=-this.learning_rate/Math.sqrt(m[B]+this.eps)*t;w[B]+=v}else{if(this.method==="adadelta"){m[B]=this.ro*m[B]+(1-this.ro)*t*t;var v=-Math.sqrt((C[B]+this.eps)/(m[B]+this.eps))*t;C[B]=this.ro*C[B]+(1-this.ro)*v*v;w[B]+=v}else{if(this.momentum>0){var v=this.momentum*m[B]-this.learning_rate*t;m[B]=v;w[B]+=v}else{w[B]+=-this.learning_rate*t}}}}F[B]=0}}}return{fwd_time:q,bwd_time:G,l2_decay_loss:k,l1_decay_loss:d,cost_loss:A,softmax_loss:A,loss:A+d+k}}};b.Trainer=c;b.SGDTrainer=c})(convnetjs);(function(a){if(typeof module==="undefined"||typeof module.exports==="undefined"){window.jsfeat=a}else{module.exports=a}})(convnetjs);
\ No newline at end of file
+var convnetjs=convnetjs||{REVISION:"ALPHA"};(function(c){var h=false;var d=0;var i=function(){if(h){h=false;return d}var l=2*Math.random()-1;var k=2*Math.random()-1;var m=l*l+k*k;if(m==0||m>1){return i()}var n=Math.sqrt(-2*Math.log(m)/m);d=k*n;h=true;return l*n};var g=function(l,k){return Math.random()*(k-l)+l};var f=function(l,k){return Math.floor(Math.random()*(k-l)+l)};var b=function(l,k){return l+i()*k};var e=function(m){if(typeof(m)==="undefined"||isNaN(m)){return[]}if(typeof ArrayBuffer==="undefined"){var k=new Array(m);for(var l=0;l<m;l++){k[l]=0}return k}else{return new Float64Array(m)}};var a=function(l){if(l.length===0){return{}}var k=l[0];var o=l[0];var m=0;var q=0;var r=l.length;for(var p=1;p<r;p++){if(l[p]>k){k=l[p];m=p}if(l[p]<o){o=l[p];q=p}}return{maxi:m,maxv:k,mini:q,minv:o,dv:k-o}};var j=function(k,p,n,o){var m=new c.Vol(k,p,n,0);for(var l=0;l<m.w.length;l++){m.w[l]=g(0,1)<o}return{weight_mask:m,bias_mask:g(0,1)<o}};c.randf=g;c.randi=f;c.randn=b;c.zeros=e;c.maxmin=a;c.bernoulliMask=j})(convnetjs);(function(b){var a=function(k,g,f,j){this.sx=k;this.sy=g;this.depth=f;var h=k*g*f;this.w=b.zeros(h);this.dw=b.zeros(h);if(typeof j==="undefined"){var e=Math.sqrt(1/(k*g*f));for(var d=0;d<h;d++){this.w[d]=b.randn(0,e)}}else{for(var d=0;d<h;d++){this.w[d]=j}}};a.prototype={get:function(c,g,f){var e=((this.sx*g)+c)*this.depth+f;return this.w[e]},set:function(c,h,g,f){var e=((this.sx*h)+c)*this.depth+g;this.w[e]=f},add:function(c,h,g,f){var e=((this.sx*h)+c)*this.depth+g;this.w[e]+=f},get_grad:function(c,g,f){var e=((this.sx*g)+c)*this.depth+f;return this.dw[e]},set_grad:function(c,h,g,f){var e=((this.sx*h)+c)*this.depth+g;this.dw[e]=f},add_grad:function(c,h,g,f){var e=((this.sx*h)+c)*this.depth+g;this.dw[e]+=f},cloneAndZero:function(){return new a(this.sx,this.sy,this.depth,0)},clone:function(){var c=new a(this.sx,this.sy,this.depth,0);var e=this.w.length;for(var d=0;d<e;d++){c.w[d]=this.w[d]}return c},addFrom:function(c){for(var d=0;d<this.w.length;d++){this.w[d]+=c.w[d]}},addFromScaled:function(d,c){for(var e=0;e<this.w.length;e++){this.w[e]+=c*d.w[e]}},setConst:function(c){for(var d=0;d<this.w.length;d++){this.w[d]=c}},toJSON:function(){var c={};c.sx=this.sx;c.sy=this.sy;c.depth=this.depth;c.w=this.w;return c},fromJSON:function(d){this.sx=d.sx;this.sy=d.sy;this.depth=d.depth;var e=this.sx*this.sy*this.depth;this.w=b.zeros(e);this.dw=b.zeros(e);for(var c=0;c<e;c++){this.w[c]=d.w[c]}}};b.Vol=a})(convnetjs);(function(c){var a=c.Vol;var b=function(f,h,n,m,g){if(typeof(g)==="undefined"){var g=false}if(typeof(n)==="undefined"){var n=c.randi(0,f.sx-h)}if(typeof(m)==="undefined"){var m=c.randi(0,f.sy-h)}var e;if(h!==f.sx||n!==0||m!==0){e=new a(h,h,f.depth,0);for(var l=0;l<h;l++){for(var k=0;k<h;k++){if(l+n<0||l+n>=f.sx||k+m<0||k+m>=f.sy){continue}for(var j=0;j<f.depth;j++){e.set(l,k,j,f.get(l+n,k+m,j))}}}}else{e=f}if(g){var i=e.cloneAndZero();for(var l=0;l<e.sx;l++){for(var k=0;k<e.sy;k++){for(var j=0;j<e.depth;j++){i.set(l,k,j,e.get(e.sx-l-1,k,j))}}}e=i}return e};var d=function(o,n){if(typeof(n)==="undefined"){var n=false}var h=document.createElement("canvas");h.width=o.width;h.height=o.height;var u=h.getContext("2d");try{u.drawImage(o,0,0)}catch(q){if(q.name==="NS_ERROR_NOT_AVAILABLE"){return false}else{throw q}}try{var v=u.getImageData(0,0,h.width,h.height)}catch(q){if(q.name==="IndexSizeError"){return false}else{throw q}}var g=v.data;var k=o.width;var s=o.height;var t=[];for(var m=0;m<g.length;m++){t.push(g[m]/255-0.5)}var r=new a(k,s,4,0);r.w=t;if(n){var f=new a(k,s,1,0);for(var m=0;m<k;m++){for(var l=0;l<s;l++){f.set(m,l,0,r.get(m,l,0))}}r=f}return r};c.augment=b;c.img_to_vol=d})(convnetjs);(function(d){var a=d.Vol;var c=function(f){var f=f||{};this.keep_probability=f.keep_probability;this.num_gaussian_samples=f.num_gaussian_samples;this.activation_layer=d.build_layer({in_sx:1,in_sy:1,in_depth:1,type:f.activation})};c.prototype={fromJSON:function(f){this.keep_probability=f.keep_probability;this.num_gaussian_samples=f.num_gaussian_samples;this.activation_layer=d.build_layer({in_sx:1,in_sy:1,in_depth:1,type:f.activation})},toJSON:function(){var f={};f.keep_probability=this.keep_probability;f.num_gaussian_samples=this.num_gaussian_samples;f.activation_type=this.activation_layer.layer_type}};var e=function(h){var h=h||{};this.out_depth=h.filters;this.sx=h.sx;this.in_depth=h.in_depth;this.in_sx=h.in_sx;this.in_sy=h.in_sy;this.sy=typeof h.sy!=="undefined"?h.sy:this.sx;this.stride=typeof h.stride!=="undefined"?h.stride:1;this.pad=typeof h.pad!=="undefined"?h.pad:0;this.l1_decay_mul=typeof h.l1_decay_mul!=="undefined"?h.l1_decay_mul:0;this.l2_decay_mul=typeof h.l2_decay_mul!=="undefined"?h.l2_decay_mul:1;this.out_sx=Math.floor((this.in_sx+this.pad*2-this.sx)/this.stride+1);this.out_sy=Math.floor((this.in_sy+this.pad*2-this.sy)/this.stride+1);this.layer_type="conv";var f=typeof h.bias_pref!=="undefined"?h.bias_pref:0;this.filters=[];for(var g=0;g<this.out_depth;g++){this.filters.push(new a(this.sx,this.sy,this.in_depth))}this.biases=new a(1,1,this.out_depth,f)};e.prototype={forward:function(m,t){this.in_act=m;var k=new a(this.out_sx,this.out_sy,this.out_depth,0);for(var p=0;p<this.out_depth;p++){var o=this.filters[p];var s=-this.pad;var q=-this.pad;for(var g=0;g<this.out_sx;s+=this.stride,g++){q=-this.pad;for(var u=0;u<this.out_sy;q+=this.stride,u++){var r=0;for(var n=0;n<o.sx;n++){for(var l=0;l<o.sy;l++){for(var j=0;j<o.depth;j++){var h=q+l;var i=s+n;if(h>=0&&h<m.sy&&i>=0&&i<m.sx){r+=o.w[((o.sx*l)+n)*o.depth+j]*m.w[((m.sx*h)+i)*m.depth+j]}}}}r+=this.biases.w[p];k.set(g,u,p,r)}}}this.out_act=k;return this.out_act},backward:function(){var l=this.in_act;l.dw=d.zeros(l.w.length);for(var o=0;o<this.out_depth;o++){var n=this.filters[o];var r=-this.pad;var q=-this.pad;for(var g=0;g<this.out_sx;r+=this.stride,g++){q=-this.pad;for(var u=0;u<this.out_sy;q+=this.stride,u++){var p=this.out_act.get_grad(g,u,o);for(var m=0;m<n.sx;m++){for(var k=0;k<n.sy;k++){for(var j=0;j<n.depth;j++){var h=q+k;var i=r+m;if(h>=0&&h<l.sy&&i>=0&&i<l.sx){var t=((l.sx*h)+i)*l.depth+j;var s=((n.sx*k)+m)*n.depth+j;n.dw[s]+=l.w[t]*p;l.dw[t]+=n.w[s]*p}}}}this.biases.dw[o]+=p}}}},getParamsAndGrads:function(){var f=[];for(var g=0;g<this.out_depth;g++){f.push({params:this.filters[g].w,grads:this.filters[g].dw,l2_decay_mul:this.l2_decay_mul,l1_decay_mul:this.l1_decay_mul})}f.push({params:this.biases.w,grads:this.biases.dw,l1_decay_mul:0,l2_decay_mul:0});return f},toJSON:function(){var g={};g.sx=this.sx;g.sy=this.sy;g.stride=this.stride;g.in_depth=this.in_depth;g.out_depth=this.out_depth;g.out_sx=this.out_sx;g.out_sy=this.out_sy;g.layer_type=this.layer_type;g.l1_decay_mul=this.l1_decay_mul;g.l2_decay_mul=this.l2_decay_mul;g.pad=this.pad;g.filters=[];for(var f=0;f<this.filters.length;f++){g.filters.push(this.filters[f].toJSON())}g.biases=this.biases.toJSON();g.drop_connect=this.drop_connect.toJSON();return g},fromJSON:function(h){this.out_depth=h.out_depth;this.out_sx=h.out_sx;this.out_sy=h.out_sy;this.layer_type=h.layer_type;this.sx=h.sx;this.sy=h.sy;this.stride=h.stride;this.in_depth=h.in_depth;this.filters=[];this.l1_decay_mul=typeof h.l1_decay_mul!=="undefined"?h.l1_decay_mul:1;this.l2_decay_mul=typeof h.l2_decay_mul!=="undefined"?h.l2_decay_mul:1;this.pad=typeof h.pad!=="undefined"?h.pad:0;for(var g=0;g<h.filters.length;g++){var f=new a(0,0,0,0);f.fromJSON(h.filters[g]);this.filters.push(f)}this.biases=new a(0,0,0,0);this.biases.fromJSON(h.biases)}};var b=function(h){var h=h||{};this.out_depth=typeof h.num_neurons!=="undefined"?h.num_neurons:h.filters;this.l1_decay_mul=typeof h.l1_decay_mul!=="undefined"?h.l1_decay_mul:0;this.l2_decay_mul=typeof h.l2_decay_mul!=="undefined"?h.l2_decay_mul:1;this.drop_connect=typeof h.drop_connect!=="undefined"?c(h.drop_connect):null;this.num_inputs=h.in_sx*h.in_sy*h.in_depth;this.out_sx=1;this.out_sy=1;this.layer_type="fc";var f=typeof h.bias_pref!=="undefined"?h.bias_pref:0;this.filters=[];for(var g=0;g<this.out_depth;g++){this.filters.push(new a(1,1,this.num_inputs))}this.biases=new a(1,1,this.out_depth,f)};b.prototype={dropConnectEnabled:function(){return this.drop_connect!==null},initializeDropConnectMask:function(){this.drop_connect_masks=[];var f=this.dropConnectEnabled()?this.drop_connect.keep_probability:1;for(var g=0;g<this.filters.length;g++){this.drop_connect_masks[g]=d.bernoulliMask(1,1,this.num_inputs,f)}},forward:function(f,g){if(g){return this.forwardTrain(f)}else{return this.forwardPredict(f)}},forwardTrain:function(j){this.in_act=j;var g=new a(1,1,this.out_depth,0);this.initializeDropConnectMask();var m=j.w;for(var k=0;k<this.out_depth;k++){var h=0;var f=this.filters[k].w;var l=this.drop_connect_masks[k].weight_mask.w;for(var n=0;n<this.num_inputs;n++){h+=m[n]*f[n]*l[n]}h+=this.biases.w[k]*this.drop_connect_masks[k].bias_mask;g.w[k]=h}this.out_act=g;return this.out_act},averageGaussianActivation:function(n,f){var o=0;var h=0;for(var k=0;k<this.out_depth;k++){o+=n[k]*f[k];h+=(n[k]*n[k])*(f[k]*f[k])}var j=this.drop_connect_prop*o;var i=Math.sqrt(this.drop_connect_prop*h);var m=new a(1,1,this.drop_connect.num_gaussian_samples,0);for(var k=0;k<m.w.length;k++){m.w[k]=d.randn(j,i)}var l=this.drop_connect.activation_layer.forward(m);var g=0;for(var k=0;k<l.w.length;k++){g+=l.w[k]}return g/l.w.length},forwardPredict:function(j){this.in_act=j;var g=new a(1,1,this.out_depth,0);var l=j.w;for(var k=0;k<this.out_depth;k++){var f=this.filters[k].w;var h=0;if(this.dropConnectEnabled()){h=this.averageGaussianActivation(l,f)}else{for(var m=0;m<this.num_inputs;m++){h+=l[m]*f[m]}h+=this.biases.w[k]}g.w[k]=h}this.out_act=g;return this.out_act},backward:function(){var f=this.in_act;f.dw=d.zeros(f.w.length);for(var g=0;g<this.out_depth;g++){var k=this.filters[g];var h=this.out_act.dw[g];var j=this.drop_connect_masks[g].weight_mask.w;for(var l=0;l<this.num_inputs;l++){f.dw[l]+=k.w[l]*h;k.dw[l]+=f.w[l]*h*j[l]}this.biases.dw[g]+=h*this.drop_connect_masks[g].bias_mask}},getParamsAndGrads:function(){var f=[];for(var g=0;g<this.out_depth;g++){f.push({params:this.filters[g].w,grads:this.filters[g].dw,l1_decay_mul:this.l1_decay_mul,l2_decay_mul:this.l2_decay_mul})}f.push({params:this.biases.w,grads:this.biases.dw,l1_decay_mul:0,l2_decay_mul:0});return f},toJSON:function(){var g={};g.out_depth=this.out_depth;g.out_sx=this.out_sx;g.out_sy=this.out_sy;g.layer_type=this.layer_type;g.num_inputs=this.num_inputs;g.l1_decay_mul=this.l1_decay_mul;g.l2_decay_mul=this.l2_decay_mul;if(this.dropConnectEnabled()){g.drop_connect=this.drop_connect.toJSON()}g.filters=[];for(var f=0;f<this.filters.length;f++){g.filters.push(this.filters[f].toJSON())}g.biases=this.biases.toJSON();return g},fromJSON:function(h){this.out_depth=h.out_depth;this.out_sx=h.out_sx;this.out_sy=h.out_sy;this.layer_type=h.layer_type;this.num_inputs=h.num_inputs;this.l1_decay_mul=typeof h.l1_decay_mul!=="undefined"?h.l1_decay_mul:1;this.l2_decay_mul=typeof h.l2_decay_mul!=="undefined"?h.l2_decay_mul:1;if(typeof h.drop_connect!=="undefined"){this.drop_connect=new c();this.drop_connect.fromJSON(h.drop_connect)}this.filters=[];for(var g=0;g<h.filters.length;g++){var f=new a(0,0,0,0);f.fromJSON(h.filters[g]);this.filters.push(f)}this.biases=new a(0,0,0,0);this.biases.fromJSON(h.biases)}};d.ConvLayer=e;d.FullyConnLayer=b})(convnetjs);(function(c){var a=c.Vol;var b=function(d){var d=d||{};this.sx=d.sx;this.in_depth=d.in_depth;this.in_sx=d.in_sx;this.in_sy=d.in_sy;this.sy=typeof d.sy!=="undefined"?d.sy:this.sx;this.stride=typeof d.stride!=="undefined"?d.stride:2;this.pad=typeof d.pad!=="undefined"?d.pad:0;this.out_depth=this.in_depth;this.out_sx=Math.floor((this.in_sx+this.pad*2-this.sx)/this.stride+1);this.out_sy=Math.floor((this.in_sy+this.pad*2-this.sy)/this.stride+1);this.layer_type="pool";this.switchx=c.zeros(this.out_sx*this.out_sy*this.out_depth);this.switchy=c.zeros(this.out_sx*this.out_sy*this.out_depth)};b.prototype={forward:function(l,u){this.in_act=l;var h=new a(this.out_sx,this.out_sy,this.out_depth,0);var i=0;for(var p=0;p<this.out_depth;p++){var s=-this.pad;var q=-this.pad;for(var e=0;e<this.out_sx;s+=this.stride,e++){q=-this.pad;for(var w=0;w<this.out_sy;q+=this.stride,w++){var r=-99999;var o=-1,k=-1;for(var m=0;m<this.sx;m++){for(var j=0;j<this.sy;j++){var f=q+j;var g=s+m;if(f>=0&&f<l.sy&&g>=0&&g<l.sx){var t=l.get(g,f,p);if(t>r){r=t;o=g;k=f}}}}this.switchx[i]=o;this.switchy[i]=k;i++;h.set(e,w,p,r)}}}this.out_act=h;return this.out_act},backward:function(){var h=this.in_act;h.dw=c.zeros(h.w.length);var f=this.out_act;var g=0;for(var j=0;j<this.out_depth;j++){var l=-this.pad;var k=-this.pad;for(var e=0;e<this.out_sx;l+=this.stride,e++){k=-this.pad;for(var m=0;m<this.out_sy;k+=this.stride,m++){var i=this.out_act.get_grad(e,m,j);h.add_grad(this.switchx[g],this.switchy[g],j,i);g++}}}},getParamsAndGrads:function(){return[]},toJSON:function(){var d={};d.sx=this.sx;d.sy=this.sy;d.stride=this.stride;d.in_depth=this.in_depth;d.out_depth=this.out_depth;d.out_sx=this.out_sx;d.out_sy=this.out_sy;d.layer_type=this.layer_type;d.pad=this.pad;return d},fromJSON:function(d){this.out_depth=d.out_depth;this.out_sx=d.out_sx;this.out_sy=d.out_sy;this.layer_type=d.layer_type;this.sx=d.sx;this.sy=d.sy;this.stride=d.stride;this.in_depth=d.in_depth;this.pad=typeof d.pad!=="undefined"?d.pad:0;this.switchx=c.zeros(this.out_sx*this.out_sy*this.out_depth);this.switchy=c.zeros(this.out_sx*this.out_sy*this.out_depth)}};c.PoolLayer=b})(convnetjs);(function(c){var a=c.Vol;var b=function(d){var d=d||{};this.out_sx=typeof d.out_sx!=="undefined"?d.out_sx:d.in_sx;this.out_sy=typeof d.out_sy!=="undefined"?d.out_sy:d.in_sy;this.out_depth=typeof d.out_depth!=="undefined"?d.out_depth:d.in_depth;this.layer_type="input"};b.prototype={forward:function(d,e){this.in_act=d;this.out_act=d;return this.out_act},backward:function(){},getParamsAndGrads:function(){return[]},toJSON:function(){var d={};d.out_depth=this.out_depth;d.out_sx=this.out_sx;d.out_sy=this.out_sy;d.layer_type=this.layer_type;return d},fromJSON:function(d){this.out_depth=d.out_depth;this.out_sx=d.out_sx;this.out_sy=d.out_sy;this.layer_type=d.layer_type}};c.InputLayer=b})(convnetjs);(function(e){var a=e.Vol;var c=function(f){var f=f||{};this.num_inputs=f.in_sx*f.in_sy*f.in_depth;this.out_depth=this.num_inputs;this.out_sx=1;this.out_sy=1;this.layer_type="softmax"};c.prototype={forward:function(h,o){this.in_act=h;var f=new a(1,1,this.out_depth,0);var j=h.w;var k=h.w[0];for(var l=1;l<this.out_depth;l++){if(j[l]>k){k=j[l]}}var n=e.zeros(this.out_depth);var g=0;for(var l=0;l<this.out_depth;l++){var m=Math.exp(j[l]-k);g+=m;n[l]=m}for(var l=0;l<this.out_depth;l++){n[l]/=g;f.w[l]=n[l]}this.es=n;this.out_act=f;return this.out_act},backward:function(k){var f=this.in_act;f.dw=e.zeros(f.w.length);for(var h=0;h<this.out_depth;h++){var g=h===k?1:0;var j=-(g-this.es[h]);f.dw[h]=j}return -Math.log(this.es[k])},getParamsAndGrads:function(){return[]},toJSON:function(){var f={};f.out_depth=this.out_depth;f.out_sx=this.out_sx;f.out_sy=this.out_sy;f.layer_type=this.layer_type;f.num_inputs=this.num_inputs;return f},fromJSON:function(f){this.out_depth=f.out_depth;this.out_sx=f.out_sx;this.out_sy=f.out_sy;this.layer_type=f.layer_type;this.num_inputs=f.num_inputs}};var d=function(f){var f=f||{};this.num_inputs=f.in_sx*f.in_sy*f.in_depth;this.out_depth=this.num_inputs;this.out_sx=1;this.out_sy=1;this.layer_type="regression"};d.prototype={forward:function(f,g){this.in_act=f;this.out_act=f;return f},backward:function(l){var f=this.in_act;f.dw=e.zeros(f.w.length);var k=0;if(l instanceof Array||l instanceof Float64Array){for(var j=0;j<this.out_depth;j++){var g=f.w[j]-l[j];f.dw[j]=g;k+=2*g*g}}else{var j=l.dim;var h=l.val;var g=f.w[j]-h;f.dw[j]=g;k+=2*g*g}return k},getParamsAndGrads:function(){return[]},toJSON:function(){var f={};f.out_depth=this.out_depth;f.out_sx=this.out_sx;f.out_sy=this.out_sy;f.layer_type=this.layer_type;f.num_inputs=this.num_inputs;return f},fromJSON:function(f){this.out_depth=f.out_depth;this.out_sx=f.out_sx;this.out_sy=f.out_sy;this.layer_type=f.layer_type;this.num_inputs=f.num_inputs}};var b=function(f){var f=f||{};this.num_inputs=f.in_sx*f.in_sy*f.in_depth;this.out_depth=this.num_inputs;this.out_sx=1;this.out_sy=1;this.layer_type="svm"};b.prototype={forward:function(f,g){this.in_act=f;this.out_act=f;return f},backward:function(l){var g=this.in_act;g.dw=e.zeros(g.w.length);var f=g.w[l];var k=1;var j=0;for(var h=0;h<this.out_depth;h++){if(-f+g.w[h]+k>0){g.dw[h]+=1;g.dw[l]-=1;j+=-f+g.w[h]+k}}return j},getParamsAndGrads:function(){return[]},toJSON:function(){var f={};f.out_depth=this.out_depth;f.out_sx=this.out_sx;f.out_sy=this.out_sy;f.layer_type=this.layer_type;f.num_inputs=this.num_inputs;return f},fromJSON:function(f){this.out_depth=f.out_depth;this.out_sx=f.out_sx;this.out_sy=f.out_sy;this.layer_type=f.layer_type;this.num_inputs=f.num_inputs}};e.RegressionLayer=d;e.SoftmaxLayer=c;e.SVMLayer=b})(convnetjs);(function(d){var a=d.Vol;var e=function(h){var h=h||{};this.out_sx=h.in_sx;this.out_sy=h.in_sy;this.out_depth=h.in_depth;this.layer_type="relu"};e.prototype={forward:function(j,l){this.in_act=j;var h=j.clone();var m=j.w.length;var n=h.w;for(var k=0;k<m;k++){if(n[k]<0){n[k]=0}}this.out_act=h;return this.out_act},backward:function(){var j=this.in_act;var h=this.out_act;var l=j.w.length;j.dw=d.zeros(l);for(var k=0;k<l;k++){if(h.w[k]<=0){j.dw[k]=0}else{j.dw[k]=h.dw[k]}}},getParamsAndGrads:function(){return[]},toJSON:function(){var h={};h.out_depth=this.out_depth;h.out_sx=this.out_sx;h.out_sy=this.out_sy;h.layer_type=this.layer_type;return h},fromJSON:function(h){this.out_depth=h.out_depth;this.out_sx=h.out_sx;this.out_sy=h.out_sy;this.layer_type=h.layer_type}};var g=function(h){var h=h||{};this.out_sx=h.in_sx;this.out_sy=h.in_sy;this.out_depth=h.in_depth;this.layer_type="sigmoid"};g.prototype={forward:function(j,m){this.in_act=j;var h=j.cloneAndZero();var n=j.w.length;var o=h.w;var l=j.w;for(var k=0;k<n;k++){o[k]=1/(1+Math.exp(-l[k]))}this.out_act=h;return this.out_act},backward:function(){var j=this.in_act;var h=this.out_act;var m=j.w.length;j.dw=d.zeros(m);for(var k=0;k<m;k++){var l=h.w[k];j.dw[k]=l*(1-l)*h.dw[k]}},getParamsAndGrads:function(){return[]},toJSON:function(){var h={};h.out_depth=this.out_depth;h.out_sx=this.out_sx;h.out_sy=this.out_sy;h.layer_type=this.layer_type;return h},fromJSON:function(h){this.out_depth=h.out_depth;this.out_sx=h.out_sx;this.out_sy=h.out_sy;this.layer_type=h.layer_type}};var f=function(h){var h=h||{};this.group_size=typeof h.group_size!=="undefined"?h.group_size:2;this.out_sx=h.in_sx;this.out_sy=h.in_sy;this.out_depth=Math.floor(h.in_depth/this.group_size);this.layer_type="maxout";this.switches=d.zeros(this.out_sx*this.out_sy*this.out_depth)};f.prototype={forward:function(l,w){this.in_act=l;var q=this.out_depth;var v=new a(this.out_sx,this.out_sy,this.out_depth,0);if(this.out_sx===1&&this.out_sy===1){for(var p=0;p<q;p++){var m=p*this.group_size;var u=l.w[m];var r=0;for(var o=1;o<this.group_size;o++){var h=l.w[m+o];if(h>u){u=h;r=o}}v.w[p]=u;this.switches[p]=m+r}}else{var k=0;for(var t=0;t<l.sx;t++){for(var s=0;s<l.sy;s++){for(var p=0;p<q;p++){var m=p*this.group_size;var u=l.get(t,s,m);var r=0;for(var o=1;o<this.group_size;o++){var h=l.get(t,s,m+o);if(h>u){u=h;r=o}}v.set(t,s,p,u);this.switches[k]=m+r;k++}}}}this.out_act=v;return this.out_act},backward:function(){var k=this.in_act;var j=this.out_act;var o=this.out_depth;k.dw=d.zeros(k.w.length);if(this.out_sx===1&&this.out_sy===1){for(var l=0;l<o;l++){var m=j.dw[l];k.dw[this.switches[l]]=m}}else{var q=0;for(var h=0;h<j.sx;h++){for(var p=0;p<j.sy;p++){for(var l=0;l<o;l++){var m=j.get_grad(h,p,l);k.set_grad(h,p,this.switches[q],m);q++}}}}},getParamsAndGrads:function(){return[]},toJSON:function(){var h={};h.out_depth=this.out_depth;h.out_sx=this.out_sx;h.out_sy=this.out_sy;h.layer_type=this.layer_type;h.group_size=this.group_size;return h},fromJSON:function(h){this.out_depth=h.out_depth;this.out_sx=h.out_sx;this.out_sy=h.out_sy;this.layer_type=h.layer_type;this.group_size=h.group_size;this.switches=d.zeros(this.group_size)}};function c(h){var i=Math.exp(2*h);return(i-1)/(i+1)}var b=function(h){var h=h||{};this.out_sx=h.in_sx;this.out_sy=h.in_sy;this.out_depth=h.in_depth;this.layer_type="tanh"};b.prototype={forward:function(j,l){this.in_act=j;var h=j.cloneAndZero();var m=j.w.length;for(var k=0;k<m;k++){h.w[k]=c(j.w[k])}this.out_act=h;return this.out_act},backward:function(){var j=this.in_act;var h=this.out_act;var m=j.w.length;j.dw=d.zeros(m);for(var k=0;k<m;k++){var l=h.w[k];j.dw[k]=(1-l*l)*h.dw[k]}},getParamsAndGrads:function(){return[]},toJSON:function(){var h={};h.out_depth=this.out_depth;h.out_sx=this.out_sx;h.out_sy=this.out_sy;h.layer_type=this.layer_type;return h},fromJSON:function(h){this.out_depth=h.out_depth;this.out_sx=h.out_sx;this.out_sy=h.out_sy;this.layer_type=h.layer_type}};d.TanhLayer=b;d.MaxoutLayer=f;d.ReluLayer=e;d.SigmoidLayer=g})(convnetjs);(function(c){var a=c.Vol;var b=function(d){var d=d||{};this.out_sx=d.in_sx;this.out_sy=d.in_sy;this.out_depth=d.in_depth;this.layer_type="dropout";this.drop_prob=typeof d.drop_prob!=="undefined"?d.drop_prob:0.5;this.dropped=c.zeros(this.out_sx*this.out_sy*this.out_depth)};b.prototype={forward:function(e,g){this.in_act=e;if(typeof(g)==="undefined"){g=false}var d=e.clone();var h=e.w.length;if(g){for(var f=0;f<h;f++){if(Math.random()<this.drop_prob){d.w[f]=0;this.dropped[f]=true}else{this.dropped[f]=false}}}else{for(var f=0;f<h;f++){d.w[f]*=this.drop_prob}}this.out_act=d;return this.out_act},backward:function(){var d=this.in_act;var f=this.out_act;var g=d.w.length;d.dw=c.zeros(g);for(var e=0;e<g;e++){if(!(this.dropped[e])){d.dw[e]=f.dw[e]}}},getParamsAndGrads:function(){return[]},toJSON:function(){var d={};d.out_depth=this.out_depth;d.out_sx=this.out_sx;d.out_sy=this.out_sy;d.layer_type=this.layer_type;d.drop_prob=this.drop_prob;return d},fromJSON:function(d){this.out_depth=d.out_depth;this.out_sx=d.out_sx;this.out_sy=d.out_sy;this.layer_type=d.layer_type;this.drop_prob=d.drop_prob}};c.DropoutLayer=b})(convnetjs);(function(c){var a=c.Vol;var b=function(d){var d=d||{};this.k=d.k;this.n=d.n;this.alpha=d.alpha;this.beta=d.beta;this.out_sx=d.in_sx;this.out_sy=d.in_sy;this.out_depth=d.in_depth;this.layer_type="lrn";if(this.n%2===0){console.log("WARNING n should be odd for LRN layer")}};b.prototype={forward:function(f,p){this.in_act=f;var e=f.cloneAndZero();this.S_cache_=f.cloneAndZero();var k=Math.floor(this.n/2);for(var n=0;n<f.sx;n++){for(var m=0;m<f.sy;m++){for(var h=0;h<f.depth;h++){var l=f.get(n,m,h);var o=0;for(var g=Math.max(0,h-k);g<=Math.min(h+k,f.depth-1);g++){var d=f.get(n,m,g);o+=d*d}o*=this.alpha/this.n;o+=this.k;this.S_cache_.set(n,m,h,o);o=Math.pow(o,this.beta);e.set(n,m,h,l/o)}}}this.out_act=e;return this.out_act},backward:function(){var f=this.in_act;f.dw=c.zeros(f.w.length);var d=this.out_act;var n=Math.floor(this.n/2);for(var r=0;r<f.sx;r++){for(var q=0;q<f.sy;q++){for(var l=0;l<f.depth;l++){var p=this.out_act.get_grad(r,q,l);var k=this.S_cache_.get(r,q,l);var e=Math.pow(k,this.beta);var s=e*e;for(var h=Math.max(0,l-n);h<=Math.min(l+n,f.depth-1);h++){var o=f.get(r,q,h);var m=-o*this.beta*Math.pow(k,this.beta-1)*this.alpha/this.n*2*o;if(h===l){m+=e}m/=s;m*=p;f.add_grad(r,q,h,m)}}}}},getParamsAndGrads:function(){return[]},toJSON:function(){var d={};d.k=this.k;d.n=this.n;d.alpha=this.alpha;d.beta=this.beta;d.out_sx=this.out_sx;d.out_sy=this.out_sy;d.out_depth=this.out_depth;d.layer_type=this.layer_type;return d},fromJSON:function(d){this.k=d.k;this.n=d.n;this.alpha=d.alpha;this.beta=d.beta;this.out_sx=d.out_sx;this.out_sy=d.out_sy;this.out_depth=d.out_depth;this.layer_type=d.layer_type}};c.LocalResponseNormalizationLayer=b})(convnetjs);(function(c){var a=c.Vol;var b=function(e){this.layers=[]};b.prototype={makeLayers:function(e){if(e.length<2){console.log("ERROR! For now at least have input and softmax layers.")}if(e[0].type!=="input"){console.log("ERROR! For now first layer should be input.")}var f=function(){var m=[];for(var l=0;l<e.length;l++){var n=e[l];if(n.type==="softmax"||n.type==="svm"){m.push({type:"fc",num_neurons:n.num_classes})}if(n.type==="regression"){m.push({type:"fc",num_neurons:n.num_neurons})}if((n.type==="fc"||n.type==="conv")&&typeof(n.bias_pref)==="undefined"){n.bias_pref=0;if(typeof n.activation==="undefined"){continue}if(typeof n.drop_connect_keep_prop!=="undefined"){n.drop_connect={};n.drop_connect.keep_probability=n.drop_connect_keep_prob;n.drop_connect.num_gaussian_samples=n.drop_connect_num_gaussian_samples;n.drop_connect.activation=n.activation}if(n.activation==="relu"){n.bias_pref=0.1}}if(typeof n.tensor!=="undefined"){if(n.tensor){m.push({type:"quadtransform"})}}m.push(n);if(typeof n.activation!=="undefined"){if(n.activation==="relu"){m.push({type:"relu"})}else{if(n.activation==="sigmoid"){m.push({type:"sigmoid"})}else{if(n.activation==="tanh"){m.push({type:"tanh"})}else{if(n.activation==="maxout"){var k=n.group_size!=="undefined"?n.group_size:2;m.push({type:"maxout",group_size:k})}else{console.log("ERROR unsupported activation "+n.activation)}}}}}if(typeof n.drop_prob!=="undefined"&&n.type!=="dropout"){m.push({type:"dropout",drop_prob:n.drop_prob})}}return m};e=f(e);this.layers=[];for(var g=0;g<e.length;g++){var j=e[g];if(g>0){var h=this.layers[g-1];j.in_sx=h.out_sx;j.in_sy=h.out_sy;j.in_depth=h.out_depth}this.layers.push(c.build_layer(j))}},forward:function(f,h){if(typeof(h)==="undefined"){h=false}var e=this.layers[0].forward(f,h);for(var g=1;g<this.layers.length;g++){e=this.layers[g].forward(e,h)}return e},backward:function(h){var g=this.layers.length;var f=this.layers[g-1].backward(h);for(var e=g-2;e>=0;e--){this.layers[e].backward()}return f},getParamsAndGrads:function(){var e=[];for(var g=0;g<this.layers.length;g++){var h=this.layers[g].getParamsAndGrads();for(var f=0;f<h.length;f++){e.push(h[f])}}return e},getPrediction:function(){var h=this.layers[this.layers.length-1];var j=h.out_act.w;var e=j[0];var f=0;for(var g=1;g<j.length;g++){if(j[g]>e){e=j[g];f=g}}return f},toJSON:function(){var f={};f.layers=[];for(var e=0;e<this.layers.length;e++){f.layers.push(this.layers[e].toJSON())}return f},fromJSON:function(j){this.layers=[];for(var h=0;h<j.layers.length;h++){var f=j.layers[h];var g=f.layer_type;var e;if(g==="input"){e=new c.InputLayer()}if(g==="relu"){e=new c.ReluLayer()}if(g==="sigmoid"){e=new c.SigmoidLayer()}if(g==="tanh"){e=new c.TanhLayer()}if(g==="dropout"){e=new c.DropoutLayer()}if(g==="conv"){e=new c.ConvLayer()}if(g==="pool"){e=new c.PoolLayer()}if(g==="lrn"){e=new c.LocalResponseNormalizationLayer()}if(g==="softmax"){e=new c.SoftmaxLayer()}if(g==="regression"){e=new c.RegressionLayer()}if(g==="fc"){e=new c.FullyConnLayer()}if(g==="maxout"){e=new c.MaxoutLayer()}if(g==="quadtransform"){e=new c.QuadTransformLayer()}if(g==="svm"){e=new c.SVMLayer()}e.fromJSON(f);this.layers.push(e)}}};var d=function(e){switch(e.type){case"fc":return new c.FullyConnLayer(e);case"lrn":return new c.LocalResponseNormalizationLayer(e);case"dropout":return new c.DropoutLayer(e);case"input":return new c.InputLayer(e);case"softmax":return new c.SoftmaxLayer(e);case"regression":return new c.RegressionLayer(e);case"conv":return new c.ConvLayer(e);case"pool":return new c.PoolLayer(e);case"relu":return new c.ReluLayer(e);case"sigmoid":return new c.SigmoidLayer(e);case"tanh":return new c.TanhLayer(e);case"maxout":return new c.MaxoutLayer(e);case"quadtransform":return new c.QuadTransformLayer(e);case"svm":return new c.SVMLayer(e);default:console.log("ERROR: UNRECOGNIZED LAYER TYPE!")}};c.Net=b;c.build_layer=d})(convnetjs);(function(b){var a=b.Vol;var c=function(e,d){this.net=e;var d=d||{};this.learning_rate=typeof d.learning_rate!=="undefined"?d.learning_rate:0.01;this.l1_decay=typeof d.l1_decay!=="undefined"?d.l1_decay:0;this.l2_decay=typeof d.l2_decay!=="undefined"?d.l2_decay:0;this.batch_size=typeof d.batch_size!=="undefined"?d.batch_size:1;this.method=typeof d.method!=="undefined"?d.method:"sgd";this.momentum=typeof d.momentum!=="undefined"?d.momentum:0.9;this.ro=typeof d.ro!=="undefined"?d.ro:0.95;this.eps=typeof d.eps!=="undefined"?d.eps:0.000001;this.k=0;this.gsum=[];this.xsum=[]};c.prototype={train:function(s,r){var h=new Date().getTime();this.net.forward(s,true);var f=new Date().getTime();var q=f-h;var h=new Date().getTime();var A=this.net.backward(r);var k=0;var d=0;var f=new Date().getTime();var G=f-h;this.k++;if(this.k%this.batch_size===0){var e=this.net.getParamsAndGrads();if(this.gsum.length===0&&(this.method!=="sgd"||this.momentum>0)){for(var E=0;E<e.length;E++){this.gsum.push(b.zeros(e[E].params.length));if(this.method==="adadelta"){this.xsum.push(b.zeros(e[E].params.length))}else{this.xsum.push([])}}}for(var E=0;E<e.length;E++){var H=e[E];var w=H.params;var F=H.grads;var z=typeof H.l2_decay_mul!=="undefined"?H.l2_decay_mul:1;var I=typeof H.l1_decay_mul!=="undefined"?H.l1_decay_mul:1;var l=this.l2_decay*z;var n=this.l1_decay*I;var u=w.length;for(var B=0;B<u;B++){k+=l*w[B]*w[B]/2;d+=n*Math.abs(w[B]);var D=n*(w[B]>0?1:-1);var o=l*(w[B]);var t=(o+D+F[B])/this.batch_size;var m=this.gsum[E];var C=this.xsum[E];if(this.method==="adagrad"){m[B]=m[B]+t*t;var v=-this.learning_rate/Math.sqrt(m[B]+this.eps)*t;w[B]+=v}else{if(this.method==="windowgrad"){m[B]=this.ro*m[B]+(1-this.ro)*t*t;var v=-this.learning_rate/Math.sqrt(m[B]+this.eps)*t;w[B]+=v}else{if(this.method==="adadelta"){m[B]=this.ro*m[B]+(1-this.ro)*t*t;var v=-Math.sqrt((C[B]+this.eps)/(m[B]+this.eps))*t;C[B]=this.ro*C[B]+(1-this.ro)*v*v;w[B]+=v}else{if(this.momentum>0){var v=this.momentum*m[B]-this.learning_rate*t;m[B]=v;w[B]+=v}else{w[B]+=-this.learning_rate*t}}}}F[B]=0}}}return{fwd_time:q,bwd_time:G,l2_decay_loss:k,l1_decay_loss:d,cost_loss:A,softmax_loss:A,loss:A+d+k}}};b.Trainer=c;b.SGDTrainer=c})(convnetjs);(function(a){if(typeof module==="undefined"||typeof module.exports==="undefined"){window.jsfeat=a}else{module.exports=a}})(convnetjs);
\ No newline at end of file
diff --git a/build/convnet.js b/build/convnet.js
index d8a7403c..93dc42d4 100644
--- a/build/convnet.js
+++ b/build/convnet.js
@@ -49,13 +49,26 @@ var convnetjs = convnetjs || { REVISION: 'ALPHA' };
       if(w[i] < minv) { minv = w[i]; mini = i; } 
     }
     return {maxi: maxi, maxv: maxv, mini: mini, minv: minv, dv:maxv-minv};
-  }
+  };
+
+
+  var bernoulliMask = function(x, y, d, probability) {
+    var weight_mask = new global.Vol(x, y, d, 0);
+    for(var i = 0; i < weight_mask.w.length; i++) {
+      weight_mask.w[i] = randf(0, 1) < probability;
+    }
+    return {
+      weight_mask: weight_mask,
+      bias_mask: randf(0, 1) < probability
+    };
+  };
 
   global.randf = randf;
   global.randi = randi;
   global.randn = randn;
   global.zeros = zeros;
   global.maxmin = maxmin;
+  global.bernoulliMask = bernoulliMask;
   
 })(convnetjs);
 (function(global) {
@@ -264,13 +277,47 @@ var convnetjs = convnetjs || { REVISION: 'ALPHA' };
 (function(global) {
   "use strict";
   var Vol = global.Vol; // convenience
-
+    
   // This file contains all layers that do dot products with input,
   // but usually in a different connectivity pattern and weight sharing
   // schemes: 
   // - FullyConn is fully connected dot products 
   // - ConvLayer does convolutions (so weight sharing spatially)
   // putting them together in one file because they are very similar
+  var DropConnect = function(opt) {
+    var opt = opt || {};
+    this.keep_probability = opt.keep_probability;
+    this.num_gaussian_samples = opt.num_gaussian_samples;
+    this.activation_layer = global.build_layer({
+      // Unused, as we just cal forward()
+      in_sx: 1,
+      in_sy: 1,
+      in_depth: 1,
+      type: opt.activation
+    });
+  };
+
+  DropConnect.prototype = {
+    fromJSON: function(json) {
+      this.keep_probability = json.keep_probability;
+      this.num_gaussian_samples = json.num_gaussian_samples;
+      this.activation_layer = global.build_layer({
+        // Unused, as we just cal forward()
+        in_sx: 1,
+        in_sy: 1,
+        in_depth: 1,
+        type: json.activation
+      });
+    },
+
+    toJSON: function() {
+      var opt = {};
+      opt.keep_probability = this.keep_probability;
+      opt.num_gaussian_samples = this.num_gaussian_samples;
+      opt.activation_type = this.activation_layer.layer_type;
+    }
+  };
+
   var ConvLayer = function(opt) {
     var opt = opt || {};
 
@@ -280,7 +327,7 @@ var convnetjs = convnetjs || { REVISION: 'ALPHA' };
     this.in_depth = opt.in_depth;
     this.in_sx = opt.in_sx;
     this.in_sy = opt.in_sy;
-    
+
     // optional
     this.sy = typeof opt.sy !== 'undefined' ? opt.sy : this.sx;
     this.stride = typeof opt.stride !== 'undefined' ? opt.stride : 1; // stride at which we apply filters to input volume
@@ -404,6 +451,7 @@ var convnetjs = convnetjs || { REVISION: 'ALPHA' };
         json.filters.push(this.filters[i].toJSON());
       }
       json.biases = this.biases.toJSON();
+      json.drop_connect = this.drop_connect.toJSON();
       return json;
     },
     fromJSON: function(json) {
@@ -428,7 +476,7 @@ var convnetjs = convnetjs || { REVISION: 'ALPHA' };
       this.biases.fromJSON(json.biases);
     }
   }
-
+      
   var FullyConnLayer = function(opt) {
     var opt = opt || {};
 
@@ -439,13 +487,12 @@ var convnetjs = convnetjs || { REVISION: 'ALPHA' };
     // optional 
     this.l1_decay_mul = typeof opt.l1_decay_mul !== 'undefined' ? opt.l1_decay_mul : 0.0;
     this.l2_decay_mul = typeof opt.l2_decay_mul !== 'undefined' ? opt.l2_decay_mul : 1.0;
-
+    this.drop_connect = typeof opt.drop_connect !== 'undefined' ? DropConnect(opt.drop_connect) : null;
     // computed
     this.num_inputs = opt.in_sx * opt.in_sy * opt.in_depth;
     this.out_sx = 1;
     this.out_sy = 1;
     this.layer_type = 'fc';
-
     // initializations
     var bias = typeof opt.bias_pref !== 'undefined' ? opt.bias_pref : 0.0;
     this.filters = [];
@@ -454,43 +501,126 @@ var convnetjs = convnetjs || { REVISION: 'ALPHA' };
   }
 
   FullyConnLayer.prototype = {
+    dropConnectEnabled: function() {
+      return this.drop_connect !== null;
+    },
+
+    initializeDropConnectMask: function() {
+      this.drop_connect_masks = [];
+      var keep_probability = this.dropConnectEnabled() ? this.drop_connect.keep_probability : 1.0;
+      for(var i = 0; i < this.filters.length; i++) {
+        this.drop_connect_masks[i] =
+          global.bernoulliMask(1, 1, this.num_inputs, keep_probability);
+      }
+    },
+
     forward: function(V, is_training) {
+      if (is_training) {
+        return this.forwardTrain(V);
+      } else {
+        return this.forwardPredict(V);
+      }
+    },
+
+    forwardTrain: function(V) {
       this.in_act = V;
       var A = new Vol(1, 1, this.out_depth, 0.0);
+      this.initializeDropConnectMask();
       var Vw = V.w;
       for(var i=0;i<this.out_depth;i++) {
         var a = 0.0;
         var wi = this.filters[i].w;
+  
+        var weight_mask = this.drop_connect_masks[i].weight_mask.w;
         for(var d=0;d<this.num_inputs;d++) {
-          a += Vw[d] * wi[d]; // for efficiency use Vols directly for now
+          a += Vw[d] * wi[d] * weight_mask[d]; // for efficiency use Vols directly for now
         }
-        a += this.biases.w[i];
+        a += this.biases.w[i] * this.drop_connect_masks[i].bias_mask;
         A.w[i] = a;
       }
       this.out_act = A;
       return this.out_act;
     },
+
+    // Algorithm 2 in Regularization of Neural Networks using
+    // DropConnect - http://cs.nyu.edu/~wanli/dropc/dropc.pdf
+    averageGaussianActivation: function(Vw, wi) {
+      var mu = 0.0;
+      var sigma_squared = 0.0;
+      for(var d = 0; d < this.out_depth; d++) {
+        mu += Vw[d] * wi[d];
+        sigma_squared += (Vw[d] * Vw[d]) * (wi[d] * wi[d]);
+      }
+      var scaled_mu = this.drop_connect_prop * mu;
+      var scaled_sigma = Math.sqrt(this.drop_connect_prop * sigma_squared);
+      var sample_units = new Vol(1, 1, this.drop_connect.num_gaussian_samples, 0.0);
+      for (var d = 0; d < sample_units.w.length; d++) {
+        sample_units.w[d] = global.randn(scaled_mu, scaled_sigma);
+      }
+
+      var output_activations = this.drop_connect.activation_layer.forward(sample_units);
+      var sum_output = 0.0;
+      for (var d = 0; d < output_activations.w.length; d++) {
+        sum_output += output_activations.w[d];
+      }
+      return sum_output / output_activations.w.length;
+    },
+
+    forwardPredict: function(V) {
+      this.in_act = V;
+      var A = new Vol(1, 1, this.out_depth, 0.0);
+      var Vw = V.w;
+      for(var i=0;i<this.out_depth;i++) {
+        var wi = this.filters[i].w;
+        var a = 0.0;
+        if (this.dropConnectEnabled()) {
+          a = this.averageGaussianActivation(Vw, wi);
+        } else {
+          // Simply compute the \sum_j W_ij v_i
+          for(var d=0;d<this.num_inputs;d++) {
+            a += Vw[d] * wi[d]; // for efficiency use Vols directly for now
+          }
+          a += this.biases.w[i];
+        }
+        A.w[i] = a;
+      }
+
+      this.out_act = A;
+      return this.out_act;
+    },
+
     backward: function() {
       var V = this.in_act;
       V.dw = global.zeros(V.w.length); // zero out the gradient in input Vol
-      
+
       // compute gradient wrt weights and data
       for(var i=0;i<this.out_depth;i++) {
         var tfi = this.filters[i];
         var chain_grad = this.out_act.dw[i];
+        var weight_mask = this.drop_connect_masks[i].weight_mask.w;
         for(var d=0;d<this.num_inputs;d++) {
           V.dw[d] += tfi.w[d]*chain_grad; // grad wrt input data
-          tfi.dw[d] += V.w[d]*chain_grad; // grad wrt params
+          tfi.dw[d] += V.w[d]*chain_grad * weight_mask[d]; // grad wrt params
         }
-        this.biases.dw[i] += chain_grad;
+        this.biases.dw[i] += chain_grad * this.drop_connect_masks[i].bias_mask;
       }
     },
     getParamsAndGrads: function() {
       var response = [];
       for(var i=0;i<this.out_depth;i++) {
-        response.push({params: this.filters[i].w, grads: this.filters[i].dw, l1_decay_mul: this.l1_decay_mul, l2_decay_mul: this.l2_decay_mul});
+        response.push({
+          params: this.filters[i].w,
+          grads: this.filters[i].dw,
+          l1_decay_mul: this.l1_decay_mul,
+          l2_decay_mul: this.l2_decay_mul
+        });
       }
-      response.push({params: this.biases.w, grads: this.biases.dw, l1_decay_mul: 0.0, l2_decay_mul: 0.0});
+      response.push({
+        params: this.biases.w,
+        grads: this.biases.dw,
+        l1_decay_mul: 0.0,
+        l2_decay_mul: 0.0
+      });
       return response;
     },
     toJSON: function() {
@@ -502,6 +632,11 @@ var convnetjs = convnetjs || { REVISION: 'ALPHA' };
       json.num_inputs = this.num_inputs;
       json.l1_decay_mul = this.l1_decay_mul;
       json.l2_decay_mul = this.l2_decay_mul;
+
+      // DropConnect
+      if (this.dropConnectEnabled()) {
+        json.drop_connect = this.drop_connect.toJSON();
+      }
       json.filters = [];
       for(var i=0;i<this.filters.length;i++) {
         json.filters.push(this.filters[i].toJSON());
@@ -517,6 +652,13 @@ var convnetjs = convnetjs || { REVISION: 'ALPHA' };
       this.num_inputs = json.num_inputs;
       this.l1_decay_mul = typeof json.l1_decay_mul !== 'undefined' ? json.l1_decay_mul : 1.0;
       this.l2_decay_mul = typeof json.l2_decay_mul !== 'undefined' ? json.l2_decay_mul : 1.0;
+
+      // DropConnect
+      if (typeof json.drop_connect !== 'undefined') {
+        this.drop_connect = new DropConnect();
+        this.drop_connect.fromJSON(json.drop_connect);
+      }
+
       this.filters = [];
       for(var i=0;i<json.filters.length;i++) {
         var v = new Vol(0,0,0,0);
@@ -530,7 +672,6 @@ var convnetjs = convnetjs || { REVISION: 'ALPHA' };
 
   global.ConvLayer = ConvLayer;
   global.FullyConnLayer = FullyConnLayer;
-  
 })(convnetjs);
 (function(global) {
   "use strict";
@@ -1409,93 +1550,6 @@ var convnetjs = convnetjs || { REVISION: 'ALPHA' };
   "use strict";
   var Vol = global.Vol; // convenience
 
-  // transforms x-> [x, x_i*x_j forall i,j]
-  // so the fully connected layer afters will essentially be doing tensor multiplies
-  var QuadTransformLayer = function(opt) {
-    var opt = opt || {};
-
-    // computed
-    this.out_sx = opt.in_sx;
-    this.out_sy = opt.in_sy;
-    // linear terms, and then quadratic terms, of which there are 1/2*n*(n+1),
-    // (offdiagonals and the diagonal total) and arithmetic series.
-    // Actually never mind, lets not be fancy here yet and just include
-    // terms x_ix_j and x_jx_i twice. Half as efficient but much less
-    // headache.
-    this.out_depth = opt.in_depth + opt.in_depth * opt.in_depth;
-    this.layer_type = 'quadtransform';
-
-  }
-  QuadTransformLayer.prototype = {
-    forward: function(V, is_training) {
-      this.in_act = V;
-      var N = this.out_depth;
-      var Ni = V.depth;
-      var V2 = new Vol(this.out_sx, this.out_sy, this.out_depth, 0.0);
-      for(var x=0;x<V.sx;x++) {
-        for(var y=0;y<V.sy;y++) {
-          for(var i=0;i<N;i++) {
-            if(i<Ni) {
-              V2.set(x,y,i,V.get(x,y,i)); // copy these over (linear terms)
-            } else {
-              var i0 = Math.floor((i-Ni)/Ni);
-              var i1 = (i-Ni) - i0*Ni;
-              V2.set(x,y,i,V.get(x,y,i0) * V.get(x,y,i1)); // quadratic
-            }
-          }
-        }
-      }
-      this.out_act = V2;
-      return this.out_act; // dummy identity function for now
-    },
-    backward: function() {
-      var V = this.in_act;
-      V.dw = global.zeros(V.w.length); // zero out gradient wrt data
-      var V2 = this.out_act;
-      var N = this.out_depth;
-      var Ni = V.depth;
-      for(var x=0;x<V.sx;x++) {
-        for(var y=0;y<V.sy;y++) {
-          for(var i=0;i<N;i++) {
-            var chain_grad = V2.get_grad(x,y,i);
-            if(i<Ni) {
-              V.add_grad(x,y,i,chain_grad);
-            } else {
-              var i0 = Math.floor((i-Ni)/Ni);
-              var i1 = (i-Ni) - i0*Ni;
-              V.add_grad(x,y,i0,V.get(x,y,i1)*chain_grad);
-              V.add_grad(x,y,i1,V.get(x,y,i0)*chain_grad);
-            }
-          }
-        }
-      }
-    },
-    getParamsAndGrads: function() {
-      return [];
-    },
-    toJSON: function() {
-      var json = {};
-      json.out_depth = this.out_depth;
-      json.out_sx = this.out_sx;
-      json.out_sy = this.out_sy;
-      json.layer_type = this.layer_type;
-      return json;
-    },
-    fromJSON: function(json) {
-      this.out_depth = json.out_depth;
-      this.out_sx = json.out_sx;
-      this.out_sy = json.out_sy;
-      this.layer_type = json.layer_type; 
-    }
-  }
-  
-
-  global.QuadTransformLayer = QuadTransformLayer;
-})(convnetjs);
-(function(global) {
-  "use strict";
-  var Vol = global.Vol; // convenience
-  
   // Net manages a set of layers
   // For now constraints: Simple linear order of layers, first layer input last layer a cost layer
   var Net = function(options) {
@@ -1503,7 +1557,7 @@ var convnetjs = convnetjs || { REVISION: 'ALPHA' };
   }
 
   Net.prototype = {
-    
+
     // takes a list of layer definitions and creates the network layer objects
     makeLayers: function(defs) {
 
@@ -1516,7 +1570,7 @@ var convnetjs = convnetjs || { REVISION: 'ALPHA' };
         var new_defs = [];
         for(var i=0;i<defs.length;i++) {
           var def = defs[i];
-          
+
           if(def.type==='softmax' || def.type==='svm') {
             // add an fc layer here, there is no reason the user should
             // have to worry about this and we almost always want to
@@ -1529,16 +1583,25 @@ var convnetjs = convnetjs || { REVISION: 'ALPHA' };
             new_defs.push({type:'fc', num_neurons: def.num_neurons});
           }
 
-          if((def.type==='fc' || def.type==='conv') 
+          if((def.type==='fc' || def.type==='conv')
               && typeof(def.bias_pref) === 'undefined'){
             def.bias_pref = 0.0;
-            if(typeof def.activation !== 'undefined' && def.activation === 'relu') {
+            if(typeof def.activation === 'undefined') {
+              continue;
+            }
+            if(typeof def.drop_connect_keep_prop !== 'undefined') {
+              def.drop_connect = {};
+              def.drop_connect.keep_probability = def.drop_connect_keep_prob;
+              def.drop_connect.num_gaussian_samples = def.drop_connect_num_gaussian_samples;
+              def.drop_connect.activation = def.activation;
+            }
+            if(def.activation === 'relu') {
               def.bias_pref = 0.1; // relus like a bit of positive bias to get gradients early
               // otherwise it's technically possible that a relu unit will never turn on (by chance)
               // and will never get any gradient and never contribute any computation. Dead relu.
             }
           }
-          
+
           if(typeof def.tensor !== 'undefined') {
             // apply quadratic transform so that the upcoming multiply will include
             // quadratic terms, equivalent to doing a tensor product
@@ -1563,7 +1626,6 @@ var convnetjs = convnetjs || { REVISION: 'ALPHA' };
           if(typeof def.drop_prob !== 'undefined' && def.type !== 'dropout') {
             new_defs.push({type:'dropout', drop_prob: def.drop_prob});
           }
-
         }
         return new_defs;
       }
@@ -1580,23 +1642,7 @@ var convnetjs = convnetjs || { REVISION: 'ALPHA' };
           def.in_depth = prev.out_depth;
         }
 
-        switch(def.type) {
-          case 'fc': this.layers.push(new global.FullyConnLayer(def)); break;
-          case 'lrn': this.layers.push(new global.LocalResponseNormalizationLayer(def)); break;
-          case 'dropout': this.layers.push(new global.DropoutLayer(def)); break;
-          case 'input': this.layers.push(new global.InputLayer(def)); break;
-          case 'softmax': this.layers.push(new global.SoftmaxLayer(def)); break;
-          case 'regression': this.layers.push(new global.RegressionLayer(def)); break;
-          case 'conv': this.layers.push(new global.ConvLayer(def)); break;
-          case 'pool': this.layers.push(new global.PoolLayer(def)); break;
-          case 'relu': this.layers.push(new global.ReluLayer(def)); break;
-          case 'sigmoid': this.layers.push(new global.SigmoidLayer(def)); break;
-          case 'tanh': this.layers.push(new global.TanhLayer(def)); break;
-          case 'maxout': this.layers.push(new global.MaxoutLayer(def)); break;
-          case 'quadtransform': this.layers.push(new global.QuadTransformLayer(def)); break;
-          case 'svm': this.layers.push(new global.SVMLayer(def)); break;
-          default: console.log('ERROR: UNRECOGNIZED LAYER TYPE!');
-        }
+        this.layers.push(global.build_layer(def));
       }
     },
 
@@ -1673,9 +1719,29 @@ var convnetjs = convnetjs || { REVISION: 'ALPHA' };
       }
     }
   }
-  
+
+  var build_layer = function(def) {
+    switch(def.type) {
+    case 'fc': return new global.FullyConnLayer(def);
+    case 'lrn': return new global.LocalResponseNormalizationLayer(def);
+    case 'dropout': return new global.DropoutLayer(def);
+    case 'input': return new global.InputLayer(def);
+    case 'softmax': return new global.SoftmaxLayer(def);
+    case 'regression': return new global.RegressionLayer(def);
+    case 'conv': return new global.ConvLayer(def);
+    case 'pool': return new global.PoolLayer(def);
+    case 'relu': return new global.ReluLayer(def);
+    case 'sigmoid': return new global.SigmoidLayer(def);
+    case 'tanh': return new global.TanhLayer(def);
+    case 'maxout': return new global.MaxoutLayer(def);
+    case 'quadtransform': return new global.QuadTransformLayer(def);
+    case 'svm': return new global.SVMLayer(def);
+    default: console.log('ERROR: UNRECOGNIZED LAYER TYPE!');
+    }
+  };
 
   global.Net = Net;
+  global.build_layer = build_layer;
 })(convnetjs);
 (function(global) {
   "use strict";
diff --git a/src/convnet_layers_dotproducts.js b/src/convnet_layers_dotproducts.js
index d94c9c06..cd0939c5 100644
--- a/src/convnet_layers_dotproducts.js
+++ b/src/convnet_layers_dotproducts.js
@@ -4,10 +4,45 @@
 
   // This file contains all layers that do dot products with input,
   // but usually in a different connectivity pattern and weight sharing
-  // schemes: 
-  // - FullyConn is fully connected dot products 
+  // schemes:
+  // - FullyConn is fully connected dot products
   // - ConvLayer does convolutions (so weight sharing spatially)
   // putting them together in one file because they are very similar
+
+  // Encapsulates DropConnect configuration, layers, etc.
+  var DropConnect = function(opt) {
+    var opt = opt || {};
+    this.keep_probability = opt.keep_probability;
+    this.num_gaussian_samples = opt.num_gaussian_samples;
+    this.activation_layer = global.build_layer({
+      // Unused, as we just cal forward()
+      in_sx: 1,
+      in_sy: 1,
+      in_depth: 1,
+      type: opt.activation
+    });
+  };
+
+  DropConnect.prototype = {
+    fromJSON: function(json) {
+      this.keep_probability = json.keep_probability;
+      this.num_gaussian_samples = json.num_gaussian_samples;
+      this.activation_layer = global.build_layer({
+        // Unused, as we just cal forward()
+        in_sx: 1,
+        in_sy: 1,
+        in_depth: 1,
+        type: json.activation
+      });
+    },
+    toJSON: function() {
+      var opt = {};
+      opt.keep_probability = this.keep_probability;
+      opt.num_gaussian_samples = this.num_gaussian_samples;
+      opt.activation_type = this.activation_layer.layer_type;
+    }
+  };
+
   var ConvLayer = function(opt) {
     var opt = opt || {};
 
@@ -141,6 +176,7 @@
         json.filters.push(this.filters[i].toJSON());
       }
       json.biases = this.biases.toJSON();
+      json.drop_connect = this.drop_connect.toJSON();
       return json;
     },
     fromJSON: function(json) {
@@ -173,61 +209,143 @@
     // ok fine we will allow 'filters' as the word as well
     this.out_depth = typeof opt.num_neurons !== 'undefined' ? opt.num_neurons : opt.filters;
 
-    // optional 
+
+    // optional
     this.l1_decay_mul = typeof opt.l1_decay_mul !== 'undefined' ? opt.l1_decay_mul : 0.0;
     this.l2_decay_mul = typeof opt.l2_decay_mul !== 'undefined' ? opt.l2_decay_mul : 1.0;
-
+    this.drop_connect = typeof opt.drop_connect !== 'undefined' ? DropConnect(opt.drop_connect) : null;
     // computed
     this.num_inputs = opt.in_sx * opt.in_sy * opt.in_depth;
     this.out_sx = 1;
     this.out_sy = 1;
     this.layer_type = 'fc';
-
     // initializations
     var bias = typeof opt.bias_pref !== 'undefined' ? opt.bias_pref : 0.0;
     this.filters = [];
     for(var i=0;i<this.out_depth ;i++) { this.filters.push(new Vol(1, 1, this.num_inputs)); }
     this.biases = new Vol(1, 1, this.out_depth, bias);
-  }
+  };
 
   FullyConnLayer.prototype = {
+    dropConnectEnabled: function() {
+      return this.drop_connect !== null;
+    },
+
+    initializeDropConnectMask: function() {
+      this.drop_connect_masks = [];
+      var keep_probability = this.dropConnectEnabled() ? this.drop_connect.keep_probability : 1.0;
+      for(var i = 0; i < this.filters.length; i++) {
+        this.drop_connect_masks[i] =
+          global.bernoulliMask(1, 1, this.num_inputs, keep_probability);
+      }
+    },
+
     forward: function(V, is_training) {
+      if (is_training) {
+        return this.forwardTrain(V);
+      } else {
+        return this.forwardPredict(V);
+      }
+    },
+
+    forwardTrain: function(V) {
       this.in_act = V;
       var A = new Vol(1, 1, this.out_depth, 0.0);
+      this.initializeDropConnectMask();
       var Vw = V.w;
       for(var i=0;i<this.out_depth;i++) {
         var a = 0.0;
         var wi = this.filters[i].w;
+        var weight_mask = this.drop_connect_masks[i].weight_mask.w;
         for(var d=0;d<this.num_inputs;d++) {
-          a += Vw[d] * wi[d]; // for efficiency use Vols directly for now
+          a += Vw[d] * wi[d] * weight_mask[d]; // for efficiency use Vols directly for now
+        }
+        a += this.biases.w[i] * this.drop_connect_masks[i].bias_mask;
+        A.w[i] = a;
+      }
+      this.out_act = A;
+      return this.out_act;
+    },
+
+    // Algorithm 2 in Regularization of Neural Networks using
+    // DropConnect - http://cs.nyu.edu/~wanli/dropc/dropc.pdf
+    averageGaussianActivation: function(Vw, wi) {
+      var mu = 0.0;
+      var sigma_squared = 0.0;
+      for(var d = 0; d < this.out_depth; d++) {
+        mu += Vw[d] * wi[d];
+        sigma_squared += (Vw[d] * Vw[d]) * (wi[d] * wi[d]);
+      }
+      var scaled_mu = this.drop_connect_prop * mu;
+      var scaled_sigma = Math.sqrt(this.drop_connect_prop * sigma_squared);
+      var sample_units = new Vol(1, 1, this.drop_connect.num_gaussian_samples, 0.0);
+      for (var d = 0; d < sample_units.w.length; d++) {
+        sample_units.w[d] = global.randn(scaled_mu, scaled_sigma);
+      }
+
+      var output_activations = this.drop_connect.activation_layer.forward(sample_units);
+      var sum_output = 0.0;
+      for (var d = 0; d < output_activations.w.length; d++) {
+        sum_output += output_activations.w[d];
+      }
+      return sum_output / output_activations.w.length;
+    },
+
+    forwardPredict: function(V) {
+      this.in_act = V;
+      var A = new Vol(1, 1, this.out_depth, 0.0);
+      var Vw = V.w;
+      for(var i=0;i<this.out_depth;i++) {
+        var wi = this.filters[i].w;
+        var a = 0.0;
+        if (this.dropConnectEnabled()) {
+          a = this.averageGaussianActivation(Vw, wi);
+        } else {
+          // Simply compute the \sum_j W_ij v_i
+          for(var d=0;d<this.num_inputs;d++) {
+            a += Vw[d] * wi[d]; // for efficiency use Vols directly for now
+          }
+          a += this.biases.w[i];
         }
-        a += this.biases.w[i];
         A.w[i] = a;
       }
+
       this.out_act = A;
       return this.out_act;
     },
+
     backward: function() {
       var V = this.in_act;
       V.dw = global.zeros(V.w.length); // zero out the gradient in input Vol
-      
+
       // compute gradient wrt weights and data
       for(var i=0;i<this.out_depth;i++) {
         var tfi = this.filters[i];
         var chain_grad = this.out_act.dw[i];
+        var weight_mask = this.drop_connect_masks[i].weight_mask.w;
         for(var d=0;d<this.num_inputs;d++) {
           V.dw[d] += tfi.w[d]*chain_grad; // grad wrt input data
-          tfi.dw[d] += V.w[d]*chain_grad; // grad wrt params
+          tfi.dw[d] += V.w[d]*chain_grad * weight_mask[d]; // grad wrt params
         }
-        this.biases.dw[i] += chain_grad;
+        this.biases.dw[i] += chain_grad * this.drop_connect_masks[i].bias_mask;
       }
     },
     getParamsAndGrads: function() {
       var response = [];
       for(var i=0;i<this.out_depth;i++) {
-        response.push({params: this.filters[i].w, grads: this.filters[i].dw, l1_decay_mul: this.l1_decay_mul, l2_decay_mul: this.l2_decay_mul});
+        response.push({
+          params: this.filters[i].w,
+          grads: this.filters[i].dw,
+          l1_decay_mul: this.l1_decay_mul,
+          l2_decay_mul: this.l2_decay_mul
+        });
       }
-      response.push({params: this.biases.w, grads: this.biases.dw, l1_decay_mul: 0.0, l2_decay_mul: 0.0});
+      response.push({
+        params: this.biases.w,
+        grads: this.biases.dw,
+        l1_decay_mul: 0.0,
+        l2_decay_mul: 0.0
+      });
       return response;
     },
     toJSON: function() {
@@ -239,6 +357,11 @@
       json.num_inputs = this.num_inputs;
       json.l1_decay_mul = this.l1_decay_mul;
       json.l2_decay_mul = this.l2_decay_mul;
+
+      // DropConnect
+      if (this.dropConnectEnabled()) {
+        json.drop_connect = this.drop_connect.toJSON();
+      }
       json.filters = [];
       for(var i=0;i<this.filters.length;i++) {
         json.filters.push(this.filters[i].toJSON());
@@ -254,6 +377,13 @@
       this.num_inputs = json.num_inputs;
       this.l1_decay_mul = typeof json.l1_decay_mul !== 'undefined' ? json.l1_decay_mul : 1.0;
       this.l2_decay_mul = typeof json.l2_decay_mul !== 'undefined' ? json.l2_decay_mul : 1.0;
+
+      // DropConnect
+      if (typeof json.drop_connect !== 'undefined') {
+        this.drop_connect = new DropConnect();
+        this.drop_connect.fromJSON(json.drop_connect);
+      }
+
       this.filters = [];
       for(var i=0;i<json.filters.length;i++) {
         var v = new Vol(0,0,0,0);
diff --git a/src/convnet_net.js b/src/convnet_net.js
index e844bea9..5e7f41ac 100644
--- a/src/convnet_net.js
+++ b/src/convnet_net.js
@@ -35,16 +35,25 @@
             new_defs.push({type:'fc', num_neurons: def.num_neurons});
           }
 
-          if((def.type==='fc' || def.type==='conv') 
+          if((def.type==='fc' || def.type==='conv')
               && typeof(def.bias_pref) === 'undefined'){
             def.bias_pref = 0.0;
-            if(typeof def.activation !== 'undefined' && def.activation === 'relu') {
+            if(typeof def.activation === 'undefined') {
+              continue;
+            }
+            if(typeof def.drop_connect_keep_prop !== 'undefined') {
+              def.drop_connect = {};
+              def.drop_connect.keep_probability = def.drop_connect_keep_prob;
+              def.drop_connect.num_gaussian_samples = def.drop_connect_num_gaussian_samples;
+              def.drop_connect.activation = def.activation;
+            }
+            if(def.activation === 'relu') {
               def.bias_pref = 0.1; // relus like a bit of positive bias to get gradients early
               // otherwise it's technically possible that a relu unit will never turn on (by chance)
               // and will never get any gradient and never contribute any computation. Dead relu.
             }
           }
-          
+
           if(typeof def.tensor !== 'undefined') {
             // apply quadratic transform so that the upcoming multiply will include
             // quadratic terms, equivalent to doing a tensor product
@@ -86,23 +95,7 @@
           def.in_depth = prev.out_depth;
         }
 
-        switch(def.type) {
-          case 'fc': this.layers.push(new global.FullyConnLayer(def)); break;
-          case 'lrn': this.layers.push(new global.LocalResponseNormalizationLayer(def)); break;
-          case 'dropout': this.layers.push(new global.DropoutLayer(def)); break;
-          case 'input': this.layers.push(new global.InputLayer(def)); break;
-          case 'softmax': this.layers.push(new global.SoftmaxLayer(def)); break;
-          case 'regression': this.layers.push(new global.RegressionLayer(def)); break;
-          case 'conv': this.layers.push(new global.ConvLayer(def)); break;
-          case 'pool': this.layers.push(new global.PoolLayer(def)); break;
-          case 'relu': this.layers.push(new global.ReluLayer(def)); break;
-          case 'sigmoid': this.layers.push(new global.SigmoidLayer(def)); break;
-          case 'tanh': this.layers.push(new global.TanhLayer(def)); break;
-          case 'maxout': this.layers.push(new global.MaxoutLayer(def)); break;
-          case 'quadtransform': this.layers.push(new global.QuadTransformLayer(def)); break;
-          case 'svm': this.layers.push(new global.SVMLayer(def)); break;
-          default: console.log('ERROR: UNRECOGNIZED LAYER TYPE!');
-        }
+        this.layers.push(global.build_layer(def));
       }
     },
 
@@ -179,7 +172,27 @@
       }
     }
   }
-  
+
+  var build_layer = function(def) {
+    switch(def.type) {
+    case 'fc': return new global.FullyConnLayer(def);
+    case 'lrn': return new global.LocalResponseNormalizationLayer(def);
+    case 'dropout': return new global.DropoutLayer(def);
+    case 'input': return new global.InputLayer(def);
+    case 'softmax': return new global.SoftmaxLayer(def);
+    case 'regression': return new global.RegressionLayer(def);
+    case 'conv': return new global.ConvLayer(def);
+    case 'pool': return new global.PoolLayer(def);
+    case 'relu': return new global.ReluLayer(def);
+    case 'sigmoid': return new global.SigmoidLayer(def);
+    case 'tanh': return new global.TanhLayer(def);
+    case 'maxout': return new global.MaxoutLayer(def);
+    case 'quadtransform': return new global.QuadTransformLayer(def);
+    case 'svm': return new global.SVMLayer(def);
+    default: console.log('ERROR: UNRECOGNIZED LAYER TYPE!');
+    }
+  };
 
   global.Net = Net;
+  global.build_layer = build_layer;
 })(convnetjs);
diff --git a/src/convnet_util.js b/src/convnet_util.js
index ef28d7bb..8d8dc4de 100644
--- a/src/convnet_util.js
+++ b/src/convnet_util.js
@@ -48,12 +48,25 @@
       if(w[i] < minv) { minv = w[i]; mini = i; } 
     }
     return {maxi: maxi, maxv: maxv, mini: mini, minv: minv, dv:maxv-minv};
-  }
+  };
+
+
+  var bernoulliMask = function(x, y, d, probability) {
+    var weight_mask = new global.Vol(x, y, d, 0);
+    for(var i = 0; i < weight_mask.w.length; i++) {
+      weight_mask.w[i] = randf(0, 1) < probability;
+    }
+    return {
+      weight_mask: weight_mask,
+      bias_mask: randf(0, 1) < probability
+    };
+  };
 
   global.randf = randf;
   global.randi = randi;
   global.randn = randn;
   global.zeros = zeros;
   global.maxmin = maxmin;
+  global.bernoulliMask = bernoulliMask;
   
 })(convnetjs);