Merge pull request #214 from Atry/hlists

Support differentiable HList

Author: Atry

DeepLearning/src/main/scala/com/thoughtworks/deeplearning/DeepLearning.scala

@@ -1,17 +1,79 @@
 package com.thoughtworks.deeplearning
+import java.io.{PrintStream, PrintWriter}
+
 import com.thoughtworks.deeplearning.DeepLearning.Tape
 import com.thoughtworks.continuation._
 import com.thoughtworks.future._
-
 import scalaz.syntax.all._
 import com.thoughtworks.raii.asynchronous._
 import simulacrum.typeclass
 
 import scala.language.implicitConversions
 import algebra.ring.MultiplicativeMonoid
+import scalaz.Semigroup
 
 object DeepLearning {
 
+  implicit object multipleExceptionThrowableSemigroup extends Semigroup[Throwable] {
+    override def append(f1: Throwable, f2: => Throwable): Throwable =
+      f1 match {
+        case me1: AbstractMultipleException =>
+          f2 match {
+            case me2: AbstractMultipleException => MultipleException(me1.throwableSet ++ me2.throwableSet)
+            case e: Throwable                   => MultipleException(me1.throwableSet + e)
+          }
+        case _: Throwable =>
+          f2 match {
+            case me2: AbstractMultipleException => MultipleException(me2.throwableSet + f1)
+            case `f1`                           => f1
+            case e: Throwable                   => MultipleException(Set(f1, e))
+          }
+      }
+  }
+
+  private final case class MultipleException(throwableSet: Set[Throwable])
+      extends DeepLearning.AbstractMultipleException
+
+  abstract class AbstractMultipleException extends RuntimeException("Multiple exceptions found") {
+
+    def throwableSet: Set[Throwable]
+
+    override def toString: String = throwableSet.mkString("\n")
+
+    override def printStackTrace(): Unit = {
+      for (throwable <- throwableSet) {
+        throwable.printStackTrace()
+      }
+    }
+
+    override def printStackTrace(s: PrintStream): Unit = {
+      for (throwable <- throwableSet) {
+        throwable.printStackTrace(s)
+      }
+    }
+
+    override def printStackTrace(s: PrintWriter): Unit = {
+      for (throwable <- throwableSet) {
+        throwable.printStackTrace(s)
+      }
+    }
+
+    override def getStackTrace: Array[StackTraceElement] = synchronized {
+      super.getStackTrace match {
+        case null =>
+          setStackTrace(throwableSet.flatMap(_.getStackTrace)(collection.breakOut))
+          super.getStackTrace
+        case stackTrace =>
+          stackTrace
+      }
+    }
+
+    override def fillInStackTrace(): this.type = {
+      this
+    }
+
+  }
+
   /** The node of wengert list created during [[DeepLearning.forward forward]] pass */
   final case class Tape[+Data, -Delta](data: Data, backward: Do[Delta] => UnitContinuation[Unit])
 

build.sbt

@@ -16,6 +16,8 @@ lazy val `plugins-Logging` = project.dependsOn(`plugins-Layers`, `plugins-Weight
 
 lazy val `plugins-Operators` = project
 
+lazy val `plugins-HLists` = project.dependsOn(DeepLearning)
+
 lazy val `plugins-FloatTraining` = project.dependsOn(`plugins-Training`)
 
 lazy val `plugins-FloatLiterals` = project.dependsOn(`DeepLearning`)
@@ -114,6 +116,7 @@ lazy val `plugins-CumulativeDoubleLayers` =
 
 lazy val `plugins-Builtins` =
   project.dependsOn(
+    `plugins-HLists`,
     `plugins-ImplicitsSingleton`,
     `plugins-Layers`,
     `plugins-Weights`,

plugins-DoubleWeights/build.sbt

@@ -4,4 +4,4 @@ libraryDependencies += "com.thoughtworks.feature" %% "implicitapply" % "2.3.0-M8
 
 libraryDependencies += "com.thoughtworks.feature" %% "factory" % "2.3.0-M8"
 
-libraryDependencies += "com.chuusai" %% "shapeless" % "2.3.2"
+libraryDependencies += "com.chuusai" %% "shapeless" % "2.3.3"

plugins-FloatWeights/build.sbt

@@ -4,4 +4,4 @@ libraryDependencies += "com.thoughtworks.feature" %% "implicitapply" % "2.3.0-M8
 
 libraryDependencies += "com.thoughtworks.feature" %% "factory" % "2.3.0-M8"
 
-libraryDependencies += "com.chuusai" %% "shapeless" % "2.3.2"
+libraryDependencies += "com.chuusai" %% "shapeless" % "2.3.3"

plugins-HLists/src/main/scala/com/thoughtworks/deeplearning/plugins/HLists.scala

@@ -0,0 +1,76 @@
+package com.thoughtworks.deeplearning.plugins
+
+import com.thoughtworks.continuation._
+import com.thoughtworks.deeplearning.DeepLearning
+import com.thoughtworks.deeplearning.DeepLearning.Tape
+import com.thoughtworks.raii.asynchronous._
+import scalaz.Applicative
+import scalaz.syntax.all._
+import scalaz.Tags.Parallel
+import shapeless.{::, HList, HNil}
+
+import java.io.{PrintStream, PrintWriter}
+
+import scalaz.Semigroup
+
+private object HLists {
+
+  implicit val doParallelApplicative =
+    asynchronousDoParallelApplicative(DeepLearning.multipleExceptionThrowableSemigroup)
+
+  private val noop: Do[HNil] => UnitContinuation[Unit] = {
+    Function.const(UnitContinuation.now(()))
+  }
+
+}
+
+/**
+  * @author 杨博 (Yang Bo)
+  */
+trait HLists {
+  import com.thoughtworks.deeplearning.plugins.HLists._
+
+  trait ImplicitsApi {
+    implicit def hnilDeepLearning[L <: HNil]: DeepLearning.Aux[L, HNil, HNil] = new DeepLearning[L] {
+      type Data = HNil
+      type Delta = HNil
+
+      def forward(differentiable: L): Do[Tape[Data, Delta]] = {
+        Do.now(Tape(HNil, noop))
+      }
+    }
+
+    implicit def hconsDeepLearning[Head, Tail <: HList, HeadData, TailData <: HList, HeadDelta, TailDelta <: HList](
+        implicit headDeepLearning: DeepLearning.Aux[Head, HeadData, HeadDelta],
+        tailDeepLearning: DeepLearning.Aux[Tail, TailData, TailDelta])
+      : DeepLearning.Aux[Head :: Tail, HeadData :: TailData, HeadDelta :: TailDelta] = new DeepLearning[Head :: Tail] {
+      type Data = HeadData :: TailData
+      type Delta = HeadDelta :: TailDelta
+
+      def forward(differentiable: Head :: Tail): Do[Tape[Data, Delta]] = {
+        val head :: tail = differentiable
+        val doHead: ParallelDo[Tape[HeadData, HeadDelta]] = Parallel(headDeepLearning.forward(head))
+
+        val doTail: ParallelDo[Tape[TailData, TailDelta]] = Parallel(tailDeepLearning.forward(tail))
+
+        Parallel.unwrap(Applicative[ParallelDo].tuple2(doHead, doTail)).map {
+          case (Tape(headData, headBackward), Tape(tailData, tailBackward)) =>
+            def backward(doDelta: Do[HeadDelta :: TailDelta]) = {
+              val continuationHead: ParallelContinuation[Unit] = Parallel(headBackward(doDelta.map(_.head)))
+              val continuationTail: ParallelContinuation[Unit] = Parallel(tailBackward(doDelta.map(_.tail)))
+              Parallel.unwrap(continuationParallelApplicative.apply2(continuationHead, continuationTail) {
+                (_: Unit, _: Unit) =>
+                  ()
+              })
+            }
+            Tape(headData :: tailData, backward)
+        }
+
+      }
+
+    }
+  }
+
+  type Implicits <: ImplicitsApi
+
+}

plugins-INDArrayLayers/src/main/scala-2.11/com/thoughtworks/deeplearning/plugins/INDArrayLayers.scala

@@ -23,43 +23,7 @@ import com.thoughtworks.dsl.Dsl
 
 object INDArrayLayers {
 
-  final case class MultipleException(throwableSet: Set[Throwable])
-      extends RuntimeException("Multiple exceptions found") {
-    override def toString: String = throwableSet.mkString("\n")
-
-    override def printStackTrace(): Unit = {
-      for (throwable <- throwableSet) {
-        throwable.printStackTrace()
-      }
-    }
-
-    override def printStackTrace(s: PrintStream): Unit = {
-      for (throwable <- throwableSet) {
-        throwable.printStackTrace(s)
-      }
-    }
-
-    override def printStackTrace(s: PrintWriter): Unit = {
-      for (throwable <- throwableSet) {
-        throwable.printStackTrace(s)
-      }
-    }
-
-    override def getStackTrace: Array[StackTraceElement] = synchronized {
-      super.getStackTrace match {
-        case null =>
-          setStackTrace(throwableSet.flatMap(_.getStackTrace)(collection.breakOut))
-          super.getStackTrace
-        case stackTrace =>
-          stackTrace
-      }
-    }
-
-    override def fillInStackTrace(): this.type = {
-      this
-    }
-
-  }
+  final case class MultipleException(throwableSet: Set[Throwable]) extends DeepLearning.AbstractMultipleException
 
   // Workaround for https://github.com/deeplearning4j/nd4j/issues/1869
   private[plugins] implicit final class Nd4jIssues1869Workaround(indArray: INDArray) {
@@ -134,23 +98,8 @@ trait INDArrayLayers extends DoubleLayers with DoubleLiterals with ImplicitsSing
   }
 
   @transient
-  private lazy val doParallelApplicative =
-    com.thoughtworks.raii.asynchronous.asynchronousDoParallelApplicative(new Semigroup[Throwable] {
-      override def append(f1: Throwable, f2: => Throwable): Throwable =
-        f1 match {
-          case MultipleException(exceptionSet1) =>
-            f2 match {
-              case MultipleException(exceptionSet2) => MultipleException(exceptionSet1 ++ exceptionSet2)
-              case e: Throwable                     => MultipleException(exceptionSet1 + e)
-            }
-          case _: Throwable =>
-            f2 match {
-              case MultipleException(exceptionSet2) => MultipleException(exceptionSet2 + f1)
-              case `f1`                             => f1
-              case e: Throwable                     => MultipleException(Set(f1, e))
-            }
-        }
-    })
+  implicit private lazy val doParallelApplicative =
+    asynchronousDoParallelApplicative(DeepLearning.multipleExceptionThrowableSemigroup)
 
   private def parallelApply2[A, B, C](doA: Do[A], doB: Do[B])(f: (A, B) => C): Do[C] = {
     Parallel.unwrap(doParallelApplicative.apply2(Parallel(doA), Parallel(doB))(f))

plugins-INDArrayWeights/build.sbt

@@ -4,7 +4,7 @@ libraryDependencies += "com.thoughtworks.feature" %% "implicitapply" % "2.3.0-M8
 
 libraryDependencies += "com.thoughtworks.feature" %% "factory" % "2.3.0-M8"
 
-libraryDependencies += "com.chuusai" %% "shapeless" % "2.3.2"
+libraryDependencies += "com.chuusai" %% "shapeless" % "2.3.3"
 
 libraryDependencies ++= {
   import Ordering.Implicits._

plugins-Operators/build.sbt

@@ -1 +1 @@
-libraryDependencies += "com.chuusai" %% "shapeless" % "2.3.2"
+libraryDependencies += "com.chuusai" %% "shapeless" % "2.3.3"