From 987f33417f43a6955e6c5b42a9288af29396512f Mon Sep 17 00:00:00 2001 From: Peter Livesey Date: Sun, 17 Feb 2019 18:01:41 +0700 Subject: [PATCH 1/3] There are 4 changes in this commit: 1. Added a podspec for NeuralNet. This should make it much easier for people to install and get started. 2. Changed the sigmoid function so it doesn't break the compiler. Currently, it throws an error saying the expression is too complex. 3. Added an initializer which allows you to copy a neural network 4. When loading from JSON, assume that the numbers are Doubles. Sometimes, these numbers are not Floats so the parsing fails. I haven't really looked into this, but I think it may happen when one of the numbers is very small and so it's parsed into a double instead. This avoids any failures to load a NeuralNetwork from disk. --- NeuralNet.podspec | 90 ++++++++++++++++++++++++++++++++++++++++ Sources/Activation.swift | 4 +- Sources/NeuralNet.swift | 20 ++++++++- Sources/Storage.swift | 10 ++--- 4 files changed, 117 insertions(+), 7 deletions(-) create mode 100644 NeuralNet.podspec diff --git a/NeuralNet.podspec b/NeuralNet.podspec new file mode 100644 index 0000000..a83ca84 --- /dev/null +++ b/NeuralNet.podspec @@ -0,0 +1,90 @@ +# +# Be sure to run `pod spec lint Toolbox.podspec' to ensure this is a +# valid spec and to remove all comments including this before submitting the spec. +# +# To learn more about Podspec attributes see https://docs.cocoapods.org/specification.html +# To see working Podspecs in the CocoaPods repo see https://github.com/CocoaPods/Specs/ +# + +Pod::Spec.new do |spec| + + spec.name = "NeuralNet" + spec.version = "0.0.1" + spec.summary = "An artificial neural network written in Swift" + + spec.description = "An artificial neural network written in Swift" + + spec.homepage = "https://github.com/Swift-AI/Swift-AI" + + spec.license = "MIT" + + spec.author = { "Collin Hundley" => "collinhundley@gmail.com" } + + # spec.platform = :ios + # spec.platform = :ios, "5.0" + + # When using multiple platforms + # spec.ios.deployment_target = "5.0" + # spec.osx.deployment_target = "10.7" + # spec.watchos.deployment_target = "2.0" + # spec.tvos.deployment_target = "9.0" + + + # ――― Source Location ―――――――――――――――――――――――――――――――――――――――――――――――――――――――――― # + # + # Specify the location from where the source should be retrieved. + # Supports git, hg, bzr, svn and HTTP. + # + + spec.source = { :git => "https://github.com/Swift-AI/NeuralNet.git" } + + + # ――― Source Code ―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――― # + # + # CocoaPods is smart about how it includes source code. For source files + # giving a folder will include any swift, h, m, mm, c & cpp files. + # For header files it will include any header in the folder. + # Not including the public_header_files will make all headers public. + # + + spec.source_files = 'Sources/**/*.swift' + + # ――― Resources ―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――― # + # + # A list of resources included with the Pod. These are copied into the + # target bundle with a build phase script. Anything else will be cleaned. + # You can preserve files from being cleaned, please don't preserve + # non-essential files like tests, examples and documentation. + # + + # spec.resource = "icon.png" + # spec.resources = "Resources/*.png" + + # spec.preserve_paths = "FilesToSave", "MoreFilesToSave" + + + # ――― Project Linking ―――――――――――――――――――――――――――――――――――――――――――――――――――――――――― # + # + # Link your library with frameworks, or libraries. Libraries do not include + # the lib prefix of their name. + # + + # spec.framework = "SomeFramework" + # spec.frameworks = "SomeFramework", "AnotherFramework" + + # spec.library = "iconv" + # spec.libraries = "iconv", "xml2" + + + # ――― Project Settings ――――――――――――――――――――――――――――――――――――――――――――――――――――――――― # + # + # If your library depends on compiler flags you can set them in the xcconfig hash + # where they will only apply to your library. If you depend on other Podspecs + # you can include multiple dependencies to ensure it works. + + # spec.requires_arc = true + + # spec.xcconfig = { "HEADER_SEARCH_PATHS" => "$(SDKROOT)/usr/include/libxml2" } + # spec.dependency "JSONKit", "~> 1.4" + +end diff --git a/Sources/Activation.swift b/Sources/Activation.swift index e76f024..349440c 100644 --- a/Sources/Activation.swift +++ b/Sources/Activation.swift @@ -302,7 +302,9 @@ public extension NeuralNet { // TODO break case .sigmoid: - result = zip(real, target).map{(-$0 * (1 - $0) * ($1 - $0))} + result = zip(real, target).map { (real: Float, target: Float) in + (-real * (1 - real) * (target - real)) + } case .softmax: vDSP_vsub(target, 1, real, 1, diff --git a/Sources/NeuralNet.swift b/Sources/NeuralNet.swift index 8933818..b537446 100644 --- a/Sources/NeuralNet.swift +++ b/Sources/NeuralNet.swift @@ -95,7 +95,25 @@ public final class NeuralNet { randomizeWeights() } } - + + /// Creates a new neural net by copying another neural net and modifying the batch size + /// This is useful if you use a large batch size to train the model but then want to infer with a smaller batch size (for instance 1) + public init(neuralNet: NeuralNet, structure: Structure, batchSize: Int) throws { + self.numLayers = neuralNet.numLayers + self.layerNodeCounts = neuralNet.layerNodeCounts + self.batchSize = batchSize + self.hiddenActivation = neuralNet.hiddenActivation + self.outputActivation = neuralNet.outputActivation + self.learningRate = neuralNet.learningRate + self.momentumFactor = neuralNet.momentumFactor + self.adjustedLearningRate = neuralNet.adjustedLearningRate + + // Initialize computed properties and caches + self.cache = Cache(structure: structure) + + try self.setWeights(neuralNet.allWeights()) + self.cache.layerBiases = neuralNet.cache.layerBiases + } } diff --git a/Sources/Storage.swift b/Sources/Storage.swift index 6ba8297..cdd8576 100644 --- a/Sources/Storage.swift +++ b/Sources/Storage.swift @@ -44,12 +44,12 @@ public extension NeuralNet { // Read all required values from JSON guard let layerNodeCounts = array[NeuralNet.layerNodeCountsKey] as? [Int], - let lr = array[NeuralNet.learningRateKey] as? Float, - let momentum = array[NeuralNet.momentumKey] as? Float, + let lr = array[NeuralNet.learningRateKey] as? Double, + let momentum = array[NeuralNet.momentumKey] as? Double, let batchSize = array[NeuralNet.batchSizeKey] as? Int, let hiddenActivationStr = array[NeuralNet.hiddenActivationKey] as? String, let outputActivationStr = array[NeuralNet.outputActivationKey] as? String, - let weights = array[NeuralNet.weightsKey] as? [[Float]] + let weights = array[NeuralNet.weightsKey] as? [[Double]] else { throw Error.initialization("One or more required NeuralNet properties are missing.") } @@ -87,10 +87,10 @@ public extension NeuralNet { // Recreate Structure object let structure = try Structure(nodes: layerNodeCounts, hiddenActivation: hiddenActivation, outputActivation: outputActivation, - batchSize: batchSize, learningRate: lr, momentum: momentum) + batchSize: batchSize, learningRate: Float(lr), momentum: Float(momentum)) // Initialize neural network - try self.init(structure: structure, weights: weights) + try self.init(structure: structure, weights: weights.map { $0.map { Float($0) }}) } From 0616b0d29bfd5ad99c1ce2ab03d71d3699629383 Mon Sep 17 00:00:00 2001 From: Peter Livesey Date: Tue, 19 Feb 2019 16:19:56 +0700 Subject: [PATCH 2/3] Adding discardableResult --- Sources/NeuralNet.swift | 1 + 1 file changed, 1 insertion(+) diff --git a/Sources/NeuralNet.swift b/Sources/NeuralNet.swift index b537446..5fbf748 100644 --- a/Sources/NeuralNet.swift +++ b/Sources/NeuralNet.swift @@ -495,6 +495,7 @@ public extension NeuralNet { /// or perform any other logic desired. /// - Returns: The total number of training epochs performed, and the final validation error. /// - Throws: An error if invalid data is provided. Checks are performed in advance to avoid problems during the training cycle. + @discardableResult public func train(_ data: Dataset, maxEpochs: Int, errorThreshold: Float, errorFunction: ErrorFunction, epochCallback: ((_ epoch: Int, _ error: Float) -> Bool)?) throws -> (epochs: Int, error: Float) { From 93aa681e788a0739c3de15f81d358c14d8d0c88b Mon Sep 17 00:00:00 2001 From: Peter Livesey Date: Fri, 22 Feb 2019 10:11:28 +0700 Subject: [PATCH 3/3] Adding biases to the disk storage Adding training error to the epoch callback. This is useful to know if the neural net is biased or has high variance (e.g. is the neural net overtraining) --- Sources/NeuralNet.swift | 49 ++++++++++++++++++++++++++++++++++++----- Sources/Storage.swift | 11 ++++++--- 2 files changed, 51 insertions(+), 9 deletions(-) diff --git a/Sources/NeuralNet.swift b/Sources/NeuralNet.swift index 5fbf748..a9dd3f4 100644 --- a/Sources/NeuralNet.swift +++ b/Sources/NeuralNet.swift @@ -74,7 +74,7 @@ public final class NeuralNet { // MARK: Initialization - public init(structure: Structure, weights: [[Float]]? = nil) throws { + public init(structure: Structure, weights: [[Float]]? = nil, biases: [[Float]]? = nil) throws { // Initialize basic properties self.numLayers = structure.numLayers self.layerNodeCounts = structure.layerNodeCounts @@ -94,6 +94,10 @@ public final class NeuralNet { } else { randomizeWeights() } + + if let biases = biases { + try self.setBiases(biases) + } } /// Creates a new neural net by copying another neural net and modifying the batch size @@ -136,6 +140,22 @@ public extension NeuralNet { public func allWeights() -> [[Float]] { return cache.layerWeights } + + /// Resets the network with the given biases (i.e. from a pre-trained network). + /// This change may safely be performed at any time. + /// + /// - Parameter weights: A 2D array of biases corresponding to each layer in the network. + public func setBiases(_ biases: [[Float]]) throws { + // TODO: ensure valid number of weights + + // Reset all weights in the network + cache.layerBiases = biases + } + + /// Returns an array of the network's current biases for each layer. + public func allBiases() -> [[Float]] { + return cache.layerBiases + } /// Randomizes all of the network's weights. fileprivate func randomizeWeights() { @@ -492,13 +512,16 @@ public extension NeuralNet { /// The handler must return a `Bool` indicating whether training should continue. /// If `false` is returned, the training routine will exit immediately and return. /// The user may implement this block to monitor the training progress, tune network parameters, - /// or perform any other logic desired. + /// or perform any other logic desired. NOTE: The validation error and training error are both + /// returned here, but the training error is taken before back propagation so may be slightly + /// lower than the actual number. In practice, this doesn't matter and is a performance + /// improvement, so you can probably ignore this fact. /// - Returns: The total number of training epochs performed, and the final validation error. /// - Throws: An error if invalid data is provided. Checks are performed in advance to avoid problems during the training cycle. @discardableResult public func train(_ data: Dataset, maxEpochs: Int, errorThreshold: Float, errorFunction: ErrorFunction, - epochCallback: ((_ epoch: Int, _ error: Float) -> Bool)?) throws -> (epochs: Int, error: Float) { + epochCallback: ((_ epoch: Int, _ validationError: Float, _ trainingError: Float) -> Bool)?) throws -> (epochs: Int, error: Float) { // Ensure valid error threshold guard errorThreshold > 0 else { throw Error.train("Training error threshold must be greater than zero.") @@ -518,14 +541,25 @@ public extension NeuralNet { // Reserve space for serializing all validation set outputs var validationOutputs = [Float](repeatElement(0, count: outputLength * batchSize * numBatches)) + + // Serialize all training labels into a single array + let trainLabels = data.trainLabels.reduce([], +) + + // Also, calculate error on the training set to report that as well + var trainingOutputs = [Float](repeatElement(0, count: outputLength * batchSize * data.trainInputs.count)) // Train until the desired error threshold is met or the max number of epochs has been executed var epochs = 1 while true { // Complete one full training epoch - for (batchinputs, batchLabels) in zip(data.trainInputs, data.trainLabels) { - try infer(batchinputs) + for (batchIndex, (batchinputs, batchLabels)) in zip(data.trainInputs, data.trainLabels).enumerated() { + let outputs = try infer(batchinputs) try backpropagate(batchLabels) + + for i in 0..