add docs

Network.hs

@@ -1,7 +1,43 @@
 {-# LANGUAGE ScopedTypeVariables, FlexibleContexts, BangPatterns #-}
 {-# OPTIONS -Wall #-}
 
-module Network where
+-- |
+-- Module      : Network
+-- Copyright   : (c) 2017 Christian Merten
+-- Maintainer  : c.merten@gmx.net
+-- Stability   : experimental
+-- Portability : GHC
+--
+-- An implementation of artifical feed-forward neural networks in pure Haskell.
+--
+-- An example is added in /XOR.hs/
+
+module Network (
+    -- * Network
+    Network(..),
+    Layer(..),
+    newNetwork,
+    output,
+    
+    -- * Learning functions
+    trainShuffled,
+    trainNTimes,
+    CostFunction(..),
+    getDelta,
+    LearningRate,
+    Lambda,
+    TrainingDataLength,
+    Sample, Samples, (-->),
+
+    -- * Activation functions
+    ActivationFunction, ActivationFunctionDerivative,
+    sigmoid,
+    sigmoid',
+
+    -- * Network serialization
+    saveNetwork,
+    loadNetwork
+) where
 
 import Data.List.Split (chunksOf)
 import Data.Binary
@@ -20,10 +56,7 @@ import Numeric.LinearAlgebra
 -- | The generic feedforward network type, a binary instance is implemented.
 -- It takes a list of layers
 -- with a minimum of one (output layer).
--- It is usually constructed using the `newNetwork` function, initializing the matrices
+-- It is usually constructed using the `newNetwork` function.
--- with some default random values.
---
--- > net <- newNetwork [2, 3, 4]
 data Network a = Network { layers :: [Layer a] }
                  deriving (Show)
 
@@ -56,20 +89,43 @@ getDelta :: Floating a => CostFunction -> a -> a -> a -> a
 getDelta QuadraticCost    z a y = (a - y) * sigmoid'(z)
 getDelta CrossEntropyCost _ a y = a - y
 
+-- | Activation function used to calculate the actual output of a neuron.
+-- Usually the 'sigmoid' function.
 type ActivationFunction a = a -> a
+
+-- | The derivative of an activation function.
 type ActivationFunctionDerivative a = a -> a
 
+-- | Training sample that can be used for the training functions.
+--
+-- > trainingData :: Samples Double
+-- > trainingData = [ fromList [0, 0] --> fromList [0],
+-- >                  fromList [0, 1] --> fromList [1],
+-- >                  fromList [1, 0] --> fromList [1],
+-- >                  fromList [1, 1] --> fromList [0]]
 type Sample a = (Vector a, Vector a)
+
+-- | A list of 'Sample's
 type Samples a = [Sample a]
 
 -- | A simple synonym for the (,) operator, used to create samples very intuitively.
 (-->) :: Vector a -> Vector a -> Sample a
 (-->) = (,)
 
+-- | The learning rate, affects the learning speed, lower learning rate results
+-- in slower learning, but usually better results after more epochs.
 type LearningRate = Double
+
+-- | Lambda value affecting the regularization while learning.
 type Lambda = Double
+
+-- | Wrapper around the training data length.
 type TrainingDataLength = Int
 
+-- | Initializes a new network with random values for weights and biases
+-- in all layers.
+--
+-- > net <- newNetwork [2, 3, 4]
 newNetwork :: [Int] -> IO (Network Double)
 newNetwork layerSizes
     | length layerSizes < 2 = error "Network too small!"
@@ -83,6 +139,8 @@ newNetwork layerSizes
             let bs = randomVector seed Gaussian outputSize
             return $ Layer ws bs
 
+-- | Calculate the output of the network based on the network, a given
+-- 'ActivationFunction' and the input vector.
 output :: (Numeric a, Num (Vector a))
        => Network a
        -> ActivationFunction a
@@ -91,14 +149,6 @@ output :: (Numeric a, Num (Vector a))
 output net act input = foldl f input (layers net)
     where f vec layer = cmap act ((weights layer #> vec) + biases layer)
 
-outputs :: (Numeric a, Num (Vector a))
-        => Network a
-        -> ActivationFunction a
-        -> Vector a
-        -> [Vector a]
-outputs net act input = scanl f input (layers net)
-    where f vec layer = cmap act ((weights layer #> vec) + biases layer)
-
 rawOutputs :: (Numeric a, Num (Vector a))
            => Network a
            -> ActivationFunction a
@@ -129,6 +179,8 @@ trainShuffled epochs debug net costFunction lambda trainSamples miniBatchSize et
         (trainShuffled (epochs - 1) debug net' costFunction lambda trainSamples miniBatchSize eta)
 
 
+-- | Pure version of 'trainShuffled', training the network /n/ times without
+-- shuffling the training set, resulting in slightly worse results.
 trainNTimes :: Int
             -> (Network Double -> Int -> String)
             -> Network Double
@@ -231,9 +283,11 @@ backprop net costFunction spl = finalNablas
               in (Layer { weights = nablaW, biases = nablaB } : nablas, delta)
 
 
+-- | The sigmoid function
 sigmoid :: Floating a => ActivationFunction a
 sigmoid x = 1 / (1 + exp (-x))
 
+-- | The derivative of the sigmoid function.
 sigmoid' :: Floating a => ActivationFunctionDerivative a
 sigmoid' x = sigmoid x * (1 - sigmoid x)
 
@@ -251,6 +305,9 @@ shuffle xs = do
     newArr :: Int -> [a] -> IO (IOArray Int a)
     newArr len lst = newListArray (1,len) lst
 
+-- | Saves the network as the given filename. When the file already exists,
+-- it looks for another filename by increasing the version, e.g
+-- /mnist.net/ becomes /mnist1.net/.
 saveNetwork :: (Element a, Binary a) => FilePath -> Network a -> IO ()
 saveNetwork fp net = do
     ex <- doesFileExist fp
@@ -265,5 +322,6 @@ newFileName fp = case fp =~ "(.+[a-z]){0,1}([0-9]*)(\\..*)" :: [[String]] of
   where version :: String -> Int
         version xs = fromMaybe 0 (readMaybe xs :: Maybe Int)
 
+-- | Load the network with the given filename.
 loadNetwork :: (Element a, Binary a) => FilePath -> IO (Network a)
 loadNetwork = decodeFile

docs/doc-index.html

@@ -1,4 +1,4 @@
 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title> (Index)</title><link href="ocean.css" rel="stylesheet" type="text/css" title="Ocean" /><script src="haddock-util.js" type="text/javascript"></script><script type="text/javascript">//<![CDATA[
 window.onload = function () {pageLoad();};
 //]]>
-</script></head><body><div id="package-header"><ul class="links" id="page-menu"><li><a href="index.html">Contents</a></li><li><a href="doc-index.html">Index</a></li></ul><p class="caption empty">&nbsp;</p></div><div id="content"><div id="index"><p class="caption">Index</p><table><tr><td class="src">--&gt;</td><td class="module"><a href="Network.html#v:-45--45--62-">Network</a></td></tr><tr><td class="src">ActivationFunction</td><td class="module"><a href="Network.html#t:ActivationFunction">Network</a></td></tr><tr><td class="src">ActivationFunctionDerivative</td><td class="module"><a href="Network.html#t:ActivationFunctionDerivative">Network</a></td></tr><tr><td class="src">backprop</td><td class="module"><a href="Network.html#v:backprop">Network</a></td></tr><tr><td class="src">biases</td><td class="module"><a href="Network.html#v:biases">Network</a></td></tr><tr><td class="src">CostFunction</td><td class="module"><a href="Network.html#t:CostFunction">Network</a></td></tr><tr><td class="src">CrossEntropyCost</td><td class="module"><a href="Network.html#v:CrossEntropyCost">Network</a></td></tr><tr><td class="src">getDelta</td><td class="module"><a href="Network.html#v:getDelta">Network</a></td></tr><tr><td class="src">Lambda</td><td class="module"><a href="Network.html#t:Lambda">Network</a></td></tr><tr><td class="src">Layer</td><td>&nbsp;</td></tr><tr><td class="alt">1 (Type/Class)</td><td class="module"><a href="Network.html#t:Layer">Network</a></td></tr><tr><td class="alt">2 (Data Constructor)</td><td class="module"><a href="Network.html#v:Layer">Network</a></td></tr><tr><td class="src">layers</td><td class="module"><a href="Network.html#v:layers">Network</a></td></tr><tr><td class="src">LearningRate</td><td class="module"><a href="Network.html#t:LearningRate">Network</a></td></tr><tr><td class="src">loadNetwork</td><td class="module"><a href="Network.html#v:loadNetwork">Network</a></td></tr><tr><td class="src">Network</td><td>&nbsp;</td></tr><tr><td class="alt">1 (Type/Class)</td><td class="module"><a href="Network.html#t:Network">Network</a></td></tr><tr><td class="alt">2 (Data Constructor)</td><td class="module"><a href="Network.html#v:Network">Network</a></td></tr><tr><td class="src">newFileName</td><td class="module"><a href="Network.html#v:newFileName">Network</a></td></tr><tr><td class="src">newNetwork</td><td class="module"><a href="Network.html#v:newNetwork">Network</a></td></tr><tr><td class="src">output</td><td class="module"><a href="Network.html#v:output">Network</a></td></tr><tr><td class="src">outputs</td><td class="module"><a href="Network.html#v:outputs">Network</a></td></tr><tr><td class="src">QuadraticCost</td><td class="module"><a href="Network.html#v:QuadraticCost">Network</a></td></tr><tr><td class="src">rawOutputs</td><td class="module"><a href="Network.html#v:rawOutputs">Network</a></td></tr><tr><td class="src">Sample</td><td class="module"><a href="Network.html#t:Sample">Network</a></td></tr><tr><td class="src">Samples</td><td class="module"><a href="Network.html#t:Samples">Network</a></td></tr><tr><td class="src">saveNetwork</td><td class="module"><a href="Network.html#v:saveNetwork">Network</a></td></tr><tr><td class="src">shuffle</td><td class="module"><a href="Network.html#v:shuffle">Network</a></td></tr><tr><td class="src">sigmoid</td><td class="module"><a href="Network.html#v:sigmoid">Network</a></td></tr><tr><td class="src">sigmoid'</td><td class="module"><a href="Network.html#v:sigmoid-39-">Network</a></td></tr><tr><td class="src">TrainingDataLength</td><td class="module"><a href="Network.html#t:TrainingDataLength">Network</a></td></tr><tr><td class="src">trainNTimes</td><td class="module"><a href="Network.html#v:trainNTimes">Network</a></td></tr><tr><td class="src">trainSGD</td><td class="module"><a href="Network.html#v:trainSGD">Network</a></td></tr><tr><td class="src">trainShuffled</td><td class="module"><a href="Network.html#v:trainShuffled">Network</a></td></tr><tr><td class="src">update</td><td class="module"><a href="Network.html#v:update">Network</a></td></tr><tr><td class="src">weights</td><td class="module"><a href="Network.html#v:weights">Network</a></td></tr></table></div></div><div id="footer"><p>Produced by <a href="http://www.haskell.org/haddock/">Haddock</a> version 2.16.1</p></div></body></html>
+</script></head><body><div id="package-header"><ul class="links" id="page-menu"><li><a href="index.html">Contents</a></li><li><a href="doc-index.html">Index</a></li></ul><p class="caption empty">&nbsp;</p></div><div id="content"><div id="index"><p class="caption">Index</p><table><tr><td class="src">--&gt;</td><td class="module"><a href="Network.html#v:-45--45--62-">Network</a></td></tr><tr><td class="src">ActivationFunction</td><td class="module"><a href="Network.html#t:ActivationFunction">Network</a></td></tr><tr><td class="src">ActivationFunctionDerivative</td><td class="module"><a href="Network.html#t:ActivationFunctionDerivative">Network</a></td></tr><tr><td class="src">biases</td><td class="module"><a href="Network.html#v:biases">Network</a></td></tr><tr><td class="src">CostFunction</td><td class="module"><a href="Network.html#t:CostFunction">Network</a></td></tr><tr><td class="src">CrossEntropyCost</td><td class="module"><a href="Network.html#v:CrossEntropyCost">Network</a></td></tr><tr><td class="src">getDelta</td><td class="module"><a href="Network.html#v:getDelta">Network</a></td></tr><tr><td class="src">Lambda</td><td class="module"><a href="Network.html#t:Lambda">Network</a></td></tr><tr><td class="src">Layer</td><td>&nbsp;</td></tr><tr><td class="alt">1 (Type/Class)</td><td class="module"><a href="Network.html#t:Layer">Network</a></td></tr><tr><td class="alt">2 (Data Constructor)</td><td class="module"><a href="Network.html#v:Layer">Network</a></td></tr><tr><td class="src">layers</td><td class="module"><a href="Network.html#v:layers">Network</a></td></tr><tr><td class="src">LearningRate</td><td class="module"><a href="Network.html#t:LearningRate">Network</a></td></tr><tr><td class="src">loadNetwork</td><td class="module"><a href="Network.html#v:loadNetwork">Network</a></td></tr><tr><td class="src">Network</td><td>&nbsp;</td></tr><tr><td class="alt">1 (Type/Class)</td><td class="module"><a href="Network.html#t:Network">Network</a></td></tr><tr><td class="alt">2 (Data Constructor)</td><td class="module"><a href="Network.html#v:Network">Network</a></td></tr><tr><td class="src">newNetwork</td><td class="module"><a href="Network.html#v:newNetwork">Network</a></td></tr><tr><td class="src">output</td><td class="module"><a href="Network.html#v:output">Network</a></td></tr><tr><td class="src">QuadraticCost</td><td class="module"><a href="Network.html#v:QuadraticCost">Network</a></td></tr><tr><td class="src">Sample</td><td class="module"><a href="Network.html#t:Sample">Network</a></td></tr><tr><td class="src">Samples</td><td class="module"><a href="Network.html#t:Samples">Network</a></td></tr><tr><td class="src">saveNetwork</td><td class="module"><a href="Network.html#v:saveNetwork">Network</a></td></tr><tr><td class="src">sigmoid</td><td class="module"><a href="Network.html#v:sigmoid">Network</a></td></tr><tr><td class="src">sigmoid'</td><td class="module"><a href="Network.html#v:sigmoid-39-">Network</a></td></tr><tr><td class="src">TrainingDataLength</td><td class="module"><a href="Network.html#t:TrainingDataLength">Network</a></td></tr><tr><td class="src">trainNTimes</td><td class="module"><a href="Network.html#v:trainNTimes">Network</a></td></tr><tr><td class="src">trainShuffled</td><td class="module"><a href="Network.html#v:trainShuffled">Network</a></td></tr><tr><td class="src">weights</td><td class="module"><a href="Network.html#v:weights">Network</a></td></tr></table></div></div><div id="footer"><p>Produced by <a href="http://www.haskell.org/haddock/">Haddock</a> version 2.16.1</p></div></body></html>

docs/mini_Network.html

@@ -1,4 +1,4 @@
 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Network</title><link href="ocean.css" rel="stylesheet" type="text/css" title="Ocean" /><script src="haddock-util.js" type="text/javascript"></script><script type="text/javascript">//<![CDATA[
 window.onload = function () {pageLoad();};
 //]]>
-</script></head><body id="mini"><div id="module-header"><p class="caption">Network</p></div><div id="interface"><div class="top"><p class="src"><span class="keyword">data</span> <a href="Network.html#t:Network" target="main">Network</a> a</p></div><div class="top"><p class="src"><span class="keyword">data</span> <a href="Network.html#t:Layer" target="main">Layer</a> a</p></div><div class="top"><p class="src"><span class="keyword">data</span> <a href="Network.html#t:CostFunction" target="main">CostFunction</a></p></div><div class="top"><p class="src"><a href="Network.html#v:getDelta" target="main">getDelta</a></p></div><div class="top"><p class="src"><span class="keyword">type</span> <a href="Network.html#t:ActivationFunction" target="main">ActivationFunction</a> a</p></div><div class="top"><p class="src"><span class="keyword">type</span> <a href="Network.html#t:ActivationFunctionDerivative" target="main">ActivationFunctionDerivative</a> a</p></div><div class="top"><p class="src"><span class="keyword">type</span> <a href="Network.html#t:Sample" target="main">Sample</a> a</p></div><div class="top"><p class="src"><span class="keyword">type</span> <a href="Network.html#t:Samples" target="main">Samples</a> a</p></div><div class="top"><p class="src"><a href="Network.html#v:-45--45--62-" target="main">(--&gt;)</a></p></div><div class="top"><p class="src"><span class="keyword">type</span> <a href="Network.html#t:LearningRate" target="main">LearningRate</a></p></div><div class="top"><p class="src"><span class="keyword">type</span> <a href="Network.html#t:Lambda" target="main">Lambda</a></p></div><div class="top"><p class="src"><span class="keyword">type</span> <a href="Network.html#t:TrainingDataLength" target="main">TrainingDataLength</a></p></div><div class="top"><p class="src"><a href="Network.html#v:newNetwork" target="main">newNetwork</a></p></div><div class="top"><p class="src"><a href="Network.html#v:output" target="main">output</a></p></div><div class="top"><p class="src"><a href="Network.html#v:outputs" target="main">outputs</a></p></div><div class="top"><p class="src"><a href="Network.html#v:rawOutputs" target="main">rawOutputs</a></p></div><div class="top"><p class="src"><a href="Network.html#v:trainShuffled" target="main">trainShuffled</a></p></div><div class="top"><p class="src"><a href="Network.html#v:trainNTimes" target="main">trainNTimes</a></p></div><div class="top"><p class="src"><a href="Network.html#v:trainSGD" target="main">trainSGD</a></p></div><div class="top"><p class="src"><a href="Network.html#v:update" target="main">update</a></p></div><div class="top"><p class="src"><a href="Network.html#v:backprop" target="main">backprop</a></p></div><div class="top"><p class="src"><a href="Network.html#v:sigmoid" target="main">sigmoid</a></p></div><div class="top"><p class="src"><a href="Network.html#v:sigmoid-39-" target="main">sigmoid'</a></p></div><div class="top"><p class="src"><a href="Network.html#v:shuffle" target="main">shuffle</a></p></div><div class="top"><p class="src"><a href="Network.html#v:saveNetwork" target="main">saveNetwork</a></p></div><div class="top"><p class="src"><a href="Network.html#v:newFileName" target="main">newFileName</a></p></div><div class="top"><p class="src"><a href="Network.html#v:loadNetwork" target="main">loadNetwork</a></p></div></div></body></html>
+</script></head><body id="mini"><div id="module-header"><p class="caption">Network</p></div><div id="interface"><h1>Network</h1><div class="top"><p class="src"><span class="keyword">data</span> <a href="Network.html#t:Network" target="main">Network</a> a</p></div><div class="top"><p class="src"><span class="keyword">data</span> <a href="Network.html#t:Layer" target="main">Layer</a> a</p></div><div class="top"><p class="src"><a href="Network.html#v:newNetwork" target="main">newNetwork</a></p></div><div class="top"><p class="src"><a href="Network.html#v:output" target="main">output</a></p></div><h1>Learning functions</h1><div class="top"><p class="src"><a href="Network.html#v:trainShuffled" target="main">trainShuffled</a></p></div><div class="top"><p class="src"><a href="Network.html#v:trainNTimes" target="main">trainNTimes</a></p></div><div class="top"><p class="src"><span class="keyword">data</span> <a href="Network.html#t:CostFunction" target="main">CostFunction</a></p></div><div class="top"><p class="src"><a href="Network.html#v:getDelta" target="main">getDelta</a></p></div><div class="top"><p class="src"><span class="keyword">type</span> <a href="Network.html#t:LearningRate" target="main">LearningRate</a></p></div><div class="top"><p class="src"><span class="keyword">type</span> <a href="Network.html#t:Lambda" target="main">Lambda</a></p></div><div class="top"><p class="src"><span class="keyword">type</span> <a href="Network.html#t:TrainingDataLength" target="main">TrainingDataLength</a></p></div><div class="top"><p class="src"><span class="keyword">type</span> <a href="Network.html#t:Sample" target="main">Sample</a> a</p></div><div class="top"><p class="src"><span class="keyword">type</span> <a href="Network.html#t:Samples" target="main">Samples</a> a</p></div><div class="top"><p class="src"><a href="Network.html#v:-45--45--62-" target="main">(--&gt;)</a></p></div><h1>Activation functions</h1><div class="top"><p class="src"><span class="keyword">type</span> <a href="Network.html#t:ActivationFunction" target="main">ActivationFunction</a> a</p></div><div class="top"><p class="src"><span class="keyword">type</span> <a href="Network.html#t:ActivationFunctionDerivative" target="main">ActivationFunctionDerivative</a> a</p></div><div class="top"><p class="src"><a href="Network.html#v:sigmoid" target="main">sigmoid</a></p></div><div class="top"><p class="src"><a href="Network.html#v:sigmoid-39-" target="main">sigmoid'</a></p></div><h1>Network serialization</h1><div class="top"><p class="src"><a href="Network.html#v:saveNetwork" target="main">saveNetwork</a></p></div><div class="top"><p class="src"><a href="Network.html#v:loadNetwork" target="main">loadNetwork</a></p></div></div></body></html>