libneuron/src/neuron.hxx

/*! @file

    @id $Id$
*/
//       1         2         3         4         5         6         7         8
// 45678901234567890123456789012345678901234567890123456789012345678901234567890

#include <matrix.hxx>

/** @mainpage Neural Network with Hidden Layers

    @section neuro-intro Overview

    @subsection nature Natural Neural Network

    From <a href="https://en.wikipedia.org/wiki/Neuron">Wikipedia</a>:
    «A neuron is an electrically excitable cell that processes and
    transmits information through electrical and chemical
    signals. These signals between neurons occur via synapses,
    specialized connections with other cells. Neurons can connect to
    each other to form neural networks. Neurons are the core
    components of the brain and spinal cord of the central nervous
    system, and of the ganglia of the peripheral nervous system.» The
    neuron connects with dendrites to the world or to the axon of
    other neuirons. The neurites (dendrite or axon) transport
    electrical stimulation to the cell, which emits the signal to the
    dendrites if the activation reaches a certain level.

    @dot
    digraph g {
      rankdir=LR;
      ranksep=0.8;
      node [shape=hexagon];
      edge [arrowhead=none];
      subgraph clusterInput {
        label="sensors";
        color="white";
        node [shape=point];
        I0; I1; I2; I3; I4; I5; I6; I7; I8 I9;
      }
      subgraph clusterOutput {
        label="actors";
        color="white";
        node [shape=point];
        O0; O1; O2; O3; O4; O5; O6;
      }
      I1 -> Cell1 [label="axon";taillabel="synapse"];
      { I2; I3; I4; } -> Cell1;
      { I5; I6; I7; I8; } -> Cell2;
      { I4; I6; I9; I0; } -> Cell3;
      Cell1 -> Cell8 [label="axon / dendrite"];
      Cell1 -> { Cell2; Cell4; Cell5; }
      Cell2 -> { Cell4; Cell5; Cell6; Cell8; }
      Cell3 -> { Cell4; Cell6; Cell7; Cell8; }
      { Cell4; Cell5; Cell6 } -> { Cell7; Cell8; }
      Cell7 -> { O0; O1; O2 };
      Cell8 -> { O3; O4; O5; };
      Cell8 -> O6 [label="dendrite"];
    }
    @enddot
    
    @subsection art Artificial Neural Network

    A complex neural network can be imitiated as a vector @c I of @c i
    input values, a vector @c O of @c o output values and any number
    @c l of hidden layers, where each of them contains @c h
    neurons.

    A neural network with double precision is inistialized as:
    @code
    NeuroNet<double, i, o, l+1, h> net;
    @endcode

    @dot
    digraph g {
      rankdir=LR;
      ranksep=1.5;
      subgraph clusterInput {
        label="Input Layer";
        I1 [label=<I<SUB>1</SUB>>];
        I2 [label=<I<SUB>2</SUB>>];
        Ix [label=<I<SUB>…</SUB>>];
        Ii [label=<I<SUB>i</SUB>>];
      }
      subgraph clusterHidden1 {
        label="First Hidden Layer";
        H11 [label=<H<SUB>11</SUB>>];
        H12 [label=<H<SUB>12</SUB>>];
        H1x [label=<H<SUB>1…</SUB>>];
        H1h [label=<H<SUB>1h</SUB>>];
      }
      subgraph clusterHidden2 {
        label="Second Hidden Layer";
        H21 [label=<H<SUB>21</SUB>>];
        H22 [label=<H<SUB>22</SUB>>];
        H2x [label=<H<SUB>2…</SUB>>];
        H2h [label=<H<SUB>2h</SUB>>];
      }
      subgraph clusterHiddenx {
        label="More Hidden Layers";
        Hx1 [label=<H<SUB>…1</SUB>>];
        Hx2 [label=<H<SUB>…2</SUB>>];
        Hxx [label=<H<SUB>……</SUB>>];
        Hxh [label=<H<SUB>…h</SUB>>];
      }
      subgraph clusterHiddenl {
        label="Last Hidden Layer";
        Hl1 [label=<H<SUB>l1</SUB>>];
        Hl2 [label=<H<SUB>l2</SUB>>];
        Hlx [label=<H<SUB>l…</SUB>>];
        Hlh [label=<H<SUB>lh</SUB>>];
      }
      subgraph clusterOutput {
        label="Output Layer";
        O1 [label=<O<SUB>1</SUB>>];
        O2 [label=<O<SUB>2</SUB>>];
        Ox [label=<O<SUB>…</SUB>>];
        Oo [label=<O<SUB>o</SUB>>];
      }
      { I1; I2; Ix; Ii; }
      -> { H11; H12; H1x; H1h; }
      -> { H21; H22; H2x; H2h; }
      -> { Hx1; Hx2; Hxx; Hxh; }
      -> { Hl1; Hl2; Hlx; Hlh; }
      -> { O1; O2; Ox; Oo; }
    }
    @enddot

    @section neuro-forward Forward Propagation
    
    The connections between two layers can be modelled as a
    Matrix. Then Matrix H<sub>1</sub> contains the weights from @c I
    to the first hidden layer, @c H<sub>2</sub> from the first to the
    second, and so on, until @c H<sub>l+1</sub> contains the weights
    from layer @c l to the output @c O.
    
    The output vector is then calculated as:
    O = I × H<sub>1</sub> × H<sub>2</sub> × H<sub>…</sub> × H<sub>l+1</sub>

    @code
    const size_type i(4);
    const size_type o(2);
    NeuroNet<double, i, o> net;
    Matrix<1, i> input(1.0, 2.0, 0.0, -1.0);
    Matrix<1, o> output = net(input);
    @endcode
    
    @section neuro-backward Back Propagation

    @page biblio Bibliography

     - <a href="http://www.tornau.name/wp-content/uploads/2009/04/studiumsmaterialien/neuronale_netze_zusammefassung.pdf">Vorlesung Neuronale Netze - Zusammenfassung - Christoph Tornau</a>
     - <a href="http://www.neuronalesnetz.de/">Neuronale Netze — Eine Einführung</a>
     - <a href="http://alphard.ethz.ch/hafner/Vorles/Optim/ANN/Artificial%20Neural%20Network%20based%20Curve%20Prediction%20Documentation.pdf">Artificial Neural Network based Curve Prediction</a>
     - <a href="http://cs231n.github.io/convolutional-networks/">Convolutional Neural Networks (CNNs / ConvNets)</a>
     - <a href="https://www.tensorflow.org/versions/r0.9/tutorials/index.html">TensorFlow utorials</a>
     - <a href="http://alphard.ethz.ch/hafner/Vorles/Optim/ANN/Artificial%20Neural%20Network%20based%20Curve%20Prediction%20Documentation.pdf">Artificial Neural Network based Curve Prediction</a>

    */
template
    <typename TYPE,
    size_t INPUT_LAYERS,
    size_t OUTPUT_LAYERS,
    size_t HIDDEN_LAYERS = INPUT_LAYERS+OUTPUT_LAYERS,
    size_t HIDDEN_LAYER_SIZE = INPUT_LAYERS+OUTPUT_LAYERS>
class NeuroNet {
};