first try of propagation

2017-01-09 16:11:06 +00:00
parent c7cc3d1690
commit fcfe5871f1
5 changed files with 96 additions and 15 deletions
--- a/2
+++ b/2
@@ -1 +1 @@
-/usr/share/automake-1.15/COPYING
+/usr/share/automake-1.14/COPYING
--- a/13
+++ b/13
@@ -1,3 +1,16 @@
+2017-01-08 08:55  
+
+	* [r12] bootstrap.sh, debian/changelog.in:
+	  rebuilt debian/changelog.in
+
+2017-01-07 14:06  
+
+	* [r11] COPYING, ChangeLog, INSTALL, ax_init_standard_project.m4,
+	  bootstrap.sh, build-in-docker.conf, build-in-docker.sh,
+	  debian/control.in, doc/plantuml.jar[ADD],
+	  resolve-debbuilddeps.sh, resolve-rpmbuilddeps.sh:
+	  build system updated
+
 2016-10-30 20:16  

 	* [r10] configure.ac:
--- a/2
+++ b/2
@@ -1 +1 @@
-/usr/share/automake-1.15/INSTALL
+/usr/share/automake-1.14/INSTALL
--- a/src/neuron.hxx
+++ b/src/neuron.hxx
@@ -6,6 +6,7 @@
 // 45678901234567890123456789012345678901234567890123456789012345678901234567890

 #include <matrix.hxx>
+#include <cmath>

 /** @mainpage Neural Network with Hidden Layers

@@ -66,9 +67,9 @@
    @c l of hidden layers, where each of them contains @c h
    neurons.

-    A neural network with double precision is inistialized as:
+    A neural network with double precision is initialized as:
    @code
-    NeuroNet<double, i, o, l+1, h> net;
+    NeuroNet<double, i, o, l, h> net;
    @endcode

    @dot
@@ -134,34 +135,80 @@
    second, and so on, until @c H<sub>l+1</sub> contains the weights
    from layer @c l to the output @c O.

-    The output vector is then calculated as:
-    O = I × H<sub>1</sub> × H<sub>2</sub> × H<sub>…</sub> × H<sub>l+1</sub>
+    There is also an activation function @f. For back propagation,
+    this function needs a first derivation @c f'.
+
+    To get the activation of the first hidden layer, the input vector
+    is multiplied with the weight matrix of the first hidden layer,
+    this results in an output vector. Then the activation function is
+    applied to all values of the output vector:
+
+    <pre>
+    V<sub>1</sub> = f(I×H<sub>1</sub>)
+    </pre>
+    
+    This is done for all layers, up to the output. The output vector
+    is then calculated as:
+    
+    <pre>
+    O = f(f(f(f(I×H<sub>1</sub>)×H<sub>2</sub>)×H<sub>…</sub>)×H<sub>l+1</sub>)
+    </pre>

    @code
    const size_type i(4);
    const size_type o(2);
    NeuroNet<double, i, o> net;
    Matrix<1, i> input(1.0, 2.0, 0.0, -1.0);
-    Matrix<1, o> output = net(input);
+    Matrix<1, o> output = feed(input);
    @endcode
    
    @section neuro-backward Back Propagation

    @page biblio Bibliography

+     - <a href="http://briandolhansky.com/blog/2014/10/30/artificial-neural-networks-matrix-form-part-5">Artificial Neural Networks: Matrix Form (Part 5)</a>
     - <a href="http://www.tornau.name/wp-content/uploads/2009/04/studiumsmaterialien/neuronale_netze_zusammefassung.pdf">Vorlesung Neuronale Netze - Zusammenfassung - Christoph Tornau</a>
     - <a href="http://www.neuronalesnetz.de/">Neuronale Netze — Eine Einführung</a>
     - <a href="http://alphard.ethz.ch/hafner/Vorles/Optim/ANN/Artificial%20Neural%20Network%20based%20Curve%20Prediction%20Documentation.pdf">Artificial Neural Network based Curve Prediction</a>
     - <a href="http://cs231n.github.io/convolutional-networks/">Convolutional Neural Networks (CNNs / ConvNets)</a>
-     - <a href="https://www.tensorflow.org/versions/r0.9/tutorials/index.html">TensorFlow utorials</a>
+     - <a href="https://www.tensorflow.org/versions/r0.9/tutorials/index.html">TensorFlow Tutorials</a>
     - <a href="http://alphard.ethz.ch/hafner/Vorles/Optim/ANN/Artificial%20Neural%20Network%20based%20Curve%20Prediction%20Documentation.pdf">Artificial Neural Network based Curve Prediction</a>

    */
+
+namespace math {
+  // tangens hyperbolicus as standard activation function
+  template<typename TYPE> TYPE tanh(const TYPE& v) {
+    return ::tanh((double)v);
+  }
+  // derivate of activation function for back propagation
+  template<typename TYPE> TYPE tanh_diff(const TYPE& v) {
+    TYPE ch(::cosh((double)v));
+    return 1/(ch*ch);
+  }
+}
+
 template
    <typename TYPE,
    size_t INPUT_LAYERS,
    size_t OUTPUT_LAYERS,
    size_t HIDDEN_LAYERS = INPUT_LAYERS+OUTPUT_LAYERS,
-    size_t HIDDEN_LAYER_SIZE = INPUT_LAYERS+OUTPUT_LAYERS>
+    size_t HIDDEN_LAYER_SIZE = INPUT_LAYERS+OUTPUT_LAYERS,
+    TYPE(*ACTIVATION)(const TYPE&) = math::tanh<TYPE>,
+    TYPE(*ACTIVATION_DIFF)(const TYPE&)  = math::tanh_diff<TYPE>>
 class NeuroNet {
+  public:
+    NeuroNet() {
+    }
+    Matrix<TYPE, 1, OUTPUT_LAYERS> feed(Matrix<TYPE, 1, INPUT_LAYERS> in) {
+      Matrix<TYPE, 1, HIDDEN_LAYER_SIZE> l((in*_wi).apply(ACTIVATION));
+      for (int i(0); i<HIDDEN_LAYERS-1; ++i)
+        l = (l*_wh[i]).apply(ACTIVATION);
+      Matrix<TYPE, 1, OUTPUT_LAYERS> out((l*_wo).apply(ACTIVATION));
+      return out;
+    }
+  private:
+    Matrix<TYPE, INPUT_LAYERS, HIDDEN_LAYER_SIZE> _wi;
+    Matrix<TYPE, HIDDEN_LAYER_SIZE, HIDDEN_LAYER_SIZE> _wh[HIDDEN_LAYERS-1];
+    Matrix<TYPE, HIDDEN_LAYER_SIZE, OUTPUT_LAYERS> _wo;
 };
--- a/test/neuron.cxx
+++ b/test/neuron.cxx
@@ -8,7 +8,7 @@
 //       1         2         3         4         5         6         7         8
 // 45678901234567890123456789012345678901234567890123456789012345678901234567890

-
+#include <neuron.hxx>

 #include <cppunit/TestFixture.h>
 #include <cppunit/ui/text/TestRunner.h>
@@ -19,15 +19,36 @@

 /// @todo Rename DummyTest and DummyTest::dummy()
 /// @todo Write test cases
-class DummyTest: public CppUnit::TestFixture { 
+class NeuroNetTest: public CppUnit::TestFixture { 
  public:
-    void dummy() {
+    void simplexor() {
+      NeuroNet<float, 2, 1> neuronet;
+      Matrix<float, 1, 2> in[] = {{1, 1},
+                                  {1, -1},
+                                  {-1, 1},
+                                  {-1, -1}};
+      Matrix<float, 1, 1> out[] = {-1,
+                                   1,
+                                   1,
+                                   -1};
+      for (int i(0); i<sizeof(in)/sizeof(*in); ++i) {
+        std::cout<<in[i]<<" → "<<out[i]<<" ~ "
+                 <<neuronet.feed(in[i]).apply([](float&v){
+                     v = v<0 ? -1.0 : 1.0;
+                   })<<std::endl;
+        auto res(neuronet.feed(in[i])
+                 .apply([](float&v){
+                     std::cout<<"v="<<v<<std::endl;
+                     v = v<0 ? -1.0 : 1.0;
+                   }));
+        CPPUNIT_ASSERT_EQUAL(out[i], res);
      }
-    CPPUNIT_TEST_SUITE(DummyTest);
-    CPPUNIT_TEST(dummy);
+    }
+    CPPUNIT_TEST_SUITE(NeuroNetTest);
+    CPPUNIT_TEST(simplexor);
    CPPUNIT_TEST_SUITE_END();
 };
-CPPUNIT_TEST_SUITE_REGISTRATION(DummyTest);
+CPPUNIT_TEST_SUITE_REGISTRATION(NeuroNetTest);

 int main(int argc, char** argv) try {
  std::ofstream ofs((*argv+std::string(".xml")).c_str());