title: FANN XOR training
body: <p>I am developing a piece of software that uses FANN, the Fast Artificial Neural Network library.
  I have tried after numerous failed attempts at writing my own ANN code to compile a FANN sample program, here the C++ XOR approximation program. Here is the source.</p>
  
  <pre class="lang-cpp prettyprint-override"><code>#include "../include/floatfann.h"
  #include "../include/fann_cpp.h"
  
  
  #include &lt;ios&gt;
  #include &lt;iostream&gt;
  #include &lt;iomanip&gt;
  using std::cout;
  using std::cerr;
  using std::endl;
  using std::setw;
  using std::left;
  using std::right;
  using std::showpos;
  using std::noshowpos;
  
  
  // Callback function that simply prints the information to cout
  int print_callback(FANN::neural_net &amp;net, FANN::training_data &amp;train,
      unsigned int max_epochs, unsigned int epochs_between_reports,
      float desired_error, unsigned int epochs, void *user_data)
  {
      cout &lt;&lt; "Epochs     " &lt;&lt; setw(8) &lt;&lt; epochs &lt;&lt; ". "
           &lt;&lt; "Current Error: " &lt;&lt; left &lt;&lt; net.get_MSE() &lt;&lt; right &lt;&lt; endl;
      return 0;
  }
  
  // Test function that demonstrates usage of the fann C++ wrapper
  void xor_test()
  {
      cout &lt;&lt; endl &lt;&lt; "XOR test started." &lt;&lt; endl;
  
      const float learning_rate = 0.7f;
      const unsigned int num_layers = 3;
      const unsigned int num_input = 2;
      const unsigned int num_hidden = 3;
      const unsigned int num_output = 1;
      const float desired_error = 0.001f;
      const unsigned int max_iterations = 300000;
      const unsigned int iterations_between_reports = 10000;
  
      ////Make array for create_standard() workaround (prevent "FANN Error 11: Unable to allocate memory.")
      const unsigned int num_input_num_hidden_num_output__array[3] = {num_input, num_hidden, num_output};
      cout &lt;&lt; endl &lt;&lt; "Creating network." &lt;&lt; endl;
  
      FANN::neural_net net;
  //    cout&lt;&lt;"Debug 1"&lt;&lt;endl;
      //net.create_standard(num_layers, num_input, num_hidden, num_output);//doesn't work
      net.create_standard_array(num_layers, num_input_num_hidden_num_output__array);//this might work -- create_standard() workaround
  
      net.set_learning_rate(learning_rate);
  
      net.set_activation_steepness_hidden(1.0);
      net.set_activation_steepness_output(1.0);
  
      //Sample Code, changed below
      net.set_activation_function_hidden(FANN::SIGMOID_SYMMETRIC_STEPWISE);
      net.set_activation_function_output(FANN::SIGMOID_SYMMETRIC_STEPWISE);
      //changed above to sigmoid
      //net.set_activation_function_hidden(FANN::SIGMOID);
      //net.set_activation_function_output(FANN::SIGMOID);
  
      // Set additional properties such as the training algorithm
      //net.set_training_algorithm(FANN::TRAIN_QUICKPROP);
  
      // Output network type and parameters
      cout &lt;&lt; endl &lt;&lt; "Network Type                         :  ";
      switch (net.get_network_type())
      {
      case FANN::LAYER://only connected to next layer
          cout &lt;&lt; "LAYER" &lt;&lt; endl;
          break;
      case FANN::SHORTCUT://connected to all other layers
          cout &lt;&lt; "SHORTCUT" &lt;&lt; endl;
          break;
      default:
          cout &lt;&lt; "UNKNOWN" &lt;&lt; endl;
          break;
      }
      net.print_parameters();
  
      cout &lt;&lt; endl &lt;&lt; "Training network." &lt;&lt; endl;
  
      FANN::training_data data;
      if (data.read_train_from_file("xor.data"))
      {
          // Initialize and train the network with the data
          net.init_weights(data);
  
          cout &lt;&lt; "Max Epochs " &lt;&lt; setw(8) &lt;&lt; max_iterations &lt;&lt; ". "
              &lt;&lt; "Desired Error: " &lt;&lt; left &lt;&lt; desired_error &lt;&lt; right &lt;&lt; endl;
          net.set_callback(print_callback, NULL);
          net.train_on_data(data, max_iterations,
              iterations_between_reports, desired_error);
  
          cout &lt;&lt; endl &lt;&lt; "Testing network. (not really)" &lt;&lt; endl;
  
          //I don't really get this code --- the funny for loop. Whatever. I'll skip it.
                  for (unsigned int i = 0; i &lt; data.length_train_data(); ++i)
                  {
                      // Run the network on the test data
                      fann_type *calc_out = net.run(data.get_input()[i]);
  
                      cout &lt;&lt; "XOR test (" &lt;&lt; showpos &lt;&lt; data.get_input()[i][0] &lt;&lt; ", "
                           &lt;&lt; data.get_input()[i][1] &lt;&lt; ") -&gt; " &lt;&lt; *calc_out
                           &lt;&lt; ", should be " &lt;&lt; data.get_output()[i][0] &lt;&lt; ", "
                           &lt;&lt; "difference = " &lt;&lt; noshowpos
                           &lt;&lt; fann_abs(*calc_out - data.get_output()[i][0]) &lt;&lt; endl;
                  }
  
          cout &lt;&lt; endl &lt;&lt; "Saving network." &lt;&lt; endl;
  
          // Save the network in floating point and fixed point
          net.save("xor_float.net");
          unsigned int decimal_point = net.save_to_fixed("xor_fixed.net");
          data.save_train_to_fixed("xor_fixed.data", decimal_point);
  
          cout &lt;&lt; endl &lt;&lt; "XOR test completed." &lt;&lt; endl;
      }
  }
  
  /* Startup function. Synchronizes C and C++ output, calls the test function
     and reports any exceptions */
  int main(int argc, char **argv)
  {
      try
      {
          std::ios::sync_with_stdio(); // Synchronize cout and printf output
          xor_test();
      }
      catch (...)
      {
          cerr &lt;&lt; endl &lt;&lt; "Abnormal exception." &lt;&lt; endl;
      }
      return 0;
  }
  </code></pre>
  
  <p>Here's my output.</p>
  
  <pre><code>XOR test started.
  
  Creating network.
  
  Network Type                         :  LAYER
  Input layer                          :   2 neurons, 1 bias
    Hidden layer                       :   3 neurons, 1 bias
  Output layer                         :   1 neurons
  Total neurons and biases             :   8
  Total connections                    :  13
  Connection rate                      :   1.000
  Network type                         :   FANN_NETTYPE_LAYER
  Training algorithm                   :   FANN_TRAIN_RPROP
  Training error function              :   FANN_ERRORFUNC_TANH
  Training stop function               :   FANN_STOPFUNC_MSE
  Bit fail limit                       :   0.350
  Learning rate                        :   0.700
  Learning momentum                    :   0.000
  Quickprop decay                      :  -0.000100
  Quickprop mu                         :   1.750
  RPROP increase factor                :   1.200
  RPROP decrease factor                :   0.500
  RPROP delta min                      :   0.000
  RPROP delta max                      :  50.000
  Cascade output change fraction       :   0.010000
  Cascade candidate change fraction    :   0.010000
  Cascade output stagnation epochs     :  12
  Cascade candidate stagnation epochs  :  12
  Cascade max output epochs            : 150
  Cascade min output epochs            :  50
  Cascade max candidate epochs         : 150
  Cascade min candidate epochs         :  50
  Cascade weight multiplier            :   0.400
  Cascade candidate limit              :1000.000
  Cascade activation functions[0]      :   FANN_SIGMOID
  Cascade activation functions[1]      :   FANN_SIGMOID_SYMMETRIC
  Cascade activation functions[2]      :   FANN_GAUSSIAN
  Cascade activation functions[3]      :   FANN_GAUSSIAN_SYMMETRIC
  Cascade activation functions[4]      :   FANN_ELLIOT
  Cascade activation functions[5]      :   FANN_ELLIOT_SYMMETRIC
  Cascade activation functions[6]      :   FANN_SIN_SYMMETRIC
  Cascade activation functions[7]      :   FANN_COS_SYMMETRIC
  Cascade activation functions[8]      :   FANN_SIN
  Cascade activation functions[9]      :   FANN_COS
  Cascade activation steepnesses[0]    :   0.250
  Cascade activation steepnesses[1]    :   0.500
  Cascade activation steepnesses[2]    :   0.750
  Cascade activation steepnesses[3]    :   1.000
  Cascade candidate groups             :   2
  Cascade no. of candidates            :  80
  
  Training network.
  Max Epochs   300000. Desired Error: 0.001
  Epochs            1. Current Error: 0.25
  Epochs        10000. Current Error: 0.25
  Epochs        20000. Current Error: 0.25
  Epochs        30000. Current Error: 0.25
  Epochs        40000. Current Error: 0.25
  Epochs        50000. Current Error: 0.25
  Epochs        60000. Current Error: 0.25
  Epochs        70000. Current Error: 0.25
  Epochs        80000. Current Error: 0.25
  Epochs        90000. Current Error: 0.25
  Epochs       100000. Current Error: 0.25
  Epochs       110000. Current Error: 0.25
  Epochs       120000. Current Error: 0.25
  Epochs       130000. Current Error: 0.25
  Epochs       140000. Current Error: 0.25
  Epochs       150000. Current Error: 0.25
  Epochs       160000. Current Error: 0.25
  Epochs       170000. Current Error: 0.25
  Epochs       180000. Current Error: 0.25
  Epochs       190000. Current Error: 0.25
  Epochs       200000. Current Error: 0.25
  Epochs       210000. Current Error: 0.25
  Epochs       220000. Current Error: 0.25
  Epochs       230000. Current Error: 0.25
  Epochs       240000. Current Error: 0.25
  Epochs       250000. Current Error: 0.25
  Epochs       260000. Current Error: 0.25
  Epochs       270000. Current Error: 0.25
  Epochs       280000. Current Error: 0.25
  Epochs       290000. Current Error: 0.25
  Epochs       300000. Current Error: 0.25
  
  Testing network. (not really)
  XOR test (+0, -1.875) -&gt; +0, should be +0, difference = -0
  XOR test (+0, -1.875) -&gt; +0, should be +0, difference = -0
  XOR test (+0, +1.875) -&gt; +0, should be +0, difference = -0
  XOR test (+0, +1.875) -&gt; +0, should be +0, difference = -0
  
  Saving network.
  
  XOR test completed.
  </code></pre>
  
  <p>The training data (<code>xor.data</code>) is here:</p>
  
  <pre><code>4 2 1    
  -1 -1    
  -1    
  -1 1    
  1    
  1 -1    
  1
  1 1    
  -1
  </code></pre>
  
  <p>What explains the eerie lack of learning in the ANN? I'm pretty convinced that I have something configured very wrong somewhere, especially given that this is the <em>sample program</em>. ANN experts, any advice?</p>