title: Number and character recognition using ANN OpenCV 3.1
body: <p>I have implemented Neural network using OpenCV ANN Library. I am newbie in this field and I learn everything about it online (Mostly StackOverflow). </p>
  
  <p>I am using this ANN for detection of number plate. I did segmentation part using OpenCV image processing library and it is working good. It performs character segmentation and gives it to the NN part of the project. NN is going to recognize the number plate.  </p>
  
  <p>I have sample images of 20x30, therefore I have 600 neurons in input layer. As there are 36 possibilities (0-9,A-Z) I have 36 output neurons. I kept 100 neurons in hidden layer. The predict function of OpenCV is giving me the same output for every segmented image. That output is also showing some large negative(&lt; -1). I have used cv::ml::ANN_MLP::SIGMOID_SYM as an activation function. 
  Please don't mind as there is lot of code wrongly commented (I am doing trial and error).
  I need to find out what is the output of predict function. Thank you for your help.</p>
  
  <pre><code>#include &lt;opencv2/opencv.hpp&gt;
  
  int inputLayerSize = 1;
  int outputLayerSize = 1;
  int numSamples = 2;
  Mat layers = Mat(3, 1, CV_32S);
  layers.row(0) =Scalar(600) ;
  layers.row(1) = Scalar(20);
  layers.row(2) = Scalar(36);
  vector&lt;int&gt; layerSizes = { 600,100,36 };
  Ptr&lt;ml::ANN_MLP&gt; nnPtr = ml::ANN_MLP::create();
  vector &lt;int&gt; n;
  
  //nnPtr-&gt;setLayerSizes(3);
  nnPtr-&gt;setLayerSizes(layers);
  nnPtr-&gt;setTrainMethod(ml::ANN_MLP::BACKPROP);
  
  nnPtr-&gt;setTermCriteria(TermCriteria(cv::TermCriteria::COUNT | cv::TermCriteria::EPS, 1000, 0.00001f));
  
  nnPtr-&gt;setActivationFunction(cv::ml::ANN_MLP::SIGMOID_SYM, 1, 1);
      nnPtr-&gt;setBackpropWeightScale(0.5f);
      nnPtr-&gt;setBackpropMomentumScale(0.5f);
  
      /*CvANN_MLP_TrainParams params = CvANN_MLP_TrainParams(
          // terminate the training after either 1000
          // iterations or a very small change in the
          // network wieghts below the specified value
          cvTermCriteria(CV_TERMCRIT_ITER + CV_TERMCRIT_EPS, 1000, 0.000001),
  
          // use backpropogation for training
          CvANN_MLP_TrainParams::BACKPROP,
  
          // co-efficents for backpropogation training
          // (refer to manual)
          0.1,
          0.1);*/
          /*  Mat samples(Size(inputLayerSize, numSamples), CV_32F);
              samples.at&lt;float&gt;(Point(0, 0)) = 0.1f;
              samples.at&lt;float&gt;(Point(0, 1)) = 0.2f;
              Mat responses(Size(outputLayerSize, numSamples), CV_32F);
              responses.at&lt;float&gt;(Point(0, 0)) = 0.2f;
              responses.at&lt;float&gt;(Point(0, 1)) = 0.4f;
              */
              //reading chaos image
                   // we will read the classification numbers into this variable as though it is a vector
                                                            // close the traning images file
              /*vector&lt;int&gt; layerInfo;
              layerInfo=nnPtr-&gt;get;
              for (int i = 0; i &lt; layerInfo.size(); i++) {
                  cout &lt;&lt; "size of 0" &lt;&lt;layerInfo[i] &lt;&lt; endl;
              }*/
  cv::imshow("chaos", matTrainingImagesAsFlattenedFloats);
  
      //  cout &lt;&lt;abc &lt;&lt; endl;
  
  
  matTrainingImagesAsFlattenedFloats.convertTo(matTrainingImagesAsFlattenedFloats, CV_32F);
  //matClassificationInts.reshape(1, 496);
  matClassificationInts.convertTo(matClassificationInts, CV_32F);
  matSamples.convertTo(matSamples, CV_32F);
  std::cout &lt;&lt; matClassificationInts.rows &lt;&lt; " " &lt;&lt; matClassificationInts.cols &lt;&lt; " ";
  std::cout &lt;&lt; matTrainingImagesAsFlattenedFloats.rows &lt;&lt; " " &lt;&lt; matTrainingImagesAsFlattenedFloats.cols &lt;&lt; " ";
  std::cout &lt;&lt; matSamples.rows &lt;&lt; " " &lt;&lt; matSamples.cols;
  imshow("Samples", matSamples);
  imshow("chaos", matTrainingImagesAsFlattenedFloats);
  Ptr&lt;ml::TrainData&gt; trainData = ml::TrainData::create(matTrainingImagesAsFlattenedFloats, ml::SampleTypes::ROW_SAMPLE, matSamples);
  nnPtr-&gt;train(trainData);
  bool m = nnPtr-&gt;isTrained();
  if (m)
      std::cout &lt;&lt; "training complete\n\n";
  //  cv::Mat matCurrentChar = Mat(cv::Size(matTrainingImagesAsFlattenedFloats.cols, matTrainingImagesAsFlattenedFloats.rows), CV_32F);
  //  cout &lt;&lt; "samples:\n" &lt;&lt; samples &lt;&lt; endl;
      //cout &lt;&lt; "\nresponses:\n" &lt;&lt; responses &lt;&lt; endl;
  
  /*  if (!nnPtr-&gt;train(trainData))
          return 1;*/
          /*  cout &lt;&lt; "\nweights[0]:\n" &lt;&lt; nnPtr-&gt;getWeights(0) &lt;&lt; endl;
              cout &lt;&lt; "\nweights[1]:\n" &lt;&lt; nnPtr-&gt;getWeights(1) &lt;&lt; endl;
              cout &lt;&lt; "\nweights[2]:\n" &lt;&lt; nnPtr-&gt;getWeights(2) &lt;&lt; endl;
              cout &lt;&lt; "\nweights[3]:\n" &lt;&lt; nnPtr-&gt;getWeights(3) &lt;&lt; endl;*/
              //predicting
  
  std::vector &lt;cv::String&gt; filename;
  cv::String folder = "./plate/";
  cv::glob(folder, filename);
  
  if (filename.empty()) {                                // if unable to open image
      std::cout &lt;&lt; "error: image not read from file\n\n";         // show error message on command line
      return(0);                                                  // and exit program
  }
  String strFinalString;
  for (int i = 0; i &lt; filename.size(); i++) {
      cv::Mat matTestingNumbers = cv::imread(filename[i]);
      cv::Mat matGrayscale;           //
      cv::Mat matBlurred;             // declare more image variables
      cv::Mat matThresh;              //
      cv::Mat matThreshCopy;
      cv::Mat matCanny;
      //
  
      cv::cvtColor(matTestingNumbers, matGrayscale, CV_BGR2GRAY);         // convert to grayscale
      matThresh = cv::Mat(cv::Size(matGrayscale.cols, matGrayscale.rows), CV_8UC1);
      for (int i = 0; i &lt; matGrayscale.cols; i++) {
  
          for (int j = 0; j &lt; matGrayscale.rows; j++) {
              if (matGrayscale.at&lt;uchar&gt;(j, i) &lt;= 130) {
  
                  matThresh.at&lt;uchar&gt;(j, i) = 255;
              }
              else {
  
                  matThresh.at&lt;uchar&gt;(j, i) = 0;
              }
  
          }
      }
      // blur
      cv::GaussianBlur(matThresh,              // input image
          matBlurred,                // output image
          cv::Size(5, 5),            // smoothing window width and height in pixels
          0);                        // sigma value, determines how much the image will be blurred, zero makes function choose the sigma value
                                     // filter image from grayscale to black and white
                                     /*   cv::adaptiveThreshold(matBlurred,                           // input image
                                     matThresh,                            // output image
                                     255,                                  // make pixels that pass the threshold full white
                                     cv::ADAPTIVE_THRESH_GAUSSIAN_C,       // use gaussian rather than mean, seems to give better results
                                     cv::THRESH_BINARY_INV,                // invert so foreground will be white, background will be black
                                     11,                                   // size of a pixel neighborhood used to calculate threshold value
                                     2);   */                                // constant subtracted from the mean or weighted mean
                                     //   cv::imshow("thresh" + std::to_string(i), matThresh);
      matThreshCopy = matThresh.clone();
      std::vector&lt;std::vector&lt;cv::Point&gt; &gt; ptContours;        // declare a vector for the contours
      std::vector&lt;cv::Vec4i&gt; v4iHierarchy;// make a copy of the thresh image, this in necessary b/c findContours modifies the image
      cv::Canny(matBlurred, matCanny, 20, 40, 3);
  
  
  
  
  
      /*std::vector&lt;std::vector&lt;cv::Point&gt; &gt; ptContours;        // declare a vector for the contours
      std::vector&lt;cv::Vec4i&gt; v4iHierarchy;                    // declare a vector for the hierarchy (we won't use this in this program but this may be helpful for reference)
  
      cv::findContours(matThreshCopy,             // input image, make sure to use a copy since the function will modify this image in the course of finding contours
      ptContours,                             // output contours
      v4iHierarchy,                           // output hierarchy
      cv::RETR_EXTERNAL,                      // retrieve the outermost contours only
      cv::CHAIN_APPROX_SIMPLE);               // compress horizontal, vertical, and diagonal segments and leave only their end points
  
      /*std::vector&lt;std::vector&lt;cv::Point&gt; &gt; contours_poly(ptContours.size());
      std::vector&lt;cv::Rect&gt; boundRect(ptContours.size());
      for (int i = 0; i &lt; ptContours.size(); i++)
      {
      approxPolyDP(cv::Mat(ptContours[i]), contours_poly[i], 3, true);
      boundRect[i] = cv::boundingRect(cv::Mat(contours_poly[i]));
      }*/
      /*for (int i = 0; i &lt; ptContours.size(); i++) {               // for each contour
      ContourWithData contourWithData;                                                    // instantiate a contour with data object
      contourWithData.ptContour = ptContours[i];                                          // assign contour to contour with data
      contourWithData.boundingRect = cv::boundingRect(contourWithData.ptContour);         // get the bounding rect
      contourWithData.fltArea = cv::contourArea(contourWithData.ptContour);               // calculate the contour area
      allContoursWithData.push_back(contourWithData);                                     // add contour with data object to list of all contours with data
      }
  
      for (int i = 0; i &lt; allContoursWithData.size(); i++) {                      // for all contours
      if (allContoursWithData[i].checkIfContourIsValid()) {                   // check if valid
      validContoursWithData.push_back(allContoursWithData[i]);            // if so, append to valid contour list
      }
      }
      //sort contours from left to right
      std::sort(validContoursWithData.begin(), validContoursWithData.end(), ContourWithData::sortByBoundingRectXPosition);
  
      //  std::string strFinalString;         // declare final string, this will have the final number sequence by the end of the program
      */
      /*for (int i = 0; i &lt; validContoursWithData.size(); i++) {            // for each contour
  
      // draw a green rect around the current char
      cv::rectangle(matTestingNumbers,                            // draw rectangle on original image
      validContoursWithData[i].boundingRect,        // rect to draw
      cv::Scalar(0, 255, 0),                        // green
      2);                                           // thickness
  
      cv::Mat matROI = matThresh(validContoursWithData[i].boundingRect);          // get ROI image of bounding rect
  
      cv::Mat matROIResized;
      cv::resize(matROI, matROIResized, cv::Size(RESIZED_IMAGE_WIDTH, RESIZED_IMAGE_HEIGHT));     // resize image, this will be more consistent for recognition and storage
      */
      cv::Mat matROIFloat;
      cv::resize(matThresh, matThresh, cv::Size(RESIZED_IMAGE_WIDTH, RESIZED_IMAGE_HEIGHT));
      matThresh.convertTo(matROIFloat, CV_32FC1, 1.0 / 255.0);             // convert Mat to float, necessary for call to find_nearest
  
      cv::Mat matROIFlattenedFloat = matROIFloat.reshape(1, 1);
      cv::Point maxLoc = { 0,0 };
      cv::Point minLoc;
      cv::Mat output = cv::Mat(cv::Size(36, 1), CV_32F);
      vector&lt;float&gt;output2;
      //  cv::Mat output2 = cv::Mat(cv::Size(36, 1), CV_32F);
      nnPtr-&gt;predict(matROIFlattenedFloat, output2);
  //  float max = output.at&lt;float&gt;(0, 0);
      int fo = 0;
      float m = output2[0];
      imshow("predicted input", matROIFlattenedFloat);
      //  float b = output.at&lt;float&gt;(0, 0);
      //  cout &lt;&lt;"\n output0,0:"&lt;&lt;b&lt;&lt;endl;
  //  minMaxLoc(output, 0, 0, &amp;minLoc, &amp;maxLoc, Mat());
      //  cout &lt;&lt; "\noutput:\n" &lt;&lt; maxLoc.x &lt;&lt; endl;
  
  
      for (int j = 1; j &lt; 36; j++) {
          float value =output2[j];
          if (value &gt; m) {
              m = value;
              fo = j;
          }
      }
      float * p = 0;
      p = &amp;m;
      cout &lt;&lt; "j value in output " &lt;&lt; fo &lt;&lt; " Max value " &lt;&lt; p &lt;&lt; endl;
      //imshow("output image" + to_string(i), output);
      //  cout &lt;&lt; "\noutput:\n" &lt;&lt; minLoc.x &lt;&lt; endl;
          //float fltCurrentChar = (float)maxLoc.x;
      output.release();
      m = 0;
      fo = 0;
  }
      //  strFinalString = strFinalString + char(int(fltCurrentChar)); // append current char to full string
  //      cv::imshow("Predict output", output);
  
  
  /*cv::Point maxLoc = {0,0};
  Mat output=Mat (cv::Size(matSamples.cols,matSamples.rows),CV_32F);
  nnPtr-&gt;predict(matTrainingImagesAsFlattenedFloats, output);
  minMaxLoc(output, 0, 0, 0, &amp;maxLoc, 0);
  cout &lt;&lt; "\noutput:\n" &lt;&lt; maxLoc.x &lt;&lt; endl;*/
  //  getchar();
  
  /*for (int i = 0; i &lt; 10;i++) {
      for (int j = 0; j &lt; 36; j++) {
          if (matCurrentChar.at&lt;float&gt;(i, j) &gt;= 0.6) {
              cout &lt;&lt; " "&lt;&lt;j&lt;&lt;" ";
          }
      }
  }*/
      waitKey(0);
      return(0);
  }
  
  
  void gen() {
  
  
  std::string dir, filepath;
  int num, imgArea, minArea;
  int pos = 0;
  bool f = true;
  struct stat filestat;
  cv::Mat imgTrainingNumbers;
  cv::Mat imgGrayscale;
  cv::Mat imgBlurred;
  cv::Mat imgThresh;
  cv::Mat imgThreshCopy;
  cv::Mat matROIResized=cv::Mat (cv::Size(RESIZED_IMAGE_WIDTH,RESIZED_IMAGE_HEIGHT),CV_8UC1);
  cv::Mat matROI;
  std::vector &lt;cv::String&gt; filename;
  std::vector&lt;std::vector&lt;cv::Point&gt; &gt; ptContours;
  std::vector&lt;cv::Vec4i&gt; v4iHierarchy;
  int count = 0, contoursCount = 0;
  matSamples = cv::Mat(cv::Size(36, 496), CV_32FC1);
  matTrainingImagesAsFlattenedFloats = cv::Mat(cv::Size(600, 496), CV_32FC1);
  
  for (int j = 0; j &lt;= 35; j++) {
  
      int tmp = j;
      cv::String folder = "./Training Data/" + std::to_string(tmp);
      cv::glob(folder, filename);
  
  
  
  
      for (int k = 0; k &lt; filename.size(); k++) {
          count++;
          // If the file is a directory (or is in some way invalid) we'll skip it 
          //  if (stat(filepath.c_str(), &amp;filestat)) continue;
          //if (S_ISDIR(filestat.st_mode))         continue;
          imgTrainingNumbers = cv::imread(filename[k]);
          imgArea = imgTrainingNumbers.cols*imgTrainingNumbers.rows;
          // read in training numbers image
          minArea = imgArea * 50 / 100;
          if (imgTrainingNumbers.empty()) {
              std::cout &lt;&lt; "error: image not read from file\n\n";
              //return(0);
          }
  
          cv::cvtColor(imgTrainingNumbers, imgGrayscale, CV_BGR2GRAY);
  
          //cv::equalizeHist(imgGrayscale, imgGrayscale);
  
          imgThresh = cv::Mat(cv::Size(imgGrayscale.cols, imgGrayscale.rows), CV_8UC1);
          /*cv::adaptiveThreshold(imgGrayscale,
          imgThresh,
          255,
          cv::ADAPTIVE_THRESH_GAUSSIAN_C,
          cv::THRESH_BINARY_INV,
          3,
          0);
          */
          for (int i = 0; i &lt; imgGrayscale.cols; i++) {
  
              for (int j = 0; j &lt; imgGrayscale.rows; j++) {
                  if (imgGrayscale.at&lt;uchar&gt;(j, i) &lt;= 130) {
  
                      imgThresh.at&lt;uchar&gt;(j, i) = 255;
                  }
                  else {
  
                      imgThresh.at&lt;uchar&gt;(j, i) = 0;
                  }
  
              }
          }
          //  cv::imshow("imgThresh"+std::to_string(count), imgThresh);         
  
          imgThreshCopy = imgThresh.clone();
  
          cv::GaussianBlur(imgThreshCopy,
              imgBlurred,
              cv::Size(5, 5),
              0);
          cv::Mat imgCanny;
          //  cv::Canny(imgBlurred,imgCanny,20,40,3);
          cv::findContours(imgBlurred,
              ptContours,
              v4iHierarchy,
              cv::RETR_EXTERNAL,
              cv::CHAIN_APPROX_SIMPLE);
  
  
  
  
  
          for (int i = 0; i &lt; ptContours.size(); i++) {
  
              if (cv::contourArea(ptContours[i]) &gt; MIN_CONTOUR_AREA) {
                  contoursCount++;
                  cv::Rect boundingRect = cv::boundingRect(ptContours[i]);
  
                  cv::rectangle(imgTrainingNumbers, boundingRect, cv::Scalar(0, 0, 255), 2);      // draw red rectangle around each contour as we ask user for input
  
                  matROI = imgThreshCopy(boundingRect);          // get ROI image of bounding rect
                  std::string path = "./" + std::to_string(contoursCount) + ".JPG";
                  cv::imwrite(path, matROI);
                  //  cv::imshow("matROI" + std::to_string(count), matROI);
                  cv::resize(matROI, matROIResized, cv::Size(RESIZED_IMAGE_WIDTH, RESIZED_IMAGE_HEIGHT));     // resize image, this will be more consistent for recognition and storage
                  std::cout &lt;&lt; filename[k] &lt;&lt; " " &lt;&lt; contoursCount &lt;&lt; "\n";
  
                  //cv::imshow("matROI", matROI);                              
                  //cv::imshow("matROIResized"+std::to_string(count), matROIResized); 
  
              //  cv::imshow("imgTrainingNumbers" + std::to_string(contoursCount), imgTrainingNumbers);
                  int intChar;
                  if (j&lt;10)
                      intChar = j + 48;
                  else {
                      intChar = j + 55;
                  }
                  /*if (intChar == 27) {        // if esc key was pressed
                  return(0);              // exit program
                  }*/
                  //   if (std::find(intValidChars.begin(), intValidChars.end(), intChar) != intValidChars.end()) {     // else if the char is in the list of chars we are looking for . . .
  
                  // append classification char to integer list of chars
  
                  cv::Mat matImageFloat; 
                  matROIResized.convertTo(matImageFloat,CV_32FC1);// now add the training image (some conversion is necessary first) . . .
                  //matROIResized.convertTo(matImageFloat, CV_32FC1);       // convert Mat to float
  
                  cv::Mat matImageFlattenedFloat = matImageFloat.reshape(1, 1); 
                  //matTrainingImagesAsFlattenedFloats.push_back(matImageFlattenedFloat);// flatten
                  try {
                      //matTrainingImagesAsFlattenedFloats.push_back(matImageFlattenedFloat);
                      std::cout &lt;&lt; matTrainingImagesAsFlattenedFloats.rows &lt;&lt; " " &lt;&lt; matTrainingImagesAsFlattenedFloats.cols;
                      //unsigned char* re;
                      int ii = 0; // Current column in training_mat
                      for (int i = 0; i&lt;matImageFloat.rows; i++) {
  
                          for (int j = 0; j &lt; matImageFloat.cols; j++) {
                              matTrainingImagesAsFlattenedFloats.at&lt;float&gt;(contoursCount-1, ii++) = matImageFloat.at&lt;float&gt;(i,j);
                          }
                      }
                  }
  
                  catch (std::exception &amp;exc) {
                      f = false;
                      exc.what();
                  }
                  if (f) {
                      matClassificationInts.push_back((float)intChar);
  
  
                          matSamples.at&lt;float&gt;(contoursCount-1, j) = 1.0;
  
                  }
                  f = true;
                  // add to Mat as though it was a vector, this is necessary due to the
                  // data types that KNearest.train accepts
              }   // end if
                  //}   // end if
          }   // end for
      }//end i
  }//end j
  </code></pre>
  
  <p>}</p>
  
  <p><a href="http://i.stack.imgur.com/GWRVt.jpg" rel="nofollow">Output of predict function</a></p>