title: OpenCV stereo image pair correction...displaying the results
body: <p>I am attempting to use OpenCV to take stereo image pairs...ie a left and a right image of the same subject...and then correct them for rotation and translation without knowing any of the properties of the camera. Once the images are corrected I should be able to display them to the user.</p>
  
  <p>So far I have merged two demo programs from the OpenCV samples directory, badly for the moment...I will clean the code up and arrange it more nicely when I get it working...and it seems to be working, however when I attempt to display the results the program crashes with a debug error. In the command window it says "OpenCV Error: Assertion failed (scn ==1 &amp;&amp; (dcn == 3 || dcn == 4)) in unknown function in file ........\opencv\modules\imgproc\src\color.cpp, line 2453"</p>
  
  <p>Commenting out various parts of the code to display the results just results in different OpenCV Errors. Here's my code. If anyone can help I will love you forever.</p>
  
  <pre><code>#include "stdafx.h"
  #include "opencv2/highgui/highgui.hpp"
  #include "opencv2/calib3d/calib3d.hpp"
  #include "opencv2/imgproc/imgproc.hpp"
  #include "opencv2/features2d/features2d.hpp"
  
  #include &lt;iostream&gt;
  
  using namespace cv;
  using namespace std;
  
  void help(char** argv)
  {
      cout &lt;&lt; "\nThis program demonstrates keypoint finding and matching between 2 images using features2d framework.\n"
       &lt;&lt; "Example of usage:\n"
       &lt;&lt; argv[0] &lt;&lt; " [detectorType] [descriptorType] [image1] [image2] [ransacReprojThreshold]\n"
       &lt;&lt; "\n"
       &lt;&lt; "Matches are filtered using homography matrix if ransacReprojThreshold&gt;=0\n"
       &lt;&lt; "Example:\n"
       &lt;&lt; "./descriptor_extractor_matcher SURF SURF  cola1.jpg cola2.jpg 3\n"
       &lt;&lt; "\n"
       &lt;&lt; "Possible detectorType values: see in documentation on createFeatureDetector().\n"
       &lt;&lt; "Possible descriptorType values: see in documentation on createDescriptorExtractor().\n" &lt;&lt; endl;
  }
  
  const string winName = "rectified";
  
  void crossCheckMatching( Ptr&lt;DescriptorMatcher&gt;&amp; descriptorMatcher,
                           const Mat&amp; descriptors1, const Mat&amp; descriptors2,
                           vector&lt;DMatch&gt;&amp; filteredMatches12, int knn=1 )
  {
      filteredMatches12.clear();
      vector&lt;vector&lt;DMatch&gt; &gt; matches12, matches21;
      descriptorMatcher-&gt;knnMatch( descriptors1, descriptors2, matches12, knn );
      descriptorMatcher-&gt;knnMatch( descriptors2, descriptors1, matches21, knn );
      for( size_t m = 0; m &lt; matches12.size(); m++ )
      {
          bool findCrossCheck = false;
          for( size_t fk = 0; fk &lt; matches12[m].size(); fk++ )
          {
              DMatch forward = matches12[m][fk];
  
              for( size_t bk = 0; bk &lt; matches21[forward.trainIdx].size(); bk++ )
              {
                  DMatch backward = matches21[forward.trainIdx][bk];
                  if( backward.trainIdx == forward.queryIdx )
                  {
                      filteredMatches12.push_back(forward);
                      findCrossCheck = true;
                      break;
                  }
              }
              if( findCrossCheck ) break;
          }
      }
  }
  
  void doIteration( const Mat&amp; leftImg, Mat&amp; rightImg,
                    vector&lt;KeyPoint&gt;&amp; keypoints1, const Mat&amp; descriptors1,
                    Ptr&lt;FeatureDetector&gt;&amp; detector, Ptr&lt;DescriptorExtractor&gt;&amp; descriptorExtractor,
                    Ptr&lt;DescriptorMatcher&gt;&amp; descriptorMatcher,
                    double ransacReprojThreshold )
  {
      assert( !leftImg.empty() );
      Mat H12;
      assert( !rightImg.empty()/* &amp;&amp; rightImg.cols==leftImg.cols &amp;&amp; rightImg.rows==leftImg.rows*/ );
  
      cout &lt;&lt; endl &lt;&lt; "&lt; Extracting keypoints from second image..." &lt;&lt; endl;
      vector&lt;KeyPoint&gt; keypoints2;
      detector-&gt;detect( rightImg, keypoints2 );
      cout &lt;&lt; keypoints2.size() &lt;&lt; " points" &lt;&lt; endl &lt;&lt; "&gt;" &lt;&lt; endl;
  
      cout &lt;&lt; "&lt; Computing descriptors for keypoints from second image..." &lt;&lt; endl;
      Mat descriptors2;
      descriptorExtractor-&gt;compute( rightImg, keypoints2, descriptors2 );
      cout &lt;&lt; "&gt;" &lt;&lt; endl;
  
      cout &lt;&lt; "&lt; Matching descriptors..." &lt;&lt; endl;
      vector&lt;DMatch&gt; filteredMatches;
      crossCheckMatching( descriptorMatcher, descriptors1, descriptors2, filteredMatches, 1 );
      cout &lt;&lt; "&gt;" &lt;&lt; endl;
  
      vector&lt;int&gt; queryIdxs( filteredMatches.size() ), trainIdxs( filteredMatches.size() );
      for( size_t i = 0; i &lt; filteredMatches.size(); i++ )
      {
          queryIdxs[i] = filteredMatches[i].queryIdx;
          trainIdxs[i] = filteredMatches[i].trainIdx;
      }
  
      cout &lt;&lt; "&lt; Computing homography (RANSAC)..." &lt;&lt; endl;
      vector&lt;Point2f&gt; points1; KeyPoint::convert(keypoints1, points1, queryIdxs);
      vector&lt;Point2f&gt; points2; KeyPoint::convert(keypoints2, points2, trainIdxs);
      H12 = findHomography( Mat(points1), Mat(points2), CV_RANSAC, ransacReprojThreshold );
      cout &lt;&lt; "&gt;" &lt;&lt; endl;
  
      //Mat drawImg;
      if( !H12.empty() ) // filter outliers
      {
          vector&lt;char&gt; matchesMask( filteredMatches.size(), 0 );
          vector&lt;Point2f&gt; points1; KeyPoint::convert(keypoints1, points1, queryIdxs);
          vector&lt;Point2f&gt; points2; KeyPoint::convert(keypoints2, points2, trainIdxs);
          Mat points1t; perspectiveTransform(Mat(points1), points1t, H12);
          for( size_t i1 = 0; i1 &lt; points1.size(); i1++ )
          {
              if( norm(points2[i1] - points1t.at&lt;Point2f&gt;((int)i1,0)) &lt; 4 ) // inlier
                  matchesMask[i1] = 1;
          }
          /* draw inliers
          drawMatches( leftImg, keypoints1, rightImg, keypoints2, filteredMatches, drawImg, CV_RGB(0, 255, 0), CV_RGB(0, 0, 255), matchesMask, 2 ); */
      }
  
      Size imageSize = leftImg.size();
      Mat F = findFundamentalMat(Mat(points1), Mat(points2), FM_8POINT, 0, 0);
      Mat H1, H2;
      stereoRectifyUncalibrated(Mat(points1), Mat(points2), F, imageSize, H1, H2, 3);
  
      Mat cameraMatrix[2], distCoeffs[2], R1, R2, P1, P2, rmap[2][2];
      cameraMatrix[0] = Mat::eye(3, 3, CV_64F);
      cameraMatrix[1] = Mat::eye(3, 3, CV_64F);
      R1 = cameraMatrix[0].inv()*H1*cameraMatrix[0];
      R2 = cameraMatrix[1].inv()*H2*cameraMatrix[1];
      P1 = cameraMatrix[0];
      P2 = cameraMatrix[1];
  
      initUndistortRectifyMap(cameraMatrix[0], distCoeffs[0], R1, P1, imageSize, CV_16SC2, rmap[0][0], rmap[0][1]);
      initUndistortRectifyMap(cameraMatrix[1], distCoeffs[1], R2, P2, imageSize, CV_16SC2, rmap[1][0], rmap[1][1]);
  
      Mat canvas, img;
      double sf;
      int i, j, w, h;
  
      sf = 600./MAX(imageSize.width, imageSize.height);
      w = cvRound(imageSize.width*sf);
      h = cvRound(imageSize.height*sf);
      canvas.create(h, w*2, CV_8UC3);
  
      for (i = 0; i &lt; 2; i++)
      {
          if (i == 0)
              img = leftImg;
          else
              img = rightImg;
  
          Mat rimg, cimg;
          remap(img, rimg, rmap[i][0], rmap[i][1], CV_INTER_LINEAR);
          cvtColor(rimg, cimg, CV_GRAY2BGR);
          Mat canvasPart = canvas(Rect(w*i, 0, w, h));
          resize(cimg, canvasPart, canvasPart.size(), 0, 0, CV_INTER_AREA);
      }
  
          for( j = 0; j &lt; canvas.rows; j += 16 )
          {
              line(canvas, Point(0, j), Point(canvas.cols, j), Scalar(0, 255, 0), 1, 8);
          }
  
          imshow(winName, canvas);
  }
  
  
  int main(int argc, char** argv)
  {
      if( argc != 6 )
      {
          help(argv);
          return -1;
      }
      double ransacReprojThreshold = atof(argv[5]);
  
  
      cout &lt;&lt; "&lt; Creating detector, descriptor extractor and descriptor matcher ..." &lt;&lt; endl;
      Ptr&lt;FeatureDetector&gt; detector = FeatureDetector::create( argv[1] );
      Ptr&lt;DescriptorExtractor&gt; descriptorExtractor = DescriptorExtractor::create( argv[2] );
      Ptr&lt;DescriptorMatcher&gt; descriptorMatcher = DescriptorMatcher::create("FlannBased");
      cout &lt;&lt; "&gt;" &lt;&lt; endl;
      if( detector.empty() || descriptorExtractor.empty() || descriptorMatcher.empty()  )
      {
          cout &lt;&lt; "Can not create detector or descriptor extractor or descriptor matcher of given types" &lt;&lt; endl;
          return -1;
      }
  
      cout &lt;&lt; "&lt; Reading the images..." &lt;&lt; endl;
      Mat leftImg = imread( argv[3] );
      Mat rightImg = imread( argv[4] );
      cout &lt;&lt; "&gt;" &lt;&lt; endl;
      if( leftImg.empty() || ( rightImg.empty()) )
      {
          cout &lt;&lt; "Can not read images" &lt;&lt; endl;
          return -1;
      }
  
      cout &lt;&lt; endl &lt;&lt; "&lt; Extracting keypoints from first image..." &lt;&lt; endl;
      vector&lt;KeyPoint&gt; keypoints1;
      detector-&gt;detect( leftImg, keypoints1 );
      cout &lt;&lt; keypoints1.size() &lt;&lt; " points" &lt;&lt; endl &lt;&lt; "&gt;" &lt;&lt; endl;
  
      cout &lt;&lt; "&lt; Computing descriptors for keypoints from first image..." &lt;&lt; endl;
      Mat descriptors1;
      descriptorExtractor-&gt;compute( leftImg, keypoints1, descriptors1 );
      cout &lt;&lt; "&gt;" &lt;&lt; endl;
  
      namedWindow(winName, CV_WINDOW_NORMAL);
      doIteration( leftImg, rightImg, keypoints1, descriptors1,
                   detector, descriptorExtractor, descriptorMatcher,
                   ransacReprojThreshold );
      for(;;)
      {
          char c = (char)waitKey(0);
          if( c == '\x1b' ) // esc
          {
              cout &lt;&lt; "Exiting ..." &lt;&lt; endl;
              return 0;
          }
      }
      waitKey(0);
      return 0;
  }
  </code></pre>
  
  <p>The main focus should probably be around the doIteration method, but I've put the rest of it in there so you can see exactly what is going on.</p>