title: Can you help me get my head around openssl public key encryption with rsa.h in c++?
body: <p>I am trying to get my head around public key encryption using the openssl implementation of rsa in C++. Can you help? So far these are my thoughts (please do correct if necessary)</p>
  
  <ol>
  <li>Alice is connected to Bob over a network</li>
  <li>Alice and Bob want secure communications</li>
  <li>Alice generates a public / private key pair and sends public key to Bob</li>
  <li>Bob receives public key and encrypts a randomly generated symmetric cypher key (e.g. blowfish) with the public key and sends the result to Alice</li>
  <li>Alice decrypts the ciphertext with the originally generated private key and obtains the symmetric blowfish key</li>
  <li>Alice and Bob now both have knowledge of symmetric blowfish key and can establish a secure communication channel </li>
  </ol>
  
  <p>Now, I have looked at the openssl/rsa.h rsa implementation (since I already have practical experience with openssl/blowfish.h), and I see these two functions:</p>
  
  <pre><code>int RSA_public_encrypt(int flen, unsigned char *from,
  unsigned char *to, RSA *rsa, int padding);
  int RSA_private_decrypt(int flen, unsigned char *from,
   unsigned char *to, RSA *rsa, int padding);
  </code></pre>
  
  <p>If Alice is to generate *rsa, how does this yield the rsa key pair? Is there something like rsa_public and rsa_private which are derived from rsa? Does *rsa contain both public and private key and the above function automatically strips out the necessary key depending on whether it requires the public or private part? Should two unique *rsa pointers be generated so that actually, we have the following:</p>
  
  <pre><code>int RSA_public_encrypt(int flen, unsigned char *from,
  unsigned char *to, RSA *rsa_public, int padding);
  int RSA_private_decrypt(int flen, unsigned char *from,
   unsigned char *to, RSA *rsa_private, int padding);
  </code></pre>
  
  <p>Secondly, in what format should the *rsa public key be sent to Bob? Must it be reinterpreted in to a character array and then sent the standard way? I've heard something about certificates -- are they anything to do with it?</p>
  
  <p>Sorry for all the questions,
  Best Wishes,
  Ben.</p>
  
  <p>EDIT: Coe I am currently employing:</p>
  
  <pre><code>/*
   *  theEncryptor.cpp
   *  
   *
   *  Created by ben on 14/01/2010.
   *  Copyright 2010 __MyCompanyName__. All rights reserved.
   *
   */
  
  #include "theEncryptor.h"
  #include &lt;iostream&gt;
  #include &lt;sys/socket.h&gt;
  #include &lt;sstream&gt;
  
  theEncryptor::theEncryptor()
  {
  
  }
  
  void
  theEncryptor::blowfish(unsigned char *data, int data_len, unsigned char* key, int enc)
  {
  
      //  hash the key first! 
      unsigned char obuf[20];
      bzero(obuf,20);
      SHA1((const unsigned char*)key, 64, obuf);
  
      BF_KEY bfkey;
      int keySize = 16;//strlen((char*)key);
      BF_set_key(&amp;bfkey, keySize, obuf);
  
      unsigned char ivec[16];
      memset(ivec, 0, 16);
  
      unsigned char* out=(unsigned char*) malloc(data_len);
      bzero(out,data_len);
      int num = 0;
      BF_cfb64_encrypt(data, out, data_len, &amp;bfkey, ivec, &amp;num, enc);
  
      //for(int i = 0;i&lt;data_len;i++)data[i]=out[i];
  
      memcpy(data, out, data_len);
      free(out);  
  
  }
  
  void
  theEncryptor::generateRSAKeyPair(int bits)
  {
      rsa = RSA_generate_key(bits, 65537, NULL, NULL);
  }
  
  
  int
  theEncryptor::publicEncrypt(unsigned char* data, unsigned char* dataEncrypted,int dataLen)
  {   
      return RSA_public_encrypt(dataLen, data, dataEncrypted, rsa, RSA_PKCS1_OAEP_PADDING);   
  }
  
  int
  theEncryptor::privateDecrypt(unsigned char* dataEncrypted,
                               unsigned char* dataDecrypted)
  {
      return RSA_private_decrypt(RSA_size(rsa), dataEncrypted, 
                                     dataDecrypted, rsa, RSA_PKCS1_OAEP_PADDING);
  }
  
  void 
  theEncryptor::receivePublicKeyAndSetRSA(int sock, int bits)
  {
      int max_hex_size = (bits / 4) + 1;
      char keybufA[max_hex_size];
      bzero(keybufA,max_hex_size);
      char keybufB[max_hex_size];
      bzero(keybufB,max_hex_size);
      int n = recv(sock,keybufA,max_hex_size,0); 
      n = send(sock,"OK",2,0);
      n = recv(sock,keybufB,max_hex_size,0); 
      n = send(sock,"OK",2,0); 
      rsa = RSA_new();
      BN_hex2bn(&amp;rsa-&gt;n, keybufA);
      BN_hex2bn(&amp;rsa-&gt;e, keybufB);
  }
  
  void 
  theEncryptor::transmitPublicKey(int sock, int bits)
  {
      const int max_hex_size = (bits / 4) + 1;
      long size = max_hex_size;
      char keyBufferA[size];
      char keyBufferB[size];
      bzero(keyBufferA,size);
      bzero(keyBufferB,size);
      sprintf(keyBufferA,"%s\r\n",BN_bn2hex(rsa-&gt;n));
      sprintf(keyBufferB,"%s\r\n",BN_bn2hex(rsa-&gt;e));
      int n = send(sock,keyBufferA,size,0);
      char recBuf[2];
      n = recv(sock,recBuf,2,0);
      n = send(sock,keyBufferB,size,0);
      n = recv(sock,recBuf,2,0);
  }
  
  void
  theEncryptor::generateRandomBlowfishKey(unsigned char* key, int bytes)
  {
              /*
      srand( (unsigned)time( NULL ) );
      std::ostringstream stm;
      for(int i = 0;i&lt;bytes;i++){
          int randomValue = 65 + rand()% 26;
          stm &lt;&lt; (char)((int)randomValue);
      }
      std::string str(stm.str());
      const char* strs = str.c_str();
      for(int i = 0;bytes;i++)key[i]=strs[i];
              */
  
      int n = RAND_bytes(key, bytes);
  
      if(n==0)std::cout&lt;&lt;"Warning key was generated with bad entropy. You should not consider communication to be secure"&lt;&lt;std::endl;
  
  }
  
  theEncryptor::~theEncryptor(){}
  </code></pre>