title: pcap_next occasionally losing packets on Linux
body: <p>Yesterday, I asked if my <a href="https://stackoverflow.com/questions/11637063/">asynchronous use of libpcap was making me lose packets</a>. Today, I looked further and it seems that the problem is not on the asynchronous use of libpcap, but on the use of <code>pcap_next_ex</code>. Occasionally (10 runs out of a 1000), <code>pcap_next_ex</code> will return before the pcap handle timeout expired, telling the program that there were no packets to be read (even though they are there).</p>
  
  <p>The following proof-of-concept reproduces the problem. It depends on libpcap, pthread, boost and <a href="http://code.google.com/p/libcrafter/" rel="nofollow noreferrer">libcrafter</a> (a nice C++ packet crafting library). Basically, it sends a set of TCP-SYN packets to a destination and tries to obtain the responses using libpcap. A thread calling <code>pcap_loop</code> is run in parallel - while the main program misses some responses (as described above), the thread captures all the packets.</p>
  
  <pre><code>#include &lt;boost/date_time/posix_time/posix_time.hpp&gt;
  #include &lt;boost/format.hpp&gt;
  #include &lt;boost/make_shared.hpp&gt;
  #include &lt;boost/shared_ptr.hpp&gt;
  
  #include &lt;crafter.h&gt;
  #include &lt;netinet/ip.h&gt;
  #include &lt;netinet/tcp.h&gt;
  #include &lt;pcap.h&gt;
  #include &lt;sys/types.h&gt;
  #include &lt;sys/socket.h&gt;
  
  using namespace boost;
  using namespace Crafter;
  using namespace std;
  
  int captureThreadCount = 0;
  
  typedef vector&lt;pair&lt;shared_ptr&lt;Packet&gt;, posix_time::time_duration&gt; &gt; PacketTimestamp;
  PacketTimestamp capturedThreadPackets;
  
  void captureThreadCallback(u_char* user, const struct pcap_pkthdr* h, const u_char* bytes)
  {
      shared_ptr&lt;Packet&gt; packet = make_shared&lt;Packet&gt;();
      packet-&gt;PacketFromIP(bytes + 16, h-&gt;caplen - 16);
      posix_time::time_duration timestamp = posix_time::seconds(h-&gt;ts.tv_sec) +
          posix_time::microseconds(h-&gt;ts.tv_usec);
      capturedThreadPackets.push_back(make_pair(packet, timestamp));
      ++captureThreadCount;
  }
  
  void* captureThread(void* arg)
  {
      pcap_t* pcap = (pcap_t*) arg;
      pcap_loop(pcap, -1, captureThreadCallback, NULL);
  }
  
  int main(int argc, char* argv[])
  {
      if (argc != 5) {
          cout &lt;&lt; "Usage: " &lt;&lt; argv[0] &lt;&lt; " &lt;source ip&gt; &lt;destination ip&gt; &lt;port&gt; &lt;# tries&gt;" &lt;&lt; endl;
          exit(1);
      }
  
      InitCrafter();
  
      // Parameters.
      string sourceIp = argv[1];
      string destinationIp = argv[2];
      int port = atoi(argv[3]);
      int nTries = atoi(argv[4]);
  
      char errorBuffer[1024];
      // Socket for sending,
      int sd = socket(PF_INET, SOCK_RAW, IPPROTO_RAW);
      // And sockaddr_in to send to.
      struct sockaddr_in sin;
      sin.sin_family = AF_INET;
      sin.sin_port = htons(port);
      sin.sin_addr.s_addr = inet_addr(destinationIp.c_str());    
      // One pcap for the main thread (calling pcap_next),
      pcap_t* pcapForNext = pcap_open_live(NULL, BUFSIZ, false, 1000, errorBuffer);
      // Another pcap for a capture thread (calling pcap_loop),
      pcap_t* pcapCapture = pcap_open_live(NULL, BUFSIZ, false, 1000, errorBuffer);
      // Both filtered for SYN+ACK or RST+ACK from destination:port to source.
      string filterExpression = (boost::format("ip src %s and dst %s and src port %d and ((tcp[tcpflags] &amp; (tcp-syn|tcp-ack) != 0) or (tcp[tcpflags] &amp; (tcp-rst|tcp-ack) != 0))") % destinationIp % sourceIp % port).str();
      struct bpf_program filter;
      pcap_compile(pcapForNext, &amp;filter, filterExpression.c_str(), false, 0);
      pcap_setfilter(pcapForNext, &amp;filter);
      pcap_setfilter(pcapCapture, &amp;filter);
      pcap_freecode(&amp;filter);
      // Don't forget the capture thread!
      pthread_t thread;
      pthread_create(&amp;thread, NULL, captureThread, pcapCapture);
  
      // Some statistics.
      int packetsSent = 0;
      int packetsReceived = 0;
      int packetsTimeout = 0;
      int packetsFailed = 0;
  
      // Let's probe.
      for (int i = 0; i &lt; nTries; ++i) {
          // Create packet,
          IP ipHeader;
          ipHeader.SetSourceIP(sourceIp);
          ipHeader.SetDestinationIP(destinationIp);
          TCP tcpHeader;
          tcpHeader.SetSrcPort(12345 + i);
          tcpHeader.SetDstPort(port);
          tcpHeader.SetFlags(TCP::SYN);
          shared_ptr&lt;Packet&gt; packet = make_shared&lt;Packet&gt;(ipHeader / tcpHeader);
          // Check the time,
          struct timeval tv;
          gettimeofday(&amp;tv, NULL);
          posix_time::time_duration sentTime =
              posix_time::seconds(tv.tv_sec) + posix_time::microseconds(tv.tv_usec);
          // And send it.
          if (packet-&gt;SocketSend(sd) &gt; 0) {
              cerr &lt;&lt; "Sent packet " &lt;&lt; i &lt;&lt; " at " &lt;&lt; sentTime &lt;&lt; "." &lt;&lt; endl;
              ++packetsSent;
          }
          else {
              cerr &lt;&lt; "Sending packet " &lt;&lt; i &lt;&lt; " failed." &lt;&lt; endl;
              continue;
          }
  
          struct pcap_pkthdr* pktHeader;
          const u_char* pktData;
          int r;
          // Wait for the response.
          if ((r = pcap_next_ex(pcapForNext, &amp;pktHeader, &amp;pktData)) &gt; 0) {
              posix_time::time_duration timestamp =
                  posix_time::seconds(pktHeader-&gt;ts.tv_sec) +
                  posix_time::microseconds(pktHeader-&gt;ts.tv_usec);
              cerr &lt;&lt; "Response " &lt;&lt; i &lt;&lt; " received at " &lt;&lt; timestamp &lt;&lt; "." &lt;&lt; endl;
              ++packetsReceived;
          }
          else if (r == 0) {
              cerr &lt;&lt; "Timeout receiving response for " &lt;&lt; i &lt;&lt; "." &lt;&lt; endl;
              ++packetsTimeout;
          }
          else {
              cerr &lt;&lt; "Failed receiving response for " &lt;&lt; i &lt;&lt; "." &lt;&lt; endl;
              ++packetsFailed;
          }
      }
  
      // Wait (to ensure "fast packets" are captured by the capture thread),
      usleep(500000); // 500 ms.
  
      for (PacketTimestamp::iterator i = capturedThreadPackets.begin();
           i != capturedThreadPackets.end(); ++i) {
          TCP* tcpLayer = GetTCP(*i-&gt;first);
          cout &lt;&lt; "Captured packet " &lt;&lt; (tcpLayer-&gt;GetDstPort() - 12345) &lt;&lt; " at " &lt;&lt; i-&gt;second &lt;&lt; "." &lt;&lt; endl;
      }
  
      cout &lt;&lt; "SNT=" &lt;&lt; packetsSent &lt;&lt;
          ", RCV=" &lt;&lt; packetsReceived &lt;&lt;
          ", TIM=" &lt;&lt; packetsTimeout &lt;&lt;
          ", FLD=" &lt;&lt; packetsFailed &lt;&lt;
          ", CAP=" &lt;&lt; captureThreadCount &lt;&lt; "." &lt;&lt; endl;
  
      CleanCrafter();
  }
  </code></pre>