title: Winsock2 - How to use IOCP on client side
body: <p>I’ve recently started learning IOCP on Windows and been reading the following article:</p>
  
  <p><a href="http://www.codeproject.com/Tips/95363/Another-TCP-echo-server-using-IOCP" rel="noreferrer">http://www.codeproject.com/Tips/95363/Another-TCP-echo-server-using-IOCP</a></p>
  
  <p>You can download the sample for the article from:</p>
  
  <p><a href="http://dl.dropbox.com/u/281215/documentation/iocp-1.00.html" rel="noreferrer">http://dl.dropbox.com/u/281215/documentation/iocp-1.00.html</a></p>
  
  <p>The sample contains two simple applications – <strong>iocp_echo_server</strong> and <strong>TcpEchoClient</strong>.</p>
  
  <p>I understand that IOCP is usually used on the server side of the client/server model but I’d like to create a client using IOCP.</p>
  
  <p>I’ve so far tried modifying the client sample above so that whenever the server sends a response to the client, it gets picked up automatically, however it doesn’t work.</p>
  
  <p>I’ve left iocp_echo_server.c as is.  My modified version of TcpEchoClient.c looks like:</p>
  
  <pre><code>//TcpEchoClient.c - a minimalistic echo client
  // -----------------------------------------------------------------------------
  
  // C language includes
  #include &lt;stdio.h&gt;
  #include &lt;winsock2.h&gt;
  #include "mswsock.h"  // for AcceptEx
  #include &lt;stdlib.h&gt; // exit
  #include &lt;string.h&gt;
  
  // Windows includes
  #include &lt;windows.h&gt;
  
  #pragma warning(disable: 4996) // sprintf
  
  // -----------------------------------------------------------------------------
  
  // configuration
  enum
  {
      BUFLEN = 1000,
      SERVICE_PORT = 4000,
      SERVER_ADDRESS = INADDR_LOOPBACK
  };
  
  enum // socket operations
  {
      OP_NONE,
      OP_ACCEPT,
      OP_READ,
      OP_WRITE
  };
  
  typedef struct _SocketState // socket state &amp; control
  {
      char operation;
      SOCKET socket;
      DWORD length;
      char buf[1024];
  } SocketState;
  
  // variables
  static HANDLE cpl_port;
  
  static SOCKET sock;
  static SocketState sock_state;
  static WSAOVERLAPPED sock_ovl;
  
  static LPFN_ACCEPTEX pfAcceptEx;
  static GUID GuidAcceptEx = WSAID_ACCEPTEX;
  
  static int msgNumber;
  static char msgBuf[BUFLEN];
  static struct sockaddr_in sin;
  
  // prototypes
  static void createConnection(void);
  static void createSocket(void);
  static void init(void);
  static void initWinsock(void);
  static void prepareEndpoint(void);
  static void recvBuffer(void);
  static void run(void);
  static void sendBuffer(void);
  
  static SOCKET create_accepting_socket(void);
  static void create_io_completion_port(void);
  static BOOL get_completion_status(DWORD*, SocketState**,WSAOVERLAPPED**);
  
  // -----------------------------------------------------------------------------
  
  void main(void)
  {
      init();
      run();
  }
  
  // -----------------------------------------------------------------------------
  
  static void createConnection(void)
  {
      printf("* connecting\n");
      if (WSAConnect(sock, (LPSOCKADDR)&amp;sin, sizeof(sin), NULL, NULL, NULL, NULL) == SOCKET_ERROR)
      {
          int err = WSAGetLastError();
          printf("* error %d in connect\n", err);
          exit(1);
      }
      printf("* connected\n");
  }
  
  // -----------------------------------------------------------------------------
  
  static void createSocket(void)
  {
      sock = WSASocket(AF_INET, SOCK_STREAM, IPPROTO_TCP, NULL, 0, WSA_FLAG_OVERLAPPED);
      if (sock == INVALID_SOCKET)
      {
          int err = WSAGetLastError();
          printf("* error %d creating socket\n", err);
          exit(1);
      }
  
      // for use by AcceptEx
      sock_state.socket = 0; // to be updated later
      sock_state.operation = OP_ACCEPT;
  
      if (CreateIoCompletionPort((HANDLE)sock, cpl_port, (ULONG_PTR)&amp;sock_state, 0) != cpl_port)
      {
          int err = WSAGetLastError();
          printf("* error %d in listener\n", err);
          exit(1);
      }
  }
  
  // -----------------------------------------------------------------------------
  
  static void init(void)
  {
      initWinsock();
      create_io_completion_port();
      createSocket();
      prepareEndpoint();
      createConnection();
  }
  
  // -----------------------------------------------------------------------------
  
  static void initWinsock(void)
  {
      WSADATA wsaData;
  
      if (WSAStartup(0x202, &amp;wsaData) == SOCKET_ERROR)
      {
          int err = WSAGetLastError();
          printf("* error %d in WSAStartup\n", err);
          exit(1);
      }
  }
  // -----------------------------------------------------------------------------
  
  static void prepareEndpoint(void)
  {
      sin.sin_family = AF_INET;
      sin.sin_addr.s_addr = htonl(SERVER_ADDRESS);
      sin.sin_port = htons(SERVICE_PORT);
  
      // bind_listening_socket()
      {
          //if (bind(sock, (SOCKADDR*)&amp;sin, sizeof(sin)) == SOCKET_ERROR)
          //{
          //    printf("* error in bind!\n");
          //    exit(1);
          //}
      }
  
      // start_listening()
      {
          //if (listen(sock, 100) == SOCKET_ERROR)
          //{
          //    printf("* error in listen!\n");
          //    exit(1);
          //}
          //printf("* started listening for connection requests...\n");
      }
  
      // load_accept_ex()
      {
          //DWORD dwBytes;
  
          // black magic for me!!!
          // You do not need to call in your code WSAIoctl. You can directly use AcceptEx and adds Mswsock.lib.
          //WSAIoctl(sock, SIO_GET_EXTENSION_FUNCTION_POINTER, &amp;GuidAcceptEx, sizeof(GuidAcceptEx), &amp;pfAcceptEx, sizeof(pfAcceptEx), &amp;dwBytes, NULL, NULL);
      }
  
      // start_accepting()
      {
          //SOCKET acceptor = create_accepting_socket();
          //DWORD expected = sizeof(struct sockaddr_in) + 16;
  
          //printf("* started accepting connections...\n");
  
          // uses listener's completion key and overlapped structure
          //sock_state.socket = acceptor;
          //memset(&amp;sock_ovl, 0, sizeof(WSAOVERLAPPED));
  
          // starts asynchronous accept
          //if (!pfAcceptEx(sock, acceptor, sock_state.buf, 0 /* no recv */, expected, expected, NULL, &amp;sock_ovl))
          //{
          //    int err = WSAGetLastError();
          //    if (err != ERROR_IO_PENDING)
          //    {
          //        printf("* error %d in AcceptEx\n", err);
          //        exit(1);
          //    }
          //}
      }
  }
  
  // -----------------------------------------------------------------------------
  
  static void recvBuffer(void)
  {
      char* buf = msgBuf;
      int pendingLen = BUFLEN;
  
      printf("* receiving reply\n");
  
      while (pendingLen &gt; 0)
      {
          int partialLen = recv(sock, buf, pendingLen, 0);
  
          if (partialLen &gt; 0)
          {
              pendingLen -= partialLen;
              buf += partialLen;
              continue;
          }
  
          // ------
  
          if (partialLen == 0)
          {
              printf("* connection closed by the server\n");
          }
          else // partialLen &lt; 0
          {
              int err = WSAGetLastError();
              printf("* error %d in recv\n", err);
          }
  
          exit(1);
      }
  }
  
  // -----------------------------------------------------------------------------
  
  static void run(void)
  {
      DWORD length;
      BOOL resultOk;
      WSAOVERLAPPED* ovl_res;
      SocketState* socketState;
  
      for (;;)
      {
          sendBuffer();
  
          resultOk = get_completion_status(&amp;length, &amp;socketState, &amp;ovl_res);
  
          recvBuffer();
      }
  }
  
  // -----------------------------------------------------------------------------
  
  static void sendBuffer(void)
  {
      char* buf = msgBuf;
      int pendingLen = BUFLEN;
  
      printf("* sending message\n");
      sprintf(msgBuf, "%05 *****", msgNumber++);
  
      while (pendingLen &gt; 0)
      {
          int partialLen = send(sock, buf, pendingLen, 0);
  
          if (partialLen &gt; 0)
          {
              pendingLen -= partialLen;
              buf += partialLen;
              continue;
          }
  
          // -----------
  
          if (partialLen == 0)
          {
              printf("* connection closed by the server\n");
          }
          else // partialLen &lt; 0
          {
              int err = WSAGetLastError();
              printf("* error %d in send\n", err);
          }
  
          exit(1);
      }
  }
  
  // -----------------------------------------------------------------------------
  
  static SOCKET create_accepting_socket(void)
  {
      SOCKET acceptor = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
      if (acceptor == INVALID_SOCKET)
      {
          printf("* error creating accept socket!\n");
          exit(1);
      }
      return acceptor;
  }
  
  // -----------------------------------------------------------------------------
  
  static void create_io_completion_port(void)
  {
      cpl_port = CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, 0, 0);
      if (!cpl_port)
      {
          int err = WSAGetLastError();
          printf("* error %d in line %d CreateIoCompletionPort\n", err, __LINE__);
          exit(1);
      }
  }
  
  // -----------------------------------------------------------------------------
  
  static BOOL get_completion_status(DWORD* length, SocketState** socketState, WSAOVERLAPPED** ovl_res)
  {
      BOOL resultOk;
      *ovl_res = NULL;
      *socketState = NULL;
  
      resultOk = GetQueuedCompletionStatus(cpl_port, length, (PULONG_PTR)socketState, ovl_res, INFINITE);
  
      if (!resultOk)
      {
          DWORD err = GetLastError();
          printf("* error %d getting completion port status!!!\n", err);
      }
  
      if (!*socketState || !*ovl_res)
      {
          printf("* don't know what to do, aborting!!!\n");
          exit(1);
      }
  
      return resultOk;
  }
  
  // -----------------------------------------------------------------------------
  // the end
  </code></pre>
  
  <p>When the server send a response by calling:</p>
  
  <pre><code>WSASend(socketState-&gt;socket, &amp;wsabuf, 1, NULL, 0, ovl, NULL)
  </code></pre>
  
  <p>I’d expect it to be picked up by the client on this line: </p>
  
  <pre><code>resultOk = get_completion_status(&amp;length, &amp;socketState, &amp;ovl_res);
  </code></pre>
  
  <p>But it doesn’t…</p>
  
  <p>Would anybody be able to tell me what I’m doing wrong?</p>
  
  <p><strong>Edit:</strong></p>
  
  <p>I’ve taken the following points:</p>
  
  <ul>
  <li>On the client side, you use WSAConnect() to create an outbound connection.</li>
  <li>Call WSARecv() and WSASend() to start reading/writing operations when needed</li>
  <li>you have to use WSASend/WSARecv if you want to use I/O completion ports.</li>
  </ul>
  
  <p>and attempted to create a simple IOCP based client:</p>
  
  <pre><code>#include &lt;iostream&gt;
  #include &lt;winsock2.h&gt;
  #pragma comment(lib,"ws2_32.lib")
  
  static DWORD WINAPI ClientWorkerThread(LPVOID lpParameter);
  
  int main(void)
  {
      WSADATA WsaDat;
      if (WSAStartup(MAKEWORD(2, 2), &amp;WsaDat) != NO_ERROR)
          return 0;
  
      // Step 1 - Create an I/O completion port.
      HANDLE hCompletionPort = CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, 0, 0);
      if (!hCompletionPort)
          return 0;
  
      // Step 2 - Find how many processors.
      SYSTEM_INFO systemInfo;
      GetSystemInfo(&amp;systemInfo);
      const int nNumberOfProcessors = systemInfo.dwNumberOfProcessors;
  
      // Step 3 - Create worker threads.
      for (int i = 0; i &lt; nNumberOfProcessors; i++)
      {
          HANDLE hThread = CreateThread(NULL, 0, ClientWorkerThread, hCompletionPort, 0, NULL);
          CloseHandle(hThread);
      }
  
      // Step 4 - Create a socket.
      SOCKET Socket = WSASocket(AF_INET, SOCK_STREAM, IPPROTO_TCP, NULL, 0, WSA_FLAG_OVERLAPPED);
      if (Socket == INVALID_SOCKET)
          return 0;
  
      struct hostent *host;
      if ((host = gethostbyname("localhost")) == NULL)
          return 0;
  
      SOCKADDR_IN SockAddr;
      SockAddr.sin_family = AF_INET;
      SockAddr.sin_addr.s_addr = *((unsigned long*)host-&gt;h_addr);
      SockAddr.sin_port = htons(8888);
  
      // Step 5 - Associate the socket with the I/O completion port.
      CreateIoCompletionPort((HANDLE)Socket, hCompletionPort, (ULONG_PTR)0, 0);
  
      if (WSAConnect(Socket, (SOCKADDR*)(&amp;SockAddr), sizeof(SockAddr), NULL, NULL, NULL, NULL) == SOCKET_ERROR)
          return 0;
  
      char buffer[1000];
      memset(buffer, 0, 999);
      WSABUF wsaBuf = {strlen(buffer), buffer};
      DWORD dwSendBytes = 0;
      DWORD dwReceivedBytes = 0;
      DWORD dwFlags = 0;
      WSAOVERLAPPED wsaOverlapped;
      SecureZeroMemory((PVOID)&amp;wsaOverlapped, sizeof(wsaOverlapped));
      wsaOverlapped.hEvent = WSACreateEvent();
  
      for(;;)
      {
          WSARecv(Socket, &amp;wsaBuf, 1, &amp;dwReceivedBytes, &amp;dwFlags, &amp;wsaOverlapped, NULL);
          std::cout &lt;&lt; wsaBuf.buf;
  
          //WSASend(Socket, &amp;wsaBuf, 1, &amp;dwSendBytes, 0, &amp;wsaOverlapped, NULL);
  
          int nError = WSAGetLastError();
          if(nError != WSAEWOULDBLOCK&amp;&amp;nError != 0)
          {
              std::cout &lt;&lt; "Winsock error code: " &lt;&lt; nError &lt;&lt; "\r\n";
              std::cout &lt;&lt; "Server disconnected!\r\n";
              shutdown(Socket, SD_SEND);
              closesocket(Socket);
  
              break;
          }
          Sleep(1000);
      }
  
      WSACleanup();
      system("PAUSE");
      return 0;
  }
  
  static DWORD WINAPI ClientWorkerThread(LPVOID lpParameter)
  {
      HANDLE hCompletionPort = (HANDLE)lpParameter;
      DWORD dwBytesTransferred = 0;
  
      while (TRUE)
      {
          BOOL bRet = GetQueuedCompletionStatus(hCompletionPort, &amp;dwBytesTransferred, (LPDWORD)0, (LPOVERLAPPED*)0, INFINITE);
      }
  
      return 0;
  }
  </code></pre>
  
  <p>I know there are several things I’m doing wrong but I don’t know what they are.</p>
  
  <p>Could somebody take a look at my code and give me some hints please?</p>
  
  <p>Many thanks</p>
  
  <p><strong>Edit 2:</strong></p>
  
  <p>Sorry this post is getting too long.</p>
  
  <p>I've had another go trying to implement an IOCP based client after reading Remy's comments below but I'm still not sure if I'm on the right track.</p>
  
  <p>I'd really appreciate it if somebody could take a look at my new code (compiles fine under VS2010 &amp; error checking omitted) below and give me some feedback.</p>
  
  <p><strong>NonBlockingClient:</strong></p>
  
  <pre><code>#include &lt;iostream&gt;
  #include &lt;winsock2.h&gt;
  #pragma comment(lib, "ws2_32.lib")
  
  static DWORD WINAPI ClientWorkerThread(LPVOID lpParameter);
  
  typedef struct _PER_HANDLE_DATA 
  {
      SOCKET Socket;
  } PER_HANDLE_DATA, * LPPER_HANDLE_DATA;
  
  typedef struct
  {
      WSAOVERLAPPED wsaOverlapped;
      WSABUF wsaBuf;
      int OperationType;
  } PER_IO_DATA, * LPPER_IO_DATA;
  
  int main(void)
  {
      WSADATA WsaDat;
      WSAStartup(MAKEWORD(2, 2), &amp;WsaDat);
  
      // Step 1 - Create an I/O completion port.
      HANDLE hCompletionPort = CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, 0, 0);
  
      // Step 2 - Find how many processors.
      SYSTEM_INFO systemInfo;
      GetSystemInfo(&amp;systemInfo);
  
      // Step 3 - Create worker threads.
      for (int i = 0; i &lt; (int)systemInfo.dwNumberOfProcessors; i++)
      {
          HANDLE hThread = CreateThread(NULL, 0, ClientWorkerThread, hCompletionPort, 0, NULL);
          CloseHandle(hThread);
      }
  
      // Step 4 - Create a socket.
      SOCKET Socket = WSASocket(AF_INET, SOCK_STREAM, IPPROTO_TCP, NULL, 0, WSA_FLAG_OVERLAPPED);
  
      PER_HANDLE_DATA *pPerHandleData = new PER_HANDLE_DATA;
      pPerHandleData-&gt;Socket = Socket;
  
      struct hostent *host;
      host = gethostbyname("localhost");
  
      SOCKADDR_IN SockAddr;
      SockAddr.sin_family = AF_INET;
      SockAddr.sin_addr.s_addr = *((unsigned long*)host-&gt;h_addr);
      SockAddr.sin_port = htons(8888);
  
      // Step 5 - Associate the socket with the I/O completion port.
      CreateIoCompletionPort((HANDLE)Socket, hCompletionPort, (DWORD)pPerHandleData, 0);
  
      WSAConnect(Socket, (SOCKADDR*)(&amp;SockAddr), sizeof(SockAddr), NULL, NULL, NULL, NULL);
  
      static char buffer[1000];
      memset(buffer, 0, 999);
  
      PER_IO_DATA *pPerIoData = new PER_IO_DATA;
  
      pPerIoData-&gt;wsaBuf.buf = buffer;
      pPerIoData-&gt;wsaBuf.len = sizeof(buffer);
  
      DWORD dwSendBytes = 0;
      DWORD dwReceivedBytes = 0;
      DWORD dwFlags = 0;
  
      SecureZeroMemory((PVOID)&amp;pPerIoData-&gt;wsaOverlapped, sizeof(pPerIoData-&gt;wsaOverlapped));
      pPerIoData-&gt;wsaOverlapped.hEvent = WSACreateEvent();
  
      WSARecv(Socket, &amp;pPerIoData-&gt;wsaBuf, 1, &amp;dwReceivedBytes, &amp;dwFlags, &amp;pPerIoData-&gt;wsaOverlapped, NULL);
      std::cout &lt;&lt; pPerIoData-&gt;wsaBuf.buf;
  
      for (;;)
      {
          int nError = WSAGetLastError();
          if (nError != WSAEWOULDBLOCK&amp;&amp;nError != 0)
          {
              std::cout &lt;&lt; "Winsock error code: " &lt;&lt; nError &lt;&lt; "\r\n";
              std::cout &lt;&lt; "Server disconnected!\r\n";
              shutdown(Socket, SD_SEND);
              closesocket(Socket);
  
              break;
          }
  
          Sleep(1000);
      }
  
      delete pPerHandleData;
      delete pPerIoData;
      WSACleanup();
      return 0;
  }
  
  static DWORD WINAPI ClientWorkerThread(LPVOID lpParameter)
  {
      HANDLE hCompletionPort = (HANDLE)lpParameter;
      DWORD bytesCopied = 0;
      OVERLAPPED *overlapped = 0;
      LPPER_HANDLE_DATA PerHandleData;
      LPPER_IO_DATA PerIoData;
      DWORD SendBytes, RecvBytes;
      DWORD Flags;
      BOOL bRet;
  
      while (TRUE)
      {
          bRet = GetQueuedCompletionStatus(hCompletionPort, &amp;bytesCopied, (LPDWORD)&amp;PerHandleData, (LPOVERLAPPED*)&amp;PerIoData, INFINITE);
  
          if (bytesCopied == 0)
          {
              break;
          }
          else
          {
              Flags = 0;
              ZeroMemory(&amp;(PerIoData-&gt;wsaOverlapped), sizeof(WSAOVERLAPPED));
  
              PerIoData-&gt;wsaBuf.len = 1000;
              WSARecv(PerHandleData-&gt;Socket, &amp;(PerIoData-&gt;wsaBuf), 1, &amp;RecvBytes, &amp;Flags, &amp;(PerIoData-&gt;wsaOverlapped), NULL);
          }
      }
  
      return 0;
  }
  </code></pre>
  
  <p><strong>NonBlockingServer:</strong></p>
  
  <pre><code>#include &lt;iostream&gt;
  #include &lt;winsock2.h&gt;
  #pragma comment(lib,"ws2_32.lib")
  
  int main()
  {
      WSADATA WsaDat;
      WSAStartup(MAKEWORD(2,2), &amp;WsaDat);
  
      SOCKET listenSocket = WSASocket(AF_INET, SOCK_STREAM, IPPROTO_TCP, NULL, 0, WSA_FLAG_OVERLAPPED);
  
      SOCKADDR_IN server;
      server.sin_family = AF_INET;
      server.sin_addr.s_addr = INADDR_ANY;
      server.sin_port = htons(8888);
  
      bind(listenSocket, (SOCKADDR*)(&amp;server), sizeof(server));
  
      listen(listenSocket, 1);
  
      SOCKET acceptSocket = SOCKET_ERROR;
      sockaddr_in saClient;
      int nClientSize = sizeof(saClient);
      while (acceptSocket == SOCKET_ERROR)
      {
          std::cout &lt;&lt; "Waiting for incoming connections...\r\n";
          acceptSocket = WSAAccept(listenSocket, (SOCKADDR*)&amp;saClient, &amp;nClientSize, NULL, NULL);
      }
  
      std::cout &lt;&lt; "Client connected!\r\n\r\n";
  
      char *szMessage = "Welcome to the server!\r\n";
      WSAOVERLAPPED SendOverlapped;
      DWORD SendBytes;
  
      WSABUF DataBuf;
      DataBuf.len = 1000;
      DataBuf.buf = szMessage;
  
      SecureZeroMemory((PVOID)&amp;SendOverlapped, sizeof(WSAOVERLAPPED));
      SendOverlapped.hEvent = WSACreateEvent();
  
      for (;;)
      {
          WSASend(acceptSocket, &amp;DataBuf, 1, &amp;SendBytes, 0, &amp;SendOverlapped, NULL);
  
          int nError = WSAGetLastError();
          if (nError != WSAEWOULDBLOCK &amp;&amp; nError != 0)
          {
              std::cout &lt;&lt; "Winsock error code: " &lt;&lt; nError &lt;&lt; "\r\n";
              std::cout &lt;&lt; "Client disconnected!\r\n";
  
              shutdown(acceptSocket, SD_SEND);
              closesocket(acceptSocket);
  
              break;
          }
  
          Sleep(1000);
      }
  
      WSACleanup();
      return 0;
  }
  </code></pre>
  
  <p>Thanks again!</p>