title: Parallel HTTP requests using System.Net.Http.HttpClient
body: <p>I'm trying to figure out the correct way to parallelize HTTP requests using <code>Task</code> and <code>async/await</code>. I'm using the <code>HttpClient</code> class which already has async methods for retrieving data. If I just call it in a foreach loop and await the response, only one request gets sent at a time (which makes sense because during the <code>await</code>, control is returning to our event loop, not to the next iteration of the foreach loop).</p>
  
  <p>My wrapper around <code>HttpClient</code> looks as such</p>
  
  <pre><code>public sealed class RestClient
  {
      private readonly HttpClient client;
  
      public RestClient(string baseUrl)
      {
          var baseUri = new Uri(baseUrl);
  
          client = new HttpClient
          {
              BaseAddress = baseUri
          };
      }
  
      public async Task&lt;Stream&gt; GetResponseStreamAsync(string uri)
      {
          var resp = await GetResponseAsync(uri);
          return await resp.Content.ReadAsStreamAsync();
      }
  
      public async Task&lt;HttpResponseMessage&gt; GetResponseAsync(string uri)
      {
          var resp = await client.GetAsync(uri);
          if (!resp.IsSuccessStatusCode)
          {
              // ...
          }
  
          return resp;
      }
  
      public async Task&lt;T&gt; GetResponseObjectAsync&lt;T&gt;(string uri)
      {
          using (var responseStream = await GetResponseStreamAsync(uri))
          using (var sr = new StreamReader(responseStream))
          using (var jr = new JsonTextReader(sr))
          {
              var serializer = new JsonSerializer {NullValueHandling = NullValueHandling.Ignore};
              return serializer.Deserialize&lt;T&gt;(jr);
          }
      }
  
      public async Task&lt;string&gt; GetResponseString(string uri)
      {
          using (var resp = await GetResponseStreamAsync(uri))
          using (var sr = new StreamReader(resp))
          {
              return sr.ReadToEnd();
          }
      }
  }
  </code></pre>
  
  <p>And the code invoked by our event loop is</p>
  
  <pre><code>public async void DoWork(Action&lt;bool&gt; onComplete)
  {
      try
      {
          var restClient = new RestClient("https://example.com");
  
          var ids = await restClient.GetResponseObjectAsync&lt;IdListResponse&gt;("/ids").Ids;
  
          Log.Info("Downloading {0:D} items", ids.Count);
          using (var fs = new FileStream(@"C:\test.json", FileMode.Create, FileAccess.Write, FileShare.Read))
          using (var sw = new StreamWriter(fs))
          {
              sw.Write("[");
  
              var first = true;
              var numCompleted = 0;
              foreach (var id in ids)
              {
                  Log.Info("Downloading item {0:D}, completed {1:D}", id, numCompleted);
                  numCompleted += 1;
                  try
                  {
                      var str = await restClient.GetResponseString($"/info/{id}");
                      if (!first)
                      {
                          sw.Write(",");
                      }
  
                      sw.Write(str);
  
                      first = false;
                  }
                  catch (HttpException e)
                  {
                      if (e.StatusCode == HttpStatusCode.Forbidden)
                      {
                          Log.Warn(e.ResponseMessage);
                      }
                      else
                      {
                          throw;
                      }
                  }
              }
  
              sw.Write("]");
          }
  
          onComplete(true);
      }
      catch (Exception e)
      {
          Log.Error(e);
          onComplete(false);
      }
  }
  </code></pre>
  
  <p>I've tried a handful of different approaches involving <code>Parallel.ForEach</code>, <code>Linq.AsParallel</code>, and wrapping the entire contents of the loop in a <code>Task</code>.</p>