title: Selenium C# - HttpWebRequest always returns 404 despite sites being browse-able
body: <p>I've created a test that checks for broken links on our website. I collect a list of links from our website's site map and in a loop, perform a HttpWebRequest which fails the test if one or more links returns anything but OK (200). The links that fail are added to a dictionary for later review (status code, link).</p>
  
  <p>Initially I was getting authorisation errors, so I added the relevant code to get through our proxy. Now, all HttpWebRequests return 404 - not just to our website, but to any website - Google/Twitter/Facebook etc. However, all of the links work in a browser. I've tried to monitor the traffic in Fiddler while running the test, but no entries are appearing in the Fiddler stream.</p>
  
  <p>On the assumption that this is a 'me'/'us' problem, rather than a fault with the HttpWebRequest class, I suspect that I am not making a fully qualified/'legitimate' HttpWebRequest.</p>
  
  <p>I've gone down the route of checking the headers, re: <a href="https://stackoverflow.com/questions/935181/how-to-view-the-headers-sent-by-httpwebrequest">How to view the headers sent by HttpWebRequest</a> but I'm not getting anywhere with that. When I query the request headers on <a href="http://rextester.com" rel="nofollow noreferrer">http://rextester.com</a>, it's empty. I only get headers on the response...</p>
  
  <pre><code>        HttpWebRequest myHttpWebRequest=(HttpWebRequest)WebRequest.Create("http://www.google.com");
          Console.WriteLine("Request headers: " + myHttpWebRequest.Headers);
          HttpWebResponse myHttpWebResponse=(HttpWebResponse)myHttpWebRequest.GetResponse();
          Console.WriteLine("Response code: " + myHttpWebResponse.StatusCode);
          Console.WriteLine("Response headers: " + myHttpWebResponse.Headers);
  </code></pre>
  
  <p>Here's the code I'm using in my test, is there anything obviously amiss here? The commented out lines were an attempt to specify headers to fix this, on the assumption I was missing something that needed to be in the request, but they made no difference.</p>
  
  <pre><code>    public void BrowseEachLinkOnPage(string page)
      {
          // Page factory does not recognise lists as IWebElement or IList&lt;IWebElement&gt; despite them being set as such. Resorting to manual collection of elements...
  
          // Default collection to initialise variable, //*[text()='donotfindthis'] should find 0 elements
          var linksToTest = driver.FindElements(By.XPath("//*[text()='donotfindthis']"));
  
          try
          {
              switch (page)
              {
                  case "site map":
                      linksToTest = driver.FindElements(By.XPath("//*[contains(@class,'sitemap__container')]//a[contains(@href,'/')]"));
                      break;
              }
          }
          catch (Exception e)
          {
              Log.UpdateTestStatus(Log.Status.Failed, "BrowseEachLinkOnPage(): - \"" + page + "\" page does not map to a predefined list of links. Check existing or create a new reference to a list of links.", driver);
              Console.WriteLine(e);
              throw;
          }
  
          Dictionary&lt;object, string&gt; invalidHttpRequestResponses = new Dictionary&lt;object, string&gt;();
          Dictionary&lt;int, string&gt; httpRequestExceptions = new Dictionary&lt;int, string&gt;();
          int numberOfHttpRequestExceptions = 0;
          int numberOfLinksToTest = linksToTest.Count();
  
          if (numberOfLinksToTest &gt; 0)
          {
              // An exception is triggered if an exception occurs within the loop at least once
              try
              {
                  for (var i = 0; i &lt; numberOfLinksToTest; i++)
                  {
                      try
                      {
                          HttpWebRequest httpReq =
                              (HttpWebRequest) WebRequest.Create(linksToTest[i].GetAttribute("href"));
                          //httpReq.AllowAutoRedirect = true;
                          httpReq.UseDefaultCredentials = false;
  
                          // AD credentials required to get through proxy, enter password when testing
                          httpReq.Credentials = new NetworkCredential("myusername", "mypassword");
  
                          WebProxy webProxy = new WebProxy("http://10.0.0.0/myproxy.pac")
                          {
                              Credentials = new NetworkCredential("myusername", "mypassword")
                          };
  
                          httpReq.Proxy = webProxy;
  
                          //httpReq.AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip;
                          //httpReq.CookieContainer = new CookieContainer();
                          //httpReq.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36";
  
                          HttpWebResponse httpRes = (HttpWebResponse) httpReq.GetResponse();
  
                          if (httpRes.StatusCode != HttpStatusCode.OK)
                          {
  invalidHttpRequestResponses.Add(httpRes.StatusCode, linksToTest[i].GetAttribute("href"));
                          }
  
                          httpRes.Close();
                      }
                      // Suppress the exception, instead capture the details of the exception for later use
                      catch (Exception e)
                      {
                          numberOfHttpRequestExceptions += 1;
                          httpRequestExceptions.Add(i, e.ToString());
                      }
                  }
              }
              // Report why the HttpWebRequests failed for further troubleshooting
              catch (Exception e)
              {
                  Log.UpdateTestStatus(Log.Status.Failed, "HttpWebRequest failed: " + numberOfHttpRequestExceptions + " exception(s) out of " + numberOfLinksToTest + " links tested.\n----- Attempt, exception: -----\n" + httpRequestExceptions, driver);
                  Console.WriteLine(e);
                  throw;
              }
          }
          else
          {
              Log.UpdateTestStatus(Log.Status.Failed, "BrowseEachLinkOnPage(): - there are no links in the collection.", driver);
              throw new Exception("BrowseEachLinkOnPage(): - there are no links in the collection.");
          }
  
          // Assuming there were no HttpWebRequest exceptions, report on any links that failed (StatusCode != to OK)
          if (invalidHttpRequestResponses.Count &gt; 0)
          {
              Log.UpdateTestStatus(Log.Status.Failed, "HttpWebRequest: - invalid response on pages (response, URL):\n" + invalidHttpRequestResponses, driver);
              throw new Exception("HttpWebRequest: - invalid response on pages (response, URL):\n" + invalidHttpRequestResponses);
          }
      }
  </code></pre>
  
  <p>thank you</p>