title: How do i search over the List<string> and check if the web address already exist or not?
body: <p>In this case its not working good it keep adding the same links to the List.</p>
  
  <pre><code>using System;
  using System.Collections.Generic;
  using System.ComponentModel;
  using System.Data;
  using System.Drawing;
  using System.Linq;
  using System.Text;
  using System.Windows.Forms;
  using HtmlAgilityPack;
  using System.IO;
  using System.Text.RegularExpressions;
  using System.Xml.Linq;
  using System.Net;
  using System.Web;
  
  
  namespace GatherLinks
  {
      public partial class Form1 : Form
      {
          int sites = 0;
          int y = 0;
          string url = @"http://www.google.co.il";
          string guys = "http://www.google.com";
  
          public Form1()
          {
              InitializeComponent();
  
              List&lt;string&gt; a = webCrawler(guys, 2);
          }
  
          private void Form1_Load(object sender, EventArgs e)
          {
  
          }
  
          private int factorial(int n)
          {
              if (n == 0) return 1;
              else y = n * factorial(n - 1);
              richTextBox1.Text = y.ToString();
              return y;
  
  
          }
  
          private List&lt;string&gt; getLinks(HtmlAgilityPack.HtmlDocument document)
          {
  
              List&lt;string&gt; mainLinks = new List&lt;string&gt;();
              var linkNodes = document.DocumentNode.SelectNodes("//a[@href]");
              if (linkNodes != null)
              {
                  foreach (HtmlNode link in linkNodes)
                  {
                      var href = link.Attributes["href"].Value;
                      mainLinks.Add(href);
                  }
              }
              return mainLinks;
  
          }
  
  
          private List&lt;string&gt; webCrawler(string url, int levels)
          {
  
                  HtmlAgilityPack.HtmlDocument doc;
                  HtmlWeb hw = new HtmlWeb();
                  List&lt;string&gt; webSites;// = new List&lt;string&gt;();
                  List&lt;string&gt; csFiles = new List&lt;string&gt;();
  
                  csFiles.Add("temp string to know that something is happening in level = " + levels.ToString());
                  csFiles.Add("current site name in this level is : " + url);
                                  try
                  {
                      doc = hw.Load(url);
                      webSites = getLinks(doc);
  
                      if (levels == 0)
                      {
                          return csFiles;
                      }
                      else
                      {
                          int actual_sites = 0;
                          for (int i = 0; i &lt; webSites.Count() &amp;&amp; i &lt; 20; i++)                         {
                              string t = webSites[i];
                                                          if ((t.StartsWith("http://") == true) || (t.StartsWith("https://") == true)) // replace this with future FilterJunkLinks function
                              {
                                 // for (int e = 0; e &lt; csFiles.Count; e++)
                                 // {
                                      if (csFiles.Contains(t))
                                      {
                                      }
                                      else
                                      {
                                          actual_sites++;
                                          csFiles.AddRange(webCrawler(t, levels - 1));
                                          Texts(richTextBox1, "Level Number " + levels + " " + t + Environment.NewLine, Color.Red);
                                      }
                                 // }
                              }
                          }
                          // report to a message box only at high levels..
                          //if (levels==1)
                          //MessageBox.Show(actual_sites.ToString());
  
                          return csFiles;
                      }
  
  
  
                  }
                  catch
                  {
                      return csFiles;
                  }
  
          }
  </code></pre>
  
  <p>And the Texts function:</p>
  
  <pre><code>public void Texts(RichTextBox box, string text, Color color)
          {
              box.SelectionStart = box.TextLength;
              box.SelectionLength = 0;
  
              box.SelectionColor = color;
              box.AppendText(text);
              box.SelectionColor = box.ForeColor;
          } 
  </code></pre>
  
  <p>I need two things to do in the webCrawler function:</p>
  
  <ol>
  <li><p>If the url variable can not be resolved then the try and catch should do the job.</p></li>
  <li><p>If the List csFiles contains already the same items do not add them again. For example if in the csFiles there already <a href="http://www.google.com" rel="nofollow">http://www.google.com</a> then do not add <a href="http://www.google.com" rel="nofollow">http://www.google.com</a> again so in the end the csFiles List will contain <a href="http://www.google.com" rel="nofollow">http://www.google.com</a> only once.</p></li>
  </ol>