libs/PHPCrawlerURLFilter.class.php
protected function urlMatchesRules(PHPCrawlerURLDescriptor $url)
  { 
    // URL-parts of the URL to check against the filter-rules
    $url_parts = PHPCrawlerUtils::splitURL($url->url_rebuild);
    
    // Kick out all links that r NOT of protocol "http" or "https"
    if ($url_parts["protocol"] != "http://" && $url_parts["protocol"] != "https://")
    {
      return false;
    }
    
    // If meta-tag "robots"->"nofollow" is present and obey_nofollow_tags is TRUE -> always kick out URL
    if ($this->obey_nofollow_tags == true &&
        isset($this->CurrentDocumentInfo->meta_attributes["robots"]) &&
        preg_match("#nofollow# i", $this->CurrentDocumentInfo->meta_attributes["robots"]))
    {
      return false;
    }
    
    // If linkcode contains "rel='nofollow'" and obey_nofollow_tags is TRUE -> always kick out URL
    if ($this->obey_nofollow_tags == true)
    {
      if (preg_match("#^<[^>]*rel\s*=\s*(?|\"\s*nofollow\s*\"|'\s*nofollow\s*'|\s*nofollow\s*)[^>]*>#", $url->linkcode))
      {
        return false;
      }
    }
    
    // Filter URLs to other domains if wanted
    if ($this->general_follow_mode >= 1)
    {
      if ($url_parts["domain"] != $this->starting_url_parts["domain"]) return false;
    }
    
    // Filter URLs to other hosts if wanted
    if ($this->general_follow_mode >= 2)
    {
      // Ignore "www." at the beginning of the host, because "www.foo.com" is the same host as "foo.com"
      if (preg_replace("#^www\.#", "", $url_parts["host"]) != preg_replace("#^www\.#", "", $this->starting_url_parts["host"]))
        return false;
    }
    
    // Filter URLs leading path-up if wanted
    if ($this->general_follow_mode == 3)
    {
      if ($url_parts["protocol"] != $this->starting_url_parts["protocol"] ||
          preg_replace("#^www\.#", "", $url_parts["host"]) != preg_replace("#^www\.#", "", $this->starting_url_parts["host"]) ||
          substr($url_parts["path"], 0, strlen($this->starting_url_parts["path"])) != $this->starting_url_parts["path"])
      {
        return false;
      }
    }
    
    // Filter URLs by url_filter_rules
    for ($x=0; $x<count($this->url_filter_rules); $x++)
    {
      if (preg_match($this->url_filter_rules[$x], $url->url_rebuild)) return false;
    }
    
    // Filter URLs by url_follow_rules
    if (count($this->url_follow_rules) > 0)
    {
      $match_found = false;
      for ($x=0; $x<count($this->url_follow_rules); $x++)
      {
        if (preg_match($this->url_follow_rules[$x], $url->url_rebuild))
        {
          $match_found = true;
          break;
        }
      }
      
      if ($match_found == false) return false;
    }
    
    return true;
  }
protected function urlMatchesRules(PHPCrawlerURLDescriptor $url)
  { 
    // URL-parts of the URL to check against the filter-rules
    $url_parts = PHPCrawlerUtils::splitURL($url->url_rebuild);
    
    // Kick out all links that r NOT of protocol "http" or "https"
    if ($url_parts["protocol"] != "http://" && $url_parts["protocol"] != "https://")
    {
      return false;
    }
    
    // If meta-tag "robots"->"nofollow" is present and obey_nofollow_tags is TRUE -> always kick out URL
    if ($this->obey_nofollow_tags == true &&
        isset($this->CurrentDocumentInfo->meta_attributes["robots"]) &&
        preg_match("#nofollow# i", $this->CurrentDocumentInfo->meta_attributes["robots"]))
    {
      return false;
    }
    
    // If linkcode contains "rel='nofollow'" and obey_nofollow_tags is TRUE -> always kick out URL
    if ($this->obey_nofollow_tags == true)
    {
      if (preg_match("#^<[^>]*rel\s*=\s*(?|\"\s*nofollow\s*\"|'\s*nofollow\s*'|\s*nofollow\s*)[^>]*>#", $url->linkcode))
      {
        return false;
      }
    }
    
    // Filter URLs to other domains if wanted
    if ($this->general_follow_mode >= 1)
    {
      if ($url_parts["domain"] != $this->starting_url_parts["domain"]) return false;
    }
    
    // Filter URLs to other hosts if wanted
    if ($this->general_follow_mode >= 2)
    {
      // Ignore "www." at the beginning of the host, because "www.foo.com" is the same host as "foo.com"
      if (preg_replace("#^www\.#", "", $url_parts["host"]) != preg_replace("#^www\.#", "", $this->starting_url_parts["host"]))
        return false;
    }
    
    // Filter URLs leading path-up if wanted
    if ($this->general_follow_mode == 3)
    {
      if ($url_parts["protocol"] != $this->starting_url_parts["protocol"] ||
          preg_replace("#^www\.#", "", $url_parts["host"]) != preg_replace("#^www\.#", "", $this->starting_url_parts["host"]) ||
          substr($url_parts["path"], 0, strlen($this->starting_url_parts["path"])) != $this->starting_url_parts["path"])
      {
        return false;
      }
    }
    
    // Filter URLs by url_filter_rules
    for ($x=0; $x<count($this->url_filter_rules); $x++)
    {
      if (preg_match($this->url_filter_rules[$x], $url->url_rebuild)) return false;
    }
    
    // Filter URLs by url_follow_rules
    if (count($this->url_follow_rules) > 0)
    {
      $match_found = false;
      for ($x=0; $x<count($this->url_follow_rules); $x++)
      {
        if (preg_match($this->url_follow_rules[$x], $url->url_rebuild))
        {
          $match_found = true;
          break;
        }
      }
      
      if ($match_found == false) return false;
    }
    
    return true;
  }
Checks whether a given URL matches the rules.

@param string $url  The URL as a PHPCrawlerURLDescriptor-object
@return bool TRUE if the URL matches the defined rules.