libs/PHPCrawlerRobotsTxtParser.class.php
protected function buildRegExpressions(&$applying_lines, $base_url)
  { 
    // First, get all "Disallow:"-pathes
    $disallow_pathes = array();
    for ($x=0; $x<count($applying_lines); $x++)
    {
      if (preg_match("#^Disallow:# i", $applying_lines[$x]))
      {
        preg_match("#^Disallow:[ ]*(.*)#", $applying_lines[$x], $match);
        $disallow_pathes[] = trim($match[1]);
      }
    }
    
    // Works like this:
    // The base-url is http://www.foo.com.
    // The driective says: "Disallow: /bla/"
    // This means: The nonFollowMatch is "#^http://www\.foo\.com/bla/#"
    
    $normalized_base_url = PHPCrawlerUtils::normalizeURL($base_url);
    
    $non_follow_expressions = array();
    
    for ($x=0; $x<count($disallow_pathes); $x++)
    {
      // If the disallow-path is empty -> simply ignore it
      if ($disallow_pathes[$x] == "") continue;
      
      $non_follow_path_complpete = $normalized_base_url.substr($disallow_pathes[$x], 1); // "http://www.foo.com/bla/"
      $non_follow_exp = preg_quote($non_follow_path_complpete, "#"); // "http://www\.foo\.com/bla/"
      $non_follow_exp = "#^".$non_follow_exp."#"; // "#^http://www\.foo\.com/bla/#"
        
      $non_follow_expressions[] = $non_follow_exp;
    }
    
    return $non_follow_expressions;
  }
protected function buildRegExpressions(&$applying_lines, $base_url)
  { 
    // First, get all "Disallow:"-pathes
    $disallow_pathes = array();
    for ($x=0; $x<count($applying_lines); $x++)
    {
      if (preg_match("#^Disallow:# i", $applying_lines[$x]))
      {
        preg_match("#^Disallow:[ ]*(.*)#", $applying_lines[$x], $match);
        $disallow_pathes[] = trim($match[1]);
      }
    }
    
    // Works like this:
    // The base-url is http://www.foo.com.
    // The driective says: "Disallow: /bla/"
    // This means: The nonFollowMatch is "#^http://www\.foo\.com/bla/#"
    
    $normalized_base_url = PHPCrawlerUtils::normalizeURL($base_url);
    
    $non_follow_expressions = array();
    
    for ($x=0; $x<count($disallow_pathes); $x++)
    {
      // If the disallow-path is empty -> simply ignore it
      if ($disallow_pathes[$x] == "") continue;
      
      $non_follow_path_complpete = $normalized_base_url.substr($disallow_pathes[$x], 1); // "http://www.foo.com/bla/"
      $non_follow_exp = preg_quote($non_follow_path_complpete, "#"); // "http://www\.foo\.com/bla/"
      $non_follow_exp = "#^".$non_follow_exp."#"; // "#^http://www\.foo\.com/bla/#"
        
      $non_follow_expressions[] = $non_follow_exp;
    }
    
    return $non_follow_expressions;
  }
Returns an array containig regular-expressions corresponding
to the given robots.txt-style "Disallow"-lines

@param array &$applying_lines Numeric array containing "disallow"-lines.
@param string $base_url       Base-URL the robots.txt-file was found in.

@return array  Numeric array containing regular-expresseions created for each "disallow"-line.