libs/PHPCrawlerRobotsTxtParser.class.php
public function parseRobotsTxt(PHPCrawlerURLDescriptor $Url, $user_agent_string)
  {
    PHPCrawlerBenchmark::start("processing_robotstxt");
    
    // URL of robots-txt
    $RobotsTxtUrl = self::getRobotsTxtURL($Url);
    
    // Get robots.txt-content related to the given URL
    $robots_txt_content = $this->getRobotsTxtContent($RobotsTxtUrl);
    
    $non_follow_reg_exps = array();
    
    // If content was found
    if ($robots_txt_content != null)
    {
      // Get all lines in the robots.txt-content that are adressed to our user-agent.
      $applying_lines = $this->getApplyingLines($robots_txt_content, $user_agent_string);
      
      // Get valid reg-expressions for the given disallow-pathes.
      $non_follow_reg_exps = $this->buildRegExpressions($applying_lines, PHPCrawlerUtils::getRootUrl($Url->url_rebuild));
    }
    
    PHPCrawlerBenchmark::stop("processing_robots.txt");
    
    return $non_follow_reg_exps;
  }
public function parseRobotsTxt(PHPCrawlerURLDescriptor $Url, $user_agent_string)
  {
    PHPCrawlerBenchmark::start("processing_robotstxt");
    
    // URL of robots-txt
    $RobotsTxtUrl = self::getRobotsTxtURL($Url);
    
    // Get robots.txt-content related to the given URL
    $robots_txt_content = $this->getRobotsTxtContent($RobotsTxtUrl);
    
    $non_follow_reg_exps = array();
    
    // If content was found
    if ($robots_txt_content != null)
    {
      // Get all lines in the robots.txt-content that are adressed to our user-agent.
      $applying_lines = $this->getApplyingLines($robots_txt_content, $user_agent_string);
      
      // Get valid reg-expressions for the given disallow-pathes.
      $non_follow_reg_exps = $this->buildRegExpressions($applying_lines, PHPCrawlerUtils::getRootUrl($Url->url_rebuild));
    }
    
    PHPCrawlerBenchmark::stop("processing_robots.txt");
    
    return $non_follow_reg_exps;
  }
Parses the robots.txt-file related to the given URL and returns regular-expression-rules
corresponding to the containing "disallow"-rules that are adressed to the given user-agent.

@param PHPCrawlerURLDescriptor $Url The URL
@param string $user_agent_string User-agent.

@return array Numeric array containing regular-expressions for each "disallow"-rule defined in the robots.txt-file
that's adressed to the given user-agent.