libs/PHPCrawlerRobotsTxtParser.class.php
protected function getApplyingLines(&$robots_txt_content, $user_agent_string)
  {
    // Split the content into its lines
    $robotstxt_lines = explode("\n", $robots_txt_content);
    
    // Flag that will get TRUE if the loop over the lines gets
    // into a section that applies to our user_agent_string 
    $matching_section = false;
    
    // Flag that indicats if the loop is in a "agent-define-section"
    // (the parts/blocks that contain the "User-agent"-lines.)
    $agent_define_section = false;
    
    // Flag that indicates if we have found a section that fits to our
    // User-agent
    $matching_section_found = false;
    
    // Array to collect all the lines that applie to our user_agent
    $applying_lines = array();
    
    // Loop over the lines
    $cnt = count($robotstxt_lines);
    for ($x=0; $x<$cnt; $x++)
    {
      $robotstxt_lines[$x] = trim($robotstxt_lines[$x]);
      
      // Check if a line begins with "User-agent"
      if (preg_match("#^User-agent:# i", $robotstxt_lines[$x]))
      {
        // If a new "user-agent" section begins -> reset matching_section-flag
        if ($agent_define_section == false)
        {
          $matching_section = false;
        }
        
        $agent_define_section = true; // Now we are in an agent-define-section
        
        // The user-agent specified in the "User-agent"-line
        preg_match("#^User-agent:[ ]*(.*)$# i", $robotstxt_lines[$x], $match);
        $user_agent_section = trim($match[1]);
        
        // if the specified user-agent in the line fits to our user-agent-String (* fits always)
        // -> switch the flag "matching_section" to true
        if ($user_agent_section == "*" || preg_match("#^".preg_quote($user_agent_section)."# i", $user_agent_string))
        {
          $matching_section = true;
          $matching_section_found = true;
        }
        
        continue; // Don't do anything else with the "User-agent"-lines, just go on
      }
      else
      {
        // We are not in an agent-define-section (anymore)
        $agent_define_section = false;
      }
      
      // If we are in a section that applies to our user_agent
      // -> store the line.
      if ($matching_section == true)
      {
        $applying_lines[] = $robotstxt_lines[$x];
      }
      
      // If we are NOT in a matching section (anymore) AND we've already found
      // and parsed a matching section -> stop looking further (thats what RFC says)
      if ($matching_section == false && $matching_section_found == true)
      {
        // break;
      }
    }
    
    return $applying_lines;
  }
protected function getApplyingLines(&$robots_txt_content, $user_agent_string)
  {
    // Split the content into its lines
    $robotstxt_lines = explode("\n", $robots_txt_content);
    
    // Flag that will get TRUE if the loop over the lines gets
    // into a section that applies to our user_agent_string 
    $matching_section = false;
    
    // Flag that indicats if the loop is in a "agent-define-section"
    // (the parts/blocks that contain the "User-agent"-lines.)
    $agent_define_section = false;
    
    // Flag that indicates if we have found a section that fits to our
    // User-agent
    $matching_section_found = false;
    
    // Array to collect all the lines that applie to our user_agent
    $applying_lines = array();
    
    // Loop over the lines
    $cnt = count($robotstxt_lines);
    for ($x=0; $x<$cnt; $x++)
    {
      $robotstxt_lines[$x] = trim($robotstxt_lines[$x]);
      
      // Check if a line begins with "User-agent"
      if (preg_match("#^User-agent:# i", $robotstxt_lines[$x]))
      {
        // If a new "user-agent" section begins -> reset matching_section-flag
        if ($agent_define_section == false)
        {
          $matching_section = false;
        }
        
        $agent_define_section = true; // Now we are in an agent-define-section
        
        // The user-agent specified in the "User-agent"-line
        preg_match("#^User-agent:[ ]*(.*)$# i", $robotstxt_lines[$x], $match);
        $user_agent_section = trim($match[1]);
        
        // if the specified user-agent in the line fits to our user-agent-String (* fits always)
        // -> switch the flag "matching_section" to true
        if ($user_agent_section == "*" || preg_match("#^".preg_quote($user_agent_section)."# i", $user_agent_string))
        {
          $matching_section = true;
          $matching_section_found = true;
        }
        
        continue; // Don't do anything else with the "User-agent"-lines, just go on
      }
      else
      {
        // We are not in an agent-define-section (anymore)
        $agent_define_section = false;
      }
      
      // If we are in a section that applies to our user_agent
      // -> store the line.
      if ($matching_section == true)
      {
        $applying_lines[] = $robotstxt_lines[$x];
      }
      
      // If we are NOT in a matching section (anymore) AND we've already found
      // and parsed a matching section -> stop looking further (thats what RFC says)
      if ($matching_section == false && $matching_section_found == true)
      {
        // break;
      }
    }
    
    return $applying_lines;
  }
Function returns all RAW lines in the given robots.txt-content that apply to
the given useragent-string.

@return array Numeric array with found lines