libs/PHPCrawlerRobotsTxtParser.class.php
protected function getRobotsTxtContent(PHPCrawlerURLDescriptor $Url)
  {
    // Request robots-txt
    $this->PageRequest->setUrl($Url);
    $PageInfo = $this->PageRequest->sendRequest();

    // Return content of the robots.txt-file if it was found, otherwie
    // reutrn NULL
    if ($PageInfo->http_status_code == 200)
    {
      return $PageInfo->content;
    }
    else
    {
      return null;
    }
  }
protected function getRobotsTxtContent(PHPCrawlerURLDescriptor $Url)
  {
    // Request robots-txt
    $this->PageRequest->setUrl($Url);
    $PageInfo = $this->PageRequest->sendRequest();

    // Return content of the robots.txt-file if it was found, otherwie
    // reutrn NULL
    if ($PageInfo->http_status_code == 200)
    {
      return $PageInfo->content;
    }
    else
    {
      return null;
    }
  }
Retreives the content of a robots.txt-file

@param PHPCrawlerURLDescriptor $Url The URL of the robots.txt-file
@return string The content of the robots.txt or NULL if no robots.txt was found.