libs/PHPCrawlerRobotsTxtParser.class.php
public static function getRobotsTxtURL(PHPCrawlerURLDescriptor $Url)
  {
    $url_parts = PHPCrawlerUtils::splitURL($Url->url_rebuild); 
    $robots_txt_url = $url_parts["protocol"].$url_parts["host"].":".$url_parts["port"] . "/robots.txt";
    
    return new PHPCrawlerURLDescriptor($robots_txt_url);
  }
public static function getRobotsTxtURL(PHPCrawlerURLDescriptor $Url)
  {
    $url_parts = PHPCrawlerUtils::splitURL($Url->url_rebuild); 
    $robots_txt_url = $url_parts["protocol"].$url_parts["host"].":".$url_parts["port"] . "/robots.txt";
    
    return new PHPCrawlerURLDescriptor($robots_txt_url);
  }
Returns the Robots.txt-URL related to the given URL

@param PHPCrawlerURLDescriptor $Url  The URL as PHPCrawlerURLDescriptor-object
@return PHPCrawlerURLDescriptor Url of the related to the passed URL.