libs/PHPCrawler.class.php
protected function processUrl(PHPCrawlerURLDescriptor $UrlDescriptor)
  { 
    PHPCrawlerBenchmark::start("processing_url");
    
    // Setup HTTP-request
    $this->PageRequest->setUrl($UrlDescriptor);
    
    // Add cookies to request
    if ($this->cookie_handling_enabled == true)
      $this->PageRequest->addCookieDescriptors($this->CookieCache->getCookiesForUrl($UrlDescriptor->url_rebuild));
    
    // Add basic-authentications to request
    $authentication = $this->UserSendDataCache->getBasicAuthenticationForUrl($UrlDescriptor->url_rebuild);
    if ($authentication != null)
    {
      $this->PageRequest->setBasicAuthentication($authentication["username"], $authentication["password"]);
    }
    
    // Add post-data to request
    $post_data = $this->UserSendDataCache->getPostDataForUrl($UrlDescriptor->url_rebuild);
    while (list($post_key, $post_value) = @each($post_data))
    {
      $this->PageRequest->addPostData($post_key, $post_value);
    }
    
    // Do request
    $PageInfo = $this->PageRequest->sendRequest();
    
    if ($this->multiprocess_mode != PHPCrawlerMultiProcessModes::MPMODE_PARENT_EXECUTES_USERCODE)
    {
      // Check for abort
      $abort_reason = $this->checkForAbort();
      if ($abort_reason !== null) return true;
      
      $this->ProcessCommunication->updateCrawlerStatus($PageInfo);
    }
    
    // Remove post and cookie-data from request-object
    $this->PageRequest->clearCookies();
    $this->PageRequest->clearPostData();
    
    // Call user-moethods if crawler doesn't run in MPMODE_PARENT_EXECUTES_USERCODE
    if ($this->multiprocess_mode != PHPCrawlerMultiProcessModes::MPMODE_PARENT_EXECUTES_USERCODE)
    {
      // Call the "abstract" method handlePageData
      $user_abort = false;
      $page_info = $PageInfo->toArray();
      $user_return_value = $this->handlePageData($page_info);
      if ($user_return_value < 0) $user_abort = true;
      
      // Call the "abstract" method handleDocumentInfo
      $user_return_value = $this->handleDocumentInfo($PageInfo);
      if ($user_return_value < 0) $user_abort = true;
      
      // Update status if user aborted process
      if ($user_abort == true) 
      {
        $this->ProcessCommunication->updateCrawlerStatus(null, PHPCrawlerAbortReasons::ABORTREASON_USERABORT);
      }
      
      // Check for abort from other processes
      if ($this->checkForAbort() !== null) return true;
    }
    
    // Filter found URLs by defined rules
    if ($this->follow_redirects_till_content == true)
    {
      $crawler_status = $this->ProcessCommunication->getCrawlerStatus();
      
      // If content wasn't found so far and content was found NOW
      if ($crawler_status->first_content_url == null && $PageInfo->http_status_code == 200)
      {
        $this->ProcessCommunication->updateCrawlerStatus(null, null, $PageInfo->url);
        $this->UrlFilter->setBaseURL($PageInfo->url); // Set current page as base-URL
        $this->UrlFilter->filterUrls($PageInfo);
        $this->follow_redirects_till_content = false; // Content was found, so this can be set to FALSE
      }
      else if ($crawler_status->first_content_url == null)
      {
        $this->UrlFilter->keepRedirectUrls($PageInfo); // Content wasn't found so far, so just keep redirect-urls 
      }
      else if ($crawler_status->first_content_url != null)
      {
        $this->follow_redirects_till_content = false;
        $this->UrlFilter->filterUrls($PageInfo);
      }
    }
    else
    {
      $this->UrlFilter->filterUrls($PageInfo);
    }
    
    // Add Cookies to Cookie-cache
    if ($this->cookie_handling_enabled == true) $this->CookieCache->addCookies($PageInfo->cookies);

    // Add filtered links to URL-cache
    $this->LinkCache->addURLs($PageInfo->links_found_url_descriptors);
    
    PHPCrawlerBenchmark::stop("processing_url");
    
    // Complete PageInfo-Object with benchmarks
    $PageInfo->benchmarks = PHPCrawlerBenchmark::getAllBenchmarks();
    
    if ($this->multiprocess_mode == PHPCrawlerMultiProcessModes::MPMODE_PARENT_EXECUTES_USERCODE)
    {
      $this->DocumentInfoQueue->addDocumentInfo($PageInfo);
    }
    
     // Mark URL as "followed"
    $this->LinkCache->markUrlAsFollowed($UrlDescriptor);
    
    PHPCrawlerBenchmark::resetAll(array("crawling_process"));
    
    return false;
  }
protected function processUrl(PHPCrawlerURLDescriptor $UrlDescriptor)
  { 
    PHPCrawlerBenchmark::start("processing_url");
    
    // Setup HTTP-request
    $this->PageRequest->setUrl($UrlDescriptor);
    
    // Add cookies to request
    if ($this->cookie_handling_enabled == true)
      $this->PageRequest->addCookieDescriptors($this->CookieCache->getCookiesForUrl($UrlDescriptor->url_rebuild));
    
    // Add basic-authentications to request
    $authentication = $this->UserSendDataCache->getBasicAuthenticationForUrl($UrlDescriptor->url_rebuild);
    if ($authentication != null)
    {
      $this->PageRequest->setBasicAuthentication($authentication["username"], $authentication["password"]);
    }
    
    // Add post-data to request
    $post_data = $this->UserSendDataCache->getPostDataForUrl($UrlDescriptor->url_rebuild);
    while (list($post_key, $post_value) = @each($post_data))
    {
      $this->PageRequest->addPostData($post_key, $post_value);
    }
    
    // Do request
    $PageInfo = $this->PageRequest->sendRequest();
    
    if ($this->multiprocess_mode != PHPCrawlerMultiProcessModes::MPMODE_PARENT_EXECUTES_USERCODE)
    {
      // Check for abort
      $abort_reason = $this->checkForAbort();
      if ($abort_reason !== null) return true;
      
      $this->ProcessCommunication->updateCrawlerStatus($PageInfo);
    }
    
    // Remove post and cookie-data from request-object
    $this->PageRequest->clearCookies();
    $this->PageRequest->clearPostData();
    
    // Call user-moethods if crawler doesn't run in MPMODE_PARENT_EXECUTES_USERCODE
    if ($this->multiprocess_mode != PHPCrawlerMultiProcessModes::MPMODE_PARENT_EXECUTES_USERCODE)
    {
      // Call the "abstract" method handlePageData
      $user_abort = false;
      $page_info = $PageInfo->toArray();
      $user_return_value = $this->handlePageData($page_info);
      if ($user_return_value < 0) $user_abort = true;
      
      // Call the "abstract" method handleDocumentInfo
      $user_return_value = $this->handleDocumentInfo($PageInfo);
      if ($user_return_value < 0) $user_abort = true;
      
      // Update status if user aborted process
      if ($user_abort == true) 
      {
        $this->ProcessCommunication->updateCrawlerStatus(null, PHPCrawlerAbortReasons::ABORTREASON_USERABORT);
      }
      
      // Check for abort from other processes
      if ($this->checkForAbort() !== null) return true;
    }
    
    // Filter found URLs by defined rules
    if ($this->follow_redirects_till_content == true)
    {
      $crawler_status = $this->ProcessCommunication->getCrawlerStatus();
      
      // If content wasn't found so far and content was found NOW
      if ($crawler_status->first_content_url == null && $PageInfo->http_status_code == 200)
      {
        $this->ProcessCommunication->updateCrawlerStatus(null, null, $PageInfo->url);
        $this->UrlFilter->setBaseURL($PageInfo->url); // Set current page as base-URL
        $this->UrlFilter->filterUrls($PageInfo);
        $this->follow_redirects_till_content = false; // Content was found, so this can be set to FALSE
      }
      else if ($crawler_status->first_content_url == null)
      {
        $this->UrlFilter->keepRedirectUrls($PageInfo); // Content wasn't found so far, so just keep redirect-urls 
      }
      else if ($crawler_status->first_content_url != null)
      {
        $this->follow_redirects_till_content = false;
        $this->UrlFilter->filterUrls($PageInfo);
      }
    }
    else
    {
      $this->UrlFilter->filterUrls($PageInfo);
    }
    
    // Add Cookies to Cookie-cache
    if ($this->cookie_handling_enabled == true) $this->CookieCache->addCookies($PageInfo->cookies);

    // Add filtered links to URL-cache
    $this->LinkCache->addURLs($PageInfo->links_found_url_descriptors);
    
    PHPCrawlerBenchmark::stop("processing_url");
    
    // Complete PageInfo-Object with benchmarks
    $PageInfo->benchmarks = PHPCrawlerBenchmark::getAllBenchmarks();
    
    if ($this->multiprocess_mode == PHPCrawlerMultiProcessModes::MPMODE_PARENT_EXECUTES_USERCODE)
    {
      $this->DocumentInfoQueue->addDocumentInfo($PageInfo);
    }
    
     // Mark URL as "followed"
    $this->LinkCache->markUrlAsFollowed($UrlDescriptor);
    
    PHPCrawlerBenchmark::resetAll(array("crawling_process"));
    
    return false;
  }
Receives and processes the given URL

@param PHPCrawlerURLDescriptor $UrlDescriptor The URL as PHPCrawlerURLDescriptor-object
@return bool TURE if the crawling-process should be aborted after processig the URL, otherwise FALSE.