title: scrapy not following links
body: <p>The following scrapy code for returning medical treatment information does return the first set of returned results, but does not follow links. Learning code and checked similar results here on stackoverflow, but integrating them did not work. True, I'm learning. Any pointers would be appreciated.  </p>
  
  <pre><code>import urlparse
  
  from scrapy.spider import BaseSpider
  from scrapy.selector import Selector
  from scrapy.http import Request
  import w3lib.url
  
  from yelp.items import YelpItem
  
  
  class YelpSpider(BaseSpider):
      name = "yelp"
      download_delay = 10
      concurrent_requests = 1
      concurrent_requests_per_domain = 1
      allowed_domains = ["yelp.com"]
      start_urls = ["http://www.yelp.com/search?find_desc=cancer+treatment&amp;find_loc=manhattan%2Cny&amp;start=0",
  "http://www.yelp.com/search?find_desc=cancer+treatment&amp;find_loc=manhattan%2Cny&amp;start=20",
  "http://www.yelp.com/search?find_desc=cancer+treatment&amp;find_loc=manhattan%2Cny&amp;start=30"]
  
      def parse(self, response):
          selector = Selector(response)
          for title in selector.css("span.indexed-biz-name"):
              page_url = urlparse.urljoin(response.url,
                                          title.xpath("a/@href").extract()[0])
              self.log("page URL: %s" % page_url)
              #continue
              yield Request(page_url,
                            callback=self.parse_page)
  
          for next_page in selector.css(u'ul &gt; li &gt; a.prev-next:contains(\u2192)'):
              next_url = urlparse.urljoin(response.url,
                                          next_page.xpath('@href').extract()[0])
              self.log("next URL: %s" % next_url)
              #continue
              yield Request(next_url,
                            callback=self.parse)
  
      def parse_page(self, response):
          selector = Selector(response)
          item = YelpItem()
          item["name"] = selector.xpath('.//h1[@itemprop="name"]/text()').extract()[0].strip()
          item["addresslocality"] = u"\n".join(
              selector.xpath('.//address[@itemprop="address"]//text()').extract()).strip()
          item["link"] = response.url
          website = selector.css('div.biz-website a')
          if website:
              website_url = website.xpath('@href').extract()[0]
              item["website"] = w3lib.url.url_query_parameter(website_url, "url")
          return item
  </code></pre>