title: Scraping multiple websites of different domains using python scrapy
body: <p>I want to scrape two different websites' emails with their corresponding links, but I get two different emails with the same link. Actually there are vaious websites to scrape, but for simplicity, I have used only two urls. The code is given below:</p>
  <pre><code>import scrapy
  import re
  import time
  urls = ['http://www.manorhouseohio.com', 'http://www.OtterCreekAL.com']
  class TheknotSpider(scrapy.Spider):
      name = 'theknot'
      def start_requests(self):
          global url
          li = ['http://www.manorhouseohio.com','http://www.OtterCreekAL.com']
          for url in range(len(li)):
              yield scrapy.Request(li[url], callback=self.parse)
      def parse(self, response):
          mail_link = response.xpath(&quot;//a[contains(@href,'mailto:')]/@href&quot;).get()
          html_text = response.xpath(&quot;//body&quot;).get()
          mail_without_link = re.findall(r&quot;[a-z0-9\.\-+_]+@[a-z0-9\.\-+_]+\.[a-z]+&quot;, html_text)
          contact = response.xpath(&quot;//a[contains(@href,'/contact')]/@href&quot;).get()
          if mail_link:
              x = mail_link[7:]
              yield {
              &quot;Main Website&quot;: urls[url],
              &quot;Email&quot;: x
              }
              time.sleep(1)
          elif mail_without_link:
              yield {
              &quot;Main Website&quot;: urls[url],
              &quot;Email&quot;: mail_without_link
              }
              time.sleep(1)
          elif contact:
                  if contact==&quot;/contact&quot; or contact==&quot;/contact/&quot; or contact==&quot;/contact-us&quot; or contact==&quot;/contact-us/&quot;:
                      contact_main = urls[url] + contact
                      yield scrapy.Request(contact_main, callback=self.parse_final)       
                  else:
                      yield scrapy.Request(contact, callback=self.parse_final)
          else:
              pass
      def parse_final(self, response):
          mail = response.xpath(&quot;//a[contains(@href,'mailto:')]/@href&quot;).get()
          text = response.xpath(&quot;//body&quot;).get()
          mail_text = re.findall(r&quot;[a-z0-9\.\-+_]+@[a-z0-9\.\-+_]+\.[a-z]+&quot;, text)
          if mail:
              x = mail[7:]
              yield {
              &quot;Main Website&quot;: urls[url],
              &quot;Email&quot;: x
              }
              time.sleep(1)
          elif mail_text:
              yield {
              &quot;Main Website&quot;: urls[url],
              &quot;Email&quot;: mail_text
              }
              time.sleep(1)
          else: pass
  </code></pre>