title: How to add server-side delay to Javascript for loop?
body: <p>I'm fiddling around with using Node.js to scrape data from an e-commerce site. I use <code>Request</code> to retrieve the DOM of the page and <code>Cheerio</code> to do server-side DOM selection.</p>
  
  <pre><code>const cheerio = require('cheerio');
  const request = require('request');
  
  // takes a URL, scrapes the page, and returns an object with the data
  let scrapePage = (url) =&gt; {
  
      return new Promise((resolve, reject) =&gt; {
  
          request(url, (error, resp, body) =&gt; {
  
              if(error){
                  reject(error);
              };
  
              let $ = cheerio.load(body); 
              let $url = url;
              let $price = $('#rt-mainbody &gt; div &gt; div.details &gt; div.Data &gt; div:nth-child(4) &gt; div.description').text();
  
              let obj = {
                  url: $url,
                  price: $price
              }
  
              resolve(obj);
  
          });
  
      });
  
  };
  
  // Runs scrapePage in a loop
  // There is a variable called arrayOfURLs defined elsewhere that contains 100s of URLs
  
  for( let i = 0; i &lt; arrayOfURLs.length; i++){
      scrapePage(arrayOfURLs[i])
          .then((obj) =&gt; {
              //write to a file
          })
          .catch((error) =&gt; {
          })
  };
  </code></pre>
  
  <p>The problem is that the server that I send requests to sometimes sends back blank data, I'm assuming because I'm sending too many requests without any kind of pause. Due to the async nature of JS I'm having a hard time figuring out how to add an effective delay between each iteration of the loop. It's not enough to just add a <code>setTimeOut</code> in a synchronous fashion because <code>setTimeOut</code> itself is async, and I'm running this on the server so there's no <code>Window</code> object.</p>
  
  <p><strong>EDIT</strong></p>
  
  <p>The code above is a simplified version of what I'm working on. The entire code is this:</p>
  
  <p><code>app.js</code></p>
  
  <pre><code>const fs = require('fs');
  const path = 'urls.txt';
  const path2 = 'results.txt';
  const scraper = require('./scraper');
  
  let scrapePage = (url) =&gt; {
      scraper.scrapePage(url)
          .then((obj) =&gt; {
              // console.log('obj from the scraper with Promises was received');
              // console.log(obj);
              // console.log('writing obj to a file');
              fs.appendFile(path2, JSON.stringify(obj) + ', ', (error) =&gt; {
                  if(error){
                      console.log(error);
                  } else {
                      // console.log('Successfully wrote to ' + path2);
                  }
              })
          })
          .catch((error) =&gt; {
              console.log('There was an error scraping obj: ');
              console.log(error);
          })  
  }
  
  fs.readFile(path, 'utf8', (err, data) =&gt; {
  
    if (err){
      throw err;
    };
  
    var urlArray = JSON.parse(data);
  
    // this returns an Unexpected Identifier error    
    // const results = await Promise.all(urlArray.map(scrapePage));
  
    // this returns an Unexpected Token Function error
    // async function scrapePages(){
    //    const results = await Promise.all(urlArray.map(scrapePage));
    // };
  
  });
  </code></pre>
  
  <p><code>scraper.js</code></p>
  
  <pre><code>const request = require('request');
  const cheerio = require('cheerio');
  
  exports.scrapePage = (url) =&gt; {
      return new Promise((resolve, reject) =&gt; {
          request(url, (error, resp, body) =&gt; {
              if(error){
                  reject(error);
              };
  
              let $ = cheerio.load(body); 
              let $url = url;
  
              let $price = $('#rt-mainbody &gt; div &gt; div.details &gt; div.itemData &gt; div:nth-child(4) &gt; div.description').text();
  
              let obj = {
                  url: $url,
                  price: $price
              }
  
              resolve(obj);
  
          })
      })
  }
  </code></pre>