lib/input-generators/robots.js
function getRobotsUrl (options, callback) {
  request({
    url: options.source,
    timeout: options.timeout()
  }, function (err, res, body) {
    var error = err || common.checkResponse(res, "text/plain");

    if (error) {
      callback(common.prependMsgToErr(error, options.source, true));
    } else {
      body.toString().split('\n').every(function(line) {
        // Process the line input, but break if base.input returns false.
        // For now, this can only happen if no outputDir is defined,
        //   which is a fatal bad option problem and will happen immediately.
        if (!oneline(line, options)) {
          error = common.prependMsgToErr(base.generatorError(), line, true);
          return false;
        }
        return true;
      });

      callback(error);
    }
  });
}
function getRobotsUrl (options, callback) {
  request({
    url: options.source,
    timeout: options.timeout()
  }, function (err, res, body) {
    var error = err || common.checkResponse(res, "text/plain");

    if (error) {
      callback(common.prependMsgToErr(error, options.source, true));
    } else {
      body.toString().split('\n').every(function(line) {
        // Process the line input, but break if base.input returns false.
        // For now, this can only happen if no outputDir is defined,
        //   which is a fatal bad option problem and will happen immediately.
        if (!oneline(line, options)) {
          error = common.prependMsgToErr(base.generatorError(), line, true);
          return false;
        }
        return true;
      });

      callback(error);
    }
  });
}
Retrieves robots.txt from url and parses it.

@param {Object} options - Robots.txt options
@param {String} options.source - The URL to a robots.txt
@param {Function} options.timeout - Returns the robots.txt request timeout.
@param {Function} callback - A completion callback.