lib/input-generators/robots.js
function generateInput (options) {
  return nodeCall(
    common.isUrl(options.source) ? getRobotsUrl : getRobotsFile,
    options
  )
    .catch(function (err) {
      options._abort(err);
    })
    .then(function () {
      base.EOI(robots);
    });
}
function generateInput (options) {
  return nodeCall(
    common.isUrl(options.source) ? getRobotsUrl : getRobotsFile,
    options
  )
    .catch(function (err) {
      options._abort(err);
    })
    .then(function () {
      base.EOI(robots);
    });
}
Generate the snapshot arguments from a robots.txt file.
Each line that has "Allow:" contains a url we need a snapshot for.
This can return true on error for true async. An async error is supplied to listener
in this case via _abort.

@param {Object} options - input generator options object.
@param {String} options.source - A url or file path.
@param {Function} options._abort - Abort function to stop the async notifier.
@returns {Promise} Resolves to undefined on completion.