lib/input-generators/robots.js
function getRobotsFile (options, callback) {
  fs.readFile(options.source, function (err, data) {
    if (!err) {
      data.toString().split('\n').every(function (line) {
        // Process the line input, but break if base.input returns false.
        // For now, this can only happen if no outputDir is defined,
        //   which is a fatal bad option problem and will happen immediately.
        if (!oneline(line, options)) {
          err = common.prependMsgToErr(base.generatorError(), line, true);
          return false;
        }
        return true;
      });
    }

    callback(err);
  });
}
function getRobotsFile (options, callback) {
  fs.readFile(options.source, function (err, data) {
    if (!err) {
      data.toString().split('\n').every(function (line) {
        // Process the line input, but break if base.input returns false.
        // For now, this can only happen if no outputDir is defined,
        //   which is a fatal bad option problem and will happen immediately.
        if (!oneline(line, options)) {
          err = common.prependMsgToErr(base.generatorError(), line, true);
          return false;
        }
        return true;
      });
    }

    callback(err);
  });
}
Reads the robots.txt file and parses it.