core/src/main/java/com/digitalpebble/stormcrawler/protocol/RobotRulesParser.java
public void setConf(Config conf) {

        // Grab the agent names we advertise to robots files.
        String agentName = ConfUtils.getString(conf, "http.agent.name");
        if (null == agentName) {
            throw new RuntimeException("Agent name not configured!");
        }

        String configuredAgentNames = ConfUtils.getString(conf,
                "http.robots.agents", "");
        StringTokenizer tok = new StringTokenizer(configuredAgentNames, ",");
        ArrayList<String> agents = new ArrayList<>();
        while (tok.hasMoreTokens()) {
            agents.add(tok.nextToken().trim());
        }

        /**
         * If there are no agents for robots-parsing, use the default
         * agent-string. If both are present, our agent-string should be the
         * first one we advertise to robots-parsing.
         */
        if (agents.isEmpty()) {
            LOG.info(
                    "No agents listed in 'http.robots.agents' property! Using http.agent.name [{}]",
                    agentName);
            this.agentNames = agentName;
        } else {
            int index = 0;
            if ((agents.get(0)).equalsIgnoreCase(agentName)) {
                index++;
            } else {
                LOG.info(
                        "Agent we advertise ({}) not listed first in 'http.robots.agents' property!",
                        agentName);
            }

            StringBuilder combinedAgentsString = new StringBuilder(agentName);
            // append all the agents from the http.robots.agents property
            for (; index < agents.size(); index++) {
                combinedAgentsString.append(", ").append(agents.get(index));
            }

            this.agentNames = combinedAgentsString.toString();
        }

        String spec = ConfUtils.getString(conf, cacheConfigParamName,
                "maximumSize=10000,expireAfterWrite=6h");
        CACHE = CacheBuilder.from(spec).build();

        spec = ConfUtils.getString(conf, errorcacheConfigParamName,
                "maximumSize=10000,expireAfterWrite=1h");
        ERRORCACHE = CacheBuilder.from(spec).build();
    }
public void setConf(Config conf) {

        // Grab the agent names we advertise to robots files.
        String agentName = ConfUtils.getString(conf, "http.agent.name");
        if (null == agentName) {
            throw new RuntimeException("Agent name not configured!");
        }

        String configuredAgentNames = ConfUtils.getString(conf,
                "http.robots.agents", "");
        StringTokenizer tok = new StringTokenizer(configuredAgentNames, ",");
        ArrayList<String> agents = new ArrayList<>();
        while (tok.hasMoreTokens()) {
            agents.add(tok.nextToken().trim());
        }

        /**
         * If there are no agents for robots-parsing, use the default
         * agent-string. If both are present, our agent-string should be the
         * first one we advertise to robots-parsing.
         */
        if (agents.isEmpty()) {
            LOG.info(
                    "No agents listed in 'http.robots.agents' property! Using http.agent.name [{}]",
                    agentName);
            this.agentNames = agentName;
        } else {
            int index = 0;
            if ((agents.get(0)).equalsIgnoreCase(agentName)) {
                index++;
            } else {
                LOG.info(
                        "Agent we advertise ({}) not listed first in 'http.robots.agents' property!",
                        agentName);
            }

            StringBuilder combinedAgentsString = new StringBuilder(agentName);
            // append all the agents from the http.robots.agents property
            for (; index < agents.size(); index++) {
                combinedAgentsString.append(", ").append(agents.get(index));
            }

            this.agentNames = combinedAgentsString.toString();
        }

        String spec = ConfUtils.getString(conf, cacheConfigParamName,
                "maximumSize=10000,expireAfterWrite=6h");
        CACHE = CacheBuilder.from(spec).build();

        spec = ConfUtils.getString(conf, errorcacheConfigParamName,
                "maximumSize=10000,expireAfterWrite=1h");
        ERRORCACHE = CacheBuilder.from(spec).build();
    }
Set the {@link Configuration} object