core/src/main/java/com/digitalpebble/stormcrawler/indexing/AbstractIndexerBolt.java
protected boolean filterDocument(Metadata meta) {
        String noindexVal = meta.getFirstValue(RobotsTags.ROBOTS_NO_INDEX);
        if ("true".equalsIgnoreCase(noindexVal))
            return false;

        if (filterKeyValue == null)
            return true;
        String[] values = meta.getValues(filterKeyValue[0]);
        // key not found
        if (values == null)
            return false;
        return ArrayUtils.contains(values, filterKeyValue[1]);
    }
protected boolean filterDocument(Metadata meta) {
        String noindexVal = meta.getFirstValue(RobotsTags.ROBOTS_NO_INDEX);
        if ("true".equalsIgnoreCase(noindexVal))
            return false;

        if (filterKeyValue == null)
            return true;
        String[] values = meta.getValues(filterKeyValue[0]);
        // key not found
        if (values == null)
            return false;
        return ArrayUtils.contains(values, filterKeyValue[1]);
    }
Determine whether a document should be indexed based on the presence of a
given key/value or the RobotsTags.ROBOTS_NO_INDEX directive.

@return true if the document should be kept.