core/src/main/java/com/digitalpebble/stormcrawler/util/RobotsTags.java
public void normaliseToMetadata(Metadata metadata) {
        metadata.setValue(ROBOTS_NO_INDEX, Boolean.toString(noIndex));
        metadata.setValue(ROBOTS_NO_CACHE, Boolean.toString(noCache));
        metadata.setValue(ROBOTS_NO_FOLLOW, Boolean.toString(noFollow));
    }
public void normaliseToMetadata(Metadata metadata) {
        metadata.setValue(ROBOTS_NO_INDEX, Boolean.toString(noIndex));
        metadata.setValue(ROBOTS_NO_CACHE, Boolean.toString(noCache));
        metadata.setValue(ROBOTS_NO_FOLLOW, Boolean.toString(noFollow));
    }
Adds a normalised representation of the directives in the metadata *