external/aws/src/main/java/com/digitalpebble/stormcrawler/aws/bolt/CloudSearchUtils.java
public static String getID(String url) {

        // the document needs an ID
        // see
        // http://docs.aws.amazon.com/cloudsearch/latest/developerguide/preparing-data.html#creating-document-batches
        // A unique ID for the document. A document ID can contain any
        // letter or number and the following characters: _ - = # ; : / ? @
        // &. Document IDs must be at least 1 and no more than 128
        // characters long.
        byte[] dig = digester.digest(url.getBytes(StandardCharsets.UTF_8));
        String ID = Hex.encodeHexString(dig);
        // is that even possible?
        if (ID.length() > 128) {
            throw new RuntimeException("ID larger than max 128 chars");
        }
        return ID;
    }
public static String getID(String url) {

        // the document needs an ID
        // see
        // http://docs.aws.amazon.com/cloudsearch/latest/developerguide/preparing-data.html#creating-document-batches
        // A unique ID for the document. A document ID can contain any
        // letter or number and the following characters: _ - = # ; : / ? @
        // &. Document IDs must be at least 1 and no more than 128
        // characters long.
        byte[] dig = digester.digest(url.getBytes(StandardCharsets.UTF_8));
        String ID = Hex.encodeHexString(dig);
        // is that even possible?
        if (ID.length() > 128) {
            throw new RuntimeException("ID larger than max 128 chars");
        }
        return ID;
    }
Returns a normalised doc ID based on the URL of a document *