external/warc/src/main/java/com/digitalpebble/stormcrawler/warc/WARCRecordFormat.java
public static String fixHttpHeaders(String headers, int contentLength) {
        int start = 0, lineEnd = 0, last = 0, trailingCrLf = 0;
        StringBuilder replace = new StringBuilder();
        while (start < headers.length()) {
            lineEnd = headers.indexOf(CRLF, start);
            trailingCrLf = 1;
            if (lineEnd == -1) {
                lineEnd = headers.length();
                trailingCrLf = 0;
            }
            int colonPos = -1;
            for (int i = start; i < lineEnd; i++) {
                if (headers.charAt(i) == ':') {
                    colonPos = i;
                    break;
                }
            }
            if (colonPos == -1) {
                boolean valid = true;
                if (start == 0) {
                    // status line (without colon)
                } else if ((lineEnd + 4) == headers.length()
                        && headers.endsWith(CRLF + CRLF)) {
                    // ok, trailing empty line
                    trailingCrLf = 2;
                } else if (start == lineEnd) {
                    // skip/remove empty line
                    valid = false;
                } else {
                    LOG.warn("Invalid header line: {}",
                            headers.substring(start, lineEnd));
                    valid = false;
                }
                if (!valid) {
                    if (last < start) {
                        replace.append(headers.substring(last, start));
                    }
                    last = lineEnd + 2 * trailingCrLf;
                }
                start = lineEnd + 2 * trailingCrLf;
                /*
                 * skip over invalid header line, no further check for
                 * problematic headers required
                 */
                continue;
            }
            String name = headers.substring(start, colonPos);
            if (PROBLEMATIC_HEADERS.matcher(name).matches()) {
                boolean needsFix = true;
                if (name.equalsIgnoreCase("content-length")) {
                    String value = headers.substring(colonPos + 1, lineEnd)
                            .trim();
                    try {
                        int l = Integer.parseInt(value);
                        if (l == contentLength) {
                            needsFix = false;
                        }
                    } catch (NumberFormatException e) {
                        // needs to be fixed
                    }
                }
                if (needsFix) {
                    if (last < start) {
                        replace.append(headers.substring(last, start));
                    }
                    last = lineEnd + 2 * trailingCrLf;
                    replace.append(X_HIDE_HEADER)
                            .append(headers.substring(start, lineEnd + 2
                                    * trailingCrLf));
                    if (trailingCrLf == 0) {
                        replace.append(CRLF);
                        trailingCrLf = 1;
                    }
                    if (name.equalsIgnoreCase("content-length")) {
                        // add effective uncompressed and unchunked length of
                        // content
                        replace.append("Content-Length").append(": ")
                                .append(contentLength).append(CRLF);
                    }
                }
            }
            start = lineEnd + 2 * trailingCrLf;
        }
        if (last > 0 || trailingCrLf != 2) {
            if (last < headers.length()) {
                // append trailing headers
                replace.append(headers.substring(last));
            }
            while (trailingCrLf < 2) {
                replace.append(CRLF);
                trailingCrLf++;
            }
            return replace.toString();
        }
        return headers;
    }
public static String fixHttpHeaders(String headers, int contentLength) {
        int start = 0, lineEnd = 0, last = 0, trailingCrLf = 0;
        StringBuilder replace = new StringBuilder();
        while (start < headers.length()) {
            lineEnd = headers.indexOf(CRLF, start);
            trailingCrLf = 1;
            if (lineEnd == -1) {
                lineEnd = headers.length();
                trailingCrLf = 0;
            }
            int colonPos = -1;
            for (int i = start; i < lineEnd; i++) {
                if (headers.charAt(i) == ':') {
                    colonPos = i;
                    break;
                }
            }
            if (colonPos == -1) {
                boolean valid = true;
                if (start == 0) {
                    // status line (without colon)
                } else if ((lineEnd + 4) == headers.length()
                        && headers.endsWith(CRLF + CRLF)) {
                    // ok, trailing empty line
                    trailingCrLf = 2;
                } else if (start == lineEnd) {
                    // skip/remove empty line
                    valid = false;
                } else {
                    LOG.warn("Invalid header line: {}",
                            headers.substring(start, lineEnd));
                    valid = false;
                }
                if (!valid) {
                    if (last < start) {
                        replace.append(headers.substring(last, start));
                    }
                    last = lineEnd + 2 * trailingCrLf;
                }
                start = lineEnd + 2 * trailingCrLf;
                /*
                 * skip over invalid header line, no further check for
                 * problematic headers required
                 */
                continue;
            }
            String name = headers.substring(start, colonPos);
            if (PROBLEMATIC_HEADERS.matcher(name).matches()) {
                boolean needsFix = true;
                if (name.equalsIgnoreCase("content-length")) {
                    String value = headers.substring(colonPos + 1, lineEnd)
                            .trim();
                    try {
                        int l = Integer.parseInt(value);
                        if (l == contentLength) {
                            needsFix = false;
                        }
                    } catch (NumberFormatException e) {
                        // needs to be fixed
                    }
                }
                if (needsFix) {
                    if (last < start) {
                        replace.append(headers.substring(last, start));
                    }
                    last = lineEnd + 2 * trailingCrLf;
                    replace.append(X_HIDE_HEADER)
                            .append(headers.substring(start, lineEnd + 2
                                    * trailingCrLf));
                    if (trailingCrLf == 0) {
                        replace.append(CRLF);
                        trailingCrLf = 1;
                    }
                    if (name.equalsIgnoreCase("content-length")) {
                        // add effective uncompressed and unchunked length of
                        // content
                        replace.append("Content-Length").append(": ")
                                .append(contentLength).append(CRLF);
                    }
                }
            }
            start = lineEnd + 2 * trailingCrLf;
        }
        if (last > 0 || trailingCrLf != 2) {
            if (last < headers.length()) {
                // append trailing headers
                replace.append(headers.substring(last));
            }
            while (trailingCrLf < 2) {
                replace.append(CRLF);
                trailingCrLf++;
            }
            return replace.toString();
        }
        return headers;
    }
Modify verbatim HTTP response headers: remove or replace headers
<code>Content-Length</code>, <code>Content-Encoding</code> and
<code>Transfer-Encoding</code> which may confuse WARC readers. Ensure
that the header end with a single empty line (<code>\r\n\r\n</code>).

@param headers
HTTP 1.1 or 1.0 response header string, CR-LF-separated lines,
first line is status line
@return safe HTTP response header