cdm/src/main/java/ucar/nc2/util/IO.java
static public long copyUrlB(String urlString, OutputStream out, int bufferSize) throws IOException {
    long count;
    URL url;

    try {
      url = new URL(urlString);
    } catch (MalformedURLException e) {
      throw new IOException("** MalformedURLException on URL <" + urlString + ">\n" + e.getMessage() + "\n");
    }

    try {
      java.net.URLConnection connection = url.openConnection();
      java.net.HttpURLConnection httpConnection = null;
      if (connection instanceof java.net.HttpURLConnection) {
        httpConnection = (java.net.HttpURLConnection) connection;
        httpConnection.addRequestProperty("Accept-Encoding", "gzip");
      }

      if (showHeaders) {
        showRequestHeaders(urlString, connection);
      }

      // get response
      if (httpConnection != null) {
        int responseCode = httpConnection.getResponseCode();
        if (responseCode / 100 != 2)
          throw new IOException("** Cant open URL <" + urlString + ">\n Response code = " + responseCode
              + "\n" + httpConnection.getResponseMessage() + "\n");
      }

      if (showHeaders && (httpConnection != null)) {
        int code = httpConnection.getResponseCode();
        String response = httpConnection.getResponseMessage();

        // response headers
        System.out.println("\nRESPONSE for " + urlString + ": ");
        System.out.println(" HTTP/1.x " + code + " " + response);
        System.out.println("Headers: ");

        for (int j = 1; ; j++) {
          String header = connection.getHeaderField(j);
          String key = connection.getHeaderFieldKey(j);
          if (header == null || key == null) break;
          System.out.println(" " + key + ": " + header);
        }
      }

      // read it
      try (InputStream is = connection.getInputStream()) {
        BufferedInputStream bis;

        // check if its gzipped
        if ("gzip".equalsIgnoreCase(connection.getContentEncoding())) {
          bis = new BufferedInputStream(new GZIPInputStream(is), 8000);
        } else {
          bis = new BufferedInputStream(is, 8000);
        }

        if (out == null)
          count = IO.copy2null(bis, bufferSize);
        else
          count = IO.copyB(bis, out, bufferSize);
      }

     } catch (java.net.ConnectException e) {
       if (showStackTrace) e.printStackTrace();
       throw new IOException("** ConnectException on URL: <" + urlString + ">\n" +
          e.getMessage() + "\nServer probably not running");

     } catch (java.net.UnknownHostException e) {
       if (showStackTrace) e.printStackTrace();
       throw new IOException("** UnknownHostException on URL: <" + urlString + ">\n");

     } catch (Exception e) {
       if (showStackTrace) e.printStackTrace();
       throw new IOException("** Exception on URL: <" + urlString + ">\n" + e);
     }

    return count;
  }
static public long copyUrlB(String urlString, OutputStream out, int bufferSize) throws IOException {
    long count;
    URL url;

    try {
      url = new URL(urlString);
    } catch (MalformedURLException e) {
      throw new IOException("** MalformedURLException on URL <" + urlString + ">\n" + e.getMessage() + "\n");
    }

    try {
      java.net.URLConnection connection = url.openConnection();
      java.net.HttpURLConnection httpConnection = null;
      if (connection instanceof java.net.HttpURLConnection) {
        httpConnection = (java.net.HttpURLConnection) connection;
        httpConnection.addRequestProperty("Accept-Encoding", "gzip");
      }

      if (showHeaders) {
        showRequestHeaders(urlString, connection);
      }

      // get response
      if (httpConnection != null) {
        int responseCode = httpConnection.getResponseCode();
        if (responseCode / 100 != 2)
          throw new IOException("** Cant open URL <" + urlString + ">\n Response code = " + responseCode
              + "\n" + httpConnection.getResponseMessage() + "\n");
      }

      if (showHeaders && (httpConnection != null)) {
        int code = httpConnection.getResponseCode();
        String response = httpConnection.getResponseMessage();

        // response headers
        System.out.println("\nRESPONSE for " + urlString + ": ");
        System.out.println(" HTTP/1.x " + code + " " + response);
        System.out.println("Headers: ");

        for (int j = 1; ; j++) {
          String header = connection.getHeaderField(j);
          String key = connection.getHeaderFieldKey(j);
          if (header == null || key == null) break;
          System.out.println(" " + key + ": " + header);
        }
      }

      // read it
      try (InputStream is = connection.getInputStream()) {
        BufferedInputStream bis;

        // check if its gzipped
        if ("gzip".equalsIgnoreCase(connection.getContentEncoding())) {
          bis = new BufferedInputStream(new GZIPInputStream(is), 8000);
        } else {
          bis = new BufferedInputStream(is, 8000);
        }

        if (out == null)
          count = IO.copy2null(bis, bufferSize);
        else
          count = IO.copyB(bis, out, bufferSize);
      }

     } catch (java.net.ConnectException e) {
       if (showStackTrace) e.printStackTrace();
       throw new IOException("** ConnectException on URL: <" + urlString + ">\n" +
          e.getMessage() + "\nServer probably not running");

     } catch (java.net.UnknownHostException e) {
       if (showStackTrace) e.printStackTrace();
       throw new IOException("** UnknownHostException on URL: <" + urlString + ">\n");

     } catch (Exception e) {
       if (showStackTrace) e.printStackTrace();
       throw new IOException("** Exception on URL: <" + urlString + ">\n" + e);
     }

    return count;
  }
copy contents of URL to output stream, specify internal buffer size. request gzip encoding

@param urlString  copy the contents of this URL
@param out        copy to this stream. If null, throw bytes away
@param bufferSize internal buffer size.
@return number of bytes copied
@throws java.io.IOException on io error