JPA : large object upload memory usage
===

I must create a file upload service able to store media files ( mostly videos, with virtually no file limit ). 

We are actually using JPA with Hibernate in the project and we must support at least Derby, PostgreSQL and Oracle so native requests are out of questions
Some corporate choices makes direct use of hibernate API impossible to use too - except for annotations ( I know Hibernate can stream large object but I can't use this )

In order to be able to handle large files without memory problems, we split the uploaded files in small parts we store in the database. Each part is about 10Mb.

My problem is that using directly the stream given by commons-fileupload, memory consumption goes off charts. I already tried with a temporary file, and in this case the memory consumption is ok.

I have been asked to identify what is the cause of the problem, but I must admit that I don't have any ideas. 
Controller :

	public void handleFileUpload(HttpServletRequest request) {
		if (!ServletFileUpload.isMultipartContent(request)) {
			throw new MediaFileWriteException("Upload request is not multipart");
		}

		DiskFileItemFactory factory = new DiskFileItemFactory();
		ServletFileUpload upload = new ServletFileUpload(factory);
		try {
			// Parse the request
			FileItemIterator iter = upload.getItemIterator(request);
			while (iter.hasNext()) {
				FileItemStream item = iter.next();
				InputStream stream = item.openStream();
				if (item.isFormField()) {
					continue;
				} else {
                                    // Here if I create a temporary file on the disk
                                    // and copy the stream into it, then open it with 
                                    // a FileInputStream, the memory consumption is 
                                    // at max 40-50Mb
                                    // If I leave it like this, it seems all the 
                                    // stream content stays loaded.
					mediaFileRepository.createMediaFile(item.getName(), item.getContentType(),stream);
					LOGGER.info("Storage of file {} done.",item.getName());
				}

			}
		} catch (Exception e) {
			throw new MediaFileWriteException("Could not upload file : ", e);
		}
	}

Repository :

    public long createMediaFile(String name, String mimeType,
			InputStream istream) {
		LOGGER.info("Saving media file {}, {} ...", name, mimeType);


		int mediaFilePartNumber = 0;
		// holds the number of bytes read after a call to istream.read()
		int streamRead = 0;
		// holds the number of bytes read of the whole media file
		int mediaFileRead = 0;
		// holds the number of bytes read for the media file part
		int mediaFilePartRead = 0;

		MediaFile mediaFile = new MediaFile();
		mediaFile.setMimeType(mimeType);
		mediaFile.setName(name);
		
		Checksum mediaFileChecksum = new CRC32();
		
		entityManager.persist(mediaFile);
		entityManager.flush();
		
		byte[] buffer = new byte[MEDIA_FILE_PART_SIZE];
		try {
			do {
				Checksum partChecksum = new CRC32();
				streamRead = 0;
				mediaFilePartRead = 0;

	
				// Filling the buffer that will be used as media file part.
				// As the read is not guaranteed to read all the data in
				// one go, we have to iterate until the buffer is full.
				int freeBufferSize = MEDIA_FILE_PART_SIZE;
				while ( freeBufferSize > 0 ) {
					streamRead = istream.read(buffer, mediaFilePartRead, freeBufferSize);

					// Must handle the end of the stream 
					if (streamRead == -1) 
						break;
					
					mediaFilePartRead += streamRead;
					mediaFileRead += streamRead;
					freeBufferSize = MEDIA_FILE_PART_SIZE - mediaFilePartRead;
				}

				// Checks if the buffer was filled. If not, we must save only
				// the read bytes. As arrays are immutable, we do it by creating
				// a copy of it.
				// This case should happen only when handling the last media
				// file part, as its size will almost always be different 
				// from MEDIA_FILE_PART_SIZE
				if (streamRead != MEDIA_FILE_PART_SIZE) {
					buffer = Arrays.copyOf(buffer, mediaFilePartRead);
				}
				
				// Updating file checksum
				partChecksum.update(buffer, 0, buffer.length);
				mediaFileChecksum.update(buffer, 0, buffer.length);
				
				// Creating media file part
				MediaFilePart mediaFilePart = new MediaFilePart();
				mediaFilePart.setData(buffer);
				mediaFilePart.setNumber(mediaFilePartNumber);
				mediaFilePart.setChecksum(Long.toHexString(partChecksum.getValue()));
				
				// Links the part with the media file. Note that here, hibernate will
				// directly store the part into the database when calling EntityManager.persist(...)
				mediaFile.getMediaFileParts().add(mediaFilePart);
	
				// Persisting data into db. The flush here is important as it ensures that
				// the mediaFilePart can be detached from jpa context safely
				LOGGER.debug("Saving media file '{}', size '{}', part n°'{}', checksum '{}'.", new Object[]{mediaFile.getName(), readableFileSize(mediaFilePartRead), mediaFilePartNumber, mediaFilePart.getChecksum()});
				entityManager.persist(mediaFilePart);
				entityManager.persist(mediaFile);
				entityManager.flush();
				entityManager.detach(mediaFilePart);
				mediaFilePartNumber++;
			} while (streamRead != -1);

			mediaFile.setSize(mediaFileRead);
			mediaFile.setChecksum(Long.toHexString(mediaFileChecksum.getValue()));

			LOGGER.info("Done saving media file '{}', '{}' in {} parts ({})...", new Object[] {name, mimeType,mediaFilePartNumber+1, readableFileSize(mediaFileRead)});
			
		} catch (Exception e) {
			throw new MediaFileWriteException("Could not upload file : ", e);
		}

		return mediaFile.getId();
	}

