最新消息： 新版网站上线了！！！

Java NIO写大文件对比（win7和mac）

测试说明

写2G文件，分批次写入，每批次写入128MB；

分别在Win7系统（3G内存，双核，32位，T系列处理器）和MacOS系统（8G内存，四核，64位，i7系列处理器）下运行测试。理论上跟硬盘类型和配置也有关系，这里不再贴出了。

测试代码

package rwbigfile;

import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.lang.reflect.Method;
import java.nio.ByteBuffer;
import java.nio.MappedByteBuffer;
import java.nio.channels.Channels;
import java.nio.channels.FileChannel;
import java.nio.channels.FileChannel.MapMode;
import java.nio.channels.ReadableByteChannel;
import java.security.AccessController;
import java.security.PrivilegedAction;

import util.StopWatch;

/**
 * NIO写大文件比较
 * @author Will
 * 
 */
public class WriteBigFileComparison {

	// data chunk be written per time
	private static final int DATA_CHUNK = 128 * 1024 * 1024; 

	// total data size is 2G
	private static final long LEN = 2L * 1024 * 1024 * 1024L; 

	
	public static void writeWithFileChannel() throws IOException {
		File file = new File("e:/test/fc.dat");
		if (file.exists()) {
			file.delete();
		}

		RandomAccessFile raf = new RandomAccessFile(file, "rw");
		FileChannel fileChannel = raf.getChannel();

		byte[] data = null;
		long len = LEN;
		ByteBuffer buf = ByteBuffer.allocate(DATA_CHUNK);
		int dataChunk = DATA_CHUNK / (1024 * 1024);
		while (len >= DATA_CHUNK) {
			System.out.println("write a data chunk: " + dataChunk + "MB");

			buf.clear(); // clear for re-write
			data = new byte[DATA_CHUNK];
			for (int i = 0; i < DATA_CHUNK; i++) {
				buf.put(data[i]);
			}

			data = null;

			buf.flip(); // switches a Buffer from writing mode to reading mode
			fileChannel.write(buf);
			fileChannel.force(true);

			len -= DATA_CHUNK;
		}

		if (len > 0) {
			System.out.println("write rest data chunk: " + len + "B");
			buf = ByteBuffer.allocateDirect((int) len);
			data = new byte[(int) len];
			for (int i = 0; i < len; i++) {
				buf.put(data[i]);
			}

			buf.flip(); // switches a Buffer from writing mode to reading mode, position to 0, limit not changed
			fileChannel.write(buf);
			fileChannel.force(true);
			data = null;
		}

		fileChannel.close();
		raf.close();
	}

	/**
	 * write big file with MappedByteBuffer
	 * @throws IOException
	 */
	public static void writeWithMappedByteBuffer() throws IOException {
		File file = new File("e:/test/mb.dat");
		if (file.exists()) {
			file.delete();
		}

		RandomAccessFile raf = new RandomAccessFile(file, "rw");
		FileChannel fileChannel = raf.getChannel();
		int pos = 0;
		MappedByteBuffer mbb = null;
		byte[] data = null;
		long len = LEN;
		int dataChunk = DATA_CHUNK / (1024 * 1024);
		while (len >= DATA_CHUNK) {
			System.out.println("write a data chunk: " + dataChunk + "MB");

			mbb = fileChannel.map(MapMode.READ_WRITE, pos, DATA_CHUNK);
			data = new byte[DATA_CHUNK];
			mbb.put(data);

			data = null;

			len -= DATA_CHUNK;
			pos += DATA_CHUNK;
		}

		if (len > 0) {
			System.out.println("write rest data chunk: " + len + "B");

			mbb = fileChannel.map(MapMode.READ_WRITE, pos, len);
			data = new byte[(int) len];
			mbb.put(data);
		}

		data = null;
		unmap(mbb);  // release MappedByteBuffer
		fileChannel.close();
	}
	
	public static void writeWithTransferTo() throws IOException {
		File file = new File("e:/test/transfer.dat");
		if (file.exists()) {
			file.delete();
		}
		
		RandomAccessFile raf = new RandomAccessFile(file, "rw");
		FileChannel toFileChannel = raf.getChannel();
		
		long len = LEN;
		byte[] data = null;
		ByteArrayInputStream bais = null;
		ReadableByteChannel fromByteChannel = null;
		long position = 0;
		int dataChunk = DATA_CHUNK / (1024 * 1024);
		while (len >= DATA_CHUNK) {
			System.out.println("write a data chunk: " + dataChunk + "MB");
			
			data = new byte[DATA_CHUNK];
			bais = new ByteArrayInputStream(data);
			fromByteChannel = Channels.newChannel(bais);
			
			long count = DATA_CHUNK;
			toFileChannel.transferFrom(fromByteChannel, position, count);
			
			data = null;
			position += DATA_CHUNK;
			len -= DATA_CHUNK;
		}
		
		if (len > 0) {
			System.out.println("write rest data chunk: " + len + "B");

			data = new byte[(int) len];
			bais = new ByteArrayInputStream(data);
			fromByteChannel = Channels.newChannel(bais);
			
			long count = len;
			toFileChannel.transferFrom(fromByteChannel, position, count);
		}
		
		data = null;
		toFileChannel.close();
		fromByteChannel.close();
	}
	
	/**
	 * 在MappedByteBuffer释放后再对它进行读操作的话就会引发jvm crash，在并发情况下很容易发生
	 * 正在释放时另一个线程正开始读取，于是crash就发生了。所以为了系统稳定性释放前一般需要检
	 * 查是否还有线程在读或写
	 * @param mappedByteBuffer
	 */
	public static void unmap(final MappedByteBuffer mappedByteBuffer) {
		try {
			if (mappedByteBuffer == null) {
				return;
			}
			
			mappedByteBuffer.force();
			AccessController.doPrivileged(new PrivilegedAction<Object>() {
				@Override
				@SuppressWarnings("restriction")
				public Object run() {
					try {
						Method getCleanerMethod = mappedByteBuffer.getClass()
								.getMethod("cleaner", new Class[0]);
						getCleanerMethod.setAccessible(true);
						sun.misc.Cleaner cleaner = 
								(sun.misc.Cleaner) getCleanerMethod
									.invoke(mappedByteBuffer, new Object[0]);
						cleaner.clean();
						
					} catch (Exception e) {
						e.printStackTrace();
					}
					System.out.println("clean MappedByteBuffer completed");
					return null;
				}
			});

		} catch (Exception e) {
			e.printStackTrace();
		}
	}

	public static void main(String[] args) throws IOException {
		StopWatch sw = new StopWatch();
		
		sw.startWithTaskName("write with file channel's write(ByteBuffer)");
		writeWithFileChannel();
		sw.stopAndPrint();
		
		sw.startWithTaskName("write with file channel's transferTo");
		writeWithTransferTo();
		sw.stopAndPrint();
		
		sw.startWithTaskName("write with MappedByteBuffer");
		writeWithMappedByteBuffer();
		sw.stopAndPrint();
	}

}

测试结果（Y轴是耗时秒数）

  • 显然writeWithMappedByteBuffer方式性能最好，且在硬件配置较高情况下优势越加明显
  • 在硬件配置较低情况下，writeWithTransferTo比writeWithFileChannel性能稍好
  • 在硬件配置较高情况下，writeWithTransferTo和writeWithFileChannel的性能基本持平
  • 此外，注意writeWithMappedByteBuffer方式除了占用JVM堆内存外，还要占用额外的native内存（Direct Byte Buffer内存）

内存映射文件使用经验

MappedByteBuffer需要占用“双倍”的内存（对象JVM堆内存和Direct Byte Buffer内存），可以通过-XX:MaxDirectMemorySize参数设置后者最大大小

不要频繁调用MappedByteBuffer的force()方法，因为这个方法会强制OS刷新内存中的数据到磁盘，从而只能获得些微的性能提升（相比IO方式），可以用后面的代码实例进行定时、定量刷新

如果突然断电或者服务器突然Down，内存映射文件数据可能还没有写入磁盘，这时就会丢失一些数据。为了降低这种风险，避免用MappedByteBuffer写超大文件，可以把大文件分割成几个小文件，但不能太小（否则将失去性能优势）

ByteBuffer的rewind()方法将position属性设回为0，因此可以重新读取buffer中的数据；limit属性保持不变，因此可读取的字节数不变

ByteBuffer的flip()方法将一个Buffer由写模式切换到读模式

ByteBuffer的clear()和compact()可以在我们读完ByteBuffer中的数据后重新切回写模式。不同的是clear()会将position设置为0，limit设为capacity，换句话说Buffer被清空了，但Buffer内的数据并没有被清空。如果Buffer中还有未被读取的数据，那调用clear()之后，这些数据会被“遗忘”，再写入就会覆盖这些未读数据。而调用compcat()之后，这些未被读取的数据仍然可以保留，因为它将所有还未被读取的数据拷贝到Buffer的左端，然后设置position为紧随未读数据之后，limit被设置为capacity，未读数据不会被覆盖

定时、定量刷新内存映射文件到磁盘

import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;

public class MappedFile {
	
	// 文件名
	private String fileName;

	// 文件所在目录路径
	private String fileDirPath;

	// 文件对象
	private File file;

	private MappedByteBuffer mappedByteBuffer;
	private FileChannel fileChannel;
	private boolean boundSuccess = false;

	// 文件最大只能为50MB
	private final static long MAX_FILE_SIZE = 1024 * 1024 * 50;
	
	// 最大的脏数据量512KB,系统必须触发一次强制刷
	private long MAX_FLUSH_DATA_SIZE = 1024 * 512;

	// 最大的刷间隔,系统必须触发一次强制刷
	private long MAX_FLUSH_TIME_GAP = 1000;

	// 文件写入位置
	private long writePosition = 0;

	// 最后一次刷数据的时候
	private long lastFlushTime;

	// 上一次刷的文件位置
	private long lastFlushFilePosition = 0;
	
	public MappedFile(String fileName, String fileDirPath) {
		super();
		this.fileName = fileName;
		this.fileDirPath = fileDirPath;
		this.file = new File(fileDirPath + "/" + fileName);
		if (!file.exists()) {
			try {
				file.createNewFile();
			} catch (IOException e) {
				e.printStackTrace();
			}
		}
		
	}

	/**
	 * 
	 * 内存映照文件绑定
	 * @return
	 */
	public synchronized boolean boundChannelToByteBuffer() {
		try {
			RandomAccessFile raf = new RandomAccessFile(file, "rw");
			this.fileChannel = raf.getChannel();
		} catch (Exception e) {
			e.printStackTrace();
			this.boundSuccess = false;
			return false;
		}

		try {
			this.mappedByteBuffer = this.fileChannel
					.map(FileChannel.MapMode.READ_WRITE, 0, MAX_FILE_SIZE);
		} catch (IOException e) {
			e.printStackTrace();
			this.boundSuccess = false;
			return false;
		}

		this.boundSuccess = true;
		return true;
	}
	
	/**
	 * 写数据：先将之前的文件删除然后重新
	 * @param data
	 * @return
	 */
	public synchronized boolean writeData(byte[] data) {
		
		return false;
	}
	
	/**
	 * 在文件末尾追加数据
	 * @param data
	 * @return
	 * @throws Exception
	 */
	public synchronized boolean appendData(byte[] data) throws Exception {
		if (!boundSuccess) {
			boundChannelToByteBuffer();
		}
		
		writePosition = writePosition + data.length;
		if (writePosition >= MAX_FILE_SIZE) {  // 如果写入data会超出文件大小限制，不写入
			flush();
			writePosition = writePosition - data.length;
			System.out.println("File=" 
								+ file.toURI().toString() 
								+ " is written full.");
			System.out.println("already write data length:" 
								+ writePosition
								+ ", max file size=" + MAX_FILE_SIZE);
			return false;
		}

		this.mappedByteBuffer.put(data);

		// 检查是否需要把内存缓冲刷到磁盘
		if ( (writePosition - lastFlushFilePosition > this.MAX_FLUSH_DATA_SIZE)
			 ||
			 (System.currentTimeMillis() - lastFlushTime > this.MAX_FLUSH_TIME_GAP
			 && writePosition > lastFlushFilePosition) ) {
			flush();  // 刷到磁盘
		}
		
		return true;
	}

	public synchronized void flush() {
		this.mappedByteBuffer.force();
		this.lastFlushTime = System.currentTimeMillis();
		this.lastFlushFilePosition = writePosition;
	}

	public long getLastFlushTime() {
		return lastFlushTime;
	}

	public String getFileName() {
		return fileName;
	}

	public String getFileDirPath() {
		return fileDirPath;
	}

	public boolean isBundSuccess() {
		return boundSuccess;
	}

	public File getFile() {
		return file;
	}

	public static long getMaxFileSize() {
		return MAX_FILE_SIZE;
	}

	public long getWritePosition() {
		return writePosition;
	}

	public long getLastFlushFilePosition() {
		return lastFlushFilePosition;
	}

	public long getMAX_FLUSH_DATA_SIZE() {
		return MAX_FLUSH_DATA_SIZE;
	}

	public long getMAX_FLUSH_TIME_GAP() {
		return MAX_FLUSH_TIME_GAP;
	}

}

以上就是本文的全部内容，希望对大家的学习有所帮助，也希望大家多多支持谷谷点程序。

转载请注明：谷谷点程序 » Java NIO写大文件对比（win7和mac）