java io读取超大文件

日常工作过程中经常使用IO读写文件,但是有时候由于文件过大,导致IO读取时出现内存溢出。
这种情况下,我们就可以使用内存映射文件来解决。

什么是内存映射文件呢?

内存映射文件,可以理解为将一个文件映射到内存地址,然后可以通过操作内存来访问文件数据。这样既不用担心内存溢出,而且还能获得更高的性能。

那如何创建和使用内存映射文件呢?

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
package com.tzq.utils;
import java.io.FileInputStream;
import java.io.IOException;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
/**
* @author tzq
*/
public class IOUtil {
private MappedByteBuffer[] mappedBufArray;
private int count = 0;
private int number;
private FileInputStream fileIn;
private long fileLength;
private int arraySize;
private byte[] array;
public IOUtil(String fileName, int arraySize) throws IOException {
this.fileIn = new FileInputStream(fileName);
FileChannel fileChannel = fileIn.getChannel();
this.fileLength = fileChannel.size();
this.number = (int) Math.ceil((double) fileLength / (double) Integer.MAX_VALUE);
this.mappedBufArray = new MappedByteBuffer[number];// 内存文件映射数组
long preLength = 0;
long regionSize = (long) Integer.MAX_VALUE;// 映射区域的大小
for (int i = 0; i < number; i++) {// 将文件的连续区域映射到内存文件映射数组中
if (fileLength - preLength < (long) Integer.MAX_VALUE) {
regionSize = fileLength - preLength;// 最后一片区域的大小
}
mappedBufArray[i] = fileChannel.map(FileChannel.MapMode.READ_ONLY, preLength, regionSize);
preLength += regionSize;// 下一片区域的开始
}
this.arraySize = arraySize;
}
public int read() throws IOException {
if (count >= number) {
return -1;
}
int limit = mappedBufArray[count].limit();
int position = mappedBufArray[count].position();
if (limit - position > arraySize) {
array = new byte[arraySize];
System.out.println(array);
mappedBufArray[count].get(array);
return arraySize;
} else {// 本内存文件映射最后一次读取数据
array = new byte[limit - position];
mappedBufArray[count].get(array);
if (count < number) {
count++;// 转换到下一个内存文件映射
}
return limit - position;
}
}
public void close() throws IOException {
fileIn.close();
array = null;
}
public byte[] getArray() {
return array;
}
public long getFileLength() {
return fileLength;
}
public static void main(String[] args) throws IOException {
IOUtil reader = new IOUtil("/Users/tzq/Downloads/data.log", 65536);
long start = System.nanoTime();
int i = 0 ;
while (reader.read() != -1){
i = i +1;
String str = new String(reader.getArray());
String[] lines = str.split("\r");
for(String line : lines){
System.out.println(line);
}
};
reader.close();
}
}