在平常的开发中会用到处理表格文件的功能，poi就是一个非常优秀的处理表格的java框架，但是当表格文件的数据量过大处理过程就会出现堆内存溢出的异常，让人痛苦了两天，最后在谷歌的帮助下找了一个解决方法，现在把这个方法分享出来希望可以帮助到大家！！
import org.apache.poi.hssf.eventusermodel.*;
import org.apache.poi.hssf.eventusermodel.dummyrecord.LastCellOfRowDummyRecord;
import org.apache.poi.hssf.eventusermodel.dummyrecord.MissingCellDummyRecord;
import org.apache.poi.hssf.model.HSSFFormulaParser;
import org.apache.poi.hssf.record.*;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.apache.poi.poifs.filesystem.POIFSFileSystem;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.PrintStream;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Date;
public class XlsToCsv implements HSSFListener {
private int minColumns;
private POIFSFileSystem fs;
private PrintStream output;
private int lastRowNumber;
private int lastColumnNumber;
public long time_cha;
public long getTime_cha() {
return time_cha;
}
public void setTime_cha(long time_cha) {
this.time_cha = time_cha;
}
/**
* Should we output the formula, or the value it has?
*/
private boolean outputFormulaValues = true;
/**
* For parsing Formulas
*/
private EventWorkbookBuilder.SheetRecordCollectingListener workbookBuildingListener;
private HSSFWorkbook stubWorkbook;
// Records we pick up as we process
private SSTRecord sstRecord;
private FormatTrackingHSSFListener formatListener;
/**
* So we known which sheet we're on
*/
private int sheetIndex = -1;
private BoundSheetRecord[] orderedBSRs;
private ArrayList boundSheetRecords = new ArrayList();
// For handling formulas with string results
private int nextRow;
private int nextColumn;
private boolean outputNextStringRecord;
private String d = "";
private final static String OUTPUT_CHARSET = "UTF-8";
// private final String OUTPUT_CHARSET = "GBK";
private int count = 0;
public static String deal_time = "";
/**
* Creates a new XLS -> CSV converter
*
* @param fs The POIFSFileSystem to process
* @param output The PrintStream to output the CSV to
* @param minColumns The minimum number of columns to output, or -1 for no minimum
*/
public XlsToCsv(POIFSFileSystem fs, PrintStream output, int minColumns) {
this.fs = fs;
this.output = output;
this.minColumns = minColumns;
}
public XlsToCsv(String inputFilePath, String outputFilePath) throws Exception {
fs = new POIFSFileSystem(new FileInputStream(inputFilePath));
output = new PrintStream(outputFilePath, OUTPUT_CHARSET);
minColumns = -1;
}
/**
* Creates a new XLS -> CSV converter
*
* @param filename The file to process
* @param minColumns The minimum number of columns to output, or -1 for no minimum
* @throws IOException
* @throws FileNotFoundException
*/
public XlsToCsv(String filename, int minColumns) throws IOException, FileNotFoundException {
this(new POIFSFileSystem(new FileInputStream(filename)), System.out, minColumns);
}
public XlsToCsv() {
}
public XlsToCsv(long time_cha) {
this.time_cha = time_cha;
}
/**
* Initiates the processing of the XLS file to CSV
*/
public void process() throws IOException {
MissingRecordAwareHSSFListener listener = new MissingRecordAwareHSSFListener(this);
formatListener = new FormatTrackingHSSFListener(listener);
HSSFEventFactory factory = new HSSFEventFactory();
HSSFRequest request = new HSSFRequest();
if (outputFormulaValues) {
request.addListenerForAllRecords(formatListener);
} else {
workbookBuildingListener = new EventWorkbookBuilder.SheetRecordCollectingListener(formatListener);
request.addListenerForAllRecords(workbookBuildingListener);
}
factory.processWorkbookEvents(request, fs);
}
/**
* Main HSSFListener method, processes events, and outputs the CSV as the
* file is processed.
*/
public void processRecord(Record record) {
int thisRow = -1;
int thisColumn = -1;
String thisStr = null;
String sign = "";
switch (record.getSid()) {
case BoundSheetRecord.sid:
//boundSheetRecords.add(record);
break;
case BOFRecord.sid:
BOFRecord br = (BOFRecord) record;
if (br.getType() == BOFRecord.TYPE_WORKSHEET) {
// Create sub workbook if required
if (workbookBuildingListener != null && stubWorkbook == null) {
stubWorkbook = workbookBuildingListener.getStubHSSFWorkbook();
}
// Output the worksheet name
// Works by ordering the BSRs by the location of
// their BOFRecords, and then knowing that we
// process BOFRecords in byte offset order
sheetIndex++;
if (orderedBSRs == null) {
orderedBSRs = BoundSheetRecord.orderByBofPosition(boundSheetRecords);
}
}
break;
case SSTRecord.sid:
sstRecord = (SSTRecord) record;
break;
case BlankRecord.sid:
BlankRecord brec = (BlankRecord) record;
thisRow = brec.getRow();
thisColumn = brec.getColumn();
// thisStr = "";
break;
case BoolErrRecord.sid:
BoolErrRecord berec = (BoolErrRecord) record;
thisRow = berec.getRow();
thisColumn = berec.getColumn();
// thisStr = "";
break;
case FormulaRecord.sid:
FormulaRecord frec = (FormulaRecord) record;
thisRow = frec.getRow();
thisColumn = frec.getColumn();
if (outputFormulaValues) {
if (Double.isNaN(frec.getValue())) {
// Formula result is a string
// This is stored in the next record
outputNextStringRecord = true;
nextRow = frec.getRow();
nextColumn = frec.getColumn();
} else {
thisStr = formatListener.formatNumberDateCell(frec);
}
} else {
// thisStr = '"' + HSSFFormulaParser.toFormulaString(stubWorkbook, frec.getParsedExpression()) + '"';
thisStr = HSSFFormulaParser.toFormulaString(stubWorkbook, frec.getParsedExpression());
}
break;
case StringRecord.sid:
if (outputNextStringRecord) {
// String for formula
StringRecord srec = (StringRecord) record;
thisStr = srec.getString();
thisRow = nextRow;
thisColumn = nextColumn;
outputNextStringRecord = false;
}
break;
case LabelRecord.sid:
LabelRecord lrec = (LabelRecord) record;
thisRow = lrec.getRow();
thisColumn = lrec.getColumn();
// thisStr = '"' + lrec.getValue() + '"';
thisStr = lrec.getValue();
break;
case LabelSSTRecord.sid:
LabelSSTRecord lsrec = (LabelSSTRecord) record;
thisRow = lsrec.getRow();
thisColumn = lsrec.getColumn();
if (sstRecord == null) {
thisStr = '"' + "(No SST Record, can't identify string)" + '"';
// } else if (sstRecord.getString(lsrec.getSSTIndex()).toString().contains("30/03/2018 - 16h34:18.440")) {
} else if (sstRecord.getString(lsrec.getSSTIndex()).toString().contains("/")) {
// thisStr = '"' + sstRecord.getString(lsrec.getSSTIndex()).toString() + '"';
thisStr = sstRecord.getString(lsrec.getSSTIndex()).toString();
//用-分割得到时间数据的年月日和小时，分钟秒的数据
String[] splitData = thisStr.split("-");
//格式化年月日的数据
String[] date = splitData[0].split("/");
//格式化小时分钟秒的数据
String[] time = splitData[1].split(":");
String[] hourAndMount = time[0].split("h");
String hours = hourAndMount[0] + ":" + hourAndMount[1] + ":" + time[1].split("\\.")[0] + "." + time[1].split("\\.")[1];
d = date[date.length - 1].trim() + "-" + date[1] + "-" + date[0];
deal_time = d + " " + hours;
thisStr = "";
} else {
//处理时间格式
thisStr = sstRecord.getString(lsrec.getSSTIndex()).toString();
// System.out.println("------->" + thisStr);
if (thisStr.contains("h")) {
String deal_get_time_split[] = thisStr.split(":");
String deal_hour_minute[] = deal_get_time_split[0].split("h");
String deal_seconds_millis[] = deal_get_time_split[1].split(",");
// String deal_seonds_millis_ = deal_seconds_millis[0] + "." + deal_seconds_millis[1];
String deal_date = d + " " + deal_hour_minute[0] + ":" + deal_hour_minute[1] + ":" + deal_seconds_millis[0] + "." + deal_seconds_millis[1];
// thisStr = d + " " + deal_hour_minute_ + ":" + deal_seonds_millis_;
try {
long t = SystemConstants.stdMSsdf.parse(deal_date).getTime() + this.getTime_cha();
thisStr = SystemConstants.stdMSsdf.format(new Date(t));
System.out.println("s===" + thisStr);
} catch (ParseException e) {
e.printStackTrace();
}
} else if (thisStr.contains("Sign") || thisStr.contains("N2")) {
//处理信号的信号名。包括信号产生时间和对应的值
if (thisStr.contains("N2")) {
thisStr = thisStr;
} else {
sign += thisStr + "_time," + thisStr + "_value";
thisStr = sign;
}
} else {
thisStr = "";
}
}
break;
case NoteRecord.sid:
NoteRecord nrec = (NoteRecord) record;
thisRow = nrec.getRow();
thisColumn = nrec.getColumn();
thisStr = '"' + "(TODO)" + '"';
break;
case NumberRecord.sid:
NumberRecord numrec = (NumberRecord) record;
thisRow = numrec.getRow();
thisColumn = numrec.getColumn();
// Format
thisStr = formatListener.formatNumberDateCell(numrec);
break;
case RKRecord.sid:
RKRecord rkrec = (RKRecord) record;
thisRow = rkrec.getRow();
thisColumn = rkrec.getColumn();
thisStr = '"' + "(TODO)" + '"';
break;
default:
break;
}
// Handle new row
if (thisRow != -1 && thisRow != lastRowNumber) {
lastColumnNumber = -1;
}
// Handle missing column
if (record instanceof MissingCellDummyRecord) {
MissingCellDummyRecord mc = (MissingCellDummyRecord) record;
thisRow = mc.getRow();
thisColumn = mc.getColumn();
thisStr = "";
}
// If we got something to print out, do so
if (thisStr != null) {
if (thisStr == "") {
} else {
// System.out.println("thisColumn :" + thisColumn + " thisStr: " + thisStr);
if (thisColumn > 0) {
//控制那一列不被写入到csv文件中，具体以thisStr来决定
if ((thisColumn - 1) % 4 == 0) {
thisStr = "";
} else {
output.print(',');
}
}
output.print(thisStr);
}
}
// Update column and row count
if (thisRow > -1) {
lastRowNumber = thisRow;
}
if (thisColumn > -1) {
lastColumnNumber = thisColumn;
}
// Handle end of row
if (record instanceof LastCellOfRowDummyRecord) {
if (((LastCellOfRowDummyRecord) record).getRow() > 7) {
// Print out any missing commas if needed
if (minColumns > 0) {
// Columns are 0 based
if (lastColumnNumber == -1) {
lastColumnNumber = 0;
}
for (int i = lastColumnNumber; i < (minColumns); i++) {
output.print(',');
}
}
// We're onto a new row
lastColumnNumber = -1;
// End the row
output.println();
}
}
}
public static void main(String[] args) throws Exception {
long star = System.currentTimeMillis();
System.out.println("开始读取的时间是======》:: " + star);
String inputPath2 = "/home/java/data/fault_database/20180330-第一次试验-试验阶段.xls";
String outputPath2 = "/home/java/data/fault_database/data.csv";
// XlsToCsv.time_cha=2000;
XlsToCsv xls2csv = new XlsToCsv(inputPath2, outputPath2);
xls2csv.setTime_cha(2000);
xls2csv.process();
System.out.println(XlsToCsv.deal_time);
long end = System.currentTimeMillis();
System.out.println("总过用时==========>: " + (end - star));
}
}
import com.monitorjbl.xlsx.StreamingReader;
import jxl.WorkbookSettings;
import jxl.read.biff.BiffException;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import java.io.*;
import java.util.*;
/**
* Created by 李泽华 on 2017/9/7.
*/
public class BigExcelReaderPoi {
private static OutputStream os;
private static OutputStreamWriter osw;
private static BufferedWriter bw;
private static WorkbookSettings ws = new WorkbookSettings();
private static jxl.Workbook wk;
private static FileInputStream in;
/**
* xls 后缀的execl表格文件内容转成csv文件
* (1)使用流的方法读取execl表格内容，然后处理表格的内容
* (2)获得第2行数据，处理得到该文件生成的时间
* (3)处理第4行数据，获取信号名，然后把信号名分别加上"_time"和"_value"
* (4)把处理好的数据按照csv文件格式要求写入文件中，在解析车台数据用用到生成的csv文件
* @param inputPath
* @param outputPath
* @return
*/
public static List<Map<String,String>> xlsConvertToCsv(String inputPath, String outputPath) {
List<Map<String,String>> startTime_outFilePath = new ArrayList<Map<String, String>>();
try {
long start_time = System.currentTimeMillis();
System.out.println("当前开始时间是=====>:" + start_time);
os = new FileOutputStream(new File(outputPath));
osw = new OutputStreamWriter(os, "UTF8");
bw = new BufferedWriter(osw);
//载入excel文件
ws.setLocale(new Locale("en", "EN"));
wk = jxl.Workbook.getWorkbook(new File(inputPath), ws);
//从工作簿workbook取得每页sheets
String date_ = "";
boolean flag = true;
String deal_time = "";
String sign_date = "";
String sign ="";
StringBuffer signBuffer = new StringBuffer();
String startTime = "";
Map<String,String> outFilePath_map = new HashMap<String, String>();
outFilePath_map.put("outFilePath",outputPath);
startTime_outFilePath.add(outFilePath_map);
for (int sheet = 0; sheet < wk.getNumberOfSheets(); sheet++) {
jxl.Sheet s = wk.getSheet(sheet);
jxl.Cell[] row = null;
//从每页sheet取得每个区块cell
for (int i = 0; i < s.getRows(); i++) {
row = s.getRow(i);
//获取开始时间 如：2018-03-30
if (i == 2){
String rowData = row[1].getContents();
//用-分割得到时间数据的年月日和小时，分钟秒的数据
String[] splitData = rowData.split("-");
//格式化年月日的数据
String[] date = splitData[0].split("/");
//格式化小时分钟秒的数据
String[] time = splitData[1].split(":");
String[] hourAndMount = time[0].split("h");
String hours = hourAndMount[0] + ":" + hourAndMount[1] + ":" + time[1].split("\\.")[0] + "." + time[1].split("\\.")[1];
deal_time = date[date.length - 1].trim() + "-" + date[1] + "-" + date[0];
startTime = deal_time + " " + hours;
}
//处理信号的名的值处理后的值如：
// Sign1_time,Sign1_value,Sign2_time,Sign2_value,Sign3_time,Sign3_value,Sign4_time,Sign4_value,Sign5_time,Sign5_value,Sign6_time,Sign6_value,Sign7_time,Sign7_value,
if (i == 4){
for (int m = 0;m<row.length; m++){
if (m % 4 == 0){
// sign += row[m].getContents() + "_time," + row[m].getContents() + "_value,";
signBuffer.append(row[m].getContents() + "_time," + row[m].getContents() + "_value,");
sign = signBuffer.toString();
}
}
bw.write(sign);
bw.newLine();
}
if ((i>8)&&(row.length > 0)) {
//写入第一列处理后的数据，并加上,
String time = "";
String deal_get_time_split[] = row[0].getContents().split(":");
String deal_hour_minute[] = deal_get_time_split[0].split("h");
String deal_hour_minute_ = deal_hour_minute[0] + ":" + deal_hour_minute[1];
String deal_seconds_millis[] = deal_get_time_split[1].split(",");
String deal_seonds_millis_ = deal_seconds_millis[0] + "." + deal_seconds_millis[1];
time = deal_time + " " + deal_hour_minute_ + ":" + deal_seonds_millis_;
bw.write(time);
bw.write(',');
bw.write(row[2].getContents());
bw.write(',');
for (int j = 1; j < row.length; j++) {
// bw.write(',');
//取出时间并处理然后写入新的文件中
if (j % 4 ==0){
String ti ="";
String deal_get_time_split_[] = row[j].getContents().split(":");
String deal_hour_minute1[] = deal_get_time_split_[0].split("h");
String deal_hour_minute_1 = deal_hour_minute1[0] + ":" + deal_hour_minute1[1];
String deal_seconds_millis1[] = deal_get_time_split_[1].split(",");
String deal_seonds_millis_1 = deal_seconds_millis1[0] + "." + deal_seconds_millis1[1];
ti = deal_time + " " + deal_hour_minute_1 + ":" + deal_seonds_millis_1;
bw.write(ti);
bw.write(',');
//获得信号的值
bw.write(row[j+2].getContents());
bw.write(',');
}
}
bw.newLine();
}
}
}
Map<String,String> startTime_map = new HashMap<String, String>();
startTime_map.put("startTime", startTime);
startTime_outFilePath.add(startTime_map);
long end_time = System.currentTimeMillis();
System.out.println("用时=======>: " + (end_time -start_time) + "ms");
bw.flush();
bw.close();
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} catch (BiffException e) {
e.printStackTrace();
}
return startTime_outFilePath;
}
/**
* .xlsx类型的execl表格数据处理成为csv文件
* @param inputPath
* @param outputPath
* @return
*/
public String xlsxConvertToCsv(String inputPath, String outputPath) {
long startTime = System.currentTimeMillis();
System.out.println("程序开始时间：" + (startTime) + "ms");
try {
os = new FileOutputStream(new File(outputPath));
osw = new OutputStreamWriter(os, "UTF8");
bw = new BufferedWriter(osw);
in = new FileInputStream(inputPath);
Workbook wk = StreamingReader.builder()
.rowCacheSize(100) //缓存到内存中的行数，默认是10
.bufferSize(4096) //读取资源时，缓存到内存的字节大小，默认是1024
.open(in); //打开资源，必须，可以是InputStream或者是File，注意：只能打开XLSX格式的文件
Sheet sheet = wk.getSheetAt(0);
bw.write(sheet.getSheetName());
bw.newLine();
//控制前几行不被写入csv文件
int count = 0;
//控制那些列不被写入csv文件
int cell_index = 0;
//遍历所有的行
for (Row row : sheet) {
//System.out.println("开始遍历第" + row.getRowNum() + "行数据：");
count++;
System.out.println("当前行数是==============》： " + count);
if (count < 8) {
continue;
}
//遍历所有的列
for (Cell cell : row) {
cell_index++;
System.out.println("当前的列号是==========》： " + cell.getColumnIndex());
String line_data = "";
// System.out.print(cell.getStringCellValue() + " ");
if (cell.getStringCellValue().contains("h")) {
String deal_get_time_split[] = cell.getStringCellValue().split(":");
String deal_hour_minute[] = deal_get_time_split[0].split("h");
String deal_hour_minute_ = deal_hour_minute[0] + ":" + deal_hour_minute[1];
// String deal_seconds_millis[] = deal_get_time_split[1].split(",");
String deal_seconds_millis[] = deal_get_time_split[1].split("\\.");
String deal_seonds_millis_ = deal_seconds_millis[0] + "." + deal_seconds_millis[1];
line_data += deal_hour_minute_ + ":" + deal_seonds_millis_;
bw.write(line_data);
bw.write(',');
} else if ((cell.getColumnIndex() + 1) % 2 == 0) {
bw.write(line_data);
// bw.write(',');
} else {
bw.write(cell.getStringCellValue());
bw.write(',');
}
}
bw.newLine();
//System.out.println(" ");
}
long endTime = System.currentTimeMillis();
System.out.println("程序运行时间：" + (endTime - startTime) + "ms"); //输出程序运行时间
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} finally {
try {
os.close();
osw.close();
bw.close();
in.close();
wk.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return outputPath;
}
//TEST
public static void main(String args[]) {
// String inputPath2 = "/home/java/data/fault_database/20180330-第一次试验-试验阶段.xls";
String inputPath2 = "d:/home/java/data/fault_database/20180330-第一次试验-试验阶段.xls";
// String outputPath2 = "/home/java/data/fault_database/data.csv";
String outputPath2 = "d:/home/java/data/fault_database/fault_databasedata.csv";
String csv = "/home/java/data/fault_database/lizehua_test.csv";
xlsConvertToCsv(inputPath2, outputPath2);
}
}