Java获取巨大的结果集

时间:2018-09-17 17:31:09

标签: java jdbc java-8 oracle12c resultset

我有一张有大量记录的表,说有将近25gb的记录和超过10亿条记录。我想使用Java来获取它们并写入csv。我想到使用java8流来停止将整个结果集带到内存中,但仍然出现Java堆空间错误。

    try {

                close=UncheckedCloseable.wrap(con);
                pSt = con.prepareStatement("select * from "+this.tblName);
                close=close.nest(pSt);
                con.setAutoCommit(false);
                pSt.setFetchSize(5000);

                ResultSet resultSet = pSt.executeQuery();
                close=close.nest(resultSet);
                Stream<String> buffer = StreamSupport.stream(new Spliterators.AbstractSpliterator<String>(
                        Long.MAX_VALUE,Spliterator.ORDERED) {

                    @Override
                    public boolean tryAdvance(Consumer<? super String> action) {
                        try {
                            if(!resultSet.next()) return false;
                            action.accept(createRecord(resultSet));
                            return true;
                        } catch(SQLException | IllegalAccessException | IllegalArgumentException | InvocationTargetException ex) {
                            throw new RuntimeException(ex);
                        }
                    }

                }, false).onClose(close);

                System.out.println(buffer.count());
                Files.write(Paths.get(home.getAbsolutePath()+"/"+tblName+".csv"), (Iterable<String>)buffer::iterator);

            }


private String createRecord(ResultSet resultSet) throws SQLException, IllegalAccessException, IllegalArgumentException, InvocationTargetException {
        StringBuffer buffer = new StringBuffer();
        int colCount = resultSet.getMetaData().getColumnCount();
        while(resultSet.next()) {
            for(int i=1; i<=colCount; i++) {
                String columnName = resultSet.getMetaData().getColumnTypeName(i);
                String getType = StringUtils.capitalize(columnName.toLowerCase());
                Method methodName = getMethodName(resultSet,getType);
                Object result = methodName.invoke(resultSet, i);
                if(result == null || result.toString().equalsIgnoreCase("null")){
                    result =  "";
                }
                if (result instanceof String) {
                    result = ((String) result).trim().replaceAll(",", "\\\\,").replace("\n", "").replace("\r", "");
                }
                buffer.append(result+",");
            }
            buffer.setLength(buffer.length()-1);
            buffer.append("\n");
        }
        return buffer.toString();
    }

0 个答案:

没有答案