记一次mybaits缓存导致的内存溢出 java.lang.OutOfMemoryError: Java heap space

原创
2019/07/22 17:59
阅读数 5.8K

先贴一下错误截图

org.springframework.web.util.NestedServletException: Handler dispatch failed; nested exception is java.lang.OutOfMemoryError: Java heap space
	at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1006)
	at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:925)
	at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:978)
	at org.springframework.web.servlet.FrameworkServlet.doPost(FrameworkServlet.java:881)
	at javax.servlet.http.HttpServlet.service(HttpServlet.java:661)
	at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:855)
	at javax.servlet.http.HttpServlet.service(HttpServlet.java:742)
	at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:231)
	at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
	at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:52)
	at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
	at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
	at com.num.context.SessionFilter.toLogin(SessionFilter.java:164)
	at com.num.context.SessionFilter.doFilterInternal(SessionFilter.java:72)
	at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
	at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
	at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
	at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:99)
	at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
	at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
	at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
	at org.springframework.web.filter.HttpPutFormContentFilter.doFilterInternal(HttpPutFormContentFilter.java:109)
	at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
	at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
	at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
	at org.springframework.web.filter.HiddenHttpMethodFilter.doFilterInternal(HiddenHttpMethodFilter.java:81)
	at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
	at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
	at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
	at org.springframework.boot.web.servlet.support.ErrorPageFilter.doFilter(ErrorPageFilter.java:115)
	at org.springframework.boot.web.servlet.support.ErrorPageFilter.access$000(ErrorPageFilter.java:59)
	at org.springframework.boot.web.servlet.support.ErrorPageFilter$1.doFilterInternal(ErrorPageFilter.java:90)
	at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
	at org.springframework.boot.web.servlet.support.ErrorPageFilter.doFilter(ErrorPageFilter.java:108)
	at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
	at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
	at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:200)
	at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
	at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
	at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
	at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:199)
	at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:96)
	at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:493)
	at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:137)
	at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:81)
	at org.apache.catalina.valves.AbstractAccessLogValve.invoke(AbstractAccessLogValve.java:660)
	at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:87)
	at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:343)
	at org.apache.coyote.http11.Http11Processor.service(Http11Processor.java:798)
	at org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:66)
	at org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:806)
	at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1498)
	at org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.OutOfMemoryError: Java heap space
	at org.apache.ibatis.reflection.MetaObject.getValue(MetaObject.java:113)
	at org.apache.ibatis.reflection.MetaObject.metaObjectForProperty(MetaObject.java:145)
	at org.apache.ibatis.reflection.wrapper.MapWrapper.hasGetter(MapWrapper.java:122)
	at org.apache.ibatis.reflection.MetaObject.hasGetter(MetaObject.java:109)
	at org.apache.ibatis.builder.SqlSourceBuilder$ParameterMappingTokenHandler.buildParameterMapping(SqlSourceBuilder.java:75)
	at org.apache.ibatis.builder.SqlSourceBuilder$ParameterMappingTokenHandler.handleToken(SqlSourceBuilder.java:67)
	at org.apache.ibatis.parsing.GenericTokenParser.parse(GenericTokenParser.java:78)
	at org.apache.ibatis.builder.SqlSourceBuilder.parse(SqlSourceBuilder.java:45)
	at org.apache.ibatis.scripting.xmltags.DynamicSqlSource.getBoundSql(DynamicSqlSource.java:44)
	at org.apache.ibatis.mapping.MappedStatement.getBoundSql(MappedStatement.java:292)
	at org.apache.ibatis.executor.statement.BaseStatementHandler.<init>(BaseStatementHandler.java:64)
	at org.apache.ibatis.executor.statement.PreparedStatementHandler.<init>(PreparedStatementHandler.java:40)
	at org.apache.ibatis.executor.statement.RoutingStatementHandler.<init>(RoutingStatementHandler.java:46)
	at org.apache.ibatis.session.Configuration.newStatementHandler(Configuration.java:545)
	at org.apache.ibatis.executor.SimpleExecutor.doUpdate(SimpleExecutor.java:48)
	at org.apache.ibatis.executor.BaseExecutor.update(BaseExecutor.java:117)
	at org.apache.ibatis.executor.CachingExecutor.update(CachingExecutor.java:76)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at org.apache.ibatis.plugin.Plugin.invoke(Plugin.java:63)
	at com.sun.proxy.$Proxy107.update(Unknown Source)
	at org.apache.ibatis.session.defaults.DefaultSqlSession.update(DefaultSqlSession.java:198)
	at org.apache.ibatis.session.defaults.DefaultSqlSession.insert(DefaultSqlSession.java:185)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at org.mybatis.spring.SqlSessionTemplate$SqlSessionInterceptor.invoke(SqlSessionTemplate.java:433)
	at com.sun.proxy.$Proxy75.insert(Unknown Source)
	at org.mybatis.spring.SqlSessionTemplate.insert(SqlSessionTemplate.java:278)

错误如上图,发生原因是因为服务器内存过下,而批量插入的数据量较大导致的。

主要代码如下

@Override
    @Transactional
    public Map<String, Object> intoDatabases(InputStream inputStream, UserSession session) throws IOException {
        Map<String, Object> map = new HashMap<String, Object>();
        try {
            HSSFWorkbook workbook = new HSSFWorkbook(inputStream);//接收到的报表对象
            inputStream.close();
            HSSFSheet sheet = workbook.getSheetAt(0);
            workbook.cloneSheet(0);
            //获得当前sheet的结束行
            int lastRowNum = sheet.getLastRowNum();
//			System.out.println(lastRowNum);
            String batch = DateUtils.getBatch();//批次

            Map<String, Integer> cityNumber = new HashMap<String, Integer>();
            Map<String, String> areaMap = new HashMap<String, String>();
            List<Number> numbers = new ArrayList<Number>();
            int nums = 0;
            for (int i = 1; i <= lastRowNum; i++) {
                HSSFRow row = sheet.getRow(i);
                Number number = new Number();
//                String city = getStringCellValue(row.getCell(1));//省份
                String city = getStringCellValue(row.getCell(2));//城市
//				System.out.println(city+"---------"+i);
                map.put("areaName", city);
                map.put("level", 2);
                String areaId = areaMapper.findByAreaName(map).getAreaId() + "";
                number.setCityId(areaId);//号码归属地城市
                number.setCity(city);//号码归属地城市
                //存储城市id以及城市name
                areaMap.put(areaId, city);
//				System.out.println(getStringCellValue(row.getCell(1))+"------------------------"+i);
                number.setSource(getStringCellValue(row.getCell(0)));//号码来源
                number.setNumber(getStringCellValue(row.getCell(3)));//号码
                number.setAreaCode(getStringCellValue(row.getCell(4)));//区号
                number.setImsi(getStringCellValue(row.getCell(5)));//IMSI
                number.setSmsc(getStringCellValue(row.getCell(6)));//短信中心
//                number.setRemarks(getStringCellValue(row.getCell(5)));//备注
                number.setBatch(batch);//批次
                number.setIsSign("1");//标记
                number.setBalance("0");//余额
                number.setStatus("0");//状态空闲
                numbers.add(number);
//				numberMapper.insert(number);

                //存储城市号码总数量
                if (cityNumber.containsKey(areaId)) {
                    cityNumber.put(areaId, cityNumber.get(areaId) + 1);
                } else {
                    cityNumber.put(areaId, 1);
                }
            }

            //存储所有的号码
            int batchCount = 3000; //每批插入数目
            int batchLastIndex = batchCount;
            List<Number> shareList = new ArrayList<>();
            for (int index = 0; index < numbers.size(); ) {
                if (batchLastIndex >= numbers.size()) {
                    batchLastIndex = numbers.size();
                    shareList = numbers.subList(index, batchLastIndex);
                    break;
                } else {
                    shareList = numbers.subList(index, batchLastIndex);
                    index = batchLastIndex;// 设置下一批下标
                    batchLastIndex = index + (batchCount - 1);
                }
                numberMapper.batchInsert(shareList);
            }
            numberMapper.batchInsert(shareList);

//            numberMapper.batchInsert(numbers);

            //插入记录表
            List<ImportRecord> list = new ArrayList<ImportRecord>();

            for (String key : cityNumber.keySet()) {
                ImportRecord importRecord = new ImportRecord();
                importRecord.setBatch(batch);//批次
                importRecord.setCityId(key);//城市id
                importRecord.setCityName(areaMap.get(key));//城市名称
                importRecord.setCount(cityNumber.get(key));//总数量
                importRecord.setAdminId(session.getAdminId());//导入人
                importRecord.setAdminName(session.getAdminName());//管理员名称
                list.add(importRecord);
            }

            importRecordMapper.batchInsert(list);

            //插入申请记录表type=5,员工导入号码
            ApplyRecord applyRecord = new ApplyRecord();
            applyRecord.setApplicant(session.getAdminId());//申请人
            applyRecord.setCreateTime(new Date());//申请时间
            applyRecord.setNum(numbers.size());//退订数量
            applyRecord.setStatus("1");//状态
            applyRecord.setBatch(batch);//批次
            applyRecord.setType(5);
            applyRecordMapper.insert(applyRecord);

            map.clear();
            map.put("message", "操作成功");
            map.put("status", "success");
            map.put("icon", "6");
        } catch (Exception e) {
            map.put("message", "操作失败");
            map.put("status", "error");
            map.put("icon", "5");
            e.printStackTrace();
            throw e;
        } finally {
            inputStream.close();
        }

        return map;
    }
<insert id="batchInsert">
        INSERT INTO num_number
        (number, segment, ascription, balance, status, begin_time, end_time,
        city_id,city, is_sign, area_code, business,imsi, smsc, batch, remarks)
        VALUES
        <foreach collection="list" index="index" item="number" separator=",">
            (#{number.number}, #{number.segment}, #{number.ascription}, #{number.balance},
            #{number.status}, #{number.beginTime}, #{number.endTime},#{number.cityId},
            #{number.city}, #{number.isSign}, #{number.areaCode},#{number.business},
            #{number.imsi}, #{number.smsc}, #{number.batch}, #{number.remarks})
        </foreach>
    </insert>

数据量大概有八万条多一点,本地测试正常,发布线上报错内存溢出。

解决方案

把list做拆分处理,分批执行批量插入的操作

java拆分list代码如下

public static void main(String[] args) {
    //list 为全量集合
    List<String> list = new ArrayList<>();
    for (int i=0; i<96; i++){
        list.add(i+"");
    }
    int batchCount = 15; //每批插入数目
    int batchLastIndex = batchCount;
    List<String> shareList = new ArrayList<>();
    for (int index = 0; index < list.size(); ) {
        if (batchLastIndex >= list.size()) {
            batchLastIndex = list.size();
            shareList = list.subList(index, batchLastIndex);
            break;
        } else {
            shareList = list.subList(index, batchLastIndex);
            index = batchLastIndex;// 设置下一批下标
            batchLastIndex = index + (batchCount - 1);
        }
        System.out.println("-------------------------"+index);
        for(int i=0; i<shareList.size(); i++){
            System.out.println(shareList.get(i));
        }
    }
    System.out.println("-----------last--------------");
    if (!EmptyUtils.isEmpty(shareList)) {
        for(int i=0; i<shareList.size(); i++){
            System.out.println(shareList.get(i));
        }
    }

}

此处不讨论其他问题,只讨论当服务器性能不足时,mybaits批量带来的内存溢出问题。

 

其实还有一种解决方法,希望大家不要打我。那就是手动拼接sql,使用jdbc操作数据库。不要让mybaits帮忙拼接sql就可以了。

 

 

展开阅读全文
打赏
0
0 收藏
分享
加载中
更多评论
打赏
0 评论
0 收藏
0
分享
返回顶部
顶部