初始化大量历史数据时踩过的坑

    技术2022-07-10  109

    1.当数据量特别大的时候要注意不要再任务的总方法上加事务,这样会导致整个任务只是用同一个数据库连接,当任务执行时间过长时,连接可能会达到最大可用时长,这个时长的配置在druid连接池表现为下面这几个属性。当连接达到最大可用时长而事务仍然没有提交时,连接池会删除这个连接,从而导致任务执行失败。

    <!-- 打开removeAbandoned功能 --> <property name="removeAbandoned" value="true" /> <!-- 2400秒,也就是40分钟 --> <property name="removeAbandonedTimeout" value="2400" /> <!-- 关闭abanded连接时输出错误日志 --> <property name="logAbandoned" value="true" />

    报错信息如下:

    20200630190028 9482462 [Druid-ConnectionPool-Destroy-821343459] ERROR -com.alibaba.druid.pool.DruidDataSource.removeAbandoned(DruidDataSource.java:2383) abandon connection, owner thread: scheduler-5, connected at : 1593505440065, open stackTrace at java.lang.Thread.getStackTrace(Thread.java:1552) at com.alibaba.druid.pool.DruidDataSource.getConnectionDirect(DruidDataSource.java:1209) at com.alibaba.druid.filter.FilterChainImpl.dataSource_connect(FilterChainImpl.java:4619) at com.alibaba.druid.filter.stat.StatFilter.dataSource_getConnection(StatFilter.java:666) at com.alibaba.druid.filter.FilterChainImpl.dataSource_connect(FilterChainImpl.java:4615) at com.alibaba.druid.pool.DruidDataSource.getConnection(DruidDataSource.java:1133) at com.alibaba.druid.pool.DruidDataSource.getConnection(DruidDataSource.java:1125) at com.alibaba.druid.pool.DruidDataSource.getConnection(DruidDataSource.java:104) at org.springframework.jdbc.datasource.DataSourceTransactionManager.doBegin(DataSourceTransactionManager.java:204) at org.springframework.transaction.support.AbstractPlatformTransactionManager.getTransaction(AbstractPlatformTransactionManager.java:373) at org.springframework.transaction.interceptor.TransactionAspectSupport.createTransactionIfNecessary(TransactionAspectSupport.java:463) at org.springframework.transaction.interceptor.TransactionAspectSupport.invokeWithinTransaction(TransactionAspectSupport.java:276) at org.springframework.transaction.interceptor.TransactionInterceptor.invoke(TransactionInterceptor.java:96) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179) at org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:653) at com.buybal.yinlian.task.job.MerchantTradeDataStatisticEveryDayTask$$EnhancerBySpringCGLIB$$1a826203.initHistoryData(<generated>) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:497) at org.springframework.scheduling.support.ScheduledMethodRunnable.run(ScheduledMethodRunnable.java:65) at org.springframework.scheduling.support.DelegatingErrorHandlingRunnable.run(DelegatingErrorHandlingRunnable.java:54) at org.springframework.scheduling.concurrent.ReschedulingRunnable.run(ReschedulingRunnable.java:81) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) ownerThread current state is BLOCKED, current stackTrace at java.util.zip.ZipFile.open(Native Method) at java.util.zip.ZipFile.<init>(ZipFile.java:219) at java.util.zip.ZipFile.<init>(ZipFile.java:149) at java.util.jar.JarFile.<init>(JarFile.java:166) at java.util.jar.JarFile.<init>(JarFile.java:130) at org.apache.tomcat.util.compat.JreCompat.jarFileNewInstance(JreCompat.java:188) at org.apache.tomcat.util.compat.JreCompat.jarFileNewInstance(JreCompat.java:173) at org.apache.catalina.webresources.AbstractArchiveResourceSet.openJarFile(AbstractArchiveResourceSet.java:313) at org.apache.catalina.webresources.AbstractSingleArchiveResourceSet.getArchiveEntry(AbstractSingleArchiveResourceSet.java:96) at org.apache.catalina.webresources.AbstractArchiveResourceSet.getResource(AbstractArchiveResourceSet.java:262) at org.apache.catalina.webresources.StandardRoot.getResourceInternal(StandardRoot.java:281) at org.apache.catalina.webresources.Cache.getResource(Cache.java:62) at org.apache.catalina.webresources.StandardRoot.getResource(StandardRoot.java:216) at org.apache.catalina.webresources.StandardRoot.getClassLoaderResource(StandardRoot.java:225) at org.apache.catalina.loader.WebappClassLoaderBase.findClassInternal(WebappClassLoaderBase.java:2227) at org.apache.catalina.loader.WebappClassLoaderBase.findClass(WebappClassLoaderBase.java:828) at org.apache.catalina.loader.WebappClassLoaderBase.loadClass(WebappClassLoaderBase.java:1277) at org.apache.catalina.loader.WebappClassLoaderBase.loadClass(WebappClassLoaderBase.java:1136) at org.apache.ibatis.logging.jdbc.ConnectionLogger.invoke(ConnectionLogger.java:72) at com.sun.proxy.$Proxy285.prepareStatement(Unknown Source) at org.apache.ibatis.executor.statement.PreparedStatementHandler.instantiateStatement(PreparedStatementHandler.java:75) at org.apache.ibatis.executor.statement.BaseStatementHandler.prepare(BaseStatementHandler.java:85) at org.apache.ibatis.executor.statement.RoutingStatementHandler.prepare(RoutingStatementHandler.java:57) at org.apache.ibatis.executor.SimpleExecutor.prepareStatement(SimpleExecutor.java:73) at org.apache.ibatis.executor.SimpleExecutor.doQuery(SimpleExecutor.java:59) at org.apache.ibatis.executor.BaseExecutor.queryFromDatabase(BaseExecutor.java:267) at org.apache.ibatis.executor.BaseExecutor.query(BaseExecutor.java:137) at org.apache.ibatis.executor.CachingExecutor.query(CachingExecutor.java:96) at org.apache.ibatis.executor.CachingExecutor.query(CachingExecutor.java:77) at sun.reflect.GeneratedMethodAccessor237.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:497) at org.apache.ibatis.plugin.Invocation.proceed(Invocation.java:49) at com.github.pagehelper.SqlUtil._processPage(SqlUtil.java:351) at com.github.pagehelper.SqlUtil.processPage(SqlUtil.java:329) at com.github.pagehelper.PageHelper.intercept(PageHelper.java:150) at org.apache.ibatis.plugin.Plugin.invoke(Plugin.java:60) at com.sun.proxy.$Proxy284.query(Unknown Source) at org.apache.ibatis.session.defaults.DefaultSqlSession.selectList(DefaultSqlSession.java:108) at org.apache.ibatis.session.defaults.DefaultSqlSession.selectList(DefaultSqlSession.java:102) at org.apache.ibatis.session.defaults.DefaultSqlSession.selectOne(DefaultSqlSession.java:66) at sun.reflect.GeneratedMethodAccessor506.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:497) at org.mybatis.spring.SqlSessionTemplate$SqlSessionInterceptor.invoke(SqlSessionTemplate.java:358) at com.sun.proxy.$Proxy227.selectOne(Unknown Source) at org.mybatis.spring.SqlSessionTemplate.selectOne(SqlSessionTemplate.java:163) at org.apache.ibatis.binding.MapperMethod.execute(MapperMethod.java:68) at org.apache.ibatis.binding.MapperProxy.invoke(MapperProxy.java:52) at com.sun.proxy.$Proxy254.selectByPrimaryKey(Unknown Source) at com.buybal.yinlian.task.service.impl.MerchantTradeDataServiceImpl.lambda$statisticTradeData$0(MerchantTradeDataServiceImpl.java:53) at com.buybal.yinlian.task.service.impl.MerchantTradeDataServiceImpl$$Lambda$69/90912826.test(Unknown Source) at java.util.stream.ReferencePipeline$2$1.accept(ReferencePipeline.java:174) at java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1374) at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:481) at java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:471) at java.util.stream.ForEachOps$ForEachOp.evaluateSequential(ForEachOps.java:151) at java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateSequential(ForEachOps.java:174) at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234) at java.util.stream.ReferencePipeline.forEach(ReferencePipeline.java:418) at com.buybal.yinlian.task.service.impl.MerchantTradeDataServiceImpl.statisticTradeData(MerchantTradeDataServiceImpl.java:58) at sun.reflect.GeneratedMethodAccessor511.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:497) at org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:317) at org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:190) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:157) at org.springframework.transaction.interceptor.TransactionInterceptor$1.proceedWithInvocation(TransactionInterceptor.java:99) at org.springframework.transaction.interceptor.TransactionAspectSupport.invokeWithinTransaction(TransactionAspectSupport.java:281) at org.springframework.transaction.interceptor.TransactionInterceptor.invoke(TransactionInterceptor.java:96) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179) at org.springframework.transaction.interceptor.TransactionInterceptor$1.proceedWithInvocation(TransactionInterceptor.java:99) at org.springframework.transaction.interceptor.TransactionAspectSupport.invokeWithinTransaction(TransactionAspectSupport.java:281) at org.springframework.transaction.interceptor.TransactionInterceptor.invoke(TransactionInterceptor.java:96) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179) at org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:92) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179) at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:207) at com.sun.proxy.$Proxy255.statisticTradeData(Unknown Source) at com.buybal.yinlian.task.job.MerchantTradeDataStatisticEveryDayTask.initHistoryData(MerchantTradeDataStatisticEveryDayTask.java:58) at com.buybal.yinlian.task.job.MerchantTradeDataStatisticEveryDayTask$$FastClassBySpringCGLIB$$1e73bfaa.invoke(<generated>) at org.springframework.cglib.proxy.MethodProxy.invoke(MethodProxy.java:204) at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.invokeJoinpoint(CglibAopProxy.java:717) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:157) at org.springframework.transaction.interceptor.TransactionInterceptor$1.proceedWithInvocation(TransactionInterceptor.java:99) at org.springframework.transaction.interceptor.TransactionAspectSupport.invokeWithinTransaction(TransactionAspectSupport.java:281) at org.springframework.transaction.interceptor.TransactionInterceptor.invoke(TransactionInterceptor.java:96) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179) at org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:653) at com.buybal.yinlian.task.job.MerchantTradeDataStatisticEveryDayTask$$EnhancerBySpringCGLIB$$1a826203.initHistoryData(<generated>) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:497) at org.springframework.scheduling.support.ScheduledMethodRunnable.run(ScheduledMethodRunnable.java:65) at org.springframework.scheduling.support.DelegatingErrorHandlingRunnable.run(DelegatingErrorHandlingRunnable.java:54) at org.springframework.scheduling.concurrent.ReschedulingRunnable.run(ReschedulingRunnable.java:81) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745)

     

    Processed: 0.016, SQL: 12