oracle - Spring batch JdbcPagingItemReader missing un commit records -

batch has 4 steps 1. basic task 2. pull records input table -> process -> out table 3. validate error count, check count records in input , output tables. 4. pull records out table , update status business tables.

i have issue count of records in out table coming correct when step 4 full records having less records. using oracle feel due commit not done when records pull.

please suggest if 1 face similar issue.

configuration follows:

<beans xmlns="http://www.springframework.org/schema/beans" xmlns:context="http://www.springframework.org/schema/context" xmlns:tx="http://www.springframework.org/schema/tx" xmlns:xsi="http://www.w3.org/2001/xmlschema-instance" xmlns:aop="http://www.springframework.org/schema/aop" xmlns:util="http://www.springframework.org/schema/util" xmlns:orcl="http://www.springframework.org/schema/data/orcl" xmlns:batch="http://www.springframework.org/schema/batch" xsi:schemalocation="http://www.springframework.org/schema/beans                            http://www.springframework.org/schema/beans/spring-beans.xsd                         http://www.springframework.org/schema/context                          http://www.springframework.org/schema/context/spring-context-2.5.xsd                         http://www.springframework.org/schema/tx                               http://www.springframework.org/schema/tx/spring-tx-3.0.xsd                        http://www.springframework.org/schema/aop                         http://www.springframework.org/schema/aop/spring-aop-3.2.xsd                        http://www.springframework.org/schema/util                         http://www.springframework.org/schema/util/spring-util.xsd                        http://www.springframework.org/schema/data/orcl                        http://www.springframework.org/schema/data/orcl/spring-data-orcl-1.0.xsd                        http://www.springframework.org/schema/batch                         http://www.springframework.org/schema/batch/spring-batch-3.0.xsd ">  <context:component-scan base-package="com.test." />  <import resource="classpath:context-datasource.xml" />  <bean id="parametersetter" class="com.test.helper.queryparamsetter"     scope="step">     <constructor-arg value="#{jobparameters['filename']}" /> </bean>  <bean id="pagingitemreader"     class="org.springframework.batch.item.database.jdbcpagingitemreader"     scope="step">     <property name="datasource" ref="datasource" />     <property name="queryprovider">         <bean             class="org.springframework.batch.item.database.support.sqlpagingqueryproviderfactorybean">             <property name="datasource" ref="datasource" />             <property name="selectclause" value="select *" />             <property name="fromclause" value="from input_table" />             <property name="whereclause"                 value="file_name= :filename , rownum &lt; 3001" />              <property name="sortkey" value="inpiut_stg_id" />          </bean>     </property>     <property name="parametervalues">         <map>             <entry key="filename" value="#{jobparameters['filename']}" />         </map>     </property>     <property name="pagesize" value="100" />     <property name="rowmapper">         <bean class="com.test.inputvomapper" />     </property> </bean>  <bean id="postitemreader"     class="org.springframework.batch.item.database.jdbcpagingitemreader"     scope="step">     <property name="datasource" ref="datasource" />     <property name="queryprovider">         <bean             class="org.springframework.batch.item.database.support.sqlpagingqueryproviderfactorybean">             <property name="datasource" ref="datasource" />             <property name="selectclause" value="select *" />             <property name="fromclause" value="from output_table" />             <property name="whereclause"                 value="file_name= :filename , txn_id not null , file_data not null" />             <property name="sortkey" value="create_dt" />          </bean>     </property>     <property name="parametervalues">         <map>             <entry key="filename" value="#{jobparameters['filename']}" />         </map>     </property>     <property name="pagesize" value="100" />     <property name="rowmapper">         <bean class="com.test.mapper.outputvomapper" />     </property> </bean>  <!-- jobexecutionlistener perform business logic before , after      job --> <bean id="joblistener" class="com.test.settlementjoblistener">      <property name="templatedir" value="templates" /> </bean>  <bean id="processeventhelper" class="com.wdpr.payment.helper.processeventhelper" />  <!-- itemwriter writes data database -->  <bean id="errorwriter"     class="org.springframework.batch.item.database.jdbcbatchitemwriter">     <property name="datasource" ref="datasource" />     <property name="sql"         value="update input_table set err_cd=:batcherrorcode, err_ds=:batcherrordesc proc_id=:rrn" />     <property name="itemsqlparametersourceprovider">         <bean             class="org.springframework.batch.item.database.beanpropertyitemsqlparametersourceprovider" />     </property> </bean> <bean id="recordwriter"     class="org.springframework.batch.item.database.jdbcbatchitemwriter">     <property name="datasource" ref="datasource" />     <property name="sql"         value="insert output_table (card_settl_id,txn_id,create_dt,create_proc_id,file_data,client_id,file_name) values (:a,:b,:c,:d,:e,:f,:g)" />     <property name="itemsqlparametersourceprovider">         <bean             class="org.springframework.batch.item.database.beanpropertyitemsqlparametersourceprovider" />     </property> </bean>  <bean id="databaseitemwriter" class="com.test.settlementdatawriter"     scope="step">     <property name="errorwriter" ref="errorwriter" />     <property name="recordwriter" ref="recordwriter" /> </bean>  <bean id="itemprocessor" class="com.test.settlementdataprocessor"     scope="step" />  <bean id="preprocessor" class="com.test.preprocessor" /> <!-- step need transaction manager --> <bean id="transactionmanager"     class="org.springframework.batch.support.transaction.resourcelesstransactionmanager" />   <bean id="statusitemwriter" class="com.test.statusitemwriter" scope="step">     <property name="processorhelper" ref="processorhelper" /> </bean>  <bean id="errorcheck" class="com.test.errorchecktasklet"     scope="step">     <property name="jdbctemplate" ref="jdbctemplate" />     <property name="sql"         value="select count(*) input_table file_name= ? , err_cd not null" />     <property name="footercount"         value="select count(*) output_table file_name= ? , file_data not null" />     <property name="countcheck"         value="select count(*) output_table file_name= ? , txn_id not null , file_data not null" />         <!-- <property name="preparedstatementsetter" ref="parametersetter" /> -->     <property name="recordwriter" ref="recordwriter" /> </bean>  <bean id="preprocessreader" class="com.test.preprocessreader"     scope="step">     <property name="jdbctemplate" ref="jdbctemplate" />     <property name="sql1"         value="select count(proc_id) input_table file_name=? , rownum &lt; 101" />     <property name="sql2"         value="select count(distinct(proc_id)) input_table file_name=? , rownum &lt; 101" />     <property name="preparedstatementsetter" ref="parametersetter" /> </bean>  <bean id="taskexecutor" class="org.springframework.core.task.simpleasynctaskexecutor" >     <property name="concurrencylimit" value="100" /> </bean>  <!-- actual job --> <batch:job id="settelmentjob">      <batch:step id="step1" next="step2">         <batch:tasklet transaction-manager="transactionmanager">             <batch:chunk reader="preprocessreader" processor="preprocessor"                 writer="recordwriter" commit-interval="1" />         </batch:tasklet>     </batch:step>      <batch:step id="step2" next="step3" >         <batch:tasklet transaction-manager="transactionmanager"             task-executor="taskexecutor" throttle-limit="100"  ><!-- throttle-limit="100" -->             <batch:chunk reader="pagingitemreader" writer="databaseitemwriter"                 processor="itemprocessor"  commit-interval="100"  /> <!-- commit-interval="100"  -->         </batch:tasklet>     </batch:step>      <batch:step id="step3" >         <batch:tasklet transaction-manager="transactionmanager"              ref="errorcheck" >         </batch:tasklet>     </batch:step>      <batch:step id="step4">         <batch:tasklet transaction-manager="transactionmanager"             task-executor="taskexecutor" throttle-limit="100" >             <batch:chunk reader="postitemreader" writer="statusitemwriter"                 commit-interval="100" />          </batch:tasklet>          </batch:step>      <batch:listeners>         <batch:listener ref="joblistener" />     </batch:listeners> </batch:job>  <!-- jobrepository , joblauncher configuration/setup classes -->  <bean id="jobrepository"     class="org.springframework.batch.core.repository.support.mapjobrepositoryfactorybean"> </bean>  <bean id="joblauncher"     class="org.springframework.batch.core.launch.support.simplejoblauncher">     <property name="jobrepository" ref="jobrepository" /> </bean> 

data source

bean id="datasource" class="org.apache.commons.dbcp2.basicdatasource" destroy-method="close"> <property name="driverclassname" value="${driverclassname}" /> <property name="url" value="${url}" /> <property name="username" value="${username}" /> <property name="password" value="${password}" /> <property name="initialsize" value="30" /> <property name="maxidle" value="10" /> <property name="validationquery" value="select 1 dual" /> <property name="maxtotal" value="50"></property> <property name="testwhileidle" value="true"/> <property name="timebetweenevictionrunsmillis" value="10000"/> <property name="removeabandonedonborrow" value="true"/> 

i believe issue you're using resourcelesstransactionmanager. there 2 issues this:

1. it means aren't using transactions (since doesn't work resources).
2. it means spring batch won't wait commit complete before proceeding since true commit isn't being called.

you're using map based jobrepository. should use transactionmanager datasource you're working oracle on. fix issue.