从头认识SpringBatch批处理框架---JobRepository数据库存储元数据并分析存储元数据表

旧城等待， 2022-08-20 12:10 311阅读 0赞

JobRepository存储执行期的元数据，提供两种默认实现,一种是存放在内存中,默认实现类为:MapJobRepositoryFactoryBean。在xml中的配置如下:

另一种是存入数据库中,可以随时监控批处理Job的执行状态,查看Job执行结果是成功还是失败,并且使得在Job失败的情况下重新启动Job成为可能。具体配置如下:

<?xml version="1.0" encoding="UTF-8"?>
    <bean:beans xmlns="http://www.springframework.org/schema/batch"    
        xmlns:bean="http://www.springframework.org/schema/beans" 
        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"    
        xmlns:p="http://www.springframework.org/schema/p" 
        xmlns:tx="http://www.springframework.org/schema/tx"    
        xmlns:aop="http://www.springframework.org/schema/aop" 
        xmlns:context="http://www.springframework.org/schema/context"    
        xsi:schemaLocation="http://www.springframework.org/schema/beans 
        http://www.springframework.org/schema/beans/spring-beans-3.0.xsd  
        http://www.springframework.org/schema/tx 
        http://www.springframework.org/schema/tx/spring-tx-3.0.xsd  
        http://www.springframework.org/schema/aop 
        http://www.springframework.org/schema/aop/spring-aop-3.0.xsd  
        http://www.springframework.org/schema/context 
        http://www.springframework.org/schema/context/spring-context-2.5.xsd
        http://www.springframework.org/schema/batch 
        http://www.springframework.org/schema/batch/spring-batch-2.2.xsd">
    	
    	
    	<job-repository id="jobRepository" data-source="dataSource"
    	    transaction-manager="transactionManager" isolation-level-for-create="SERIALIZABLE"
    	    table-prefix="BATCH_" max-varchar-length="1000"
    	/>
    	
    	
    	<bean:bean id="jobLauncher" 
    		class="org.springframework.batch.core.launch.support.SimpleJobLauncher">
    		<bean:property name="jobRepository" ref="jobRepository"/>
    	</bean:bean>
    	
    	
    	<bean:bean id="transactionManager" class="org.springframework.jdbc.datasource.DataSourceTransactionManager">
    		<bean:property name="dataSource" ref="dataSource" />
    	</bean:bean>
    	
    	
        <bean:bean id="dataSource" class="org.springframework.jdbc.datasource.DriverManagerDataSource">
      		<bean:property name="driverClassName">
       			<bean:value>com.mysql.jdbc.Driver</bean:value>
      		</bean:property>
      		<bean:property name="url">
       			<bean:value>jdbc:mysql://localhost:3306/mydatabase</bean:value>
      		</bean:property>
      		<bean:property name="username" value="root"></bean:property>
      		<bean:property name="password" value="wbw123456"></bean:property>
     	</bean:bean>
    </bean:beans>

注意：在使用数据库的仓库时，需要初始化数据库表,数据库表脚本位置放在org\\springframework\\batch\\spring-batch-core\\3.0.6.RELEASE\\spring-batch-core-3.0.6.RELEASE.jar目录下,并且针对不同的数据库提供了不同的数据库脚本

![Center][]

在这里我们选用schema-mysql.sql 具体的建表语句如下:

-- Autogenerated: do not edit this file
    
    CREATE TABLE BATCH_JOB_INSTANCE  (
    	JOB_INSTANCE_ID BIGINT  NOT NULL PRIMARY KEY ,
    	VERSION BIGINT ,
    	JOB_NAME VARCHAR(100) NOT NULL,
    	JOB_KEY VARCHAR(32) NOT NULL,
    	constraint JOB_INST_UN unique (JOB_NAME, JOB_KEY)
    ) ENGINE=InnoDB;
    
    CREATE TABLE BATCH_JOB_EXECUTION  (
    	JOB_EXECUTION_ID BIGINT  NOT NULL PRIMARY KEY ,
    	VERSION BIGINT  ,
    	JOB_INSTANCE_ID BIGINT NOT NULL,
    	CREATE_TIME DATETIME NOT NULL,
    	START_TIME DATETIME DEFAULT NULL ,
    	END_TIME DATETIME DEFAULT NULL ,
    	STATUS VARCHAR(10) ,
    	EXIT_CODE VARCHAR(2500) ,
    	EXIT_MESSAGE VARCHAR(2500) ,
    	LAST_UPDATED DATETIME,
    	JOB_CONFIGURATION_LOCATION VARCHAR(2500) NULL,
    	constraint JOB_INST_EXEC_FK foreign key (JOB_INSTANCE_ID)
    	references BATCH_JOB_INSTANCE(JOB_INSTANCE_ID)
    ) ENGINE=InnoDB;
    
    CREATE TABLE BATCH_JOB_EXECUTION_PARAMS  (
    	JOB_EXECUTION_ID BIGINT NOT NULL ,
    	TYPE_CD VARCHAR(6) NOT NULL ,
    	KEY_NAME VARCHAR(100) NOT NULL ,
    	STRING_VAL VARCHAR(250) ,
    	DATE_VAL DATETIME DEFAULT NULL ,
    	LONG_VAL BIGINT ,
    	DOUBLE_VAL DOUBLE PRECISION ,
    	IDENTIFYING CHAR(1) NOT NULL ,
    	constraint JOB_EXEC_PARAMS_FK foreign key (JOB_EXECUTION_ID)
    	references BATCH_JOB_EXECUTION(JOB_EXECUTION_ID)
    ) ENGINE=InnoDB;
    
    CREATE TABLE BATCH_STEP_EXECUTION  (
    	STEP_EXECUTION_ID BIGINT  NOT NULL PRIMARY KEY ,
    	VERSION BIGINT NOT NULL,
    	STEP_NAME VARCHAR(100) NOT NULL,
    	JOB_EXECUTION_ID BIGINT NOT NULL,
    	START_TIME DATETIME NOT NULL ,
    	END_TIME DATETIME DEFAULT NULL ,
    	STATUS VARCHAR(10) ,
    	COMMIT_COUNT BIGINT ,
    	READ_COUNT BIGINT ,
    	FILTER_COUNT BIGINT ,
    	WRITE_COUNT BIGINT ,
    	READ_SKIP_COUNT BIGINT ,
    	WRITE_SKIP_COUNT BIGINT ,
    	PROCESS_SKIP_COUNT BIGINT ,
    	ROLLBACK_COUNT BIGINT ,
    	EXIT_CODE VARCHAR(2500) ,
    	EXIT_MESSAGE VARCHAR(2500) ,
    	LAST_UPDATED DATETIME,
    	constraint JOB_EXEC_STEP_FK foreign key (JOB_EXECUTION_ID)
    	references BATCH_JOB_EXECUTION(JOB_EXECUTION_ID)
    ) ENGINE=InnoDB;
    
    CREATE TABLE BATCH_STEP_EXECUTION_CONTEXT  (
    	STEP_EXECUTION_ID BIGINT NOT NULL PRIMARY KEY,
    	SHORT_CONTEXT VARCHAR(2500) NOT NULL,
    	SERIALIZED_CONTEXT TEXT ,
    	constraint STEP_EXEC_CTX_FK foreign key (STEP_EXECUTION_ID)
    	references BATCH_STEP_EXECUTION(STEP_EXECUTION_ID)
    ) ENGINE=InnoDB;
    
    CREATE TABLE BATCH_JOB_EXECUTION_CONTEXT  (
    	JOB_EXECUTION_ID BIGINT NOT NULL PRIMARY KEY,
    	SHORT_CONTEXT VARCHAR(2500) NOT NULL,
    	SERIALIZED_CONTEXT TEXT ,
    	constraint JOB_EXEC_CTX_FK foreign key (JOB_EXECUTION_ID)
    	references BATCH_JOB_EXECUTION(JOB_EXECUTION_ID)
    ) ENGINE=InnoDB;
    
    CREATE TABLE BATCH_STEP_EXECUTION_SEQ (
    	ID BIGINT NOT NULL,
    	UNIQUE_KEY CHAR(1) NOT NULL,
    	constraint UNIQUE_KEY_UN unique (UNIQUE_KEY)
    ) ENGINE=InnoDB;
    
    INSERT INTO BATCH_STEP_EXECUTION_SEQ (ID, UNIQUE_KEY) select * from (select 0 as ID, '0' as UNIQUE_KEY) as tmp where not exists(select * from BATCH_STEP_EXECUTION_SEQ);
    
    CREATE TABLE BATCH_JOB_EXECUTION_SEQ (
    	ID BIGINT NOT NULL,
    	UNIQUE_KEY CHAR(1) NOT NULL,
    	constraint UNIQUE_KEY_UN unique (UNIQUE_KEY)
    ) ENGINE=InnoDB;
    
    INSERT INTO BATCH_JOB_EXECUTION_SEQ (ID, UNIQUE_KEY) select * from (select 0 as ID, '0' as UNIQUE_KEY) as tmp where not exists(select * from BATCH_JOB_EXECUTION_SEQ);
    
    CREATE TABLE BATCH_JOB_SEQ (
    	ID BIGINT NOT NULL,
    	UNIQUE_KEY CHAR(1) NOT NULL,
    	constraint UNIQUE_KEY_UN unique (UNIQUE_KEY)
    ) ENGINE=InnoDB;
    
    INSERT INTO BATCH_JOB_SEQ (ID, UNIQUE_KEY) select * from (select 0 as ID, '0' as UNIQUE_KEY) as tmp where not exists(select * from BATCH_JOB_SEQ);

SpringBatch框架进行元数据的管理共有九张表:

![Center 1][]

下面分别对9张表的数据结构讲解哈:

**BATCH\_JOB\_INSTANCE(作业实例表,存放JOB的实例信息)**

<table> 
 <tbody> 
  <tr> 
   <td>字段名称</td> 
   <td>字段 类型</td> 
   <td><br> 字段说明<br> <br> </td> 
  </tr> 
  <tr> 
   <td>JOB_INSTANCE_ID</td> 
   <td>bigint&nbsp;&nbsp;</td> 
   <td>主键 ，作业实例编号,根据BATCH_JOB_SEQ自动生成</td> 
  </tr> 
  <tr> 
   <td>VERSION &nbsp; &nbsp; &nbsp;</td> 
   <td>bigint&nbsp;</td> 
   <td>版本号</td> 
  </tr> 
  <tr> 
   <td>JOB_NAME</td> 
   <td><br> varchar<br> <br> </td> 
   <td>作业名称 即在配置文件定义的JobId字段内容</td> 
  </tr> 
  <tr> 
   <td>JOB_KEY</td> 
   <td>varchar</td> 
   <td>作业标识,根据作业参数序列化生成的标识,需要注意通过JOB_NAME+JOB_KEY能够唯一区分一个作业实例</td> 
  </tr> 
 </tbody> 
</table>

注意:JOB\_KEY------如果同一个Job,则JOB\_KEY一定不能相同,籍作业参数不能相同,如果不是同一个Job,JOB\_KEY可以相同,也就是作业参数可以相同

**BATCH\_JOB\_EXECUTION\_PARAMS(作业参数表,用于存放每个JOB执行的时候的参数信息,即对于的JOB实例)**

<table> 
 <tbody> 
  <tr> 
   <td>字段名称</td> 
   <td>字段类型</td> 
   <td>字段说明</td> 
  </tr> 
  <tr> 
   <td>JOB_EXECUTION_ID</td> 
   <td>bigint</td> 
   <td>外键ID,作业执行器ID编号，一个作业实例可能又会多行参数记录</td> 
  </tr> 
  <tr> 
   <td>TYPE_CD</td> 
   <td>varchar</td> 
   <td>参数的类型，总共有四种 String、Date、Double、Long</td> 
  </tr> 
  <tr> 
   <td>KEY_NAME</td> 
   <td>varchar</td> 
   <td>参数的名字</td> 
  </tr> 
  <tr> 
   <td>STRING_VAL</td> 
   <td>varchar</td> 
   <td>参数如果是String,就存放String类型的参数值</td> 
  </tr> 
  <tr> 
   <td>DATE_VAL</td> 
   <td>datetime</td> 
   <td>参数如果是Date,就存放Date类型的参数值</td> 
  </tr> 
  <tr> 
   <td>LONG_VAL</td> 
   <td>bigint</td> 
   <td>参数如果是Long,就存放Long类型的参数值</td> 
  </tr> 
  <tr> 
   <td>DOUBLE_VAL</td> 
   <td>double</td> 
   <td>参数如果是Double,就存放Double类型的参数值</td> 
  </tr> 
 </tbody> 
</table>

IDENTIFYING                           char              用于标识作业参数是否标识作业实例

\------------------------------------------------------------------------------------------------------------------------------------------------------------------

**BATCH\_JOB\_EXECUTION(作业执行器表,存放当前作业的执行信息,创建时间、执行时间、执行结束时间、执行状态、执行Job实例等信息)**

<table> 
 <tbody> 
  <tr> 
   <td>字段名称</td> 
   <td>字段类型</td> 
   <td>字段说明</td> 
  </tr> 
  <tr> 
   <td>JOB_EXECUTION_ID</td> 
   <td>bigint</td> 
   <td>主键</td> 
  </tr> 
  <tr> 
   <td>VERSION</td> 
   <td>bigint</td> 
   <td>版本号</td> 
  </tr> 
  <tr> 
   <td>JOB_INSTANCE_ID</td> 
   <td>bigint</td> 
   <td>作业实例ID</td> 
  </tr> 
  <tr> 
   <td>CREATE_TIME</td> 
   <td>datetime</td> 
   <td>作业执行器创建时间</td> 
  </tr> 
  <tr> 
   <td>START_TIME</td> 
   <td>datetime</td> 
   <td>作业执行器开始执行时间</td> 
  </tr> 
  <tr> 
   <td>END_TIME</td> 
   <td>datetime</td> 
   <td>作业执行器结束时间&nbsp;</td> 
  </tr> 
  <tr> 
   <td>STATUS</td> 
   <td>varchar</td> 
   <td>作业执行器执行的状态:COMPLETED、FAILED、STARTING、UNKNOW等</td> 
  </tr> 
  <tr> 
   <td>EXIT_CODE</td> 
   <td>varchar</td> 
   <td>作业执行器退出代码 如：UNKNOW、EXECUTING、COMPLETED等</td> 
  </tr> 
  <tr> 
   <td>EXIT_MESSAGE</td> 
   <td>varchar</td> 
   <td>作业执行器退出信息,通常存放异常信息</td> 
  </tr> 
  <tr> 
   <td>LAST_UPDATED</td> 
   <td>datetime</td> 
   <td>本条记录上次更新时间</td> 
  </tr> 
  <tr> 
   <td>JOB_CONFIGURATION_LOCATION</td> 
   <td>varchar</td> 
   <td>作业配置文件的位置</td> 
  </tr> 
  <tr> 
   <td>&nbsp;</td> 
   <td>&nbsp;</td> 
   <td>&nbsp;</td> 
  </tr> 
 </tbody> 
</table>

运用BATCH\_JOB\_INSTANCE和BATCH\_JOB\_EXECUTION两张表联合查询Job信息

SELECT
    	bi.JOB_NAME,bi.JOB_KEY,be.CREATE_TIME,be.START_TIME,be.END_TIME,be.`STATUS`
    FROM
    	batch_job_instance bi
    LEFT JOIN batch_job_execution be ON bi.JOB_INSTANCE_ID = be.JOB_INSTANCE_ID
    where bi.JOB_NAME='billJob'

**BATCH\_JOB\_EXECUTION\_CONTENXT(作业执行上下文表,存放上下文信息)**

<table> 
 <tbody> 
  <tr> 
   <td>字段名称</td> 
   <td>字段类型</td> 
   <td>字段说明</td> 
  </tr> 
  <tr> 
   <td>JOB_EXECUTION_ID</td> 
   <td>bigint</td> 
   <td>外键ID,作业执行器编号</td> 
  </tr> 
  <tr> 
   <td>SHORT_CONTEXT</td> 
   <td>varchar</td> 
   <td>作业执行器上下文字符串格式</td> 
  </tr> 
  <tr> 
   <td>SERIALIZED_CONTEXT</td> 
   <td>text</td> 
   <td>序列化的作业执行上下文</td> 
  </tr> 
 </tbody> 
</table>

**BATCH\_JOB\_EXECUTION\_SEQ表（用于BATCH\_JOB\_EXECUTION和BATCH\_JOB\_EXECUTION\_CONTEXT提供主键生成）**

**BATCH\_JOB\_SEQ表（用于BATCH\_JOB\_INSTNCE和BATCH\_JOB\_EXECUTION\_PARAMS提供主键生成）**

**BATCH\_STEP\_EXECUTION(作业步执行器，存放每个Step执行器的信息：开始时间、执行完成时间、读/写次数等)**

<table style="font-weight:bold"> 
 <tbody> 
  <tr> 
   <td>字段名称</td> 
   <td>字段类型</td> 
   <td>字段说明</td> 
  </tr> 
  <tr> 
   <td>STEP_EXECUTION_ID</td> 
   <td>bigint</td> 
   <td>&nbsp;主键</td> 
  </tr> 
  <tr> 
   <td>VERSION</td> 
   <td>bigint</td> 
   <td>版本</td> 
  </tr> 
  <tr> 
   <td>STEP_NAME</td> 
   <td>varchar</td> 
   <td>作业步的名称</td> 
  </tr> 
  <tr> 
   <td>JOB_EXECUTION_ID</td> 
   <td>bigint</td> 
   <td>作业执行器ID</td> 
  </tr> 
  <tr> 
   <td>START_TIME</td> 
   <td>datetime</td> 
   <td>作业步执行器执行开始时间</td> 
  </tr> 
  <tr> 
   <td>END_TIME</td> 
   <td>datetime</td> 
   <td>作业步执行器执行结束时间</td> 
  </tr> 
  <tr> 
   <td>STATUS</td> 
   <td>varchar</td> 
   <td>作业步执行器执行状态<br> &nbsp;COMPLETED, STARTING, STARTED, STOPPING, STOPPED, FAILED, ABANDONED, UNKNOWN<br> &nbsp;存在于org.springframework.batch.core.BatchStatus</td> 
  </tr> 
  <tr> 
   <td>COMMIT_COUNT</td> 
   <td>bigint</td> 
   <td>事务提交次数</td> 
  </tr> 
  <tr> 
   <td>READ_COUNT</td> 
   <td>bigint</td> 
   <td>读数据次数</td> 
  </tr> 
  <tr> 
   <td>FILTER_COUNT</td> 
   <td>bigint</td> 
   <td>过滤掉的数据次数</td> 
  </tr> 
  <tr> 
   <td>WRITE_COUNT</td> 
   <td>bigint</td> 
   <td>写数据次数</td> 
  </tr> 
  <tr> 
   <td>READ_SKIP_COUNT</td> 
   <td>bigint</td> 
   <td>读数据跳过的次数</td> 
  </tr> 
  <tr> 
   <td>WRITE_SKIP_COUNT</td> 
   <td>bigint</td> 
   <td>写数据跳过的次数</td> 
  </tr> 
  <tr> 
   <td>PROCESS_SKIP_COUNT</td> 
   <td>bigint</td> 
   <td>处理数据跳过的次数</td> 
  </tr> 
  <tr> 
   <td>ROLLBACK_COUNT</td> 
   <td>bigint</td> 
   <td>事务回滚的次数</td> 
  </tr> 
  <tr> 
   <td>EXIT_CODE</td> 
   <td>varchar</td> 
   <td>作业步执行器退出编码，状态码存在于org.springframework.batch.core.ExitStauts</td> 
  </tr> 
  <tr> 
   <td>EXIT_MESSAGE</td> 
   <td>varchar</td> 
   <td>作业步执行器退出描述,一般是异常信息</td> 
  </tr> 
  <tr> 
   <td>LAST_UPDATED</td> 
   <td>datetime</td> 
   <td>本条记录上次更新时间</td> 
  </tr> 
  <tr> 
   <td>&nbsp;</td> 
   <td>&nbsp;</td> 
   <td>&nbsp;</td> 
  </tr> 
 </tbody> 
</table>

**BATCH\_STEP\_EXECUTION\_CONTEXT(作业步执行器上下文,存放作业步执行的上下文信息)**

<table> 
 <tbody> 
  <tr> 
   <td>字段名称</td> 
   <td>字段类型</td> 
   <td>字段说明</td> 
  </tr> 
  <tr> 
   <td>STEP_EXECUTION_ID</td> 
   <td>bigint</td> 
   <td>作业步执行器ID</td> 
  </tr> 
  <tr> 
   <td>SHORT_CONTEXT</td> 
   <td>varchar</td> 
   <td>作业步执行器上下文字符串格式</td> 
  </tr> 
  <tr> 
   <td>SERIALIZED_CONTEXT</td> 
   <td>text</td> 
   <td>序列化<span style="font-size:14px">作业步执行器上下文</span></td> 
  </tr> 
 </tbody> 
</table>

**BATCH\_STEP\_EXECUTION\_SEQ（用于BATCH\_STEP\_EXECUTION和BATCH\_STEP\_EXECUTION\_CONTEXT提供主键）**

[Center]: /images/20220731/1c6c248110694cc0999d4771bbf1964f.png
[Center 1]: /images/20220731/fceb4ce445f64a86af5ac8ba59d3ee3d.png