文档章节

spring batch +boot

j
 jackking2017
发布于 2017/07/24 09:34
字数 588
阅读 21
收藏 0

<?xml version="1.0" encoding="UTF-8"?>

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <modelVersion>4.0.0</modelVersion>

<groupId>org.podcastpedia.batch</groupId>
<artifactId>podcastpedia-batch</artifactId>
<version>0.1.0</version>

<properties>
	<sprinb.boot.version>1.1.6.RELEASE</sprinb.boot.version>
	<java.version>1.7</java.version>
</properties>
<parent>
    <groupId>org.springframework.boot</groupId>
    <artifactId>spring-boot-starter-parent</artifactId>
    <version>1.1.6.RELEASE</version>
</parent>

<dependencies>

    <dependency>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-batch</artifactId>

    </dependency>  
	<dependency>
	  <groupId>org.springframework.boot</groupId>
	  <artifactId>spring-boot-starter-data-jpa</artifactId>		   
	</dependency>        

	<dependency>
		<groupId>org.apache.httpcomponents</groupId>
		<artifactId>httpclient</artifactId>
		<version>4.3.5</version>
	</dependency>		
	<dependency>
		<groupId>org.apache.httpcomponents</groupId>
		<artifactId>httpcore</artifactId>
		<version>4.3.2</version>
	</dependency>
	<!-- velocity -->
	<dependency>
		<groupId>org.apache.velocity</groupId>
		<artifactId>velocity</artifactId>
		<version>1.7</version>		
	</dependency>
	<dependency>
		<groupId>org.apache.velocity</groupId>
		<artifactId>velocity-tools</artifactId>
		<version>2.0</version>
	    <exclusions>
	        <exclusion>
				<groupId>org.apache.struts</groupId>
				<artifactId>struts-core</artifactId>
	        </exclusion>
	    </exclusions>				
	</dependency>

	<!-- Project rome rss, atom -->
	<dependency>
		<groupId>rome</groupId>
		<artifactId>rome</artifactId>
		<version>1.0</version>
	</dependency>
	<!-- option this fetcher thing -->
	<dependency>
		<groupId>rome</groupId>
		<artifactId>rome-fetcher</artifactId>
		<version>1.0</version>
	</dependency>
	<dependency>
		<groupId>org.jdom</groupId>
		<artifactId>jdom</artifactId>
		<version>1.1</version>
	</dependency>		
	<!-- PID 1 -->
	<dependency>
		<groupId>xerces</groupId>
		<artifactId>xercesImpl</artifactId>
		<version>2.9.1</version>
	</dependency>

	<!-- MySQL JDBC connector -->
	<dependency>
		<groupId>mysql</groupId>
		<artifactId>mysql-connector-java</artifactId>
		<version>5.1.31</version>
	</dependency>
	<dependency>
		<groupId>org.springframework.boot</groupId>
		<artifactId>spring-boot-starter-freemarker</artifactId>   
	</dependency>
	<dependency>
		<groupId>org.springframework.boot</groupId>
		<artifactId>spring-boot-starter-remote-shell</artifactId>
	    <exclusions>
	        <exclusion>
				<groupId>javax.mail</groupId>
				<artifactId>mail</artifactId>
	        </exclusion>
	    </exclusions>				
	</dependency>
	<dependency>
		<groupId>javax.mail</groupId>
		<artifactId>mail</artifactId>
		<version>1.4.7</version>
	</dependency>		
	<dependency>
		<groupId>javax.inject</groupId>
		<artifactId>javax.inject</artifactId>
		<version>1</version>
	</dependency>		
	<dependency>
		<groupId>org.twitter4j</groupId>
		<artifactId>twitter4j-core</artifactId>
		<version>[4.0,)</version>
	</dependency>
	<dependency>
		<groupId>org.springframework.boot</groupId>
		<artifactId>spring-boot-starter-test</artifactId>
	</dependency>
</dependencies>

<build>
    <plugins>
        <plugin>
            <artifactId>maven-compiler-plugin</artifactId>
        </plugin>
        <plugin>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-maven-plugin</artifactId>
        </plugin>
    </plugins>
</build>

</project>

package org.podcastpedia.batch.jobs.addpodcast;

import org.podcastpedia.batch.common.configuration.DatabaseAccessConfiguration; import org.podcastpedia.batch.common.listeners.LogProcessListener; import org.podcastpedia.batch.common.listeners.ProtocolListener; import org.podcastpedia.batch.jobs.addpodcast.model.SuggestedPodcast; import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; import org.springframework.batch.item.ItemProcessor; import org.springframework.batch.item.ItemReader; import org.springframework.batch.item.ItemWriter; import org.springframework.batch.item.file.FlatFileItemReader; import org.springframework.batch.item.file.LineMapper; import org.springframework.batch.item.file.mapping.BeanWrapperFieldSetMapper; import org.springframework.batch.item.file.mapping.DefaultLineMapper; import org.springframework.batch.item.file.transform.DelimitedLineTokenizer; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; import org.springframework.core.io.ClassPathResource;

import com.mysql.jdbc.exceptions.jdbc4.MySQLIntegrityConstraintViolationException;

@Configuration @EnableBatchProcessing @Import({DatabaseAccessConfiguration.class, ServicesConfiguration.class}) public class AddPodcastJobConfiguration {

@Autowired
private JobBuilderFactory jobs;

@Autowired
private StepBuilderFactory stepBuilderFactory;

// tag::jobstep[]
@Bean
public Job addNewPodcastJob(){
	return jobs.get("addNewPodcastJob")
			.listener(protocolListener())
			.start(step())
			.build();
}

@Bean
public Step step(){
	return stepBuilderFactory.get("step")
			.<SuggestedPodcast,SuggestedPodcast>chunk(1) //important to be one in this case to commit after every line read
			.reader(reader())
			.processor(processor())
			.writer(writer())
			.listener(logProcessListener())
			.faultTolerant()
			.skipLimit(10) //default is set to 0
			.skip(MySQLIntegrityConstraintViolationException.class)
			.build();
}
// end::jobstep[]

// tag::readerwriterprocessor[]
@Bean
public ItemReader<SuggestedPodcast> reader(){
	FlatFileItemReader<SuggestedPodcast> reader = new FlatFileItemReader<SuggestedPodcast>();
	reader.setLinesToSkip(1);//first line is title definition
	reader.setResource(new ClassPathResource("suggested-podcasts.txt"));
	reader.setLineMapper(lineMapper());
	return reader;
}

@Bean
public LineMapper<SuggestedPodcast> lineMapper() {
	DefaultLineMapper<SuggestedPodcast> lineMapper = new DefaultLineMapper<SuggestedPodcast>();

	DelimitedLineTokenizer lineTokenizer = new DelimitedLineTokenizer();
	lineTokenizer.setDelimiter(";");
	lineTokenizer.setStrict(false);
	lineTokenizer.setNames(new String[]{"FEED_URL", "IDENTIFIER_ON_PODCASTPEDIA", "CATEGORIES", "LANGUAGE", "MEDIA_TYPE", "UPDATE_FREQUENCY", "KEYWORDS", "FB_PAGE", "TWITTER_PAGE", "GPLUS_PAGE", "NAME_SUBMITTER", "EMAIL_SUBMITTER"});

	BeanWrapperFieldSetMapper<SuggestedPodcast> fieldSetMapper = new BeanWrapperFieldSetMapper<SuggestedPodcast>();
	fieldSetMapper.setTargetType(SuggestedPodcast.class);

	lineMapper.setLineTokenizer(lineTokenizer);
	lineMapper.setFieldSetMapper(suggestedPodcastFieldSetMapper());

	return lineMapper;
}

@Bean
public SuggestedPodcastFieldSetMapper suggestedPodcastFieldSetMapper() {
	return new SuggestedPodcastFieldSetMapper();
}

/** configure the processor related stuff */
@Bean
public ItemProcessor<SuggestedPodcast, SuggestedPodcast> processor() {
    return new SuggestedPodcastItemProcessor();
}

@Bean
public ItemWriter<SuggestedPodcast> writer() {
	return new Writer();
}
// end::readerwriterprocessor[]

@Bean
public ProtocolListener protocolListener(){
	return new ProtocolListener();
}

@Bean
public LogProcessListener logProcessListener(){
	return new LogProcessListener();
}    

}

@Bean public Job addNewPodcastJob(){ return jobs.get("addNewPodcastJob") .listener(protocolListener()) .start(step()) .build(); }

@Bean public Step step(){ return stepBuilderFactory.get("step") .<SuggestedPodcast,SuggestedPodcast>chunk(1) //important to be one in this case to commit after every line read .reader(reader()) .processor(processor()) .writer(writer()) .listener(logProcessListener()) .faultTolerant() .skipLimit(10) //default is set to 0 .skip(MySQLIntegrityConstraintViolationException.class) .build(); }

© 著作权归作者所有

共有 人打赏支持
j
粉丝 2
博文 36
码字总数 4400
作品 0
广州
程序员
私信 提问
Spring Batch 4.1.0.RC1,用于编写批处理应用的框架

Spring Batch 4.1.0.RC1 已发布,可从 Maven Central, Github 和 Pivotal 下载仓库获取。Spring Batch 是一个使用 Spring 和 Java 编写离线和批处理应用程序的框架。 更新内容 在这个版本中,...

局长
09/27
1K
0
Spring Batch 4.0.0 正式发布,批处理框架

Spring Batch 4.0.0 已发布,Spring Batch 是一个轻量级的,完全面向 Spring 的批处理框架,可以应用于企业级大量的数据处理系统。Spring Batch 以 POJO 和 Spring 框架为基础,使开发者更容...

王练
2017/12/05
3.6K
5
Spring Batch 4.1 GA 发布,用于编写批处理应用的框架

Spring Batch 4.1 GA 正式发布了,可以在 Spring Boot 2.1 中使用 Spring Batch 4.1 GA 版本。 Spring Batch 4.1 GA 的更新亮点: 增加新的 注解用于简化测试批处理组件 增加新的 注解,用于...

达尔文
11/01
1K
0
Spring Batch 3.0.9 和 4.0.1 发布,常规维护版本

Spring Batch 3.0.9 和 4.0.1 已发布,可从 Maven Central, Github 和 Pivotal 下载仓库获取。 这两个维护版本修复了一些错误、增强功能和更新文档。有关变更的完整列表,请参阅 3.0.9 和 4....

局长
03/08
1K
2
Spring IO Platform 1.1.5/2.0.1 发布

Spring IO Platform 1.1.5/2.0.1 发布,现已提供在 repo.spring.io 和 Maven Central。Spring IO Platform 1.1.5 是个维护版本,主要包括最新的维护版本: Spring Batch 3.0.6.RELEASE Sprin......

oschina
2015/12/19
1K
2

没有更多内容

加载失败,请刷新页面

加载更多

Caffe(二)-Python-自定义网络

这里我们用一个例子先来体验一下 首先定义一下我们的环境变量 $PYTHONPATH,我这儿是Windows开发环境,至于Windows Caffe怎么编译由读者自己下去搞定 我使用的控制台是 Windows PowerShell 添...

Pulsar-V
36分钟前
3
0
ActiveMQ从入门到精通(二)之可靠性机制

ActiveMQ的可靠性机制 缘由( 确认JMS消息) 只要消息被确认之后,才认为消息被成功消费了。消息的成功消费包括三个阶段:客户端接收消息、客户端处理消息以及客户端确认消息。在事务性会话中...

一看就喷亏的小猿
44分钟前
1
0
源码分析 Mybatis 的 foreach 为什么会出现性能问题

背景 最近在做一个类似于综合报表之类的东西,需要查询所有的记录(数据库记录有限制),大概有1W条记录,该报表需要三个表的数据,也就是根据这 1W 个 ID 去执行查询三次数据库,其中,有一...

TSMYK
今天
7
0
IC-CAD Methodology企业实战之openlava

在云计算解决安全问题并成为IC界主流运算平台之前,私有的服务器集群系统仍然是各大IC公司的计算资源平台首选。 现在主流的服务器集群管理系统包括lsf,openlava,SkyForm,三者都属于lsf一系...

李艳青1987
今天
7
0
http response stream 字节流 接收与解码

在接收图片、音频、视频的时候,需要用到二进制流。 浏览器会发给客户端 字节Byte流,一串串的发过来_int8格式 -128~127(十进制),也就是8bit(位)。 客户端接收的时候,对接收到的字节收集,...

大灰狼wow
今天
9
0

没有更多内容

加载失败,请刷新页面

加载更多

返回顶部
顶部