<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <modelVersion>4.0.0</modelVersion>
<groupId>org.podcastpedia.batch</groupId>
<artifactId>podcastpedia-batch</artifactId>
<version>0.1.0</version>
<properties>
<sprinb.boot.version>1.1.6.RELEASE</sprinb.boot.version>
<java.version>1.7</java.version>
</properties>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>1.1.6.RELEASE</version>
</parent>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-batch</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-jpa</artifactId>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.3.5</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcore</artifactId>
<version>4.3.2</version>
</dependency>
<!-- velocity -->
<dependency>
<groupId>org.apache.velocity</groupId>
<artifactId>velocity</artifactId>
<version>1.7</version>
</dependency>
<dependency>
<groupId>org.apache.velocity</groupId>
<artifactId>velocity-tools</artifactId>
<version>2.0</version>
<exclusions>
<exclusion>
<groupId>org.apache.struts</groupId>
<artifactId>struts-core</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- Project rome rss, atom -->
<dependency>
<groupId>rome</groupId>
<artifactId>rome</artifactId>
<version>1.0</version>
</dependency>
<!-- option this fetcher thing -->
<dependency>
<groupId>rome</groupId>
<artifactId>rome-fetcher</artifactId>
<version>1.0</version>
</dependency>
<dependency>
<groupId>org.jdom</groupId>
<artifactId>jdom</artifactId>
<version>1.1</version>
</dependency>
<!-- PID 1 -->
<dependency>
<groupId>xerces</groupId>
<artifactId>xercesImpl</artifactId>
<version>2.9.1</version>
</dependency>
<!-- MySQL JDBC connector -->
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>5.1.31</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-freemarker</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-remote-shell</artifactId>
<exclusions>
<exclusion>
<groupId>javax.mail</groupId>
<artifactId>mail</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>javax.mail</groupId>
<artifactId>mail</artifactId>
<version>1.4.7</version>
</dependency>
<dependency>
<groupId>javax.inject</groupId>
<artifactId>javax.inject</artifactId>
<version>1</version>
</dependency>
<dependency>
<groupId>org.twitter4j</groupId>
<artifactId>twitter4j-core</artifactId>
<version>[4.0,)</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
</project>
package org.podcastpedia.batch.jobs.addpodcast;
import org.podcastpedia.batch.common.configuration.DatabaseAccessConfiguration; import org.podcastpedia.batch.common.listeners.LogProcessListener; import org.podcastpedia.batch.common.listeners.ProtocolListener; import org.podcastpedia.batch.jobs.addpodcast.model.SuggestedPodcast; import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; import org.springframework.batch.item.ItemProcessor; import org.springframework.batch.item.ItemReader; import org.springframework.batch.item.ItemWriter; import org.springframework.batch.item.file.FlatFileItemReader; import org.springframework.batch.item.file.LineMapper; import org.springframework.batch.item.file.mapping.BeanWrapperFieldSetMapper; import org.springframework.batch.item.file.mapping.DefaultLineMapper; import org.springframework.batch.item.file.transform.DelimitedLineTokenizer; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; import org.springframework.core.io.ClassPathResource;
import com.mysql.jdbc.exceptions.jdbc4.MySQLIntegrityConstraintViolationException;
@Configuration @EnableBatchProcessing @Import({DatabaseAccessConfiguration.class, ServicesConfiguration.class}) public class AddPodcastJobConfiguration {
@Autowired
private JobBuilderFactory jobs;
@Autowired
private StepBuilderFactory stepBuilderFactory;
// tag::jobstep[]
@Bean
public Job addNewPodcastJob(){
return jobs.get("addNewPodcastJob")
.listener(protocolListener())
.start(step())
.build();
}
@Bean
public Step step(){
return stepBuilderFactory.get("step")
.<SuggestedPodcast,SuggestedPodcast>chunk(1) //important to be one in this case to commit after every line read
.reader(reader())
.processor(processor())
.writer(writer())
.listener(logProcessListener())
.faultTolerant()
.skipLimit(10) //default is set to 0
.skip(MySQLIntegrityConstraintViolationException.class)
.build();
}
// end::jobstep[]
// tag::readerwriterprocessor[]
@Bean
public ItemReader<SuggestedPodcast> reader(){
FlatFileItemReader<SuggestedPodcast> reader = new FlatFileItemReader<SuggestedPodcast>();
reader.setLinesToSkip(1);//first line is title definition
reader.setResource(new ClassPathResource("suggested-podcasts.txt"));
reader.setLineMapper(lineMapper());
return reader;
}
@Bean
public LineMapper<SuggestedPodcast> lineMapper() {
DefaultLineMapper<SuggestedPodcast> lineMapper = new DefaultLineMapper<SuggestedPodcast>();
DelimitedLineTokenizer lineTokenizer = new DelimitedLineTokenizer();
lineTokenizer.setDelimiter(";");
lineTokenizer.setStrict(false);
lineTokenizer.setNames(new String[]{"FEED_URL", "IDENTIFIER_ON_PODCASTPEDIA", "CATEGORIES", "LANGUAGE", "MEDIA_TYPE", "UPDATE_FREQUENCY", "KEYWORDS", "FB_PAGE", "TWITTER_PAGE", "GPLUS_PAGE", "NAME_SUBMITTER", "EMAIL_SUBMITTER"});
BeanWrapperFieldSetMapper<SuggestedPodcast> fieldSetMapper = new BeanWrapperFieldSetMapper<SuggestedPodcast>();
fieldSetMapper.setTargetType(SuggestedPodcast.class);
lineMapper.setLineTokenizer(lineTokenizer);
lineMapper.setFieldSetMapper(suggestedPodcastFieldSetMapper());
return lineMapper;
}
@Bean
public SuggestedPodcastFieldSetMapper suggestedPodcastFieldSetMapper() {
return new SuggestedPodcastFieldSetMapper();
}
/** configure the processor related stuff */
@Bean
public ItemProcessor<SuggestedPodcast, SuggestedPodcast> processor() {
return new SuggestedPodcastItemProcessor();
}
@Bean
public ItemWriter<SuggestedPodcast> writer() {
return new Writer();
}
// end::readerwriterprocessor[]
@Bean
public ProtocolListener protocolListener(){
return new ProtocolListener();
}
@Bean
public LogProcessListener logProcessListener(){
return new LogProcessListener();
}
}
@Bean public Job addNewPodcastJob(){ return jobs.get("addNewPodcastJob") .listener(protocolListener()) .start(step()) .build(); }
@Bean public Step step(){ return stepBuilderFactory.get("step") .<SuggestedPodcast,SuggestedPodcast>chunk(1) //important to be one in this case to commit after every line read .reader(reader()) .processor(processor()) .writer(writer()) .listener(logProcessListener()) .faultTolerant() .skipLimit(10) //default is set to 0 .skip(MySQLIntegrityConstraintViolationException.class) .build(); }
