Overview

In this tutorial i will show you how to read multi files by using Spring Batch Framework.

Here i have two csv files which called users1.csv & users2.csv.

Technology Used

• spring-boot-spring-batch
• JDK 1.8
• Maven 3
• Mysql
• IDEA

Project Structure

users1.csv

jackrutorial1,male
jackrutorial2,female
jackrutorial3,male

users2.csv

jackrutorial4,male
jackrutorial5,female
jackrutorial6,male

User.java

package com.zhiyueinfo.springbatch.csvtomysql.model;
​
public class User {
  private String name;
  private String gender;
​
  public String getGender() {
    return gender;
  }
​
  public void setGender(String gender) {
    this.gender = gender;
  }
​
  public String getName() {
    return name;
  }
​
  public void setName(String name) {
    this.name = name;
  }
}

BatchConfiguration.java

Code below defines we read file from classpath which fit the regular expression 'users*.csv'

@Value("classpath:users*.csv")
private Resource[] inputResources;

Code below defines the MultiResourceItemReaderbean. And it delegate to the reader() to actually read the files.

@Bean
  public MultiResourceItemReader<User> multiResourceItemReader() {
    MultiResourceItemReader<User> resourceItemReader = new MultiResourceItemReader<User();
    resourceItemReader.setResources(inputResources);
    resourceItemReader.setDelegate(reader());
    return resourceItemReader;
  }

BatchConfiguration.java

import com.zhiyueinfo.springbatch.csvtomysql.listener.JobResultListener;
import com.zhiyueinfo.springbatch.csvtomysql.listener.StepResultListener;
import com.zhiyueinfo.springbatch.csvtomysql.listener.StepSkipListener;
import com.zhiyueinfo.springbatch.csvtomysql.model.User;
import com.zhiyueinfo.springbatch.csvtomysql.processor.UserItemProcessor;
import com.zhiyueinfo.springbatch.csvtomysql.writer.ConsoleItemWriter;
import javax.sql.DataSource;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.core.launch.support.RunIdIncrementer;
import org.springframework.batch.item.database.BeanPropertyItemSqlParameterSourceProvider;
import org.springframework.batch.item.database.JdbcBatchItemWriter;
import org.springframework.batch.item.file.FlatFileItemReader;
import org.springframework.batch.item.file.FlatFileParseException;
import org.springframework.batch.item.file.MultiResourceItemReader;
import org.springframework.batch.item.file.mapping.BeanWrapperFieldSetMapper;
import org.springframework.batch.item.file.mapping.DefaultLineMapper;
import org.springframework.batch.item.file.transform.DelimitedLineTokenizer;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.io.Resource;
import org.springframework.jdbc.datasource.DriverManagerDataSource;
​
​
@Configuration
@EnableBatchProcessing
public class BatchConfiguration {
​
  @Autowired
  public JobBuilderFactory jobBuilderFactory;
​
  @Autowired
  public StepBuilderFactory stepBuilderFactory;
​
  @Autowired
  public DataSource dataSource;
​
  @Value("classpath:users*.csv")
  private Resource[] inputResources;
​
  @Bean
  public DataSource dataSource() {
    final DriverManagerDataSource dataSource = new DriverManagerDataSource();
    dataSource.setDriverClassName("com.mysql.jdbc.Driver");
    dataSource.setUrl("jdbc:mysql://localhost:3307/springbatch");
    dataSource.setUsername("root");
    dataSource.setPassword("123456");
​
    return dataSource;
  }
​
  @Bean
  public FlatFileItemReader<User> reader() {
    FlatFileItemReader<User> reader = new FlatFileItemReader<User>();
    reader.setLineMapper(new DefaultLineMapper<User>() {{
      setLineTokenizer(new DelimitedLineTokenizer() {{
        setNames(new String[]{"name", "gender"});
      }});
      setFieldSetMapper(new BeanWrapperFieldSetMapper<User>() {{
        setTargetType(User.class);
      }});
​
    }});
​
    return reader;
  }
​
  @Bean
  public MultiResourceItemReader<User> multiResourceItemReader() {
    MultiResourceItemReader<User> resourceItemReader = new MultiResourceItemReader<User>();
    resourceItemReader.setResources(inputResources);
    resourceItemReader.setDelegate(reader());
    return resourceItemReader;
  }
​
  @Bean
  public UserItemProcessor processor() {
    return new UserItemProcessor();
  }
​
  @Bean
  public JdbcBatchItemWriter<User> writer() {
    JdbcBatchItemWriter<User> writer = new JdbcBatchItemWriter<User>();
    writer
        .setItemSqlParameterSourceProvider(new BeanPropertyItemSqlParameterSourceProvider<User>());
    writer.setSql("INSERT INTO user(name,gender) VALUES (:name,:gender)");
    writer.setDataSource(dataSource);
​
    return writer;
  }
​
​
  @Bean
  public Step step1() {
    return stepBuilderFactory.get("step1")
        .listener(new StepResultListener())
//        .listener(new StepItemWriteListener())
//        .listener(new StepItemReadListener())
//        .listener(new StepItemProcessListener())
        .<User, User>chunk(6)
        .reader(multiResourceItemReader())
        .faultTolerant()
        .skipLimit(1)
        .skip(FlatFileParseException.class)
        .listener(new StepSkipListener())
        .processor(processor())
        .writer(customWriter())
        .build();
  }
​
  @Bean
  public Job importUserJob() {
    return jobBuilderFactory.get("importUserJob")
        .incrementer(new RunIdIncrementer())
        .listener(new JobResultListener())
        .flow(step1())
        .end()
        .build();
  }
​
  @Bean
  public ConsoleItemWriter<User> customWriter() {
    return new ConsoleItemWriter<User>();
  }
​
}

ConsoleItemWriter.java

import com.zhiyueinfo.springbatch.csvtomysql.model.User;
import java.util.List;
 
import org.springframework.batch.item.ItemWriter;
 
public class ConsoleItemWriter<T> implements ItemWriter<T> {
    @Override
    public void write(List<? extends T> items) throws Exception {
        for (T item : items) {
            User itemnew = (User) item;
            System.out.println(itemnew.getName());
        }
    }
}

Run it

Demo code

Github

springbatchtutorial_multireader-master