CODEWITHSUNDEEP
springspring-bootspring-batch
abhinav jain
abhinav jain

Reputation: 15

How to reset MultiResourceItemReader for each job run . Step scope not working

How can I initilize the MultiResourceItemReader for each job run . currently with this setup its still using the same instance for each job run

I put the @StepScope still its using the same old list of files which it has already been processed. I am not sure what else I have to add in this code

I also tried with the @JobScope also it did not work out. there is something fundamental I am missing 

@Configuration
@EnableBatchProcessing
public class BatchConfiguration {

    @Autowired
    public JobBuilderFactory jobBuilderFactory;
    @Autowired
    public StepBuilderFactory stepBuilderFactory;


    @Value("file:ftp-inbound/*.csv")
    @Autowired
    private Resource[] inputResources;

    @Autowired
    private StepBuilderFactory steps;
    @Autowired
    private JobBuilderFactory jobs;
    @Autowired
    private ResourceLoader resourceLoader;


    @Bean
    public FlatFileItemReader<AccommodationRoomAvailability> itemReader() throws UnexpectedInputException, ParseException, IOException {
        FlatFileItemReader<AccommodationRoomAvailability> reader = new FlatFileItemReader<AccommodationRoomAvailability>();
        DelimitedLineTokenizer tokenizer = new DelimitedLineTokenizer();
        String[] tokens = {"Product ID", "Allotment", "Kamertype", "Zoeknaam", "Hotel", "Datum", "Beschikbaar", "Nachten"};
        tokenizer.setNames(tokens);
        tokenizer.setDelimiter(";");
        tokenizer.setStrict(true);
        reader.setLinesToSkip(1);
        DefaultLineMapper<AccommodationRoomAvailability> lineMapper = new DefaultLineMapper<AccommodationRoomAvailability>();
        lineMapper.setLineTokenizer(tokenizer);
        lineMapper.setFieldSetMapper(new RecordFieldSetMapper());
        reader.setLineMapper(lineMapper);

        return reader;
    }

    @Bean
    @Qualifier("multiResourceReader")
    @StepScope
    public MultiResourceItemReader<AccommodationRoomAvailability> multiResourceItemReader() throws Exception {
        MultiResourceItemReader<AccommodationRoomAvailability> resourceItemReader = new MultiResourceItemReader<AccommodationRoomAvailability>();
        resourceItemReader.setResources(inputResources);
        resourceItemReader.setDelegate(itemReader());
        resourceItemReader.setStrict(false);
        resourceItemReader.setSaveState(false);

        // resourceItemReader.read();
        return resourceItemReader;
    }

    @Bean
    public ItemProcessor<AccommodationRoomAvailability, String> itemProcessor() {
        return new AvailabilityProcessor();
    }

    @Bean
    public ItemWriter itemWriter() {
        return new ItemWriter() {
            @Override
            public void write(List list) throws Exception {


            }
        };
    }

    @Bean
    protected Step step1(@Qualifier("multiResourceReader") MultiResourceItemReader<AccommodationRoomAvailability> reader, ItemProcessor<AccommodationRoomAvailability, String> processor,
                         ItemWriter writer) {
        return steps.get("step1")/*.listener(new StepListener())*/.<AccommodationRoomAvailability, String>chunk(30000).reader(reader)
                .processor(processor)
                .writer(writer)
                .build();
    }

    @Bean
    public Step step2() throws IOException {

        FileDeletingTasklet task = new FileDeletingTasklet();
        task.setResources(inputResources);
        return stepBuilderFactory.get("step2")
                .tasklet(task)
                .build();
    }

    @Bean(name = "job")
    public Job job(@Qualifier("step1") Step step1, Step step2) throws IOException {
        return jobs.get("job")
                .start(step1).on("*").to(step2).end()
                // .flow(step1).on("").to(step2()).end()
                .build();
    }

}

Upvotes: 0

Views: 768

Answers (1)

Mahmoud Ben Hassine
Mahmoud Ben Hassine

Reputation: 31600

Once your application context is created, the injected resources @Value("file:ftp-inbound/*.csv") will be the same during the whole lifetime of your app. That's why the reader will always read the same values.

You need to pass these resources as a parameter to your job and late-bind them in your reader with Step scope. In your example it would be something like:

@Bean
@Qualifier("multiResourceReader")
@StepScope
public MultiResourceItemReader<AccommodationRoomAvailability> multiResourceItemReader(@Value("#{jobParameters['inputResources']}") Resource[] inputResources) throws Exception {
    MultiResourceItemReader<AccommodationRoomAvailability> resourceItemReader = new MultiResourceItemReader<AccommodationRoomAvailability>();
    resourceItemReader.setResources(inputResources);
    resourceItemReader.setDelegate(itemReader());
    resourceItemReader.setStrict(false);
    resourceItemReader.setSaveState(false);

    return resourceItemReader;
}

Then pass input resources as a parameter to your job:

JobParameters jobParameters = new JobParametersBuilder()
            .addString("inputResources", "file:ftp-inbound/*.csv")
            .toJobParameters();

currently with this setup its still using the same instance for each job run

That's because your resources are always the same when they are injected in a field of your configuration class. If you use the job parameters approach I mentioned in the previous example, you will have a different instance if you run the job with different set of files.

Upvotes: 2

Related Questions